drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c

   1 /*
   2  * Copyright 2012 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Ben Skeggs
  23  */
  24
  25 #include "nvc0.h"
  26 #include "fuc/hubnvc0.fuc.h"
  27 #include "fuc/gpcnvc0.fuc.h"
  28
  29 /*******************************************************************************
  30  * Graphics object classes
  31  ******************************************************************************/
  32
  33 static struct nouveau_oclass
  34 nvc0_graph_sclass[] = {
  35         { 0x902d, &nouveau_object_ofuncs },
  36         { 0x9039, &nouveau_object_ofuncs },
  37         { 0x9097, &nouveau_object_ofuncs },
  38         { 0x90c0, &nouveau_object_ofuncs },
  39         {}
  40 };
  41
  42 static struct nouveau_oclass
  43 nvc1_graph_sclass[] = {
  44         { 0x902d, &nouveau_object_ofuncs },
  45         { 0x9039, &nouveau_object_ofuncs },
  46         { 0x9097, &nouveau_object_ofuncs },
  47         { 0x90c0, &nouveau_object_ofuncs },
  48         { 0x9197, &nouveau_object_ofuncs },
  49         {}
  50 };
  51
  52 static struct nouveau_oclass
  53 nvc8_graph_sclass[] = {
  54         { 0x902d, &nouveau_object_ofuncs },
  55         { 0x9039, &nouveau_object_ofuncs },
  56         { 0x9097, &nouveau_object_ofuncs },
  57         { 0x90c0, &nouveau_object_ofuncs },
  58         { 0x9197, &nouveau_object_ofuncs },
  59         { 0x9297, &nouveau_object_ofuncs },
  60         {}
  61 };
  62
  63 u64
  64 nvc0_graph_units(struct nouveau_graph *graph)
  65 {
  66         struct nvc0_graph_priv *priv = (void *)graph;
  67         u64 cfg;
  68
  69         cfg  = (u32)priv->gpc_nr;
  70         cfg |= (u32)priv->tpc_total << 8;
  71         cfg |= (u64)priv->rop_nr << 32;
  72
  73         return cfg;
  74 }
  75
  76 /*******************************************************************************
  77  * PGRAPH context
  78  ******************************************************************************/
  79
  80 int
  81 nvc0_graph_context_ctor(struct nouveau_object *parent,
  82                         struct nouveau_object *engine,
  83                         struct nouveau_oclass *oclass, void *args, u32 size,
  84                         struct nouveau_object **pobject)
  85 {
  86         struct nouveau_vm *vm = nouveau_client(parent)->vm;
  87         struct nvc0_graph_priv *priv = (void *)engine;
  88         struct nvc0_graph_data *data = priv->mmio_data;
  89         struct nvc0_graph_mmio *mmio = priv->mmio_list;
  90         struct nvc0_graph_chan *chan;
  91         int ret, i;
  92
  93         /* allocate memory for context, and fill with default values */
  94         ret = nouveau_graph_context_create(parent, engine, oclass, NULL,
  95                                            priv->size, 0x100,
  96                                            NVOBJ_FLAG_ZERO_ALLOC, &chan);
  97         *pobject = nv_object(chan);
  98         if (ret)
  99                 return ret;
 100
 101         /* allocate memory for a "mmio list" buffer that's used by the HUB
 102          * fuc to modify some per-context register settings on first load
 103          * of the context.
 104          */
 105         ret = nouveau_gpuobj_new(nv_object(chan), NULL, 0x1000, 0x100, 0,
 106                                 &chan->mmio);
 107         if (ret)
 108                 return ret;
 109
 110         ret = nouveau_gpuobj_map_vm(nv_gpuobj(chan->mmio), vm,
 111                                     NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS,
 112                                     &chan->mmio_vma);
 113         if (ret)
 114                 return ret;
 115
 116         /* allocate buffers referenced by mmio list */
 117         for (i = 0; data->size && i < ARRAY_SIZE(priv->mmio_data); i++) {
 118                 ret = nouveau_gpuobj_new(nv_object(chan), NULL, data->size,
 119                                          data->align, 0, &chan->data[i].mem);
 120                 if (ret)
 121                         return ret;
 122
 123                 ret = nouveau_gpuobj_map_vm(chan->data[i].mem, vm, data->access,
 124                                            &chan->data[i].vma);
 125                 if (ret)
 126                         return ret;
 127
 128                 data++;
 129         }
 130
 131         /* finally, fill in the mmio list and point the context at it */
 132         for (i = 0; mmio->addr && i < ARRAY_SIZE(priv->mmio_list); i++) {
 133                 u32 addr = mmio->addr;
 134                 u32 data = mmio->data;
 135
 136                 if (mmio->shift) {
 137                         u64 info = chan->data[mmio->buffer].vma.offset;
 138                         data |= info >> mmio->shift;
 139                 }
 140
 141                 nv_wo32(chan->mmio, chan->mmio_nr++ * 4, addr);
 142                 nv_wo32(chan->mmio, chan->mmio_nr++ * 4, data);
 143                 mmio++;
 144         }
 145
 146         for (i = 0; i < priv->size; i += 4)
 147                 nv_wo32(chan, i, priv->data[i / 4]);
 148
 149         if (!priv->firmware) {
 150                 nv_wo32(chan, 0x00, chan->mmio_nr / 2);
 151                 nv_wo32(chan, 0x04, chan->mmio_vma.offset >> 8);
 152         } else {
 153                 nv_wo32(chan, 0xf4, 0);
 154                 nv_wo32(chan, 0xf8, 0);
 155                 nv_wo32(chan, 0x10, chan->mmio_nr / 2);
 156                 nv_wo32(chan, 0x14, lower_32_bits(chan->mmio_vma.offset));
 157                 nv_wo32(chan, 0x18, upper_32_bits(chan->mmio_vma.offset));
 158                 nv_wo32(chan, 0x1c, 1);
 159                 nv_wo32(chan, 0x20, 0);
 160                 nv_wo32(chan, 0x28, 0);
 161                 nv_wo32(chan, 0x2c, 0);
 162         }
 163
 164         return 0;
 165 }
 166
 167 void
 168 nvc0_graph_context_dtor(struct nouveau_object *object)
 169 {
 170         struct nvc0_graph_chan *chan = (void *)object;
 171         int i;
 172
 173         for (i = 0; i < ARRAY_SIZE(chan->data); i++) {
 174                 nouveau_gpuobj_unmap(&chan->data[i].vma);
 175                 nouveau_gpuobj_ref(NULL, &chan->data[i].mem);
 176         }
 177
 178         nouveau_gpuobj_unmap(&chan->mmio_vma);
 179         nouveau_gpuobj_ref(NULL, &chan->mmio);
 180
 181         nouveau_graph_context_destroy(&chan->base);
 182 }
 183
 184 static struct nouveau_oclass
 185 nvc0_graph_cclass = {
 186         .ofuncs = &(struct nouveau_ofuncs) {
 187                 .ctor = nvc0_graph_context_ctor,
 188                 .dtor = nvc0_graph_context_dtor,
 189                 .init = _nouveau_graph_context_init,
 190                 .fini = _nouveau_graph_context_fini,
 191                 .rd32 = _nouveau_graph_context_rd32,
 192                 .wr32 = _nouveau_graph_context_wr32,
 193         },
 194 };
 195
 196 /*******************************************************************************
 197  * PGRAPH engine/subdev functions
 198  ******************************************************************************/
 199
 200 static void
 201 nvc0_graph_ctxctl_debug_unit(struct nvc0_graph_priv *priv, u32 base)
 202 {
 203         nv_error(priv, "%06x - done 0x%08x\n", base,
 204                  nv_rd32(priv, base + 0x400));
 205         nv_error(priv, "%06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
 206                  nv_rd32(priv, base + 0x800), nv_rd32(priv, base + 0x804),
 207                  nv_rd32(priv, base + 0x808), nv_rd32(priv, base + 0x80c));
 208         nv_error(priv, "%06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
 209                  nv_rd32(priv, base + 0x810), nv_rd32(priv, base + 0x814),
 210                  nv_rd32(priv, base + 0x818), nv_rd32(priv, base + 0x81c));
 211 }
 212
 213 void
 214 nvc0_graph_ctxctl_debug(struct nvc0_graph_priv *priv)
 215 {
 216         u32 gpcnr = nv_rd32(priv, 0x409604) & 0xffff;
 217         u32 gpc;
 218
 219         nvc0_graph_ctxctl_debug_unit(priv, 0x409000);
 220         for (gpc = 0; gpc < gpcnr; gpc++)
 221                 nvc0_graph_ctxctl_debug_unit(priv, 0x502000 + (gpc * 0x8000));
 222 }
 223
 224 static void
 225 nvc0_graph_ctxctl_isr(struct nvc0_graph_priv *priv)
 226 {
 227         u32 ustat = nv_rd32(priv, 0x409c18);
 228
 229         if (ustat & 0x00000001)
 230                 nv_error(priv, "CTXCTRL ucode error\n");
 231         if (ustat & 0x00080000)
 232                 nv_error(priv, "CTXCTRL watchdog timeout\n");
 233         if (ustat & ~0x00080001)
 234                 nv_error(priv, "CTXCTRL 0x%08x\n", ustat);
 235
 236         nvc0_graph_ctxctl_debug(priv);
 237         nv_wr32(priv, 0x409c20, ustat);
 238 }
 239
 240 static const struct nouveau_enum nvc0_mp_warp_error[] = {
 241         { 0x00, "NO_ERROR" },
 242         { 0x01, "STACK_MISMATCH" },
 243         { 0x05, "MISALIGNED_PC" },
 244         { 0x08, "MISALIGNED_GPR" },
 245         { 0x09, "INVALID_OPCODE" },
 246         { 0x0d, "GPR_OUT_OF_BOUNDS" },
 247         { 0x0e, "MEM_OUT_OF_BOUNDS" },
 248         { 0x0f, "UNALIGNED_MEM_ACCESS" },
 249         { 0x11, "INVALID_PARAM" },
 250         {}
 251 };
 252
 253 static const struct nouveau_bitfield nvc0_mp_global_error[] = {
 254         { 0x00000004, "MULTIPLE_WARP_ERRORS" },
 255         { 0x00000008, "OUT_OF_STACK_SPACE" },
 256         {}
 257 };
 258
 259 static void
 260 nvc0_graph_trap_mp(struct nvc0_graph_priv *priv, int gpc, int tpc)
 261 {
 262         u32 werr = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x648));
 263         u32 gerr = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x650));
 264
 265         nv_error(priv, "GPC%i/TPC%i/MP trap:", gpc, tpc);
 266         nouveau_bitfield_print(nvc0_mp_global_error, gerr);
 267         if (werr) {
 268                 pr_cont(" ");
 269                 nouveau_enum_print(nvc0_mp_warp_error, werr & 0xffff);
 270         }
 271         pr_cont("\n");
 272
 273         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x648), 0x00000000);
 274         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x650), gerr);
 275 }
 276
 277 static void
 278 nvc0_graph_trap_tpc(struct nvc0_graph_priv *priv, int gpc, int tpc)
 279 {
 280         u32 stat = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0508));
 281
 282         if (stat & 0x00000001) {
 283                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0224));
 284                 nv_error(priv, "GPC%d/TPC%d/TEX: 0x%08x\n", gpc, tpc, trap);
 285                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0224), 0xc0000000);
 286                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), 0x00000001);
 287                 stat &= ~0x00000001;
 288         }
 289
 290         if (stat & 0x00000002) {
 291                 nvc0_graph_trap_mp(priv, gpc, tpc);
 292                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), 0x00000002);
 293                 stat &= ~0x00000002;
 294         }
 295
 296         if (stat & 0x00000004) {
 297                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0084));
 298                 nv_error(priv, "GPC%d/TPC%d/POLY: 0x%08x\n", gpc, tpc, trap);
 299                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0084), 0xc0000000);
 300                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), 0x00000004);
 301                 stat &= ~0x00000004;
 302         }
 303
 304         if (stat & 0x00000008) {
 305                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x048c));
 306                 nv_error(priv, "GPC%d/TPC%d/L1C: 0x%08x\n", gpc, tpc, trap);
 307                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x048c), 0xc0000000);
 308                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), 0x00000008);
 309                 stat &= ~0x00000008;
 310         }
 311
 312         if (stat) {
 313                 nv_error(priv, "GPC%d/TPC%d/0x%08x: unknown\n", gpc, tpc, stat);
 314                 nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), stat);
 315         }
 316 }
 317
 318 static void
 319 nvc0_graph_trap_gpc(struct nvc0_graph_priv *priv, int gpc)
 320 {
 321         u32 stat = nv_rd32(priv, GPC_UNIT(gpc, 0x2c90));
 322         int tpc;
 323
 324         if (stat & 0x00000001) {
 325                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0420));
 326                 nv_error(priv, "GPC%d/PROP: 0x%08x\n", gpc, trap);
 327                 nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
 328                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0x00000001);
 329                 stat &= ~0x00000001;
 330         }
 331
 332         if (stat & 0x00000002) {
 333                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0900));
 334                 nv_error(priv, "GPC%d/ZCULL: 0x%08x\n", gpc, trap);
 335                 nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
 336                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0x00000002);
 337                 stat &= ~0x00000002;
 338         }
 339
 340         if (stat & 0x00000004) {
 341                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x1028));
 342                 nv_error(priv, "GPC%d/CCACHE: 0x%08x\n", gpc, trap);
 343                 nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
 344                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0x00000004);
 345                 stat &= ~0x00000004;
 346         }
 347
 348         if (stat & 0x00000008) {
 349                 u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0824));
 350                 nv_error(priv, "GPC%d/ESETUP: 0x%08x\n", gpc, trap);
 351                 nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
 352                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0x00000008);
 353                 stat &= ~0x00000009;
 354         }
 355
 356         for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
 357                 u32 mask = 0x00010000 << tpc;
 358                 if (stat & mask) {
 359                         nvc0_graph_trap_tpc(priv, gpc, tpc);
 360                         nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), mask);
 361                         stat &= ~mask;
 362                 }
 363         }
 364
 365         if (stat) {
 366                 nv_error(priv, "GPC%d/0x%08x: unknown\n", gpc, stat);
 367                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), stat);
 368         }
 369 }
 370
 371 static void
 372 nvc0_graph_trap_intr(struct nvc0_graph_priv *priv)
 373 {
 374         u32 trap = nv_rd32(priv, 0x400108);
 375         int rop, gpc;
 376
 377         if (trap & 0x00000001) {
 378                 u32 stat = nv_rd32(priv, 0x404000);
 379                 nv_error(priv, "DISPATCH 0x%08x\n", stat);
 380                 nv_wr32(priv, 0x404000, 0xc0000000);
 381                 nv_wr32(priv, 0x400108, 0x00000001);
 382                 trap &= ~0x00000001;
 383         }
 384
 385         if (trap & 0x00000002) {
 386                 u32 stat = nv_rd32(priv, 0x404600);
 387                 nv_error(priv, "M2MF 0x%08x\n", stat);
 388                 nv_wr32(priv, 0x404600, 0xc0000000);
 389                 nv_wr32(priv, 0x400108, 0x00000002);
 390                 trap &= ~0x00000002;
 391         }
 392
 393         if (trap & 0x00000008) {
 394                 u32 stat = nv_rd32(priv, 0x408030);
 395                 nv_error(priv, "CCACHE 0x%08x\n", stat);
 396                 nv_wr32(priv, 0x408030, 0xc0000000);
 397                 nv_wr32(priv, 0x400108, 0x00000008);
 398                 trap &= ~0x00000008;
 399         }
 400
 401         if (trap & 0x00000010) {
 402                 u32 stat = nv_rd32(priv, 0x405840);
 403                 nv_error(priv, "SHADER 0x%08x\n", stat);
 404                 nv_wr32(priv, 0x405840, 0xc0000000);
 405                 nv_wr32(priv, 0x400108, 0x00000010);
 406                 trap &= ~0x00000010;
 407         }
 408
 409         if (trap & 0x00000040) {
 410                 u32 stat = nv_rd32(priv, 0x40601c);
 411                 nv_error(priv, "UNK6 0x%08x\n", stat);
 412                 nv_wr32(priv, 0x40601c, 0xc0000000);
 413                 nv_wr32(priv, 0x400108, 0x00000040);
 414                 trap &= ~0x00000040;
 415         }
 416
 417         if (trap & 0x00000080) {
 418                 u32 stat = nv_rd32(priv, 0x404490);
 419                 nv_error(priv, "MACRO 0x%08x\n", stat);
 420                 nv_wr32(priv, 0x404490, 0xc0000000);
 421                 nv_wr32(priv, 0x400108, 0x00000080);
 422                 trap &= ~0x00000080;
 423         }
 424
 425         if (trap & 0x01000000) {
 426                 u32 stat = nv_rd32(priv, 0x400118);
 427                 for (gpc = 0; stat && gpc < priv->gpc_nr; gpc++) {
 428                         u32 mask = 0x00000001 << gpc;
 429                         if (stat & mask) {
 430                                 nvc0_graph_trap_gpc(priv, gpc);
 431                                 nv_wr32(priv, 0x400118, mask);
 432                                 stat &= ~mask;
 433                         }
 434                 }
 435                 nv_wr32(priv, 0x400108, 0x01000000);
 436                 trap &= ~0x01000000;
 437         }
 438
 439         if (trap & 0x02000000) {
 440                 for (rop = 0; rop < priv->rop_nr; rop++) {
 441                         u32 statz = nv_rd32(priv, ROP_UNIT(rop, 0x070));
 442                         u32 statc = nv_rd32(priv, ROP_UNIT(rop, 0x144));
 443                         nv_error(priv, "ROP%d 0x%08x 0x%08x\n",
 444                                  rop, statz, statc);
 445                         nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
 446                         nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
 447                 }
 448                 nv_wr32(priv, 0x400108, 0x02000000);
 449                 trap &= ~0x02000000;
 450         }
 451
 452         if (trap) {
 453                 nv_error(priv, "TRAP UNHANDLED 0x%08x\n", trap);
 454                 nv_wr32(priv, 0x400108, trap);
 455         }
 456 }
 457
 458 static void
 459 nvc0_graph_intr(struct nouveau_subdev *subdev)
 460 {
 461         struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
 462         struct nouveau_engine *engine = nv_engine(subdev);
 463         struct nouveau_object *engctx;
 464         struct nouveau_handle *handle;
 465         struct nvc0_graph_priv *priv = (void *)subdev;
 466         u64 inst = nv_rd32(priv, 0x409b00) & 0x0fffffff;
 467         u32 stat = nv_rd32(priv, 0x400100);
 468         u32 addr = nv_rd32(priv, 0x400704);
 469         u32 mthd = (addr & 0x00003ffc);
 470         u32 subc = (addr & 0x00070000) >> 16;
 471         u32 data = nv_rd32(priv, 0x400708);
 472         u32 code = nv_rd32(priv, 0x400110);
 473         u32 class = nv_rd32(priv, 0x404200 + (subc * 4));
 474         int chid;
 475
 476         engctx = nouveau_engctx_get(engine, inst);
 477         chid   = pfifo->chid(pfifo, engctx);
 478
 479         if (stat & 0x00000010) {
 480                 handle = nouveau_handle_get_class(engctx, class);
 481                 if (!handle || nv_call(handle->object, mthd, data)) {
 482                         nv_error(priv,
 483                                  "ILLEGAL_MTHD ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
 484                                  chid, inst << 12, nouveau_client_name(engctx),
 485                                  subc, class, mthd, data);
 486                 }
 487                 nouveau_handle_put(handle);
 488                 nv_wr32(priv, 0x400100, 0x00000010);
 489                 stat &= ~0x00000010;
 490         }
 491
 492         if (stat & 0x00000020) {
 493                 nv_error(priv,
 494                          "ILLEGAL_CLASS ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
 495                          chid, inst << 12, nouveau_client_name(engctx), subc,
 496                          class, mthd, data);
 497                 nv_wr32(priv, 0x400100, 0x00000020);
 498                 stat &= ~0x00000020;
 499         }
 500
 501         if (stat & 0x00100000) {
 502                 nv_error(priv, "DATA_ERROR [");
 503                 nouveau_enum_print(nv50_data_error_names, code);
 504                 pr_cont("] ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
 505                         chid, inst << 12, nouveau_client_name(engctx), subc,
 506                         class, mthd, data);
 507                 nv_wr32(priv, 0x400100, 0x00100000);
 508                 stat &= ~0x00100000;
 509         }
 510
 511         if (stat & 0x00200000) {
 512                 nv_error(priv, "TRAP ch %d [0x%010llx %s]\n", chid, inst << 12,
 513                          nouveau_client_name(engctx));
 514                 nvc0_graph_trap_intr(priv);
 515                 nv_wr32(priv, 0x400100, 0x00200000);
 516                 stat &= ~0x00200000;
 517         }
 518
 519         if (stat & 0x00080000) {
 520                 nvc0_graph_ctxctl_isr(priv);
 521                 nv_wr32(priv, 0x400100, 0x00080000);
 522                 stat &= ~0x00080000;
 523         }
 524
 525         if (stat) {
 526                 nv_error(priv, "unknown stat 0x%08x\n", stat);
 527                 nv_wr32(priv, 0x400100, stat);
 528         }
 529
 530         nv_wr32(priv, 0x400500, 0x00010001);
 531         nouveau_engctx_put(engctx);
 532 }
 533
 534 int
 535 nvc0_graph_ctor_fw(struct nvc0_graph_priv *priv, const char *fwname,
 536                    struct nvc0_graph_fuc *fuc)
 537 {
 538         struct nouveau_device *device = nv_device(priv);
 539         const struct firmware *fw;
 540         char f[32];
 541         int ret;
 542
 543         snprintf(f, sizeof(f), "nouveau/nv%02x_%s", device->chipset, fwname);
 544         ret = request_firmware(&fw, f, &device->pdev->dev);
 545         if (ret) {
 546                 snprintf(f, sizeof(f), "nouveau/%s", fwname);
 547                 ret = request_firmware(&fw, f, &device->pdev->dev);
 548                 if (ret) {
 549                         nv_error(priv, "failed to load %s\n", fwname);
 550                         return ret;
 551                 }
 552         }
 553
 554         fuc->size = fw->size;
 555         fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
 556         release_firmware(fw);
 557         return (fuc->data != NULL) ? 0 : -ENOMEM;
 558 }
 559
 560 static int
 561 nvc0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 562                 struct nouveau_oclass *oclass, void *data, u32 size,
 563                 struct nouveau_object **pobject)
 564 {
 565         struct nouveau_device *device = nv_device(parent);
 566         struct nvc0_graph_priv *priv;
 567         bool enable = device->chipset != 0xd7;
 568         int ret, i;
 569
 570         ret = nouveau_graph_create(parent, engine, oclass, enable, &priv);
 571         *pobject = nv_object(priv);
 572         if (ret)
 573                 return ret;
 574
 575         nv_subdev(priv)->unit = 0x18001000;
 576         nv_subdev(priv)->intr = nvc0_graph_intr;
 577         nv_engine(priv)->cclass = &nvc0_graph_cclass;
 578
 579         priv->base.units = nvc0_graph_units;
 580
 581         if (nouveau_boolopt(device->cfgopt, "NvGrUseFW", false)) {
 582                 nv_info(priv, "using external firmware\n");
 583                 if (nvc0_graph_ctor_fw(priv, "fuc409c", &priv->fuc409c) ||
 584                     nvc0_graph_ctor_fw(priv, "fuc409d", &priv->fuc409d) ||
 585                     nvc0_graph_ctor_fw(priv, "fuc41ac", &priv->fuc41ac) ||
 586                     nvc0_graph_ctor_fw(priv, "fuc41ad", &priv->fuc41ad))
 587                         return -EINVAL;
 588                 priv->firmware = true;
 589         }
 590
 591         switch (nvc0_graph_class(priv)) {
 592         case 0x9097:
 593                 nv_engine(priv)->sclass = nvc0_graph_sclass;
 594                 break;
 595         case 0x9197:
 596                 nv_engine(priv)->sclass = nvc1_graph_sclass;
 597                 break;
 598         case 0x9297:
 599                 nv_engine(priv)->sclass = nvc8_graph_sclass;
 600                 break;
 601         }
 602
 603         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 256, 0,
 604                                 &priv->unk4188b4);
 605         if (ret)
 606                 return ret;
 607
 608         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 256, 0,
 609                                 &priv->unk4188b8);
 610         if (ret)
 611                 return ret;
 612
 613         for (i = 0; i < 0x1000; i += 4) {
 614                 nv_wo32(priv->unk4188b4, i, 0x00000010);
 615                 nv_wo32(priv->unk4188b8, i, 0x00000010);
 616         }
 617
 618         priv->rop_nr = (nv_rd32(priv, 0x409604) & 0x001f0000) >> 16;
 619         priv->gpc_nr =  nv_rd32(priv, 0x409604) & 0x0000001f;
 620         for (i = 0; i < priv->gpc_nr; i++) {
 621                 priv->tpc_nr[i]  = nv_rd32(priv, GPC_UNIT(i, 0x2608));
 622                 priv->tpc_total += priv->tpc_nr[i];
 623         }
 624
 625         /*XXX: these need figuring out... though it might not even matter */
 626         switch (nv_device(priv)->chipset) {
 627         case 0xc0:
 628                 if (priv->tpc_total == 11) { /* 465, 3/4/4/0, 4 */
 629                         priv->magic_not_rop_nr = 0x07;
 630                 } else
 631                 if (priv->tpc_total == 14) { /* 470, 3/3/4/4, 5 */
 632                         priv->magic_not_rop_nr = 0x05;
 633                 } else
 634                 if (priv->tpc_total == 15) { /* 480, 3/4/4/4, 6 */
 635                         priv->magic_not_rop_nr = 0x06;
 636                 }
 637                 break;
 638         case 0xc3: /* 450, 4/0/0/0, 2 */
 639                 priv->magic_not_rop_nr = 0x03;
 640                 break;
 641         case 0xc4: /* 460, 3/4/0/0, 4 */
 642                 priv->magic_not_rop_nr = 0x01;
 643                 break;
 644         case 0xc1: /* 2/0/0/0, 1 */
 645                 priv->magic_not_rop_nr = 0x01;
 646                 break;
 647         case 0xc8: /* 4/4/3/4, 5 */
 648                 priv->magic_not_rop_nr = 0x06;
 649                 break;
 650         case 0xce: /* 4/4/0/0, 4 */
 651                 priv->magic_not_rop_nr = 0x03;
 652                 break;
 653         case 0xcf: /* 4/0/0/0, 3 */
 654                 priv->magic_not_rop_nr = 0x03;
 655                 break;
 656         case 0xd9: /* 1/0/0/0, 1 */
 657                 priv->magic_not_rop_nr = 0x01;
 658                 break;
 659         }
 660
 661         return 0;
 662 }
 663
 664 static void
 665 nvc0_graph_dtor_fw(struct nvc0_graph_fuc *fuc)
 666 {
 667         kfree(fuc->data);
 668         fuc->data = NULL;
 669 }
 670
 671 void
 672 nvc0_graph_dtor(struct nouveau_object *object)
 673 {
 674         struct nvc0_graph_priv *priv = (void *)object;
 675
 676         kfree(priv->data);
 677
 678         nvc0_graph_dtor_fw(&priv->fuc409c);
 679         nvc0_graph_dtor_fw(&priv->fuc409d);
 680         nvc0_graph_dtor_fw(&priv->fuc41ac);
 681         nvc0_graph_dtor_fw(&priv->fuc41ad);
 682
 683         nouveau_gpuobj_ref(NULL, &priv->unk4188b8);
 684         nouveau_gpuobj_ref(NULL, &priv->unk4188b4);
 685
 686         nouveau_graph_destroy(&priv->base);
 687 }
 688
 689 static void
 690 nvc0_graph_init_obj418880(struct nvc0_graph_priv *priv)
 691 {
 692         int i;
 693
 694         nv_wr32(priv, GPC_BCAST(0x0880), 0x00000000);
 695         nv_wr32(priv, GPC_BCAST(0x08a4), 0x00000000);
 696         for (i = 0; i < 4; i++)
 697                 nv_wr32(priv, GPC_BCAST(0x0888) + (i * 4), 0x00000000);
 698         nv_wr32(priv, GPC_BCAST(0x08b4), priv->unk4188b4->addr >> 8);
 699         nv_wr32(priv, GPC_BCAST(0x08b8), priv->unk4188b8->addr >> 8);
 700 }
 701
 702 static void
 703 nvc0_graph_init_regs(struct nvc0_graph_priv *priv)
 704 {
 705         nv_wr32(priv, 0x400080, 0x003083c2);
 706         nv_wr32(priv, 0x400088, 0x00006fe7);
 707         nv_wr32(priv, 0x40008c, 0x00000000);
 708         nv_wr32(priv, 0x400090, 0x00000030);
 709         nv_wr32(priv, 0x40013c, 0x013901f7);
 710         nv_wr32(priv, 0x400140, 0x00000100);
 711         nv_wr32(priv, 0x400144, 0x00000000);
 712         nv_wr32(priv, 0x400148, 0x00000110);
 713         nv_wr32(priv, 0x400138, 0x00000000);
 714         nv_wr32(priv, 0x400130, 0x00000000);
 715         nv_wr32(priv, 0x400134, 0x00000000);
 716         nv_wr32(priv, 0x400124, 0x00000002);
 717 }
 718
 719 static void
 720 nvc0_graph_init_unk40xx(struct nvc0_graph_priv *priv)
 721 {
 722         nv_wr32(priv, 0x40415c, 0x00000000);
 723         nv_wr32(priv, 0x404170, 0x00000000);
 724 }
 725
 726 static void
 727 nvc0_graph_init_unk44xx(struct nvc0_graph_priv *priv)
 728 {
 729         nv_wr32(priv, 0x404488, 0x00000000);
 730         nv_wr32(priv, 0x40448c, 0x00000000);
 731 }
 732
 733 static void
 734 nvc0_graph_init_unk78xx(struct nvc0_graph_priv *priv)
 735 {
 736         nv_wr32(priv, 0x407808, 0x00000000);
 737 }
 738
 739 static void
 740 nvc0_graph_init_unk60xx(struct nvc0_graph_priv *priv)
 741 {
 742         nv_wr32(priv, 0x406024, 0x00000000);
 743 }
 744
 745 static void
 746 nvc0_graph_init_unk64xx(struct nvc0_graph_priv *priv)
 747 {
 748         switch (nv_device(priv)->chipset) {
 749         case 0xd9:
 750         case 0xd7:
 751                 nv_wr32(priv, 0x4064f0, 0x00000000);
 752                 nv_wr32(priv, 0x4064f4, 0x00000000);
 753                 nv_wr32(priv, 0x4064f8, 0x00000000);
 754                 break;
 755         case 0xc0:
 756         case 0xc3:
 757         case 0xc4:
 758         case 0xc1:
 759         case 0xc8:
 760         case 0xce:
 761         case 0xcf:
 762                 break;
 763         default:
 764                 BUG_ON(1);
 765                 break;
 766         }
 767 }
 768
 769 static void
 770 nvc0_graph_init_unk58xx(struct nvc0_graph_priv *priv)
 771 {
 772         nv_wr32(priv, 0x405844, 0x00ffffff);
 773         nv_wr32(priv, 0x405850, 0x00000000);
 774         switch (nv_device(priv)->chipset) {
 775         case 0xc3:
 776         case 0xc4:
 777         case 0xc1:
 778         case 0xce:
 779         case 0xcf:
 780         case 0xd9:
 781         case 0xd7:
 782                 nv_wr32(priv, 0x405900, 0x00002834);
 783                 break;
 784         case 0xc0:
 785         case 0xc8:
 786                 break;
 787         default:
 788                 BUG_ON(1);
 789                 break;
 790         }
 791         nv_wr32(priv, 0x405908, 0x00000000);
 792         switch (nv_device(priv)->chipset) {
 793         case 0xd9:
 794         case 0xd7:
 795                 nv_wr32(priv, 0x405928, 0x00000000);
 796                 nv_wr32(priv, 0x40592c, 0x00000000);
 797                 break;
 798         case 0xc0:
 799         case 0xc3:
 800         case 0xc4:
 801         case 0xc1:
 802         case 0xc8:
 803         case 0xce:
 804         case 0xcf:
 805                 break;
 806         default:
 807                 BUG_ON(1);
 808                 break;
 809         }
 810 }
 811
 812 static void
 813 nvc0_graph_init_unk80xx(struct nvc0_graph_priv *priv)
 814 {
 815         nv_wr32(priv, 0x40803c, 0x00000000);
 816 }
 817
 818 static void
 819 nvc0_graph_init_gpc(struct nvc0_graph_priv *priv)
 820 {
 821         switch (nv_device(priv)->chipset) {
 822         case 0xd9:
 823         case 0xd7:
 824                 nv_wr32(priv, 0x418408, 0x00000000);
 825                 break;
 826         case 0xc0:
 827         case 0xc3:
 828         case 0xc4:
 829         case 0xc1:
 830         case 0xc8:
 831         case 0xce:
 832         case 0xcf:
 833                 break;
 834         default:
 835                 BUG_ON(1);
 836                 break;
 837         }
 838         nv_wr32(priv, 0x4184a0, 0x00000000);
 839         switch (nv_device(priv)->chipset) {
 840         case 0xd9:
 841         case 0xd7:
 842                 nv_wr32(priv, 0x4184a4, 0x00000000);
 843                 nv_wr32(priv, 0x4184a8, 0x00000000);
 844                 break;
 845         case 0xc0:
 846         case 0xc3:
 847         case 0xc4:
 848         case 0xc1:
 849         case 0xc8:
 850         case 0xce:
 851         case 0xcf:
 852                 break;
 853         default:
 854                 BUG_ON(1);
 855                 break;
 856         }
 857         nv_wr32(priv, 0x418604, 0x00000000);
 858         nv_wr32(priv, 0x418680, 0x00000000);
 859         switch (nv_device(priv)->chipset) {
 860         case 0xd9:
 861         case 0xd7:
 862         case 0xc1:
 863                 nv_wr32(priv, 0x418714, 0x00000000);
 864                 break;
 865         case 0xc0:
 866         case 0xc3:
 867         case 0xc4:
 868         case 0xc8:
 869         case 0xce:
 870         case 0xcf:
 871                 nv_wr32(priv, 0x418714, 0x80000000);
 872                 break;
 873         default:
 874                 BUG_ON(1);
 875                 break;
 876         }
 877         nv_wr32(priv, 0x418384, 0x00000000);
 878         nv_wr32(priv, 0x418814, 0x00000000);
 879         nv_wr32(priv, 0x418818, 0x00000000);
 880         nv_wr32(priv, 0x41881c, 0x00000000);
 881         nv_wr32(priv, 0x418b04, 0x00000000);
 882         switch (nv_device(priv)->chipset) {
 883         case 0xd9:
 884         case 0xd7:
 885         case 0xc1:
 886         case 0xc8:
 887                 nv_wr32(priv, 0x4188c8, 0x00000000);
 888                 break;
 889         case 0xc0:
 890         case 0xc3:
 891         case 0xc4:
 892         case 0xce:
 893         case 0xcf:
 894                 nv_wr32(priv, 0x4188c8, 0x80000000);
 895                 break;
 896         default:
 897                 BUG_ON(1);
 898                 break;
 899         }
 900         nv_wr32(priv, 0x4188cc, 0x00000000);
 901         nv_wr32(priv, 0x4188d0, 0x00010000);
 902         nv_wr32(priv, 0x4188d4, 0x00000001);
 903         nv_wr32(priv, 0x418910, 0x00010001);
 904         nv_wr32(priv, 0x418914, 0x00000301);
 905         nv_wr32(priv, 0x418918, 0x00800000);
 906         nv_wr32(priv, 0x418980, 0x77777770);
 907         nv_wr32(priv, 0x418984, 0x77777777);
 908         nv_wr32(priv, 0x418988, 0x77777777);
 909         nv_wr32(priv, 0x41898c, 0x77777777);
 910         nv_wr32(priv, 0x418c04, 0x00000000);
 911         switch (nv_device(priv)->chipset) {
 912         case 0xd9:
 913         case 0xd7:
 914                 nv_wr32(priv, 0x418c64, 0x00000000);
 915                 nv_wr32(priv, 0x418c68, 0x00000000);
 916                 break;
 917         case 0xc0:
 918         case 0xc3:
 919         case 0xc4:
 920         case 0xc1:
 921         case 0xc8:
 922         case 0xce:
 923         case 0xcf:
 924                 break;
 925         default:
 926                 BUG_ON(1);
 927                 break;
 928         }
 929         nv_wr32(priv, 0x418c88, 0x00000000);
 930         switch (nv_device(priv)->chipset) {
 931         case 0xd9:
 932         case 0xd7:
 933                 nv_wr32(priv, 0x418cb4, 0x00000000);
 934                 nv_wr32(priv, 0x418cb8, 0x00000000);
 935                 break;
 936         case 0xc0:
 937         case 0xc3:
 938         case 0xc4:
 939         case 0xc1:
 940         case 0xc8:
 941         case 0xce:
 942         case 0xcf:
 943                 break;
 944         default:
 945                 BUG_ON(1);
 946                 break;
 947         }
 948         nv_wr32(priv, 0x418d00, 0x00000000);
 949         switch (nv_device(priv)->chipset) {
 950         case 0xd9:
 951         case 0xd7:
 952                 nv_wr32(priv, 0x418d28, 0x00000000);
 953                 nv_wr32(priv, 0x418d2c, 0x00000000);
 954                 nv_wr32(priv, 0x418f00, 0x00000000);
 955                 break;
 956         case 0xc0:
 957         case 0xc3:
 958         case 0xc4:
 959         case 0xc1:
 960         case 0xc8:
 961         case 0xce:
 962         case 0xcf:
 963                 break;
 964         default:
 965                 BUG_ON(1);
 966                 break;
 967         }
 968         nv_wr32(priv, 0x418f08, 0x00000000);
 969         switch (nv_device(priv)->chipset) {
 970         case 0xd9:
 971         case 0xd7:
 972                 nv_wr32(priv, 0x418f20, 0x00000000);
 973                 nv_wr32(priv, 0x418f24, 0x00000000);
 974                 /*fall-through*/
 975         case 0xc1:
 976                 nv_wr32(priv, 0x418e00, 0x00000003);
 977                 break;
 978         case 0xc0:
 979         case 0xc3:
 980         case 0xc4:
 981         case 0xc8:
 982         case 0xce:
 983         case 0xcf:
 984                 nv_wr32(priv, 0x418e00, 0x00000050);
 985                 break;
 986         default:
 987                 BUG_ON(1);
 988                 break;
 989         }
 990         nv_wr32(priv, 0x418e08, 0x00000000);
 991         switch (nv_device(priv)->chipset) {
 992         case 0xd9:
 993         case 0xd7:
 994                 nv_wr32(priv, 0x418e1c, 0x00000000);
 995                 nv_wr32(priv, 0x418e20, 0x00000000);
 996                 break;
 997         case 0xc0:
 998         case 0xc3:
 999         case 0xc4:
1000         case 0xc1:
1001         case 0xc8:
1002         case 0xce:
1003         case 0xcf:
1004                 break;
1005         default:
1006                 BUG_ON(1);
1007                 break;
1008         }
1009         nv_wr32(priv, 0x41900c, 0x00000000);
1010         nv_wr32(priv, 0x419018, 0x00000000);
1011 }
1012
1013 static void
1014 nvc0_graph_init_tpc(struct nvc0_graph_priv *priv)
1015 {
1016         nv_wr32(priv, 0x419d08, 0x00000000);
1017         nv_wr32(priv, 0x419d0c, 0x00000000);
1018         nv_wr32(priv, 0x419d10, 0x00000014);
1019         nv_wr32(priv, 0x419ab0, 0x00000000);
1020         switch (nv_device(priv)->chipset) {
1021         case 0xc3:
1022         case 0xc4:
1023         case 0xc1:
1024         case 0xce:
1025         case 0xcf:
1026         case 0xd9:
1027         case 0xd7:
1028                 nv_wr32(priv, 0x419ac8, 0x00000000);
1029                 break;
1030         case 0xc0:
1031         case 0xc8:
1032                 break;
1033         default:
1034                 BUG_ON(1);
1035                 break;
1036         }
1037         nv_wr32(priv, 0x419ab8, 0x000000e7);
1038         nv_wr32(priv, 0x419abc, 0x00000000);
1039         nv_wr32(priv, 0x419ac0, 0x00000000);
1040         switch (nv_device(priv)->chipset) {
1041         case 0xd9:
1042         case 0xd7:
1043                 nv_wr32(priv, 0x419ab4, 0x00000000);
1044                 nv_wr32(priv, 0x41980c, 0x00000010);
1045                 break;
1046         case 0xc0:
1047         case 0xc3:
1048         case 0xc4:
1049         case 0xc1:
1050         case 0xc8:
1051         case 0xce:
1052         case 0xcf:
1053                 nv_wr32(priv, 0x41980c, 0x00000000);
1054                 break;
1055         default:
1056                 BUG_ON(1);
1057                 break;
1058         }
1059         nv_wr32(priv, 0x419810, 0x00000000);
1060         switch (nv_device(priv)->chipset) {
1061         case 0xd9:
1062         case 0xd7:
1063         case 0xc1:
1064                 nv_wr32(priv, 0x419814, 0x00000004);
1065                 break;
1066         case 0xc0:
1067         case 0xc3:
1068         case 0xc4:
1069         case 0xc8:
1070         case 0xce:
1071         case 0xcf:
1072                 nv_wr32(priv, 0x419814, 0x00000000);
1073                 break;
1074         default:
1075                 BUG_ON(1);
1076                 break;
1077         }
1078         nv_wr32(priv, 0x419844, 0x00000000);
1079         switch (nv_device(priv)->chipset) {
1080         case 0xd9:
1081         case 0xd7:
1082                 nv_wr32(priv, 0x41984c, 0x0000a918);
1083                 break;
1084         case 0xc0:
1085         case 0xc3:
1086         case 0xc4:
1087         case 0xc1:
1088         case 0xc8:
1089         case 0xce:
1090         case 0xcf:
1091                 nv_wr32(priv, 0x41984c, 0x00005bc5);
1092                 break;
1093         default:
1094                 BUG_ON(1);
1095                 break;
1096         }
1097         nv_wr32(priv, 0x419850, 0x00000000);
1098         nv_wr32(priv, 0x419854, 0x00000000);
1099         nv_wr32(priv, 0x419858, 0x00000000);
1100         nv_wr32(priv, 0x41985c, 0x00000000);
1101         switch (nv_device(priv)->chipset) {
1102         case 0xc3:
1103         case 0xc4:
1104         case 0xc1:
1105         case 0xce:
1106         case 0xcf:
1107         case 0xd9:
1108         case 0xd7:
1109                 nv_wr32(priv, 0x419880, 0x00000002);
1110                 break;
1111         case 0xc0:
1112         case 0xc8:
1113                 break;
1114         default:
1115                 BUG_ON(1);
1116                 break;
1117         }
1118         nv_wr32(priv, 0x419c98, 0x00000000);
1119         nv_wr32(priv, 0x419ca8, 0x80000000);
1120         nv_wr32(priv, 0x419cb4, 0x00000000);
1121         nv_wr32(priv, 0x419cb8, 0x00008bf4);
1122         nv_wr32(priv, 0x419cbc, 0x28137606);
1123         nv_wr32(priv, 0x419cc0, 0x00000000);
1124         nv_wr32(priv, 0x419cc4, 0x00000000);
1125         nv_wr32(priv, 0x419bd4, 0x00800000);
1126         nv_wr32(priv, 0x419bdc, 0x00000000);
1127         switch (nv_device(priv)->chipset) {
1128         case 0xd9:
1129         case 0xd7:
1130                 nv_wr32(priv, 0x419bf8, 0x00000000);
1131                 nv_wr32(priv, 0x419bfc, 0x00000000);
1132                 break;
1133         case 0xc0:
1134         case 0xc3:
1135         case 0xc4:
1136         case 0xc1:
1137         case 0xc8:
1138         case 0xce:
1139         case 0xcf:
1140                 break;
1141         default:
1142                 BUG_ON(1);
1143                 break;
1144         }
1145         nv_wr32(priv, 0x419d2c, 0x00000000);
1146         switch (nv_device(priv)->chipset) {
1147         case 0xd9:
1148         case 0xd7:
1149                 nv_wr32(priv, 0x419d48, 0x00000000);
1150                 nv_wr32(priv, 0x419d4c, 0x00000000);
1151                 break;
1152         case 0xc0:
1153         case 0xc3:
1154         case 0xc4:
1155         case 0xc1:
1156         case 0xc8:
1157         case 0xce:
1158         case 0xcf:
1159                 break;
1160         default:
1161                 BUG_ON(1);
1162                 break;
1163         }
1164         nv_wr32(priv, 0x419c0c, 0x00000000);
1165         nv_wr32(priv, 0x419e00, 0x00000000);
1166         nv_wr32(priv, 0x419ea0, 0x00000000);
1167         nv_wr32(priv, 0x419ea4, 0x00000100);
1168         switch (nv_device(priv)->chipset) {
1169         case 0xd9:
1170         case 0xd7:
1171                 nv_wr32(priv, 0x419ea8, 0x02001100);
1172                 break;
1173         case 0xc0:
1174         case 0xc3:
1175         case 0xc4:
1176         case 0xc1:
1177         case 0xc8:
1178         case 0xce:
1179         case 0xcf:
1180                 nv_wr32(priv, 0x419ea8, 0x00001100);
1181                 break;
1182         default:
1183                 BUG_ON(1);
1184                 break;
1185         }
1186
1187         switch (nv_device(priv)->chipset) {
1188         case 0xc8:
1189                 nv_wr32(priv, 0x419eac, 0x11100f02);
1190                 break;
1191         case 0xc0:
1192         case 0xc3:
1193         case 0xc4:
1194         case 0xc1:
1195         case 0xce:
1196         case 0xcf:
1197         case 0xd9:
1198         case 0xd7:
1199                 nv_wr32(priv, 0x419eac, 0x11100702);
1200                 break;
1201         default:
1202                 BUG_ON(1);
1203                 break;
1204         }
1205         nv_wr32(priv, 0x419eb0, 0x00000003);
1206         nv_wr32(priv, 0x419eb4, 0x00000000);
1207         nv_wr32(priv, 0x419eb8, 0x00000000);
1208         nv_wr32(priv, 0x419ebc, 0x00000000);
1209         nv_wr32(priv, 0x419ec0, 0x00000000);
1210         switch (nv_device(priv)->chipset) {
1211         case 0xc3:
1212         case 0xc4:
1213         case 0xc1:
1214         case 0xce:
1215         case 0xcf:
1216         case 0xd9:
1217         case 0xd7:
1218                 nv_wr32(priv, 0x419ec8, 0x0e063818);
1219                 nv_wr32(priv, 0x419ecc, 0x0e060e06);
1220                 nv_wr32(priv, 0x419ed0, 0x00003818);
1221                 break;
1222         case 0xc0:
1223         case 0xc8:
1224                 nv_wr32(priv, 0x419ec8, 0x06060618);
1225                 nv_wr32(priv, 0x419ed0, 0x0eff0e38);
1226                 break;
1227         default:
1228                 BUG_ON(1);
1229                 break;
1230         }
1231         nv_wr32(priv, 0x419ed4, 0x011104f1);
1232         nv_wr32(priv, 0x419edc, 0x00000000);
1233         nv_wr32(priv, 0x419f00, 0x00000000);
1234         nv_wr32(priv, 0x419f2c, 0x00000000);
1235 }
1236
1237 static void
1238 nvc0_graph_init_unk88xx(struct nvc0_graph_priv *priv)
1239 {
1240         nv_wr32(priv, 0x40880c, 0x00000000);
1241         nv_wr32(priv, 0x408910, 0x00000000);
1242         nv_wr32(priv, 0x408914, 0x00000000);
1243         nv_wr32(priv, 0x408918, 0x00000000);
1244         nv_wr32(priv, 0x40891c, 0x00000000);
1245         nv_wr32(priv, 0x408920, 0x00000000);
1246         nv_wr32(priv, 0x408924, 0x00000000);
1247         nv_wr32(priv, 0x408928, 0x00000000);
1248         nv_wr32(priv, 0x40892c, 0x00000000);
1249         nv_wr32(priv, 0x408930, 0x00000000);
1250         nv_wr32(priv, 0x408950, 0x00000000);
1251         nv_wr32(priv, 0x408954, 0x0000ffff);
1252         nv_wr32(priv, 0x408984, 0x00000000);
1253         nv_wr32(priv, 0x408988, 0x08040201);
1254         nv_wr32(priv, 0x40898c, 0x80402010);
1255 }
1256
1257 static void
1258 nvc0_graph_init_gpc_0(struct nvc0_graph_priv *priv)
1259 {
1260         const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tpc_total);
1261         u32 data[TPC_MAX / 8];
1262         u8  tpcnr[GPC_MAX];
1263         int i, gpc, tpc;
1264
1265         nv_wr32(priv, TPC_UNIT(0, 0, 0x5c), 1); /* affects TFB offset queries */
1266
1267         /*
1268          *      TP      ROP UNKVAL(magic_not_rop_nr)
1269          * 450: 4/0/0/0 2        3
1270          * 460: 3/4/0/0 4        1
1271          * 465: 3/4/4/0 4        7
1272          * 470: 3/3/4/4 5        5
1273          * 480: 3/4/4/4 6        6
1274          */
1275
1276         memset(data, 0x00, sizeof(data));
1277         memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
1278         for (i = 0, gpc = -1; i < priv->tpc_total; i++) {
1279                 do {
1280                         gpc = (gpc + 1) % priv->gpc_nr;
1281                 } while (!tpcnr[gpc]);
1282                 tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
1283
1284                 data[i / 8] |= tpc << ((i % 8) * 4);
1285         }
1286
1287         nv_wr32(priv, GPC_BCAST(0x0980), data[0]);
1288         nv_wr32(priv, GPC_BCAST(0x0984), data[1]);
1289         nv_wr32(priv, GPC_BCAST(0x0988), data[2]);
1290         nv_wr32(priv, GPC_BCAST(0x098c), data[3]);
1291
1292         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
1293                 nv_wr32(priv, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
1294                                                   priv->tpc_nr[gpc]);
1295                 nv_wr32(priv, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tpc_total);
1296                 nv_wr32(priv, GPC_UNIT(gpc, 0x0918), magicgpc918);
1297         }
1298
1299         nv_wr32(priv, GPC_BCAST(0x1bd4), magicgpc918);
1300         nv_wr32(priv, GPC_BCAST(0x08ac), nv_rd32(priv, 0x100800));
1301 }
1302
1303 static void
1304 nvc0_graph_init_units(struct nvc0_graph_priv *priv)
1305 {
1306         nv_wr32(priv, 0x409c24, 0x000f0000);
1307         nv_wr32(priv, 0x404000, 0xc0000000); /* DISPATCH */
1308         nv_wr32(priv, 0x404600, 0xc0000000); /* M2MF */
1309         nv_wr32(priv, 0x408030, 0xc0000000);
1310         nv_wr32(priv, 0x40601c, 0xc0000000);
1311         nv_wr32(priv, 0x404490, 0xc0000000); /* MACRO */
1312         nv_wr32(priv, 0x406018, 0xc0000000);
1313         nv_wr32(priv, 0x405840, 0xc0000000);
1314         nv_wr32(priv, 0x405844, 0x00ffffff);
1315         nv_mask(priv, 0x419cc0, 0x00000008, 0x00000008);
1316         nv_mask(priv, 0x419eb4, 0x00001000, 0x00001000);
1317 }
1318
1319 static void
1320 nvc0_graph_init_gpc_1(struct nvc0_graph_priv *priv)
1321 {
1322         int gpc, tpc;
1323
1324         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
1325                 nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
1326                 nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
1327                 nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
1328                 nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
1329                 for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
1330                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
1331                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
1332                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
1333                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
1334                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
1335                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
1336                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
1337                 }
1338                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
1339                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
1340         }
1341 }
1342
1343 static void
1344 nvc0_graph_init_rop(struct nvc0_graph_priv *priv)
1345 {
1346         int rop;
1347
1348         for (rop = 0; rop < priv->rop_nr; rop++) {
1349                 nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
1350                 nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
1351                 nv_wr32(priv, ROP_UNIT(rop, 0x204), 0xffffffff);
1352                 nv_wr32(priv, ROP_UNIT(rop, 0x208), 0xffffffff);
1353         }
1354 }
1355
1356 void
1357 nvc0_graph_init_fw(struct nvc0_graph_priv *priv, u32 fuc_base,
1358                    struct nvc0_graph_fuc *code, struct nvc0_graph_fuc *data)
1359 {
1360         int i;
1361
1362         nv_wr32(priv, fuc_base + 0x01c0, 0x01000000);
1363         for (i = 0; i < data->size / 4; i++)
1364                 nv_wr32(priv, fuc_base + 0x01c4, data->data[i]);
1365
1366         nv_wr32(priv, fuc_base + 0x0180, 0x01000000);
1367         for (i = 0; i < code->size / 4; i++) {
1368                 if ((i & 0x3f) == 0)
1369                         nv_wr32(priv, fuc_base + 0x0188, i >> 6);
1370                 nv_wr32(priv, fuc_base + 0x0184, code->data[i]);
1371         }
1372 }
1373
1374 static int
1375 nvc0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
1376 {
1377         u32 r000260;
1378         int i;
1379
1380         if (priv->firmware) {
1381                 /* load fuc microcode */
1382                 r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
1383                 nvc0_graph_init_fw(priv, 0x409000, &priv->fuc409c,
1384                                                    &priv->fuc409d);
1385                 nvc0_graph_init_fw(priv, 0x41a000, &priv->fuc41ac,
1386                                                    &priv->fuc41ad);
1387                 nv_wr32(priv, 0x000260, r000260);
1388
1389                 /* start both of them running */
1390                 nv_wr32(priv, 0x409840, 0xffffffff);
1391                 nv_wr32(priv, 0x41a10c, 0x00000000);
1392                 nv_wr32(priv, 0x40910c, 0x00000000);
1393                 nv_wr32(priv, 0x41a100, 0x00000002);
1394                 nv_wr32(priv, 0x409100, 0x00000002);
1395                 if (!nv_wait(priv, 0x409800, 0x00000001, 0x00000001))
1396                         nv_warn(priv, "0x409800 wait failed\n");
1397
1398                 nv_wr32(priv, 0x409840, 0xffffffff);
1399                 nv_wr32(priv, 0x409500, 0x7fffffff);
1400                 nv_wr32(priv, 0x409504, 0x00000021);
1401
1402                 nv_wr32(priv, 0x409840, 0xffffffff);
1403                 nv_wr32(priv, 0x409500, 0x00000000);
1404                 nv_wr32(priv, 0x409504, 0x00000010);
1405                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
1406                         nv_error(priv, "fuc09 req 0x10 timeout\n");
1407                         return -EBUSY;
1408                 }
1409                 priv->size = nv_rd32(priv, 0x409800);
1410
1411                 nv_wr32(priv, 0x409840, 0xffffffff);
1412                 nv_wr32(priv, 0x409500, 0x00000000);
1413                 nv_wr32(priv, 0x409504, 0x00000016);
1414                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
1415                         nv_error(priv, "fuc09 req 0x16 timeout\n");
1416                         return -EBUSY;
1417                 }
1418
1419                 nv_wr32(priv, 0x409840, 0xffffffff);
1420                 nv_wr32(priv, 0x409500, 0x00000000);
1421                 nv_wr32(priv, 0x409504, 0x00000025);
1422                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
1423                         nv_error(priv, "fuc09 req 0x25 timeout\n");
1424                         return -EBUSY;
1425                 }
1426
1427                 if (priv->data == NULL) {
1428                         int ret = nvc0_grctx_generate(priv);
1429                         if (ret) {
1430                                 nv_error(priv, "failed to construct context\n");
1431                                 return ret;
1432                         }
1433                 }
1434
1435                 return 0;
1436         }
1437
1438         /* load HUB microcode */
1439         r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
1440         nv_wr32(priv, 0x4091c0, 0x01000000);
1441         for (i = 0; i < sizeof(nvc0_grhub_data) / 4; i++)
1442                 nv_wr32(priv, 0x4091c4, nvc0_grhub_data[i]);
1443
1444         nv_wr32(priv, 0x409180, 0x01000000);
1445         for (i = 0; i < sizeof(nvc0_grhub_code) / 4; i++) {
1446                 if ((i & 0x3f) == 0)
1447                         nv_wr32(priv, 0x409188, i >> 6);
1448                 nv_wr32(priv, 0x409184, nvc0_grhub_code[i]);
1449         }
1450
1451         /* load GPC microcode */
1452         nv_wr32(priv, 0x41a1c0, 0x01000000);
1453         for (i = 0; i < sizeof(nvc0_grgpc_data) / 4; i++)
1454                 nv_wr32(priv, 0x41a1c4, nvc0_grgpc_data[i]);
1455
1456         nv_wr32(priv, 0x41a180, 0x01000000);
1457         for (i = 0; i < sizeof(nvc0_grgpc_code) / 4; i++) {
1458                 if ((i & 0x3f) == 0)
1459                         nv_wr32(priv, 0x41a188, i >> 6);
1460                 nv_wr32(priv, 0x41a184, nvc0_grgpc_code[i]);
1461         }
1462         nv_wr32(priv, 0x000260, r000260);
1463
1464         /* start HUB ucode running, it'll init the GPCs */
1465         nv_wr32(priv, 0x409800, nv_device(priv)->chipset);
1466         nv_wr32(priv, 0x40910c, 0x00000000);
1467         nv_wr32(priv, 0x409100, 0x00000002);
1468         if (!nv_wait(priv, 0x409800, 0x80000000, 0x80000000)) {
1469                 nv_error(priv, "HUB_INIT timed out\n");
1470                 nvc0_graph_ctxctl_debug(priv);
1471                 return -EBUSY;
1472         }
1473
1474         priv->size = nv_rd32(priv, 0x409804);
1475         if (priv->data == NULL) {
1476                 int ret = nvc0_grctx_generate(priv);
1477                 if (ret) {
1478                         nv_error(priv, "failed to construct context\n");
1479                         return ret;
1480                 }
1481         }
1482
1483         return 0;
1484 }
1485
1486 static int
1487 nvc0_graph_init(struct nouveau_object *object)
1488 {
1489         struct nvc0_graph_priv *priv = (void *)object;
1490         int ret;
1491
1492         ret = nouveau_graph_init(&priv->base);
1493         if (ret)
1494                 return ret;
1495
1496         nvc0_graph_init_obj418880(priv);
1497         nvc0_graph_init_regs(priv);
1498         nvc0_graph_init_unk40xx(priv);
1499         nvc0_graph_init_unk44xx(priv);
1500         nvc0_graph_init_unk78xx(priv);
1501         nvc0_graph_init_unk60xx(priv);
1502         nvc0_graph_init_unk64xx(priv);
1503         nvc0_graph_init_unk58xx(priv);
1504         nvc0_graph_init_unk80xx(priv);
1505         nvc0_graph_init_gpc(priv);
1506         nvc0_graph_init_tpc(priv);
1507         nvc0_graph_init_unk88xx(priv);
1508         nvc0_graph_init_gpc_0(priv);
1509         /*nvc0_graph_init_unitplemented_c242(priv);*/
1510
1511         nv_wr32(priv, 0x400500, 0x00010001);
1512         nv_wr32(priv, 0x400100, 0xffffffff);
1513         nv_wr32(priv, 0x40013c, 0xffffffff);
1514
1515         nvc0_graph_init_units(priv);
1516         nvc0_graph_init_gpc_1(priv);
1517         nvc0_graph_init_rop(priv);
1518
1519         nv_wr32(priv, 0x400108, 0xffffffff);
1520         nv_wr32(priv, 0x400138, 0xffffffff);
1521         nv_wr32(priv, 0x400118, 0xffffffff);
1522         nv_wr32(priv, 0x400130, 0xffffffff);
1523         nv_wr32(priv, 0x40011c, 0xffffffff);
1524         nv_wr32(priv, 0x400134, 0xffffffff);
1525         nv_wr32(priv, 0x400054, 0x34ce3464);
1526
1527         ret = nvc0_graph_init_ctxctl(priv);
1528         if (ret)
1529                 return ret;
1530
1531         return 0;
1532 }
1533
1534 struct nouveau_oclass
1535 nvc0_graph_oclass = {
1536         .handle = NV_ENGINE(GR, 0xc0),
1537         .ofuncs = &(struct nouveau_ofuncs) {
1538                 .ctor = nvc0_graph_ctor,
1539                 .dtor = nvc0_graph_dtor,
1540                 .init = nvc0_graph_init,
1541                 .fini = _nouveau_graph_fini,
1542         },
1543 };