drivers/gpu/drm/nouveau/core/engine/graph/nve0.c

   1 /*
   2  * Copyright 2012 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Ben Skeggs
  23  */
  24
  25 #include "nvc0.h"
  26 #include "fuc/hubnve0.fuc.h"
  27 #include "fuc/gpcnve0.fuc.h"
  28
  29 /*******************************************************************************
  30  * Graphics object classes
  31  ******************************************************************************/
  32
  33 static struct nouveau_oclass
  34 nve0_graph_sclass[] = {
  35         { 0x902d, &nouveau_object_ofuncs },
  36         { 0xa040, &nouveau_object_ofuncs },
  37         { 0xa097, &nouveau_object_ofuncs },
  38         { 0xa0c0, &nouveau_object_ofuncs },
  39         {}
  40 };
  41
  42 /*******************************************************************************
  43  * PGRAPH context
  44  ******************************************************************************/
  45
  46 static struct nouveau_oclass
  47 nve0_graph_cclass = {
  48         .handle = NV_ENGCTX(GR, 0xe0),
  49         .ofuncs = &(struct nouveau_ofuncs) {
  50                 .ctor = nvc0_graph_context_ctor,
  51                 .dtor = nvc0_graph_context_dtor,
  52                 .init = _nouveau_graph_context_init,
  53                 .fini = _nouveau_graph_context_fini,
  54                 .rd32 = _nouveau_graph_context_rd32,
  55                 .wr32 = _nouveau_graph_context_wr32,
  56         },
  57 };
  58
  59 /*******************************************************************************
  60  * PGRAPH engine/subdev functions
  61  ******************************************************************************/
  62
  63 static void
  64 nve0_graph_ctxctl_isr(struct nvc0_graph_priv *priv)
  65 {
  66         u32 ustat = nv_rd32(priv, 0x409c18);
  67
  68         if (ustat & 0x00000001)
  69                 nv_error(priv, "CTXCTRL ucode error\n");
  70         if (ustat & 0x00080000)
  71                 nv_error(priv, "CTXCTRL watchdog timeout\n");
  72         if (ustat & ~0x00080001)
  73                 nv_error(priv, "CTXCTRL 0x%08x\n", ustat);
  74
  75         nvc0_graph_ctxctl_debug(priv);
  76         nv_wr32(priv, 0x409c20, ustat);
  77 }
  78
  79 static const struct nouveau_enum nve0_mp_warp_error[] = {
  80         { 0x00, "NO_ERROR" },
  81         { 0x01, "STACK_MISMATCH" },
  82         { 0x05, "MISALIGNED_PC" },
  83         { 0x08, "MISALIGNED_GPR" },
  84         { 0x09, "INVALID_OPCODE" },
  85         { 0x0d, "GPR_OUT_OF_BOUNDS" },
  86         { 0x0e, "MEM_OUT_OF_BOUNDS" },
  87         { 0x0f, "UNALIGNED_MEM_ACCESS" },
  88         { 0x11, "INVALID_PARAM" },
  89         {}
  90 };
  91
  92 static const struct nouveau_enum nve0_mp_global_error[] = {
  93         { 2, "MULTIPLE_WARP_ERRORS" },
  94         { 3, "OUT_OF_STACK_SPACE" },
  95         {}
  96 };
  97
  98 static const struct nouveau_enum nve0_gpc_rop_error[] = {
  99         { 1, "RT_PITCH_OVERRUN" },
 100         { 4, "RT_WIDTH_OVERRUN" },
 101         { 5, "RT_HEIGHT_OVERRUN" },
 102         { 7, "ZETA_STORAGE_TYPE_MISMATCH" },
 103         { 8, "RT_STORAGE_TYPE_MISMATCH" },
 104         { 10, "RT_LINEAR_MISMATCH" },
 105         {}
 106 };
 107
 108 static const struct nouveau_enum nve0_sked_error[] = {
 109         { 7, "CONSTANT_BUFFER_SIZE" },
 110         { 9, "LOCAL_MEMORY_SIZE_POS" },
 111         { 10, "LOCAL_MEMORY_SIZE_NEG" },
 112         { 11, "WARP_CSTACK_SIZE" },
 113         { 12, "TOTAL_TEMP_SIZE" },
 114         { 13, "REGISTER_COUNT" },
 115         { 18, "TOTAL_THREADS" },
 116         { 20, "PROGRAM_OFFSET" },
 117         { 21, "SHARED_MEMORY_SIZE" },
 118         { 25, "SHARED_CONFIG_TOO_SMALL" },
 119         { 26, "TOTAL_REGISTER_COUNT" },
 120         {}
 121 };
 122
 123 static void
 124 nve0_graph_mp_trap(struct nvc0_graph_priv *priv, int gpc, int tp)
 125 {
 126         int i;
 127         u32 werr = nv_rd32(priv, TPC_UNIT(gpc, tp, 0x648));
 128         u32 gerr = nv_rd32(priv, TPC_UNIT(gpc, tp, 0x650));
 129
 130         nv_error(priv, "GPC%i/TP%i/MP trap:", gpc, tp);
 131
 132         for (i = 0; i <= 31; ++i) {
 133                 if (!(gerr & (1 << i)))
 134                         continue;
 135                 pr_cont(" ");
 136                 nouveau_enum_print(nve0_mp_global_error, i);
 137         }
 138         if (werr) {
 139                 pr_cont(" ");
 140                 nouveau_enum_print(nve0_mp_warp_error, werr & 0xffff);
 141         }
 142         pr_cont("\n");
 143
 144         /* disable MP trap to avoid spam */
 145         nv_mask(priv, TPC_UNIT(gpc, tp, 0x50c), 0x2, 0x0);
 146
 147         /* TODO: figure out how to resume after an MP trap */
 148 }
 149
 150 static void
 151 nve0_graph_tp_trap(struct nvc0_graph_priv *priv, int gpc, int tp)
 152 {
 153         u32 stat = nv_rd32(priv, TPC_UNIT(gpc, tp, 0x508));
 154
 155         if (stat & 0x1) {
 156                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tp, 0x224));
 157                 nv_error(priv, "GPC%i/TP%i/TEX trap: %08x\n",
 158                          gpc, tp, trap);
 159
 160                 nv_wr32(priv, TPC_UNIT(gpc, tp, 0x224), 0xc0000000);
 161                 stat &= ~0x1;
 162         }
 163
 164         if (stat & 0x2) {
 165                 nve0_graph_mp_trap(priv, gpc, tp);
 166                 stat &= ~0x2;
 167         }
 168
 169         if (stat & 0x4) {
 170                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tp, 0x084));
 171                 nv_error(priv, "GPC%i/TP%i/POLY trap: %08x\n",
 172                          gpc, tp, trap);
 173
 174                 nv_wr32(priv, TPC_UNIT(gpc, tp, 0x084), 0xc0000000);
 175                 stat &= ~0x4;
 176         }
 177
 178         if (stat & 0x8) {
 179                 u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tp, 0x48c));
 180                 nv_error(priv, "GPC%i/TP%i/L1C trap: %08x\n",
 181                          gpc, tp, trap);
 182
 183                 nv_wr32(priv, TPC_UNIT(gpc, tp, 0x48c), 0xc0000000);
 184                 stat &= ~0x8;
 185         }
 186
 187         if (stat) {
 188                 nv_error(priv, "GPC%i/TP%i: unknown stat %08x\n",
 189                          gpc, tp, stat);
 190         }
 191 }
 192
 193 static void
 194 nve0_graph_gpc_trap(struct nvc0_graph_priv *priv)
 195 {
 196         const u32 mask = nv_rd32(priv, 0x400118);
 197         int gpc;
 198
 199         for (gpc = 0; gpc < 4; ++gpc) {
 200                 u32 stat;
 201                 int tp;
 202
 203                 if (!(mask & (1 << gpc)))
 204                         continue;
 205                 stat = nv_rd32(priv, GPC_UNIT(gpc, 0x2c90));
 206
 207                 if (stat & 0x0001) {
 208                         u32 trap[4];
 209                         int i;
 210
 211                         trap[0] = nv_rd32(priv, GPC_UNIT(gpc, 0x0420));
 212                         trap[1] = nv_rd32(priv, GPC_UNIT(gpc, 0x0434));
 213                         trap[2] = nv_rd32(priv, GPC_UNIT(gpc, 0x0438));
 214                         trap[3] = nv_rd32(priv, GPC_UNIT(gpc, 0x043c));
 215
 216                         nv_error(priv, "GPC%i/PROP trap:", gpc);
 217                         for (i = 0; i <= 29; ++i) {
 218                                 if (!(trap[0] & (1 << i)))
 219                                         continue;
 220                                 pr_cont(" ");
 221                                 nouveau_enum_print(nve0_gpc_rop_error, i);
 222                         }
 223                         pr_cont("\n");
 224
 225                         nv_error(priv, "x = %u, y = %u, "
 226                                  "format = %x, storage type = %x\n",
 227                                  trap[1] & 0xffff,
 228                                  trap[1] >> 16,
 229                                  (trap[2] >> 8) & 0x3f,
 230                                  trap[3] & 0xff);
 231
 232                         nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
 233                         stat &= ~0x0001;
 234                 }
 235
 236                 if (stat & 0x0002) {
 237                         u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0900));
 238                         nv_error(priv, "GPC%i/ZCULL trap: %08x\n", gpc,
 239                                  trap);
 240                         nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
 241                         stat &= ~0x0002;
 242                 }
 243
 244                 if (stat & 0x0004) {
 245                         u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x1028));
 246                         nv_error(priv, "GPC%i/CCACHE trap: %08x\n", gpc,
 247                                  trap);
 248                         nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
 249                         stat &= ~0x0004;
 250                 }
 251
 252                 if (stat & 0x0008) {
 253                         u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0824));
 254                         nv_error(priv, "GPC%i/ESETUP trap %08x\n", gpc,
 255                                  trap);
 256                         nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
 257                         stat &= ~0x0008;
 258                 }
 259
 260                 for (tp = 0; tp < 8; ++tp) {
 261                         if (stat & (1 << (16 + tp)))
 262                                 nve0_graph_tp_trap(priv, gpc, tp);
 263                 }
 264                 stat &= ~0xff0000;
 265
 266                 if (stat) {
 267                         nv_error(priv, "GPC%i: unknown stat %08x\n",
 268                                  gpc, stat);
 269                 }
 270         }
 271 }
 272
 273
 274 static void
 275 nve0_graph_trap_isr(struct nvc0_graph_priv *priv, int chid, u64 inst,
 276                 struct nouveau_object *engctx)
 277 {
 278         u32 trap = nv_rd32(priv, 0x400108);
 279         int i;
 280         int rop;
 281
 282         if (trap & 0x00000001) {
 283                 u32 stat = nv_rd32(priv, 0x404000);
 284                 nv_error(priv, "DISPATCH ch %d [0x%010llx %s] 0x%08x\n",
 285                          chid, inst, nouveau_client_name(engctx), stat);
 286                 nv_wr32(priv, 0x404000, 0xc0000000);
 287                 nv_wr32(priv, 0x400108, 0x00000001);
 288                 trap &= ~0x00000001;
 289         }
 290
 291         if (trap & 0x00000010) {
 292                 u32 stat = nv_rd32(priv, 0x405840);
 293                 nv_error(priv, "SHADER ch %d [0x%010llx %s] 0x%08x\n",
 294                          chid, inst, nouveau_client_name(engctx), stat);
 295                 nv_wr32(priv, 0x405840, 0xc0000000);
 296                 nv_wr32(priv, 0x400108, 0x00000010);
 297                 trap &= ~0x00000010;
 298         }
 299
 300         if (trap & 0x00000100) {
 301                 u32 stat = nv_rd32(priv, 0x407020);
 302                 nv_error(priv, "SKED ch %d [0x%010llx %s]:",
 303                          chid, inst, nouveau_client_name(engctx));
 304
 305                 for (i = 0; i <= 29; ++i) {
 306                         if (!(stat & (1 << i)))
 307                                 continue;
 308                         pr_cont(" ");
 309                         nouveau_enum_print(nve0_sked_error, i);
 310                 }
 311                 pr_cont("\n");
 312
 313                 if (stat & 0x3fffffff)
 314                         nv_wr32(priv, 0x407020, 0x40000000);
 315                 nv_wr32(priv, 0x400108, 0x00000100);
 316                 trap &= ~0x00000100;
 317         }
 318
 319         if (trap & 0x01000000) {
 320                 nv_error(priv, "GPC ch %d [0x%010llx %s]:\n",
 321                          chid, inst, nouveau_client_name(engctx));
 322                 nve0_graph_gpc_trap(priv);
 323                 trap &= ~0x01000000;
 324         }
 325
 326         if (trap & 0x02000000) {
 327                 for (rop = 0; rop < priv->rop_nr; rop++) {
 328                         u32 statz = nv_rd32(priv, ROP_UNIT(rop, 0x070));
 329                         u32 statc = nv_rd32(priv, ROP_UNIT(rop, 0x144));
 330                         nv_error(priv,
 331                                  "ROP%d ch %d [0x%010llx %s] 0x%08x 0x%08x\n",
 332                                  rop, chid, inst, nouveau_client_name(engctx),
 333                                  statz, statc);
 334                         nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
 335                         nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
 336                 }
 337                 nv_wr32(priv, 0x400108, 0x02000000);
 338                 trap &= ~0x02000000;
 339         }
 340
 341         if (trap) {
 342                 nv_error(priv, "TRAP ch %d [0x%010llx %s] 0x%08x\n",
 343                          chid, inst, nouveau_client_name(engctx), trap);
 344                 nv_wr32(priv, 0x400108, trap);
 345         }
 346 }
 347
 348 static void
 349 nve0_graph_intr(struct nouveau_subdev *subdev)
 350 {
 351         struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
 352         struct nouveau_engine *engine = nv_engine(subdev);
 353         struct nouveau_object *engctx;
 354         struct nouveau_handle *handle;
 355         struct nvc0_graph_priv *priv = (void *)subdev;
 356         u64 inst = nv_rd32(priv, 0x409b00) & 0x0fffffff;
 357         u32 stat = nv_rd32(priv, 0x400100);
 358         u32 addr = nv_rd32(priv, 0x400704);
 359         u32 mthd = (addr & 0x00003ffc);
 360         u32 subc = (addr & 0x00070000) >> 16;
 361         u32 data = nv_rd32(priv, 0x400708);
 362         u32 code = nv_rd32(priv, 0x400110);
 363         u32 class = nv_rd32(priv, 0x404200 + (subc * 4));
 364         int chid;
 365
 366         engctx = nouveau_engctx_get(engine, inst);
 367         chid   = pfifo->chid(pfifo, engctx);
 368
 369         if (stat & 0x00000010) {
 370                 handle = nouveau_handle_get_class(engctx, class);
 371                 if (!handle || nv_call(handle->object, mthd, data)) {
 372                         nv_error(priv,
 373                                  "ILLEGAL_MTHD ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
 374                                  chid, inst, nouveau_client_name(engctx), subc,
 375                                  class, mthd, data);
 376                 }
 377                 nouveau_handle_put(handle);
 378                 nv_wr32(priv, 0x400100, 0x00000010);
 379                 stat &= ~0x00000010;
 380         }
 381
 382         if (stat & 0x00000020) {
 383                 nv_error(priv,
 384                          "ILLEGAL_CLASS ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
 385                          chid, inst, nouveau_client_name(engctx), subc, class,
 386                          mthd, data);
 387                 nv_wr32(priv, 0x400100, 0x00000020);
 388                 stat &= ~0x00000020;
 389         }
 390
 391         if (stat & 0x00100000) {
 392                 nv_error(priv, "DATA_ERROR [");
 393                 nouveau_enum_print(nv50_data_error_names, code);
 394                 pr_cont("] ch %d [0x%010llx %s] subc %d class 0x%04x mthd 0x%04x data 0x%08x\n",
 395                         chid, inst, nouveau_client_name(engctx), subc, class,
 396                         mthd, data);
 397                 nv_wr32(priv, 0x400100, 0x00100000);
 398                 stat &= ~0x00100000;
 399         }
 400
 401         if (stat & 0x00200000) {
 402                 nve0_graph_trap_isr(priv, chid, inst, engctx);
 403                 nv_wr32(priv, 0x400100, 0x00200000);
 404                 stat &= ~0x00200000;
 405         }
 406
 407         if (stat & 0x00080000) {
 408                 nve0_graph_ctxctl_isr(priv);
 409                 nv_wr32(priv, 0x400100, 0x00080000);
 410                 stat &= ~0x00080000;
 411         }
 412
 413         if (stat) {
 414                 nv_error(priv, "unknown stat 0x%08x\n", stat);
 415                 nv_wr32(priv, 0x400100, stat);
 416         }
 417
 418         nv_wr32(priv, 0x400500, 0x00010001);
 419         nouveau_engctx_put(engctx);
 420 }
 421
 422 static int
 423 nve0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 424                struct nouveau_oclass *oclass, void *data, u32 size,
 425                struct nouveau_object **pobject)
 426 {
 427         struct nouveau_device *device = nv_device(parent);
 428         struct nvc0_graph_priv *priv;
 429         int ret, i;
 430
 431         ret = nouveau_graph_create(parent, engine, oclass, true, &priv);
 432         *pobject = nv_object(priv);
 433         if (ret)
 434                 return ret;
 435
 436         nv_subdev(priv)->unit = 0x18001000;
 437         nv_subdev(priv)->intr = nve0_graph_intr;
 438         nv_engine(priv)->cclass = &nve0_graph_cclass;
 439         nv_engine(priv)->sclass = nve0_graph_sclass;
 440
 441         priv->base.units = nvc0_graph_units;
 442
 443         if (nouveau_boolopt(device->cfgopt, "NvGrUseFW", false)) {
 444                 nv_info(priv, "using external firmware\n");
 445                 if (nvc0_graph_ctor_fw(priv, "fuc409c", &priv->fuc409c) ||
 446                     nvc0_graph_ctor_fw(priv, "fuc409d", &priv->fuc409d) ||
 447                     nvc0_graph_ctor_fw(priv, "fuc41ac", &priv->fuc41ac) ||
 448                     nvc0_graph_ctor_fw(priv, "fuc41ad", &priv->fuc41ad))
 449                         return -EINVAL;
 450                 priv->firmware = true;
 451         }
 452
 453         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 256, 0,
 454                                 &priv->unk4188b4);
 455         if (ret)
 456                 return ret;
 457
 458         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 256, 0,
 459                                 &priv->unk4188b8);
 460         if (ret)
 461                 return ret;
 462
 463         for (i = 0; i < 0x1000; i += 4) {
 464                 nv_wo32(priv->unk4188b4, i, 0x00000010);
 465                 nv_wo32(priv->unk4188b8, i, 0x00000010);
 466         }
 467
 468         priv->gpc_nr =  nv_rd32(priv, 0x409604) & 0x0000001f;
 469         priv->rop_nr = (nv_rd32(priv, 0x409604) & 0x001f0000) >> 16;
 470         for (i = 0; i < priv->gpc_nr; i++) {
 471                 priv->tpc_nr[i] = nv_rd32(priv, GPC_UNIT(i, 0x2608));
 472                 priv->tpc_total += priv->tpc_nr[i];
 473         }
 474
 475         switch (nv_device(priv)->chipset) {
 476         case 0xe4:
 477                 if (priv->tpc_total == 8)
 478                         priv->magic_not_rop_nr = 3;
 479                 else
 480                 if (priv->tpc_total == 7)
 481                         priv->magic_not_rop_nr = 1;
 482                 break;
 483         case 0xe7:
 484         case 0xe6:
 485                 priv->magic_not_rop_nr = 1;
 486                 break;
 487         default:
 488                 break;
 489         }
 490
 491         return 0;
 492 }
 493
 494 static void
 495 nve0_graph_init_obj418880(struct nvc0_graph_priv *priv)
 496 {
 497         int i;
 498
 499         nv_wr32(priv, GPC_BCAST(0x0880), 0x00000000);
 500         nv_wr32(priv, GPC_BCAST(0x08a4), 0x00000000);
 501         for (i = 0; i < 4; i++)
 502                 nv_wr32(priv, GPC_BCAST(0x0888) + (i * 4), 0x00000000);
 503         nv_wr32(priv, GPC_BCAST(0x08b4), priv->unk4188b4->addr >> 8);
 504         nv_wr32(priv, GPC_BCAST(0x08b8), priv->unk4188b8->addr >> 8);
 505 }
 506
 507 static void
 508 nve0_graph_init_regs(struct nvc0_graph_priv *priv)
 509 {
 510         nv_wr32(priv, 0x400080, 0x003083c2);
 511         nv_wr32(priv, 0x400088, 0x0001ffe7);
 512         nv_wr32(priv, 0x40008c, 0x00000000);
 513         nv_wr32(priv, 0x400090, 0x00000030);
 514         nv_wr32(priv, 0x40013c, 0x003901f7);
 515         nv_wr32(priv, 0x400140, 0x00000100);
 516         nv_wr32(priv, 0x400144, 0x00000000);
 517         nv_wr32(priv, 0x400148, 0x00000110);
 518         nv_wr32(priv, 0x400138, 0x00000000);
 519         nv_wr32(priv, 0x400130, 0x00000000);
 520         nv_wr32(priv, 0x400134, 0x00000000);
 521         nv_wr32(priv, 0x400124, 0x00000002);
 522 }
 523
 524 static void
 525 nve0_graph_init_units(struct nvc0_graph_priv *priv)
 526 {
 527         nv_wr32(priv, 0x409ffc, 0x00000000);
 528         nv_wr32(priv, 0x409c14, 0x00003e3e);
 529         nv_wr32(priv, 0x409c24, 0x000f0000);
 530
 531         nv_wr32(priv, 0x404000, 0xc0000000);
 532         nv_wr32(priv, 0x404600, 0xc0000000);
 533         nv_wr32(priv, 0x408030, 0xc0000000);
 534         nv_wr32(priv, 0x404490, 0xc0000000);
 535         nv_wr32(priv, 0x406018, 0xc0000000);
 536         nv_wr32(priv, 0x407020, 0xc0000000);
 537         nv_wr32(priv, 0x405840, 0xc0000000);
 538         nv_wr32(priv, 0x405844, 0x00ffffff);
 539
 540         nv_mask(priv, 0x419cc0, 0x00000008, 0x00000008);
 541         nv_mask(priv, 0x419eb4, 0x00001000, 0x00001000);
 542
 543 }
 544
 545 static void
 546 nve0_graph_init_gpc_0(struct nvc0_graph_priv *priv)
 547 {
 548         const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tpc_total);
 549         u32 data[TPC_MAX / 8];
 550         u8  tpcnr[GPC_MAX];
 551         int i, gpc, tpc;
 552
 553         nv_wr32(priv, GPC_UNIT(0, 0x3018), 0x00000001);
 554
 555         memset(data, 0x00, sizeof(data));
 556         memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
 557         for (i = 0, gpc = -1; i < priv->tpc_total; i++) {
 558                 do {
 559                         gpc = (gpc + 1) % priv->gpc_nr;
 560                 } while (!tpcnr[gpc]);
 561                 tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
 562
 563                 data[i / 8] |= tpc << ((i % 8) * 4);
 564         }
 565
 566         nv_wr32(priv, GPC_BCAST(0x0980), data[0]);
 567         nv_wr32(priv, GPC_BCAST(0x0984), data[1]);
 568         nv_wr32(priv, GPC_BCAST(0x0988), data[2]);
 569         nv_wr32(priv, GPC_BCAST(0x098c), data[3]);
 570
 571         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
 572                 nv_wr32(priv, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
 573                                                   priv->tpc_nr[gpc]);
 574                 nv_wr32(priv, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tpc_total);
 575                 nv_wr32(priv, GPC_UNIT(gpc, 0x0918), magicgpc918);
 576         }
 577
 578         nv_wr32(priv, GPC_BCAST(0x3fd4), magicgpc918);
 579         nv_wr32(priv, GPC_BCAST(0x08ac), nv_rd32(priv, 0x100800));
 580 }
 581
 582 static void
 583 nve0_graph_init_gpc_1(struct nvc0_graph_priv *priv)
 584 {
 585         int gpc, tpc;
 586
 587         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
 588                 nv_wr32(priv, GPC_UNIT(gpc, 0x3038), 0xc0000000);
 589                 nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
 590                 nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
 591                 nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
 592                 nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
 593                 for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
 594                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
 595                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
 596                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
 597                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
 598                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
 599                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
 600                         nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
 601                 }
 602                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
 603                 nv_wr32(priv, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
 604         }
 605 }
 606
 607 static void
 608 nve0_graph_init_rop(struct nvc0_graph_priv *priv)
 609 {
 610         int rop;
 611
 612         for (rop = 0; rop < priv->rop_nr; rop++) {
 613                 nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
 614                 nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
 615                 nv_wr32(priv, ROP_UNIT(rop, 0x204), 0xffffffff);
 616                 nv_wr32(priv, ROP_UNIT(rop, 0x208), 0xffffffff);
 617         }
 618 }
 619
 620 static int
 621 nve0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
 622 {
 623         u32 r000260;
 624         int i;
 625
 626         if (priv->firmware) {
 627                 /* load fuc microcode */
 628                 r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
 629                 nvc0_graph_init_fw(priv, 0x409000, &priv->fuc409c, &priv->fuc409d);
 630                 nvc0_graph_init_fw(priv, 0x41a000, &priv->fuc41ac, &priv->fuc41ad);
 631                 nv_wr32(priv, 0x000260, r000260);
 632
 633                 /* start both of them running */
 634                 nv_wr32(priv, 0x409840, 0xffffffff);
 635                 nv_wr32(priv, 0x41a10c, 0x00000000);
 636                 nv_wr32(priv, 0x40910c, 0x00000000);
 637                 nv_wr32(priv, 0x41a100, 0x00000002);
 638                 nv_wr32(priv, 0x409100, 0x00000002);
 639                 if (!nv_wait(priv, 0x409800, 0x00000001, 0x00000001))
 640                         nv_error(priv, "0x409800 wait failed\n");
 641
 642                 nv_wr32(priv, 0x409840, 0xffffffff);
 643                 nv_wr32(priv, 0x409500, 0x7fffffff);
 644                 nv_wr32(priv, 0x409504, 0x00000021);
 645
 646                 nv_wr32(priv, 0x409840, 0xffffffff);
 647                 nv_wr32(priv, 0x409500, 0x00000000);
 648                 nv_wr32(priv, 0x409504, 0x00000010);
 649                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 650                         nv_error(priv, "fuc09 req 0x10 timeout\n");
 651                         return -EBUSY;
 652                 }
 653                 priv->size = nv_rd32(priv, 0x409800);
 654
 655                 nv_wr32(priv, 0x409840, 0xffffffff);
 656                 nv_wr32(priv, 0x409500, 0x00000000);
 657                 nv_wr32(priv, 0x409504, 0x00000016);
 658                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 659                         nv_error(priv, "fuc09 req 0x16 timeout\n");
 660                         return -EBUSY;
 661                 }
 662
 663                 nv_wr32(priv, 0x409840, 0xffffffff);
 664                 nv_wr32(priv, 0x409500, 0x00000000);
 665                 nv_wr32(priv, 0x409504, 0x00000025);
 666                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 667                         nv_error(priv, "fuc09 req 0x25 timeout\n");
 668                         return -EBUSY;
 669                 }
 670
 671                 nv_wr32(priv, 0x409800, 0x00000000);
 672                 nv_wr32(priv, 0x409500, 0x00000001);
 673                 nv_wr32(priv, 0x409504, 0x00000030);
 674                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 675                         nv_error(priv, "fuc09 req 0x30 timeout\n");
 676                         return -EBUSY;
 677                 }
 678
 679                 nv_wr32(priv, 0x409810, 0xb00095c8);
 680                 nv_wr32(priv, 0x409800, 0x00000000);
 681                 nv_wr32(priv, 0x409500, 0x00000001);
 682                 nv_wr32(priv, 0x409504, 0x00000031);
 683                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 684                         nv_error(priv, "fuc09 req 0x31 timeout\n");
 685                         return -EBUSY;
 686                 }
 687
 688                 nv_wr32(priv, 0x409810, 0x00080420);
 689                 nv_wr32(priv, 0x409800, 0x00000000);
 690                 nv_wr32(priv, 0x409500, 0x00000001);
 691                 nv_wr32(priv, 0x409504, 0x00000032);
 692                 if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
 693                         nv_error(priv, "fuc09 req 0x32 timeout\n");
 694                         return -EBUSY;
 695                 }
 696
 697                 nv_wr32(priv, 0x409614, 0x00000070);
 698                 nv_wr32(priv, 0x409614, 0x00000770);
 699                 nv_wr32(priv, 0x40802c, 0x00000001);
 700
 701                 if (priv->data == NULL) {
 702                         int ret = nve0_grctx_generate(priv);
 703                         if (ret) {
 704                                 nv_error(priv, "failed to construct context\n");
 705                                 return ret;
 706                         }
 707                 }
 708
 709                 return 0;
 710         }
 711
 712         /* load HUB microcode */
 713         r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
 714         nv_wr32(priv, 0x4091c0, 0x01000000);
 715         for (i = 0; i < sizeof(nve0_grhub_data) / 4; i++)
 716                 nv_wr32(priv, 0x4091c4, nve0_grhub_data[i]);
 717
 718         nv_wr32(priv, 0x409180, 0x01000000);
 719         for (i = 0; i < sizeof(nve0_grhub_code) / 4; i++) {
 720                 if ((i & 0x3f) == 0)
 721                         nv_wr32(priv, 0x409188, i >> 6);
 722                 nv_wr32(priv, 0x409184, nve0_grhub_code[i]);
 723         }
 724
 725         /* load GPC microcode */
 726         nv_wr32(priv, 0x41a1c0, 0x01000000);
 727         for (i = 0; i < sizeof(nve0_grgpc_data) / 4; i++)
 728                 nv_wr32(priv, 0x41a1c4, nve0_grgpc_data[i]);
 729
 730         nv_wr32(priv, 0x41a180, 0x01000000);
 731         for (i = 0; i < sizeof(nve0_grgpc_code) / 4; i++) {
 732                 if ((i & 0x3f) == 0)
 733                         nv_wr32(priv, 0x41a188, i >> 6);
 734                 nv_wr32(priv, 0x41a184, nve0_grgpc_code[i]);
 735         }
 736         nv_wr32(priv, 0x000260, r000260);
 737
 738         /* start HUB ucode running, it'll init the GPCs */
 739         nv_wr32(priv, 0x409800, nv_device(priv)->chipset);
 740         nv_wr32(priv, 0x40910c, 0x00000000);
 741         nv_wr32(priv, 0x409100, 0x00000002);
 742         if (!nv_wait(priv, 0x409800, 0x80000000, 0x80000000)) {
 743                 nv_error(priv, "HUB_INIT timed out\n");
 744                 nvc0_graph_ctxctl_debug(priv);
 745                 return -EBUSY;
 746         }
 747
 748         priv->size = nv_rd32(priv, 0x409804);
 749         if (priv->data == NULL) {
 750                 int ret = nve0_grctx_generate(priv);
 751                 if (ret) {
 752                         nv_error(priv, "failed to construct context\n");
 753                         return ret;
 754                 }
 755         }
 756
 757         return 0;
 758 }
 759
 760 static int
 761 nve0_graph_init(struct nouveau_object *object)
 762 {
 763         struct nvc0_graph_priv *priv = (void *)object;
 764         int ret;
 765
 766         ret = nouveau_graph_init(&priv->base);
 767         if (ret)
 768                 return ret;
 769
 770         nve0_graph_init_obj418880(priv);
 771         nve0_graph_init_regs(priv);
 772         nve0_graph_init_gpc_0(priv);
 773
 774         nv_wr32(priv, 0x400500, 0x00010001);
 775         nv_wr32(priv, 0x400100, 0xffffffff);
 776         nv_wr32(priv, 0x40013c, 0xffffffff);
 777
 778         nve0_graph_init_units(priv);
 779         nve0_graph_init_gpc_1(priv);
 780         nve0_graph_init_rop(priv);
 781
 782         nv_wr32(priv, 0x400108, 0xffffffff);
 783         nv_wr32(priv, 0x400138, 0xffffffff);
 784         nv_wr32(priv, 0x400118, 0xffffffff);
 785         nv_wr32(priv, 0x400130, 0xffffffff);
 786         nv_wr32(priv, 0x40011c, 0xffffffff);
 787         nv_wr32(priv, 0x400134, 0xffffffff);
 788         nv_wr32(priv, 0x400054, 0x34ce3464);
 789
 790         ret = nve0_graph_init_ctxctl(priv);
 791         if (ret)
 792                 return ret;
 793
 794         return 0;
 795 }
 796
 797 struct nouveau_oclass
 798 nve0_graph_oclass = {
 799         .handle = NV_ENGINE(GR, 0xe0),
 800         .ofuncs = &(struct nouveau_ofuncs) {
 801                 .ctor = nve0_graph_ctor,
 802                 .dtor = nvc0_graph_dtor,
 803                 .init = nve0_graph_init,
 804                 .fini = _nouveau_graph_fini,
 805         },
 806 };