drivers/gpu/drm/i915/intel_lrc.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Ben Widawsky <ben@bwidawsk.net>
  25  *    Michel Thierry <michel.thierry@intel.com>
  26  *    Thomas Daniel <thomas.daniel@intel.com>
  27  *    Oscar Mateo <oscar.mateo@intel.com>
  28  *
  29  */
  30
  31 /*
  32  * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
  33  * These expanded contexts enable a number of new abilities, especially
  34  * "Execlists" (also implemented in this file).
  35  *
  36  * Execlists are the new method by which, on gen8+ hardware, workloads are
  37  * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
  38  */
  39
  40 #include <drm/drmP.h>
  41 #include <drm/i915_drm.h>
  42 #include "i915_drv.h"
  43
  44 #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
  45 #define GEN8_LR_CONTEXT_OTHER_SIZE (2 * PAGE_SIZE)
  46
  47 #define GEN8_LR_CONTEXT_ALIGN 4096
  48
  49 #define RING_ELSP(ring)                 ((ring)->mmio_base+0x230)
  50 #define RING_CONTEXT_CONTROL(ring)      ((ring)->mmio_base+0x244)
  51
  52 #define CTX_LRI_HEADER_0                0x01
  53 #define CTX_CONTEXT_CONTROL             0x02
  54 #define CTX_RING_HEAD                   0x04
  55 #define CTX_RING_TAIL                   0x06
  56 #define CTX_RING_BUFFER_START           0x08
  57 #define CTX_RING_BUFFER_CONTROL         0x0a
  58 #define CTX_BB_HEAD_U                   0x0c
  59 #define CTX_BB_HEAD_L                   0x0e
  60 #define CTX_BB_STATE                    0x10
  61 #define CTX_SECOND_BB_HEAD_U            0x12
  62 #define CTX_SECOND_BB_HEAD_L            0x14
  63 #define CTX_SECOND_BB_STATE             0x16
  64 #define CTX_BB_PER_CTX_PTR              0x18
  65 #define CTX_RCS_INDIRECT_CTX            0x1a
  66 #define CTX_RCS_INDIRECT_CTX_OFFSET     0x1c
  67 #define CTX_LRI_HEADER_1                0x21
  68 #define CTX_CTX_TIMESTAMP               0x22
  69 #define CTX_PDP3_UDW                    0x24
  70 #define CTX_PDP3_LDW                    0x26
  71 #define CTX_PDP2_UDW                    0x28
  72 #define CTX_PDP2_LDW                    0x2a
  73 #define CTX_PDP1_UDW                    0x2c
  74 #define CTX_PDP1_LDW                    0x2e
  75 #define CTX_PDP0_UDW                    0x30
  76 #define CTX_PDP0_LDW                    0x32
  77 #define CTX_LRI_HEADER_2                0x41
  78 #define CTX_R_PWR_CLK_STATE             0x42
  79 #define CTX_GPGPU_CSR_BASE_ADDRESS      0x44
  80
  81 int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists)
  82 {
  83         WARN_ON(i915.enable_ppgtt == -1);
  84
  85         if (enable_execlists == 0)
  86                 return 0;
  87
  88         if (HAS_LOGICAL_RING_CONTEXTS(dev) && USES_PPGTT(dev))
  89                 return 1;
  90
  91         return 0;
  92 }
  93
  94 int intel_execlists_submission(struct drm_device *dev, struct drm_file *file,
  95                                struct intel_engine_cs *ring,
  96                                struct intel_context *ctx,
  97                                struct drm_i915_gem_execbuffer2 *args,
  98                                struct list_head *vmas,
  99                                struct drm_i915_gem_object *batch_obj,
 100                                u64 exec_start, u32 flags)
 101 {
 102         /* TODO */
 103         return 0;
 104 }
 105
 106 void intel_logical_ring_stop(struct intel_engine_cs *ring)
 107 {
 108         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 109         int ret;
 110
 111         if (!intel_ring_initialized(ring))
 112                 return;
 113
 114         ret = intel_ring_idle(ring);
 115         if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
 116                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
 117                           ring->name, ret);
 118
 119         /* TODO: Is this correct with Execlists enabled? */
 120         I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
 121         if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
 122                 DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
 123                 return;
 124         }
 125         I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
 126 }
 127
 128 void intel_logical_ring_advance_and_submit(struct intel_ringbuffer *ringbuf)
 129 {
 130         intel_logical_ring_advance(ringbuf);
 131
 132         if (intel_ring_stopped(ringbuf->ring))
 133                 return;
 134
 135         /* TODO: how to submit a context to the ELSP is not here yet */
 136 }
 137
 138 static int logical_ring_alloc_seqno(struct intel_engine_cs *ring)
 139 {
 140         if (ring->outstanding_lazy_seqno)
 141                 return 0;
 142
 143         if (ring->preallocated_lazy_request == NULL) {
 144                 struct drm_i915_gem_request *request;
 145
 146                 request = kmalloc(sizeof(*request), GFP_KERNEL);
 147                 if (request == NULL)
 148                         return -ENOMEM;
 149
 150                 ring->preallocated_lazy_request = request;
 151         }
 152
 153         return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
 154 }
 155
 156 static int logical_ring_wait_request(struct intel_ringbuffer *ringbuf,
 157                                      int bytes)
 158 {
 159         struct intel_engine_cs *ring = ringbuf->ring;
 160         struct drm_i915_gem_request *request;
 161         u32 seqno = 0;
 162         int ret;
 163
 164         if (ringbuf->last_retired_head != -1) {
 165                 ringbuf->head = ringbuf->last_retired_head;
 166                 ringbuf->last_retired_head = -1;
 167
 168                 ringbuf->space = intel_ring_space(ringbuf);
 169                 if (ringbuf->space >= bytes)
 170                         return 0;
 171         }
 172
 173         list_for_each_entry(request, &ring->request_list, list) {
 174                 if (__intel_ring_space(request->tail, ringbuf->tail,
 175                                        ringbuf->size) >= bytes) {
 176                         seqno = request->seqno;
 177                         break;
 178                 }
 179         }
 180
 181         if (seqno == 0)
 182                 return -ENOSPC;
 183
 184         ret = i915_wait_seqno(ring, seqno);
 185         if (ret)
 186                 return ret;
 187
 188         /* TODO: make sure we update the right ringbuffer's last_retired_head
 189          * when retiring requests */
 190         i915_gem_retire_requests_ring(ring);
 191         ringbuf->head = ringbuf->last_retired_head;
 192         ringbuf->last_retired_head = -1;
 193
 194         ringbuf->space = intel_ring_space(ringbuf);
 195         return 0;
 196 }
 197
 198 static int logical_ring_wait_for_space(struct intel_ringbuffer *ringbuf,
 199                                        int bytes)
 200 {
 201         struct intel_engine_cs *ring = ringbuf->ring;
 202         struct drm_device *dev = ring->dev;
 203         struct drm_i915_private *dev_priv = dev->dev_private;
 204         unsigned long end;
 205         int ret;
 206
 207         ret = logical_ring_wait_request(ringbuf, bytes);
 208         if (ret != -ENOSPC)
 209                 return ret;
 210
 211         /* Force the context submission in case we have been skipping it */
 212         intel_logical_ring_advance_and_submit(ringbuf);
 213
 214         /* With GEM the hangcheck timer should kick us out of the loop,
 215          * leaving it early runs the risk of corrupting GEM state (due
 216          * to running on almost untested codepaths). But on resume
 217          * timers don't work yet, so prevent a complete hang in that
 218          * case by choosing an insanely large timeout. */
 219         end = jiffies + 60 * HZ;
 220
 221         do {
 222                 ringbuf->head = I915_READ_HEAD(ring);
 223                 ringbuf->space = intel_ring_space(ringbuf);
 224                 if (ringbuf->space >= bytes) {
 225                         ret = 0;
 226                         break;
 227                 }
 228
 229                 msleep(1);
 230
 231                 if (dev_priv->mm.interruptible && signal_pending(current)) {
 232                         ret = -ERESTARTSYS;
 233                         break;
 234                 }
 235
 236                 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
 237                                            dev_priv->mm.interruptible);
 238                 if (ret)
 239                         break;
 240
 241                 if (time_after(jiffies, end)) {
 242                         ret = -EBUSY;
 243                         break;
 244                 }
 245         } while (1);
 246
 247         return ret;
 248 }
 249
 250 static int logical_ring_wrap_buffer(struct intel_ringbuffer *ringbuf)
 251 {
 252         uint32_t __iomem *virt;
 253         int rem = ringbuf->size - ringbuf->tail;
 254
 255         if (ringbuf->space < rem) {
 256                 int ret = logical_ring_wait_for_space(ringbuf, rem);
 257
 258                 if (ret)
 259                         return ret;
 260         }
 261
 262         virt = ringbuf->virtual_start + ringbuf->tail;
 263         rem /= 4;
 264         while (rem--)
 265                 iowrite32(MI_NOOP, virt++);
 266
 267         ringbuf->tail = 0;
 268         ringbuf->space = intel_ring_space(ringbuf);
 269
 270         return 0;
 271 }
 272
 273 static int logical_ring_prepare(struct intel_ringbuffer *ringbuf, int bytes)
 274 {
 275         int ret;
 276
 277         if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {
 278                 ret = logical_ring_wrap_buffer(ringbuf);
 279                 if (unlikely(ret))
 280                         return ret;
 281         }
 282
 283         if (unlikely(ringbuf->space < bytes)) {
 284                 ret = logical_ring_wait_for_space(ringbuf, bytes);
 285                 if (unlikely(ret))
 286                         return ret;
 287         }
 288
 289         return 0;
 290 }
 291
 292 int intel_logical_ring_begin(struct intel_ringbuffer *ringbuf, int num_dwords)
 293 {
 294         struct intel_engine_cs *ring = ringbuf->ring;
 295         struct drm_device *dev = ring->dev;
 296         struct drm_i915_private *dev_priv = dev->dev_private;
 297         int ret;
 298
 299         ret = i915_gem_check_wedge(&dev_priv->gpu_error,
 300                                    dev_priv->mm.interruptible);
 301         if (ret)
 302                 return ret;
 303
 304         ret = logical_ring_prepare(ringbuf, num_dwords * sizeof(uint32_t));
 305         if (ret)
 306                 return ret;
 307
 308         /* Preallocate the olr before touching the ring */
 309         ret = logical_ring_alloc_seqno(ring);
 310         if (ret)
 311                 return ret;
 312
 313         ringbuf->space -= num_dwords * sizeof(uint32_t);
 314         return 0;
 315 }
 316
 317 static int gen8_init_common_ring(struct intel_engine_cs *ring)
 318 {
 319         struct drm_device *dev = ring->dev;
 320         struct drm_i915_private *dev_priv = dev->dev_private;
 321
 322         I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
 323         I915_WRITE(RING_HWSTAM(ring->mmio_base), 0xffffffff);
 324
 325         I915_WRITE(RING_MODE_GEN7(ring),
 326                    _MASKED_BIT_DISABLE(GFX_REPLAY_MODE) |
 327                    _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
 328         POSTING_READ(RING_MODE_GEN7(ring));
 329         DRM_DEBUG_DRIVER("Execlists enabled for %s\n", ring->name);
 330
 331         memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
 332
 333         return 0;
 334 }
 335
 336 static int gen8_init_render_ring(struct intel_engine_cs *ring)
 337 {
 338         struct drm_device *dev = ring->dev;
 339         struct drm_i915_private *dev_priv = dev->dev_private;
 340         int ret;
 341
 342         ret = gen8_init_common_ring(ring);
 343         if (ret)
 344                 return ret;
 345
 346         /* We need to disable the AsyncFlip performance optimisations in order
 347          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
 348          * programmed to '1' on all products.
 349          *
 350          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
 351          */
 352         I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
 353
 354         ret = intel_init_pipe_control(ring);
 355         if (ret)
 356                 return ret;
 357
 358         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 359
 360         return ret;
 361 }
 362
 363 static bool gen8_logical_ring_get_irq(struct intel_engine_cs *ring)
 364 {
 365         struct drm_device *dev = ring->dev;
 366         struct drm_i915_private *dev_priv = dev->dev_private;
 367         unsigned long flags;
 368
 369         if (!dev->irq_enabled)
 370                 return false;
 371
 372         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 373         if (ring->irq_refcount++ == 0) {
 374                 I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
 375                 POSTING_READ(RING_IMR(ring->mmio_base));
 376         }
 377         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 378
 379         return true;
 380 }
 381
 382 static void gen8_logical_ring_put_irq(struct intel_engine_cs *ring)
 383 {
 384         struct drm_device *dev = ring->dev;
 385         struct drm_i915_private *dev_priv = dev->dev_private;
 386         unsigned long flags;
 387
 388         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 389         if (--ring->irq_refcount == 0) {
 390                 I915_WRITE_IMR(ring, ~ring->irq_keep_mask);
 391                 POSTING_READ(RING_IMR(ring->mmio_base));
 392         }
 393         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 394 }
 395
 396 static int gen8_emit_flush(struct intel_ringbuffer *ringbuf,
 397                            u32 invalidate_domains,
 398                            u32 unused)
 399 {
 400         struct intel_engine_cs *ring = ringbuf->ring;
 401         struct drm_device *dev = ring->dev;
 402         struct drm_i915_private *dev_priv = dev->dev_private;
 403         uint32_t cmd;
 404         int ret;
 405
 406         ret = intel_logical_ring_begin(ringbuf, 4);
 407         if (ret)
 408                 return ret;
 409
 410         cmd = MI_FLUSH_DW + 1;
 411
 412         if (ring == &dev_priv->ring[VCS]) {
 413                 if (invalidate_domains & I915_GEM_GPU_DOMAINS)
 414                         cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
 415                                 MI_FLUSH_DW_STORE_INDEX |
 416                                 MI_FLUSH_DW_OP_STOREDW;
 417         } else {
 418                 if (invalidate_domains & I915_GEM_DOMAIN_RENDER)
 419                         cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
 420                                 MI_FLUSH_DW_OP_STOREDW;
 421         }
 422
 423         intel_logical_ring_emit(ringbuf, cmd);
 424         intel_logical_ring_emit(ringbuf,
 425                                 I915_GEM_HWS_SCRATCH_ADDR |
 426                                 MI_FLUSH_DW_USE_GTT);
 427         intel_logical_ring_emit(ringbuf, 0); /* upper addr */
 428         intel_logical_ring_emit(ringbuf, 0); /* value */
 429         intel_logical_ring_advance(ringbuf);
 430
 431         return 0;
 432 }
 433
 434 static int gen8_emit_flush_render(struct intel_ringbuffer *ringbuf,
 435                                   u32 invalidate_domains,
 436                                   u32 flush_domains)
 437 {
 438         struct intel_engine_cs *ring = ringbuf->ring;
 439         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 440         u32 flags = 0;
 441         int ret;
 442
 443         flags |= PIPE_CONTROL_CS_STALL;
 444
 445         if (flush_domains) {
 446                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 447                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 448         }
 449
 450         if (invalidate_domains) {
 451                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 452                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 453                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 454                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 455                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 456                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 457                 flags |= PIPE_CONTROL_QW_WRITE;
 458                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
 459         }
 460
 461         ret = intel_logical_ring_begin(ringbuf, 6);
 462         if (ret)
 463                 return ret;
 464
 465         intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
 466         intel_logical_ring_emit(ringbuf, flags);
 467         intel_logical_ring_emit(ringbuf, scratch_addr);
 468         intel_logical_ring_emit(ringbuf, 0);
 469         intel_logical_ring_emit(ringbuf, 0);
 470         intel_logical_ring_emit(ringbuf, 0);
 471         intel_logical_ring_advance(ringbuf);
 472
 473         return 0;
 474 }
 475
 476 static u32 gen8_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
 477 {
 478         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 479 }
 480
 481 static void gen8_set_seqno(struct intel_engine_cs *ring, u32 seqno)
 482 {
 483         intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
 484 }
 485
 486 static int gen8_emit_request(struct intel_ringbuffer *ringbuf)
 487 {
 488         struct intel_engine_cs *ring = ringbuf->ring;
 489         u32 cmd;
 490         int ret;
 491
 492         ret = intel_logical_ring_begin(ringbuf, 6);
 493         if (ret)
 494                 return ret;
 495
 496         cmd = MI_STORE_DWORD_IMM_GEN8;
 497         cmd |= MI_GLOBAL_GTT;
 498
 499         intel_logical_ring_emit(ringbuf, cmd);
 500         intel_logical_ring_emit(ringbuf,
 501                                 (ring->status_page.gfx_addr +
 502                                 (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)));
 503         intel_logical_ring_emit(ringbuf, 0);
 504         intel_logical_ring_emit(ringbuf, ring->outstanding_lazy_seqno);
 505         intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
 506         intel_logical_ring_emit(ringbuf, MI_NOOP);
 507         intel_logical_ring_advance_and_submit(ringbuf);
 508
 509         return 0;
 510 }
 511
 512 void intel_logical_ring_cleanup(struct intel_engine_cs *ring)
 513 {
 514         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 515
 516         if (!intel_ring_initialized(ring))
 517                 return;
 518
 519         intel_logical_ring_stop(ring);
 520         WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
 521         ring->preallocated_lazy_request = NULL;
 522         ring->outstanding_lazy_seqno = 0;
 523
 524         if (ring->cleanup)
 525                 ring->cleanup(ring);
 526
 527         i915_cmd_parser_fini_ring(ring);
 528
 529         if (ring->status_page.obj) {
 530                 kunmap(sg_page(ring->status_page.obj->pages->sgl));
 531                 ring->status_page.obj = NULL;
 532         }
 533 }
 534
 535 static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring)
 536 {
 537         int ret;
 538         struct intel_context *dctx = ring->default_context;
 539         struct drm_i915_gem_object *dctx_obj;
 540
 541         /* Intentionally left blank. */
 542         ring->buffer = NULL;
 543
 544         ring->dev = dev;
 545         INIT_LIST_HEAD(&ring->active_list);
 546         INIT_LIST_HEAD(&ring->request_list);
 547         init_waitqueue_head(&ring->irq_queue);
 548
 549         ret = intel_lr_context_deferred_create(dctx, ring);
 550         if (ret)
 551                 return ret;
 552
 553         /* The status page is offset 0 from the context object in LRCs. */
 554         dctx_obj = dctx->engine[ring->id].state;
 555         ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(dctx_obj);
 556         ring->status_page.page_addr = kmap(sg_page(dctx_obj->pages->sgl));
 557         if (ring->status_page.page_addr == NULL)
 558                 return -ENOMEM;
 559         ring->status_page.obj = dctx_obj;
 560
 561         ret = i915_cmd_parser_init_ring(ring);
 562         if (ret)
 563                 return ret;
 564
 565         if (ring->init) {
 566                 ret = ring->init(ring);
 567                 if (ret)
 568                         return ret;
 569         }
 570
 571         return 0;
 572 }
 573
 574 static int logical_render_ring_init(struct drm_device *dev)
 575 {
 576         struct drm_i915_private *dev_priv = dev->dev_private;
 577         struct intel_engine_cs *ring = &dev_priv->ring[RCS];
 578
 579         ring->name = "render ring";
 580         ring->id = RCS;
 581         ring->mmio_base = RENDER_RING_BASE;
 582         ring->irq_enable_mask =
 583                 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
 584         ring->irq_keep_mask =
 585                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
 586         if (HAS_L3_DPF(dev))
 587                 ring->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
 588
 589         ring->init = gen8_init_render_ring;
 590         ring->cleanup = intel_fini_pipe_control;
 591         ring->get_seqno = gen8_get_seqno;
 592         ring->set_seqno = gen8_set_seqno;
 593         ring->emit_request = gen8_emit_request;
 594         ring->emit_flush = gen8_emit_flush_render;
 595         ring->irq_get = gen8_logical_ring_get_irq;
 596         ring->irq_put = gen8_logical_ring_put_irq;
 597
 598         return logical_ring_init(dev, ring);
 599 }
 600
 601 static int logical_bsd_ring_init(struct drm_device *dev)
 602 {
 603         struct drm_i915_private *dev_priv = dev->dev_private;
 604         struct intel_engine_cs *ring = &dev_priv->ring[VCS];
 605
 606         ring->name = "bsd ring";
 607         ring->id = VCS;
 608         ring->mmio_base = GEN6_BSD_RING_BASE;
 609         ring->irq_enable_mask =
 610                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
 611         ring->irq_keep_mask =
 612                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
 613
 614         ring->init = gen8_init_common_ring;
 615         ring->get_seqno = gen8_get_seqno;
 616         ring->set_seqno = gen8_set_seqno;
 617         ring->emit_request = gen8_emit_request;
 618         ring->emit_flush = gen8_emit_flush;
 619         ring->irq_get = gen8_logical_ring_get_irq;
 620         ring->irq_put = gen8_logical_ring_put_irq;
 621
 622         return logical_ring_init(dev, ring);
 623 }
 624
 625 static int logical_bsd2_ring_init(struct drm_device *dev)
 626 {
 627         struct drm_i915_private *dev_priv = dev->dev_private;
 628         struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
 629
 630         ring->name = "bds2 ring";
 631         ring->id = VCS2;
 632         ring->mmio_base = GEN8_BSD2_RING_BASE;
 633         ring->irq_enable_mask =
 634                 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
 635         ring->irq_keep_mask =
 636                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
 637
 638         ring->init = gen8_init_common_ring;
 639         ring->get_seqno = gen8_get_seqno;
 640         ring->set_seqno = gen8_set_seqno;
 641         ring->emit_request = gen8_emit_request;
 642         ring->emit_flush = gen8_emit_flush;
 643         ring->irq_get = gen8_logical_ring_get_irq;
 644         ring->irq_put = gen8_logical_ring_put_irq;
 645
 646         return logical_ring_init(dev, ring);
 647 }
 648
 649 static int logical_blt_ring_init(struct drm_device *dev)
 650 {
 651         struct drm_i915_private *dev_priv = dev->dev_private;
 652         struct intel_engine_cs *ring = &dev_priv->ring[BCS];
 653
 654         ring->name = "blitter ring";
 655         ring->id = BCS;
 656         ring->mmio_base = BLT_RING_BASE;
 657         ring->irq_enable_mask =
 658                 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
 659         ring->irq_keep_mask =
 660                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
 661
 662         ring->init = gen8_init_common_ring;
 663         ring->get_seqno = gen8_get_seqno;
 664         ring->set_seqno = gen8_set_seqno;
 665         ring->emit_request = gen8_emit_request;
 666         ring->emit_flush = gen8_emit_flush;
 667         ring->irq_get = gen8_logical_ring_get_irq;
 668         ring->irq_put = gen8_logical_ring_put_irq;
 669
 670         return logical_ring_init(dev, ring);
 671 }
 672
 673 static int logical_vebox_ring_init(struct drm_device *dev)
 674 {
 675         struct drm_i915_private *dev_priv = dev->dev_private;
 676         struct intel_engine_cs *ring = &dev_priv->ring[VECS];
 677
 678         ring->name = "video enhancement ring";
 679         ring->id = VECS;
 680         ring->mmio_base = VEBOX_RING_BASE;
 681         ring->irq_enable_mask =
 682                 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
 683         ring->irq_keep_mask =
 684                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
 685
 686         ring->init = gen8_init_common_ring;
 687         ring->get_seqno = gen8_get_seqno;
 688         ring->set_seqno = gen8_set_seqno;
 689         ring->emit_request = gen8_emit_request;
 690         ring->emit_flush = gen8_emit_flush;
 691         ring->irq_get = gen8_logical_ring_get_irq;
 692         ring->irq_put = gen8_logical_ring_put_irq;
 693
 694         return logical_ring_init(dev, ring);
 695 }
 696
 697 int intel_logical_rings_init(struct drm_device *dev)
 698 {
 699         struct drm_i915_private *dev_priv = dev->dev_private;
 700         int ret;
 701
 702         ret = logical_render_ring_init(dev);
 703         if (ret)
 704                 return ret;
 705
 706         if (HAS_BSD(dev)) {
 707                 ret = logical_bsd_ring_init(dev);
 708                 if (ret)
 709                         goto cleanup_render_ring;
 710         }
 711
 712         if (HAS_BLT(dev)) {
 713                 ret = logical_blt_ring_init(dev);
 714                 if (ret)
 715                         goto cleanup_bsd_ring;
 716         }
 717
 718         if (HAS_VEBOX(dev)) {
 719                 ret = logical_vebox_ring_init(dev);
 720                 if (ret)
 721                         goto cleanup_blt_ring;
 722         }
 723
 724         if (HAS_BSD2(dev)) {
 725                 ret = logical_bsd2_ring_init(dev);
 726                 if (ret)
 727                         goto cleanup_vebox_ring;
 728         }
 729
 730         ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
 731         if (ret)
 732                 goto cleanup_bsd2_ring;
 733
 734         return 0;
 735
 736 cleanup_bsd2_ring:
 737         intel_logical_ring_cleanup(&dev_priv->ring[VCS2]);
 738 cleanup_vebox_ring:
 739         intel_logical_ring_cleanup(&dev_priv->ring[VECS]);
 740 cleanup_blt_ring:
 741         intel_logical_ring_cleanup(&dev_priv->ring[BCS]);
 742 cleanup_bsd_ring:
 743         intel_logical_ring_cleanup(&dev_priv->ring[VCS]);
 744 cleanup_render_ring:
 745         intel_logical_ring_cleanup(&dev_priv->ring[RCS]);
 746
 747         return ret;
 748 }
 749
 750 static int
 751 populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_obj,
 752                     struct intel_engine_cs *ring, struct intel_ringbuffer *ringbuf)
 753 {
 754         struct drm_i915_gem_object *ring_obj = ringbuf->obj;
 755         struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(ctx);
 756         struct page *page;
 757         uint32_t *reg_state;
 758         int ret;
 759
 760         ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
 761         if (ret) {
 762                 DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
 763                 return ret;
 764         }
 765
 766         ret = i915_gem_object_get_pages(ctx_obj);
 767         if (ret) {
 768                 DRM_DEBUG_DRIVER("Could not get object pages\n");
 769                 return ret;
 770         }
 771
 772         i915_gem_object_pin_pages(ctx_obj);
 773
 774         /* The second page of the context object contains some fields which must
 775          * be set up prior to the first execution. */
 776         page = i915_gem_object_get_page(ctx_obj, 1);
 777         reg_state = kmap_atomic(page);
 778
 779         /* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM
 780          * commands followed by (reg, value) pairs. The values we are setting here are
 781          * only for the first context restore: on a subsequent save, the GPU will
 782          * recreate this batchbuffer with new values (including all the missing
 783          * MI_LOAD_REGISTER_IMM commands that we are not initializing here). */
 784         if (ring->id == RCS)
 785                 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(14);
 786         else
 787                 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(11);
 788         reg_state[CTX_LRI_HEADER_0] |= MI_LRI_FORCE_POSTED;
 789         reg_state[CTX_CONTEXT_CONTROL] = RING_CONTEXT_CONTROL(ring);
 790         reg_state[CTX_CONTEXT_CONTROL+1] =
 791                         _MASKED_BIT_ENABLE((1<<3) | MI_RESTORE_INHIBIT);
 792         reg_state[CTX_RING_HEAD] = RING_HEAD(ring->mmio_base);
 793         reg_state[CTX_RING_HEAD+1] = 0;
 794         reg_state[CTX_RING_TAIL] = RING_TAIL(ring->mmio_base);
 795         reg_state[CTX_RING_TAIL+1] = 0;
 796         reg_state[CTX_RING_BUFFER_START] = RING_START(ring->mmio_base);
 797         reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(ring_obj);
 798         reg_state[CTX_RING_BUFFER_CONTROL] = RING_CTL(ring->mmio_base);
 799         reg_state[CTX_RING_BUFFER_CONTROL+1] =
 800                         ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID;
 801         reg_state[CTX_BB_HEAD_U] = ring->mmio_base + 0x168;
 802         reg_state[CTX_BB_HEAD_U+1] = 0;
 803         reg_state[CTX_BB_HEAD_L] = ring->mmio_base + 0x140;
 804         reg_state[CTX_BB_HEAD_L+1] = 0;
 805         reg_state[CTX_BB_STATE] = ring->mmio_base + 0x110;
 806         reg_state[CTX_BB_STATE+1] = (1<<5);
 807         reg_state[CTX_SECOND_BB_HEAD_U] = ring->mmio_base + 0x11c;
 808         reg_state[CTX_SECOND_BB_HEAD_U+1] = 0;
 809         reg_state[CTX_SECOND_BB_HEAD_L] = ring->mmio_base + 0x114;
 810         reg_state[CTX_SECOND_BB_HEAD_L+1] = 0;
 811         reg_state[CTX_SECOND_BB_STATE] = ring->mmio_base + 0x118;
 812         reg_state[CTX_SECOND_BB_STATE+1] = 0;
 813         if (ring->id == RCS) {
 814                 /* TODO: according to BSpec, the register state context
 815                  * for CHV does not have these. OTOH, these registers do
 816                  * exist in CHV. I'm waiting for a clarification */
 817                 reg_state[CTX_BB_PER_CTX_PTR] = ring->mmio_base + 0x1c0;
 818                 reg_state[CTX_BB_PER_CTX_PTR+1] = 0;
 819                 reg_state[CTX_RCS_INDIRECT_CTX] = ring->mmio_base + 0x1c4;
 820                 reg_state[CTX_RCS_INDIRECT_CTX+1] = 0;
 821                 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET] = ring->mmio_base + 0x1c8;
 822                 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] = 0;
 823         }
 824         reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9);
 825         reg_state[CTX_LRI_HEADER_1] |= MI_LRI_FORCE_POSTED;
 826         reg_state[CTX_CTX_TIMESTAMP] = ring->mmio_base + 0x3a8;
 827         reg_state[CTX_CTX_TIMESTAMP+1] = 0;
 828         reg_state[CTX_PDP3_UDW] = GEN8_RING_PDP_UDW(ring, 3);
 829         reg_state[CTX_PDP3_LDW] = GEN8_RING_PDP_LDW(ring, 3);
 830         reg_state[CTX_PDP2_UDW] = GEN8_RING_PDP_UDW(ring, 2);
 831         reg_state[CTX_PDP2_LDW] = GEN8_RING_PDP_LDW(ring, 2);
 832         reg_state[CTX_PDP1_UDW] = GEN8_RING_PDP_UDW(ring, 1);
 833         reg_state[CTX_PDP1_LDW] = GEN8_RING_PDP_LDW(ring, 1);
 834         reg_state[CTX_PDP0_UDW] = GEN8_RING_PDP_UDW(ring, 0);
 835         reg_state[CTX_PDP0_LDW] = GEN8_RING_PDP_LDW(ring, 0);
 836         reg_state[CTX_PDP3_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[3]);
 837         reg_state[CTX_PDP3_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[3]);
 838         reg_state[CTX_PDP2_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[2]);
 839         reg_state[CTX_PDP2_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[2]);
 840         reg_state[CTX_PDP1_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[1]);
 841         reg_state[CTX_PDP1_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[1]);
 842         reg_state[CTX_PDP0_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[0]);
 843         reg_state[CTX_PDP0_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[0]);
 844         if (ring->id == RCS) {
 845                 reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
 846                 reg_state[CTX_R_PWR_CLK_STATE] = 0x20c8;
 847                 reg_state[CTX_R_PWR_CLK_STATE+1] = 0;
 848         }
 849
 850         kunmap_atomic(reg_state);
 851
 852         ctx_obj->dirty = 1;
 853         set_page_dirty(page);
 854         i915_gem_object_unpin_pages(ctx_obj);
 855
 856         return 0;
 857 }
 858
 859 void intel_lr_context_free(struct intel_context *ctx)
 860 {
 861         int i;
 862
 863         for (i = 0; i < I915_NUM_RINGS; i++) {
 864                 struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state;
 865                 struct intel_ringbuffer *ringbuf = ctx->engine[i].ringbuf;
 866
 867                 if (ctx_obj) {
 868                         intel_destroy_ringbuffer_obj(ringbuf);
 869                         kfree(ringbuf);
 870                         i915_gem_object_ggtt_unpin(ctx_obj);
 871                         drm_gem_object_unreference(&ctx_obj->base);
 872                 }
 873         }
 874 }
 875
 876 static uint32_t get_lr_context_size(struct intel_engine_cs *ring)
 877 {
 878         int ret = 0;
 879
 880         WARN_ON(INTEL_INFO(ring->dev)->gen != 8);
 881
 882         switch (ring->id) {
 883         case RCS:
 884                 ret = GEN8_LR_CONTEXT_RENDER_SIZE;
 885                 break;
 886         case VCS:
 887         case BCS:
 888         case VECS:
 889         case VCS2:
 890                 ret = GEN8_LR_CONTEXT_OTHER_SIZE;
 891                 break;
 892         }
 893
 894         return ret;
 895 }
 896
 897 int intel_lr_context_deferred_create(struct intel_context *ctx,
 898                                      struct intel_engine_cs *ring)
 899 {
 900         struct drm_device *dev = ring->dev;
 901         struct drm_i915_gem_object *ctx_obj;
 902         uint32_t context_size;
 903         struct intel_ringbuffer *ringbuf;
 904         int ret;
 905
 906         WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL);
 907         if (ctx->engine[ring->id].state)
 908                 return 0;
 909
 910         context_size = round_up(get_lr_context_size(ring), 4096);
 911
 912         ctx_obj = i915_gem_alloc_context_obj(dev, context_size);
 913         if (IS_ERR(ctx_obj)) {
 914                 ret = PTR_ERR(ctx_obj);
 915                 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed: %d\n", ret);
 916                 return ret;
 917         }
 918
 919         ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, 0);
 920         if (ret) {
 921                 DRM_DEBUG_DRIVER("Pin LRC backing obj failed: %d\n", ret);
 922                 drm_gem_object_unreference(&ctx_obj->base);
 923                 return ret;
 924         }
 925
 926         ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
 927         if (!ringbuf) {
 928                 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
 929                                 ring->name);
 930                 i915_gem_object_ggtt_unpin(ctx_obj);
 931                 drm_gem_object_unreference(&ctx_obj->base);
 932                 ret = -ENOMEM;
 933                 return ret;
 934         }
 935
 936         ringbuf->ring = ring;
 937         ringbuf->size = 32 * PAGE_SIZE;
 938         ringbuf->effective_size = ringbuf->size;
 939         ringbuf->head = 0;
 940         ringbuf->tail = 0;
 941         ringbuf->space = ringbuf->size;
 942         ringbuf->last_retired_head = -1;
 943
 944         /* TODO: For now we put this in the mappable region so that we can reuse
 945          * the existing ringbuffer code which ioremaps it. When we start
 946          * creating many contexts, this will no longer work and we must switch
 947          * to a kmapish interface.
 948          */
 949         ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
 950         if (ret) {
 951                 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer obj %s: %d\n",
 952                                 ring->name, ret);
 953                 goto error;
 954         }
 955
 956         ret = populate_lr_context(ctx, ctx_obj, ring, ringbuf);
 957         if (ret) {
 958                 DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
 959                 intel_destroy_ringbuffer_obj(ringbuf);
 960                 goto error;
 961         }
 962
 963         ctx->engine[ring->id].ringbuf = ringbuf;
 964         ctx->engine[ring->id].state = ctx_obj;
 965
 966         return 0;
 967
 968 error:
 969         kfree(ringbuf);
 970         i915_gem_object_ggtt_unpin(ctx_obj);
 971         drm_gem_object_unreference(&ctx_obj->base);
 972         return ret;
 973 }