drivers/gpu/drm/i915/intel_lrc.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Ben Widawsky <ben@bwidawsk.net>
  25  *    Michel Thierry <michel.thierry@intel.com>
  26  *    Thomas Daniel <thomas.daniel@intel.com>
  27  *    Oscar Mateo <oscar.mateo@intel.com>
  28  *
  29  */
  30
  31 /*
  32  * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
  33  * These expanded contexts enable a number of new abilities, especially
  34  * "Execlists" (also implemented in this file).
  35  *
  36  * Execlists are the new method by which, on gen8+ hardware, workloads are
  37  * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
  38  */
  39
  40 #include <drm/drmP.h>
  41 #include <drm/i915_drm.h>
  42 #include "i915_drv.h"
  43
  44 #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
  45 #define GEN8_LR_CONTEXT_OTHER_SIZE (2 * PAGE_SIZE)
  46
  47 #define GEN8_LR_CONTEXT_ALIGN 4096
  48
  49 #define RING_ELSP(ring)                 ((ring)->mmio_base+0x230)
  50 #define RING_CONTEXT_CONTROL(ring)      ((ring)->mmio_base+0x244)
  51
  52 #define CTX_LRI_HEADER_0                0x01
  53 #define CTX_CONTEXT_CONTROL             0x02
  54 #define CTX_RING_HEAD                   0x04
  55 #define CTX_RING_TAIL                   0x06
  56 #define CTX_RING_BUFFER_START           0x08
  57 #define CTX_RING_BUFFER_CONTROL         0x0a
  58 #define CTX_BB_HEAD_U                   0x0c
  59 #define CTX_BB_HEAD_L                   0x0e
  60 #define CTX_BB_STATE                    0x10
  61 #define CTX_SECOND_BB_HEAD_U            0x12
  62 #define CTX_SECOND_BB_HEAD_L            0x14
  63 #define CTX_SECOND_BB_STATE             0x16
  64 #define CTX_BB_PER_CTX_PTR              0x18
  65 #define CTX_RCS_INDIRECT_CTX            0x1a
  66 #define CTX_RCS_INDIRECT_CTX_OFFSET     0x1c
  67 #define CTX_LRI_HEADER_1                0x21
  68 #define CTX_CTX_TIMESTAMP               0x22
  69 #define CTX_PDP3_UDW                    0x24
  70 #define CTX_PDP3_LDW                    0x26
  71 #define CTX_PDP2_UDW                    0x28
  72 #define CTX_PDP2_LDW                    0x2a
  73 #define CTX_PDP1_UDW                    0x2c
  74 #define CTX_PDP1_LDW                    0x2e
  75 #define CTX_PDP0_UDW                    0x30
  76 #define CTX_PDP0_LDW                    0x32
  77 #define CTX_LRI_HEADER_2                0x41
  78 #define CTX_R_PWR_CLK_STATE             0x42
  79 #define CTX_GPGPU_CSR_BASE_ADDRESS      0x44
  80
  81 int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists)
  82 {
  83         WARN_ON(i915.enable_ppgtt == -1);
  84
  85         if (enable_execlists == 0)
  86                 return 0;
  87
  88         if (HAS_LOGICAL_RING_CONTEXTS(dev) && USES_PPGTT(dev))
  89                 return 1;
  90
  91         return 0;
  92 }
  93
  94 int intel_execlists_submission(struct drm_device *dev, struct drm_file *file,
  95                                struct intel_engine_cs *ring,
  96                                struct intel_context *ctx,
  97                                struct drm_i915_gem_execbuffer2 *args,
  98                                struct list_head *vmas,
  99                                struct drm_i915_gem_object *batch_obj,
 100                                u64 exec_start, u32 flags)
 101 {
 102         /* TODO */
 103         return 0;
 104 }
 105
 106 void intel_logical_ring_stop(struct intel_engine_cs *ring)
 107 {
 108         /* TODO */
 109 }
 110
 111 void intel_logical_ring_advance_and_submit(struct intel_ringbuffer *ringbuf)
 112 {
 113         intel_logical_ring_advance(ringbuf);
 114
 115         if (intel_ring_stopped(ringbuf->ring))
 116                 return;
 117
 118         /* TODO: how to submit a context to the ELSP is not here yet */
 119 }
 120
 121 static int logical_ring_alloc_seqno(struct intel_engine_cs *ring)
 122 {
 123         if (ring->outstanding_lazy_seqno)
 124                 return 0;
 125
 126         if (ring->preallocated_lazy_request == NULL) {
 127                 struct drm_i915_gem_request *request;
 128
 129                 request = kmalloc(sizeof(*request), GFP_KERNEL);
 130                 if (request == NULL)
 131                         return -ENOMEM;
 132
 133                 ring->preallocated_lazy_request = request;
 134         }
 135
 136         return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
 137 }
 138
 139 static int logical_ring_wait_request(struct intel_ringbuffer *ringbuf,
 140                                      int bytes)
 141 {
 142         struct intel_engine_cs *ring = ringbuf->ring;
 143         struct drm_i915_gem_request *request;
 144         u32 seqno = 0;
 145         int ret;
 146
 147         if (ringbuf->last_retired_head != -1) {
 148                 ringbuf->head = ringbuf->last_retired_head;
 149                 ringbuf->last_retired_head = -1;
 150
 151                 ringbuf->space = intel_ring_space(ringbuf);
 152                 if (ringbuf->space >= bytes)
 153                         return 0;
 154         }
 155
 156         list_for_each_entry(request, &ring->request_list, list) {
 157                 if (__intel_ring_space(request->tail, ringbuf->tail,
 158                                        ringbuf->size) >= bytes) {
 159                         seqno = request->seqno;
 160                         break;
 161                 }
 162         }
 163
 164         if (seqno == 0)
 165                 return -ENOSPC;
 166
 167         ret = i915_wait_seqno(ring, seqno);
 168         if (ret)
 169                 return ret;
 170
 171         /* TODO: make sure we update the right ringbuffer's last_retired_head
 172          * when retiring requests */
 173         i915_gem_retire_requests_ring(ring);
 174         ringbuf->head = ringbuf->last_retired_head;
 175         ringbuf->last_retired_head = -1;
 176
 177         ringbuf->space = intel_ring_space(ringbuf);
 178         return 0;
 179 }
 180
 181 static int logical_ring_wait_for_space(struct intel_ringbuffer *ringbuf,
 182                                        int bytes)
 183 {
 184         struct intel_engine_cs *ring = ringbuf->ring;
 185         struct drm_device *dev = ring->dev;
 186         struct drm_i915_private *dev_priv = dev->dev_private;
 187         unsigned long end;
 188         int ret;
 189
 190         ret = logical_ring_wait_request(ringbuf, bytes);
 191         if (ret != -ENOSPC)
 192                 return ret;
 193
 194         /* Force the context submission in case we have been skipping it */
 195         intel_logical_ring_advance_and_submit(ringbuf);
 196
 197         /* With GEM the hangcheck timer should kick us out of the loop,
 198          * leaving it early runs the risk of corrupting GEM state (due
 199          * to running on almost untested codepaths). But on resume
 200          * timers don't work yet, so prevent a complete hang in that
 201          * case by choosing an insanely large timeout. */
 202         end = jiffies + 60 * HZ;
 203
 204         do {
 205                 ringbuf->head = I915_READ_HEAD(ring);
 206                 ringbuf->space = intel_ring_space(ringbuf);
 207                 if (ringbuf->space >= bytes) {
 208                         ret = 0;
 209                         break;
 210                 }
 211
 212                 msleep(1);
 213
 214                 if (dev_priv->mm.interruptible && signal_pending(current)) {
 215                         ret = -ERESTARTSYS;
 216                         break;
 217                 }
 218
 219                 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
 220                                            dev_priv->mm.interruptible);
 221                 if (ret)
 222                         break;
 223
 224                 if (time_after(jiffies, end)) {
 225                         ret = -EBUSY;
 226                         break;
 227                 }
 228         } while (1);
 229
 230         return ret;
 231 }
 232
 233 static int logical_ring_wrap_buffer(struct intel_ringbuffer *ringbuf)
 234 {
 235         uint32_t __iomem *virt;
 236         int rem = ringbuf->size - ringbuf->tail;
 237
 238         if (ringbuf->space < rem) {
 239                 int ret = logical_ring_wait_for_space(ringbuf, rem);
 240
 241                 if (ret)
 242                         return ret;
 243         }
 244
 245         virt = ringbuf->virtual_start + ringbuf->tail;
 246         rem /= 4;
 247         while (rem--)
 248                 iowrite32(MI_NOOP, virt++);
 249
 250         ringbuf->tail = 0;
 251         ringbuf->space = intel_ring_space(ringbuf);
 252
 253         return 0;
 254 }
 255
 256 static int logical_ring_prepare(struct intel_ringbuffer *ringbuf, int bytes)
 257 {
 258         int ret;
 259
 260         if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {
 261                 ret = logical_ring_wrap_buffer(ringbuf);
 262                 if (unlikely(ret))
 263                         return ret;
 264         }
 265
 266         if (unlikely(ringbuf->space < bytes)) {
 267                 ret = logical_ring_wait_for_space(ringbuf, bytes);
 268                 if (unlikely(ret))
 269                         return ret;
 270         }
 271
 272         return 0;
 273 }
 274
 275 int intel_logical_ring_begin(struct intel_ringbuffer *ringbuf, int num_dwords)
 276 {
 277         struct intel_engine_cs *ring = ringbuf->ring;
 278         struct drm_device *dev = ring->dev;
 279         struct drm_i915_private *dev_priv = dev->dev_private;
 280         int ret;
 281
 282         ret = i915_gem_check_wedge(&dev_priv->gpu_error,
 283                                    dev_priv->mm.interruptible);
 284         if (ret)
 285                 return ret;
 286
 287         ret = logical_ring_prepare(ringbuf, num_dwords * sizeof(uint32_t));
 288         if (ret)
 289                 return ret;
 290
 291         /* Preallocate the olr before touching the ring */
 292         ret = logical_ring_alloc_seqno(ring);
 293         if (ret)
 294                 return ret;
 295
 296         ringbuf->space -= num_dwords * sizeof(uint32_t);
 297         return 0;
 298 }
 299
 300 static int gen8_init_common_ring(struct intel_engine_cs *ring)
 301 {
 302         struct drm_device *dev = ring->dev;
 303         struct drm_i915_private *dev_priv = dev->dev_private;
 304
 305         I915_WRITE(RING_MODE_GEN7(ring),
 306                    _MASKED_BIT_DISABLE(GFX_REPLAY_MODE) |
 307                    _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
 308         POSTING_READ(RING_MODE_GEN7(ring));
 309         DRM_DEBUG_DRIVER("Execlists enabled for %s\n", ring->name);
 310
 311         memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
 312
 313         return 0;
 314 }
 315
 316 static int gen8_init_render_ring(struct intel_engine_cs *ring)
 317 {
 318         struct drm_device *dev = ring->dev;
 319         struct drm_i915_private *dev_priv = dev->dev_private;
 320         int ret;
 321
 322         ret = gen8_init_common_ring(ring);
 323         if (ret)
 324                 return ret;
 325
 326         /* We need to disable the AsyncFlip performance optimisations in order
 327          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
 328          * programmed to '1' on all products.
 329          *
 330          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
 331          */
 332         I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
 333
 334         ret = intel_init_pipe_control(ring);
 335         if (ret)
 336                 return ret;
 337
 338         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 339
 340         return ret;
 341 }
 342
 343 static u32 gen8_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
 344 {
 345         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 346 }
 347
 348 static void gen8_set_seqno(struct intel_engine_cs *ring, u32 seqno)
 349 {
 350         intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
 351 }
 352
 353 void intel_logical_ring_cleanup(struct intel_engine_cs *ring)
 354 {
 355         if (!intel_ring_initialized(ring))
 356                 return;
 357
 358         /* TODO: make sure the ring is stopped */
 359         ring->preallocated_lazy_request = NULL;
 360         ring->outstanding_lazy_seqno = 0;
 361
 362         if (ring->cleanup)
 363                 ring->cleanup(ring);
 364
 365         i915_cmd_parser_fini_ring(ring);
 366
 367         if (ring->status_page.obj) {
 368                 kunmap(sg_page(ring->status_page.obj->pages->sgl));
 369                 ring->status_page.obj = NULL;
 370         }
 371 }
 372
 373 static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring)
 374 {
 375         int ret;
 376         struct intel_context *dctx = ring->default_context;
 377         struct drm_i915_gem_object *dctx_obj;
 378
 379         /* Intentionally left blank. */
 380         ring->buffer = NULL;
 381
 382         ring->dev = dev;
 383         INIT_LIST_HEAD(&ring->active_list);
 384         INIT_LIST_HEAD(&ring->request_list);
 385         init_waitqueue_head(&ring->irq_queue);
 386
 387         ret = intel_lr_context_deferred_create(dctx, ring);
 388         if (ret)
 389                 return ret;
 390
 391         /* The status page is offset 0 from the context object in LRCs. */
 392         dctx_obj = dctx->engine[ring->id].state;
 393         ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(dctx_obj);
 394         ring->status_page.page_addr = kmap(sg_page(dctx_obj->pages->sgl));
 395         if (ring->status_page.page_addr == NULL)
 396                 return -ENOMEM;
 397         ring->status_page.obj = dctx_obj;
 398
 399         ret = i915_cmd_parser_init_ring(ring);
 400         if (ret)
 401                 return ret;
 402
 403         if (ring->init) {
 404                 ret = ring->init(ring);
 405                 if (ret)
 406                         return ret;
 407         }
 408
 409         return 0;
 410 }
 411
 412 static int logical_render_ring_init(struct drm_device *dev)
 413 {
 414         struct drm_i915_private *dev_priv = dev->dev_private;
 415         struct intel_engine_cs *ring = &dev_priv->ring[RCS];
 416
 417         ring->name = "render ring";
 418         ring->id = RCS;
 419         ring->mmio_base = RENDER_RING_BASE;
 420         ring->irq_enable_mask =
 421                 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
 422
 423         ring->init = gen8_init_render_ring;
 424         ring->cleanup = intel_fini_pipe_control;
 425         ring->get_seqno = gen8_get_seqno;
 426         ring->set_seqno = gen8_set_seqno;
 427
 428         return logical_ring_init(dev, ring);
 429 }
 430
 431 static int logical_bsd_ring_init(struct drm_device *dev)
 432 {
 433         struct drm_i915_private *dev_priv = dev->dev_private;
 434         struct intel_engine_cs *ring = &dev_priv->ring[VCS];
 435
 436         ring->name = "bsd ring";
 437         ring->id = VCS;
 438         ring->mmio_base = GEN6_BSD_RING_BASE;
 439         ring->irq_enable_mask =
 440                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
 441
 442         ring->init = gen8_init_common_ring;
 443         ring->get_seqno = gen8_get_seqno;
 444         ring->set_seqno = gen8_set_seqno;
 445
 446         return logical_ring_init(dev, ring);
 447 }
 448
 449 static int logical_bsd2_ring_init(struct drm_device *dev)
 450 {
 451         struct drm_i915_private *dev_priv = dev->dev_private;
 452         struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
 453
 454         ring->name = "bds2 ring";
 455         ring->id = VCS2;
 456         ring->mmio_base = GEN8_BSD2_RING_BASE;
 457         ring->irq_enable_mask =
 458                 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
 459
 460         ring->init = gen8_init_common_ring;
 461         ring->get_seqno = gen8_get_seqno;
 462         ring->set_seqno = gen8_set_seqno;
 463
 464         return logical_ring_init(dev, ring);
 465 }
 466
 467 static int logical_blt_ring_init(struct drm_device *dev)
 468 {
 469         struct drm_i915_private *dev_priv = dev->dev_private;
 470         struct intel_engine_cs *ring = &dev_priv->ring[BCS];
 471
 472         ring->name = "blitter ring";
 473         ring->id = BCS;
 474         ring->mmio_base = BLT_RING_BASE;
 475         ring->irq_enable_mask =
 476                 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
 477
 478         ring->init = gen8_init_common_ring;
 479         ring->get_seqno = gen8_get_seqno;
 480         ring->set_seqno = gen8_set_seqno;
 481
 482         return logical_ring_init(dev, ring);
 483 }
 484
 485 static int logical_vebox_ring_init(struct drm_device *dev)
 486 {
 487         struct drm_i915_private *dev_priv = dev->dev_private;
 488         struct intel_engine_cs *ring = &dev_priv->ring[VECS];
 489
 490         ring->name = "video enhancement ring";
 491         ring->id = VECS;
 492         ring->mmio_base = VEBOX_RING_BASE;
 493         ring->irq_enable_mask =
 494                 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
 495
 496         ring->init = gen8_init_common_ring;
 497         ring->get_seqno = gen8_get_seqno;
 498         ring->set_seqno = gen8_set_seqno;
 499
 500         return logical_ring_init(dev, ring);
 501 }
 502
 503 int intel_logical_rings_init(struct drm_device *dev)
 504 {
 505         struct drm_i915_private *dev_priv = dev->dev_private;
 506         int ret;
 507
 508         ret = logical_render_ring_init(dev);
 509         if (ret)
 510                 return ret;
 511
 512         if (HAS_BSD(dev)) {
 513                 ret = logical_bsd_ring_init(dev);
 514                 if (ret)
 515                         goto cleanup_render_ring;
 516         }
 517
 518         if (HAS_BLT(dev)) {
 519                 ret = logical_blt_ring_init(dev);
 520                 if (ret)
 521                         goto cleanup_bsd_ring;
 522         }
 523
 524         if (HAS_VEBOX(dev)) {
 525                 ret = logical_vebox_ring_init(dev);
 526                 if (ret)
 527                         goto cleanup_blt_ring;
 528         }
 529
 530         if (HAS_BSD2(dev)) {
 531                 ret = logical_bsd2_ring_init(dev);
 532                 if (ret)
 533                         goto cleanup_vebox_ring;
 534         }
 535
 536         ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
 537         if (ret)
 538                 goto cleanup_bsd2_ring;
 539
 540         return 0;
 541
 542 cleanup_bsd2_ring:
 543         intel_logical_ring_cleanup(&dev_priv->ring[VCS2]);
 544 cleanup_vebox_ring:
 545         intel_logical_ring_cleanup(&dev_priv->ring[VECS]);
 546 cleanup_blt_ring:
 547         intel_logical_ring_cleanup(&dev_priv->ring[BCS]);
 548 cleanup_bsd_ring:
 549         intel_logical_ring_cleanup(&dev_priv->ring[VCS]);
 550 cleanup_render_ring:
 551         intel_logical_ring_cleanup(&dev_priv->ring[RCS]);
 552
 553         return ret;
 554 }
 555
 556 static int
 557 populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_obj,
 558                     struct intel_engine_cs *ring, struct intel_ringbuffer *ringbuf)
 559 {
 560         struct drm_i915_gem_object *ring_obj = ringbuf->obj;
 561         struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(ctx);
 562         struct page *page;
 563         uint32_t *reg_state;
 564         int ret;
 565
 566         ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
 567         if (ret) {
 568                 DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
 569                 return ret;
 570         }
 571
 572         ret = i915_gem_object_get_pages(ctx_obj);
 573         if (ret) {
 574                 DRM_DEBUG_DRIVER("Could not get object pages\n");
 575                 return ret;
 576         }
 577
 578         i915_gem_object_pin_pages(ctx_obj);
 579
 580         /* The second page of the context object contains some fields which must
 581          * be set up prior to the first execution. */
 582         page = i915_gem_object_get_page(ctx_obj, 1);
 583         reg_state = kmap_atomic(page);
 584
 585         /* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM
 586          * commands followed by (reg, value) pairs. The values we are setting here are
 587          * only for the first context restore: on a subsequent save, the GPU will
 588          * recreate this batchbuffer with new values (including all the missing
 589          * MI_LOAD_REGISTER_IMM commands that we are not initializing here). */
 590         if (ring->id == RCS)
 591                 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(14);
 592         else
 593                 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(11);
 594         reg_state[CTX_LRI_HEADER_0] |= MI_LRI_FORCE_POSTED;
 595         reg_state[CTX_CONTEXT_CONTROL] = RING_CONTEXT_CONTROL(ring);
 596         reg_state[CTX_CONTEXT_CONTROL+1] =
 597                         _MASKED_BIT_ENABLE((1<<3) | MI_RESTORE_INHIBIT);
 598         reg_state[CTX_RING_HEAD] = RING_HEAD(ring->mmio_base);
 599         reg_state[CTX_RING_HEAD+1] = 0;
 600         reg_state[CTX_RING_TAIL] = RING_TAIL(ring->mmio_base);
 601         reg_state[CTX_RING_TAIL+1] = 0;
 602         reg_state[CTX_RING_BUFFER_START] = RING_START(ring->mmio_base);
 603         reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(ring_obj);
 604         reg_state[CTX_RING_BUFFER_CONTROL] = RING_CTL(ring->mmio_base);
 605         reg_state[CTX_RING_BUFFER_CONTROL+1] =
 606                         ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID;
 607         reg_state[CTX_BB_HEAD_U] = ring->mmio_base + 0x168;
 608         reg_state[CTX_BB_HEAD_U+1] = 0;
 609         reg_state[CTX_BB_HEAD_L] = ring->mmio_base + 0x140;
 610         reg_state[CTX_BB_HEAD_L+1] = 0;
 611         reg_state[CTX_BB_STATE] = ring->mmio_base + 0x110;
 612         reg_state[CTX_BB_STATE+1] = (1<<5);
 613         reg_state[CTX_SECOND_BB_HEAD_U] = ring->mmio_base + 0x11c;
 614         reg_state[CTX_SECOND_BB_HEAD_U+1] = 0;
 615         reg_state[CTX_SECOND_BB_HEAD_L] = ring->mmio_base + 0x114;
 616         reg_state[CTX_SECOND_BB_HEAD_L+1] = 0;
 617         reg_state[CTX_SECOND_BB_STATE] = ring->mmio_base + 0x118;
 618         reg_state[CTX_SECOND_BB_STATE+1] = 0;
 619         if (ring->id == RCS) {
 620                 /* TODO: according to BSpec, the register state context
 621                  * for CHV does not have these. OTOH, these registers do
 622                  * exist in CHV. I'm waiting for a clarification */
 623                 reg_state[CTX_BB_PER_CTX_PTR] = ring->mmio_base + 0x1c0;
 624                 reg_state[CTX_BB_PER_CTX_PTR+1] = 0;
 625                 reg_state[CTX_RCS_INDIRECT_CTX] = ring->mmio_base + 0x1c4;
 626                 reg_state[CTX_RCS_INDIRECT_CTX+1] = 0;
 627                 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET] = ring->mmio_base + 0x1c8;
 628                 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] = 0;
 629         }
 630         reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9);
 631         reg_state[CTX_LRI_HEADER_1] |= MI_LRI_FORCE_POSTED;
 632         reg_state[CTX_CTX_TIMESTAMP] = ring->mmio_base + 0x3a8;
 633         reg_state[CTX_CTX_TIMESTAMP+1] = 0;
 634         reg_state[CTX_PDP3_UDW] = GEN8_RING_PDP_UDW(ring, 3);
 635         reg_state[CTX_PDP3_LDW] = GEN8_RING_PDP_LDW(ring, 3);
 636         reg_state[CTX_PDP2_UDW] = GEN8_RING_PDP_UDW(ring, 2);
 637         reg_state[CTX_PDP2_LDW] = GEN8_RING_PDP_LDW(ring, 2);
 638         reg_state[CTX_PDP1_UDW] = GEN8_RING_PDP_UDW(ring, 1);
 639         reg_state[CTX_PDP1_LDW] = GEN8_RING_PDP_LDW(ring, 1);
 640         reg_state[CTX_PDP0_UDW] = GEN8_RING_PDP_UDW(ring, 0);
 641         reg_state[CTX_PDP0_LDW] = GEN8_RING_PDP_LDW(ring, 0);
 642         reg_state[CTX_PDP3_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[3]);
 643         reg_state[CTX_PDP3_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[3]);
 644         reg_state[CTX_PDP2_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[2]);
 645         reg_state[CTX_PDP2_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[2]);
 646         reg_state[CTX_PDP1_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[1]);
 647         reg_state[CTX_PDP1_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[1]);
 648         reg_state[CTX_PDP0_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[0]);
 649         reg_state[CTX_PDP0_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[0]);
 650         if (ring->id == RCS) {
 651                 reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
 652                 reg_state[CTX_R_PWR_CLK_STATE] = 0x20c8;
 653                 reg_state[CTX_R_PWR_CLK_STATE+1] = 0;
 654         }
 655
 656         kunmap_atomic(reg_state);
 657
 658         ctx_obj->dirty = 1;
 659         set_page_dirty(page);
 660         i915_gem_object_unpin_pages(ctx_obj);
 661
 662         return 0;
 663 }
 664
 665 void intel_lr_context_free(struct intel_context *ctx)
 666 {
 667         int i;
 668
 669         for (i = 0; i < I915_NUM_RINGS; i++) {
 670                 struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state;
 671                 struct intel_ringbuffer *ringbuf = ctx->engine[i].ringbuf;
 672
 673                 if (ctx_obj) {
 674                         intel_destroy_ringbuffer_obj(ringbuf);
 675                         kfree(ringbuf);
 676                         i915_gem_object_ggtt_unpin(ctx_obj);
 677                         drm_gem_object_unreference(&ctx_obj->base);
 678                 }
 679         }
 680 }
 681
 682 static uint32_t get_lr_context_size(struct intel_engine_cs *ring)
 683 {
 684         int ret = 0;
 685
 686         WARN_ON(INTEL_INFO(ring->dev)->gen != 8);
 687
 688         switch (ring->id) {
 689         case RCS:
 690                 ret = GEN8_LR_CONTEXT_RENDER_SIZE;
 691                 break;
 692         case VCS:
 693         case BCS:
 694         case VECS:
 695         case VCS2:
 696                 ret = GEN8_LR_CONTEXT_OTHER_SIZE;
 697                 break;
 698         }
 699
 700         return ret;
 701 }
 702
 703 int intel_lr_context_deferred_create(struct intel_context *ctx,
 704                                      struct intel_engine_cs *ring)
 705 {
 706         struct drm_device *dev = ring->dev;
 707         struct drm_i915_gem_object *ctx_obj;
 708         uint32_t context_size;
 709         struct intel_ringbuffer *ringbuf;
 710         int ret;
 711
 712         WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL);
 713         if (ctx->engine[ring->id].state)
 714                 return 0;
 715
 716         context_size = round_up(get_lr_context_size(ring), 4096);
 717
 718         ctx_obj = i915_gem_alloc_context_obj(dev, context_size);
 719         if (IS_ERR(ctx_obj)) {
 720                 ret = PTR_ERR(ctx_obj);
 721                 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed: %d\n", ret);
 722                 return ret;
 723         }
 724
 725         ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, 0);
 726         if (ret) {
 727                 DRM_DEBUG_DRIVER("Pin LRC backing obj failed: %d\n", ret);
 728                 drm_gem_object_unreference(&ctx_obj->base);
 729                 return ret;
 730         }
 731
 732         ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
 733         if (!ringbuf) {
 734                 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
 735                                 ring->name);
 736                 i915_gem_object_ggtt_unpin(ctx_obj);
 737                 drm_gem_object_unreference(&ctx_obj->base);
 738                 ret = -ENOMEM;
 739                 return ret;
 740         }
 741
 742         ringbuf->ring = ring;
 743         ringbuf->size = 32 * PAGE_SIZE;
 744         ringbuf->effective_size = ringbuf->size;
 745         ringbuf->head = 0;
 746         ringbuf->tail = 0;
 747         ringbuf->space = ringbuf->size;
 748         ringbuf->last_retired_head = -1;
 749
 750         /* TODO: For now we put this in the mappable region so that we can reuse
 751          * the existing ringbuffer code which ioremaps it. When we start
 752          * creating many contexts, this will no longer work and we must switch
 753          * to a kmapish interface.
 754          */
 755         ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
 756         if (ret) {
 757                 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer obj %s: %d\n",
 758                                 ring->name, ret);
 759                 goto error;
 760         }
 761
 762         ret = populate_lr_context(ctx, ctx_obj, ring, ringbuf);
 763         if (ret) {
 764                 DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
 765                 intel_destroy_ringbuffer_obj(ringbuf);
 766                 goto error;
 767         }
 768
 769         ctx->engine[ring->id].ringbuf = ringbuf;
 770         ctx->engine[ring->id].state = ctx_obj;
 771
 772         return 0;
 773
 774 error:
 775         kfree(ringbuf);
 776         i915_gem_object_ggtt_unpin(ctx_obj);
 777         drm_gem_object_unreference(&ctx_obj->base);
 778         return ret;
 779 }