drivers/gpu/drm/i915/intel_lrc.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Ben Widawsky <ben@bwidawsk.net>
  25  *    Michel Thierry <michel.thierry@intel.com>
  26  *    Thomas Daniel <thomas.daniel@intel.com>
  27  *    Oscar Mateo <oscar.mateo@intel.com>
  28  *
  29  */
  30
  31 /*
  32  * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
  33  * These expanded contexts enable a number of new abilities, especially
  34  * "Execlists" (also implemented in this file).
  35  *
  36  * Execlists are the new method by which, on gen8+ hardware, workloads are
  37  * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
  38  */
  39
  40 #include <drm/drmP.h>
  41 #include <drm/i915_drm.h>
  42 #include "i915_drv.h"
  43
  44 #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
  45 #define GEN8_LR_CONTEXT_OTHER_SIZE (2 * PAGE_SIZE)
  46
  47 #define GEN8_LR_CONTEXT_ALIGN 4096
  48
  49 #define RING_ELSP(ring)                 ((ring)->mmio_base+0x230)
  50 #define RING_CONTEXT_CONTROL(ring)      ((ring)->mmio_base+0x244)
  51
  52 #define CTX_LRI_HEADER_0                0x01
  53 #define CTX_CONTEXT_CONTROL             0x02
  54 #define CTX_RING_HEAD                   0x04
  55 #define CTX_RING_TAIL                   0x06
  56 #define CTX_RING_BUFFER_START           0x08
  57 #define CTX_RING_BUFFER_CONTROL         0x0a
  58 #define CTX_BB_HEAD_U                   0x0c
  59 #define CTX_BB_HEAD_L                   0x0e
  60 #define CTX_BB_STATE                    0x10
  61 #define CTX_SECOND_BB_HEAD_U            0x12
  62 #define CTX_SECOND_BB_HEAD_L            0x14
  63 #define CTX_SECOND_BB_STATE             0x16
  64 #define CTX_BB_PER_CTX_PTR              0x18
  65 #define CTX_RCS_INDIRECT_CTX            0x1a
  66 #define CTX_RCS_INDIRECT_CTX_OFFSET     0x1c
  67 #define CTX_LRI_HEADER_1                0x21
  68 #define CTX_CTX_TIMESTAMP               0x22
  69 #define CTX_PDP3_UDW                    0x24
  70 #define CTX_PDP3_LDW                    0x26
  71 #define CTX_PDP2_UDW                    0x28
  72 #define CTX_PDP2_LDW                    0x2a
  73 #define CTX_PDP1_UDW                    0x2c
  74 #define CTX_PDP1_LDW                    0x2e
  75 #define CTX_PDP0_UDW                    0x30
  76 #define CTX_PDP0_LDW                    0x32
  77 #define CTX_LRI_HEADER_2                0x41
  78 #define CTX_R_PWR_CLK_STATE             0x42
  79 #define CTX_GPGPU_CSR_BASE_ADDRESS      0x44
  80
  81 int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists)
  82 {
  83         WARN_ON(i915.enable_ppgtt == -1);
  84
  85         if (enable_execlists == 0)
  86                 return 0;
  87
  88         if (HAS_LOGICAL_RING_CONTEXTS(dev) && USES_PPGTT(dev) &&
  89             i915.use_mmio_flip >= 0)
  90                 return 1;
  91
  92         return 0;
  93 }
  94
  95 static int logical_ring_invalidate_all_caches(struct intel_ringbuffer *ringbuf)
  96 {
  97         struct intel_engine_cs *ring = ringbuf->ring;
  98         uint32_t flush_domains;
  99         int ret;
 100
 101         flush_domains = 0;
 102         if (ring->gpu_caches_dirty)
 103                 flush_domains = I915_GEM_GPU_DOMAINS;
 104
 105         ret = ring->emit_flush(ringbuf, I915_GEM_GPU_DOMAINS, flush_domains);
 106         if (ret)
 107                 return ret;
 108
 109         ring->gpu_caches_dirty = false;
 110         return 0;
 111 }
 112
 113 static int execlists_move_to_gpu(struct intel_ringbuffer *ringbuf,
 114                                  struct list_head *vmas)
 115 {
 116         struct intel_engine_cs *ring = ringbuf->ring;
 117         struct i915_vma *vma;
 118         uint32_t flush_domains = 0;
 119         bool flush_chipset = false;
 120         int ret;
 121
 122         list_for_each_entry(vma, vmas, exec_list) {
 123                 struct drm_i915_gem_object *obj = vma->obj;
 124
 125                 ret = i915_gem_object_sync(obj, ring);
 126                 if (ret)
 127                         return ret;
 128
 129                 if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
 130                         flush_chipset |= i915_gem_clflush_object(obj, false);
 131
 132                 flush_domains |= obj->base.write_domain;
 133         }
 134
 135         if (flush_domains & I915_GEM_DOMAIN_GTT)
 136                 wmb();
 137
 138         /* Unconditionally invalidate gpu caches and ensure that we do flush
 139          * any residual writes from the previous batch.
 140          */
 141         return logical_ring_invalidate_all_caches(ringbuf);
 142 }
 143
 144 int intel_execlists_submission(struct drm_device *dev, struct drm_file *file,
 145                                struct intel_engine_cs *ring,
 146                                struct intel_context *ctx,
 147                                struct drm_i915_gem_execbuffer2 *args,
 148                                struct list_head *vmas,
 149                                struct drm_i915_gem_object *batch_obj,
 150                                u64 exec_start, u32 flags)
 151 {
 152         struct drm_i915_private *dev_priv = dev->dev_private;
 153         struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
 154         int instp_mode;
 155         u32 instp_mask;
 156         int ret;
 157
 158         instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
 159         instp_mask = I915_EXEC_CONSTANTS_MASK;
 160         switch (instp_mode) {
 161         case I915_EXEC_CONSTANTS_REL_GENERAL:
 162         case I915_EXEC_CONSTANTS_ABSOLUTE:
 163         case I915_EXEC_CONSTANTS_REL_SURFACE:
 164                 if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) {
 165                         DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
 166                         return -EINVAL;
 167                 }
 168
 169                 if (instp_mode != dev_priv->relative_constants_mode) {
 170                         if (instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
 171                                 DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
 172                                 return -EINVAL;
 173                         }
 174
 175                         /* The HW changed the meaning on this bit on gen6 */
 176                         instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
 177                 }
 178                 break;
 179         default:
 180                 DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
 181                 return -EINVAL;
 182         }
 183
 184         if (args->num_cliprects != 0) {
 185                 DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
 186                 return -EINVAL;
 187         } else {
 188                 if (args->DR4 == 0xffffffff) {
 189                         DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
 190                         args->DR4 = 0;
 191                 }
 192
 193                 if (args->DR1 || args->DR4 || args->cliprects_ptr) {
 194                         DRM_DEBUG("0 cliprects but dirt in cliprects fields\n");
 195                         return -EINVAL;
 196                 }
 197         }
 198
 199         if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
 200                 DRM_DEBUG("sol reset is gen7 only\n");
 201                 return -EINVAL;
 202         }
 203
 204         ret = execlists_move_to_gpu(ringbuf, vmas);
 205         if (ret)
 206                 return ret;
 207
 208         if (ring == &dev_priv->ring[RCS] &&
 209             instp_mode != dev_priv->relative_constants_mode) {
 210                 ret = intel_logical_ring_begin(ringbuf, 4);
 211                 if (ret)
 212                         return ret;
 213
 214                 intel_logical_ring_emit(ringbuf, MI_NOOP);
 215                 intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(1));
 216                 intel_logical_ring_emit(ringbuf, INSTPM);
 217                 intel_logical_ring_emit(ringbuf, instp_mask << 16 | instp_mode);
 218                 intel_logical_ring_advance(ringbuf);
 219
 220                 dev_priv->relative_constants_mode = instp_mode;
 221         }
 222
 223         ret = ring->emit_bb_start(ringbuf, exec_start, flags);
 224         if (ret)
 225                 return ret;
 226
 227         i915_gem_execbuffer_move_to_active(vmas, ring);
 228         i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj);
 229
 230         return 0;
 231 }
 232
 233 void intel_logical_ring_stop(struct intel_engine_cs *ring)
 234 {
 235         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 236         int ret;
 237
 238         if (!intel_ring_initialized(ring))
 239                 return;
 240
 241         ret = intel_ring_idle(ring);
 242         if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
 243                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
 244                           ring->name, ret);
 245
 246         /* TODO: Is this correct with Execlists enabled? */
 247         I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
 248         if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
 249                 DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
 250                 return;
 251         }
 252         I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
 253 }
 254
 255 void intel_logical_ring_advance_and_submit(struct intel_ringbuffer *ringbuf)
 256 {
 257         intel_logical_ring_advance(ringbuf);
 258
 259         if (intel_ring_stopped(ringbuf->ring))
 260                 return;
 261
 262         /* TODO: how to submit a context to the ELSP is not here yet */
 263 }
 264
 265 static int logical_ring_alloc_seqno(struct intel_engine_cs *ring)
 266 {
 267         if (ring->outstanding_lazy_seqno)
 268                 return 0;
 269
 270         if (ring->preallocated_lazy_request == NULL) {
 271                 struct drm_i915_gem_request *request;
 272
 273                 request = kmalloc(sizeof(*request), GFP_KERNEL);
 274                 if (request == NULL)
 275                         return -ENOMEM;
 276
 277                 ring->preallocated_lazy_request = request;
 278         }
 279
 280         return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
 281 }
 282
 283 static int logical_ring_wait_request(struct intel_ringbuffer *ringbuf,
 284                                      int bytes)
 285 {
 286         struct intel_engine_cs *ring = ringbuf->ring;
 287         struct drm_i915_gem_request *request;
 288         u32 seqno = 0;
 289         int ret;
 290
 291         if (ringbuf->last_retired_head != -1) {
 292                 ringbuf->head = ringbuf->last_retired_head;
 293                 ringbuf->last_retired_head = -1;
 294
 295                 ringbuf->space = intel_ring_space(ringbuf);
 296                 if (ringbuf->space >= bytes)
 297                         return 0;
 298         }
 299
 300         list_for_each_entry(request, &ring->request_list, list) {
 301                 if (__intel_ring_space(request->tail, ringbuf->tail,
 302                                        ringbuf->size) >= bytes) {
 303                         seqno = request->seqno;
 304                         break;
 305                 }
 306         }
 307
 308         if (seqno == 0)
 309                 return -ENOSPC;
 310
 311         ret = i915_wait_seqno(ring, seqno);
 312         if (ret)
 313                 return ret;
 314
 315         /* TODO: make sure we update the right ringbuffer's last_retired_head
 316          * when retiring requests */
 317         i915_gem_retire_requests_ring(ring);
 318         ringbuf->head = ringbuf->last_retired_head;
 319         ringbuf->last_retired_head = -1;
 320
 321         ringbuf->space = intel_ring_space(ringbuf);
 322         return 0;
 323 }
 324
 325 static int logical_ring_wait_for_space(struct intel_ringbuffer *ringbuf,
 326                                        int bytes)
 327 {
 328         struct intel_engine_cs *ring = ringbuf->ring;
 329         struct drm_device *dev = ring->dev;
 330         struct drm_i915_private *dev_priv = dev->dev_private;
 331         unsigned long end;
 332         int ret;
 333
 334         ret = logical_ring_wait_request(ringbuf, bytes);
 335         if (ret != -ENOSPC)
 336                 return ret;
 337
 338         /* Force the context submission in case we have been skipping it */
 339         intel_logical_ring_advance_and_submit(ringbuf);
 340
 341         /* With GEM the hangcheck timer should kick us out of the loop,
 342          * leaving it early runs the risk of corrupting GEM state (due
 343          * to running on almost untested codepaths). But on resume
 344          * timers don't work yet, so prevent a complete hang in that
 345          * case by choosing an insanely large timeout. */
 346         end = jiffies + 60 * HZ;
 347
 348         do {
 349                 ringbuf->head = I915_READ_HEAD(ring);
 350                 ringbuf->space = intel_ring_space(ringbuf);
 351                 if (ringbuf->space >= bytes) {
 352                         ret = 0;
 353                         break;
 354                 }
 355
 356                 msleep(1);
 357
 358                 if (dev_priv->mm.interruptible && signal_pending(current)) {
 359                         ret = -ERESTARTSYS;
 360                         break;
 361                 }
 362
 363                 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
 364                                            dev_priv->mm.interruptible);
 365                 if (ret)
 366                         break;
 367
 368                 if (time_after(jiffies, end)) {
 369                         ret = -EBUSY;
 370                         break;
 371                 }
 372         } while (1);
 373
 374         return ret;
 375 }
 376
 377 static int logical_ring_wrap_buffer(struct intel_ringbuffer *ringbuf)
 378 {
 379         uint32_t __iomem *virt;
 380         int rem = ringbuf->size - ringbuf->tail;
 381
 382         if (ringbuf->space < rem) {
 383                 int ret = logical_ring_wait_for_space(ringbuf, rem);
 384
 385                 if (ret)
 386                         return ret;
 387         }
 388
 389         virt = ringbuf->virtual_start + ringbuf->tail;
 390         rem /= 4;
 391         while (rem--)
 392                 iowrite32(MI_NOOP, virt++);
 393
 394         ringbuf->tail = 0;
 395         ringbuf->space = intel_ring_space(ringbuf);
 396
 397         return 0;
 398 }
 399
 400 static int logical_ring_prepare(struct intel_ringbuffer *ringbuf, int bytes)
 401 {
 402         int ret;
 403
 404         if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {
 405                 ret = logical_ring_wrap_buffer(ringbuf);
 406                 if (unlikely(ret))
 407                         return ret;
 408         }
 409
 410         if (unlikely(ringbuf->space < bytes)) {
 411                 ret = logical_ring_wait_for_space(ringbuf, bytes);
 412                 if (unlikely(ret))
 413                         return ret;
 414         }
 415
 416         return 0;
 417 }
 418
 419 int intel_logical_ring_begin(struct intel_ringbuffer *ringbuf, int num_dwords)
 420 {
 421         struct intel_engine_cs *ring = ringbuf->ring;
 422         struct drm_device *dev = ring->dev;
 423         struct drm_i915_private *dev_priv = dev->dev_private;
 424         int ret;
 425
 426         ret = i915_gem_check_wedge(&dev_priv->gpu_error,
 427                                    dev_priv->mm.interruptible);
 428         if (ret)
 429                 return ret;
 430
 431         ret = logical_ring_prepare(ringbuf, num_dwords * sizeof(uint32_t));
 432         if (ret)
 433                 return ret;
 434
 435         /* Preallocate the olr before touching the ring */
 436         ret = logical_ring_alloc_seqno(ring);
 437         if (ret)
 438                 return ret;
 439
 440         ringbuf->space -= num_dwords * sizeof(uint32_t);
 441         return 0;
 442 }
 443
 444 static int gen8_init_common_ring(struct intel_engine_cs *ring)
 445 {
 446         struct drm_device *dev = ring->dev;
 447         struct drm_i915_private *dev_priv = dev->dev_private;
 448
 449         I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
 450         I915_WRITE(RING_HWSTAM(ring->mmio_base), 0xffffffff);
 451
 452         I915_WRITE(RING_MODE_GEN7(ring),
 453                    _MASKED_BIT_DISABLE(GFX_REPLAY_MODE) |
 454                    _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
 455         POSTING_READ(RING_MODE_GEN7(ring));
 456         DRM_DEBUG_DRIVER("Execlists enabled for %s\n", ring->name);
 457
 458         memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
 459
 460         return 0;
 461 }
 462
 463 static int gen8_init_render_ring(struct intel_engine_cs *ring)
 464 {
 465         struct drm_device *dev = ring->dev;
 466         struct drm_i915_private *dev_priv = dev->dev_private;
 467         int ret;
 468
 469         ret = gen8_init_common_ring(ring);
 470         if (ret)
 471                 return ret;
 472
 473         /* We need to disable the AsyncFlip performance optimisations in order
 474          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
 475          * programmed to '1' on all products.
 476          *
 477          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
 478          */
 479         I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
 480
 481         ret = intel_init_pipe_control(ring);
 482         if (ret)
 483                 return ret;
 484
 485         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 486
 487         return ret;
 488 }
 489
 490 static int gen8_emit_bb_start(struct intel_ringbuffer *ringbuf,
 491                               u64 offset, unsigned flags)
 492 {
 493         bool ppgtt = !(flags & I915_DISPATCH_SECURE);
 494         int ret;
 495
 496         ret = intel_logical_ring_begin(ringbuf, 4);
 497         if (ret)
 498                 return ret;
 499
 500         /* FIXME(BDW): Address space and security selectors. */
 501         intel_logical_ring_emit(ringbuf, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
 502         intel_logical_ring_emit(ringbuf, lower_32_bits(offset));
 503         intel_logical_ring_emit(ringbuf, upper_32_bits(offset));
 504         intel_logical_ring_emit(ringbuf, MI_NOOP);
 505         intel_logical_ring_advance(ringbuf);
 506
 507         return 0;
 508 }
 509
 510 static bool gen8_logical_ring_get_irq(struct intel_engine_cs *ring)
 511 {
 512         struct drm_device *dev = ring->dev;
 513         struct drm_i915_private *dev_priv = dev->dev_private;
 514         unsigned long flags;
 515
 516         if (!dev->irq_enabled)
 517                 return false;
 518
 519         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 520         if (ring->irq_refcount++ == 0) {
 521                 I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
 522                 POSTING_READ(RING_IMR(ring->mmio_base));
 523         }
 524         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 525
 526         return true;
 527 }
 528
 529 static void gen8_logical_ring_put_irq(struct intel_engine_cs *ring)
 530 {
 531         struct drm_device *dev = ring->dev;
 532         struct drm_i915_private *dev_priv = dev->dev_private;
 533         unsigned long flags;
 534
 535         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 536         if (--ring->irq_refcount == 0) {
 537                 I915_WRITE_IMR(ring, ~ring->irq_keep_mask);
 538                 POSTING_READ(RING_IMR(ring->mmio_base));
 539         }
 540         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 541 }
 542
 543 static int gen8_emit_flush(struct intel_ringbuffer *ringbuf,
 544                            u32 invalidate_domains,
 545                            u32 unused)
 546 {
 547         struct intel_engine_cs *ring = ringbuf->ring;
 548         struct drm_device *dev = ring->dev;
 549         struct drm_i915_private *dev_priv = dev->dev_private;
 550         uint32_t cmd;
 551         int ret;
 552
 553         ret = intel_logical_ring_begin(ringbuf, 4);
 554         if (ret)
 555                 return ret;
 556
 557         cmd = MI_FLUSH_DW + 1;
 558
 559         if (ring == &dev_priv->ring[VCS]) {
 560                 if (invalidate_domains & I915_GEM_GPU_DOMAINS)
 561                         cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
 562                                 MI_FLUSH_DW_STORE_INDEX |
 563                                 MI_FLUSH_DW_OP_STOREDW;
 564         } else {
 565                 if (invalidate_domains & I915_GEM_DOMAIN_RENDER)
 566                         cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
 567                                 MI_FLUSH_DW_OP_STOREDW;
 568         }
 569
 570         intel_logical_ring_emit(ringbuf, cmd);
 571         intel_logical_ring_emit(ringbuf,
 572                                 I915_GEM_HWS_SCRATCH_ADDR |
 573                                 MI_FLUSH_DW_USE_GTT);
 574         intel_logical_ring_emit(ringbuf, 0); /* upper addr */
 575         intel_logical_ring_emit(ringbuf, 0); /* value */
 576         intel_logical_ring_advance(ringbuf);
 577
 578         return 0;
 579 }
 580
 581 static int gen8_emit_flush_render(struct intel_ringbuffer *ringbuf,
 582                                   u32 invalidate_domains,
 583                                   u32 flush_domains)
 584 {
 585         struct intel_engine_cs *ring = ringbuf->ring;
 586         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 587         u32 flags = 0;
 588         int ret;
 589
 590         flags |= PIPE_CONTROL_CS_STALL;
 591
 592         if (flush_domains) {
 593                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 594                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 595         }
 596
 597         if (invalidate_domains) {
 598                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 599                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 600                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 601                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 602                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 603                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 604                 flags |= PIPE_CONTROL_QW_WRITE;
 605                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
 606         }
 607
 608         ret = intel_logical_ring_begin(ringbuf, 6);
 609         if (ret)
 610                 return ret;
 611
 612         intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
 613         intel_logical_ring_emit(ringbuf, flags);
 614         intel_logical_ring_emit(ringbuf, scratch_addr);
 615         intel_logical_ring_emit(ringbuf, 0);
 616         intel_logical_ring_emit(ringbuf, 0);
 617         intel_logical_ring_emit(ringbuf, 0);
 618         intel_logical_ring_advance(ringbuf);
 619
 620         return 0;
 621 }
 622
 623 static u32 gen8_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
 624 {
 625         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 626 }
 627
 628 static void gen8_set_seqno(struct intel_engine_cs *ring, u32 seqno)
 629 {
 630         intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
 631 }
 632
 633 static int gen8_emit_request(struct intel_ringbuffer *ringbuf)
 634 {
 635         struct intel_engine_cs *ring = ringbuf->ring;
 636         u32 cmd;
 637         int ret;
 638
 639         ret = intel_logical_ring_begin(ringbuf, 6);
 640         if (ret)
 641                 return ret;
 642
 643         cmd = MI_STORE_DWORD_IMM_GEN8;
 644         cmd |= MI_GLOBAL_GTT;
 645
 646         intel_logical_ring_emit(ringbuf, cmd);
 647         intel_logical_ring_emit(ringbuf,
 648                                 (ring->status_page.gfx_addr +
 649                                 (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)));
 650         intel_logical_ring_emit(ringbuf, 0);
 651         intel_logical_ring_emit(ringbuf, ring->outstanding_lazy_seqno);
 652         intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
 653         intel_logical_ring_emit(ringbuf, MI_NOOP);
 654         intel_logical_ring_advance_and_submit(ringbuf);
 655
 656         return 0;
 657 }
 658
 659 void intel_logical_ring_cleanup(struct intel_engine_cs *ring)
 660 {
 661         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 662
 663         if (!intel_ring_initialized(ring))
 664                 return;
 665
 666         intel_logical_ring_stop(ring);
 667         WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
 668         ring->preallocated_lazy_request = NULL;
 669         ring->outstanding_lazy_seqno = 0;
 670
 671         if (ring->cleanup)
 672                 ring->cleanup(ring);
 673
 674         i915_cmd_parser_fini_ring(ring);
 675
 676         if (ring->status_page.obj) {
 677                 kunmap(sg_page(ring->status_page.obj->pages->sgl));
 678                 ring->status_page.obj = NULL;
 679         }
 680 }
 681
 682 static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring)
 683 {
 684         int ret;
 685         struct intel_context *dctx = ring->default_context;
 686         struct drm_i915_gem_object *dctx_obj;
 687
 688         /* Intentionally left blank. */
 689         ring->buffer = NULL;
 690
 691         ring->dev = dev;
 692         INIT_LIST_HEAD(&ring->active_list);
 693         INIT_LIST_HEAD(&ring->request_list);
 694         init_waitqueue_head(&ring->irq_queue);
 695
 696         ret = intel_lr_context_deferred_create(dctx, ring);
 697         if (ret)
 698                 return ret;
 699
 700         /* The status page is offset 0 from the context object in LRCs. */
 701         dctx_obj = dctx->engine[ring->id].state;
 702         ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(dctx_obj);
 703         ring->status_page.page_addr = kmap(sg_page(dctx_obj->pages->sgl));
 704         if (ring->status_page.page_addr == NULL)
 705                 return -ENOMEM;
 706         ring->status_page.obj = dctx_obj;
 707
 708         ret = i915_cmd_parser_init_ring(ring);
 709         if (ret)
 710                 return ret;
 711
 712         if (ring->init) {
 713                 ret = ring->init(ring);
 714                 if (ret)
 715                         return ret;
 716         }
 717
 718         return 0;
 719 }
 720
 721 static int logical_render_ring_init(struct drm_device *dev)
 722 {
 723         struct drm_i915_private *dev_priv = dev->dev_private;
 724         struct intel_engine_cs *ring = &dev_priv->ring[RCS];
 725
 726         ring->name = "render ring";
 727         ring->id = RCS;
 728         ring->mmio_base = RENDER_RING_BASE;
 729         ring->irq_enable_mask =
 730                 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
 731         ring->irq_keep_mask =
 732                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
 733         if (HAS_L3_DPF(dev))
 734                 ring->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
 735
 736         ring->init = gen8_init_render_ring;
 737         ring->cleanup = intel_fini_pipe_control;
 738         ring->get_seqno = gen8_get_seqno;
 739         ring->set_seqno = gen8_set_seqno;
 740         ring->emit_request = gen8_emit_request;
 741         ring->emit_flush = gen8_emit_flush_render;
 742         ring->irq_get = gen8_logical_ring_get_irq;
 743         ring->irq_put = gen8_logical_ring_put_irq;
 744         ring->emit_bb_start = gen8_emit_bb_start;
 745
 746         return logical_ring_init(dev, ring);
 747 }
 748
 749 static int logical_bsd_ring_init(struct drm_device *dev)
 750 {
 751         struct drm_i915_private *dev_priv = dev->dev_private;
 752         struct intel_engine_cs *ring = &dev_priv->ring[VCS];
 753
 754         ring->name = "bsd ring";
 755         ring->id = VCS;
 756         ring->mmio_base = GEN6_BSD_RING_BASE;
 757         ring->irq_enable_mask =
 758                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
 759         ring->irq_keep_mask =
 760                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
 761
 762         ring->init = gen8_init_common_ring;
 763         ring->get_seqno = gen8_get_seqno;
 764         ring->set_seqno = gen8_set_seqno;
 765         ring->emit_request = gen8_emit_request;
 766         ring->emit_flush = gen8_emit_flush;
 767         ring->irq_get = gen8_logical_ring_get_irq;
 768         ring->irq_put = gen8_logical_ring_put_irq;
 769         ring->emit_bb_start = gen8_emit_bb_start;
 770
 771         return logical_ring_init(dev, ring);
 772 }
 773
 774 static int logical_bsd2_ring_init(struct drm_device *dev)
 775 {
 776         struct drm_i915_private *dev_priv = dev->dev_private;
 777         struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
 778
 779         ring->name = "bds2 ring";
 780         ring->id = VCS2;
 781         ring->mmio_base = GEN8_BSD2_RING_BASE;
 782         ring->irq_enable_mask =
 783                 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
 784         ring->irq_keep_mask =
 785                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
 786
 787         ring->init = gen8_init_common_ring;
 788         ring->get_seqno = gen8_get_seqno;
 789         ring->set_seqno = gen8_set_seqno;
 790         ring->emit_request = gen8_emit_request;
 791         ring->emit_flush = gen8_emit_flush;
 792         ring->irq_get = gen8_logical_ring_get_irq;
 793         ring->irq_put = gen8_logical_ring_put_irq;
 794         ring->emit_bb_start = gen8_emit_bb_start;
 795
 796         return logical_ring_init(dev, ring);
 797 }
 798
 799 static int logical_blt_ring_init(struct drm_device *dev)
 800 {
 801         struct drm_i915_private *dev_priv = dev->dev_private;
 802         struct intel_engine_cs *ring = &dev_priv->ring[BCS];
 803
 804         ring->name = "blitter ring";
 805         ring->id = BCS;
 806         ring->mmio_base = BLT_RING_BASE;
 807         ring->irq_enable_mask =
 808                 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
 809         ring->irq_keep_mask =
 810                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
 811
 812         ring->init = gen8_init_common_ring;
 813         ring->get_seqno = gen8_get_seqno;
 814         ring->set_seqno = gen8_set_seqno;
 815         ring->emit_request = gen8_emit_request;
 816         ring->emit_flush = gen8_emit_flush;
 817         ring->irq_get = gen8_logical_ring_get_irq;
 818         ring->irq_put = gen8_logical_ring_put_irq;
 819         ring->emit_bb_start = gen8_emit_bb_start;
 820
 821         return logical_ring_init(dev, ring);
 822 }
 823
 824 static int logical_vebox_ring_init(struct drm_device *dev)
 825 {
 826         struct drm_i915_private *dev_priv = dev->dev_private;
 827         struct intel_engine_cs *ring = &dev_priv->ring[VECS];
 828
 829         ring->name = "video enhancement ring";
 830         ring->id = VECS;
 831         ring->mmio_base = VEBOX_RING_BASE;
 832         ring->irq_enable_mask =
 833                 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
 834         ring->irq_keep_mask =
 835                 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
 836
 837         ring->init = gen8_init_common_ring;
 838         ring->get_seqno = gen8_get_seqno;
 839         ring->set_seqno = gen8_set_seqno;
 840         ring->emit_request = gen8_emit_request;
 841         ring->emit_flush = gen8_emit_flush;
 842         ring->irq_get = gen8_logical_ring_get_irq;
 843         ring->irq_put = gen8_logical_ring_put_irq;
 844         ring->emit_bb_start = gen8_emit_bb_start;
 845
 846         return logical_ring_init(dev, ring);
 847 }
 848
 849 int intel_logical_rings_init(struct drm_device *dev)
 850 {
 851         struct drm_i915_private *dev_priv = dev->dev_private;
 852         int ret;
 853
 854         ret = logical_render_ring_init(dev);
 855         if (ret)
 856                 return ret;
 857
 858         if (HAS_BSD(dev)) {
 859                 ret = logical_bsd_ring_init(dev);
 860                 if (ret)
 861                         goto cleanup_render_ring;
 862         }
 863
 864         if (HAS_BLT(dev)) {
 865                 ret = logical_blt_ring_init(dev);
 866                 if (ret)
 867                         goto cleanup_bsd_ring;
 868         }
 869
 870         if (HAS_VEBOX(dev)) {
 871                 ret = logical_vebox_ring_init(dev);
 872                 if (ret)
 873                         goto cleanup_blt_ring;
 874         }
 875
 876         if (HAS_BSD2(dev)) {
 877                 ret = logical_bsd2_ring_init(dev);
 878                 if (ret)
 879                         goto cleanup_vebox_ring;
 880         }
 881
 882         ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
 883         if (ret)
 884                 goto cleanup_bsd2_ring;
 885
 886         return 0;
 887
 888 cleanup_bsd2_ring:
 889         intel_logical_ring_cleanup(&dev_priv->ring[VCS2]);
 890 cleanup_vebox_ring:
 891         intel_logical_ring_cleanup(&dev_priv->ring[VECS]);
 892 cleanup_blt_ring:
 893         intel_logical_ring_cleanup(&dev_priv->ring[BCS]);
 894 cleanup_bsd_ring:
 895         intel_logical_ring_cleanup(&dev_priv->ring[VCS]);
 896 cleanup_render_ring:
 897         intel_logical_ring_cleanup(&dev_priv->ring[RCS]);
 898
 899         return ret;
 900 }
 901
 902 static int
 903 populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_obj,
 904                     struct intel_engine_cs *ring, struct intel_ringbuffer *ringbuf)
 905 {
 906         struct drm_i915_gem_object *ring_obj = ringbuf->obj;
 907         struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
 908         struct page *page;
 909         uint32_t *reg_state;
 910         int ret;
 911
 912         ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
 913         if (ret) {
 914                 DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
 915                 return ret;
 916         }
 917
 918         ret = i915_gem_object_get_pages(ctx_obj);
 919         if (ret) {
 920                 DRM_DEBUG_DRIVER("Could not get object pages\n");
 921                 return ret;
 922         }
 923
 924         i915_gem_object_pin_pages(ctx_obj);
 925
 926         /* The second page of the context object contains some fields which must
 927          * be set up prior to the first execution. */
 928         page = i915_gem_object_get_page(ctx_obj, 1);
 929         reg_state = kmap_atomic(page);
 930
 931         /* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM
 932          * commands followed by (reg, value) pairs. The values we are setting here are
 933          * only for the first context restore: on a subsequent save, the GPU will
 934          * recreate this batchbuffer with new values (including all the missing
 935          * MI_LOAD_REGISTER_IMM commands that we are not initializing here). */
 936         if (ring->id == RCS)
 937                 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(14);
 938         else
 939                 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(11);
 940         reg_state[CTX_LRI_HEADER_0] |= MI_LRI_FORCE_POSTED;
 941         reg_state[CTX_CONTEXT_CONTROL] = RING_CONTEXT_CONTROL(ring);
 942         reg_state[CTX_CONTEXT_CONTROL+1] =
 943                         _MASKED_BIT_ENABLE((1<<3) | MI_RESTORE_INHIBIT);
 944         reg_state[CTX_RING_HEAD] = RING_HEAD(ring->mmio_base);
 945         reg_state[CTX_RING_HEAD+1] = 0;
 946         reg_state[CTX_RING_TAIL] = RING_TAIL(ring->mmio_base);
 947         reg_state[CTX_RING_TAIL+1] = 0;
 948         reg_state[CTX_RING_BUFFER_START] = RING_START(ring->mmio_base);
 949         reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(ring_obj);
 950         reg_state[CTX_RING_BUFFER_CONTROL] = RING_CTL(ring->mmio_base);
 951         reg_state[CTX_RING_BUFFER_CONTROL+1] =
 952                         ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID;
 953         reg_state[CTX_BB_HEAD_U] = ring->mmio_base + 0x168;
 954         reg_state[CTX_BB_HEAD_U+1] = 0;
 955         reg_state[CTX_BB_HEAD_L] = ring->mmio_base + 0x140;
 956         reg_state[CTX_BB_HEAD_L+1] = 0;
 957         reg_state[CTX_BB_STATE] = ring->mmio_base + 0x110;
 958         reg_state[CTX_BB_STATE+1] = (1<<5);
 959         reg_state[CTX_SECOND_BB_HEAD_U] = ring->mmio_base + 0x11c;
 960         reg_state[CTX_SECOND_BB_HEAD_U+1] = 0;
 961         reg_state[CTX_SECOND_BB_HEAD_L] = ring->mmio_base + 0x114;
 962         reg_state[CTX_SECOND_BB_HEAD_L+1] = 0;
 963         reg_state[CTX_SECOND_BB_STATE] = ring->mmio_base + 0x118;
 964         reg_state[CTX_SECOND_BB_STATE+1] = 0;
 965         if (ring->id == RCS) {
 966                 /* TODO: according to BSpec, the register state context
 967                  * for CHV does not have these. OTOH, these registers do
 968                  * exist in CHV. I'm waiting for a clarification */
 969                 reg_state[CTX_BB_PER_CTX_PTR] = ring->mmio_base + 0x1c0;
 970                 reg_state[CTX_BB_PER_CTX_PTR+1] = 0;
 971                 reg_state[CTX_RCS_INDIRECT_CTX] = ring->mmio_base + 0x1c4;
 972                 reg_state[CTX_RCS_INDIRECT_CTX+1] = 0;
 973                 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET] = ring->mmio_base + 0x1c8;
 974                 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] = 0;
 975         }
 976         reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9);
 977         reg_state[CTX_LRI_HEADER_1] |= MI_LRI_FORCE_POSTED;
 978         reg_state[CTX_CTX_TIMESTAMP] = ring->mmio_base + 0x3a8;
 979         reg_state[CTX_CTX_TIMESTAMP+1] = 0;
 980         reg_state[CTX_PDP3_UDW] = GEN8_RING_PDP_UDW(ring, 3);
 981         reg_state[CTX_PDP3_LDW] = GEN8_RING_PDP_LDW(ring, 3);
 982         reg_state[CTX_PDP2_UDW] = GEN8_RING_PDP_UDW(ring, 2);
 983         reg_state[CTX_PDP2_LDW] = GEN8_RING_PDP_LDW(ring, 2);
 984         reg_state[CTX_PDP1_UDW] = GEN8_RING_PDP_UDW(ring, 1);
 985         reg_state[CTX_PDP1_LDW] = GEN8_RING_PDP_LDW(ring, 1);
 986         reg_state[CTX_PDP0_UDW] = GEN8_RING_PDP_UDW(ring, 0);
 987         reg_state[CTX_PDP0_LDW] = GEN8_RING_PDP_LDW(ring, 0);
 988         reg_state[CTX_PDP3_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[3]);
 989         reg_state[CTX_PDP3_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[3]);
 990         reg_state[CTX_PDP2_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[2]);
 991         reg_state[CTX_PDP2_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[2]);
 992         reg_state[CTX_PDP1_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[1]);
 993         reg_state[CTX_PDP1_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[1]);
 994         reg_state[CTX_PDP0_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[0]);
 995         reg_state[CTX_PDP0_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[0]);
 996         if (ring->id == RCS) {
 997                 reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
 998                 reg_state[CTX_R_PWR_CLK_STATE] = 0x20c8;
 999                 reg_state[CTX_R_PWR_CLK_STATE+1] = 0;
1000         }
1001
1002         kunmap_atomic(reg_state);
1003
1004         ctx_obj->dirty = 1;
1005         set_page_dirty(page);
1006         i915_gem_object_unpin_pages(ctx_obj);
1007
1008         return 0;
1009 }
1010
1011 void intel_lr_context_free(struct intel_context *ctx)
1012 {
1013         int i;
1014
1015         for (i = 0; i < I915_NUM_RINGS; i++) {
1016                 struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state;
1017                 struct intel_ringbuffer *ringbuf = ctx->engine[i].ringbuf;
1018
1019                 if (ctx_obj) {
1020                         intel_destroy_ringbuffer_obj(ringbuf);
1021                         kfree(ringbuf);
1022                         i915_gem_object_ggtt_unpin(ctx_obj);
1023                         drm_gem_object_unreference(&ctx_obj->base);
1024                 }
1025         }
1026 }
1027
1028 static uint32_t get_lr_context_size(struct intel_engine_cs *ring)
1029 {
1030         int ret = 0;
1031
1032         WARN_ON(INTEL_INFO(ring->dev)->gen != 8);
1033
1034         switch (ring->id) {
1035         case RCS:
1036                 ret = GEN8_LR_CONTEXT_RENDER_SIZE;
1037                 break;
1038         case VCS:
1039         case BCS:
1040         case VECS:
1041         case VCS2:
1042                 ret = GEN8_LR_CONTEXT_OTHER_SIZE;
1043                 break;
1044         }
1045
1046         return ret;
1047 }
1048
1049 int intel_lr_context_deferred_create(struct intel_context *ctx,
1050                                      struct intel_engine_cs *ring)
1051 {
1052         struct drm_device *dev = ring->dev;
1053         struct drm_i915_gem_object *ctx_obj;
1054         uint32_t context_size;
1055         struct intel_ringbuffer *ringbuf;
1056         int ret;
1057
1058         WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL);
1059         if (ctx->engine[ring->id].state)
1060                 return 0;
1061
1062         context_size = round_up(get_lr_context_size(ring), 4096);
1063
1064         ctx_obj = i915_gem_alloc_context_obj(dev, context_size);
1065         if (IS_ERR(ctx_obj)) {
1066                 ret = PTR_ERR(ctx_obj);
1067                 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed: %d\n", ret);
1068                 return ret;
1069         }
1070
1071         ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, 0);
1072         if (ret) {
1073                 DRM_DEBUG_DRIVER("Pin LRC backing obj failed: %d\n", ret);
1074                 drm_gem_object_unreference(&ctx_obj->base);
1075                 return ret;
1076         }
1077
1078         ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
1079         if (!ringbuf) {
1080                 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
1081                                 ring->name);
1082                 i915_gem_object_ggtt_unpin(ctx_obj);
1083                 drm_gem_object_unreference(&ctx_obj->base);
1084                 ret = -ENOMEM;
1085                 return ret;
1086         }
1087
1088         ringbuf->ring = ring;
1089         ringbuf->FIXME_lrc_ctx = ctx;
1090
1091         ringbuf->size = 32 * PAGE_SIZE;
1092         ringbuf->effective_size = ringbuf->size;
1093         ringbuf->head = 0;
1094         ringbuf->tail = 0;
1095         ringbuf->space = ringbuf->size;
1096         ringbuf->last_retired_head = -1;
1097
1098         /* TODO: For now we put this in the mappable region so that we can reuse
1099          * the existing ringbuffer code which ioremaps it. When we start
1100          * creating many contexts, this will no longer work and we must switch
1101          * to a kmapish interface.
1102          */
1103         ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
1104         if (ret) {
1105                 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer obj %s: %d\n",
1106                                 ring->name, ret);
1107                 goto error;
1108         }
1109
1110         ret = populate_lr_context(ctx, ctx_obj, ring, ringbuf);
1111         if (ret) {
1112                 DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
1113                 intel_destroy_ringbuffer_obj(ringbuf);
1114                 goto error;
1115         }
1116
1117         ctx->engine[ring->id].ringbuf = ringbuf;
1118         ctx->engine[ring->id].state = ctx_obj;
1119
1120         return 0;
1121
1122 error:
1123         kfree(ringbuf);
1124         i915_gem_object_ggtt_unpin(ctx_obj);
1125         drm_gem_object_unreference(&ctx_obj->base);
1126         return ret;
1127 }