drm/i915: add yesno utility function
[deliverable/linux.git] / drivers / gpu / drm / i915 / intel_lrc.c
CommitLineData
b20385f1
OM
1/*
2 * Copyright © 2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Ben Widawsky <ben@bwidawsk.net>
25 * Michel Thierry <michel.thierry@intel.com>
26 * Thomas Daniel <thomas.daniel@intel.com>
27 * Oscar Mateo <oscar.mateo@intel.com>
28 *
29 */
30
73e4d07f
OM
31/**
32 * DOC: Logical Rings, Logical Ring Contexts and Execlists
33 *
34 * Motivation:
b20385f1
OM
35 * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
36 * These expanded contexts enable a number of new abilities, especially
37 * "Execlists" (also implemented in this file).
38 *
73e4d07f
OM
39 * One of the main differences with the legacy HW contexts is that logical
40 * ring contexts incorporate many more things to the context's state, like
41 * PDPs or ringbuffer control registers:
42 *
43 * The reason why PDPs are included in the context is straightforward: as
44 * PPGTTs (per-process GTTs) are actually per-context, having the PDPs
45 * contained there mean you don't need to do a ppgtt->switch_mm yourself,
46 * instead, the GPU will do it for you on the context switch.
47 *
48 * But, what about the ringbuffer control registers (head, tail, etc..)?
49 * shouldn't we just need a set of those per engine command streamer? This is
50 * where the name "Logical Rings" starts to make sense: by virtualizing the
51 * rings, the engine cs shifts to a new "ring buffer" with every context
52 * switch. When you want to submit a workload to the GPU you: A) choose your
53 * context, B) find its appropriate virtualized ring, C) write commands to it
54 * and then, finally, D) tell the GPU to switch to that context.
55 *
56 * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch
57 * to a contexts is via a context execution list, ergo "Execlists".
58 *
59 * LRC implementation:
60 * Regarding the creation of contexts, we have:
61 *
62 * - One global default context.
63 * - One local default context for each opened fd.
64 * - One local extra context for each context create ioctl call.
65 *
66 * Now that ringbuffers belong per-context (and not per-engine, like before)
67 * and that contexts are uniquely tied to a given engine (and not reusable,
68 * like before) we need:
69 *
70 * - One ringbuffer per-engine inside each context.
71 * - One backing object per-engine inside each context.
72 *
73 * The global default context starts its life with these new objects fully
74 * allocated and populated. The local default context for each opened fd is
75 * more complex, because we don't know at creation time which engine is going
76 * to use them. To handle this, we have implemented a deferred creation of LR
77 * contexts:
78 *
79 * The local context starts its life as a hollow or blank holder, that only
80 * gets populated for a given engine once we receive an execbuffer. If later
81 * on we receive another execbuffer ioctl for the same context but a different
82 * engine, we allocate/populate a new ringbuffer and context backing object and
83 * so on.
84 *
85 * Finally, regarding local contexts created using the ioctl call: as they are
86 * only allowed with the render ring, we can allocate & populate them right
87 * away (no need to defer anything, at least for now).
88 *
89 * Execlists implementation:
b20385f1
OM
90 * Execlists are the new method by which, on gen8+ hardware, workloads are
91 * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
73e4d07f
OM
92 * This method works as follows:
93 *
94 * When a request is committed, its commands (the BB start and any leading or
95 * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer
96 * for the appropriate context. The tail pointer in the hardware context is not
97 * updated at this time, but instead, kept by the driver in the ringbuffer
98 * structure. A structure representing this request is added to a request queue
99 * for the appropriate engine: this structure contains a copy of the context's
100 * tail after the request was written to the ring buffer and a pointer to the
101 * context itself.
102 *
103 * If the engine's request queue was empty before the request was added, the
104 * queue is processed immediately. Otherwise the queue will be processed during
105 * a context switch interrupt. In any case, elements on the queue will get sent
106 * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a
107 * globally unique 20-bits submission ID.
108 *
109 * When execution of a request completes, the GPU updates the context status
110 * buffer with a context complete event and generates a context switch interrupt.
111 * During the interrupt handling, the driver examines the events in the buffer:
112 * for each context complete event, if the announced ID matches that on the head
113 * of the request queue, then that request is retired and removed from the queue.
114 *
115 * After processing, if any requests were retired and the queue is not empty
116 * then a new execution list can be submitted. The two requests at the front of
117 * the queue are next to be submitted but since a context may not occur twice in
118 * an execution list, if subsequent requests have the same ID as the first then
119 * the two requests must be combined. This is done simply by discarding requests
120 * at the head of the queue until either only one requests is left (in which case
121 * we use a NULL second context) or the first two requests have unique IDs.
122 *
123 * By always executing the first two requests in the queue the driver ensures
124 * that the GPU is kept as busy as possible. In the case where a single context
125 * completes but a second context is still executing, the request for this second
126 * context will be at the head of the queue when we remove the first one. This
127 * request will then be resubmitted along with a new request for a different context,
128 * which will cause the hardware to continue executing the second request and queue
129 * the new request (the GPU detects the condition of a context getting preempted
130 * with the same context and optimizes the context switch flow by not doing
131 * preemption, but just sampling the new tail pointer).
132 *
b20385f1
OM
133 */
134
135#include <drm/drmP.h>
136#include <drm/i915_drm.h>
137#include "i915_drv.h"
3bbaba0c 138#include "intel_mocs.h"
127f1003 139
468c6816 140#define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
8c857917
OM
141#define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
142#define GEN8_LR_CONTEXT_OTHER_SIZE (2 * PAGE_SIZE)
143
e981e7b1
TD
144#define RING_EXECLIST_QFULL (1 << 0x2)
145#define RING_EXECLIST1_VALID (1 << 0x3)
146#define RING_EXECLIST0_VALID (1 << 0x4)
147#define RING_EXECLIST_ACTIVE_STATUS (3 << 0xE)
148#define RING_EXECLIST1_ACTIVE (1 << 0x11)
149#define RING_EXECLIST0_ACTIVE (1 << 0x12)
150
151#define GEN8_CTX_STATUS_IDLE_ACTIVE (1 << 0)
152#define GEN8_CTX_STATUS_PREEMPTED (1 << 1)
153#define GEN8_CTX_STATUS_ELEMENT_SWITCH (1 << 2)
154#define GEN8_CTX_STATUS_ACTIVE_IDLE (1 << 3)
155#define GEN8_CTX_STATUS_COMPLETE (1 << 4)
156#define GEN8_CTX_STATUS_LITE_RESTORE (1 << 15)
8670d6f9
OM
157
158#define CTX_LRI_HEADER_0 0x01
159#define CTX_CONTEXT_CONTROL 0x02
160#define CTX_RING_HEAD 0x04
161#define CTX_RING_TAIL 0x06
162#define CTX_RING_BUFFER_START 0x08
163#define CTX_RING_BUFFER_CONTROL 0x0a
164#define CTX_BB_HEAD_U 0x0c
165#define CTX_BB_HEAD_L 0x0e
166#define CTX_BB_STATE 0x10
167#define CTX_SECOND_BB_HEAD_U 0x12
168#define CTX_SECOND_BB_HEAD_L 0x14
169#define CTX_SECOND_BB_STATE 0x16
170#define CTX_BB_PER_CTX_PTR 0x18
171#define CTX_RCS_INDIRECT_CTX 0x1a
172#define CTX_RCS_INDIRECT_CTX_OFFSET 0x1c
173#define CTX_LRI_HEADER_1 0x21
174#define CTX_CTX_TIMESTAMP 0x22
175#define CTX_PDP3_UDW 0x24
176#define CTX_PDP3_LDW 0x26
177#define CTX_PDP2_UDW 0x28
178#define CTX_PDP2_LDW 0x2a
179#define CTX_PDP1_UDW 0x2c
180#define CTX_PDP1_LDW 0x2e
181#define CTX_PDP0_UDW 0x30
182#define CTX_PDP0_LDW 0x32
183#define CTX_LRI_HEADER_2 0x41
184#define CTX_R_PWR_CLK_STATE 0x42
185#define CTX_GPGPU_CSR_BASE_ADDRESS 0x44
186
84b790f8
BW
187#define GEN8_CTX_VALID (1<<0)
188#define GEN8_CTX_FORCE_PD_RESTORE (1<<1)
189#define GEN8_CTX_FORCE_RESTORE (1<<2)
190#define GEN8_CTX_L3LLC_COHERENT (1<<5)
191#define GEN8_CTX_PRIVILEGE (1<<8)
e5815a2e
MT
192
193#define ASSIGN_CTX_PDP(ppgtt, reg_state, n) { \
d852c7bf 194 const u64 _addr = i915_page_dir_dma_addr((ppgtt), (n)); \
e5815a2e
MT
195 reg_state[CTX_PDP ## n ## _UDW+1] = upper_32_bits(_addr); \
196 reg_state[CTX_PDP ## n ## _LDW+1] = lower_32_bits(_addr); \
197}
198
2dba3239
MT
199#define ASSIGN_CTX_PML4(ppgtt, reg_state) { \
200 reg_state[CTX_PDP0_UDW + 1] = upper_32_bits(px_dma(&ppgtt->pml4)); \
201 reg_state[CTX_PDP0_LDW + 1] = lower_32_bits(px_dma(&ppgtt->pml4)); \
202}
203
84b790f8
BW
204enum {
205 ADVANCED_CONTEXT = 0,
2dba3239 206 LEGACY_32B_CONTEXT,
84b790f8
BW
207 ADVANCED_AD_CONTEXT,
208 LEGACY_64B_CONTEXT
209};
2dba3239
MT
210#define GEN8_CTX_ADDRESSING_MODE_SHIFT 3
211#define GEN8_CTX_ADDRESSING_MODE(dev) (USES_FULL_48BIT_PPGTT(dev) ?\
212 LEGACY_64B_CONTEXT :\
213 LEGACY_32B_CONTEXT)
84b790f8
BW
214enum {
215 FAULT_AND_HANG = 0,
216 FAULT_AND_HALT, /* Debug only */
217 FAULT_AND_STREAM,
218 FAULT_AND_CONTINUE /* Unsupported */
219};
220#define GEN8_CTX_ID_SHIFT 32
17ee950d 221#define CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x17
84b790f8 222
8ba319da 223static int intel_lr_context_pin(struct drm_i915_gem_request *rq);
7ba717cf 224
73e4d07f
OM
225/**
226 * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists
227 * @dev: DRM device.
228 * @enable_execlists: value of i915.enable_execlists module parameter.
229 *
230 * Only certain platforms support Execlists (the prerequisites being
27401d12 231 * support for Logical Ring Contexts and Aliasing PPGTT or better).
73e4d07f
OM
232 *
233 * Return: 1 if Execlists is supported and has to be enabled.
234 */
127f1003
OM
235int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists)
236{
bd84b1e9
DV
237 WARN_ON(i915.enable_ppgtt == -1);
238
70ee45e1
DL
239 if (INTEL_INFO(dev)->gen >= 9)
240 return 1;
241
127f1003
OM
242 if (enable_execlists == 0)
243 return 0;
244
14bf993e
OM
245 if (HAS_LOGICAL_RING_CONTEXTS(dev) && USES_PPGTT(dev) &&
246 i915.use_mmio_flip >= 0)
127f1003
OM
247 return 1;
248
249 return 0;
250}
ede7d42b 251
73e4d07f
OM
252/**
253 * intel_execlists_ctx_id() - get the Execlists Context ID
254 * @ctx_obj: Logical Ring Context backing object.
255 *
256 * Do not confuse with ctx->id! Unfortunately we have a name overload
257 * here: the old context ID we pass to userspace as a handler so that
258 * they can refer to a context, and the new context ID we pass to the
259 * ELSP so that the GPU can inform us of the context status via
260 * interrupts.
261 *
262 * Return: 20-bits globally unique context ID.
263 */
84b790f8
BW
264u32 intel_execlists_ctx_id(struct drm_i915_gem_object *ctx_obj)
265{
d1675198
AD
266 u32 lrca = i915_gem_obj_ggtt_offset(ctx_obj) +
267 LRC_PPHWSP_PN * PAGE_SIZE;
84b790f8
BW
268
269 /* LRCA is required to be 4K aligned so the more significant 20 bits
270 * are globally unique */
271 return lrca >> 12;
272}
273
919f1f55
DG
274uint64_t intel_lr_context_descriptor(struct intel_context *ctx,
275 struct intel_engine_cs *ring)
84b790f8 276{
203a571b 277 struct drm_device *dev = ring->dev;
919f1f55 278 struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state;
84b790f8 279 uint64_t desc;
d1675198
AD
280 uint64_t lrca = i915_gem_obj_ggtt_offset(ctx_obj) +
281 LRC_PPHWSP_PN * PAGE_SIZE;
acdd884a
MT
282
283 WARN_ON(lrca & 0xFFFFFFFF00000FFFULL);
84b790f8
BW
284
285 desc = GEN8_CTX_VALID;
2dba3239 286 desc |= GEN8_CTX_ADDRESSING_MODE(dev) << GEN8_CTX_ADDRESSING_MODE_SHIFT;
51847fb9
AS
287 if (IS_GEN8(ctx_obj->base.dev))
288 desc |= GEN8_CTX_L3LLC_COHERENT;
84b790f8
BW
289 desc |= GEN8_CTX_PRIVILEGE;
290 desc |= lrca;
291 desc |= (u64)intel_execlists_ctx_id(ctx_obj) << GEN8_CTX_ID_SHIFT;
292
293 /* TODO: WaDisableLiteRestore when we start using semaphore
294 * signalling between Command Streamers */
295 /* desc |= GEN8_CTX_FORCE_RESTORE; */
296
203a571b
NH
297 /* WaEnableForceRestoreInCtxtDescForVCS:skl */
298 if (IS_GEN9(dev) &&
299 INTEL_REVID(dev) <= SKL_REVID_B0 &&
300 (ring->id == BCS || ring->id == VCS ||
301 ring->id == VECS || ring->id == VCS2))
302 desc |= GEN8_CTX_FORCE_RESTORE;
303
84b790f8
BW
304 return desc;
305}
306
cc3c4253
MK
307static void execlists_elsp_write(struct drm_i915_gem_request *rq0,
308 struct drm_i915_gem_request *rq1)
84b790f8 309{
cc3c4253
MK
310
311 struct intel_engine_cs *ring = rq0->ring;
6e7cc470
TU
312 struct drm_device *dev = ring->dev;
313 struct drm_i915_private *dev_priv = dev->dev_private;
1cff8cc3 314 uint64_t desc[2];
84b790f8 315
1cff8cc3 316 if (rq1) {
919f1f55 317 desc[1] = intel_lr_context_descriptor(rq1->ctx, rq1->ring);
1cff8cc3
MK
318 rq1->elsp_submitted++;
319 } else {
320 desc[1] = 0;
321 }
84b790f8 322
919f1f55 323 desc[0] = intel_lr_context_descriptor(rq0->ctx, rq0->ring);
1cff8cc3 324 rq0->elsp_submitted++;
84b790f8 325
1cff8cc3 326 /* You must always write both descriptors in the order below. */
a6111f7b
CW
327 spin_lock(&dev_priv->uncore.lock);
328 intel_uncore_forcewake_get__locked(dev_priv, FORCEWAKE_ALL);
1cff8cc3
MK
329 I915_WRITE_FW(RING_ELSP(ring), upper_32_bits(desc[1]));
330 I915_WRITE_FW(RING_ELSP(ring), lower_32_bits(desc[1]));
6daccb0b 331
1cff8cc3 332 I915_WRITE_FW(RING_ELSP(ring), upper_32_bits(desc[0]));
84b790f8 333 /* The context is automatically loaded after the following */
1cff8cc3 334 I915_WRITE_FW(RING_ELSP(ring), lower_32_bits(desc[0]));
84b790f8 335
1cff8cc3 336 /* ELSP is a wo register, use another nearby reg for posting */
a6111f7b
CW
337 POSTING_READ_FW(RING_EXECLIST_STATUS(ring));
338 intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL);
339 spin_unlock(&dev_priv->uncore.lock);
84b790f8
BW
340}
341
05d9824b 342static int execlists_update_context(struct drm_i915_gem_request *rq)
ae1250b9 343{
05d9824b
MK
344 struct intel_engine_cs *ring = rq->ring;
345 struct i915_hw_ppgtt *ppgtt = rq->ctx->ppgtt;
346 struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring->id].state;
347 struct drm_i915_gem_object *rb_obj = rq->ringbuf->obj;
ae1250b9
OM
348 struct page *page;
349 uint32_t *reg_state;
350
05d9824b
MK
351 BUG_ON(!ctx_obj);
352 WARN_ON(!i915_gem_obj_is_pinned(ctx_obj));
353 WARN_ON(!i915_gem_obj_is_pinned(rb_obj));
354
d1675198 355 page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
ae1250b9
OM
356 reg_state = kmap_atomic(page);
357
05d9824b
MK
358 reg_state[CTX_RING_TAIL+1] = rq->tail;
359 reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(rb_obj);
ae1250b9 360
2dba3239
MT
361 if (ppgtt && !USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
362 /* True 32b PPGTT with dynamic page allocation: update PDP
363 * registers and point the unallocated PDPs to scratch page.
364 * PML4 is allocated during ppgtt init, so this is not needed
365 * in 48-bit mode.
366 */
d7b2633d
MT
367 ASSIGN_CTX_PDP(ppgtt, reg_state, 3);
368 ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
369 ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
370 ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
371 }
372
ae1250b9
OM
373 kunmap_atomic(reg_state);
374
375 return 0;
376}
377
d8cb8875
MK
378static void execlists_submit_requests(struct drm_i915_gem_request *rq0,
379 struct drm_i915_gem_request *rq1)
84b790f8 380{
05d9824b 381 execlists_update_context(rq0);
d8cb8875 382
cc3c4253 383 if (rq1)
05d9824b 384 execlists_update_context(rq1);
84b790f8 385
cc3c4253 386 execlists_elsp_write(rq0, rq1);
84b790f8
BW
387}
388
acdd884a
MT
389static void execlists_context_unqueue(struct intel_engine_cs *ring)
390{
6d3d8274
NH
391 struct drm_i915_gem_request *req0 = NULL, *req1 = NULL;
392 struct drm_i915_gem_request *cursor = NULL, *tmp = NULL;
e981e7b1
TD
393
394 assert_spin_locked(&ring->execlist_lock);
acdd884a 395
779949f4
PA
396 /*
397 * If irqs are not active generate a warning as batches that finish
398 * without the irqs may get lost and a GPU Hang may occur.
399 */
400 WARN_ON(!intel_irqs_enabled(ring->dev->dev_private));
401
acdd884a
MT
402 if (list_empty(&ring->execlist_queue))
403 return;
404
405 /* Try to read in pairs */
406 list_for_each_entry_safe(cursor, tmp, &ring->execlist_queue,
407 execlist_link) {
408 if (!req0) {
409 req0 = cursor;
6d3d8274 410 } else if (req0->ctx == cursor->ctx) {
acdd884a
MT
411 /* Same ctx: ignore first request, as second request
412 * will update tail past first request's workload */
e1fee72c 413 cursor->elsp_submitted = req0->elsp_submitted;
acdd884a 414 list_del(&req0->execlist_link);
c86ee3a9
TD
415 list_add_tail(&req0->execlist_link,
416 &ring->execlist_retired_req_list);
acdd884a
MT
417 req0 = cursor;
418 } else {
419 req1 = cursor;
420 break;
421 }
422 }
423
53292cdb
MT
424 if (IS_GEN8(ring->dev) || IS_GEN9(ring->dev)) {
425 /*
426 * WaIdleLiteRestore: make sure we never cause a lite
427 * restore with HEAD==TAIL
428 */
d63f820f 429 if (req0->elsp_submitted) {
53292cdb
MT
430 /*
431 * Apply the wa NOOPS to prevent ring:HEAD == req:TAIL
432 * as we resubmit the request. See gen8_emit_request()
433 * for where we prepare the padding after the end of the
434 * request.
435 */
436 struct intel_ringbuffer *ringbuf;
437
438 ringbuf = req0->ctx->engine[ring->id].ringbuf;
439 req0->tail += 8;
440 req0->tail &= ringbuf->size - 1;
441 }
442 }
443
e1fee72c
OM
444 WARN_ON(req1 && req1->elsp_submitted);
445
d8cb8875 446 execlists_submit_requests(req0, req1);
acdd884a
MT
447}
448
e981e7b1
TD
449static bool execlists_check_remove_request(struct intel_engine_cs *ring,
450 u32 request_id)
451{
6d3d8274 452 struct drm_i915_gem_request *head_req;
e981e7b1
TD
453
454 assert_spin_locked(&ring->execlist_lock);
455
456 head_req = list_first_entry_or_null(&ring->execlist_queue,
6d3d8274 457 struct drm_i915_gem_request,
e981e7b1
TD
458 execlist_link);
459
460 if (head_req != NULL) {
461 struct drm_i915_gem_object *ctx_obj =
6d3d8274 462 head_req->ctx->engine[ring->id].state;
e981e7b1 463 if (intel_execlists_ctx_id(ctx_obj) == request_id) {
e1fee72c
OM
464 WARN(head_req->elsp_submitted == 0,
465 "Never submitted head request\n");
466
467 if (--head_req->elsp_submitted <= 0) {
468 list_del(&head_req->execlist_link);
c86ee3a9
TD
469 list_add_tail(&head_req->execlist_link,
470 &ring->execlist_retired_req_list);
e1fee72c
OM
471 return true;
472 }
e981e7b1
TD
473 }
474 }
475
476 return false;
477}
478
73e4d07f 479/**
3f7531c3 480 * intel_lrc_irq_handler() - handle Context Switch interrupts
73e4d07f
OM
481 * @ring: Engine Command Streamer to handle.
482 *
483 * Check the unread Context Status Buffers and manage the submission of new
484 * contexts to the ELSP accordingly.
485 */
3f7531c3 486void intel_lrc_irq_handler(struct intel_engine_cs *ring)
e981e7b1
TD
487{
488 struct drm_i915_private *dev_priv = ring->dev->dev_private;
489 u32 status_pointer;
490 u8 read_pointer;
491 u8 write_pointer;
492 u32 status;
493 u32 status_id;
494 u32 submit_contexts = 0;
495
496 status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(ring));
497
498 read_pointer = ring->next_context_status_buffer;
499 write_pointer = status_pointer & 0x07;
500 if (read_pointer > write_pointer)
501 write_pointer += 6;
502
503 spin_lock(&ring->execlist_lock);
504
505 while (read_pointer < write_pointer) {
506 read_pointer++;
507 status = I915_READ(RING_CONTEXT_STATUS_BUF(ring) +
508 (read_pointer % 6) * 8);
509 status_id = I915_READ(RING_CONTEXT_STATUS_BUF(ring) +
510 (read_pointer % 6) * 8 + 4);
511
031a8936
MK
512 if (status & GEN8_CTX_STATUS_IDLE_ACTIVE)
513 continue;
514
e1fee72c
OM
515 if (status & GEN8_CTX_STATUS_PREEMPTED) {
516 if (status & GEN8_CTX_STATUS_LITE_RESTORE) {
517 if (execlists_check_remove_request(ring, status_id))
518 WARN(1, "Lite Restored request removed from queue\n");
519 } else
520 WARN(1, "Preemption without Lite Restore\n");
521 }
522
523 if ((status & GEN8_CTX_STATUS_ACTIVE_IDLE) ||
524 (status & GEN8_CTX_STATUS_ELEMENT_SWITCH)) {
e981e7b1
TD
525 if (execlists_check_remove_request(ring, status_id))
526 submit_contexts++;
527 }
528 }
529
530 if (submit_contexts != 0)
531 execlists_context_unqueue(ring);
532
533 spin_unlock(&ring->execlist_lock);
534
535 WARN(submit_contexts > 2, "More than two context complete events?\n");
536 ring->next_context_status_buffer = write_pointer % 6;
537
538 I915_WRITE(RING_CONTEXT_STATUS_PTR(ring),
cc53699b 539 _MASKED_FIELD(0x07 << 8, ((u32)ring->next_context_status_buffer & 0x07) << 8));
e981e7b1
TD
540}
541
ae70797d 542static int execlists_context_queue(struct drm_i915_gem_request *request)
acdd884a 543{
ae70797d 544 struct intel_engine_cs *ring = request->ring;
6d3d8274 545 struct drm_i915_gem_request *cursor;
f1ad5a1f 546 int num_elements = 0;
acdd884a 547
ae70797d 548 if (request->ctx != ring->default_context)
8ba319da 549 intel_lr_context_pin(request);
9bb1af44
JH
550
551 i915_gem_request_reference(request);
552
b5eba372 553 spin_lock_irq(&ring->execlist_lock);
acdd884a 554
f1ad5a1f
OM
555 list_for_each_entry(cursor, &ring->execlist_queue, execlist_link)
556 if (++num_elements > 2)
557 break;
558
559 if (num_elements > 2) {
6d3d8274 560 struct drm_i915_gem_request *tail_req;
f1ad5a1f
OM
561
562 tail_req = list_last_entry(&ring->execlist_queue,
6d3d8274 563 struct drm_i915_gem_request,
f1ad5a1f
OM
564 execlist_link);
565
ae70797d 566 if (request->ctx == tail_req->ctx) {
f1ad5a1f 567 WARN(tail_req->elsp_submitted != 0,
7ba717cf 568 "More than 2 already-submitted reqs queued\n");
f1ad5a1f 569 list_del(&tail_req->execlist_link);
c86ee3a9
TD
570 list_add_tail(&tail_req->execlist_link,
571 &ring->execlist_retired_req_list);
f1ad5a1f
OM
572 }
573 }
574
6d3d8274 575 list_add_tail(&request->execlist_link, &ring->execlist_queue);
f1ad5a1f 576 if (num_elements == 0)
acdd884a
MT
577 execlists_context_unqueue(ring);
578
b5eba372 579 spin_unlock_irq(&ring->execlist_lock);
acdd884a
MT
580
581 return 0;
582}
583
2f20055d 584static int logical_ring_invalidate_all_caches(struct drm_i915_gem_request *req)
ba8b7ccb 585{
2f20055d 586 struct intel_engine_cs *ring = req->ring;
ba8b7ccb
OM
587 uint32_t flush_domains;
588 int ret;
589
590 flush_domains = 0;
591 if (ring->gpu_caches_dirty)
592 flush_domains = I915_GEM_GPU_DOMAINS;
593
7deb4d39 594 ret = ring->emit_flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
ba8b7ccb
OM
595 if (ret)
596 return ret;
597
598 ring->gpu_caches_dirty = false;
599 return 0;
600}
601
535fbe82 602static int execlists_move_to_gpu(struct drm_i915_gem_request *req,
ba8b7ccb
OM
603 struct list_head *vmas)
604{
535fbe82 605 const unsigned other_rings = ~intel_ring_flag(req->ring);
ba8b7ccb
OM
606 struct i915_vma *vma;
607 uint32_t flush_domains = 0;
608 bool flush_chipset = false;
609 int ret;
610
611 list_for_each_entry(vma, vmas, exec_list) {
612 struct drm_i915_gem_object *obj = vma->obj;
613
03ade511 614 if (obj->active & other_rings) {
91af127f 615 ret = i915_gem_object_sync(obj, req->ring, &req);
03ade511
CW
616 if (ret)
617 return ret;
618 }
ba8b7ccb
OM
619
620 if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
621 flush_chipset |= i915_gem_clflush_object(obj, false);
622
623 flush_domains |= obj->base.write_domain;
624 }
625
626 if (flush_domains & I915_GEM_DOMAIN_GTT)
627 wmb();
628
629 /* Unconditionally invalidate gpu caches and ensure that we do flush
630 * any residual writes from the previous batch.
631 */
2f20055d 632 return logical_ring_invalidate_all_caches(req);
ba8b7ccb
OM
633}
634
40e895ce 635int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request)
bc0dce3f 636{
bc0dce3f
JH
637 int ret;
638
f3cc01f0
MK
639 request->ringbuf = request->ctx->engine[request->ring->id].ringbuf;
640
40e895ce 641 if (request->ctx != request->ring->default_context) {
8ba319da 642 ret = intel_lr_context_pin(request);
6689cb2b 643 if (ret)
bc0dce3f 644 return ret;
bc0dce3f
JH
645 }
646
bc0dce3f
JH
647 return 0;
648}
649
ae70797d 650static int logical_ring_wait_for_space(struct drm_i915_gem_request *req,
595e1eeb 651 int bytes)
bc0dce3f 652{
ae70797d
JH
653 struct intel_ringbuffer *ringbuf = req->ringbuf;
654 struct intel_engine_cs *ring = req->ring;
655 struct drm_i915_gem_request *target;
b4716185
CW
656 unsigned space;
657 int ret;
bc0dce3f
JH
658
659 if (intel_ring_space(ringbuf) >= bytes)
660 return 0;
661
79bbcc29
JH
662 /* The whole point of reserving space is to not wait! */
663 WARN_ON(ringbuf->reserved_in_use);
664
ae70797d 665 list_for_each_entry(target, &ring->request_list, list) {
bc0dce3f
JH
666 /*
667 * The request queue is per-engine, so can contain requests
668 * from multiple ringbuffers. Here, we must ignore any that
669 * aren't from the ringbuffer we're considering.
670 */
ae70797d 671 if (target->ringbuf != ringbuf)
bc0dce3f
JH
672 continue;
673
674 /* Would completion of this request free enough space? */
ae70797d 675 space = __intel_ring_space(target->postfix, ringbuf->tail,
b4716185
CW
676 ringbuf->size);
677 if (space >= bytes)
bc0dce3f 678 break;
bc0dce3f
JH
679 }
680
ae70797d 681 if (WARN_ON(&target->list == &ring->request_list))
bc0dce3f
JH
682 return -ENOSPC;
683
ae70797d 684 ret = i915_wait_request(target);
bc0dce3f
JH
685 if (ret)
686 return ret;
687
b4716185
CW
688 ringbuf->space = space;
689 return 0;
bc0dce3f
JH
690}
691
692/*
693 * intel_logical_ring_advance_and_submit() - advance the tail and submit the workload
ae70797d 694 * @request: Request to advance the logical ringbuffer of.
bc0dce3f
JH
695 *
696 * The tail is updated in our logical ringbuffer struct, not in the actual context. What
697 * really happens during submission is that the context and current tail will be placed
698 * on a queue waiting for the ELSP to be ready to accept a new context submission. At that
699 * point, the tail *inside* the context is updated and the ELSP written to.
700 */
701static void
ae70797d 702intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request)
bc0dce3f 703{
ae70797d 704 struct intel_engine_cs *ring = request->ring;
d1675198 705 struct drm_i915_private *dev_priv = request->i915;
bc0dce3f 706
ae70797d 707 intel_logical_ring_advance(request->ringbuf);
bc0dce3f 708
d1675198
AD
709 request->tail = request->ringbuf->tail;
710
bc0dce3f
JH
711 if (intel_ring_stopped(ring))
712 return;
713
d1675198
AD
714 if (dev_priv->guc.execbuf_client)
715 i915_guc_submit(dev_priv->guc.execbuf_client, request);
716 else
717 execlists_context_queue(request);
bc0dce3f
JH
718}
719
79bbcc29 720static void __wrap_ring_buffer(struct intel_ringbuffer *ringbuf)
bc0dce3f
JH
721{
722 uint32_t __iomem *virt;
723 int rem = ringbuf->size - ringbuf->tail;
724
bc0dce3f
JH
725 virt = ringbuf->virtual_start + ringbuf->tail;
726 rem /= 4;
727 while (rem--)
728 iowrite32(MI_NOOP, virt++);
729
730 ringbuf->tail = 0;
731 intel_ring_update_space(ringbuf);
bc0dce3f
JH
732}
733
ae70797d 734static int logical_ring_prepare(struct drm_i915_gem_request *req, int bytes)
bc0dce3f 735{
ae70797d 736 struct intel_ringbuffer *ringbuf = req->ringbuf;
79bbcc29
JH
737 int remain_usable = ringbuf->effective_size - ringbuf->tail;
738 int remain_actual = ringbuf->size - ringbuf->tail;
739 int ret, total_bytes, wait_bytes = 0;
740 bool need_wrap = false;
29b1b415 741
79bbcc29
JH
742 if (ringbuf->reserved_in_use)
743 total_bytes = bytes;
744 else
745 total_bytes = bytes + ringbuf->reserved_size;
29b1b415 746
79bbcc29
JH
747 if (unlikely(bytes > remain_usable)) {
748 /*
749 * Not enough space for the basic request. So need to flush
750 * out the remainder and then wait for base + reserved.
751 */
752 wait_bytes = remain_actual + total_bytes;
753 need_wrap = true;
754 } else {
755 if (unlikely(total_bytes > remain_usable)) {
756 /*
757 * The base request will fit but the reserved space
758 * falls off the end. So only need to to wait for the
759 * reserved size after flushing out the remainder.
760 */
761 wait_bytes = remain_actual + ringbuf->reserved_size;
762 need_wrap = true;
763 } else if (total_bytes > ringbuf->space) {
764 /* No wrapping required, just waiting. */
765 wait_bytes = total_bytes;
29b1b415 766 }
bc0dce3f
JH
767 }
768
79bbcc29
JH
769 if (wait_bytes) {
770 ret = logical_ring_wait_for_space(req, wait_bytes);
bc0dce3f
JH
771 if (unlikely(ret))
772 return ret;
79bbcc29
JH
773
774 if (need_wrap)
775 __wrap_ring_buffer(ringbuf);
bc0dce3f
JH
776 }
777
778 return 0;
779}
780
781/**
782 * intel_logical_ring_begin() - prepare the logical ringbuffer to accept some commands
783 *
4d616a29 784 * @request: The request to start some new work for
4d78c8dc 785 * @ctx: Logical ring context whose ringbuffer is being prepared.
bc0dce3f
JH
786 * @num_dwords: number of DWORDs that we plan to write to the ringbuffer.
787 *
788 * The ringbuffer might not be ready to accept the commands right away (maybe it needs to
789 * be wrapped, or wait a bit for the tail to be updated). This function takes care of that
790 * and also preallocates a request (every workload submission is still mediated through
791 * requests, same as it did with legacy ringbuffer submission).
792 *
793 * Return: non-zero if the ringbuffer is not ready to be written to.
794 */
3bbaba0c 795int intel_logical_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
bc0dce3f 796{
4d616a29 797 struct drm_i915_private *dev_priv;
bc0dce3f
JH
798 int ret;
799
4d616a29
JH
800 WARN_ON(req == NULL);
801 dev_priv = req->ring->dev->dev_private;
802
bc0dce3f
JH
803 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
804 dev_priv->mm.interruptible);
805 if (ret)
806 return ret;
807
ae70797d 808 ret = logical_ring_prepare(req, num_dwords * sizeof(uint32_t));
bc0dce3f
JH
809 if (ret)
810 return ret;
811
4d616a29 812 req->ringbuf->space -= num_dwords * sizeof(uint32_t);
bc0dce3f
JH
813 return 0;
814}
815
ccd98fe4
JH
816int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request)
817{
818 /*
819 * The first call merely notes the reserve request and is common for
820 * all back ends. The subsequent localised _begin() call actually
821 * ensures that the reservation is available. Without the begin, if
822 * the request creator immediately submitted the request without
823 * adding any commands to it then there might not actually be
824 * sufficient room for the submission commands.
825 */
826 intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST);
827
828 return intel_logical_ring_begin(request, 0);
829}
830
73e4d07f
OM
831/**
832 * execlists_submission() - submit a batchbuffer for execution, Execlists style
833 * @dev: DRM device.
834 * @file: DRM file.
835 * @ring: Engine Command Streamer to submit to.
836 * @ctx: Context to employ for this submission.
837 * @args: execbuffer call arguments.
838 * @vmas: list of vmas.
839 * @batch_obj: the batchbuffer to submit.
840 * @exec_start: batchbuffer start virtual address pointer.
8e004efc 841 * @dispatch_flags: translated execbuffer call flags.
73e4d07f
OM
842 *
843 * This is the evil twin version of i915_gem_ringbuffer_submission. It abstracts
844 * away the submission details of the execbuffer ioctl call.
845 *
846 * Return: non-zero if the submission fails.
847 */
5f19e2bf 848int intel_execlists_submission(struct i915_execbuffer_params *params,
454afebd 849 struct drm_i915_gem_execbuffer2 *args,
5f19e2bf 850 struct list_head *vmas)
454afebd 851{
5f19e2bf
JH
852 struct drm_device *dev = params->dev;
853 struct intel_engine_cs *ring = params->ring;
ba8b7ccb 854 struct drm_i915_private *dev_priv = dev->dev_private;
5f19e2bf
JH
855 struct intel_ringbuffer *ringbuf = params->ctx->engine[ring->id].ringbuf;
856 u64 exec_start;
ba8b7ccb
OM
857 int instp_mode;
858 u32 instp_mask;
859 int ret;
860
861 instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
862 instp_mask = I915_EXEC_CONSTANTS_MASK;
863 switch (instp_mode) {
864 case I915_EXEC_CONSTANTS_REL_GENERAL:
865 case I915_EXEC_CONSTANTS_ABSOLUTE:
866 case I915_EXEC_CONSTANTS_REL_SURFACE:
867 if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) {
868 DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
869 return -EINVAL;
870 }
871
872 if (instp_mode != dev_priv->relative_constants_mode) {
873 if (instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
874 DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
875 return -EINVAL;
876 }
877
878 /* The HW changed the meaning on this bit on gen6 */
879 instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
880 }
881 break;
882 default:
883 DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
884 return -EINVAL;
885 }
886
887 if (args->num_cliprects != 0) {
888 DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
889 return -EINVAL;
890 } else {
891 if (args->DR4 == 0xffffffff) {
892 DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
893 args->DR4 = 0;
894 }
895
896 if (args->DR1 || args->DR4 || args->cliprects_ptr) {
897 DRM_DEBUG("0 cliprects but dirt in cliprects fields\n");
898 return -EINVAL;
899 }
900 }
901
902 if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
903 DRM_DEBUG("sol reset is gen7 only\n");
904 return -EINVAL;
905 }
906
535fbe82 907 ret = execlists_move_to_gpu(params->request, vmas);
ba8b7ccb
OM
908 if (ret)
909 return ret;
910
911 if (ring == &dev_priv->ring[RCS] &&
912 instp_mode != dev_priv->relative_constants_mode) {
4d616a29 913 ret = intel_logical_ring_begin(params->request, 4);
ba8b7ccb
OM
914 if (ret)
915 return ret;
916
917 intel_logical_ring_emit(ringbuf, MI_NOOP);
918 intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(1));
919 intel_logical_ring_emit(ringbuf, INSTPM);
920 intel_logical_ring_emit(ringbuf, instp_mask << 16 | instp_mode);
921 intel_logical_ring_advance(ringbuf);
922
923 dev_priv->relative_constants_mode = instp_mode;
924 }
925
5f19e2bf
JH
926 exec_start = params->batch_obj_vm_offset +
927 args->batch_start_offset;
928
be795fc1 929 ret = ring->emit_bb_start(params->request, exec_start, params->dispatch_flags);
ba8b7ccb
OM
930 if (ret)
931 return ret;
932
95c24161 933 trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
5e4be7bd 934
8a8edb59 935 i915_gem_execbuffer_move_to_active(vmas, params->request);
adeca76d 936 i915_gem_execbuffer_retire_commands(params);
ba8b7ccb 937
454afebd
OM
938 return 0;
939}
940
c86ee3a9
TD
941void intel_execlists_retire_requests(struct intel_engine_cs *ring)
942{
6d3d8274 943 struct drm_i915_gem_request *req, *tmp;
c86ee3a9
TD
944 struct list_head retired_list;
945
946 WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
947 if (list_empty(&ring->execlist_retired_req_list))
948 return;
949
950 INIT_LIST_HEAD(&retired_list);
b5eba372 951 spin_lock_irq(&ring->execlist_lock);
c86ee3a9 952 list_replace_init(&ring->execlist_retired_req_list, &retired_list);
b5eba372 953 spin_unlock_irq(&ring->execlist_lock);
c86ee3a9
TD
954
955 list_for_each_entry_safe(req, tmp, &retired_list, execlist_link) {
6d3d8274 956 struct intel_context *ctx = req->ctx;
7ba717cf
TD
957 struct drm_i915_gem_object *ctx_obj =
958 ctx->engine[ring->id].state;
959
960 if (ctx_obj && (ctx != ring->default_context))
8ba319da 961 intel_lr_context_unpin(req);
c86ee3a9 962 list_del(&req->execlist_link);
f8210795 963 i915_gem_request_unreference(req);
c86ee3a9
TD
964 }
965}
966
454afebd
OM
967void intel_logical_ring_stop(struct intel_engine_cs *ring)
968{
9832b9da
OM
969 struct drm_i915_private *dev_priv = ring->dev->dev_private;
970 int ret;
971
972 if (!intel_ring_initialized(ring))
973 return;
974
975 ret = intel_ring_idle(ring);
976 if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
977 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
978 ring->name, ret);
979
980 /* TODO: Is this correct with Execlists enabled? */
981 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
982 if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
983 DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
984 return;
985 }
986 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
454afebd
OM
987}
988
4866d729 989int logical_ring_flush_all_caches(struct drm_i915_gem_request *req)
48e29f55 990{
4866d729 991 struct intel_engine_cs *ring = req->ring;
48e29f55
OM
992 int ret;
993
994 if (!ring->gpu_caches_dirty)
995 return 0;
996
7deb4d39 997 ret = ring->emit_flush(req, 0, I915_GEM_GPU_DOMAINS);
48e29f55
OM
998 if (ret)
999 return ret;
1000
1001 ring->gpu_caches_dirty = false;
1002 return 0;
1003}
1004
8ba319da 1005static int intel_lr_context_pin(struct drm_i915_gem_request *rq)
dcb4c12a 1006{
d1675198 1007 struct drm_i915_private *dev_priv = rq->i915;
8ba319da
MK
1008 struct intel_engine_cs *ring = rq->ring;
1009 struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring->id].state;
1010 struct intel_ringbuffer *ringbuf = rq->ringbuf;
dcb4c12a
OM
1011 int ret = 0;
1012
1013 WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
8ba319da 1014 if (rq->ctx->engine[ring->id].pin_count++ == 0) {
d1675198
AD
1015 ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN,
1016 PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
dcb4c12a 1017 if (ret)
a7cbedec 1018 goto reset_pin_count;
7ba717cf
TD
1019
1020 ret = intel_pin_and_map_ringbuffer_obj(ring->dev, ringbuf);
1021 if (ret)
1022 goto unpin_ctx_obj;
d1675198
AD
1023
1024 /* Invalidate GuC TLB. */
1025 if (i915.enable_guc_submission)
1026 I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
dcb4c12a
OM
1027 }
1028
7ba717cf
TD
1029 return ret;
1030
1031unpin_ctx_obj:
1032 i915_gem_object_ggtt_unpin(ctx_obj);
a7cbedec 1033reset_pin_count:
8ba319da 1034 rq->ctx->engine[ring->id].pin_count = 0;
7ba717cf 1035
dcb4c12a
OM
1036 return ret;
1037}
1038
8ba319da 1039void intel_lr_context_unpin(struct drm_i915_gem_request *rq)
dcb4c12a 1040{
8ba319da
MK
1041 struct intel_engine_cs *ring = rq->ring;
1042 struct drm_i915_gem_object *ctx_obj = rq->ctx->engine[ring->id].state;
1043 struct intel_ringbuffer *ringbuf = rq->ringbuf;
dcb4c12a
OM
1044
1045 if (ctx_obj) {
1046 WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex));
8ba319da 1047 if (--rq->ctx->engine[ring->id].pin_count == 0) {
7ba717cf 1048 intel_unpin_ringbuffer_obj(ringbuf);
dcb4c12a 1049 i915_gem_object_ggtt_unpin(ctx_obj);
7ba717cf 1050 }
dcb4c12a
OM
1051 }
1052}
1053
e2be4faf 1054static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req)
771b9a53
MT
1055{
1056 int ret, i;
e2be4faf
JH
1057 struct intel_engine_cs *ring = req->ring;
1058 struct intel_ringbuffer *ringbuf = req->ringbuf;
771b9a53
MT
1059 struct drm_device *dev = ring->dev;
1060 struct drm_i915_private *dev_priv = dev->dev_private;
1061 struct i915_workarounds *w = &dev_priv->workarounds;
1062
e6c1abb7 1063 if (WARN_ON_ONCE(w->count == 0))
771b9a53
MT
1064 return 0;
1065
1066 ring->gpu_caches_dirty = true;
4866d729 1067 ret = logical_ring_flush_all_caches(req);
771b9a53
MT
1068 if (ret)
1069 return ret;
1070
4d616a29 1071 ret = intel_logical_ring_begin(req, w->count * 2 + 2);
771b9a53
MT
1072 if (ret)
1073 return ret;
1074
1075 intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(w->count));
1076 for (i = 0; i < w->count; i++) {
1077 intel_logical_ring_emit(ringbuf, w->reg[i].addr);
1078 intel_logical_ring_emit(ringbuf, w->reg[i].value);
1079 }
1080 intel_logical_ring_emit(ringbuf, MI_NOOP);
1081
1082 intel_logical_ring_advance(ringbuf);
1083
1084 ring->gpu_caches_dirty = true;
4866d729 1085 ret = logical_ring_flush_all_caches(req);
771b9a53
MT
1086 if (ret)
1087 return ret;
1088
1089 return 0;
1090}
1091
83b8a982 1092#define wa_ctx_emit(batch, index, cmd) \
17ee950d 1093 do { \
83b8a982
AS
1094 int __index = (index)++; \
1095 if (WARN_ON(__index >= (PAGE_SIZE / sizeof(uint32_t)))) { \
17ee950d
AS
1096 return -ENOSPC; \
1097 } \
83b8a982 1098 batch[__index] = (cmd); \
17ee950d
AS
1099 } while (0)
1100
9e000847
AS
1101
1102/*
1103 * In this WA we need to set GEN8_L3SQCREG4[21:21] and reset it after
1104 * PIPE_CONTROL instruction. This is required for the flush to happen correctly
1105 * but there is a slight complication as this is applied in WA batch where the
1106 * values are only initialized once so we cannot take register value at the
1107 * beginning and reuse it further; hence we save its value to memory, upload a
1108 * constant value with bit21 set and then we restore it back with the saved value.
1109 * To simplify the WA, a constant value is formed by using the default value
1110 * of this register. This shouldn't be a problem because we are only modifying
1111 * it for a short period and this batch in non-premptible. We can ofcourse
1112 * use additional instructions that read the actual value of the register
1113 * at that time and set our bit of interest but it makes the WA complicated.
1114 *
1115 * This WA is also required for Gen9 so extracting as a function avoids
1116 * code duplication.
1117 */
1118static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *ring,
1119 uint32_t *const batch,
1120 uint32_t index)
1121{
1122 uint32_t l3sqc4_flush = (0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES);
1123
a4106a78
AS
1124 /*
1125 * WaDisableLSQCROPERFforOCL:skl
1126 * This WA is implemented in skl_init_clock_gating() but since
1127 * this batch updates GEN8_L3SQCREG4 with default value we need to
1128 * set this bit here to retain the WA during flush.
1129 */
1130 if (IS_SKYLAKE(ring->dev) && INTEL_REVID(ring->dev) <= SKL_REVID_E0)
1131 l3sqc4_flush |= GEN8_LQSC_RO_PERF_DIS;
1132
f1afe24f 1133 wa_ctx_emit(batch, index, (MI_STORE_REGISTER_MEM_GEN8 |
83b8a982
AS
1134 MI_SRM_LRM_GLOBAL_GTT));
1135 wa_ctx_emit(batch, index, GEN8_L3SQCREG4);
1136 wa_ctx_emit(batch, index, ring->scratch.gtt_offset + 256);
1137 wa_ctx_emit(batch, index, 0);
1138
1139 wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
1140 wa_ctx_emit(batch, index, GEN8_L3SQCREG4);
1141 wa_ctx_emit(batch, index, l3sqc4_flush);
1142
1143 wa_ctx_emit(batch, index, GFX_OP_PIPE_CONTROL(6));
1144 wa_ctx_emit(batch, index, (PIPE_CONTROL_CS_STALL |
1145 PIPE_CONTROL_DC_FLUSH_ENABLE));
1146 wa_ctx_emit(batch, index, 0);
1147 wa_ctx_emit(batch, index, 0);
1148 wa_ctx_emit(batch, index, 0);
1149 wa_ctx_emit(batch, index, 0);
1150
f1afe24f 1151 wa_ctx_emit(batch, index, (MI_LOAD_REGISTER_MEM_GEN8 |
83b8a982
AS
1152 MI_SRM_LRM_GLOBAL_GTT));
1153 wa_ctx_emit(batch, index, GEN8_L3SQCREG4);
1154 wa_ctx_emit(batch, index, ring->scratch.gtt_offset + 256);
1155 wa_ctx_emit(batch, index, 0);
9e000847
AS
1156
1157 return index;
1158}
1159
17ee950d
AS
1160static inline uint32_t wa_ctx_start(struct i915_wa_ctx_bb *wa_ctx,
1161 uint32_t offset,
1162 uint32_t start_alignment)
1163{
1164 return wa_ctx->offset = ALIGN(offset, start_alignment);
1165}
1166
1167static inline int wa_ctx_end(struct i915_wa_ctx_bb *wa_ctx,
1168 uint32_t offset,
1169 uint32_t size_alignment)
1170{
1171 wa_ctx->size = offset - wa_ctx->offset;
1172
1173 WARN(wa_ctx->size % size_alignment,
1174 "wa_ctx_bb failed sanity checks: size %d is not aligned to %d\n",
1175 wa_ctx->size, size_alignment);
1176 return 0;
1177}
1178
1179/**
1180 * gen8_init_indirectctx_bb() - initialize indirect ctx batch with WA
1181 *
1182 * @ring: only applicable for RCS
1183 * @wa_ctx: structure representing wa_ctx
1184 * offset: specifies start of the batch, should be cache-aligned. This is updated
1185 * with the offset value received as input.
1186 * size: size of the batch in DWORDS but HW expects in terms of cachelines
1187 * @batch: page in which WA are loaded
1188 * @offset: This field specifies the start of the batch, it should be
1189 * cache-aligned otherwise it is adjusted accordingly.
1190 * Typically we only have one indirect_ctx and per_ctx batch buffer which are
1191 * initialized at the beginning and shared across all contexts but this field
1192 * helps us to have multiple batches at different offsets and select them based
1193 * on a criteria. At the moment this batch always start at the beginning of the page
1194 * and at this point we don't have multiple wa_ctx batch buffers.
1195 *
1196 * The number of WA applied are not known at the beginning; we use this field
1197 * to return the no of DWORDS written.
4d78c8dc 1198 *
17ee950d
AS
1199 * It is to be noted that this batch does not contain MI_BATCH_BUFFER_END
1200 * so it adds NOOPs as padding to make it cacheline aligned.
1201 * MI_BATCH_BUFFER_END will be added to perctx batch and both of them together
1202 * makes a complete batch buffer.
1203 *
1204 * Return: non-zero if we exceed the PAGE_SIZE limit.
1205 */
1206
1207static int gen8_init_indirectctx_bb(struct intel_engine_cs *ring,
1208 struct i915_wa_ctx_bb *wa_ctx,
1209 uint32_t *const batch,
1210 uint32_t *offset)
1211{
0160f055 1212 uint32_t scratch_addr;
17ee950d
AS
1213 uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
1214
7ad00d1a 1215 /* WaDisableCtxRestoreArbitration:bdw,chv */
83b8a982 1216 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE);
17ee950d 1217
c82435bb
AS
1218 /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */
1219 if (IS_BROADWELL(ring->dev)) {
9e000847
AS
1220 index = gen8_emit_flush_coherentl3_wa(ring, batch, index);
1221 if (index < 0)
1222 return index;
c82435bb
AS
1223 }
1224
0160f055
AS
1225 /* WaClearSlmSpaceAtContextSwitch:bdw,chv */
1226 /* Actual scratch location is at 128 bytes offset */
1227 scratch_addr = ring->scratch.gtt_offset + 2*CACHELINE_BYTES;
1228
83b8a982
AS
1229 wa_ctx_emit(batch, index, GFX_OP_PIPE_CONTROL(6));
1230 wa_ctx_emit(batch, index, (PIPE_CONTROL_FLUSH_L3 |
1231 PIPE_CONTROL_GLOBAL_GTT_IVB |
1232 PIPE_CONTROL_CS_STALL |
1233 PIPE_CONTROL_QW_WRITE));
1234 wa_ctx_emit(batch, index, scratch_addr);
1235 wa_ctx_emit(batch, index, 0);
1236 wa_ctx_emit(batch, index, 0);
1237 wa_ctx_emit(batch, index, 0);
0160f055 1238
17ee950d
AS
1239 /* Pad to end of cacheline */
1240 while (index % CACHELINE_DWORDS)
83b8a982 1241 wa_ctx_emit(batch, index, MI_NOOP);
17ee950d
AS
1242
1243 /*
1244 * MI_BATCH_BUFFER_END is not required in Indirect ctx BB because
1245 * execution depends on the length specified in terms of cache lines
1246 * in the register CTX_RCS_INDIRECT_CTX
1247 */
1248
1249 return wa_ctx_end(wa_ctx, *offset = index, CACHELINE_DWORDS);
1250}
1251
1252/**
1253 * gen8_init_perctx_bb() - initialize per ctx batch with WA
1254 *
1255 * @ring: only applicable for RCS
1256 * @wa_ctx: structure representing wa_ctx
1257 * offset: specifies start of the batch, should be cache-aligned.
1258 * size: size of the batch in DWORDS but HW expects in terms of cachelines
4d78c8dc 1259 * @batch: page in which WA are loaded
17ee950d
AS
1260 * @offset: This field specifies the start of this batch.
1261 * This batch is started immediately after indirect_ctx batch. Since we ensure
1262 * that indirect_ctx ends on a cacheline this batch is aligned automatically.
1263 *
1264 * The number of DWORDS written are returned using this field.
1265 *
1266 * This batch is terminated with MI_BATCH_BUFFER_END and so we need not add padding
1267 * to align it with cacheline as padding after MI_BATCH_BUFFER_END is redundant.
1268 */
1269static int gen8_init_perctx_bb(struct intel_engine_cs *ring,
1270 struct i915_wa_ctx_bb *wa_ctx,
1271 uint32_t *const batch,
1272 uint32_t *offset)
1273{
1274 uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
1275
7ad00d1a 1276 /* WaDisableCtxRestoreArbitration:bdw,chv */
83b8a982 1277 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE);
7ad00d1a 1278
83b8a982 1279 wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END);
17ee950d
AS
1280
1281 return wa_ctx_end(wa_ctx, *offset = index, 1);
1282}
1283
0504cffc
AS
1284static int gen9_init_indirectctx_bb(struct intel_engine_cs *ring,
1285 struct i915_wa_ctx_bb *wa_ctx,
1286 uint32_t *const batch,
1287 uint32_t *offset)
1288{
a4106a78 1289 int ret;
0907c8f7 1290 struct drm_device *dev = ring->dev;
0504cffc
AS
1291 uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
1292
0907c8f7
AS
1293 /* WaDisableCtxRestoreArbitration:skl,bxt */
1294 if ((IS_SKYLAKE(dev) && (INTEL_REVID(dev) <= SKL_REVID_D0)) ||
1295 (IS_BROXTON(dev) && (INTEL_REVID(dev) == BXT_REVID_A0)))
1296 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE);
0504cffc 1297
a4106a78
AS
1298 /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt */
1299 ret = gen8_emit_flush_coherentl3_wa(ring, batch, index);
1300 if (ret < 0)
1301 return ret;
1302 index = ret;
1303
0504cffc
AS
1304 /* Pad to end of cacheline */
1305 while (index % CACHELINE_DWORDS)
1306 wa_ctx_emit(batch, index, MI_NOOP);
1307
1308 return wa_ctx_end(wa_ctx, *offset = index, CACHELINE_DWORDS);
1309}
1310
1311static int gen9_init_perctx_bb(struct intel_engine_cs *ring,
1312 struct i915_wa_ctx_bb *wa_ctx,
1313 uint32_t *const batch,
1314 uint32_t *offset)
1315{
0907c8f7 1316 struct drm_device *dev = ring->dev;
0504cffc
AS
1317 uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
1318
9b01435d
AS
1319 /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
1320 if ((IS_SKYLAKE(dev) && (INTEL_REVID(dev) <= SKL_REVID_B0)) ||
1321 (IS_BROXTON(dev) && (INTEL_REVID(dev) == BXT_REVID_A0))) {
1322 wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
1323 wa_ctx_emit(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0);
1324 wa_ctx_emit(batch, index,
1325 _MASKED_BIT_ENABLE(DISABLE_PIXEL_MASK_CAMMING));
1326 wa_ctx_emit(batch, index, MI_NOOP);
1327 }
1328
0907c8f7
AS
1329 /* WaDisableCtxRestoreArbitration:skl,bxt */
1330 if ((IS_SKYLAKE(dev) && (INTEL_REVID(dev) <= SKL_REVID_D0)) ||
1331 (IS_BROXTON(dev) && (INTEL_REVID(dev) == BXT_REVID_A0)))
1332 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE);
1333
0504cffc
AS
1334 wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END);
1335
1336 return wa_ctx_end(wa_ctx, *offset = index, 1);
1337}
1338
17ee950d
AS
1339static int lrc_setup_wa_ctx_obj(struct intel_engine_cs *ring, u32 size)
1340{
1341 int ret;
1342
1343 ring->wa_ctx.obj = i915_gem_alloc_object(ring->dev, PAGE_ALIGN(size));
1344 if (!ring->wa_ctx.obj) {
1345 DRM_DEBUG_DRIVER("alloc LRC WA ctx backing obj failed.\n");
1346 return -ENOMEM;
1347 }
1348
1349 ret = i915_gem_obj_ggtt_pin(ring->wa_ctx.obj, PAGE_SIZE, 0);
1350 if (ret) {
1351 DRM_DEBUG_DRIVER("pin LRC WA ctx backing obj failed: %d\n",
1352 ret);
1353 drm_gem_object_unreference(&ring->wa_ctx.obj->base);
1354 return ret;
1355 }
1356
1357 return 0;
1358}
1359
1360static void lrc_destroy_wa_ctx_obj(struct intel_engine_cs *ring)
1361{
1362 if (ring->wa_ctx.obj) {
1363 i915_gem_object_ggtt_unpin(ring->wa_ctx.obj);
1364 drm_gem_object_unreference(&ring->wa_ctx.obj->base);
1365 ring->wa_ctx.obj = NULL;
1366 }
1367}
1368
1369static int intel_init_workaround_bb(struct intel_engine_cs *ring)
1370{
1371 int ret;
1372 uint32_t *batch;
1373 uint32_t offset;
1374 struct page *page;
1375 struct i915_ctx_workarounds *wa_ctx = &ring->wa_ctx;
1376
1377 WARN_ON(ring->id != RCS);
1378
5e60d790 1379 /* update this when WA for higher Gen are added */
0504cffc
AS
1380 if (INTEL_INFO(ring->dev)->gen > 9) {
1381 DRM_ERROR("WA batch buffer is not initialized for Gen%d\n",
1382 INTEL_INFO(ring->dev)->gen);
5e60d790 1383 return 0;
0504cffc 1384 }
5e60d790 1385
c4db7599
AS
1386 /* some WA perform writes to scratch page, ensure it is valid */
1387 if (ring->scratch.obj == NULL) {
1388 DRM_ERROR("scratch page not allocated for %s\n", ring->name);
1389 return -EINVAL;
1390 }
1391
17ee950d
AS
1392 ret = lrc_setup_wa_ctx_obj(ring, PAGE_SIZE);
1393 if (ret) {
1394 DRM_DEBUG_DRIVER("Failed to setup context WA page: %d\n", ret);
1395 return ret;
1396 }
1397
1398 page = i915_gem_object_get_page(wa_ctx->obj, 0);
1399 batch = kmap_atomic(page);
1400 offset = 0;
1401
1402 if (INTEL_INFO(ring->dev)->gen == 8) {
1403 ret = gen8_init_indirectctx_bb(ring,
1404 &wa_ctx->indirect_ctx,
1405 batch,
1406 &offset);
1407 if (ret)
1408 goto out;
1409
1410 ret = gen8_init_perctx_bb(ring,
1411 &wa_ctx->per_ctx,
1412 batch,
1413 &offset);
1414 if (ret)
1415 goto out;
0504cffc
AS
1416 } else if (INTEL_INFO(ring->dev)->gen == 9) {
1417 ret = gen9_init_indirectctx_bb(ring,
1418 &wa_ctx->indirect_ctx,
1419 batch,
1420 &offset);
1421 if (ret)
1422 goto out;
1423
1424 ret = gen9_init_perctx_bb(ring,
1425 &wa_ctx->per_ctx,
1426 batch,
1427 &offset);
1428 if (ret)
1429 goto out;
17ee950d
AS
1430 }
1431
1432out:
1433 kunmap_atomic(batch);
1434 if (ret)
1435 lrc_destroy_wa_ctx_obj(ring);
1436
1437 return ret;
1438}
1439
9b1136d5
OM
1440static int gen8_init_common_ring(struct intel_engine_cs *ring)
1441{
1442 struct drm_device *dev = ring->dev;
1443 struct drm_i915_private *dev_priv = dev->dev_private;
1444
73d477f6
OM
1445 I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
1446 I915_WRITE(RING_HWSTAM(ring->mmio_base), 0xffffffff);
1447
2e5356da
AS
1448 if (ring->status_page.obj) {
1449 I915_WRITE(RING_HWS_PGA(ring->mmio_base),
1450 (u32)ring->status_page.gfx_addr);
1451 POSTING_READ(RING_HWS_PGA(ring->mmio_base));
1452 }
1453
9b1136d5
OM
1454 I915_WRITE(RING_MODE_GEN7(ring),
1455 _MASKED_BIT_DISABLE(GFX_REPLAY_MODE) |
1456 _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
1457 POSTING_READ(RING_MODE_GEN7(ring));
c0a03a2e 1458 ring->next_context_status_buffer = 0;
9b1136d5
OM
1459 DRM_DEBUG_DRIVER("Execlists enabled for %s\n", ring->name);
1460
1461 memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
1462
1463 return 0;
1464}
1465
1466static int gen8_init_render_ring(struct intel_engine_cs *ring)
1467{
1468 struct drm_device *dev = ring->dev;
1469 struct drm_i915_private *dev_priv = dev->dev_private;
1470 int ret;
1471
1472 ret = gen8_init_common_ring(ring);
1473 if (ret)
1474 return ret;
1475
1476 /* We need to disable the AsyncFlip performance optimisations in order
1477 * to use MI_WAIT_FOR_EVENT within the CS. It should already be
1478 * programmed to '1' on all products.
1479 *
1480 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
1481 */
1482 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
1483
9b1136d5
OM
1484 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1485
771b9a53 1486 return init_workarounds_ring(ring);
9b1136d5
OM
1487}
1488
82ef822e
DL
1489static int gen9_init_render_ring(struct intel_engine_cs *ring)
1490{
1491 int ret;
1492
1493 ret = gen8_init_common_ring(ring);
1494 if (ret)
1495 return ret;
1496
1497 return init_workarounds_ring(ring);
1498}
1499
7a01a0a2
MT
1500static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req)
1501{
1502 struct i915_hw_ppgtt *ppgtt = req->ctx->ppgtt;
1503 struct intel_engine_cs *ring = req->ring;
1504 struct intel_ringbuffer *ringbuf = req->ringbuf;
1505 const int num_lri_cmds = GEN8_LEGACY_PDPES * 2;
1506 int i, ret;
1507
1508 ret = intel_logical_ring_begin(req, num_lri_cmds * 2 + 2);
1509 if (ret)
1510 return ret;
1511
1512 intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(num_lri_cmds));
1513 for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
1514 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
1515
1516 intel_logical_ring_emit(ringbuf, GEN8_RING_PDP_UDW(ring, i));
1517 intel_logical_ring_emit(ringbuf, upper_32_bits(pd_daddr));
1518 intel_logical_ring_emit(ringbuf, GEN8_RING_PDP_LDW(ring, i));
1519 intel_logical_ring_emit(ringbuf, lower_32_bits(pd_daddr));
1520 }
1521
1522 intel_logical_ring_emit(ringbuf, MI_NOOP);
1523 intel_logical_ring_advance(ringbuf);
1524
1525 return 0;
1526}
1527
be795fc1 1528static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
8e004efc 1529 u64 offset, unsigned dispatch_flags)
15648585 1530{
be795fc1 1531 struct intel_ringbuffer *ringbuf = req->ringbuf;
8e004efc 1532 bool ppgtt = !(dispatch_flags & I915_DISPATCH_SECURE);
15648585
OM
1533 int ret;
1534
7a01a0a2
MT
1535 /* Don't rely in hw updating PDPs, specially in lite-restore.
1536 * Ideally, we should set Force PD Restore in ctx descriptor,
1537 * but we can't. Force Restore would be a second option, but
1538 * it is unsafe in case of lite-restore (because the ctx is
2dba3239
MT
1539 * not idle). PML4 is allocated during ppgtt init so this is
1540 * not needed in 48-bit.*/
7a01a0a2
MT
1541 if (req->ctx->ppgtt &&
1542 (intel_ring_flag(req->ring) & req->ctx->ppgtt->pd_dirty_rings)) {
2dba3239
MT
1543 if (!USES_FULL_48BIT_PPGTT(req->i915)) {
1544 ret = intel_logical_ring_emit_pdps(req);
1545 if (ret)
1546 return ret;
1547 }
7a01a0a2
MT
1548
1549 req->ctx->ppgtt->pd_dirty_rings &= ~intel_ring_flag(req->ring);
1550 }
1551
4d616a29 1552 ret = intel_logical_ring_begin(req, 4);
15648585
OM
1553 if (ret)
1554 return ret;
1555
1556 /* FIXME(BDW): Address space and security selectors. */
6922528a
AJ
1557 intel_logical_ring_emit(ringbuf, MI_BATCH_BUFFER_START_GEN8 |
1558 (ppgtt<<8) |
1559 (dispatch_flags & I915_DISPATCH_RS ?
1560 MI_BATCH_RESOURCE_STREAMER : 0));
15648585
OM
1561 intel_logical_ring_emit(ringbuf, lower_32_bits(offset));
1562 intel_logical_ring_emit(ringbuf, upper_32_bits(offset));
1563 intel_logical_ring_emit(ringbuf, MI_NOOP);
1564 intel_logical_ring_advance(ringbuf);
1565
1566 return 0;
1567}
1568
73d477f6
OM
1569static bool gen8_logical_ring_get_irq(struct intel_engine_cs *ring)
1570{
1571 struct drm_device *dev = ring->dev;
1572 struct drm_i915_private *dev_priv = dev->dev_private;
1573 unsigned long flags;
1574
7cd512f1 1575 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
73d477f6
OM
1576 return false;
1577
1578 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1579 if (ring->irq_refcount++ == 0) {
1580 I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
1581 POSTING_READ(RING_IMR(ring->mmio_base));
1582 }
1583 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1584
1585 return true;
1586}
1587
1588static void gen8_logical_ring_put_irq(struct intel_engine_cs *ring)
1589{
1590 struct drm_device *dev = ring->dev;
1591 struct drm_i915_private *dev_priv = dev->dev_private;
1592 unsigned long flags;
1593
1594 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1595 if (--ring->irq_refcount == 0) {
1596 I915_WRITE_IMR(ring, ~ring->irq_keep_mask);
1597 POSTING_READ(RING_IMR(ring->mmio_base));
1598 }
1599 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1600}
1601
7deb4d39 1602static int gen8_emit_flush(struct drm_i915_gem_request *request,
4712274c
OM
1603 u32 invalidate_domains,
1604 u32 unused)
1605{
7deb4d39 1606 struct intel_ringbuffer *ringbuf = request->ringbuf;
4712274c
OM
1607 struct intel_engine_cs *ring = ringbuf->ring;
1608 struct drm_device *dev = ring->dev;
1609 struct drm_i915_private *dev_priv = dev->dev_private;
1610 uint32_t cmd;
1611 int ret;
1612
4d616a29 1613 ret = intel_logical_ring_begin(request, 4);
4712274c
OM
1614 if (ret)
1615 return ret;
1616
1617 cmd = MI_FLUSH_DW + 1;
1618
f0a1fb10
CW
1619 /* We always require a command barrier so that subsequent
1620 * commands, such as breadcrumb interrupts, are strictly ordered
1621 * wrt the contents of the write cache being flushed to memory
1622 * (and thus being coherent from the CPU).
1623 */
1624 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
1625
1626 if (invalidate_domains & I915_GEM_GPU_DOMAINS) {
1627 cmd |= MI_INVALIDATE_TLB;
1628 if (ring == &dev_priv->ring[VCS])
1629 cmd |= MI_INVALIDATE_BSD;
4712274c
OM
1630 }
1631
1632 intel_logical_ring_emit(ringbuf, cmd);
1633 intel_logical_ring_emit(ringbuf,
1634 I915_GEM_HWS_SCRATCH_ADDR |
1635 MI_FLUSH_DW_USE_GTT);
1636 intel_logical_ring_emit(ringbuf, 0); /* upper addr */
1637 intel_logical_ring_emit(ringbuf, 0); /* value */
1638 intel_logical_ring_advance(ringbuf);
1639
1640 return 0;
1641}
1642
7deb4d39 1643static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
4712274c
OM
1644 u32 invalidate_domains,
1645 u32 flush_domains)
1646{
7deb4d39 1647 struct intel_ringbuffer *ringbuf = request->ringbuf;
4712274c
OM
1648 struct intel_engine_cs *ring = ringbuf->ring;
1649 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
9647ff36 1650 bool vf_flush_wa;
4712274c
OM
1651 u32 flags = 0;
1652 int ret;
1653
1654 flags |= PIPE_CONTROL_CS_STALL;
1655
1656 if (flush_domains) {
1657 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
1658 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
1659 }
1660
1661 if (invalidate_domains) {
1662 flags |= PIPE_CONTROL_TLB_INVALIDATE;
1663 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
1664 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
1665 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
1666 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
1667 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
1668 flags |= PIPE_CONTROL_QW_WRITE;
1669 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
1670 }
1671
9647ff36
ID
1672 /*
1673 * On GEN9+ Before VF_CACHE_INVALIDATE we need to emit a NULL pipe
1674 * control.
1675 */
1676 vf_flush_wa = INTEL_INFO(ring->dev)->gen >= 9 &&
1677 flags & PIPE_CONTROL_VF_CACHE_INVALIDATE;
1678
4d616a29 1679 ret = intel_logical_ring_begin(request, vf_flush_wa ? 12 : 6);
4712274c
OM
1680 if (ret)
1681 return ret;
1682
9647ff36
ID
1683 if (vf_flush_wa) {
1684 intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
1685 intel_logical_ring_emit(ringbuf, 0);
1686 intel_logical_ring_emit(ringbuf, 0);
1687 intel_logical_ring_emit(ringbuf, 0);
1688 intel_logical_ring_emit(ringbuf, 0);
1689 intel_logical_ring_emit(ringbuf, 0);
1690 }
1691
4712274c
OM
1692 intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
1693 intel_logical_ring_emit(ringbuf, flags);
1694 intel_logical_ring_emit(ringbuf, scratch_addr);
1695 intel_logical_ring_emit(ringbuf, 0);
1696 intel_logical_ring_emit(ringbuf, 0);
1697 intel_logical_ring_emit(ringbuf, 0);
1698 intel_logical_ring_advance(ringbuf);
1699
1700 return 0;
1701}
1702
e94e37ad
OM
1703static u32 gen8_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
1704{
1705 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
1706}
1707
1708static void gen8_set_seqno(struct intel_engine_cs *ring, u32 seqno)
1709{
1710 intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
1711}
1712
319404df
ID
1713static u32 bxt_a_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
1714{
1715
1716 /*
1717 * On BXT A steppings there is a HW coherency issue whereby the
1718 * MI_STORE_DATA_IMM storing the completed request's seqno
1719 * occasionally doesn't invalidate the CPU cache. Work around this by
1720 * clflushing the corresponding cacheline whenever the caller wants
1721 * the coherency to be guaranteed. Note that this cacheline is known
1722 * to be clean at this point, since we only write it in
1723 * bxt_a_set_seqno(), where we also do a clflush after the write. So
1724 * this clflush in practice becomes an invalidate operation.
1725 */
1726
1727 if (!lazy_coherency)
1728 intel_flush_status_page(ring, I915_GEM_HWS_INDEX);
1729
1730 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
1731}
1732
1733static void bxt_a_set_seqno(struct intel_engine_cs *ring, u32 seqno)
1734{
1735 intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
1736
1737 /* See bxt_a_get_seqno() explaining the reason for the clflush. */
1738 intel_flush_status_page(ring, I915_GEM_HWS_INDEX);
1739}
1740
c4e76638 1741static int gen8_emit_request(struct drm_i915_gem_request *request)
4da46e1e 1742{
c4e76638 1743 struct intel_ringbuffer *ringbuf = request->ringbuf;
4da46e1e
OM
1744 struct intel_engine_cs *ring = ringbuf->ring;
1745 u32 cmd;
1746 int ret;
1747
53292cdb
MT
1748 /*
1749 * Reserve space for 2 NOOPs at the end of each request to be
1750 * used as a workaround for not being allowed to do lite
1751 * restore with HEAD==TAIL (WaIdleLiteRestore).
1752 */
4d616a29 1753 ret = intel_logical_ring_begin(request, 8);
4da46e1e
OM
1754 if (ret)
1755 return ret;
1756
8edfbb8b 1757 cmd = MI_STORE_DWORD_IMM_GEN4;
4da46e1e
OM
1758 cmd |= MI_GLOBAL_GTT;
1759
1760 intel_logical_ring_emit(ringbuf, cmd);
1761 intel_logical_ring_emit(ringbuf,
1762 (ring->status_page.gfx_addr +
1763 (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)));
1764 intel_logical_ring_emit(ringbuf, 0);
c4e76638 1765 intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request));
4da46e1e
OM
1766 intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
1767 intel_logical_ring_emit(ringbuf, MI_NOOP);
ae70797d 1768 intel_logical_ring_advance_and_submit(request);
4da46e1e 1769
53292cdb
MT
1770 /*
1771 * Here we add two extra NOOPs as padding to avoid
1772 * lite restore of a context with HEAD==TAIL.
1773 */
1774 intel_logical_ring_emit(ringbuf, MI_NOOP);
1775 intel_logical_ring_emit(ringbuf, MI_NOOP);
1776 intel_logical_ring_advance(ringbuf);
1777
4da46e1e
OM
1778 return 0;
1779}
1780
be01363f 1781static int intel_lr_context_render_state_init(struct drm_i915_gem_request *req)
cef437ad 1782{
cef437ad 1783 struct render_state so;
cef437ad
DL
1784 int ret;
1785
be01363f 1786 ret = i915_gem_render_state_prepare(req->ring, &so);
cef437ad
DL
1787 if (ret)
1788 return ret;
1789
1790 if (so.rodata == NULL)
1791 return 0;
1792
be795fc1 1793 ret = req->ring->emit_bb_start(req, so.ggtt_offset,
be01363f 1794 I915_DISPATCH_SECURE);
cef437ad
DL
1795 if (ret)
1796 goto out;
1797
84e81020
AS
1798 ret = req->ring->emit_bb_start(req,
1799 (so.ggtt_offset + so.aux_batch_offset),
1800 I915_DISPATCH_SECURE);
1801 if (ret)
1802 goto out;
1803
b2af0376 1804 i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), req);
cef437ad 1805
cef437ad
DL
1806out:
1807 i915_gem_render_state_fini(&so);
1808 return ret;
1809}
1810
8753181e 1811static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
e7778be1
TD
1812{
1813 int ret;
1814
e2be4faf 1815 ret = intel_logical_ring_workarounds_emit(req);
e7778be1
TD
1816 if (ret)
1817 return ret;
1818
3bbaba0c
PA
1819 ret = intel_rcs_context_init_mocs(req);
1820 /*
1821 * Failing to program the MOCS is non-fatal.The system will not
1822 * run at peak performance. So generate an error and carry on.
1823 */
1824 if (ret)
1825 DRM_ERROR("MOCS failed to program: expect performance issues.\n");
1826
be01363f 1827 return intel_lr_context_render_state_init(req);
e7778be1
TD
1828}
1829
73e4d07f
OM
1830/**
1831 * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer
1832 *
1833 * @ring: Engine Command Streamer.
1834 *
1835 */
454afebd
OM
1836void intel_logical_ring_cleanup(struct intel_engine_cs *ring)
1837{
6402c330 1838 struct drm_i915_private *dev_priv;
9832b9da 1839
48d82387
OM
1840 if (!intel_ring_initialized(ring))
1841 return;
1842
6402c330
JH
1843 dev_priv = ring->dev->dev_private;
1844
9832b9da
OM
1845 intel_logical_ring_stop(ring);
1846 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
48d82387
OM
1847
1848 if (ring->cleanup)
1849 ring->cleanup(ring);
1850
1851 i915_cmd_parser_fini_ring(ring);
06fbca71 1852 i915_gem_batch_pool_fini(&ring->batch_pool);
48d82387
OM
1853
1854 if (ring->status_page.obj) {
1855 kunmap(sg_page(ring->status_page.obj->pages->sgl));
1856 ring->status_page.obj = NULL;
1857 }
17ee950d
AS
1858
1859 lrc_destroy_wa_ctx_obj(ring);
454afebd
OM
1860}
1861
1862static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring)
1863{
48d82387 1864 int ret;
48d82387
OM
1865
1866 /* Intentionally left blank. */
1867 ring->buffer = NULL;
1868
1869 ring->dev = dev;
1870 INIT_LIST_HEAD(&ring->active_list);
1871 INIT_LIST_HEAD(&ring->request_list);
06fbca71 1872 i915_gem_batch_pool_init(dev, &ring->batch_pool);
48d82387
OM
1873 init_waitqueue_head(&ring->irq_queue);
1874
acdd884a 1875 INIT_LIST_HEAD(&ring->execlist_queue);
c86ee3a9 1876 INIT_LIST_HEAD(&ring->execlist_retired_req_list);
acdd884a
MT
1877 spin_lock_init(&ring->execlist_lock);
1878
48d82387
OM
1879 ret = i915_cmd_parser_init_ring(ring);
1880 if (ret)
1881 return ret;
1882
564ddb2f
OM
1883 ret = intel_lr_context_deferred_create(ring->default_context, ring);
1884
1885 return ret;
454afebd
OM
1886}
1887
1888static int logical_render_ring_init(struct drm_device *dev)
1889{
1890 struct drm_i915_private *dev_priv = dev->dev_private;
1891 struct intel_engine_cs *ring = &dev_priv->ring[RCS];
99be1dfe 1892 int ret;
454afebd
OM
1893
1894 ring->name = "render ring";
1895 ring->id = RCS;
1896 ring->mmio_base = RENDER_RING_BASE;
1897 ring->irq_enable_mask =
1898 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
73d477f6
OM
1899 ring->irq_keep_mask =
1900 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
1901 if (HAS_L3_DPF(dev))
1902 ring->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
454afebd 1903
82ef822e
DL
1904 if (INTEL_INFO(dev)->gen >= 9)
1905 ring->init_hw = gen9_init_render_ring;
1906 else
1907 ring->init_hw = gen8_init_render_ring;
e7778be1 1908 ring->init_context = gen8_init_rcs_context;
9b1136d5 1909 ring->cleanup = intel_fini_pipe_control;
319404df
ID
1910 if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) {
1911 ring->get_seqno = bxt_a_get_seqno;
1912 ring->set_seqno = bxt_a_set_seqno;
1913 } else {
1914 ring->get_seqno = gen8_get_seqno;
1915 ring->set_seqno = gen8_set_seqno;
1916 }
4da46e1e 1917 ring->emit_request = gen8_emit_request;
4712274c 1918 ring->emit_flush = gen8_emit_flush_render;
73d477f6
OM
1919 ring->irq_get = gen8_logical_ring_get_irq;
1920 ring->irq_put = gen8_logical_ring_put_irq;
15648585 1921 ring->emit_bb_start = gen8_emit_bb_start;
9b1136d5 1922
99be1dfe 1923 ring->dev = dev;
c4db7599
AS
1924
1925 ret = intel_init_pipe_control(ring);
99be1dfe
DV
1926 if (ret)
1927 return ret;
1928
17ee950d
AS
1929 ret = intel_init_workaround_bb(ring);
1930 if (ret) {
1931 /*
1932 * We continue even if we fail to initialize WA batch
1933 * because we only expect rare glitches but nothing
1934 * critical to prevent us from using GPU
1935 */
1936 DRM_ERROR("WA batch buffer initialization failed: %d\n",
1937 ret);
1938 }
1939
c4db7599
AS
1940 ret = logical_ring_init(dev, ring);
1941 if (ret) {
17ee950d 1942 lrc_destroy_wa_ctx_obj(ring);
c4db7599 1943 }
17ee950d
AS
1944
1945 return ret;
454afebd
OM
1946}
1947
1948static int logical_bsd_ring_init(struct drm_device *dev)
1949{
1950 struct drm_i915_private *dev_priv = dev->dev_private;
1951 struct intel_engine_cs *ring = &dev_priv->ring[VCS];
1952
1953 ring->name = "bsd ring";
1954 ring->id = VCS;
1955 ring->mmio_base = GEN6_BSD_RING_BASE;
1956 ring->irq_enable_mask =
1957 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
73d477f6
OM
1958 ring->irq_keep_mask =
1959 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
454afebd 1960
ecfe00d8 1961 ring->init_hw = gen8_init_common_ring;
319404df
ID
1962 if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) {
1963 ring->get_seqno = bxt_a_get_seqno;
1964 ring->set_seqno = bxt_a_set_seqno;
1965 } else {
1966 ring->get_seqno = gen8_get_seqno;
1967 ring->set_seqno = gen8_set_seqno;
1968 }
4da46e1e 1969 ring->emit_request = gen8_emit_request;
4712274c 1970 ring->emit_flush = gen8_emit_flush;
73d477f6
OM
1971 ring->irq_get = gen8_logical_ring_get_irq;
1972 ring->irq_put = gen8_logical_ring_put_irq;
15648585 1973 ring->emit_bb_start = gen8_emit_bb_start;
9b1136d5 1974
454afebd
OM
1975 return logical_ring_init(dev, ring);
1976}
1977
1978static int logical_bsd2_ring_init(struct drm_device *dev)
1979{
1980 struct drm_i915_private *dev_priv = dev->dev_private;
1981 struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
1982
1983 ring->name = "bds2 ring";
1984 ring->id = VCS2;
1985 ring->mmio_base = GEN8_BSD2_RING_BASE;
1986 ring->irq_enable_mask =
1987 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
73d477f6
OM
1988 ring->irq_keep_mask =
1989 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
454afebd 1990
ecfe00d8 1991 ring->init_hw = gen8_init_common_ring;
e94e37ad
OM
1992 ring->get_seqno = gen8_get_seqno;
1993 ring->set_seqno = gen8_set_seqno;
4da46e1e 1994 ring->emit_request = gen8_emit_request;
4712274c 1995 ring->emit_flush = gen8_emit_flush;
73d477f6
OM
1996 ring->irq_get = gen8_logical_ring_get_irq;
1997 ring->irq_put = gen8_logical_ring_put_irq;
15648585 1998 ring->emit_bb_start = gen8_emit_bb_start;
9b1136d5 1999
454afebd
OM
2000 return logical_ring_init(dev, ring);
2001}
2002
2003static int logical_blt_ring_init(struct drm_device *dev)
2004{
2005 struct drm_i915_private *dev_priv = dev->dev_private;
2006 struct intel_engine_cs *ring = &dev_priv->ring[BCS];
2007
2008 ring->name = "blitter ring";
2009 ring->id = BCS;
2010 ring->mmio_base = BLT_RING_BASE;
2011 ring->irq_enable_mask =
2012 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
73d477f6
OM
2013 ring->irq_keep_mask =
2014 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
454afebd 2015
ecfe00d8 2016 ring->init_hw = gen8_init_common_ring;
319404df
ID
2017 if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) {
2018 ring->get_seqno = bxt_a_get_seqno;
2019 ring->set_seqno = bxt_a_set_seqno;
2020 } else {
2021 ring->get_seqno = gen8_get_seqno;
2022 ring->set_seqno = gen8_set_seqno;
2023 }
4da46e1e 2024 ring->emit_request = gen8_emit_request;
4712274c 2025 ring->emit_flush = gen8_emit_flush;
73d477f6
OM
2026 ring->irq_get = gen8_logical_ring_get_irq;
2027 ring->irq_put = gen8_logical_ring_put_irq;
15648585 2028 ring->emit_bb_start = gen8_emit_bb_start;
9b1136d5 2029
454afebd
OM
2030 return logical_ring_init(dev, ring);
2031}
2032
2033static int logical_vebox_ring_init(struct drm_device *dev)
2034{
2035 struct drm_i915_private *dev_priv = dev->dev_private;
2036 struct intel_engine_cs *ring = &dev_priv->ring[VECS];
2037
2038 ring->name = "video enhancement ring";
2039 ring->id = VECS;
2040 ring->mmio_base = VEBOX_RING_BASE;
2041 ring->irq_enable_mask =
2042 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
73d477f6
OM
2043 ring->irq_keep_mask =
2044 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
454afebd 2045
ecfe00d8 2046 ring->init_hw = gen8_init_common_ring;
319404df
ID
2047 if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) {
2048 ring->get_seqno = bxt_a_get_seqno;
2049 ring->set_seqno = bxt_a_set_seqno;
2050 } else {
2051 ring->get_seqno = gen8_get_seqno;
2052 ring->set_seqno = gen8_set_seqno;
2053 }
4da46e1e 2054 ring->emit_request = gen8_emit_request;
4712274c 2055 ring->emit_flush = gen8_emit_flush;
73d477f6
OM
2056 ring->irq_get = gen8_logical_ring_get_irq;
2057 ring->irq_put = gen8_logical_ring_put_irq;
15648585 2058 ring->emit_bb_start = gen8_emit_bb_start;
9b1136d5 2059
454afebd
OM
2060 return logical_ring_init(dev, ring);
2061}
2062
73e4d07f
OM
2063/**
2064 * intel_logical_rings_init() - allocate, populate and init the Engine Command Streamers
2065 * @dev: DRM device.
2066 *
2067 * This function inits the engines for an Execlists submission style (the equivalent in the
2068 * legacy ringbuffer submission world would be i915_gem_init_rings). It does it only for
2069 * those engines that are present in the hardware.
2070 *
2071 * Return: non-zero if the initialization failed.
2072 */
454afebd
OM
2073int intel_logical_rings_init(struct drm_device *dev)
2074{
2075 struct drm_i915_private *dev_priv = dev->dev_private;
2076 int ret;
2077
2078 ret = logical_render_ring_init(dev);
2079 if (ret)
2080 return ret;
2081
2082 if (HAS_BSD(dev)) {
2083 ret = logical_bsd_ring_init(dev);
2084 if (ret)
2085 goto cleanup_render_ring;
2086 }
2087
2088 if (HAS_BLT(dev)) {
2089 ret = logical_blt_ring_init(dev);
2090 if (ret)
2091 goto cleanup_bsd_ring;
2092 }
2093
2094 if (HAS_VEBOX(dev)) {
2095 ret = logical_vebox_ring_init(dev);
2096 if (ret)
2097 goto cleanup_blt_ring;
2098 }
2099
2100 if (HAS_BSD2(dev)) {
2101 ret = logical_bsd2_ring_init(dev);
2102 if (ret)
2103 goto cleanup_vebox_ring;
2104 }
2105
2106 ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
2107 if (ret)
2108 goto cleanup_bsd2_ring;
2109
2110 return 0;
2111
2112cleanup_bsd2_ring:
2113 intel_logical_ring_cleanup(&dev_priv->ring[VCS2]);
2114cleanup_vebox_ring:
2115 intel_logical_ring_cleanup(&dev_priv->ring[VECS]);
2116cleanup_blt_ring:
2117 intel_logical_ring_cleanup(&dev_priv->ring[BCS]);
2118cleanup_bsd_ring:
2119 intel_logical_ring_cleanup(&dev_priv->ring[VCS]);
2120cleanup_render_ring:
2121 intel_logical_ring_cleanup(&dev_priv->ring[RCS]);
2122
2123 return ret;
2124}
2125
0cea6502
JM
2126static u32
2127make_rpcs(struct drm_device *dev)
2128{
2129 u32 rpcs = 0;
2130
2131 /*
2132 * No explicit RPCS request is needed to ensure full
2133 * slice/subslice/EU enablement prior to Gen9.
2134 */
2135 if (INTEL_INFO(dev)->gen < 9)
2136 return 0;
2137
2138 /*
2139 * Starting in Gen9, render power gating can leave
2140 * slice/subslice/EU in a partially enabled state. We
2141 * must make an explicit request through RPCS for full
2142 * enablement.
2143 */
2144 if (INTEL_INFO(dev)->has_slice_pg) {
2145 rpcs |= GEN8_RPCS_S_CNT_ENABLE;
2146 rpcs |= INTEL_INFO(dev)->slice_total <<
2147 GEN8_RPCS_S_CNT_SHIFT;
2148 rpcs |= GEN8_RPCS_ENABLE;
2149 }
2150
2151 if (INTEL_INFO(dev)->has_subslice_pg) {
2152 rpcs |= GEN8_RPCS_SS_CNT_ENABLE;
2153 rpcs |= INTEL_INFO(dev)->subslice_per_slice <<
2154 GEN8_RPCS_SS_CNT_SHIFT;
2155 rpcs |= GEN8_RPCS_ENABLE;
2156 }
2157
2158 if (INTEL_INFO(dev)->has_eu_pg) {
2159 rpcs |= INTEL_INFO(dev)->eu_per_subslice <<
2160 GEN8_RPCS_EU_MIN_SHIFT;
2161 rpcs |= INTEL_INFO(dev)->eu_per_subslice <<
2162 GEN8_RPCS_EU_MAX_SHIFT;
2163 rpcs |= GEN8_RPCS_ENABLE;
2164 }
2165
2166 return rpcs;
2167}
2168
8670d6f9
OM
2169static int
2170populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_obj,
2171 struct intel_engine_cs *ring, struct intel_ringbuffer *ringbuf)
2172{
2d965536
TD
2173 struct drm_device *dev = ring->dev;
2174 struct drm_i915_private *dev_priv = dev->dev_private;
ae6c4806 2175 struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
8670d6f9
OM
2176 struct page *page;
2177 uint32_t *reg_state;
2178 int ret;
2179
2d965536
TD
2180 if (!ppgtt)
2181 ppgtt = dev_priv->mm.aliasing_ppgtt;
2182
8670d6f9
OM
2183 ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
2184 if (ret) {
2185 DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
2186 return ret;
2187 }
2188
2189 ret = i915_gem_object_get_pages(ctx_obj);
2190 if (ret) {
2191 DRM_DEBUG_DRIVER("Could not get object pages\n");
2192 return ret;
2193 }
2194
2195 i915_gem_object_pin_pages(ctx_obj);
2196
2197 /* The second page of the context object contains some fields which must
2198 * be set up prior to the first execution. */
d1675198 2199 page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
8670d6f9
OM
2200 reg_state = kmap_atomic(page);
2201
2202 /* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM
2203 * commands followed by (reg, value) pairs. The values we are setting here are
2204 * only for the first context restore: on a subsequent save, the GPU will
2205 * recreate this batchbuffer with new values (including all the missing
2206 * MI_LOAD_REGISTER_IMM commands that we are not initializing here). */
2207 if (ring->id == RCS)
2208 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(14);
2209 else
2210 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(11);
2211 reg_state[CTX_LRI_HEADER_0] |= MI_LRI_FORCE_POSTED;
2212 reg_state[CTX_CONTEXT_CONTROL] = RING_CONTEXT_CONTROL(ring);
2213 reg_state[CTX_CONTEXT_CONTROL+1] =
5baa22c5 2214 _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
6922528a
AJ
2215 CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
2216 CTX_CTRL_RS_CTX_ENABLE);
8670d6f9
OM
2217 reg_state[CTX_RING_HEAD] = RING_HEAD(ring->mmio_base);
2218 reg_state[CTX_RING_HEAD+1] = 0;
2219 reg_state[CTX_RING_TAIL] = RING_TAIL(ring->mmio_base);
2220 reg_state[CTX_RING_TAIL+1] = 0;
2221 reg_state[CTX_RING_BUFFER_START] = RING_START(ring->mmio_base);
7ba717cf
TD
2222 /* Ring buffer start address is not known until the buffer is pinned.
2223 * It is written to the context image in execlists_update_context()
2224 */
8670d6f9
OM
2225 reg_state[CTX_RING_BUFFER_CONTROL] = RING_CTL(ring->mmio_base);
2226 reg_state[CTX_RING_BUFFER_CONTROL+1] =
2227 ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID;
2228 reg_state[CTX_BB_HEAD_U] = ring->mmio_base + 0x168;
2229 reg_state[CTX_BB_HEAD_U+1] = 0;
2230 reg_state[CTX_BB_HEAD_L] = ring->mmio_base + 0x140;
2231 reg_state[CTX_BB_HEAD_L+1] = 0;
2232 reg_state[CTX_BB_STATE] = ring->mmio_base + 0x110;
2233 reg_state[CTX_BB_STATE+1] = (1<<5);
2234 reg_state[CTX_SECOND_BB_HEAD_U] = ring->mmio_base + 0x11c;
2235 reg_state[CTX_SECOND_BB_HEAD_U+1] = 0;
2236 reg_state[CTX_SECOND_BB_HEAD_L] = ring->mmio_base + 0x114;
2237 reg_state[CTX_SECOND_BB_HEAD_L+1] = 0;
2238 reg_state[CTX_SECOND_BB_STATE] = ring->mmio_base + 0x118;
2239 reg_state[CTX_SECOND_BB_STATE+1] = 0;
2240 if (ring->id == RCS) {
8670d6f9
OM
2241 reg_state[CTX_BB_PER_CTX_PTR] = ring->mmio_base + 0x1c0;
2242 reg_state[CTX_BB_PER_CTX_PTR+1] = 0;
2243 reg_state[CTX_RCS_INDIRECT_CTX] = ring->mmio_base + 0x1c4;
2244 reg_state[CTX_RCS_INDIRECT_CTX+1] = 0;
2245 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET] = ring->mmio_base + 0x1c8;
2246 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] = 0;
17ee950d
AS
2247 if (ring->wa_ctx.obj) {
2248 struct i915_ctx_workarounds *wa_ctx = &ring->wa_ctx;
2249 uint32_t ggtt_offset = i915_gem_obj_ggtt_offset(wa_ctx->obj);
2250
2251 reg_state[CTX_RCS_INDIRECT_CTX+1] =
2252 (ggtt_offset + wa_ctx->indirect_ctx.offset * sizeof(uint32_t)) |
2253 (wa_ctx->indirect_ctx.size / CACHELINE_DWORDS);
2254
2255 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] =
2256 CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT << 6;
2257
2258 reg_state[CTX_BB_PER_CTX_PTR+1] =
2259 (ggtt_offset + wa_ctx->per_ctx.offset * sizeof(uint32_t)) |
2260 0x01;
2261 }
8670d6f9
OM
2262 }
2263 reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9);
2264 reg_state[CTX_LRI_HEADER_1] |= MI_LRI_FORCE_POSTED;
2265 reg_state[CTX_CTX_TIMESTAMP] = ring->mmio_base + 0x3a8;
2266 reg_state[CTX_CTX_TIMESTAMP+1] = 0;
2267 reg_state[CTX_PDP3_UDW] = GEN8_RING_PDP_UDW(ring, 3);
2268 reg_state[CTX_PDP3_LDW] = GEN8_RING_PDP_LDW(ring, 3);
2269 reg_state[CTX_PDP2_UDW] = GEN8_RING_PDP_UDW(ring, 2);
2270 reg_state[CTX_PDP2_LDW] = GEN8_RING_PDP_LDW(ring, 2);
2271 reg_state[CTX_PDP1_UDW] = GEN8_RING_PDP_UDW(ring, 1);
2272 reg_state[CTX_PDP1_LDW] = GEN8_RING_PDP_LDW(ring, 1);
2273 reg_state[CTX_PDP0_UDW] = GEN8_RING_PDP_UDW(ring, 0);
2274 reg_state[CTX_PDP0_LDW] = GEN8_RING_PDP_LDW(ring, 0);
d7b2633d 2275
2dba3239
MT
2276 if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
2277 /* 64b PPGTT (48bit canonical)
2278 * PDP0_DESCRIPTOR contains the base address to PML4 and
2279 * other PDP Descriptors are ignored.
2280 */
2281 ASSIGN_CTX_PML4(ppgtt, reg_state);
2282 } else {
2283 /* 32b PPGTT
2284 * PDP*_DESCRIPTOR contains the base address of space supported.
2285 * With dynamic page allocation, PDPs may not be allocated at
2286 * this point. Point the unallocated PDPs to the scratch page
2287 */
2288 ASSIGN_CTX_PDP(ppgtt, reg_state, 3);
2289 ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
2290 ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
2291 ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
2292 }
2293
8670d6f9
OM
2294 if (ring->id == RCS) {
2295 reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
0cea6502
JM
2296 reg_state[CTX_R_PWR_CLK_STATE] = GEN8_R_PWR_CLK_STATE;
2297 reg_state[CTX_R_PWR_CLK_STATE+1] = make_rpcs(dev);
8670d6f9
OM
2298 }
2299
2300 kunmap_atomic(reg_state);
2301
2302 ctx_obj->dirty = 1;
2303 set_page_dirty(page);
2304 i915_gem_object_unpin_pages(ctx_obj);
2305
2306 return 0;
2307}
2308
73e4d07f
OM
2309/**
2310 * intel_lr_context_free() - free the LRC specific bits of a context
2311 * @ctx: the LR context to free.
2312 *
2313 * The real context freeing is done in i915_gem_context_free: this only
2314 * takes care of the bits that are LRC related: the per-engine backing
2315 * objects and the logical ringbuffer.
2316 */
ede7d42b
OM
2317void intel_lr_context_free(struct intel_context *ctx)
2318{
8c857917
OM
2319 int i;
2320
2321 for (i = 0; i < I915_NUM_RINGS; i++) {
2322 struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state;
84c2377f 2323
8c857917 2324 if (ctx_obj) {
dcb4c12a
OM
2325 struct intel_ringbuffer *ringbuf =
2326 ctx->engine[i].ringbuf;
2327 struct intel_engine_cs *ring = ringbuf->ring;
2328
7ba717cf
TD
2329 if (ctx == ring->default_context) {
2330 intel_unpin_ringbuffer_obj(ringbuf);
2331 i915_gem_object_ggtt_unpin(ctx_obj);
2332 }
a7cbedec 2333 WARN_ON(ctx->engine[ring->id].pin_count);
84c2377f
OM
2334 intel_destroy_ringbuffer_obj(ringbuf);
2335 kfree(ringbuf);
8c857917
OM
2336 drm_gem_object_unreference(&ctx_obj->base);
2337 }
2338 }
2339}
2340
2341static uint32_t get_lr_context_size(struct intel_engine_cs *ring)
2342{
2343 int ret = 0;
2344
468c6816 2345 WARN_ON(INTEL_INFO(ring->dev)->gen < 8);
8c857917
OM
2346
2347 switch (ring->id) {
2348 case RCS:
468c6816
MN
2349 if (INTEL_INFO(ring->dev)->gen >= 9)
2350 ret = GEN9_LR_CONTEXT_RENDER_SIZE;
2351 else
2352 ret = GEN8_LR_CONTEXT_RENDER_SIZE;
8c857917
OM
2353 break;
2354 case VCS:
2355 case BCS:
2356 case VECS:
2357 case VCS2:
2358 ret = GEN8_LR_CONTEXT_OTHER_SIZE;
2359 break;
2360 }
2361
2362 return ret;
ede7d42b
OM
2363}
2364
70b0ea86 2365static void lrc_setup_hardware_status_page(struct intel_engine_cs *ring,
1df06b75
TD
2366 struct drm_i915_gem_object *default_ctx_obj)
2367{
2368 struct drm_i915_private *dev_priv = ring->dev->dev_private;
d1675198 2369 struct page *page;
1df06b75 2370
d1675198
AD
2371 /* The HWSP is part of the default context object in LRC mode. */
2372 ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(default_ctx_obj)
2373 + LRC_PPHWSP_PN * PAGE_SIZE;
2374 page = i915_gem_object_get_page(default_ctx_obj, LRC_PPHWSP_PN);
2375 ring->status_page.page_addr = kmap(page);
1df06b75
TD
2376 ring->status_page.obj = default_ctx_obj;
2377
2378 I915_WRITE(RING_HWS_PGA(ring->mmio_base),
2379 (u32)ring->status_page.gfx_addr);
2380 POSTING_READ(RING_HWS_PGA(ring->mmio_base));
1df06b75
TD
2381}
2382
73e4d07f
OM
2383/**
2384 * intel_lr_context_deferred_create() - create the LRC specific bits of a context
2385 * @ctx: LR context to create.
2386 * @ring: engine to be used with the context.
2387 *
2388 * This function can be called more than once, with different engines, if we plan
2389 * to use the context with them. The context backing objects and the ringbuffers
2390 * (specially the ringbuffer backing objects) suck a lot of memory up, and that's why
2391 * the creation is a deferred call: it's better to make sure first that we need to use
2392 * a given ring with the context.
2393 *
32197aab 2394 * Return: non-zero on error.
73e4d07f 2395 */
ede7d42b
OM
2396int intel_lr_context_deferred_create(struct intel_context *ctx,
2397 struct intel_engine_cs *ring)
2398{
dcb4c12a 2399 const bool is_global_default_ctx = (ctx == ring->default_context);
8c857917 2400 struct drm_device *dev = ring->dev;
d1675198 2401 struct drm_i915_private *dev_priv = dev->dev_private;
8c857917
OM
2402 struct drm_i915_gem_object *ctx_obj;
2403 uint32_t context_size;
84c2377f 2404 struct intel_ringbuffer *ringbuf;
8c857917
OM
2405 int ret;
2406
ede7d42b 2407 WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL);
bfc882b4 2408 WARN_ON(ctx->engine[ring->id].state);
ede7d42b 2409
8c857917
OM
2410 context_size = round_up(get_lr_context_size(ring), 4096);
2411
d1675198
AD
2412 /* One extra page as the sharing data between driver and GuC */
2413 context_size += PAGE_SIZE * LRC_PPHWSP_PN;
2414
149c86e7 2415 ctx_obj = i915_gem_alloc_object(dev, context_size);
3126a660
DC
2416 if (!ctx_obj) {
2417 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n");
2418 return -ENOMEM;
8c857917
OM
2419 }
2420
dcb4c12a 2421 if (is_global_default_ctx) {
d1675198
AD
2422 ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN,
2423 PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
dcb4c12a
OM
2424 if (ret) {
2425 DRM_DEBUG_DRIVER("Pin LRC backing obj failed: %d\n",
2426 ret);
2427 drm_gem_object_unreference(&ctx_obj->base);
2428 return ret;
2429 }
d1675198
AD
2430
2431 /* Invalidate GuC TLB. */
2432 if (i915.enable_guc_submission)
2433 I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
8c857917
OM
2434 }
2435
84c2377f
OM
2436 ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
2437 if (!ringbuf) {
2438 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
2439 ring->name);
84c2377f 2440 ret = -ENOMEM;
7ba717cf 2441 goto error_unpin_ctx;
84c2377f
OM
2442 }
2443
0c7dd53b 2444 ringbuf->ring = ring;
582d67f0 2445
d1675198 2446 ringbuf->size = 4 * PAGE_SIZE;
84c2377f
OM
2447 ringbuf->effective_size = ringbuf->size;
2448 ringbuf->head = 0;
2449 ringbuf->tail = 0;
84c2377f 2450 ringbuf->last_retired_head = -1;
ebd0fd4b 2451 intel_ring_update_space(ringbuf);
84c2377f 2452
7ba717cf
TD
2453 if (ringbuf->obj == NULL) {
2454 ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
2455 if (ret) {
2456 DRM_DEBUG_DRIVER(
2457 "Failed to allocate ringbuffer obj %s: %d\n",
84c2377f 2458 ring->name, ret);
7ba717cf
TD
2459 goto error_free_rbuf;
2460 }
2461
2462 if (is_global_default_ctx) {
2463 ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);
2464 if (ret) {
2465 DRM_ERROR(
2466 "Failed to pin and map ringbuffer %s: %d\n",
2467 ring->name, ret);
2468 goto error_destroy_rbuf;
2469 }
2470 }
2471
8670d6f9
OM
2472 }
2473
2474 ret = populate_lr_context(ctx, ctx_obj, ring, ringbuf);
2475 if (ret) {
2476 DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
8670d6f9 2477 goto error;
84c2377f
OM
2478 }
2479
2480 ctx->engine[ring->id].ringbuf = ringbuf;
8c857917 2481 ctx->engine[ring->id].state = ctx_obj;
ede7d42b 2482
70b0ea86
DV
2483 if (ctx == ring->default_context)
2484 lrc_setup_hardware_status_page(ring, ctx_obj);
e7778be1 2485 else if (ring->id == RCS && !ctx->rcs_initialized) {
771b9a53 2486 if (ring->init_context) {
76c39168
JH
2487 struct drm_i915_gem_request *req;
2488
2489 ret = i915_gem_request_alloc(ring, ctx, &req);
2490 if (ret)
2491 return ret;
2492
8753181e 2493 ret = ring->init_context(req);
e7778be1 2494 if (ret) {
771b9a53 2495 DRM_ERROR("ring init context: %d\n", ret);
76c39168 2496 i915_gem_request_cancel(req);
e7778be1
TD
2497 ctx->engine[ring->id].ringbuf = NULL;
2498 ctx->engine[ring->id].state = NULL;
2499 goto error;
2500 }
76c39168 2501
75289874 2502 i915_add_request_no_flush(req);
771b9a53
MT
2503 }
2504
564ddb2f
OM
2505 ctx->rcs_initialized = true;
2506 }
2507
ede7d42b 2508 return 0;
8670d6f9
OM
2509
2510error:
7ba717cf
TD
2511 if (is_global_default_ctx)
2512 intel_unpin_ringbuffer_obj(ringbuf);
2513error_destroy_rbuf:
2514 intel_destroy_ringbuffer_obj(ringbuf);
2515error_free_rbuf:
8670d6f9 2516 kfree(ringbuf);
7ba717cf 2517error_unpin_ctx:
dcb4c12a
OM
2518 if (is_global_default_ctx)
2519 i915_gem_object_ggtt_unpin(ctx_obj);
8670d6f9
OM
2520 drm_gem_object_unreference(&ctx_obj->base);
2521 return ret;
ede7d42b 2522}
3e5b6f05
TD
2523
2524void intel_lr_context_reset(struct drm_device *dev,
2525 struct intel_context *ctx)
2526{
2527 struct drm_i915_private *dev_priv = dev->dev_private;
2528 struct intel_engine_cs *ring;
2529 int i;
2530
2531 for_each_ring(ring, dev_priv, i) {
2532 struct drm_i915_gem_object *ctx_obj =
2533 ctx->engine[ring->id].state;
2534 struct intel_ringbuffer *ringbuf =
2535 ctx->engine[ring->id].ringbuf;
2536 uint32_t *reg_state;
2537 struct page *page;
2538
2539 if (!ctx_obj)
2540 continue;
2541
2542 if (i915_gem_object_get_pages(ctx_obj)) {
2543 WARN(1, "Failed get_pages for context obj\n");
2544 continue;
2545 }
d1675198 2546 page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
3e5b6f05
TD
2547 reg_state = kmap_atomic(page);
2548
2549 reg_state[CTX_RING_HEAD+1] = 0;
2550 reg_state[CTX_RING_TAIL+1] = 0;
2551
2552 kunmap_atomic(reg_state);
2553
2554 ringbuf->head = 0;
2555 ringbuf->tail = 0;
2556 }
2557}
This page took 0.257169 seconds and 5 git commands to generate.