projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Kill intel_prepare_ddi()
[deliverable/linux.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
CommitLineData
62fdfeaf
EA		 1/*
2 * Copyright © 2008-2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Zou Nan hai <nanhai.zou@intel.com>
26 * Xiang Hai hao<haihao.xiang@intel.com>
27 *
28 */
29
a4d8a0fe 30#include <linux/log2.h>
760285e7 31#include <drm/drmP.h>
62fdfeaf 32#include "i915_drv.h"
760285e7 33#include <drm/i915_drm.h>
62fdfeaf 34#include "i915_trace.h"
881f47b6 35#include "intel_drv.h"
62fdfeaf 36
82e104cc 37int __intel_ring_space(int head, int tail, int size)
c7dca47b 38{
4f54741e
DG		 39 int space = head - tail;
40 if (space <= 0)
1cf0ba14 41 space += size;
4f54741e 42 return space - I915_RING_FREE_SPACE;
c7dca47b
CW		 43}
44
ebd0fd4b
DG		 45void intel_ring_update_space(struct intel_ringbuffer *ringbuf)
46{
47 if (ringbuf->last_retired_head != -1) {
48 ringbuf->head = ringbuf->last_retired_head;
49 ringbuf->last_retired_head = -1;
50 }
51
52 ringbuf->space = __intel_ring_space(ringbuf->head & HEAD_ADDR,
53 ringbuf->tail, ringbuf->size);
54}
55
82e104cc 56int intel_ring_space(struct intel_ringbuffer *ringbuf)
1cf0ba14 57{
ebd0fd4b
DG		 58 intel_ring_update_space(ringbuf);
59 return ringbuf->space;
1cf0ba14
CW		 60}
61
82e104cc 62bool intel_ring_stopped(struct intel_engine_cs *ring)
09246732
CW		 63{
64 struct drm_i915_private *dev_priv = ring->dev->dev_private;
88b4aa87
MK		 65 return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);
66}
09246732 67
6258fbe2 68static void __intel_ring_advance(struct intel_engine_cs *ring)
88b4aa87 69{
93b0a4e0
OM		 70 struct intel_ringbuffer *ringbuf = ring->buffer;
71 ringbuf->tail &= ringbuf->size - 1;
88b4aa87 72 if (intel_ring_stopped(ring))
09246732 73 return;
93b0a4e0 74 ring->write_tail(ring, ringbuf->tail);
09246732
CW		 75}
76
b72f3acb 77static int
a84c3ae1 78gen2_render_ring_flush(struct drm_i915_gem_request *req,
46f0f8d1
CW		 79 u32 invalidate_domains,
80 u32 flush_domains)
81{
a84c3ae1 82 struct intel_engine_cs *ring = req->ring;
46f0f8d1
CW		 83 u32 cmd;
84 int ret;
85
86 cmd = MI_FLUSH;
31b14c9f 87 if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
46f0f8d1
CW		 88 cmd |= MI_NO_WRITE_FLUSH;
89
90 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
91 cmd |= MI_READ_FLUSH;
92
5fb9de1a 93 ret = intel_ring_begin(req, 2);
46f0f8d1
CW		 94 if (ret)
95 return ret;
96
97 intel_ring_emit(ring, cmd);
98 intel_ring_emit(ring, MI_NOOP);
99 intel_ring_advance(ring);
100
101 return 0;
102}
103
104static int
a84c3ae1 105gen4_render_ring_flush(struct drm_i915_gem_request *req,
46f0f8d1
CW		 106 u32 invalidate_domains,
107 u32 flush_domains)
62fdfeaf 108{
a84c3ae1 109 struct intel_engine_cs *ring = req->ring;
78501eac 110 struct drm_device *dev = ring->dev;
6f392d54 111 u32 cmd;
b72f3acb 112 int ret;
6f392d54 113
36d527de
CW		 114 /*
115 * read/write caches:
116 *
117 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
118 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
119 * also flushed at 2d versus 3d pipeline switches.
120 *
121 * read-only caches:
122 *
123 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
124 * MI_READ_FLUSH is set, and is always flushed on 965.
125 *
126 * I915_GEM_DOMAIN_COMMAND may not exist?
127 *
128 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
129 * invalidated when MI_EXE_FLUSH is set.
130 *
131 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
132 * invalidated with every MI_FLUSH.
133 *
134 * TLBs:
135 *
136 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
137 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
138 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
139 * are flushed at any MI_FLUSH.
140 */
141
142 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
46f0f8d1 143 if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
36d527de 144 cmd &= ~MI_NO_WRITE_FLUSH;
36d527de
CW		 145 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
146 cmd |= MI_EXE_FLUSH;
62fdfeaf 147
36d527de
CW		 148 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
149 (IS_G4X(dev) || IS_GEN5(dev)))
150 cmd |= MI_INVALIDATE_ISP;
70eac33e 151
5fb9de1a 152 ret = intel_ring_begin(req, 2);
36d527de
CW		 153 if (ret)
154 return ret;
b72f3acb 155
36d527de
CW		 156 intel_ring_emit(ring, cmd);
157 intel_ring_emit(ring, MI_NOOP);
158 intel_ring_advance(ring);
b72f3acb
CW		 159
160 return 0;
8187a2b7
ZN		 161}
162
8d315287
JB		 163/**
164 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
165 * implementing two workarounds on gen6. From section 1.4.7.1
166 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
167 *
168 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
169 * produced by non-pipelined state commands), software needs to first
170 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
171 * 0.
172 *
173 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
174 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
175 *
176 * And the workaround for these two requires this workaround first:
177 *
178 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
179 * BEFORE the pipe-control with a post-sync op and no write-cache
180 * flushes.
181 *
182 * And this last workaround is tricky because of the requirements on
183 * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
184 * volume 2 part 1:
185 *
186 * "1 of the following must also be set:
187 * - Render Target Cache Flush Enable ([12] of DW1)
188 * - Depth Cache Flush Enable ([0] of DW1)
189 * - Stall at Pixel Scoreboard ([1] of DW1)
190 * - Depth Stall ([13] of DW1)
191 * - Post-Sync Operation ([13] of DW1)
192 * - Notify Enable ([8] of DW1)"
193 *
194 * The cache flushes require the workaround flush that triggered this
195 * one, so we can't use it. Depth stall would trigger the same.
196 * Post-sync nonzero is what triggered this second workaround, so we
197 * can't use that one either. Notify enable is IRQs, which aren't
198 * really our business. That leaves only stall at scoreboard.
199 */
200static int
f2cf1fcc 201intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
8d315287 202{
f2cf1fcc 203 struct intel_engine_cs *ring = req->ring;
18393f63 204 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
8d315287
JB		 205 int ret;
206
5fb9de1a 207 ret = intel_ring_begin(req, 6);
8d315287
JB		 208 if (ret)
209 return ret;
210
211 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
212 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
213 PIPE_CONTROL_STALL_AT_SCOREBOARD);
214 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
215 intel_ring_emit(ring, 0); /* low dword */
216 intel_ring_emit(ring, 0); /* high dword */
217 intel_ring_emit(ring, MI_NOOP);
218 intel_ring_advance(ring);
219
5fb9de1a 220 ret = intel_ring_begin(req, 6);
8d315287
JB		 221 if (ret)
222 return ret;
223
224 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
225 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
226 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
227 intel_ring_emit(ring, 0);
228 intel_ring_emit(ring, 0);
229 intel_ring_emit(ring, MI_NOOP);
230 intel_ring_advance(ring);
231
232 return 0;
233}
234
235static int
a84c3ae1
JH		 236gen6_render_ring_flush(struct drm_i915_gem_request *req,
237 u32 invalidate_domains, u32 flush_domains)
8d315287 238{
a84c3ae1 239 struct intel_engine_cs *ring = req->ring;
8d315287 240 u32 flags = 0;
18393f63 241 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
8d315287
JB		 242 int ret;
243
b3111509 244 /* Force SNB workarounds for PIPE_CONTROL flushes */
f2cf1fcc 245 ret = intel_emit_post_sync_nonzero_flush(req);
b3111509
PZ		 246 if (ret)
247 return ret;
248
8d315287
JB		 249 /* Just flush everything. Experiments have shown that reducing the
250 * number of bits based on the write domains has little performance
251 * impact.
252 */
7d54a904
CW		 253 if (flush_domains) {
254 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
255 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
256 /*
257 * Ensure that any following seqno writes only happen
258 * when the render cache is indeed flushed.
259 */
97f209bc 260 flags |= PIPE_CONTROL_CS_STALL;
7d54a904
CW		 261 }
262 if (invalidate_domains) {
263 flags |= PIPE_CONTROL_TLB_INVALIDATE;
264 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
265 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
266 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
267 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
268 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
269 /*
270 * TLB invalidate requires a post-sync write.
271 */
3ac78313 272 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
7d54a904 273 }
8d315287 274
5fb9de1a 275 ret = intel_ring_begin(req, 4);
8d315287
JB		 276 if (ret)
277 return ret;
278
6c6cf5aa 279 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
8d315287
JB		 280 intel_ring_emit(ring, flags);
281 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
6c6cf5aa 282 intel_ring_emit(ring, 0);
8d315287
JB		 283 intel_ring_advance(ring);
284
285 return 0;
286}
287
f3987631 288static int
f2cf1fcc 289gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req)
f3987631 290{
f2cf1fcc 291 struct intel_engine_cs *ring = req->ring;
f3987631
PZ		 292 int ret;
293
5fb9de1a 294 ret = intel_ring_begin(req, 4);
f3987631
PZ		 295 if (ret)
296 return ret;
297
298 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
299 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
300 PIPE_CONTROL_STALL_AT_SCOREBOARD);
301 intel_ring_emit(ring, 0);
302 intel_ring_emit(ring, 0);
303 intel_ring_advance(ring);
304
305 return 0;
306}
307
4772eaeb 308static int
a84c3ae1 309gen7_render_ring_flush(struct drm_i915_gem_request *req,
4772eaeb
PZ		 310 u32 invalidate_domains, u32 flush_domains)
311{
a84c3ae1 312 struct intel_engine_cs *ring = req->ring;
4772eaeb 313 u32 flags = 0;
18393f63 314 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
4772eaeb
PZ		 315 int ret;
316
f3987631
PZ		 317 /*
318 * Ensure that any following seqno writes only happen when the render
319 * cache is indeed flushed.
320 *
321 * Workaround: 4th PIPE_CONTROL command (except the ones with only
322 * read-cache invalidate bits set) must have the CS_STALL bit set. We
323 * don't try to be clever and just set it unconditionally.
324 */
325 flags |= PIPE_CONTROL_CS_STALL;
326
4772eaeb
PZ		 327 /* Just flush everything. Experiments have shown that reducing the
328 * number of bits based on the write domains has little performance
329 * impact.
330 */
331 if (flush_domains) {
332 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
333 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
40a24488 334 flags |= PIPE_CONTROL_FLUSH_ENABLE;
4772eaeb
PZ		 335 }
336 if (invalidate_domains) {
337 flags |= PIPE_CONTROL_TLB_INVALIDATE;
338 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
339 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
340 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
341 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
342 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
148b83d0 343 flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR;
4772eaeb
PZ		 344 /*
345 * TLB invalidate requires a post-sync write.
346 */
347 flags |= PIPE_CONTROL_QW_WRITE;
b9e1faa7 348 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
f3987631 349
add284a3
CW		 350 flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;
351
f3987631
PZ		 352 /* Workaround: we must issue a pipe_control with CS-stall bit
353 * set before a pipe_control command that has the state cache
354 * invalidate bit set. */
f2cf1fcc 355 gen7_render_ring_cs_stall_wa(req);
4772eaeb
PZ		 356 }
357
5fb9de1a 358 ret = intel_ring_begin(req, 4);
4772eaeb
PZ		 359 if (ret)
360 return ret;
361
362 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
363 intel_ring_emit(ring, flags);
b9e1faa7 364 intel_ring_emit(ring, scratch_addr);
4772eaeb
PZ		 365 intel_ring_emit(ring, 0);
366 intel_ring_advance(ring);
367
368 return 0;
369}
370
884ceace 371static int
f2cf1fcc 372gen8_emit_pipe_control(struct drm_i915_gem_request *req,
884ceace
KG		 373 u32 flags, u32 scratch_addr)
374{
f2cf1fcc 375 struct intel_engine_cs *ring = req->ring;
884ceace
KG		 376 int ret;
377
5fb9de1a 378 ret = intel_ring_begin(req, 6);
884ceace
KG		 379 if (ret)
380 return ret;
381
382 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
383 intel_ring_emit(ring, flags);
384 intel_ring_emit(ring, scratch_addr);
385 intel_ring_emit(ring, 0);
386 intel_ring_emit(ring, 0);
387 intel_ring_emit(ring, 0);
388 intel_ring_advance(ring);
389
390 return 0;
391}
392
a5f3d68e 393static int
a84c3ae1 394gen8_render_ring_flush(struct drm_i915_gem_request *req,
a5f3d68e
BW		 395 u32 invalidate_domains, u32 flush_domains)
396{
397 u32 flags = 0;
f2cf1fcc 398 u32 scratch_addr = req->ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
02c9f7e3 399 int ret;
a5f3d68e
BW		 400
401 flags |= PIPE_CONTROL_CS_STALL;
402
403 if (flush_domains) {
404 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
405 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
40a24488 406 flags |= PIPE_CONTROL_FLUSH_ENABLE;
a5f3d68e
BW		 407 }
408 if (invalidate_domains) {
409 flags |= PIPE_CONTROL_TLB_INVALIDATE;
410 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
411 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
412 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
413 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
414 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
415 flags |= PIPE_CONTROL_QW_WRITE;
416 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
02c9f7e3
KG		 417
418 /* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
f2cf1fcc 419 ret = gen8_emit_pipe_control(req,
02c9f7e3
KG		 420 PIPE_CONTROL_CS_STALL |
421 PIPE_CONTROL_STALL_AT_SCOREBOARD,
422 0);
423 if (ret)
424 return ret;
a5f3d68e
BW		 425 }
426
f2cf1fcc 427 return gen8_emit_pipe_control(req, flags, scratch_addr);
a5f3d68e
BW		 428}
429
a4872ba6 430static void ring_write_tail(struct intel_engine_cs *ring,
297b0c5b 431 u32 value)
d46eefa2 432{
4640c4ff 433 struct drm_i915_private *dev_priv = ring->dev->dev_private;
297b0c5b 434 I915_WRITE_TAIL(ring, value);
d46eefa2
XH		 435}
436
a4872ba6 437u64 intel_ring_get_active_head(struct intel_engine_cs *ring)
8187a2b7 438{
4640c4ff 439 struct drm_i915_private *dev_priv = ring->dev->dev_private;
50877445 440 u64 acthd;
8187a2b7 441
50877445
CW		 442 if (INTEL_INFO(ring->dev)->gen >= 8)
443 acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),
444 RING_ACTHD_UDW(ring->mmio_base));
445 else if (INTEL_INFO(ring->dev)->gen >= 4)
446 acthd = I915_READ(RING_ACTHD(ring->mmio_base));
447 else
448 acthd = I915_READ(ACTHD);
449
450 return acthd;
8187a2b7
ZN		 451}
452
a4872ba6 453static void ring_setup_phys_status_page(struct intel_engine_cs *ring)
035dc1e0
DV		 454{
455 struct drm_i915_private *dev_priv = ring->dev->dev_private;
456 u32 addr;
457
458 addr = dev_priv->status_page_dmah->busaddr;
459 if (INTEL_INFO(ring->dev)->gen >= 4)
460 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
461 I915_WRITE(HWS_PGA, addr);
462}
463
af75f269
DL		 464static void intel_ring_setup_status_page(struct intel_engine_cs *ring)
465{
466 struct drm_device *dev = ring->dev;
467 struct drm_i915_private *dev_priv = ring->dev->dev_private;
f0f59a00 468 i915_reg_t mmio;
af75f269
DL		 469
470 /* The ring status page addresses are no longer next to the rest of
471 * the ring registers as of gen7.
472 */
473 if (IS_GEN7(dev)) {
474 switch (ring->id) {
475 case RCS:
476 mmio = RENDER_HWS_PGA_GEN7;
477 break;
478 case BCS:
479 mmio = BLT_HWS_PGA_GEN7;
480 break;
481 /*
482 * VCS2 actually doesn't exist on Gen7. Only shut up
483 * gcc switch check warning
484 */
485 case VCS2:
486 case VCS:
487 mmio = BSD_HWS_PGA_GEN7;
488 break;
489 case VECS:
490 mmio = VEBOX_HWS_PGA_GEN7;
491 break;
492 }
493 } else if (IS_GEN6(ring->dev)) {
494 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
495 } else {
496 /* XXX: gen8 returns to sanity */
497 mmio = RING_HWS_PGA(ring->mmio_base);
498 }
499
500 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
501 POSTING_READ(mmio);
502
503 /*
504 * Flush the TLB for this page
505 *
506 * FIXME: These two bits have disappeared on gen8, so a question
507 * arises: do we still need this and if so how should we go about
508 * invalidating the TLB?
509 */
510 if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
f0f59a00 511 i915_reg_t reg = RING_INSTPM(ring->mmio_base);
af75f269
DL		 512
513 /* ring should be idle before issuing a sync flush*/
514 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
515
516 I915_WRITE(reg,
517 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
518 INSTPM_SYNC_FLUSH));
519 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
520 1000))
521 DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
522 ring->name);
523 }
524}
525
a4872ba6 526static bool stop_ring(struct intel_engine_cs *ring)
8187a2b7 527{
9991ae78 528 struct drm_i915_private *dev_priv = to_i915(ring->dev);
8187a2b7 529
9991ae78
CW		 530 if (!IS_GEN2(ring->dev)) {
531 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
403bdd10
DV		 532 if (wait_for((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
533 DRM_ERROR("%s : timed out trying to stop ring\n", ring->name);
9bec9b13
CW		 534 /* Sometimes we observe that the idle flag is not
535 * set even though the ring is empty. So double
536 * check before giving up.
537 */
538 if (I915_READ_HEAD(ring) != I915_READ_TAIL(ring))
539 return false;
9991ae78
CW		 540 }
541 }
b7884eb4 542
7f2ab699 543 I915_WRITE_CTL(ring, 0);
570ef608 544 I915_WRITE_HEAD(ring, 0);
78501eac 545 ring->write_tail(ring, 0);
8187a2b7 546
9991ae78
CW		 547 if (!IS_GEN2(ring->dev)) {
548 (void)I915_READ_CTL(ring);
549 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
550 }
a51435a3 551
9991ae78
CW		 552 return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0;
553}
8187a2b7 554
a4872ba6 555static int init_ring_common(struct intel_engine_cs *ring)
9991ae78
CW		 556{
557 struct drm_device *dev = ring->dev;
558 struct drm_i915_private *dev_priv = dev->dev_private;
93b0a4e0
OM		 559 struct intel_ringbuffer *ringbuf = ring->buffer;
560 struct drm_i915_gem_object *obj = ringbuf->obj;
9991ae78
CW		 561 int ret = 0;
562
59bad947 563 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
9991ae78
CW		 564
565 if (!stop_ring(ring)) {
566 /* G45 ring initialization often fails to reset head to zero */
6fd0d56e
CW		 567 DRM_DEBUG_KMS("%s head not reset to zero "
568 "ctl %08x head %08x tail %08x start %08x\n",
569 ring->name,
570 I915_READ_CTL(ring),
571 I915_READ_HEAD(ring),
572 I915_READ_TAIL(ring),
573 I915_READ_START(ring));
8187a2b7 574
9991ae78 575 if (!stop_ring(ring)) {
6fd0d56e
CW		 576 DRM_ERROR("failed to set %s head to zero "
577 "ctl %08x head %08x tail %08x start %08x\n",
578 ring->name,
579 I915_READ_CTL(ring),
580 I915_READ_HEAD(ring),
581 I915_READ_TAIL(ring),
582 I915_READ_START(ring));
9991ae78
CW		 583 ret = -EIO;
584 goto out;
6fd0d56e 585 }
8187a2b7
ZN		 586 }
587
9991ae78
CW		 588 if (I915_NEED_GFX_HWS(dev))
589 intel_ring_setup_status_page(ring);
590 else
591 ring_setup_phys_status_page(ring);
592
ece4a17d
JK		 593 /* Enforce ordering by reading HEAD register back */
594 I915_READ_HEAD(ring);
595
0d8957c8
DV		 596 /* Initialize the ring. This must happen _after_ we've cleared the ring
597 * registers with the above sequence (the readback of the HEAD registers
598 * also enforces ordering), otherwise the hw might lose the new ring
599 * register values. */
f343c5f6 600 I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
95468892
CW		 601
602 /* WaClearRingBufHeadRegAtInit:ctg,elk */
603 if (I915_READ_HEAD(ring))
604 DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
605 ring->name, I915_READ_HEAD(ring));
606 I915_WRITE_HEAD(ring, 0);
607 (void)I915_READ_HEAD(ring);
608
7f2ab699 609 I915_WRITE_CTL(ring,
93b0a4e0 610 ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
5d031e5b 611 | RING_VALID);
8187a2b7 612
8187a2b7 613 /* If the head is still not zero, the ring is dead */
f01db988 614 if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
f343c5f6 615 I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
f01db988 616 (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
e74cfed5 617 DRM_ERROR("%s initialization failed "
48e48a0b
CW		 618 "ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
619 ring->name,
620 I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID,
621 I915_READ_HEAD(ring), I915_READ_TAIL(ring),
622 I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj));
b7884eb4
DV		 623 ret = -EIO;
624 goto out;
8187a2b7
ZN		 625 }
626
ebd0fd4b 627 ringbuf->last_retired_head = -1;
5c6c6003
CW		 628 ringbuf->head = I915_READ_HEAD(ring);
629 ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
ebd0fd4b 630 intel_ring_update_space(ringbuf);
1ec14ad3 631
50f018df
CW		 632 memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
633
b7884eb4 634out:
59bad947 635 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
b7884eb4
DV		 636
637 return ret;
8187a2b7
ZN		 638}
639
9b1136d5
OM		 640void
641intel_fini_pipe_control(struct intel_engine_cs *ring)
642{
643 struct drm_device *dev = ring->dev;
644
645 if (ring->scratch.obj == NULL)
646 return;
647
648 if (INTEL_INFO(dev)->gen >= 5) {
649 kunmap(sg_page(ring->scratch.obj->pages->sgl));
650 i915_gem_object_ggtt_unpin(ring->scratch.obj);
651 }
652
653 drm_gem_object_unreference(&ring->scratch.obj->base);
654 ring->scratch.obj = NULL;
655}
656
657int
658intel_init_pipe_control(struct intel_engine_cs *ring)
c6df541c 659{
c6df541c
CW		 660 int ret;
661
bfc882b4 662 WARN_ON(ring->scratch.obj);
c6df541c 663
0d1aacac
CW		 664 ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
665 if (ring->scratch.obj == NULL) {
c6df541c
CW		 666 DRM_ERROR("Failed to allocate seqno page\n");
667 ret = -ENOMEM;
668 goto err;
669 }
e4ffd173 670
a9cc726c
DV		 671 ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
672 if (ret)
673 goto err_unref;
c6df541c 674
1ec9e26d 675 ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0);
c6df541c
CW		 676 if (ret)
677 goto err_unref;
678
0d1aacac
CW		 679 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
680 ring->scratch.cpu_page = kmap(sg_page(ring->scratch.obj->pages->sgl));
681 if (ring->scratch.cpu_page == NULL) {
56b085a0 682 ret = -ENOMEM;
c6df541c 683 goto err_unpin;
56b085a0 684 }
c6df541c 685
2b1086cc 686 DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
0d1aacac 687 ring->name, ring->scratch.gtt_offset);
c6df541c
CW		 688 return 0;
689
690err_unpin:
d7f46fc4 691 i915_gem_object_ggtt_unpin(ring->scratch.obj);
c6df541c 692err_unref:
0d1aacac 693 drm_gem_object_unreference(&ring->scratch.obj->base);
c6df541c 694err:
c6df541c
CW		 695 return ret;
696}
697
e2be4faf 698static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
86d7f238 699{
7225342a 700 int ret, i;
e2be4faf 701 struct intel_engine_cs *ring = req->ring;
888b5995
AS		 702 struct drm_device *dev = ring->dev;
703 struct drm_i915_private *dev_priv = dev->dev_private;
7225342a 704 struct i915_workarounds *w = &dev_priv->workarounds;
888b5995 705
02235808 706 if (w->count == 0)
7225342a 707 return 0;
888b5995 708
7225342a 709 ring->gpu_caches_dirty = true;
4866d729 710 ret = intel_ring_flush_all_caches(req);
7225342a
MK		 711 if (ret)
712 return ret;
888b5995 713
5fb9de1a 714 ret = intel_ring_begin(req, (w->count * 2 + 2));
7225342a
MK		 715 if (ret)
716 return ret;
717
22a916aa 718 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count));
7225342a 719 for (i = 0; i < w->count; i++) {
f92a9162 720 intel_ring_emit_reg(ring, w->reg[i].addr);
7225342a
MK		 721 intel_ring_emit(ring, w->reg[i].value);
722 }
22a916aa 723 intel_ring_emit(ring, MI_NOOP);
7225342a
MK		 724
725 intel_ring_advance(ring);
726
727 ring->gpu_caches_dirty = true;
4866d729 728 ret = intel_ring_flush_all_caches(req);
7225342a
MK		 729 if (ret)
730 return ret;
888b5995 731
7225342a 732 DRM_DEBUG_DRIVER("Number of Workarounds emitted: %d\n", w->count);
888b5995 733
7225342a 734 return 0;
86d7f238
AS		 735}
736
8753181e 737static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
8f0e2b9d
DV		 738{
739 int ret;
740
e2be4faf 741 ret = intel_ring_workarounds_emit(req);
8f0e2b9d
DV		 742 if (ret != 0)
743 return ret;
744
be01363f 745 ret = i915_gem_render_state_init(req);
8f0e2b9d
DV		 746 if (ret)
747 DRM_ERROR("init render state: %d\n", ret);
748
749 return ret;
750}
751
7225342a 752static int wa_add(struct drm_i915_private *dev_priv,
f0f59a00
VS		 753 i915_reg_t addr,
754 const u32 mask, const u32 val)
7225342a
MK		 755{
756 const u32 idx = dev_priv->workarounds.count;
757
758 if (WARN_ON(idx >= I915_MAX_WA_REGS))
759 return -ENOSPC;
760
761 dev_priv->workarounds.reg[idx].addr = addr;
762 dev_priv->workarounds.reg[idx].value = val;
763 dev_priv->workarounds.reg[idx].mask = mask;
764
765 dev_priv->workarounds.count++;
766
767 return 0;
86d7f238
AS		 768}
769
ca5a0fbd 770#define WA_REG(addr, mask, val) do { \
cf4b0de6 771 const int r = wa_add(dev_priv, (addr), (mask), (val)); \
7225342a
MK		 772 if (r) \
773 return r; \
ca5a0fbd 774 } while (0)
7225342a
MK		 775
776#define WA_SET_BIT_MASKED(addr, mask) \
26459343 777 WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
7225342a
MK		 778
779#define WA_CLR_BIT_MASKED(addr, mask) \
26459343 780 WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
7225342a 781
98533251 782#define WA_SET_FIELD_MASKED(addr, mask, value) \
cf4b0de6 783 WA_REG(addr, mask, _MASKED_FIELD(mask, value))
7225342a 784
cf4b0de6
DL		 785#define WA_SET_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) | (mask))
786#define WA_CLR_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) & ~(mask))
7225342a 787
cf4b0de6 788#define WA_WRITE(addr, val) WA_REG(addr, 0xffffffff, val)
7225342a 789
e9a64ada
AS		 790static int gen8_init_workarounds(struct intel_engine_cs *ring)
791{
68c6198b
AS		 792 struct drm_device *dev = ring->dev;
793 struct drm_i915_private *dev_priv = dev->dev_private;
794
795 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
e9a64ada 796
717d84d6
AS		 797 /* WaDisableAsyncFlipPerfMode:bdw,chv */
798 WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
799
d0581194
AS		 800 /* WaDisablePartialInstShootdown:bdw,chv */
801 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
802 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
803
a340af58
AS		 804 /* Use Force Non-Coherent whenever executing a 3D context. This is a
805 * workaround for for a possible hang in the unlikely event a TLB
806 * invalidation occurs during a PSD flush.
807 */
808 /* WaForceEnableNonCoherent:bdw,chv */
120f5d28 809 /* WaHdcDisableFetchWhenMasked:bdw,chv */
a340af58 810 WA_SET_BIT_MASKED(HDC_CHICKEN0,
120f5d28 811 HDC_DONOT_FETCH_MEM_WHEN_MASKED |
a340af58
AS		 812 HDC_FORCE_NON_COHERENT);
813
6def8fdd
AS		 814 /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
815 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
816 * polygons in the same 8x4 pixel/sample area to be processed without
817 * stalling waiting for the earlier ones to write to Hierarchical Z
818 * buffer."
819 *
820 * This optimization is off by default for BDW and CHV; turn it on.
821 */
822 WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
823
48404636
AS		 824 /* Wa4x4STCOptimizationDisable:bdw,chv */
825 WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
826
7eebcde6
AS		 827 /*
828 * BSpec recommends 8x4 when MSAA is used,
829 * however in practice 16x4 seems fastest.
830 *
831 * Note that PS/WM thread counts depend on the WIZ hashing
832 * disable bit, which we don't touch here, but it's good
833 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
834 */
835 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
836 GEN6_WIZ_HASHING_MASK,
837 GEN6_WIZ_HASHING_16x4);
838
e9a64ada
AS		 839 return 0;
840}
841
00e1e623 842static int bdw_init_workarounds(struct intel_engine_cs *ring)
86d7f238 843{
e9a64ada 844 int ret;
888b5995
AS		 845 struct drm_device *dev = ring->dev;
846 struct drm_i915_private *dev_priv = dev->dev_private;
86d7f238 847
e9a64ada
AS		 848 ret = gen8_init_workarounds(ring);
849 if (ret)
850 return ret;
851
101b376d 852 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
d0581194 853 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
86d7f238 854
101b376d 855 /* WaDisableDopClockGating:bdw */
7225342a
MK		 856 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
857 DOP_CLOCK_GATING_DISABLE);
86d7f238 858
7225342a
MK		 859 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
860 GEN8_SAMPLER_POWER_BYPASS_DIS);
86d7f238 861
7225342a 862 WA_SET_BIT_MASKED(HDC_CHICKEN0,
35cb6f3b
DL		 863 /* WaForceContextSaveRestoreNonCoherent:bdw */
864 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
35cb6f3b 865 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
7225342a 866 (IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
86d7f238 867
86d7f238
AS		 868 return 0;
869}
870
00e1e623
VS		 871static int chv_init_workarounds(struct intel_engine_cs *ring)
872{
e9a64ada 873 int ret;
00e1e623
VS		 874 struct drm_device *dev = ring->dev;
875 struct drm_i915_private *dev_priv = dev->dev_private;
876
e9a64ada
AS		 877 ret = gen8_init_workarounds(ring);
878 if (ret)
879 return ret;
880
00e1e623 881 /* WaDisableThreadStallDopClockGating:chv */
d0581194 882 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
00e1e623 883
d60de81d
KG		 884 /* Improve HiZ throughput on CHV. */
885 WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
886
7225342a
MK		 887 return 0;
888}
889
3b106531
HN		 890static int gen9_init_workarounds(struct intel_engine_cs *ring)
891{
ab0dfafe
HN		 892 struct drm_device *dev = ring->dev;
893 struct drm_i915_private *dev_priv = dev->dev_private;
8ea6f892 894 uint32_t tmp;
ab0dfafe 895
9c4cbf82
MK		 896 /* WaEnableLbsSlaRetryTimerDecrement:skl */
897 I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
898 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
899
900 /* WaDisableKillLogic:bxt,skl */
901 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
902 ECOCHK_DIS_TLB);
903
b0e6f6d4 904 /* WaDisablePartialInstShootdown:skl,bxt */
ab0dfafe
HN		 905 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
906 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
907
a119a6e6 908 /* Syncing dependencies between camera and graphics:skl,bxt */
8424171e
NH		 909 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
910 GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
911
e87a005d
JN		 912 /* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */
913 if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
914 IS_BXT_REVID(dev, 0, BXT_REVID_A1))
a86eb582
DL		 915 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
916 GEN9_DG_MIRROR_FIX_ENABLE);
1de4582f 917
e87a005d
JN		 918 /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
919 if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
920 IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
183c6dac
DL		 921 WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
922 GEN9_RHWO_OPTIMIZATION_DISABLE);
9b01435d
AS		 923 /*
924 * WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
925 * but we do that in per ctx batchbuffer as there is an issue
926 * with this register not getting restored on ctx restore
927 */
183c6dac
DL		 928 }
929
e87a005d
JN		 930 /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt */
931 if (IS_SKL_REVID(dev, SKL_REVID_C0, REVID_FOREVER) || IS_BROXTON(dev))
cac23df4
NH		 932 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
933 GEN9_ENABLE_YV12_BUGFIX);
cac23df4 934
5068368c 935 /* Wa4x4STCOptimizationDisable:skl,bxt */
27160c96 936 /* WaDisablePartialResolveInVc:skl,bxt */
60294683
AS		 937 WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
938 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
9370cd98 939
16be17af 940 /* WaCcsTlbPrefetchDisable:skl,bxt */
e2db7071
DL		 941 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
942 GEN9_CCS_TLB_PREFETCH_ENABLE);
943
5a2ae95e 944 /* WaDisableMaskBasedCammingInRCC:skl,bxt */
e87a005d
JN		 945 if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_C0) ||
946 IS_BXT_REVID(dev, 0, BXT_REVID_A1))
38a39a7b
BW		 947 WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
948 PIXEL_MASK_CAMMING_DISABLE);
949
8ea6f892
ID		 950 /* WaForceContextSaveRestoreNonCoherent:skl,bxt */
951 tmp = HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT;
e87a005d
JN		 952 if (IS_SKL_REVID(dev, SKL_REVID_F0, SKL_REVID_F0) ||
953 IS_BXT_REVID(dev, BXT_REVID_B0, REVID_FOREVER))
8ea6f892
ID		 954 tmp |= HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE;
955 WA_SET_BIT_MASKED(HDC_CHICKEN0, tmp);
956
8c761609 957 /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt */
e87a005d 958 if (IS_SKYLAKE(dev) || IS_BXT_REVID(dev, 0, BXT_REVID_B0))
8c761609
AS		 959 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
960 GEN8_SAMPLER_POWER_BYPASS_DIS);
8c761609 961
6b6d5626
RB		 962 /* WaDisableSTUnitPowerOptimization:skl,bxt */
963 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
964
3b106531
HN		 965 return 0;
966}
967
b7668791
DL		 968static int skl_tune_iz_hashing(struct intel_engine_cs *ring)
969{
970 struct drm_device *dev = ring->dev;
971 struct drm_i915_private *dev_priv = dev->dev_private;
972 u8 vals[3] = { 0, 0, 0 };
973 unsigned int i;
974
975 for (i = 0; i < 3; i++) {
976 u8 ss;
977
978 /*
979 * Only consider slices where one, and only one, subslice has 7
980 * EUs
981 */
a4d8a0fe 982 if (!is_power_of_2(dev_priv->info.subslice_7eu[i]))
b7668791
DL		 983 continue;
984
985 /*
986 * subslice_7eu[i] != 0 (because of the check above) and
987 * ss_max == 4 (maximum number of subslices possible per slice)
988 *
989 * -> 0 <= ss <= 3;
990 */
991 ss = ffs(dev_priv->info.subslice_7eu[i]) - 1;
992 vals[i] = 3 - ss;
993 }
994
995 if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
996 return 0;
997
998 /* Tune IZ hashing. See intel_device_info_runtime_init() */
999 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
1000 GEN9_IZ_HASHING_MASK(2) |
1001 GEN9_IZ_HASHING_MASK(1) |
1002 GEN9_IZ_HASHING_MASK(0),
1003 GEN9_IZ_HASHING(2, vals[2]) |
1004 GEN9_IZ_HASHING(1, vals[1]) |
1005 GEN9_IZ_HASHING(0, vals[0]));
1006
1007 return 0;
1008}
1009
8d205494
DL		 1010static int skl_init_workarounds(struct intel_engine_cs *ring)
1011{
aa0011a8 1012 int ret;
d0bbbc4f
DL		 1013 struct drm_device *dev = ring->dev;
1014 struct drm_i915_private *dev_priv = dev->dev_private;
1015
aa0011a8
AS		 1016 ret = gen9_init_workarounds(ring);
1017 if (ret)
1018 return ret;
8d205494 1019
e87a005d 1020 if (IS_SKL_REVID(dev, 0, SKL_REVID_D0)) {
9c4cbf82
MK		 1021 /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
1022 I915_WRITE(FF_SLICE_CS_CHICKEN2,
1023 _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
1024 }
1025
1026 /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
1027 * involving this register should also be added to WA batch as required.
1028 */
e87a005d 1029 if (IS_SKL_REVID(dev, 0, SKL_REVID_E0))
9c4cbf82
MK		 1030 /* WaDisableLSQCROPERFforOCL:skl */
1031 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
1032 GEN8_LQSC_RO_PERF_DIS);
1033
1034 /* WaEnableGapsTsvCreditFix:skl */
e87a005d 1035 if (IS_SKL_REVID(dev, SKL_REVID_C0, REVID_FOREVER)) {
9c4cbf82
MK		 1036 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1037 GEN9_GAPS_TSV_CREDIT_DISABLE));
1038 }
1039
d0bbbc4f 1040 /* WaDisablePowerCompilerClockGating:skl */
e87a005d 1041 if (IS_SKL_REVID(dev, SKL_REVID_B0, SKL_REVID_B0))
d0bbbc4f
DL		 1042 WA_SET_BIT_MASKED(HIZ_CHICKEN,
1043 BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE);
1044
e238659d 1045 if (IS_SKL_REVID(dev, 0, SKL_REVID_F0)) {
b62adbd1
NH		 1046 /*
1047 *Use Force Non-Coherent whenever executing a 3D context. This
1048 * is a workaround for a possible hang in the unlikely event
1049 * a TLB invalidation occurs during a PSD flush.
1050 */
1051 /* WaForceEnableNonCoherent:skl */
1052 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1053 HDC_FORCE_NON_COHERENT);
e238659d
MK		 1054
1055 /* WaDisableHDCInvalidation:skl */
1056 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
1057 BDW_DISABLE_HDC_INVALIDATION);
b62adbd1
NH		 1058 }
1059
e87a005d
JN		 1060 /* WaBarrierPerformanceFixDisable:skl */
1061 if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_D0))
5b6fd12a
VS		 1062 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1063 HDC_FENCE_DEST_SLM_DISABLE |
1064 HDC_BARRIER_PERFORMANCE_DISABLE);
1065
9bd9dfb4 1066 /* WaDisableSbeCacheDispatchPortSharing:skl */
e87a005d 1067 if (IS_SKL_REVID(dev, 0, SKL_REVID_F0))
9bd9dfb4
MK		 1068 WA_SET_BIT_MASKED(
1069 GEN7_HALF_SLICE_CHICKEN1,
1070 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
9bd9dfb4 1071
b7668791 1072 return skl_tune_iz_hashing(ring);
7225342a
MK		 1073}
1074
cae0437f
NH		 1075static int bxt_init_workarounds(struct intel_engine_cs *ring)
1076{
aa0011a8 1077 int ret;
dfb601e6
NH		 1078 struct drm_device *dev = ring->dev;
1079 struct drm_i915_private *dev_priv = dev->dev_private;
1080
aa0011a8
AS		 1081 ret = gen9_init_workarounds(ring);
1082 if (ret)
1083 return ret;
cae0437f 1084
9c4cbf82
MK		 1085 /* WaStoreMultiplePTEenable:bxt */
1086 /* This is a requirement according to Hardware specification */
cbdc12a9 1087 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
9c4cbf82
MK		 1088 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
1089
1090 /* WaSetClckGatingDisableMedia:bxt */
cbdc12a9 1091 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
9c4cbf82
MK		 1092 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
1093 ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
1094 }
1095
dfb601e6
NH		 1096 /* WaDisableThreadStallDopClockGating:bxt */
1097 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1098 STALL_DOP_GATING_DISABLE);
1099
983b4b9d 1100 /* WaDisableSbeCacheDispatchPortSharing:bxt */
e87a005d 1101 if (IS_BXT_REVID(dev, 0, BXT_REVID_B0)) {
983b4b9d
NH		 1102 WA_SET_BIT_MASKED(
1103 GEN7_HALF_SLICE_CHICKEN1,
1104 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1105 }
1106
cae0437f
NH		 1107 return 0;
1108}
1109
771b9a53 1110int init_workarounds_ring(struct intel_engine_cs *ring)
7225342a
MK		 1111{
1112 struct drm_device *dev = ring->dev;
1113 struct drm_i915_private *dev_priv = dev->dev_private;
1114
1115 WARN_ON(ring->id != RCS);
1116
1117 dev_priv->workarounds.count = 0;
1118
1119 if (IS_BROADWELL(dev))
1120 return bdw_init_workarounds(ring);
1121
1122 if (IS_CHERRYVIEW(dev))
1123 return chv_init_workarounds(ring);
00e1e623 1124
8d205494
DL		 1125 if (IS_SKYLAKE(dev))
1126 return skl_init_workarounds(ring);
cae0437f
NH		 1127
1128 if (IS_BROXTON(dev))
1129 return bxt_init_workarounds(ring);
3b106531 1130
00e1e623
VS		 1131 return 0;
1132}
1133
a4872ba6 1134static int init_render_ring(struct intel_engine_cs *ring)
8187a2b7 1135{
78501eac 1136 struct drm_device *dev = ring->dev;
1ec14ad3 1137 struct drm_i915_private *dev_priv = dev->dev_private;
78501eac 1138 int ret = init_ring_common(ring);
9c33baa6
KZ		 1139 if (ret)
1140 return ret;
a69ffdbf 1141
61a563a2
AG		 1142 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
1143 if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
6b26c86d 1144 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
1c8c38c5
CW		 1145
1146 /* We need to disable the AsyncFlip performance optimisations in order
1147 * to use MI_WAIT_FOR_EVENT within the CS. It should already be
1148 * programmed to '1' on all products.
8693a824 1149 *
2441f877 1150 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1c8c38c5 1151 */
2441f877 1152 if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8)
1c8c38c5
CW		 1153 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
1154
f05bb0c7 1155 /* Required for the hardware to program scanline values for waiting */
01fa0302 1156 /* WaEnableFlushTlbInvalidationMode:snb */
f05bb0c7
CW		 1157 if (INTEL_INFO(dev)->gen == 6)
1158 I915_WRITE(GFX_MODE,
aa83e30d 1159 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
f05bb0c7 1160
01fa0302 1161 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1c8c38c5
CW		 1162 if (IS_GEN7(dev))
1163 I915_WRITE(GFX_MODE_GEN7,
01fa0302 1164 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
1c8c38c5 1165 _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
78501eac 1166
5e13a0c5 1167 if (IS_GEN6(dev)) {
3a69ddd6
KG		 1168 /* From the Sandybridge PRM, volume 1 part 3, page 24:
1169 * "If this bit is set, STCunit will have LRA as replacement
1170 * policy. [...] This bit must be reset. LRA replacement
1171 * policy is not supported."
1172 */
1173 I915_WRITE(CACHE_MODE_0,
5e13a0c5 1174 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
84f9f938
BW		 1175 }
1176
9cc83020 1177 if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8)
6b26c86d 1178 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
84f9f938 1179
040d2baa 1180 if (HAS_L3_DPF(dev))
35a85ac6 1181 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
15b9f80e 1182
7225342a 1183 return init_workarounds_ring(ring);
8187a2b7
ZN		 1184}
1185
a4872ba6 1186static void render_ring_cleanup(struct intel_engine_cs *ring)
c6df541c 1187{
b45305fc 1188 struct drm_device *dev = ring->dev;
3e78998a
BW		 1189 struct drm_i915_private *dev_priv = dev->dev_private;
1190
1191 if (dev_priv->semaphore_obj) {
1192 i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);
1193 drm_gem_object_unreference(&dev_priv->semaphore_obj->base);
1194 dev_priv->semaphore_obj = NULL;
1195 }
b45305fc 1196
9b1136d5 1197 intel_fini_pipe_control(ring);
c6df541c
CW		 1198}
1199
f7169687 1200static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
3e78998a
BW		 1201 unsigned int num_dwords)
1202{
1203#define MBOX_UPDATE_DWORDS 8
f7169687 1204 struct intel_engine_cs *signaller = signaller_req->ring;
3e78998a
BW		 1205 struct drm_device *dev = signaller->dev;
1206 struct drm_i915_private *dev_priv = dev->dev_private;
1207 struct intel_engine_cs *waiter;
1208 int i, ret, num_rings;
1209
1210 num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
1211 num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
1212#undef MBOX_UPDATE_DWORDS
1213
5fb9de1a 1214 ret = intel_ring_begin(signaller_req, num_dwords);
3e78998a
BW		 1215 if (ret)
1216 return ret;
1217
1218 for_each_ring(waiter, dev_priv, i) {
6259cead 1219 u32 seqno;
3e78998a
BW		 1220 u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
1221 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1222 continue;
1223
f7169687 1224 seqno = i915_gem_request_get_seqno(signaller_req);
3e78998a
BW		 1225 intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
1226 intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB |
1227 PIPE_CONTROL_QW_WRITE |
1228 PIPE_CONTROL_FLUSH_ENABLE);
1229 intel_ring_emit(signaller, lower_32_bits(gtt_offset));
1230 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
6259cead 1231 intel_ring_emit(signaller, seqno);
3e78998a
BW		 1232 intel_ring_emit(signaller, 0);
1233 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
1234 MI_SEMAPHORE_TARGET(waiter->id));
1235 intel_ring_emit(signaller, 0);
1236 }
1237
1238 return 0;
1239}
1240
f7169687 1241static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
3e78998a
BW		 1242 unsigned int num_dwords)
1243{
1244#define MBOX_UPDATE_DWORDS 6
f7169687 1245 struct intel_engine_cs *signaller = signaller_req->ring;
3e78998a
BW		 1246 struct drm_device *dev = signaller->dev;
1247 struct drm_i915_private *dev_priv = dev->dev_private;
1248 struct intel_engine_cs *waiter;
1249 int i, ret, num_rings;
1250
1251 num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
1252 num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
1253#undef MBOX_UPDATE_DWORDS
1254
5fb9de1a 1255 ret = intel_ring_begin(signaller_req, num_dwords);
3e78998a
BW		 1256 if (ret)
1257 return ret;
1258
1259 for_each_ring(waiter, dev_priv, i) {
6259cead 1260 u32 seqno;
3e78998a
BW		 1261 u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
1262 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1263 continue;
1264
f7169687 1265 seqno = i915_gem_request_get_seqno(signaller_req);
3e78998a
BW		 1266 intel_ring_emit(signaller, (MI_FLUSH_DW + 1) |
1267 MI_FLUSH_DW_OP_STOREDW);
1268 intel_ring_emit(signaller, lower_32_bits(gtt_offset) |
1269 MI_FLUSH_DW_USE_GTT);
1270 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
6259cead 1271 intel_ring_emit(signaller, seqno);
3e78998a
BW		 1272 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
1273 MI_SEMAPHORE_TARGET(waiter->id));
1274 intel_ring_emit(signaller, 0);
1275 }
1276
1277 return 0;
1278}
1279
f7169687 1280static int gen6_signal(struct drm_i915_gem_request *signaller_req,
024a43e1 1281 unsigned int num_dwords)
1ec14ad3 1282{
f7169687 1283 struct intel_engine_cs *signaller = signaller_req->ring;
024a43e1
BW		 1284 struct drm_device *dev = signaller->dev;
1285 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 1286 struct intel_engine_cs *useless;
a1444b79 1287 int i, ret, num_rings;
78325f2d 1288
a1444b79
BW		 1289#define MBOX_UPDATE_DWORDS 3
1290 num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
1291 num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
1292#undef MBOX_UPDATE_DWORDS
024a43e1 1293
5fb9de1a 1294 ret = intel_ring_begin(signaller_req, num_dwords);
024a43e1
BW		 1295 if (ret)
1296 return ret;
024a43e1 1297
78325f2d 1298 for_each_ring(useless, dev_priv, i) {
f0f59a00
VS		 1299 i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[i];
1300
1301 if (i915_mmio_reg_valid(mbox_reg)) {
f7169687 1302 u32 seqno = i915_gem_request_get_seqno(signaller_req);
f0f59a00 1303
78325f2d 1304 intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
f92a9162 1305 intel_ring_emit_reg(signaller, mbox_reg);
6259cead 1306 intel_ring_emit(signaller, seqno);
78325f2d
BW		 1307 }
1308 }
024a43e1 1309
a1444b79
BW		 1310 /* If num_dwords was rounded, make sure the tail pointer is correct */
1311 if (num_rings % 2 == 0)
1312 intel_ring_emit(signaller, MI_NOOP);
1313
024a43e1 1314 return 0;
1ec14ad3
CW		 1315}
1316
c8c99b0f
BW		 1317/**
1318 * gen6_add_request - Update the semaphore mailbox registers
ee044a88
JH		 1319 *
1320 * @request - request to write to the ring
c8c99b0f
BW		 1321 *
1322 * Update the mailbox registers in the *other* rings with the current seqno.
1323 * This acts like a signal in the canonical semaphore.
1324 */
1ec14ad3 1325static int
ee044a88 1326gen6_add_request(struct drm_i915_gem_request *req)
1ec14ad3 1327{
ee044a88 1328 struct intel_engine_cs *ring = req->ring;
024a43e1 1329 int ret;
52ed2325 1330
707d9cf9 1331 if (ring->semaphore.signal)
f7169687 1332 ret = ring->semaphore.signal(req, 4);
707d9cf9 1333 else
5fb9de1a 1334 ret = intel_ring_begin(req, 4);
707d9cf9 1335
1ec14ad3
CW		 1336 if (ret)
1337 return ret;
1338
1ec14ad3
CW		 1339 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1340 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
ee044a88 1341 intel_ring_emit(ring, i915_gem_request_get_seqno(req));
1ec14ad3 1342 intel_ring_emit(ring, MI_USER_INTERRUPT);
09246732 1343 __intel_ring_advance(ring);
1ec14ad3 1344
1ec14ad3
CW		 1345 return 0;
1346}
1347
f72b3435
MK		 1348static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
1349 u32 seqno)
1350{
1351 struct drm_i915_private *dev_priv = dev->dev_private;
1352 return dev_priv->last_seqno < seqno;
1353}
1354
c8c99b0f
BW		 1355/**
1356 * intel_ring_sync - sync the waiter to the signaller on seqno
1357 *
1358 * @waiter - ring that is waiting
1359 * @signaller - ring which has, or will signal
1360 * @seqno - seqno which the waiter will block on
1361 */
5ee426ca
BW		 1362
1363static int
599d924c 1364gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
5ee426ca
BW		 1365 struct intel_engine_cs *signaller,
1366 u32 seqno)
1367{
599d924c 1368 struct intel_engine_cs *waiter = waiter_req->ring;
5ee426ca
BW		 1369 struct drm_i915_private *dev_priv = waiter->dev->dev_private;
1370 int ret;
1371
5fb9de1a 1372 ret = intel_ring_begin(waiter_req, 4);
5ee426ca
BW		 1373 if (ret)
1374 return ret;
1375
1376 intel_ring_emit(waiter, MI_SEMAPHORE_WAIT |
1377 MI_SEMAPHORE_GLOBAL_GTT |
bae4fcd2 1378 MI_SEMAPHORE_POLL |
5ee426ca
BW		 1379 MI_SEMAPHORE_SAD_GTE_SDD);
1380 intel_ring_emit(waiter, seqno);
1381 intel_ring_emit(waiter,
1382 lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
1383 intel_ring_emit(waiter,
1384 upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
1385 intel_ring_advance(waiter);
1386 return 0;
1387}
1388
c8c99b0f 1389static int
599d924c 1390gen6_ring_sync(struct drm_i915_gem_request *waiter_req,
a4872ba6 1391 struct intel_engine_cs *signaller,
686cb5f9 1392 u32 seqno)
1ec14ad3 1393{
599d924c 1394 struct intel_engine_cs *waiter = waiter_req->ring;
c8c99b0f
BW		 1395 u32 dw1 = MI_SEMAPHORE_MBOX |
1396 MI_SEMAPHORE_COMPARE |
1397 MI_SEMAPHORE_REGISTER;
ebc348b2
BW		 1398 u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
1399 int ret;
1ec14ad3 1400
1500f7ea
BW		 1401 /* Throughout all of the GEM code, seqno passed implies our current
1402 * seqno is >= the last seqno executed. However for hardware the
1403 * comparison is strictly greater than.
1404 */
1405 seqno -= 1;
1406
ebc348b2 1407 WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
686cb5f9 1408
5fb9de1a 1409 ret = intel_ring_begin(waiter_req, 4);
1ec14ad3
CW		 1410 if (ret)
1411 return ret;
1412
f72b3435
MK		 1413 /* If seqno wrap happened, omit the wait with no-ops */
1414 if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
ebc348b2 1415 intel_ring_emit(waiter, dw1 | wait_mbox);
f72b3435
MK		 1416 intel_ring_emit(waiter, seqno);
1417 intel_ring_emit(waiter, 0);
1418 intel_ring_emit(waiter, MI_NOOP);
1419 } else {
1420 intel_ring_emit(waiter, MI_NOOP);
1421 intel_ring_emit(waiter, MI_NOOP);
1422 intel_ring_emit(waiter, MI_NOOP);
1423 intel_ring_emit(waiter, MI_NOOP);
1424 }
c8c99b0f 1425 intel_ring_advance(waiter);
1ec14ad3
CW		 1426
1427 return 0;
1428}
1429
c6df541c
CW		 1430#define PIPE_CONTROL_FLUSH(ring__, addr__) \
1431do { \
fcbc34e4
KG		 1432 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \
1433 PIPE_CONTROL_DEPTH_STALL); \
c6df541c
CW		 1434 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
1435 intel_ring_emit(ring__, 0); \
1436 intel_ring_emit(ring__, 0); \
1437} while (0)
1438
1439static int
ee044a88 1440pc_render_add_request(struct drm_i915_gem_request *req)
c6df541c 1441{
ee044a88 1442 struct intel_engine_cs *ring = req->ring;
18393f63 1443 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
c6df541c
CW		 1444 int ret;
1445
1446 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
1447 * incoherent with writes to memory, i.e. completely fubar,
1448 * so we need to use PIPE_NOTIFY instead.
1449 *
1450 * However, we also need to workaround the qword write
1451 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
1452 * memory before requesting an interrupt.
1453 */
5fb9de1a 1454 ret = intel_ring_begin(req, 32);
c6df541c
CW		 1455 if (ret)
1456 return ret;
1457
fcbc34e4 1458 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
9d971b37
KG		 1459 PIPE_CONTROL_WRITE_FLUSH |
1460 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
0d1aacac 1461 intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
ee044a88 1462 intel_ring_emit(ring, i915_gem_request_get_seqno(req));
c6df541c
CW		 1463 intel_ring_emit(ring, 0);
1464 PIPE_CONTROL_FLUSH(ring, scratch_addr);
18393f63 1465 scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
c6df541c 1466 PIPE_CONTROL_FLUSH(ring, scratch_addr);
18393f63 1467 scratch_addr += 2 * CACHELINE_BYTES;
c6df541c 1468 PIPE_CONTROL_FLUSH(ring, scratch_addr);
18393f63 1469 scratch_addr += 2 * CACHELINE_BYTES;
c6df541c 1470 PIPE_CONTROL_FLUSH(ring, scratch_addr);
18393f63 1471 scratch_addr += 2 * CACHELINE_BYTES;
c6df541c 1472 PIPE_CONTROL_FLUSH(ring, scratch_addr);
18393f63 1473 scratch_addr += 2 * CACHELINE_BYTES;
c6df541c 1474 PIPE_CONTROL_FLUSH(ring, scratch_addr);
a71d8d94 1475
fcbc34e4 1476 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
9d971b37
KG		 1477 PIPE_CONTROL_WRITE_FLUSH |
1478 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
c6df541c 1479 PIPE_CONTROL_NOTIFY);
0d1aacac 1480 intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
ee044a88 1481 intel_ring_emit(ring, i915_gem_request_get_seqno(req));
c6df541c 1482 intel_ring_emit(ring, 0);
09246732 1483 __intel_ring_advance(ring);
c6df541c 1484
c6df541c
CW		 1485 return 0;
1486}
1487
4cd53c0c 1488static u32
a4872ba6 1489gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
4cd53c0c 1490{
4cd53c0c
DV		 1491 /* Workaround to force correct ordering between irq and seqno writes on
1492 * ivb (and maybe also on snb) by reading from a CS register (like
1493 * ACTHD) before reading the status page. */
50877445
CW		 1494 if (!lazy_coherency) {
1495 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1496 POSTING_READ(RING_ACTHD(ring->mmio_base));
1497 }
1498
4cd53c0c
DV		 1499 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
1500}
1501
8187a2b7 1502static u32
a4872ba6 1503ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
8187a2b7 1504{
1ec14ad3
CW		 1505 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
1506}
1507
b70ec5bf 1508static void
a4872ba6 1509ring_set_seqno(struct intel_engine_cs *ring, u32 seqno)
b70ec5bf
MK		 1510{
1511 intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
1512}
1513
c6df541c 1514static u32
a4872ba6 1515pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
c6df541c 1516{
0d1aacac 1517 return ring->scratch.cpu_page[0];
c6df541c
CW		 1518}
1519
b70ec5bf 1520static void
a4872ba6 1521pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno)
b70ec5bf 1522{
0d1aacac 1523 ring->scratch.cpu_page[0] = seqno;
b70ec5bf
MK		 1524}
1525
e48d8634 1526static bool
a4872ba6 1527gen5_ring_get_irq(struct intel_engine_cs *ring)
e48d8634
DV		 1528{
1529 struct drm_device *dev = ring->dev;
4640c4ff 1530 struct drm_i915_private *dev_priv = dev->dev_private;
7338aefa 1531 unsigned long flags;
e48d8634 1532
7cd512f1 1533 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
e48d8634
DV		 1534 return false;
1535
7338aefa 1536 spin_lock_irqsave(&dev_priv->irq_lock, flags);
43eaea13 1537 if (ring->irq_refcount++ == 0)
480c8033 1538 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
7338aefa 1539 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
e48d8634
DV		 1540
1541 return true;
1542}
1543
1544static void
a4872ba6 1545gen5_ring_put_irq(struct intel_engine_cs *ring)
e48d8634
DV		 1546{
1547 struct drm_device *dev = ring->dev;
4640c4ff 1548 struct drm_i915_private *dev_priv = dev->dev_private;
7338aefa 1549 unsigned long flags;
e48d8634 1550
7338aefa 1551 spin_lock_irqsave(&dev_priv->irq_lock, flags);
43eaea13 1552 if (--ring->irq_refcount == 0)
480c8033 1553 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
7338aefa 1554 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
e48d8634
DV		 1555}
1556
b13c2b96 1557static bool
a4872ba6 1558i9xx_ring_get_irq(struct intel_engine_cs *ring)
62fdfeaf 1559{
78501eac 1560 struct drm_device *dev = ring->dev;
4640c4ff 1561 struct drm_i915_private *dev_priv = dev->dev_private;
7338aefa 1562 unsigned long flags;
62fdfeaf 1563
7cd512f1 1564 if (!intel_irqs_enabled(dev_priv))
b13c2b96
CW		 1565 return false;
1566
7338aefa 1567 spin_lock_irqsave(&dev_priv->irq_lock, flags);
c7113cc3 1568 if (ring->irq_refcount++ == 0) {
f637fde4
DV		 1569 dev_priv->irq_mask &= ~ring->irq_enable_mask;
1570 I915_WRITE(IMR, dev_priv->irq_mask);
1571 POSTING_READ(IMR);
1572 }
7338aefa 1573 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
b13c2b96
CW		 1574
1575 return true;
62fdfeaf
EA		 1576}
1577
8187a2b7 1578static void
a4872ba6 1579i9xx_ring_put_irq(struct intel_engine_cs *ring)
62fdfeaf 1580{
78501eac 1581 struct drm_device *dev = ring->dev;
4640c4ff 1582 struct drm_i915_private *dev_priv = dev->dev_private;
7338aefa 1583 unsigned long flags;
62fdfeaf 1584
7338aefa 1585 spin_lock_irqsave(&dev_priv->irq_lock, flags);
c7113cc3 1586 if (--ring->irq_refcount == 0) {
f637fde4
DV		 1587 dev_priv->irq_mask |= ring->irq_enable_mask;
1588 I915_WRITE(IMR, dev_priv->irq_mask);
1589 POSTING_READ(IMR);
1590 }
7338aefa 1591 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
62fdfeaf
EA		 1592}
1593
c2798b19 1594static bool
a4872ba6 1595i8xx_ring_get_irq(struct intel_engine_cs *ring)
c2798b19
CW		 1596{
1597 struct drm_device *dev = ring->dev;
4640c4ff 1598 struct drm_i915_private *dev_priv = dev->dev_private;
7338aefa 1599 unsigned long flags;
c2798b19 1600
7cd512f1 1601 if (!intel_irqs_enabled(dev_priv))
c2798b19
CW		 1602 return false;
1603
7338aefa 1604 spin_lock_irqsave(&dev_priv->irq_lock, flags);
c7113cc3 1605 if (ring->irq_refcount++ == 0) {
c2798b19
CW		 1606 dev_priv->irq_mask &= ~ring->irq_enable_mask;
1607 I915_WRITE16(IMR, dev_priv->irq_mask);
1608 POSTING_READ16(IMR);
1609 }
7338aefa 1610 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
c2798b19
CW		 1611
1612 return true;
1613}
1614
1615static void
a4872ba6 1616i8xx_ring_put_irq(struct intel_engine_cs *ring)
c2798b19
CW		 1617{
1618 struct drm_device *dev = ring->dev;
4640c4ff 1619 struct drm_i915_private *dev_priv = dev->dev_private;
7338aefa 1620 unsigned long flags;
c2798b19 1621
7338aefa 1622 spin_lock_irqsave(&dev_priv->irq_lock, flags);
c7113cc3 1623 if (--ring->irq_refcount == 0) {
c2798b19
CW		 1624 dev_priv->irq_mask |= ring->irq_enable_mask;
1625 I915_WRITE16(IMR, dev_priv->irq_mask);
1626 POSTING_READ16(IMR);
1627 }
7338aefa 1628 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
c2798b19
CW		 1629}
1630
b72f3acb 1631static int
a84c3ae1 1632bsd_ring_flush(struct drm_i915_gem_request *req,
78501eac
CW		 1633 u32 invalidate_domains,
1634 u32 flush_domains)
d1b851fc 1635{
a84c3ae1 1636 struct intel_engine_cs *ring = req->ring;
b72f3acb
CW		 1637 int ret;
1638
5fb9de1a 1639 ret = intel_ring_begin(req, 2);
b72f3acb
CW		 1640 if (ret)
1641 return ret;
1642
1643 intel_ring_emit(ring, MI_FLUSH);
1644 intel_ring_emit(ring, MI_NOOP);
1645 intel_ring_advance(ring);
1646 return 0;
d1b851fc
ZN		 1647}
1648
3cce469c 1649static int
ee044a88 1650i9xx_add_request(struct drm_i915_gem_request *req)
d1b851fc 1651{
ee044a88 1652 struct intel_engine_cs *ring = req->ring;
3cce469c
CW		 1653 int ret;
1654
5fb9de1a 1655 ret = intel_ring_begin(req, 4);
3cce469c
CW		 1656 if (ret)
1657 return ret;
6f392d54 1658
3cce469c
CW		 1659 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1660 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
ee044a88 1661 intel_ring_emit(ring, i915_gem_request_get_seqno(req));
3cce469c 1662 intel_ring_emit(ring, MI_USER_INTERRUPT);
09246732 1663 __intel_ring_advance(ring);
d1b851fc 1664
3cce469c 1665 return 0;
d1b851fc
ZN		 1666}
1667
0f46832f 1668static bool
a4872ba6 1669gen6_ring_get_irq(struct intel_engine_cs *ring)
0f46832f
CW		 1670{
1671 struct drm_device *dev = ring->dev;
4640c4ff 1672 struct drm_i915_private *dev_priv = dev->dev_private;
7338aefa 1673 unsigned long flags;
0f46832f 1674
7cd512f1
DV		 1675 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1676 return false;
0f46832f 1677
7338aefa 1678 spin_lock_irqsave(&dev_priv->irq_lock, flags);
c7113cc3 1679 if (ring->irq_refcount++ == 0) {
040d2baa 1680 if (HAS_L3_DPF(dev) && ring->id == RCS)
cc609d5d
BW		 1681 I915_WRITE_IMR(ring,
1682 ~(ring->irq_enable_mask |
35a85ac6 1683 GT_PARITY_ERROR(dev)));
15b9f80e
BW		 1684 else
1685 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
480c8033 1686 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
0f46832f 1687 }
7338aefa 1688 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
0f46832f
CW		 1689
1690 return true;
1691}
1692
1693static void
a4872ba6 1694gen6_ring_put_irq(struct intel_engine_cs *ring)
0f46832f
CW		 1695{
1696 struct drm_device *dev = ring->dev;
4640c4ff 1697 struct drm_i915_private *dev_priv = dev->dev_private;
7338aefa 1698 unsigned long flags;
0f46832f 1699
7338aefa 1700 spin_lock_irqsave(&dev_priv->irq_lock, flags);
c7113cc3 1701 if (--ring->irq_refcount == 0) {
040d2baa 1702 if (HAS_L3_DPF(dev) && ring->id == RCS)
35a85ac6 1703 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
15b9f80e
BW		 1704 else
1705 I915_WRITE_IMR(ring, ~0);
480c8033 1706 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
1ec14ad3 1707 }
7338aefa 1708 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
d1b851fc
ZN		 1709}
1710
a19d2933 1711static bool
a4872ba6 1712hsw_vebox_get_irq(struct intel_engine_cs *ring)
a19d2933
BW		 1713{
1714 struct drm_device *dev = ring->dev;
1715 struct drm_i915_private *dev_priv = dev->dev_private;
1716 unsigned long flags;
1717
7cd512f1 1718 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
a19d2933
BW		 1719 return false;
1720
59cdb63d 1721 spin_lock_irqsave(&dev_priv->irq_lock, flags);
c7113cc3 1722 if (ring->irq_refcount++ == 0) {
a19d2933 1723 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
480c8033 1724 gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask);
a19d2933 1725 }
59cdb63d 1726 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
a19d2933
BW		 1727
1728 return true;
1729}
1730
1731static void
a4872ba6 1732hsw_vebox_put_irq(struct intel_engine_cs *ring)
a19d2933
BW		 1733{
1734 struct drm_device *dev = ring->dev;
1735 struct drm_i915_private *dev_priv = dev->dev_private;
1736 unsigned long flags;
1737
59cdb63d 1738 spin_lock_irqsave(&dev_priv->irq_lock, flags);
c7113cc3 1739 if (--ring->irq_refcount == 0) {
a19d2933 1740 I915_WRITE_IMR(ring, ~0);
480c8033 1741 gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask);
a19d2933 1742 }
59cdb63d 1743 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
a19d2933
BW		 1744}
1745
abd58f01 1746static bool
a4872ba6 1747gen8_ring_get_irq(struct intel_engine_cs *ring)
abd58f01
BW		 1748{
1749 struct drm_device *dev = ring->dev;
1750 struct drm_i915_private *dev_priv = dev->dev_private;
1751 unsigned long flags;
1752
7cd512f1 1753 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
abd58f01
BW		 1754 return false;
1755
1756 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1757 if (ring->irq_refcount++ == 0) {
1758 if (HAS_L3_DPF(dev) && ring->id == RCS) {
1759 I915_WRITE_IMR(ring,
1760 ~(ring->irq_enable_mask |
1761 GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
1762 } else {
1763 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1764 }
1765 POSTING_READ(RING_IMR(ring->mmio_base));
1766 }
1767 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1768
1769 return true;
1770}
1771
1772static void
a4872ba6 1773gen8_ring_put_irq(struct intel_engine_cs *ring)
abd58f01
BW		 1774{
1775 struct drm_device *dev = ring->dev;
1776 struct drm_i915_private *dev_priv = dev->dev_private;
1777 unsigned long flags;
1778
1779 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1780 if (--ring->irq_refcount == 0) {
1781 if (HAS_L3_DPF(dev) && ring->id == RCS) {
1782 I915_WRITE_IMR(ring,
1783 ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
1784 } else {
1785 I915_WRITE_IMR(ring, ~0);
1786 }
1787 POSTING_READ(RING_IMR(ring->mmio_base));
1788 }
1789 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1790}
1791
d1b851fc 1792static int
53fddaf7 1793i965_dispatch_execbuffer(struct drm_i915_gem_request *req,
9bcb144c 1794 u64 offset, u32 length,
8e004efc 1795 unsigned dispatch_flags)
d1b851fc 1796{
53fddaf7 1797 struct intel_engine_cs *ring = req->ring;
e1f99ce6 1798 int ret;
78501eac 1799
5fb9de1a 1800 ret = intel_ring_begin(req, 2);
e1f99ce6
CW		 1801 if (ret)
1802 return ret;
1803
78501eac 1804 intel_ring_emit(ring,
65f56876
CW		 1805 MI_BATCH_BUFFER_START |
1806 MI_BATCH_GTT |
8e004efc
JH		 1807 (dispatch_flags & I915_DISPATCH_SECURE ?
1808 0 : MI_BATCH_NON_SECURE_I965));
c4e7a414 1809 intel_ring_emit(ring, offset);
78501eac
CW		 1810 intel_ring_advance(ring);
1811
d1b851fc
ZN		 1812 return 0;
1813}
1814
b45305fc
DV		 1815/* Just userspace ABI convention to limit the wa batch bo to a resonable size */
1816#define I830_BATCH_LIMIT (256*1024)
c4d69da1
CW		 1817#define I830_TLB_ENTRIES (2)
1818#define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
8187a2b7 1819static int
53fddaf7 1820i830_dispatch_execbuffer(struct drm_i915_gem_request *req,
8e004efc
JH		 1821 u64 offset, u32 len,
1822 unsigned dispatch_flags)
62fdfeaf 1823{
53fddaf7 1824 struct intel_engine_cs *ring = req->ring;
c4d69da1 1825 u32 cs_offset = ring->scratch.gtt_offset;
c4e7a414 1826 int ret;
62fdfeaf 1827
5fb9de1a 1828 ret = intel_ring_begin(req, 6);
c4d69da1
CW		 1829 if (ret)
1830 return ret;
62fdfeaf 1831
c4d69da1
CW		 1832 /* Evict the invalid PTE TLBs */
1833 intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA);
1834 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096);
1835 intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); /* load each page */
1836 intel_ring_emit(ring, cs_offset);
1837 intel_ring_emit(ring, 0xdeadbeef);
1838 intel_ring_emit(ring, MI_NOOP);
1839 intel_ring_advance(ring);
b45305fc 1840
8e004efc 1841 if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
b45305fc
DV		 1842 if (len > I830_BATCH_LIMIT)
1843 return -ENOSPC;
1844
5fb9de1a 1845 ret = intel_ring_begin(req, 6 + 2);
b45305fc
DV		 1846 if (ret)
1847 return ret;
c4d69da1
CW		 1848
1849 /* Blit the batch (which has now all relocs applied) to the
1850 * stable batch scratch bo area (so that the CS never
1851 * stumbles over its tlb invalidation bug) ...
1852 */
1853 intel_ring_emit(ring, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA);
1854 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096);
611a7a4f 1855 intel_ring_emit(ring, DIV_ROUND_UP(len, 4096) << 16 | 4096);
b45305fc 1856 intel_ring_emit(ring, cs_offset);
b45305fc
DV		 1857 intel_ring_emit(ring, 4096);
1858 intel_ring_emit(ring, offset);
c4d69da1 1859
b45305fc 1860 intel_ring_emit(ring, MI_FLUSH);
c4d69da1
CW		 1861 intel_ring_emit(ring, MI_NOOP);
1862 intel_ring_advance(ring);
b45305fc
DV		 1863
1864 /* ... and execute it. */
c4d69da1 1865 offset = cs_offset;
b45305fc 1866 }
e1f99ce6 1867
5fb9de1a 1868 ret = intel_ring_begin(req, 4);
c4d69da1
CW		 1869 if (ret)
1870 return ret;
1871
1872 intel_ring_emit(ring, MI_BATCH_BUFFER);
8e004efc
JH		 1873 intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
1874 0 : MI_BATCH_NON_SECURE));
c4d69da1
CW		 1875 intel_ring_emit(ring, offset + len - 8);
1876 intel_ring_emit(ring, MI_NOOP);
1877 intel_ring_advance(ring);
1878
fb3256da
DV		 1879 return 0;
1880}
1881
1882static int
53fddaf7 1883i915_dispatch_execbuffer(struct drm_i915_gem_request *req,
9bcb144c 1884 u64 offset, u32 len,
8e004efc 1885 unsigned dispatch_flags)
fb3256da 1886{
53fddaf7 1887 struct intel_engine_cs *ring = req->ring;
fb3256da
DV		 1888 int ret;
1889
5fb9de1a 1890 ret = intel_ring_begin(req, 2);
fb3256da
DV		 1891 if (ret)
1892 return ret;
1893
65f56876 1894 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
8e004efc
JH		 1895 intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
1896 0 : MI_BATCH_NON_SECURE));
c4e7a414 1897 intel_ring_advance(ring);
62fdfeaf 1898
62fdfeaf
EA		 1899 return 0;
1900}
1901
a4872ba6 1902static void cleanup_status_page(struct intel_engine_cs *ring)
62fdfeaf 1903{
05394f39 1904 struct drm_i915_gem_object *obj;
62fdfeaf 1905
8187a2b7
ZN		 1906 obj = ring->status_page.obj;
1907 if (obj == NULL)
62fdfeaf 1908 return;
62fdfeaf 1909
9da3da66 1910 kunmap(sg_page(obj->pages->sgl));
d7f46fc4 1911 i915_gem_object_ggtt_unpin(obj);
05394f39 1912 drm_gem_object_unreference(&obj->base);
8187a2b7 1913 ring->status_page.obj = NULL;
62fdfeaf
EA		 1914}
1915
a4872ba6 1916static int init_status_page(struct intel_engine_cs *ring)
62fdfeaf 1917{
05394f39 1918 struct drm_i915_gem_object *obj;
62fdfeaf 1919
e3efda49 1920 if ((obj = ring->status_page.obj) == NULL) {
1f767e02 1921 unsigned flags;
e3efda49 1922 int ret;
e4ffd173 1923
e3efda49
CW		 1924 obj = i915_gem_alloc_object(ring->dev, 4096);
1925 if (obj == NULL) {
1926 DRM_ERROR("Failed to allocate status page\n");
1927 return -ENOMEM;
1928 }
62fdfeaf 1929
e3efda49
CW		 1930 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
1931 if (ret)
1932 goto err_unref;
1933
1f767e02
CW		 1934 flags = 0;
1935 if (!HAS_LLC(ring->dev))
1936 /* On g33, we cannot place HWS above 256MiB, so
1937 * restrict its pinning to the low mappable arena.
1938 * Though this restriction is not documented for
1939 * gen4, gen5, or byt, they also behave similarly
1940 * and hang if the HWS is placed at the top of the
1941 * GTT. To generalise, it appears that all !llc
1942 * platforms have issues with us placing the HWS
1943 * above the mappable region (even though we never
1944 * actualy map it).
1945 */
1946 flags |= PIN_MAPPABLE;
1947 ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
e3efda49
CW		 1948 if (ret) {
1949err_unref:
1950 drm_gem_object_unreference(&obj->base);
1951 return ret;
1952 }
1953
1954 ring->status_page.obj = obj;
1955 }
62fdfeaf 1956
f343c5f6 1957 ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
9da3da66 1958 ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
8187a2b7 1959 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
62fdfeaf 1960
8187a2b7
ZN		 1961 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
1962 ring->name, ring->status_page.gfx_addr);
62fdfeaf
EA		 1963
1964 return 0;
62fdfeaf
EA		 1965}
1966
a4872ba6 1967static int init_phys_status_page(struct intel_engine_cs *ring)
6b8294a4
CW		 1968{
1969 struct drm_i915_private *dev_priv = ring->dev->dev_private;
6b8294a4
CW		 1970
1971 if (!dev_priv->status_page_dmah) {
1972 dev_priv->status_page_dmah =
1973 drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
1974 if (!dev_priv->status_page_dmah)
1975 return -ENOMEM;
1976 }
1977
6b8294a4
CW		 1978 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1979 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1980
1981 return 0;
1982}
1983
7ba717cf 1984void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
2919d291 1985{
def0c5f6
CW		 1986 if (HAS_LLC(ringbuf->obj->base.dev) && !ringbuf->obj->stolen)
1987 vunmap(ringbuf->virtual_start);
1988 else
1989 iounmap(ringbuf->virtual_start);
7ba717cf 1990 ringbuf->virtual_start = NULL;
2919d291 1991 i915_gem_object_ggtt_unpin(ringbuf->obj);
7ba717cf
TD		 1992}
1993
def0c5f6
CW		 1994static u32 *vmap_obj(struct drm_i915_gem_object *obj)
1995{
1996 struct sg_page_iter sg_iter;
1997 struct page **pages;
1998 void *addr;
1999 int i;
2000
2001 pages = drm_malloc_ab(obj->base.size >> PAGE_SHIFT, sizeof(*pages));
2002 if (pages == NULL)
2003 return NULL;
2004
2005 i = 0;
2006 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0)
2007 pages[i++] = sg_page_iter_page(&sg_iter);
2008
2009 addr = vmap(pages, i, 0, PAGE_KERNEL);
2010 drm_free_large(pages);
2011
2012 return addr;
2013}
2014
7ba717cf
TD		 2015int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
2016 struct intel_ringbuffer *ringbuf)
2017{
2018 struct drm_i915_private *dev_priv = to_i915(dev);
2019 struct drm_i915_gem_object *obj = ringbuf->obj;
2020 int ret;
2021
def0c5f6
CW		 2022 if (HAS_LLC(dev_priv) && !obj->stolen) {
2023 ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, 0);
2024 if (ret)
2025 return ret;
7ba717cf 2026
def0c5f6
CW		 2027 ret = i915_gem_object_set_to_cpu_domain(obj, true);
2028 if (ret) {
2029 i915_gem_object_ggtt_unpin(obj);
2030 return ret;
2031 }
2032
2033 ringbuf->virtual_start = vmap_obj(obj);
2034 if (ringbuf->virtual_start == NULL) {
2035 i915_gem_object_ggtt_unpin(obj);
2036 return -ENOMEM;
2037 }
2038 } else {
2039 ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
2040 if (ret)
2041 return ret;
7ba717cf 2042
def0c5f6
CW		 2043 ret = i915_gem_object_set_to_gtt_domain(obj, true);
2044 if (ret) {
2045 i915_gem_object_ggtt_unpin(obj);
2046 return ret;
2047 }
2048
2049 ringbuf->virtual_start = ioremap_wc(dev_priv->gtt.mappable_base +
2050 i915_gem_obj_ggtt_offset(obj), ringbuf->size);
2051 if (ringbuf->virtual_start == NULL) {
2052 i915_gem_object_ggtt_unpin(obj);
2053 return -EINVAL;
2054 }
7ba717cf
TD		 2055 }
2056
2057 return 0;
2058}
2059
01101fa7 2060static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
7ba717cf 2061{
2919d291
OM		 2062 drm_gem_object_unreference(&ringbuf->obj->base);
2063 ringbuf->obj = NULL;
2064}
2065
01101fa7
CW		 2066static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
2067 struct intel_ringbuffer *ringbuf)
62fdfeaf 2068{
05394f39 2069 struct drm_i915_gem_object *obj;
62fdfeaf 2070
ebc052e0
CW		 2071 obj = NULL;
2072 if (!HAS_LLC(dev))
93b0a4e0 2073 obj = i915_gem_object_create_stolen(dev, ringbuf->size);
ebc052e0 2074 if (obj == NULL)
93b0a4e0 2075 obj = i915_gem_alloc_object(dev, ringbuf->size);
e3efda49
CW		 2076 if (obj == NULL)
2077 return -ENOMEM;
8187a2b7 2078
24f3a8cf
AG		 2079 /* mark ring buffers as read-only from GPU side by default */
2080 obj->gt_ro = 1;
2081
93b0a4e0 2082 ringbuf->obj = obj;
e3efda49 2083
7ba717cf 2084 return 0;
e3efda49
CW		 2085}
2086
01101fa7
CW		 2087struct intel_ringbuffer *
2088intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
2089{
2090 struct intel_ringbuffer *ring;
2091 int ret;
2092
2093 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
608c1a52
CW		 2094 if (ring == NULL) {
2095 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
2096 engine->name);
01101fa7 2097 return ERR_PTR(-ENOMEM);
608c1a52 2098 }
01101fa7
CW		 2099
2100 ring->ring = engine;
608c1a52 2101 list_add(&ring->link, &engine->buffers);
01101fa7
CW		 2102
2103 ring->size = size;
2104 /* Workaround an erratum on the i830 which causes a hang if
2105 * the TAIL pointer points to within the last 2 cachelines
2106 * of the buffer.
2107 */
2108 ring->effective_size = size;
2109 if (IS_I830(engine->dev) || IS_845G(engine->dev))
2110 ring->effective_size -= 2 * CACHELINE_BYTES;
2111
2112 ring->last_retired_head = -1;
2113 intel_ring_update_space(ring);
2114
2115 ret = intel_alloc_ringbuffer_obj(engine->dev, ring);
2116 if (ret) {
608c1a52
CW		 2117 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n",
2118 engine->name, ret);
2119 list_del(&ring->link);
01101fa7
CW		 2120 kfree(ring);
2121 return ERR_PTR(ret);
2122 }
2123
2124 return ring;
2125}
2126
2127void
2128intel_ringbuffer_free(struct intel_ringbuffer *ring)
2129{
2130 intel_destroy_ringbuffer_obj(ring);
608c1a52 2131 list_del(&ring->link);
01101fa7
CW		 2132 kfree(ring);
2133}
2134
e3efda49 2135static int intel_init_ring_buffer(struct drm_device *dev,
a4872ba6 2136 struct intel_engine_cs *ring)
e3efda49 2137{
bfc882b4 2138 struct intel_ringbuffer *ringbuf;
e3efda49
CW		 2139 int ret;
2140
bfc882b4
DV		 2141 WARN_ON(ring->buffer);
2142
e3efda49
CW		 2143 ring->dev = dev;
2144 INIT_LIST_HEAD(&ring->active_list);
2145 INIT_LIST_HEAD(&ring->request_list);
cc9130be 2146 INIT_LIST_HEAD(&ring->execlist_queue);
608c1a52 2147 INIT_LIST_HEAD(&ring->buffers);
06fbca71 2148 i915_gem_batch_pool_init(dev, &ring->batch_pool);
ebc348b2 2149 memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
e3efda49
CW		 2150
2151 init_waitqueue_head(&ring->irq_queue);
2152
01101fa7 2153 ringbuf = intel_engine_create_ringbuffer(ring, 32 * PAGE_SIZE);
b0366a54
DG		 2154 if (IS_ERR(ringbuf)) {
2155 ret = PTR_ERR(ringbuf);
2156 goto error;
2157 }
01101fa7
CW		 2158 ring->buffer = ringbuf;
2159
e3efda49
CW		 2160 if (I915_NEED_GFX_HWS(dev)) {
2161 ret = init_status_page(ring);
2162 if (ret)
8ee14975 2163 goto error;
e3efda49
CW		 2164 } else {
2165 BUG_ON(ring->id != RCS);
2166 ret = init_phys_status_page(ring);
2167 if (ret)
8ee14975 2168 goto error;
e3efda49
CW		 2169 }
2170
bfc882b4
DV		 2171 ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);
2172 if (ret) {
2173 DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n",
2174 ring->name, ret);
2175 intel_destroy_ringbuffer_obj(ringbuf);
2176 goto error;
e3efda49 2177 }
62fdfeaf 2178
44e895a8
BV		 2179 ret = i915_cmd_parser_init_ring(ring);
2180 if (ret)
8ee14975
OM		 2181 goto error;
2182
8ee14975 2183 return 0;
351e3db2 2184
8ee14975 2185error:
b0366a54 2186 intel_cleanup_ring_buffer(ring);
8ee14975 2187 return ret;
62fdfeaf
EA		 2188}
2189
a4872ba6 2190void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
62fdfeaf 2191{
6402c330 2192 struct drm_i915_private *dev_priv;
33626e6a 2193
93b0a4e0 2194 if (!intel_ring_initialized(ring))
62fdfeaf
EA		 2195 return;
2196
6402c330 2197 dev_priv = to_i915(ring->dev);
6402c330 2198
b0366a54
DG		 2199 if (ring->buffer) {
2200 intel_stop_ring_buffer(ring);
2201 WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
33626e6a 2202
b0366a54
DG		 2203 intel_unpin_ringbuffer_obj(ring->buffer);
2204 intel_ringbuffer_free(ring->buffer);
2205 ring->buffer = NULL;
2206 }
78501eac 2207
8d19215b
ZN		 2208 if (ring->cleanup)
2209 ring->cleanup(ring);
2210
78501eac 2211 cleanup_status_page(ring);
44e895a8
BV		 2212
2213 i915_cmd_parser_fini_ring(ring);
06fbca71 2214 i915_gem_batch_pool_fini(&ring->batch_pool);
b0366a54 2215 ring->dev = NULL;
62fdfeaf
EA		 2216}
2217
595e1eeb 2218static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
a71d8d94 2219{
93b0a4e0 2220 struct intel_ringbuffer *ringbuf = ring->buffer;
a71d8d94 2221 struct drm_i915_gem_request *request;
b4716185
CW		 2222 unsigned space;
2223 int ret;
a71d8d94 2224
ebd0fd4b
DG		 2225 if (intel_ring_space(ringbuf) >= n)
2226 return 0;
a71d8d94 2227
79bbcc29
JH		 2228 /* The whole point of reserving space is to not wait! */
2229 WARN_ON(ringbuf->reserved_in_use);
2230
a71d8d94 2231 list_for_each_entry(request, &ring->request_list, list) {
b4716185
CW		 2232 space = __intel_ring_space(request->postfix, ringbuf->tail,
2233 ringbuf->size);
2234 if (space >= n)
a71d8d94 2235 break;
a71d8d94
CW		 2236 }
2237
595e1eeb 2238 if (WARN_ON(&request->list == &ring->request_list))
a71d8d94
CW		 2239 return -ENOSPC;
2240
a4b3a571 2241 ret = i915_wait_request(request);
a71d8d94
CW		 2242 if (ret)
2243 return ret;
2244
b4716185 2245 ringbuf->space = space;
a71d8d94
CW		 2246 return 0;
2247}
2248
79bbcc29 2249static void __wrap_ring_buffer(struct intel_ringbuffer *ringbuf)
3e960501
CW		 2250{
2251 uint32_t __iomem *virt;
93b0a4e0 2252 int rem = ringbuf->size - ringbuf->tail;
3e960501 2253
93b0a4e0 2254 virt = ringbuf->virtual_start + ringbuf->tail;
3e960501
CW		 2255 rem /= 4;
2256 while (rem--)
2257 iowrite32(MI_NOOP, virt++);
2258
93b0a4e0 2259 ringbuf->tail = 0;
ebd0fd4b 2260 intel_ring_update_space(ringbuf);
3e960501
CW		 2261}
2262
a4872ba6 2263int intel_ring_idle(struct intel_engine_cs *ring)
3e960501 2264{
a4b3a571 2265 struct drm_i915_gem_request *req;
3e960501 2266
3e960501
CW		 2267 /* Wait upon the last request to be completed */
2268 if (list_empty(&ring->request_list))
2269 return 0;
2270
a4b3a571 2271 req = list_entry(ring->request_list.prev,
b4716185
CW		 2272 struct drm_i915_gem_request,
2273 list);
2274
2275 /* Make sure we do not trigger any retires */
2276 return __i915_wait_request(req,
2277 atomic_read(&to_i915(ring->dev)->gpu_error.reset_counter),
2278 to_i915(ring->dev)->mm.interruptible,
2279 NULL, NULL);
3e960501
CW		 2280}
2281
6689cb2b 2282int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request)
9d773091 2283{
6689cb2b 2284 request->ringbuf = request->ring->buffer;
9eba5d4a 2285 return 0;
9d773091
CW		 2286}
2287
ccd98fe4
JH		 2288int intel_ring_reserve_space(struct drm_i915_gem_request *request)
2289{
2290 /*
2291 * The first call merely notes the reserve request and is common for
2292 * all back ends. The subsequent localised _begin() call actually
2293 * ensures that the reservation is available. Without the begin, if
2294 * the request creator immediately submitted the request without
2295 * adding any commands to it then there might not actually be
2296 * sufficient room for the submission commands.
2297 */
2298 intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST);
2299
2300 return intel_ring_begin(request, 0);
2301}
2302
29b1b415
JH		 2303void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size)
2304{
ccd98fe4 2305 WARN_ON(ringbuf->reserved_size);
29b1b415
JH		 2306 WARN_ON(ringbuf->reserved_in_use);
2307
2308 ringbuf->reserved_size = size;
29b1b415
JH		 2309}
2310
2311void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf)
2312{
2313 WARN_ON(ringbuf->reserved_in_use);
2314
2315 ringbuf->reserved_size = 0;
2316 ringbuf->reserved_in_use = false;
2317}
2318
2319void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf)
2320{
2321 WARN_ON(ringbuf->reserved_in_use);
2322
2323 ringbuf->reserved_in_use = true;
2324 ringbuf->reserved_tail = ringbuf->tail;
2325}
2326
2327void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf)
2328{
2329 WARN_ON(!ringbuf->reserved_in_use);
79bbcc29
JH		 2330 if (ringbuf->tail > ringbuf->reserved_tail) {
2331 WARN(ringbuf->tail > ringbuf->reserved_tail + ringbuf->reserved_size,
2332 "request reserved size too small: %d vs %d!\n",
2333 ringbuf->tail - ringbuf->reserved_tail, ringbuf->reserved_size);
2334 } else {
2335 /*
2336 * The ring was wrapped while the reserved space was in use.
2337 * That means that some unknown amount of the ring tail was
2338 * no-op filled and skipped. Thus simply adding the ring size
2339 * to the tail and doing the above space check will not work.
2340 * Rather than attempt to track how much tail was skipped,
2341 * it is much simpler to say that also skipping the sanity
2342 * check every once in a while is not a big issue.
2343 */
2344 }
29b1b415
JH		 2345
2346 ringbuf->reserved_size = 0;
2347 ringbuf->reserved_in_use = false;
2348}
2349
2350static int __intel_ring_prepare(struct intel_engine_cs *ring, int bytes)
cbcc80df 2351{
93b0a4e0 2352 struct intel_ringbuffer *ringbuf = ring->buffer;
79bbcc29
JH		 2353 int remain_usable = ringbuf->effective_size - ringbuf->tail;
2354 int remain_actual = ringbuf->size - ringbuf->tail;
2355 int ret, total_bytes, wait_bytes = 0;
2356 bool need_wrap = false;
29b1b415 2357
79bbcc29
JH		 2358 if (ringbuf->reserved_in_use)
2359 total_bytes = bytes;
2360 else
2361 total_bytes = bytes + ringbuf->reserved_size;
29b1b415 2362
79bbcc29
JH		 2363 if (unlikely(bytes > remain_usable)) {
2364 /*
2365 * Not enough space for the basic request. So need to flush
2366 * out the remainder and then wait for base + reserved.
2367 */
2368 wait_bytes = remain_actual + total_bytes;
2369 need_wrap = true;
2370 } else {
2371 if (unlikely(total_bytes > remain_usable)) {
2372 /*
2373 * The base request will fit but the reserved space
2374 * falls off the end. So only need to to wait for the
2375 * reserved size after flushing out the remainder.
2376 */
2377 wait_bytes = remain_actual + ringbuf->reserved_size;
2378 need_wrap = true;
2379 } else if (total_bytes > ringbuf->space) {
2380 /* No wrapping required, just waiting. */
2381 wait_bytes = total_bytes;
29b1b415 2382 }
cbcc80df
MK		 2383 }
2384
79bbcc29
JH		 2385 if (wait_bytes) {
2386 ret = ring_wait_for_space(ring, wait_bytes);
cbcc80df
MK		 2387 if (unlikely(ret))
2388 return ret;
79bbcc29
JH		 2389
2390 if (need_wrap)
2391 __wrap_ring_buffer(ringbuf);
cbcc80df
MK		 2392 }
2393
cbcc80df
MK		 2394 return 0;
2395}
2396
5fb9de1a 2397int intel_ring_begin(struct drm_i915_gem_request *req,
e1f99ce6 2398 int num_dwords)
8187a2b7 2399{
5fb9de1a
JH		 2400 struct intel_engine_cs *ring;
2401 struct drm_i915_private *dev_priv;
e1f99ce6 2402 int ret;
78501eac 2403
5fb9de1a
JH		 2404 WARN_ON(req == NULL);
2405 ring = req->ring;
2406 dev_priv = ring->dev->dev_private;
2407
33196ded
DV		 2408 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
2409 dev_priv->mm.interruptible);
de2b9985
DV		 2410 if (ret)
2411 return ret;
21dd3734 2412
304d695c
CW		 2413 ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
2414 if (ret)
2415 return ret;
2416
ee1b1e5e 2417 ring->buffer->space -= num_dwords * sizeof(uint32_t);
304d695c 2418 return 0;
8187a2b7 2419}
78501eac 2420
753b1ad4 2421/* Align the ring tail to a cacheline boundary */
bba09b12 2422int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
753b1ad4 2423{
bba09b12 2424 struct intel_engine_cs *ring = req->ring;
ee1b1e5e 2425 int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
753b1ad4
VS		 2426 int ret;
2427
2428 if (num_dwords == 0)
2429 return 0;
2430
18393f63 2431 num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
5fb9de1a 2432 ret = intel_ring_begin(req, num_dwords);
753b1ad4
VS		 2433 if (ret)
2434 return ret;
2435
2436 while (num_dwords--)
2437 intel_ring_emit(ring, MI_NOOP);
2438
2439 intel_ring_advance(ring);
2440
2441 return 0;
2442}
2443
a4872ba6 2444void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno)
498d2ac1 2445{
3b2cc8ab
OM		 2446 struct drm_device *dev = ring->dev;
2447 struct drm_i915_private *dev_priv = dev->dev_private;
498d2ac1 2448
3b2cc8ab 2449 if (INTEL_INFO(dev)->gen == 6 || INTEL_INFO(dev)->gen == 7) {
f7e98ad4
MK		 2450 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
2451 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
3b2cc8ab 2452 if (HAS_VEBOX(dev))
5020150b 2453 I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
e1f99ce6 2454 }
d97ed339 2455
f7e98ad4 2456 ring->set_seqno(ring, seqno);
92cab734 2457 ring->hangcheck.seqno = seqno;
8187a2b7 2458}
62fdfeaf 2459
a4872ba6 2460static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring,
297b0c5b 2461 u32 value)
881f47b6 2462{
4640c4ff 2463 struct drm_i915_private *dev_priv = ring->dev->dev_private;
881f47b6
XH		 2464
2465 /* Every tail move must follow the sequence below */
12f55818
CW		 2466
2467 /* Disable notification that the ring is IDLE. The GT
2468 * will then assume that it is busy and bring it out of rc6.
2469 */
0206e353 2470 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
12f55818
CW		 2471 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2472
2473 /* Clear the context id. Here be magic! */
2474 I915_WRITE64(GEN6_BSD_RNCID, 0x0);
0206e353 2475
12f55818 2476 /* Wait for the ring not to be idle, i.e. for it to wake up. */
0206e353 2477 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
12f55818
CW		 2478 GEN6_BSD_SLEEP_INDICATOR) == 0,
2479 50))
2480 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
0206e353 2481
12f55818 2482 /* Now that the ring is fully powered up, update the tail */
0206e353 2483 I915_WRITE_TAIL(ring, value);
12f55818
CW		 2484 POSTING_READ(RING_TAIL(ring->mmio_base));
2485
2486 /* Let the ring send IDLE messages to the GT again,
2487 * and so let it sleep to conserve power when idle.
2488 */
0206e353 2489 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
12f55818 2490 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
881f47b6
XH		 2491}
2492
a84c3ae1 2493static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
ea251324 2494 u32 invalidate, u32 flush)
881f47b6 2495{
a84c3ae1 2496 struct intel_engine_cs *ring = req->ring;
71a77e07 2497 uint32_t cmd;
b72f3acb
CW		 2498 int ret;
2499
5fb9de1a 2500 ret = intel_ring_begin(req, 4);
b72f3acb
CW		 2501 if (ret)
2502 return ret;
2503
71a77e07 2504 cmd = MI_FLUSH_DW;
075b3bba
BW		 2505 if (INTEL_INFO(ring->dev)->gen >= 8)
2506 cmd += 1;
f0a1fb10
CW		 2507
2508 /* We always require a command barrier so that subsequent
2509 * commands, such as breadcrumb interrupts, are strictly ordered
2510 * wrt the contents of the write cache being flushed to memory
2511 * (and thus being coherent from the CPU).
2512 */
2513 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
2514
9a289771
JB		 2515 /*
2516 * Bspec vol 1c.5 - video engine command streamer:
2517 * "If ENABLED, all TLBs will be invalidated once the flush
2518 * operation is complete. This bit is only valid when the
2519 * Post-Sync Operation field is a value of 1h or 3h."
2520 */
71a77e07 2521 if (invalidate & I915_GEM_GPU_DOMAINS)
f0a1fb10
CW		 2522 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
2523
71a77e07 2524 intel_ring_emit(ring, cmd);
9a289771 2525 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
075b3bba
BW		 2526 if (INTEL_INFO(ring->dev)->gen >= 8) {
2527 intel_ring_emit(ring, 0); /* upper addr */
2528 intel_ring_emit(ring, 0); /* value */
2529 } else {
2530 intel_ring_emit(ring, 0);
2531 intel_ring_emit(ring, MI_NOOP);
2532 }
b72f3acb
CW		 2533 intel_ring_advance(ring);
2534 return 0;
881f47b6
XH		 2535}
2536
1c7a0623 2537static int
53fddaf7 2538gen8_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
9bcb144c 2539 u64 offset, u32 len,
8e004efc 2540 unsigned dispatch_flags)
1c7a0623 2541{
53fddaf7 2542 struct intel_engine_cs *ring = req->ring;
8e004efc
JH		 2543 bool ppgtt = USES_PPGTT(ring->dev) &&
2544 !(dispatch_flags & I915_DISPATCH_SECURE);
1c7a0623
BW		 2545 int ret;
2546
5fb9de1a 2547 ret = intel_ring_begin(req, 4);
1c7a0623
BW		 2548 if (ret)
2549 return ret;
2550
2551 /* FIXME(BDW): Address space and security selectors. */
919032ec
AJ		 2552 intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8) |
2553 (dispatch_flags & I915_DISPATCH_RS ?
2554 MI_BATCH_RESOURCE_STREAMER : 0));
9bcb144c
BW		 2555 intel_ring_emit(ring, lower_32_bits(offset));
2556 intel_ring_emit(ring, upper_32_bits(offset));
1c7a0623
BW		 2557 intel_ring_emit(ring, MI_NOOP);
2558 intel_ring_advance(ring);
2559
2560 return 0;
2561}
2562
d7d4eedd 2563static int
53fddaf7 2564hsw_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
8e004efc
JH		 2565 u64 offset, u32 len,
2566 unsigned dispatch_flags)
d7d4eedd 2567{
53fddaf7 2568 struct intel_engine_cs *ring = req->ring;
d7d4eedd
CW		 2569 int ret;
2570
5fb9de1a 2571 ret = intel_ring_begin(req, 2);
d7d4eedd
CW		 2572 if (ret)
2573 return ret;
2574
2575 intel_ring_emit(ring,
77072258 2576 MI_BATCH_BUFFER_START |
8e004efc 2577 (dispatch_flags & I915_DISPATCH_SECURE ?
919032ec
AJ		 2578 0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW) |
2579 (dispatch_flags & I915_DISPATCH_RS ?
2580 MI_BATCH_RESOURCE_STREAMER : 0));
d7d4eedd
CW		 2581 /* bit0-7 is the length on GEN6+ */
2582 intel_ring_emit(ring, offset);
2583 intel_ring_advance(ring);
2584
2585 return 0;
2586}
2587
881f47b6 2588static int
53fddaf7 2589gen6_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
9bcb144c 2590 u64 offset, u32 len,
8e004efc 2591 unsigned dispatch_flags)
881f47b6 2592{
53fddaf7 2593 struct intel_engine_cs *ring = req->ring;
0206e353 2594 int ret;
ab6f8e32 2595
5fb9de1a 2596 ret = intel_ring_begin(req, 2);
0206e353
AJ		 2597 if (ret)
2598 return ret;
e1f99ce6 2599
d7d4eedd
CW		 2600 intel_ring_emit(ring,
2601 MI_BATCH_BUFFER_START |
8e004efc
JH		 2602 (dispatch_flags & I915_DISPATCH_SECURE ?
2603 0 : MI_BATCH_NON_SECURE_I965));
0206e353
AJ		 2604 /* bit0-7 is the length on GEN6+ */
2605 intel_ring_emit(ring, offset);
2606 intel_ring_advance(ring);
ab6f8e32 2607
0206e353 2608 return 0;
881f47b6
XH		 2609}
2610
549f7365
CW		 2611/* Blitter support (SandyBridge+) */
2612
a84c3ae1 2613static int gen6_ring_flush(struct drm_i915_gem_request *req,
ea251324 2614 u32 invalidate, u32 flush)
8d19215b 2615{
a84c3ae1 2616 struct intel_engine_cs *ring = req->ring;
fd3da6c9 2617 struct drm_device *dev = ring->dev;
71a77e07 2618 uint32_t cmd;
b72f3acb
CW		 2619 int ret;
2620
5fb9de1a 2621 ret = intel_ring_begin(req, 4);
b72f3acb
CW		 2622 if (ret)
2623 return ret;
2624
71a77e07 2625 cmd = MI_FLUSH_DW;
dbef0f15 2626 if (INTEL_INFO(dev)->gen >= 8)
075b3bba 2627 cmd += 1;
f0a1fb10
CW		 2628
2629 /* We always require a command barrier so that subsequent
2630 * commands, such as breadcrumb interrupts, are strictly ordered
2631 * wrt the contents of the write cache being flushed to memory
2632 * (and thus being coherent from the CPU).
2633 */
2634 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
2635
9a289771
JB		 2636 /*
2637 * Bspec vol 1c.3 - blitter engine command streamer:
2638 * "If ENABLED, all TLBs will be invalidated once the flush
2639 * operation is complete. This bit is only valid when the
2640 * Post-Sync Operation field is a value of 1h or 3h."
2641 */
71a77e07 2642 if (invalidate & I915_GEM_DOMAIN_RENDER)
f0a1fb10 2643 cmd |= MI_INVALIDATE_TLB;
71a77e07 2644 intel_ring_emit(ring, cmd);
9a289771 2645 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
dbef0f15 2646 if (INTEL_INFO(dev)->gen >= 8) {
075b3bba
BW		 2647 intel_ring_emit(ring, 0); /* upper addr */
2648 intel_ring_emit(ring, 0); /* value */
2649 } else {
2650 intel_ring_emit(ring, 0);
2651 intel_ring_emit(ring, MI_NOOP);
2652 }
b72f3acb 2653 intel_ring_advance(ring);
fd3da6c9 2654
b72f3acb 2655 return 0;
8d19215b
ZN		 2656}
2657
5c1143bb
XH		 2658int intel_init_render_ring_buffer(struct drm_device *dev)
2659{
4640c4ff 2660 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 2661 struct intel_engine_cs *ring = &dev_priv->ring[RCS];
3e78998a
BW		 2662 struct drm_i915_gem_object *obj;
2663 int ret;
5c1143bb 2664
59465b5f
DV		 2665 ring->name = "render ring";
2666 ring->id = RCS;
2667 ring->mmio_base = RENDER_RING_BASE;
2668
707d9cf9 2669 if (INTEL_INFO(dev)->gen >= 8) {
3e78998a
BW		 2670 if (i915_semaphore_is_enabled(dev)) {
2671 obj = i915_gem_alloc_object(dev, 4096);
2672 if (obj == NULL) {
2673 DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
2674 i915.semaphores = 0;
2675 } else {
2676 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2677 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
2678 if (ret != 0) {
2679 drm_gem_object_unreference(&obj->base);
2680 DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
2681 i915.semaphores = 0;
2682 } else
2683 dev_priv->semaphore_obj = obj;
2684 }
2685 }
7225342a 2686
8f0e2b9d 2687 ring->init_context = intel_rcs_ctx_init;
707d9cf9
BW		 2688 ring->add_request = gen6_add_request;
2689 ring->flush = gen8_render_ring_flush;
2690 ring->irq_get = gen8_ring_get_irq;
2691 ring->irq_put = gen8_ring_put_irq;
2692 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2693 ring->get_seqno = gen6_ring_get_seqno;
2694 ring->set_seqno = ring_set_seqno;
2695 if (i915_semaphore_is_enabled(dev)) {
3e78998a 2696 WARN_ON(!dev_priv->semaphore_obj);
5ee426ca 2697 ring->semaphore.sync_to = gen8_ring_sync;
3e78998a
BW		 2698 ring->semaphore.signal = gen8_rcs_signal;
2699 GEN8_RING_SEMAPHORE_INIT;
707d9cf9
BW		 2700 }
2701 } else if (INTEL_INFO(dev)->gen >= 6) {
4f91fc6d 2702 ring->init_context = intel_rcs_ctx_init;
1ec14ad3 2703 ring->add_request = gen6_add_request;
4772eaeb 2704 ring->flush = gen7_render_ring_flush;
6c6cf5aa 2705 if (INTEL_INFO(dev)->gen == 6)
b3111509 2706 ring->flush = gen6_render_ring_flush;
707d9cf9
BW		 2707 ring->irq_get = gen6_ring_get_irq;
2708 ring->irq_put = gen6_ring_put_irq;
cc609d5d 2709 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
4cd53c0c 2710 ring->get_seqno = gen6_ring_get_seqno;
b70ec5bf 2711 ring->set_seqno = ring_set_seqno;
707d9cf9
BW		 2712 if (i915_semaphore_is_enabled(dev)) {
2713 ring->semaphore.sync_to = gen6_ring_sync;
2714 ring->semaphore.signal = gen6_signal;
2715 /*
2716 * The current semaphore is only applied on pre-gen8
2717 * platform. And there is no VCS2 ring on the pre-gen8
2718 * platform. So the semaphore between RCS and VCS2 is
2719 * initialized as INVALID. Gen8 will initialize the
2720 * sema between VCS2 and RCS later.
2721 */
2722 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
2723 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
2724 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
2725 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
2726 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2727 ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
2728 ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
2729 ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
2730 ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
2731 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2732 }
c6df541c
CW		 2733 } else if (IS_GEN5(dev)) {
2734 ring->add_request = pc_render_add_request;
46f0f8d1 2735 ring->flush = gen4_render_ring_flush;
c6df541c 2736 ring->get_seqno = pc_render_get_seqno;
b70ec5bf 2737 ring->set_seqno = pc_render_set_seqno;
e48d8634
DV		 2738 ring->irq_get = gen5_ring_get_irq;
2739 ring->irq_put = gen5_ring_put_irq;
cc609d5d
BW		 2740 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
2741 GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
59465b5f 2742 } else {
8620a3a9 2743 ring->add_request = i9xx_add_request;
46f0f8d1
CW		 2744 if (INTEL_INFO(dev)->gen < 4)
2745 ring->flush = gen2_render_ring_flush;
2746 else
2747 ring->flush = gen4_render_ring_flush;
59465b5f 2748 ring->get_seqno = ring_get_seqno;
b70ec5bf 2749 ring->set_seqno = ring_set_seqno;
c2798b19
CW		 2750 if (IS_GEN2(dev)) {
2751 ring->irq_get = i8xx_ring_get_irq;
2752 ring->irq_put = i8xx_ring_put_irq;
2753 } else {
2754 ring->irq_get = i9xx_ring_get_irq;
2755 ring->irq_put = i9xx_ring_put_irq;
2756 }
e3670319 2757 ring->irq_enable_mask = I915_USER_INTERRUPT;
1ec14ad3 2758 }
59465b5f 2759 ring->write_tail = ring_write_tail;
707d9cf9 2760
d7d4eedd
CW		 2761 if (IS_HASWELL(dev))
2762 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
1c7a0623
BW		 2763 else if (IS_GEN8(dev))
2764 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
d7d4eedd 2765 else if (INTEL_INFO(dev)->gen >= 6)
fb3256da
DV		 2766 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2767 else if (INTEL_INFO(dev)->gen >= 4)
2768 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
2769 else if (IS_I830(dev) || IS_845G(dev))
2770 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
2771 else
2772 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
ecfe00d8 2773 ring->init_hw = init_render_ring;
59465b5f
DV		 2774 ring->cleanup = render_ring_cleanup;
2775
b45305fc
DV		 2776 /* Workaround batchbuffer to combat CS tlb bug. */
2777 if (HAS_BROKEN_CS_TLB(dev)) {
c4d69da1 2778 obj = i915_gem_alloc_object(dev, I830_WA_SIZE);
b45305fc
DV		 2779 if (obj == NULL) {
2780 DRM_ERROR("Failed to allocate batch bo\n");
2781 return -ENOMEM;
2782 }
2783
be1fa129 2784 ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
b45305fc
DV		 2785 if (ret != 0) {
2786 drm_gem_object_unreference(&obj->base);
2787 DRM_ERROR("Failed to ping batch bo\n");
2788 return ret;
2789 }
2790
0d1aacac
CW		 2791 ring->scratch.obj = obj;
2792 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
b45305fc
DV		 2793 }
2794
99be1dfe
DV		 2795 ret = intel_init_ring_buffer(dev, ring);
2796 if (ret)
2797 return ret;
2798
2799 if (INTEL_INFO(dev)->gen >= 5) {
2800 ret = intel_init_pipe_control(ring);
2801 if (ret)
2802 return ret;
2803 }
2804
2805 return 0;
5c1143bb
XH		 2806}
2807
2808int intel_init_bsd_ring_buffer(struct drm_device *dev)
2809{
4640c4ff 2810 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 2811 struct intel_engine_cs *ring = &dev_priv->ring[VCS];
5c1143bb 2812
58fa3835
DV		 2813 ring->name = "bsd ring";
2814 ring->id = VCS;
2815
0fd2c201 2816 ring->write_tail = ring_write_tail;
780f18c8 2817 if (INTEL_INFO(dev)->gen >= 6) {
58fa3835 2818 ring->mmio_base = GEN6_BSD_RING_BASE;
0fd2c201
DV		 2819 /* gen6 bsd needs a special wa for tail updates */
2820 if (IS_GEN6(dev))
2821 ring->write_tail = gen6_bsd_ring_write_tail;
ea251324 2822 ring->flush = gen6_bsd_ring_flush;
58fa3835
DV		 2823 ring->add_request = gen6_add_request;
2824 ring->get_seqno = gen6_ring_get_seqno;
b70ec5bf 2825 ring->set_seqno = ring_set_seqno;
abd58f01
BW		 2826 if (INTEL_INFO(dev)->gen >= 8) {
2827 ring->irq_enable_mask =
2828 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
2829 ring->irq_get = gen8_ring_get_irq;
2830 ring->irq_put = gen8_ring_put_irq;
1c7a0623
BW		 2831 ring->dispatch_execbuffer =
2832 gen8_ring_dispatch_execbuffer;
707d9cf9 2833 if (i915_semaphore_is_enabled(dev)) {
5ee426ca 2834 ring->semaphore.sync_to = gen8_ring_sync;
3e78998a
BW		 2835 ring->semaphore.signal = gen8_xcs_signal;
2836 GEN8_RING_SEMAPHORE_INIT;
707d9cf9 2837 }
abd58f01
BW		 2838 } else {
2839 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
2840 ring->irq_get = gen6_ring_get_irq;
2841 ring->irq_put = gen6_ring_put_irq;
1c7a0623
BW		 2842 ring->dispatch_execbuffer =
2843 gen6_ring_dispatch_execbuffer;
707d9cf9
BW		 2844 if (i915_semaphore_is_enabled(dev)) {
2845 ring->semaphore.sync_to = gen6_ring_sync;
2846 ring->semaphore.signal = gen6_signal;
2847 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
2848 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
2849 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
2850 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
2851 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2852 ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
2853 ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
2854 ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
2855 ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
2856 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2857 }
abd58f01 2858 }
58fa3835
DV		 2859 } else {
2860 ring->mmio_base = BSD_RING_BASE;
58fa3835 2861 ring->flush = bsd_ring_flush;
8620a3a9 2862 ring->add_request = i9xx_add_request;
58fa3835 2863 ring->get_seqno = ring_get_seqno;
b70ec5bf 2864 ring->set_seqno = ring_set_seqno;
e48d8634 2865 if (IS_GEN5(dev)) {
cc609d5d 2866 ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
e48d8634
DV		 2867 ring->irq_get = gen5_ring_get_irq;
2868 ring->irq_put = gen5_ring_put_irq;
2869 } else {
e3670319 2870 ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
e48d8634
DV		 2871 ring->irq_get = i9xx_ring_get_irq;
2872 ring->irq_put = i9xx_ring_put_irq;
2873 }
fb3256da 2874 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
58fa3835 2875 }
ecfe00d8 2876 ring->init_hw = init_ring_common;
58fa3835 2877
1ec14ad3 2878 return intel_init_ring_buffer(dev, ring);
5c1143bb 2879}
549f7365 2880
845f74a7 2881/**
62659920 2882 * Initialize the second BSD ring (eg. Broadwell GT3, Skylake GT3)
845f74a7
ZY		 2883 */
2884int intel_init_bsd2_ring_buffer(struct drm_device *dev)
2885{
2886 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 2887 struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
845f74a7 2888
f7b64236 2889 ring->name = "bsd2 ring";
845f74a7
ZY		 2890 ring->id = VCS2;
2891
2892 ring->write_tail = ring_write_tail;
2893 ring->mmio_base = GEN8_BSD2_RING_BASE;
2894 ring->flush = gen6_bsd_ring_flush;
2895 ring->add_request = gen6_add_request;
2896 ring->get_seqno = gen6_ring_get_seqno;
2897 ring->set_seqno = ring_set_seqno;
2898 ring->irq_enable_mask =
2899 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
2900 ring->irq_get = gen8_ring_get_irq;
2901 ring->irq_put = gen8_ring_put_irq;
2902 ring->dispatch_execbuffer =
2903 gen8_ring_dispatch_execbuffer;
3e78998a 2904 if (i915_semaphore_is_enabled(dev)) {
5ee426ca 2905 ring->semaphore.sync_to = gen8_ring_sync;
3e78998a
BW		 2906 ring->semaphore.signal = gen8_xcs_signal;
2907 GEN8_RING_SEMAPHORE_INIT;
2908 }
ecfe00d8 2909 ring->init_hw = init_ring_common;
845f74a7
ZY		 2910
2911 return intel_init_ring_buffer(dev, ring);
2912}
2913
549f7365
CW		 2914int intel_init_blt_ring_buffer(struct drm_device *dev)
2915{
4640c4ff 2916 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 2917 struct intel_engine_cs *ring = &dev_priv->ring[BCS];
549f7365 2918
3535d9dd
DV		 2919 ring->name = "blitter ring";
2920 ring->id = BCS;
2921
2922 ring->mmio_base = BLT_RING_BASE;
2923 ring->write_tail = ring_write_tail;
ea251324 2924 ring->flush = gen6_ring_flush;
3535d9dd
DV		 2925 ring->add_request = gen6_add_request;
2926 ring->get_seqno = gen6_ring_get_seqno;
b70ec5bf 2927 ring->set_seqno = ring_set_seqno;
abd58f01
BW		 2928 if (INTEL_INFO(dev)->gen >= 8) {
2929 ring->irq_enable_mask =
2930 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
2931 ring->irq_get = gen8_ring_get_irq;
2932 ring->irq_put = gen8_ring_put_irq;
1c7a0623 2933 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
707d9cf9 2934 if (i915_semaphore_is_enabled(dev)) {
5ee426ca 2935 ring->semaphore.sync_to = gen8_ring_sync;
3e78998a
BW		 2936 ring->semaphore.signal = gen8_xcs_signal;
2937 GEN8_RING_SEMAPHORE_INIT;
707d9cf9 2938 }
abd58f01
BW		 2939 } else {
2940 ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
2941 ring->irq_get = gen6_ring_get_irq;
2942 ring->irq_put = gen6_ring_put_irq;
1c7a0623 2943 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
707d9cf9
BW		 2944 if (i915_semaphore_is_enabled(dev)) {
2945 ring->semaphore.signal = gen6_signal;
2946 ring->semaphore.sync_to = gen6_ring_sync;
2947 /*
2948 * The current semaphore is only applied on pre-gen8
2949 * platform. And there is no VCS2 ring on the pre-gen8
2950 * platform. So the semaphore between BCS and VCS2 is
2951 * initialized as INVALID. Gen8 will initialize the
2952 * sema between BCS and VCS2 later.
2953 */
2954 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
2955 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
2956 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
2957 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
2958 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2959 ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
2960 ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
2961 ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
2962 ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
2963 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2964 }
abd58f01 2965 }
ecfe00d8 2966 ring->init_hw = init_ring_common;
549f7365 2967
1ec14ad3 2968 return intel_init_ring_buffer(dev, ring);
549f7365 2969}
a7b9761d 2970
9a8a2213
BW		 2971int intel_init_vebox_ring_buffer(struct drm_device *dev)
2972{
4640c4ff 2973 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 2974 struct intel_engine_cs *ring = &dev_priv->ring[VECS];
9a8a2213
BW		 2975
2976 ring->name = "video enhancement ring";
2977 ring->id = VECS;
2978
2979 ring->mmio_base = VEBOX_RING_BASE;
2980 ring->write_tail = ring_write_tail;
2981 ring->flush = gen6_ring_flush;
2982 ring->add_request = gen6_add_request;
2983 ring->get_seqno = gen6_ring_get_seqno;
2984 ring->set_seqno = ring_set_seqno;
abd58f01
BW		 2985
2986 if (INTEL_INFO(dev)->gen >= 8) {
2987 ring->irq_enable_mask =
40c499f9 2988 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
abd58f01
BW		 2989 ring->irq_get = gen8_ring_get_irq;
2990 ring->irq_put = gen8_ring_put_irq;
1c7a0623 2991 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
707d9cf9 2992 if (i915_semaphore_is_enabled(dev)) {
5ee426ca 2993 ring->semaphore.sync_to = gen8_ring_sync;
3e78998a
BW		 2994 ring->semaphore.signal = gen8_xcs_signal;
2995 GEN8_RING_SEMAPHORE_INIT;
707d9cf9 2996 }
abd58f01
BW		 2997 } else {
2998 ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
2999 ring->irq_get = hsw_vebox_get_irq;
3000 ring->irq_put = hsw_vebox_put_irq;
1c7a0623 3001 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
707d9cf9
BW		 3002 if (i915_semaphore_is_enabled(dev)) {
3003 ring->semaphore.sync_to = gen6_ring_sync;
3004 ring->semaphore.signal = gen6_signal;
3005 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
3006 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
3007 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
3008 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
3009 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3010 ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
3011 ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
3012 ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
3013 ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
3014 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3015 }
abd58f01 3016 }
ecfe00d8 3017 ring->init_hw = init_ring_common;
9a8a2213
BW		 3018
3019 return intel_init_ring_buffer(dev, ring);
3020}
3021
a7b9761d 3022int
4866d729 3023intel_ring_flush_all_caches(struct drm_i915_gem_request *req)
a7b9761d 3024{
4866d729 3025 struct intel_engine_cs *ring = req->ring;
a7b9761d
CW		 3026 int ret;
3027
3028 if (!ring->gpu_caches_dirty)
3029 return 0;
3030
a84c3ae1 3031 ret = ring->flush(req, 0, I915_GEM_GPU_DOMAINS);
a7b9761d
CW		 3032 if (ret)
3033 return ret;
3034
a84c3ae1 3035 trace_i915_gem_ring_flush(req, 0, I915_GEM_GPU_DOMAINS);
a7b9761d
CW		 3036
3037 ring->gpu_caches_dirty = false;
3038 return 0;
3039}
3040
3041int
2f20055d 3042intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req)
a7b9761d 3043{
2f20055d 3044 struct intel_engine_cs *ring = req->ring;
a7b9761d
CW		 3045 uint32_t flush_domains;
3046 int ret;
3047
3048 flush_domains = 0;
3049 if (ring->gpu_caches_dirty)
3050 flush_domains = I915_GEM_GPU_DOMAINS;
3051
a84c3ae1 3052 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
a7b9761d
CW		 3053 if (ret)
3054 return ret;
3055
a84c3ae1 3056 trace_i915_gem_ring_flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
a7b9761d
CW		 3057
3058 ring->gpu_caches_dirty = false;
3059 return 0;
3060}
e3efda49
CW		 3061
3062void
a4872ba6 3063intel_stop_ring_buffer(struct intel_engine_cs *ring)
e3efda49
CW		 3064{
3065 int ret;
3066
3067 if (!intel_ring_initialized(ring))
3068 return;
3069
3070 ret = intel_ring_idle(ring);
3071 if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
3072 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
3073 ring->name, ret);
3074
3075 stop_ring(ring);
3076}
Linux kernel - Deliverables
This page took 0.661997 seconds and 5 git commands to generate.