projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/i915: Fix VCS2's ring name.
[deliverable/linux.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
1 /*
2 * Copyright © 2008-2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Zou Nan hai <nanhai.zou@intel.com>
26 * Xiang Hai hao<haihao.xiang@intel.com>
27 *
28 */
29
30 #include <drm/drmP.h>
31 #include "i915_drv.h"
32 #include <drm/i915_drm.h>
33 #include "i915_trace.h"
34 #include "intel_drv.h"
35
36 /* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
37 * but keeps the logic simple. Indeed, the whole purpose of this macro is just
38 * to give some inclination as to some of the magic values used in the various
39 * workarounds!
40 */
41 #define CACHELINE_BYTES 64
42
43 static inline int __ring_space(int head, int tail, int size)
44 {
45 int space = head - (tail + I915_RING_FREE_SPACE);
46 if (space < 0)
47 space += size;
48 return space;
49 }
50
51 static inline int ring_space(struct intel_engine_cs *ring)
52 {
53 struct intel_ringbuffer *ringbuf = ring->buffer;
54 return __ring_space(ringbuf->head & HEAD_ADDR, ringbuf->tail, ringbuf->size);
55 }
56
57 static bool intel_ring_stopped(struct intel_engine_cs *ring)
58 {
59 struct drm_i915_private *dev_priv = ring->dev->dev_private;
60 return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);
61 }
62
63 void __intel_ring_advance(struct intel_engine_cs *ring)
64 {
65 struct intel_ringbuffer *ringbuf = ring->buffer;
66 ringbuf->tail &= ringbuf->size - 1;
67 if (intel_ring_stopped(ring))
68 return;
69 ring->write_tail(ring, ringbuf->tail);
70 }
71
72 static int
73 gen2_render_ring_flush(struct intel_engine_cs *ring,
74 u32 invalidate_domains,
75 u32 flush_domains)
76 {
77 u32 cmd;
78 int ret;
79
80 cmd = MI_FLUSH;
81 if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
82 cmd |= MI_NO_WRITE_FLUSH;
83
84 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
85 cmd |= MI_READ_FLUSH;
86
87 ret = intel_ring_begin(ring, 2);
88 if (ret)
89 return ret;
90
91 intel_ring_emit(ring, cmd);
92 intel_ring_emit(ring, MI_NOOP);
93 intel_ring_advance(ring);
94
95 return 0;
96 }
97
98 static int
99 gen4_render_ring_flush(struct intel_engine_cs *ring,
100 u32 invalidate_domains,
101 u32 flush_domains)
102 {
103 struct drm_device *dev = ring->dev;
104 u32 cmd;
105 int ret;
106
107 /*
108 * read/write caches:
109 *
110 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
111 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
112 * also flushed at 2d versus 3d pipeline switches.
113 *
114 * read-only caches:
115 *
116 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
117 * MI_READ_FLUSH is set, and is always flushed on 965.
118 *
119 * I915_GEM_DOMAIN_COMMAND may not exist?
120 *
121 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
122 * invalidated when MI_EXE_FLUSH is set.
123 *
124 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
125 * invalidated with every MI_FLUSH.
126 *
127 * TLBs:
128 *
129 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
130 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
131 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
132 * are flushed at any MI_FLUSH.
133 */
134
135 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
136 if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
137 cmd &= ~MI_NO_WRITE_FLUSH;
138 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
139 cmd |= MI_EXE_FLUSH;
140
141 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
142 (IS_G4X(dev) || IS_GEN5(dev)))
143 cmd |= MI_INVALIDATE_ISP;
144
145 ret = intel_ring_begin(ring, 2);
146 if (ret)
147 return ret;
148
149 intel_ring_emit(ring, cmd);
150 intel_ring_emit(ring, MI_NOOP);
151 intel_ring_advance(ring);
152
153 return 0;
154 }
155
156 /**
157 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
158 * implementing two workarounds on gen6. From section 1.4.7.1
159 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
160 *
161 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
162 * produced by non-pipelined state commands), software needs to first
163 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
164 * 0.
165 *
166 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
167 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
168 *
169 * And the workaround for these two requires this workaround first:
170 *
171 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
172 * BEFORE the pipe-control with a post-sync op and no write-cache
173 * flushes.
174 *
175 * And this last workaround is tricky because of the requirements on
176 * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
177 * volume 2 part 1:
178 *
179 * "1 of the following must also be set:
180 * - Render Target Cache Flush Enable ([12] of DW1)
181 * - Depth Cache Flush Enable ([0] of DW1)
182 * - Stall at Pixel Scoreboard ([1] of DW1)
183 * - Depth Stall ([13] of DW1)
184 * - Post-Sync Operation ([13] of DW1)
185 * - Notify Enable ([8] of DW1)"
186 *
187 * The cache flushes require the workaround flush that triggered this
188 * one, so we can't use it. Depth stall would trigger the same.
189 * Post-sync nonzero is what triggered this second workaround, so we
190 * can't use that one either. Notify enable is IRQs, which aren't
191 * really our business. That leaves only stall at scoreboard.
192 */
193 static int
194 intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring)
195 {
196 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
197 int ret;
198
199
200 ret = intel_ring_begin(ring, 6);
201 if (ret)
202 return ret;
203
204 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
205 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
206 PIPE_CONTROL_STALL_AT_SCOREBOARD);
207 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
208 intel_ring_emit(ring, 0); /* low dword */
209 intel_ring_emit(ring, 0); /* high dword */
210 intel_ring_emit(ring, MI_NOOP);
211 intel_ring_advance(ring);
212
213 ret = intel_ring_begin(ring, 6);
214 if (ret)
215 return ret;
216
217 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
218 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
219 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
220 intel_ring_emit(ring, 0);
221 intel_ring_emit(ring, 0);
222 intel_ring_emit(ring, MI_NOOP);
223 intel_ring_advance(ring);
224
225 return 0;
226 }
227
228 static int
229 gen6_render_ring_flush(struct intel_engine_cs *ring,
230 u32 invalidate_domains, u32 flush_domains)
231 {
232 u32 flags = 0;
233 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
234 int ret;
235
236 /* Force SNB workarounds for PIPE_CONTROL flushes */
237 ret = intel_emit_post_sync_nonzero_flush(ring);
238 if (ret)
239 return ret;
240
241 /* Just flush everything. Experiments have shown that reducing the
242 * number of bits based on the write domains has little performance
243 * impact.
244 */
245 if (flush_domains) {
246 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
247 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
248 /*
249 * Ensure that any following seqno writes only happen
250 * when the render cache is indeed flushed.
251 */
252 flags |= PIPE_CONTROL_CS_STALL;
253 }
254 if (invalidate_domains) {
255 flags |= PIPE_CONTROL_TLB_INVALIDATE;
256 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
257 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
258 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
259 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
260 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
261 /*
262 * TLB invalidate requires a post-sync write.
263 */
264 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
265 }
266
267 ret = intel_ring_begin(ring, 4);
268 if (ret)
269 return ret;
270
271 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
272 intel_ring_emit(ring, flags);
273 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
274 intel_ring_emit(ring, 0);
275 intel_ring_advance(ring);
276
277 return 0;
278 }
279
280 static int
281 gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring)
282 {
283 int ret;
284
285 ret = intel_ring_begin(ring, 4);
286 if (ret)
287 return ret;
288
289 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
290 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
291 PIPE_CONTROL_STALL_AT_SCOREBOARD);
292 intel_ring_emit(ring, 0);
293 intel_ring_emit(ring, 0);
294 intel_ring_advance(ring);
295
296 return 0;
297 }
298
299 static int gen7_ring_fbc_flush(struct intel_engine_cs *ring, u32 value)
300 {
301 int ret;
302
303 if (!ring->fbc_dirty)
304 return 0;
305
306 ret = intel_ring_begin(ring, 6);
307 if (ret)
308 return ret;
309 /* WaFbcNukeOn3DBlt:ivb/hsw */
310 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
311 intel_ring_emit(ring, MSG_FBC_REND_STATE);
312 intel_ring_emit(ring, value);
313 intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT);
314 intel_ring_emit(ring, MSG_FBC_REND_STATE);
315 intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
316 intel_ring_advance(ring);
317
318 ring->fbc_dirty = false;
319 return 0;
320 }
321
322 static int
323 gen7_render_ring_flush(struct intel_engine_cs *ring,
324 u32 invalidate_domains, u32 flush_domains)
325 {
326 u32 flags = 0;
327 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
328 int ret;
329
330 /*
331 * Ensure that any following seqno writes only happen when the render
332 * cache is indeed flushed.
333 *
334 * Workaround: 4th PIPE_CONTROL command (except the ones with only
335 * read-cache invalidate bits set) must have the CS_STALL bit set. We
336 * don't try to be clever and just set it unconditionally.
337 */
338 flags |= PIPE_CONTROL_CS_STALL;
339
340 /* Just flush everything. Experiments have shown that reducing the
341 * number of bits based on the write domains has little performance
342 * impact.
343 */
344 if (flush_domains) {
345 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
346 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
347 }
348 if (invalidate_domains) {
349 flags |= PIPE_CONTROL_TLB_INVALIDATE;
350 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
351 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
352 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
353 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
354 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
355 /*
356 * TLB invalidate requires a post-sync write.
357 */
358 flags |= PIPE_CONTROL_QW_WRITE;
359 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
360
361 /* Workaround: we must issue a pipe_control with CS-stall bit
362 * set before a pipe_control command that has the state cache
363 * invalidate bit set. */
364 gen7_render_ring_cs_stall_wa(ring);
365 }
366
367 ret = intel_ring_begin(ring, 4);
368 if (ret)
369 return ret;
370
371 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
372 intel_ring_emit(ring, flags);
373 intel_ring_emit(ring, scratch_addr);
374 intel_ring_emit(ring, 0);
375 intel_ring_advance(ring);
376
377 if (!invalidate_domains && flush_domains)
378 return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
379
380 return 0;
381 }
382
383 static int
384 gen8_render_ring_flush(struct intel_engine_cs *ring,
385 u32 invalidate_domains, u32 flush_domains)
386 {
387 u32 flags = 0;
388 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
389 int ret;
390
391 flags |= PIPE_CONTROL_CS_STALL;
392
393 if (flush_domains) {
394 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
395 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
396 }
397 if (invalidate_domains) {
398 flags |= PIPE_CONTROL_TLB_INVALIDATE;
399 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
400 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
401 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
402 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
403 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
404 flags |= PIPE_CONTROL_QW_WRITE;
405 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
406 }
407
408 ret = intel_ring_begin(ring, 6);
409 if (ret)
410 return ret;
411
412 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
413 intel_ring_emit(ring, flags);
414 intel_ring_emit(ring, scratch_addr);
415 intel_ring_emit(ring, 0);
416 intel_ring_emit(ring, 0);
417 intel_ring_emit(ring, 0);
418 intel_ring_advance(ring);
419
420 return 0;
421
422 }
423
424 static void ring_write_tail(struct intel_engine_cs *ring,
425 u32 value)
426 {
427 struct drm_i915_private *dev_priv = ring->dev->dev_private;
428 I915_WRITE_TAIL(ring, value);
429 }
430
431 u64 intel_ring_get_active_head(struct intel_engine_cs *ring)
432 {
433 struct drm_i915_private *dev_priv = ring->dev->dev_private;
434 u64 acthd;
435
436 if (INTEL_INFO(ring->dev)->gen >= 8)
437 acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),
438 RING_ACTHD_UDW(ring->mmio_base));
439 else if (INTEL_INFO(ring->dev)->gen >= 4)
440 acthd = I915_READ(RING_ACTHD(ring->mmio_base));
441 else
442 acthd = I915_READ(ACTHD);
443
444 return acthd;
445 }
446
447 static void ring_setup_phys_status_page(struct intel_engine_cs *ring)
448 {
449 struct drm_i915_private *dev_priv = ring->dev->dev_private;
450 u32 addr;
451
452 addr = dev_priv->status_page_dmah->busaddr;
453 if (INTEL_INFO(ring->dev)->gen >= 4)
454 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
455 I915_WRITE(HWS_PGA, addr);
456 }
457
458 static bool stop_ring(struct intel_engine_cs *ring)
459 {
460 struct drm_i915_private *dev_priv = to_i915(ring->dev);
461
462 if (!IS_GEN2(ring->dev)) {
463 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
464 if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
465 DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
466 return false;
467 }
468 }
469
470 I915_WRITE_CTL(ring, 0);
471 I915_WRITE_HEAD(ring, 0);
472 ring->write_tail(ring, 0);
473
474 if (!IS_GEN2(ring->dev)) {
475 (void)I915_READ_CTL(ring);
476 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
477 }
478
479 return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0;
480 }
481
482 static int init_ring_common(struct intel_engine_cs *ring)
483 {
484 struct drm_device *dev = ring->dev;
485 struct drm_i915_private *dev_priv = dev->dev_private;
486 struct intel_ringbuffer *ringbuf = ring->buffer;
487 struct drm_i915_gem_object *obj = ringbuf->obj;
488 int ret = 0;
489
490 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
491
492 if (!stop_ring(ring)) {
493 /* G45 ring initialization often fails to reset head to zero */
494 DRM_DEBUG_KMS("%s head not reset to zero "
495 "ctl %08x head %08x tail %08x start %08x\n",
496 ring->name,
497 I915_READ_CTL(ring),
498 I915_READ_HEAD(ring),
499 I915_READ_TAIL(ring),
500 I915_READ_START(ring));
501
502 if (!stop_ring(ring)) {
503 DRM_ERROR("failed to set %s head to zero "
504 "ctl %08x head %08x tail %08x start %08x\n",
505 ring->name,
506 I915_READ_CTL(ring),
507 I915_READ_HEAD(ring),
508 I915_READ_TAIL(ring),
509 I915_READ_START(ring));
510 ret = -EIO;
511 goto out;
512 }
513 }
514
515 if (I915_NEED_GFX_HWS(dev))
516 intel_ring_setup_status_page(ring);
517 else
518 ring_setup_phys_status_page(ring);
519
520 /* Initialize the ring. This must happen _after_ we've cleared the ring
521 * registers with the above sequence (the readback of the HEAD registers
522 * also enforces ordering), otherwise the hw might lose the new ring
523 * register values. */
524 I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
525 I915_WRITE_CTL(ring,
526 ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
527 | RING_VALID);
528
529 /* If the head is still not zero, the ring is dead */
530 if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
531 I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
532 (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
533 DRM_ERROR("%s initialization failed "
534 "ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
535 ring->name,
536 I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID,
537 I915_READ_HEAD(ring), I915_READ_TAIL(ring),
538 I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj));
539 ret = -EIO;
540 goto out;
541 }
542
543 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
544 i915_kernel_lost_context(ring->dev);
545 else {
546 ringbuf->head = I915_READ_HEAD(ring);
547 ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
548 ringbuf->space = ring_space(ring);
549 ringbuf->last_retired_head = -1;
550 }
551
552 memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
553
554 out:
555 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
556
557 return ret;
558 }
559
560 static int
561 init_pipe_control(struct intel_engine_cs *ring)
562 {
563 int ret;
564
565 if (ring->scratch.obj)
566 return 0;
567
568 ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
569 if (ring->scratch.obj == NULL) {
570 DRM_ERROR("Failed to allocate seqno page\n");
571 ret = -ENOMEM;
572 goto err;
573 }
574
575 ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
576 if (ret)
577 goto err_unref;
578
579 ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0);
580 if (ret)
581 goto err_unref;
582
583 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
584 ring->scratch.cpu_page = kmap(sg_page(ring->scratch.obj->pages->sgl));
585 if (ring->scratch.cpu_page == NULL) {
586 ret = -ENOMEM;
587 goto err_unpin;
588 }
589
590 DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
591 ring->name, ring->scratch.gtt_offset);
592 return 0;
593
594 err_unpin:
595 i915_gem_object_ggtt_unpin(ring->scratch.obj);
596 err_unref:
597 drm_gem_object_unreference(&ring->scratch.obj->base);
598 err:
599 return ret;
600 }
601
602 static int init_render_ring(struct intel_engine_cs *ring)
603 {
604 struct drm_device *dev = ring->dev;
605 struct drm_i915_private *dev_priv = dev->dev_private;
606 int ret = init_ring_common(ring);
607 if (ret)
608 return ret;
609
610 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
611 if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
612 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
613
614 /* We need to disable the AsyncFlip performance optimisations in order
615 * to use MI_WAIT_FOR_EVENT within the CS. It should already be
616 * programmed to '1' on all products.
617 *
618 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
619 */
620 if (INTEL_INFO(dev)->gen >= 6)
621 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
622
623 /* Required for the hardware to program scanline values for waiting */
624 /* WaEnableFlushTlbInvalidationMode:snb */
625 if (INTEL_INFO(dev)->gen == 6)
626 I915_WRITE(GFX_MODE,
627 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
628
629 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
630 if (IS_GEN7(dev))
631 I915_WRITE(GFX_MODE_GEN7,
632 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
633 _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
634
635 if (INTEL_INFO(dev)->gen >= 5) {
636 ret = init_pipe_control(ring);
637 if (ret)
638 return ret;
639 }
640
641 if (IS_GEN6(dev)) {
642 /* From the Sandybridge PRM, volume 1 part 3, page 24:
643 * "If this bit is set, STCunit will have LRA as replacement
644 * policy. [...] This bit must be reset. LRA replacement
645 * policy is not supported."
646 */
647 I915_WRITE(CACHE_MODE_0,
648 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
649 }
650
651 if (INTEL_INFO(dev)->gen >= 6)
652 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
653
654 if (HAS_L3_DPF(dev))
655 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
656
657 return ret;
658 }
659
660 static void render_ring_cleanup(struct intel_engine_cs *ring)
661 {
662 struct drm_device *dev = ring->dev;
663
664 if (ring->scratch.obj == NULL)
665 return;
666
667 if (INTEL_INFO(dev)->gen >= 5) {
668 kunmap(sg_page(ring->scratch.obj->pages->sgl));
669 i915_gem_object_ggtt_unpin(ring->scratch.obj);
670 }
671
672 drm_gem_object_unreference(&ring->scratch.obj->base);
673 ring->scratch.obj = NULL;
674 }
675
676 static int gen6_signal(struct intel_engine_cs *signaller,
677 unsigned int num_dwords)
678 {
679 struct drm_device *dev = signaller->dev;
680 struct drm_i915_private *dev_priv = dev->dev_private;
681 struct intel_engine_cs *useless;
682 int i, ret;
683
684 /* NB: In order to be able to do semaphore MBOX updates for varying
685 * number of rings, it's easiest if we round up each individual update
686 * to a multiple of 2 (since ring updates must always be a multiple of
687 * 2) even though the actual update only requires 3 dwords.
688 */
689 #define MBOX_UPDATE_DWORDS 4
690 if (i915_semaphore_is_enabled(dev))
691 num_dwords += ((I915_NUM_RINGS-1) * MBOX_UPDATE_DWORDS);
692 else
693 return intel_ring_begin(signaller, num_dwords);
694
695 ret = intel_ring_begin(signaller, num_dwords);
696 if (ret)
697 return ret;
698 #undef MBOX_UPDATE_DWORDS
699
700 for_each_ring(useless, dev_priv, i) {
701 u32 mbox_reg = signaller->semaphore.mbox.signal[i];
702 if (mbox_reg != GEN6_NOSYNC) {
703 intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
704 intel_ring_emit(signaller, mbox_reg);
705 intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
706 intel_ring_emit(signaller, MI_NOOP);
707 } else {
708 intel_ring_emit(signaller, MI_NOOP);
709 intel_ring_emit(signaller, MI_NOOP);
710 intel_ring_emit(signaller, MI_NOOP);
711 intel_ring_emit(signaller, MI_NOOP);
712 }
713 }
714
715 return 0;
716 }
717
718 /**
719 * gen6_add_request - Update the semaphore mailbox registers
720 *
721 * @ring - ring that is adding a request
722 * @seqno - return seqno stuck into the ring
723 *
724 * Update the mailbox registers in the *other* rings with the current seqno.
725 * This acts like a signal in the canonical semaphore.
726 */
727 static int
728 gen6_add_request(struct intel_engine_cs *ring)
729 {
730 int ret;
731
732 ret = ring->semaphore.signal(ring, 4);
733 if (ret)
734 return ret;
735
736 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
737 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
738 intel_ring_emit(ring, ring->outstanding_lazy_seqno);
739 intel_ring_emit(ring, MI_USER_INTERRUPT);
740 __intel_ring_advance(ring);
741
742 return 0;
743 }
744
745 static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
746 u32 seqno)
747 {
748 struct drm_i915_private *dev_priv = dev->dev_private;
749 return dev_priv->last_seqno < seqno;
750 }
751
752 /**
753 * intel_ring_sync - sync the waiter to the signaller on seqno
754 *
755 * @waiter - ring that is waiting
756 * @signaller - ring which has, or will signal
757 * @seqno - seqno which the waiter will block on
758 */
759 static int
760 gen6_ring_sync(struct intel_engine_cs *waiter,
761 struct intel_engine_cs *signaller,
762 u32 seqno)
763 {
764 u32 dw1 = MI_SEMAPHORE_MBOX |
765 MI_SEMAPHORE_COMPARE |
766 MI_SEMAPHORE_REGISTER;
767 u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
768 int ret;
769
770 /* Throughout all of the GEM code, seqno passed implies our current
771 * seqno is >= the last seqno executed. However for hardware the
772 * comparison is strictly greater than.
773 */
774 seqno -= 1;
775
776 WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
777
778 ret = intel_ring_begin(waiter, 4);
779 if (ret)
780 return ret;
781
782 /* If seqno wrap happened, omit the wait with no-ops */
783 if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
784 intel_ring_emit(waiter, dw1 | wait_mbox);
785 intel_ring_emit(waiter, seqno);
786 intel_ring_emit(waiter, 0);
787 intel_ring_emit(waiter, MI_NOOP);
788 } else {
789 intel_ring_emit(waiter, MI_NOOP);
790 intel_ring_emit(waiter, MI_NOOP);
791 intel_ring_emit(waiter, MI_NOOP);
792 intel_ring_emit(waiter, MI_NOOP);
793 }
794 intel_ring_advance(waiter);
795
796 return 0;
797 }
798
799 #define PIPE_CONTROL_FLUSH(ring__, addr__) \
800 do { \
801 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \
802 PIPE_CONTROL_DEPTH_STALL); \
803 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
804 intel_ring_emit(ring__, 0); \
805 intel_ring_emit(ring__, 0); \
806 } while (0)
807
808 static int
809 pc_render_add_request(struct intel_engine_cs *ring)
810 {
811 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
812 int ret;
813
814 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
815 * incoherent with writes to memory, i.e. completely fubar,
816 * so we need to use PIPE_NOTIFY instead.
817 *
818 * However, we also need to workaround the qword write
819 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
820 * memory before requesting an interrupt.
821 */
822 ret = intel_ring_begin(ring, 32);
823 if (ret)
824 return ret;
825
826 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
827 PIPE_CONTROL_WRITE_FLUSH |
828 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
829 intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
830 intel_ring_emit(ring, ring->outstanding_lazy_seqno);
831 intel_ring_emit(ring, 0);
832 PIPE_CONTROL_FLUSH(ring, scratch_addr);
833 scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
834 PIPE_CONTROL_FLUSH(ring, scratch_addr);
835 scratch_addr += 2 * CACHELINE_BYTES;
836 PIPE_CONTROL_FLUSH(ring, scratch_addr);
837 scratch_addr += 2 * CACHELINE_BYTES;
838 PIPE_CONTROL_FLUSH(ring, scratch_addr);
839 scratch_addr += 2 * CACHELINE_BYTES;
840 PIPE_CONTROL_FLUSH(ring, scratch_addr);
841 scratch_addr += 2 * CACHELINE_BYTES;
842 PIPE_CONTROL_FLUSH(ring, scratch_addr);
843
844 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
845 PIPE_CONTROL_WRITE_FLUSH |
846 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
847 PIPE_CONTROL_NOTIFY);
848 intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
849 intel_ring_emit(ring, ring->outstanding_lazy_seqno);
850 intel_ring_emit(ring, 0);
851 __intel_ring_advance(ring);
852
853 return 0;
854 }
855
856 static u32
857 gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
858 {
859 /* Workaround to force correct ordering between irq and seqno writes on
860 * ivb (and maybe also on snb) by reading from a CS register (like
861 * ACTHD) before reading the status page. */
862 if (!lazy_coherency) {
863 struct drm_i915_private *dev_priv = ring->dev->dev_private;
864 POSTING_READ(RING_ACTHD(ring->mmio_base));
865 }
866
867 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
868 }
869
870 static u32
871 ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
872 {
873 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
874 }
875
876 static void
877 ring_set_seqno(struct intel_engine_cs *ring, u32 seqno)
878 {
879 intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
880 }
881
882 static u32
883 pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
884 {
885 return ring->scratch.cpu_page[0];
886 }
887
888 static void
889 pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno)
890 {
891 ring->scratch.cpu_page[0] = seqno;
892 }
893
894 static bool
895 gen5_ring_get_irq(struct intel_engine_cs *ring)
896 {
897 struct drm_device *dev = ring->dev;
898 struct drm_i915_private *dev_priv = dev->dev_private;
899 unsigned long flags;
900
901 if (!dev->irq_enabled)
902 return false;
903
904 spin_lock_irqsave(&dev_priv->irq_lock, flags);
905 if (ring->irq_refcount++ == 0)
906 ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask);
907 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
908
909 return true;
910 }
911
912 static void
913 gen5_ring_put_irq(struct intel_engine_cs *ring)
914 {
915 struct drm_device *dev = ring->dev;
916 struct drm_i915_private *dev_priv = dev->dev_private;
917 unsigned long flags;
918
919 spin_lock_irqsave(&dev_priv->irq_lock, flags);
920 if (--ring->irq_refcount == 0)
921 ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask);
922 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
923 }
924
925 static bool
926 i9xx_ring_get_irq(struct intel_engine_cs *ring)
927 {
928 struct drm_device *dev = ring->dev;
929 struct drm_i915_private *dev_priv = dev->dev_private;
930 unsigned long flags;
931
932 if (!dev->irq_enabled)
933 return false;
934
935 spin_lock_irqsave(&dev_priv->irq_lock, flags);
936 if (ring->irq_refcount++ == 0) {
937 dev_priv->irq_mask &= ~ring->irq_enable_mask;
938 I915_WRITE(IMR, dev_priv->irq_mask);
939 POSTING_READ(IMR);
940 }
941 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
942
943 return true;
944 }
945
946 static void
947 i9xx_ring_put_irq(struct intel_engine_cs *ring)
948 {
949 struct drm_device *dev = ring->dev;
950 struct drm_i915_private *dev_priv = dev->dev_private;
951 unsigned long flags;
952
953 spin_lock_irqsave(&dev_priv->irq_lock, flags);
954 if (--ring->irq_refcount == 0) {
955 dev_priv->irq_mask |= ring->irq_enable_mask;
956 I915_WRITE(IMR, dev_priv->irq_mask);
957 POSTING_READ(IMR);
958 }
959 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
960 }
961
962 static bool
963 i8xx_ring_get_irq(struct intel_engine_cs *ring)
964 {
965 struct drm_device *dev = ring->dev;
966 struct drm_i915_private *dev_priv = dev->dev_private;
967 unsigned long flags;
968
969 if (!dev->irq_enabled)
970 return false;
971
972 spin_lock_irqsave(&dev_priv->irq_lock, flags);
973 if (ring->irq_refcount++ == 0) {
974 dev_priv->irq_mask &= ~ring->irq_enable_mask;
975 I915_WRITE16(IMR, dev_priv->irq_mask);
976 POSTING_READ16(IMR);
977 }
978 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
979
980 return true;
981 }
982
983 static void
984 i8xx_ring_put_irq(struct intel_engine_cs *ring)
985 {
986 struct drm_device *dev = ring->dev;
987 struct drm_i915_private *dev_priv = dev->dev_private;
988 unsigned long flags;
989
990 spin_lock_irqsave(&dev_priv->irq_lock, flags);
991 if (--ring->irq_refcount == 0) {
992 dev_priv->irq_mask |= ring->irq_enable_mask;
993 I915_WRITE16(IMR, dev_priv->irq_mask);
994 POSTING_READ16(IMR);
995 }
996 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
997 }
998
999 void intel_ring_setup_status_page(struct intel_engine_cs *ring)
1000 {
1001 struct drm_device *dev = ring->dev;
1002 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1003 u32 mmio = 0;
1004
1005 /* The ring status page addresses are no longer next to the rest of
1006 * the ring registers as of gen7.
1007 */
1008 if (IS_GEN7(dev)) {
1009 switch (ring->id) {
1010 case RCS:
1011 mmio = RENDER_HWS_PGA_GEN7;
1012 break;
1013 case BCS:
1014 mmio = BLT_HWS_PGA_GEN7;
1015 break;
1016 /*
1017 * VCS2 actually doesn't exist on Gen7. Only shut up
1018 * gcc switch check warning
1019 */
1020 case VCS2:
1021 case VCS:
1022 mmio = BSD_HWS_PGA_GEN7;
1023 break;
1024 case VECS:
1025 mmio = VEBOX_HWS_PGA_GEN7;
1026 break;
1027 }
1028 } else if (IS_GEN6(ring->dev)) {
1029 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
1030 } else {
1031 /* XXX: gen8 returns to sanity */
1032 mmio = RING_HWS_PGA(ring->mmio_base);
1033 }
1034
1035 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
1036 POSTING_READ(mmio);
1037
1038 /*
1039 * Flush the TLB for this page
1040 *
1041 * FIXME: These two bits have disappeared on gen8, so a question
1042 * arises: do we still need this and if so how should we go about
1043 * invalidating the TLB?
1044 */
1045 if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
1046 u32 reg = RING_INSTPM(ring->mmio_base);
1047
1048 /* ring should be idle before issuing a sync flush*/
1049 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
1050
1051 I915_WRITE(reg,
1052 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
1053 INSTPM_SYNC_FLUSH));
1054 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
1055 1000))
1056 DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
1057 ring->name);
1058 }
1059 }
1060
1061 static int
1062 bsd_ring_flush(struct intel_engine_cs *ring,
1063 u32 invalidate_domains,
1064 u32 flush_domains)
1065 {
1066 int ret;
1067
1068 ret = intel_ring_begin(ring, 2);
1069 if (ret)
1070 return ret;
1071
1072 intel_ring_emit(ring, MI_FLUSH);
1073 intel_ring_emit(ring, MI_NOOP);
1074 intel_ring_advance(ring);
1075 return 0;
1076 }
1077
1078 static int
1079 i9xx_add_request(struct intel_engine_cs *ring)
1080 {
1081 int ret;
1082
1083 ret = intel_ring_begin(ring, 4);
1084 if (ret)
1085 return ret;
1086
1087 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1088 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1089 intel_ring_emit(ring, ring->outstanding_lazy_seqno);
1090 intel_ring_emit(ring, MI_USER_INTERRUPT);
1091 __intel_ring_advance(ring);
1092
1093 return 0;
1094 }
1095
1096 static bool
1097 gen6_ring_get_irq(struct intel_engine_cs *ring)
1098 {
1099 struct drm_device *dev = ring->dev;
1100 struct drm_i915_private *dev_priv = dev->dev_private;
1101 unsigned long flags;
1102
1103 if (!dev->irq_enabled)
1104 return false;
1105
1106 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1107 if (ring->irq_refcount++ == 0) {
1108 if (HAS_L3_DPF(dev) && ring->id == RCS)
1109 I915_WRITE_IMR(ring,
1110 ~(ring->irq_enable_mask |
1111 GT_PARITY_ERROR(dev)));
1112 else
1113 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1114 ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask);
1115 }
1116 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1117
1118 return true;
1119 }
1120
1121 static void
1122 gen6_ring_put_irq(struct intel_engine_cs *ring)
1123 {
1124 struct drm_device *dev = ring->dev;
1125 struct drm_i915_private *dev_priv = dev->dev_private;
1126 unsigned long flags;
1127
1128 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1129 if (--ring->irq_refcount == 0) {
1130 if (HAS_L3_DPF(dev) && ring->id == RCS)
1131 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
1132 else
1133 I915_WRITE_IMR(ring, ~0);
1134 ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask);
1135 }
1136 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1137 }
1138
1139 static bool
1140 hsw_vebox_get_irq(struct intel_engine_cs *ring)
1141 {
1142 struct drm_device *dev = ring->dev;
1143 struct drm_i915_private *dev_priv = dev->dev_private;
1144 unsigned long flags;
1145
1146 if (!dev->irq_enabled)
1147 return false;
1148
1149 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1150 if (ring->irq_refcount++ == 0) {
1151 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1152 snb_enable_pm_irq(dev_priv, ring->irq_enable_mask);
1153 }
1154 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1155
1156 return true;
1157 }
1158
1159 static void
1160 hsw_vebox_put_irq(struct intel_engine_cs *ring)
1161 {
1162 struct drm_device *dev = ring->dev;
1163 struct drm_i915_private *dev_priv = dev->dev_private;
1164 unsigned long flags;
1165
1166 if (!dev->irq_enabled)
1167 return;
1168
1169 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1170 if (--ring->irq_refcount == 0) {
1171 I915_WRITE_IMR(ring, ~0);
1172 snb_disable_pm_irq(dev_priv, ring->irq_enable_mask);
1173 }
1174 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1175 }
1176
1177 static bool
1178 gen8_ring_get_irq(struct intel_engine_cs *ring)
1179 {
1180 struct drm_device *dev = ring->dev;
1181 struct drm_i915_private *dev_priv = dev->dev_private;
1182 unsigned long flags;
1183
1184 if (!dev->irq_enabled)
1185 return false;
1186
1187 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1188 if (ring->irq_refcount++ == 0) {
1189 if (HAS_L3_DPF(dev) && ring->id == RCS) {
1190 I915_WRITE_IMR(ring,
1191 ~(ring->irq_enable_mask |
1192 GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
1193 } else {
1194 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1195 }
1196 POSTING_READ(RING_IMR(ring->mmio_base));
1197 }
1198 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1199
1200 return true;
1201 }
1202
1203 static void
1204 gen8_ring_put_irq(struct intel_engine_cs *ring)
1205 {
1206 struct drm_device *dev = ring->dev;
1207 struct drm_i915_private *dev_priv = dev->dev_private;
1208 unsigned long flags;
1209
1210 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1211 if (--ring->irq_refcount == 0) {
1212 if (HAS_L3_DPF(dev) && ring->id == RCS) {
1213 I915_WRITE_IMR(ring,
1214 ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
1215 } else {
1216 I915_WRITE_IMR(ring, ~0);
1217 }
1218 POSTING_READ(RING_IMR(ring->mmio_base));
1219 }
1220 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1221 }
1222
1223 static int
1224 i965_dispatch_execbuffer(struct intel_engine_cs *ring,
1225 u64 offset, u32 length,
1226 unsigned flags)
1227 {
1228 int ret;
1229
1230 ret = intel_ring_begin(ring, 2);
1231 if (ret)
1232 return ret;
1233
1234 intel_ring_emit(ring,
1235 MI_BATCH_BUFFER_START |
1236 MI_BATCH_GTT |
1237 (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
1238 intel_ring_emit(ring, offset);
1239 intel_ring_advance(ring);
1240
1241 return 0;
1242 }
1243
1244 /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
1245 #define I830_BATCH_LIMIT (256*1024)
1246 static int
1247 i830_dispatch_execbuffer(struct intel_engine_cs *ring,
1248 u64 offset, u32 len,
1249 unsigned flags)
1250 {
1251 int ret;
1252
1253 if (flags & I915_DISPATCH_PINNED) {
1254 ret = intel_ring_begin(ring, 4);
1255 if (ret)
1256 return ret;
1257
1258 intel_ring_emit(ring, MI_BATCH_BUFFER);
1259 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1260 intel_ring_emit(ring, offset + len - 8);
1261 intel_ring_emit(ring, MI_NOOP);
1262 intel_ring_advance(ring);
1263 } else {
1264 u32 cs_offset = ring->scratch.gtt_offset;
1265
1266 if (len > I830_BATCH_LIMIT)
1267 return -ENOSPC;
1268
1269 ret = intel_ring_begin(ring, 9+3);
1270 if (ret)
1271 return ret;
1272 /* Blit the batch (which has now all relocs applied) to the stable batch
1273 * scratch bo area (so that the CS never stumbles over its tlb
1274 * invalidation bug) ... */
1275 intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD |
1276 XY_SRC_COPY_BLT_WRITE_ALPHA |
1277 XY_SRC_COPY_BLT_WRITE_RGB);
1278 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_GXCOPY | 4096);
1279 intel_ring_emit(ring, 0);
1280 intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) | 1024);
1281 intel_ring_emit(ring, cs_offset);
1282 intel_ring_emit(ring, 0);
1283 intel_ring_emit(ring, 4096);
1284 intel_ring_emit(ring, offset);
1285 intel_ring_emit(ring, MI_FLUSH);
1286
1287 /* ... and execute it. */
1288 intel_ring_emit(ring, MI_BATCH_BUFFER);
1289 intel_ring_emit(ring, cs_offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1290 intel_ring_emit(ring, cs_offset + len - 8);
1291 intel_ring_advance(ring);
1292 }
1293
1294 return 0;
1295 }
1296
1297 static int
1298 i915_dispatch_execbuffer(struct intel_engine_cs *ring,
1299 u64 offset, u32 len,
1300 unsigned flags)
1301 {
1302 int ret;
1303
1304 ret = intel_ring_begin(ring, 2);
1305 if (ret)
1306 return ret;
1307
1308 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
1309 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1310 intel_ring_advance(ring);
1311
1312 return 0;
1313 }
1314
1315 static void cleanup_status_page(struct intel_engine_cs *ring)
1316 {
1317 struct drm_i915_gem_object *obj;
1318
1319 obj = ring->status_page.obj;
1320 if (obj == NULL)
1321 return;
1322
1323 kunmap(sg_page(obj->pages->sgl));
1324 i915_gem_object_ggtt_unpin(obj);
1325 drm_gem_object_unreference(&obj->base);
1326 ring->status_page.obj = NULL;
1327 }
1328
1329 static int init_status_page(struct intel_engine_cs *ring)
1330 {
1331 struct drm_i915_gem_object *obj;
1332
1333 if ((obj = ring->status_page.obj) == NULL) {
1334 int ret;
1335
1336 obj = i915_gem_alloc_object(ring->dev, 4096);
1337 if (obj == NULL) {
1338 DRM_ERROR("Failed to allocate status page\n");
1339 return -ENOMEM;
1340 }
1341
1342 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
1343 if (ret)
1344 goto err_unref;
1345
1346 ret = i915_gem_obj_ggtt_pin(obj, 4096, 0);
1347 if (ret) {
1348 err_unref:
1349 drm_gem_object_unreference(&obj->base);
1350 return ret;
1351 }
1352
1353 ring->status_page.obj = obj;
1354 }
1355
1356 ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
1357 ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
1358 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1359
1360 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
1361 ring->name, ring->status_page.gfx_addr);
1362
1363 return 0;
1364 }
1365
1366 static int init_phys_status_page(struct intel_engine_cs *ring)
1367 {
1368 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1369
1370 if (!dev_priv->status_page_dmah) {
1371 dev_priv->status_page_dmah =
1372 drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
1373 if (!dev_priv->status_page_dmah)
1374 return -ENOMEM;
1375 }
1376
1377 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1378 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1379
1380 return 0;
1381 }
1382
1383 static int allocate_ring_buffer(struct intel_engine_cs *ring)
1384 {
1385 struct drm_device *dev = ring->dev;
1386 struct drm_i915_private *dev_priv = to_i915(dev);
1387 struct intel_ringbuffer *ringbuf = ring->buffer;
1388 struct drm_i915_gem_object *obj;
1389 int ret;
1390
1391 if (intel_ring_initialized(ring))
1392 return 0;
1393
1394 obj = NULL;
1395 if (!HAS_LLC(dev))
1396 obj = i915_gem_object_create_stolen(dev, ringbuf->size);
1397 if (obj == NULL)
1398 obj = i915_gem_alloc_object(dev, ringbuf->size);
1399 if (obj == NULL)
1400 return -ENOMEM;
1401
1402 /* mark ring buffers as read-only from GPU side by default */
1403 obj->gt_ro = 1;
1404
1405 ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
1406 if (ret)
1407 goto err_unref;
1408
1409 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1410 if (ret)
1411 goto err_unpin;
1412
1413 ringbuf->virtual_start =
1414 ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj),
1415 ringbuf->size);
1416 if (ringbuf->virtual_start == NULL) {
1417 ret = -EINVAL;
1418 goto err_unpin;
1419 }
1420
1421 ringbuf->obj = obj;
1422 return 0;
1423
1424 err_unpin:
1425 i915_gem_object_ggtt_unpin(obj);
1426 err_unref:
1427 drm_gem_object_unreference(&obj->base);
1428 return ret;
1429 }
1430
1431 static int intel_init_ring_buffer(struct drm_device *dev,
1432 struct intel_engine_cs *ring)
1433 {
1434 struct intel_ringbuffer *ringbuf = ring->buffer;
1435 int ret;
1436
1437 if (ringbuf == NULL) {
1438 ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
1439 if (!ringbuf)
1440 return -ENOMEM;
1441 ring->buffer = ringbuf;
1442 }
1443
1444 ring->dev = dev;
1445 INIT_LIST_HEAD(&ring->active_list);
1446 INIT_LIST_HEAD(&ring->request_list);
1447 ringbuf->size = 32 * PAGE_SIZE;
1448 memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
1449
1450 init_waitqueue_head(&ring->irq_queue);
1451
1452 if (I915_NEED_GFX_HWS(dev)) {
1453 ret = init_status_page(ring);
1454 if (ret)
1455 goto error;
1456 } else {
1457 BUG_ON(ring->id != RCS);
1458 ret = init_phys_status_page(ring);
1459 if (ret)
1460 goto error;
1461 }
1462
1463 ret = allocate_ring_buffer(ring);
1464 if (ret) {
1465 DRM_ERROR("Failed to allocate ringbuffer %s: %d\n", ring->name, ret);
1466 goto error;
1467 }
1468
1469 /* Workaround an erratum on the i830 which causes a hang if
1470 * the TAIL pointer points to within the last 2 cachelines
1471 * of the buffer.
1472 */
1473 ringbuf->effective_size = ringbuf->size;
1474 if (IS_I830(dev) || IS_845G(dev))
1475 ringbuf->effective_size -= 2 * CACHELINE_BYTES;
1476
1477 ret = i915_cmd_parser_init_ring(ring);
1478 if (ret)
1479 goto error;
1480
1481 ret = ring->init(ring);
1482 if (ret)
1483 goto error;
1484
1485 return 0;
1486
1487 error:
1488 kfree(ringbuf);
1489 ring->buffer = NULL;
1490 return ret;
1491 }
1492
1493 void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
1494 {
1495 struct drm_i915_private *dev_priv = to_i915(ring->dev);
1496 struct intel_ringbuffer *ringbuf = ring->buffer;
1497
1498 if (!intel_ring_initialized(ring))
1499 return;
1500
1501 intel_stop_ring_buffer(ring);
1502 WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
1503
1504 iounmap(ringbuf->virtual_start);
1505
1506 i915_gem_object_ggtt_unpin(ringbuf->obj);
1507 drm_gem_object_unreference(&ringbuf->obj->base);
1508 ringbuf->obj = NULL;
1509 ring->preallocated_lazy_request = NULL;
1510 ring->outstanding_lazy_seqno = 0;
1511
1512 if (ring->cleanup)
1513 ring->cleanup(ring);
1514
1515 cleanup_status_page(ring);
1516
1517 i915_cmd_parser_fini_ring(ring);
1518
1519 kfree(ringbuf);
1520 ring->buffer = NULL;
1521 }
1522
1523 static int intel_ring_wait_request(struct intel_engine_cs *ring, int n)
1524 {
1525 struct intel_ringbuffer *ringbuf = ring->buffer;
1526 struct drm_i915_gem_request *request;
1527 u32 seqno = 0;
1528 int ret;
1529
1530 if (ringbuf->last_retired_head != -1) {
1531 ringbuf->head = ringbuf->last_retired_head;
1532 ringbuf->last_retired_head = -1;
1533
1534 ringbuf->space = ring_space(ring);
1535 if (ringbuf->space >= n)
1536 return 0;
1537 }
1538
1539 list_for_each_entry(request, &ring->request_list, list) {
1540 if (__ring_space(request->tail, ringbuf->tail, ringbuf->size) >= n) {
1541 seqno = request->seqno;
1542 break;
1543 }
1544 }
1545
1546 if (seqno == 0)
1547 return -ENOSPC;
1548
1549 ret = i915_wait_seqno(ring, seqno);
1550 if (ret)
1551 return ret;
1552
1553 i915_gem_retire_requests_ring(ring);
1554 ringbuf->head = ringbuf->last_retired_head;
1555 ringbuf->last_retired_head = -1;
1556
1557 ringbuf->space = ring_space(ring);
1558 return 0;
1559 }
1560
1561 static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
1562 {
1563 struct drm_device *dev = ring->dev;
1564 struct drm_i915_private *dev_priv = dev->dev_private;
1565 struct intel_ringbuffer *ringbuf = ring->buffer;
1566 unsigned long end;
1567 int ret;
1568
1569 ret = intel_ring_wait_request(ring, n);
1570 if (ret != -ENOSPC)
1571 return ret;
1572
1573 /* force the tail write in case we have been skipping them */
1574 __intel_ring_advance(ring);
1575
1576 /* With GEM the hangcheck timer should kick us out of the loop,
1577 * leaving it early runs the risk of corrupting GEM state (due
1578 * to running on almost untested codepaths). But on resume
1579 * timers don't work yet, so prevent a complete hang in that
1580 * case by choosing an insanely large timeout. */
1581 end = jiffies + 60 * HZ;
1582
1583 trace_i915_ring_wait_begin(ring);
1584 do {
1585 ringbuf->head = I915_READ_HEAD(ring);
1586 ringbuf->space = ring_space(ring);
1587 if (ringbuf->space >= n) {
1588 ret = 0;
1589 break;
1590 }
1591
1592 if (!drm_core_check_feature(dev, DRIVER_MODESET) &&
1593 dev->primary->master) {
1594 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1595 if (master_priv->sarea_priv)
1596 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1597 }
1598
1599 msleep(1);
1600
1601 if (dev_priv->mm.interruptible && signal_pending(current)) {
1602 ret = -ERESTARTSYS;
1603 break;
1604 }
1605
1606 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
1607 dev_priv->mm.interruptible);
1608 if (ret)
1609 break;
1610
1611 if (time_after(jiffies, end)) {
1612 ret = -EBUSY;
1613 break;
1614 }
1615 } while (1);
1616 trace_i915_ring_wait_end(ring);
1617 return ret;
1618 }
1619
1620 static int intel_wrap_ring_buffer(struct intel_engine_cs *ring)
1621 {
1622 uint32_t __iomem *virt;
1623 struct intel_ringbuffer *ringbuf = ring->buffer;
1624 int rem = ringbuf->size - ringbuf->tail;
1625
1626 if (ringbuf->space < rem) {
1627 int ret = ring_wait_for_space(ring, rem);
1628 if (ret)
1629 return ret;
1630 }
1631
1632 virt = ringbuf->virtual_start + ringbuf->tail;
1633 rem /= 4;
1634 while (rem--)
1635 iowrite32(MI_NOOP, virt++);
1636
1637 ringbuf->tail = 0;
1638 ringbuf->space = ring_space(ring);
1639
1640 return 0;
1641 }
1642
1643 int intel_ring_idle(struct intel_engine_cs *ring)
1644 {
1645 u32 seqno;
1646 int ret;
1647
1648 /* We need to add any requests required to flush the objects and ring */
1649 if (ring->outstanding_lazy_seqno) {
1650 ret = i915_add_request(ring, NULL);
1651 if (ret)
1652 return ret;
1653 }
1654
1655 /* Wait upon the last request to be completed */
1656 if (list_empty(&ring->request_list))
1657 return 0;
1658
1659 seqno = list_entry(ring->request_list.prev,
1660 struct drm_i915_gem_request,
1661 list)->seqno;
1662
1663 return i915_wait_seqno(ring, seqno);
1664 }
1665
1666 static int
1667 intel_ring_alloc_seqno(struct intel_engine_cs *ring)
1668 {
1669 if (ring->outstanding_lazy_seqno)
1670 return 0;
1671
1672 if (ring->preallocated_lazy_request == NULL) {
1673 struct drm_i915_gem_request *request;
1674
1675 request = kmalloc(sizeof(*request), GFP_KERNEL);
1676 if (request == NULL)
1677 return -ENOMEM;
1678
1679 ring->preallocated_lazy_request = request;
1680 }
1681
1682 return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
1683 }
1684
1685 static int __intel_ring_prepare(struct intel_engine_cs *ring,
1686 int bytes)
1687 {
1688 struct intel_ringbuffer *ringbuf = ring->buffer;
1689 int ret;
1690
1691 if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {
1692 ret = intel_wrap_ring_buffer(ring);
1693 if (unlikely(ret))
1694 return ret;
1695 }
1696
1697 if (unlikely(ringbuf->space < bytes)) {
1698 ret = ring_wait_for_space(ring, bytes);
1699 if (unlikely(ret))
1700 return ret;
1701 }
1702
1703 return 0;
1704 }
1705
1706 int intel_ring_begin(struct intel_engine_cs *ring,
1707 int num_dwords)
1708 {
1709 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1710 int ret;
1711
1712 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
1713 dev_priv->mm.interruptible);
1714 if (ret)
1715 return ret;
1716
1717 ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
1718 if (ret)
1719 return ret;
1720
1721 /* Preallocate the olr before touching the ring */
1722 ret = intel_ring_alloc_seqno(ring);
1723 if (ret)
1724 return ret;
1725
1726 ring->buffer->space -= num_dwords * sizeof(uint32_t);
1727 return 0;
1728 }
1729
1730 /* Align the ring tail to a cacheline boundary */
1731 int intel_ring_cacheline_align(struct intel_engine_cs *ring)
1732 {
1733 int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
1734 int ret;
1735
1736 if (num_dwords == 0)
1737 return 0;
1738
1739 num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
1740 ret = intel_ring_begin(ring, num_dwords);
1741 if (ret)
1742 return ret;
1743
1744 while (num_dwords--)
1745 intel_ring_emit(ring, MI_NOOP);
1746
1747 intel_ring_advance(ring);
1748
1749 return 0;
1750 }
1751
1752 void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno)
1753 {
1754 struct drm_device *dev = ring->dev;
1755 struct drm_i915_private *dev_priv = dev->dev_private;
1756
1757 BUG_ON(ring->outstanding_lazy_seqno);
1758
1759 if (INTEL_INFO(dev)->gen == 6 || INTEL_INFO(dev)->gen == 7) {
1760 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
1761 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
1762 if (HAS_VEBOX(dev))
1763 I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
1764 }
1765
1766 ring->set_seqno(ring, seqno);
1767 ring->hangcheck.seqno = seqno;
1768 }
1769
1770 static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring,
1771 u32 value)
1772 {
1773 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1774
1775 /* Every tail move must follow the sequence below */
1776
1777 /* Disable notification that the ring is IDLE. The GT
1778 * will then assume that it is busy and bring it out of rc6.
1779 */
1780 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1781 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1782
1783 /* Clear the context id. Here be magic! */
1784 I915_WRITE64(GEN6_BSD_RNCID, 0x0);
1785
1786 /* Wait for the ring not to be idle, i.e. for it to wake up. */
1787 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1788 GEN6_BSD_SLEEP_INDICATOR) == 0,
1789 50))
1790 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
1791
1792 /* Now that the ring is fully powered up, update the tail */
1793 I915_WRITE_TAIL(ring, value);
1794 POSTING_READ(RING_TAIL(ring->mmio_base));
1795
1796 /* Let the ring send IDLE messages to the GT again,
1797 * and so let it sleep to conserve power when idle.
1798 */
1799 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1800 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1801 }
1802
1803 static int gen6_bsd_ring_flush(struct intel_engine_cs *ring,
1804 u32 invalidate, u32 flush)
1805 {
1806 uint32_t cmd;
1807 int ret;
1808
1809 ret = intel_ring_begin(ring, 4);
1810 if (ret)
1811 return ret;
1812
1813 cmd = MI_FLUSH_DW;
1814 if (INTEL_INFO(ring->dev)->gen >= 8)
1815 cmd += 1;
1816 /*
1817 * Bspec vol 1c.5 - video engine command streamer:
1818 * "If ENABLED, all TLBs will be invalidated once the flush
1819 * operation is complete. This bit is only valid when the
1820 * Post-Sync Operation field is a value of 1h or 3h."
1821 */
1822 if (invalidate & I915_GEM_GPU_DOMAINS)
1823 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
1824 MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
1825 intel_ring_emit(ring, cmd);
1826 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
1827 if (INTEL_INFO(ring->dev)->gen >= 8) {
1828 intel_ring_emit(ring, 0); /* upper addr */
1829 intel_ring_emit(ring, 0); /* value */
1830 } else {
1831 intel_ring_emit(ring, 0);
1832 intel_ring_emit(ring, MI_NOOP);
1833 }
1834 intel_ring_advance(ring);
1835 return 0;
1836 }
1837
1838 static int
1839 gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
1840 u64 offset, u32 len,
1841 unsigned flags)
1842 {
1843 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1844 bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL &&
1845 !(flags & I915_DISPATCH_SECURE);
1846 int ret;
1847
1848 ret = intel_ring_begin(ring, 4);
1849 if (ret)
1850 return ret;
1851
1852 /* FIXME(BDW): Address space and security selectors. */
1853 intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
1854 intel_ring_emit(ring, lower_32_bits(offset));
1855 intel_ring_emit(ring, upper_32_bits(offset));
1856 intel_ring_emit(ring, MI_NOOP);
1857 intel_ring_advance(ring);
1858
1859 return 0;
1860 }
1861
1862 static int
1863 hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
1864 u64 offset, u32 len,
1865 unsigned flags)
1866 {
1867 int ret;
1868
1869 ret = intel_ring_begin(ring, 2);
1870 if (ret)
1871 return ret;
1872
1873 intel_ring_emit(ring,
1874 MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW |
1875 (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
1876 /* bit0-7 is the length on GEN6+ */
1877 intel_ring_emit(ring, offset);
1878 intel_ring_advance(ring);
1879
1880 return 0;
1881 }
1882
1883 static int
1884 gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
1885 u64 offset, u32 len,
1886 unsigned flags)
1887 {
1888 int ret;
1889
1890 ret = intel_ring_begin(ring, 2);
1891 if (ret)
1892 return ret;
1893
1894 intel_ring_emit(ring,
1895 MI_BATCH_BUFFER_START |
1896 (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
1897 /* bit0-7 is the length on GEN6+ */
1898 intel_ring_emit(ring, offset);
1899 intel_ring_advance(ring);
1900
1901 return 0;
1902 }
1903
1904 /* Blitter support (SandyBridge+) */
1905
1906 static int gen6_ring_flush(struct intel_engine_cs *ring,
1907 u32 invalidate, u32 flush)
1908 {
1909 struct drm_device *dev = ring->dev;
1910 uint32_t cmd;
1911 int ret;
1912
1913 ret = intel_ring_begin(ring, 4);
1914 if (ret)
1915 return ret;
1916
1917 cmd = MI_FLUSH_DW;
1918 if (INTEL_INFO(ring->dev)->gen >= 8)
1919 cmd += 1;
1920 /*
1921 * Bspec vol 1c.3 - blitter engine command streamer:
1922 * "If ENABLED, all TLBs will be invalidated once the flush
1923 * operation is complete. This bit is only valid when the
1924 * Post-Sync Operation field is a value of 1h or 3h."
1925 */
1926 if (invalidate & I915_GEM_DOMAIN_RENDER)
1927 cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
1928 MI_FLUSH_DW_OP_STOREDW;
1929 intel_ring_emit(ring, cmd);
1930 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
1931 if (INTEL_INFO(ring->dev)->gen >= 8) {
1932 intel_ring_emit(ring, 0); /* upper addr */
1933 intel_ring_emit(ring, 0); /* value */
1934 } else {
1935 intel_ring_emit(ring, 0);
1936 intel_ring_emit(ring, MI_NOOP);
1937 }
1938 intel_ring_advance(ring);
1939
1940 if (IS_GEN7(dev) && !invalidate && flush)
1941 return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN);
1942
1943 return 0;
1944 }
1945
1946 int intel_init_render_ring_buffer(struct drm_device *dev)
1947 {
1948 struct drm_i915_private *dev_priv = dev->dev_private;
1949 struct intel_engine_cs *ring = &dev_priv->ring[RCS];
1950
1951 ring->name = "render ring";
1952 ring->id = RCS;
1953 ring->mmio_base = RENDER_RING_BASE;
1954
1955 if (INTEL_INFO(dev)->gen >= 6) {
1956 ring->add_request = gen6_add_request;
1957 ring->flush = gen7_render_ring_flush;
1958 if (INTEL_INFO(dev)->gen == 6)
1959 ring->flush = gen6_render_ring_flush;
1960 if (INTEL_INFO(dev)->gen >= 8) {
1961 ring->flush = gen8_render_ring_flush;
1962 ring->irq_get = gen8_ring_get_irq;
1963 ring->irq_put = gen8_ring_put_irq;
1964 } else {
1965 ring->irq_get = gen6_ring_get_irq;
1966 ring->irq_put = gen6_ring_put_irq;
1967 }
1968 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
1969 ring->get_seqno = gen6_ring_get_seqno;
1970 ring->set_seqno = ring_set_seqno;
1971 ring->semaphore.sync_to = gen6_ring_sync;
1972 ring->semaphore.signal = gen6_signal;
1973 /*
1974 * The current semaphore is only applied on pre-gen8 platform.
1975 * And there is no VCS2 ring on the pre-gen8 platform. So the
1976 * semaphore between RCS and VCS2 is initialized as INVALID.
1977 * Gen8 will initialize the sema between VCS2 and RCS later.
1978 */
1979 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
1980 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
1981 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
1982 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
1983 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
1984 ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
1985 ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
1986 ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
1987 ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
1988 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
1989 } else if (IS_GEN5(dev)) {
1990 ring->add_request = pc_render_add_request;
1991 ring->flush = gen4_render_ring_flush;
1992 ring->get_seqno = pc_render_get_seqno;
1993 ring->set_seqno = pc_render_set_seqno;
1994 ring->irq_get = gen5_ring_get_irq;
1995 ring->irq_put = gen5_ring_put_irq;
1996 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
1997 GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
1998 } else {
1999 ring->add_request = i9xx_add_request;
2000 if (INTEL_INFO(dev)->gen < 4)
2001 ring->flush = gen2_render_ring_flush;
2002 else
2003 ring->flush = gen4_render_ring_flush;
2004 ring->get_seqno = ring_get_seqno;
2005 ring->set_seqno = ring_set_seqno;
2006 if (IS_GEN2(dev)) {
2007 ring->irq_get = i8xx_ring_get_irq;
2008 ring->irq_put = i8xx_ring_put_irq;
2009 } else {
2010 ring->irq_get = i9xx_ring_get_irq;
2011 ring->irq_put = i9xx_ring_put_irq;
2012 }
2013 ring->irq_enable_mask = I915_USER_INTERRUPT;
2014 }
2015 ring->write_tail = ring_write_tail;
2016 if (IS_HASWELL(dev))
2017 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
2018 else if (IS_GEN8(dev))
2019 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2020 else if (INTEL_INFO(dev)->gen >= 6)
2021 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2022 else if (INTEL_INFO(dev)->gen >= 4)
2023 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
2024 else if (IS_I830(dev) || IS_845G(dev))
2025 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
2026 else
2027 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
2028 ring->init = init_render_ring;
2029 ring->cleanup = render_ring_cleanup;
2030
2031 /* Workaround batchbuffer to combat CS tlb bug. */
2032 if (HAS_BROKEN_CS_TLB(dev)) {
2033 struct drm_i915_gem_object *obj;
2034 int ret;
2035
2036 obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT);
2037 if (obj == NULL) {
2038 DRM_ERROR("Failed to allocate batch bo\n");
2039 return -ENOMEM;
2040 }
2041
2042 ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
2043 if (ret != 0) {
2044 drm_gem_object_unreference(&obj->base);
2045 DRM_ERROR("Failed to ping batch bo\n");
2046 return ret;
2047 }
2048
2049 ring->scratch.obj = obj;
2050 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
2051 }
2052
2053 return intel_init_ring_buffer(dev, ring);
2054 }
2055
2056 int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
2057 {
2058 struct drm_i915_private *dev_priv = dev->dev_private;
2059 struct intel_engine_cs *ring = &dev_priv->ring[RCS];
2060 struct intel_ringbuffer *ringbuf = ring->buffer;
2061 int ret;
2062
2063 if (ringbuf == NULL) {
2064 ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
2065 if (!ringbuf)
2066 return -ENOMEM;
2067 ring->buffer = ringbuf;
2068 }
2069
2070 ring->name = "render ring";
2071 ring->id = RCS;
2072 ring->mmio_base = RENDER_RING_BASE;
2073
2074 if (INTEL_INFO(dev)->gen >= 6) {
2075 /* non-kms not supported on gen6+ */
2076 ret = -ENODEV;
2077 goto err_ringbuf;
2078 }
2079
2080 /* Note: gem is not supported on gen5/ilk without kms (the corresponding
2081 * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
2082 * the special gen5 functions. */
2083 ring->add_request = i9xx_add_request;
2084 if (INTEL_INFO(dev)->gen < 4)
2085 ring->flush = gen2_render_ring_flush;
2086 else
2087 ring->flush = gen4_render_ring_flush;
2088 ring->get_seqno = ring_get_seqno;
2089 ring->set_seqno = ring_set_seqno;
2090 if (IS_GEN2(dev)) {
2091 ring->irq_get = i8xx_ring_get_irq;
2092 ring->irq_put = i8xx_ring_put_irq;
2093 } else {
2094 ring->irq_get = i9xx_ring_get_irq;
2095 ring->irq_put = i9xx_ring_put_irq;
2096 }
2097 ring->irq_enable_mask = I915_USER_INTERRUPT;
2098 ring->write_tail = ring_write_tail;
2099 if (INTEL_INFO(dev)->gen >= 4)
2100 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
2101 else if (IS_I830(dev) || IS_845G(dev))
2102 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
2103 else
2104 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
2105 ring->init = init_render_ring;
2106 ring->cleanup = render_ring_cleanup;
2107
2108 ring->dev = dev;
2109 INIT_LIST_HEAD(&ring->active_list);
2110 INIT_LIST_HEAD(&ring->request_list);
2111
2112 ringbuf->size = size;
2113 ringbuf->effective_size = ringbuf->size;
2114 if (IS_I830(ring->dev) || IS_845G(ring->dev))
2115 ringbuf->effective_size -= 2 * CACHELINE_BYTES;
2116
2117 ringbuf->virtual_start = ioremap_wc(start, size);
2118 if (ringbuf->virtual_start == NULL) {
2119 DRM_ERROR("can not ioremap virtual address for"
2120 " ring buffer\n");
2121 ret = -ENOMEM;
2122 goto err_ringbuf;
2123 }
2124
2125 if (!I915_NEED_GFX_HWS(dev)) {
2126 ret = init_phys_status_page(ring);
2127 if (ret)
2128 goto err_vstart;
2129 }
2130
2131 return 0;
2132
2133 err_vstart:
2134 iounmap(ringbuf->virtual_start);
2135 err_ringbuf:
2136 kfree(ringbuf);
2137 ring->buffer = NULL;
2138 return ret;
2139 }
2140
2141 int intel_init_bsd_ring_buffer(struct drm_device *dev)
2142 {
2143 struct drm_i915_private *dev_priv = dev->dev_private;
2144 struct intel_engine_cs *ring = &dev_priv->ring[VCS];
2145
2146 ring->name = "bsd ring";
2147 ring->id = VCS;
2148
2149 ring->write_tail = ring_write_tail;
2150 if (INTEL_INFO(dev)->gen >= 6) {
2151 ring->mmio_base = GEN6_BSD_RING_BASE;
2152 /* gen6 bsd needs a special wa for tail updates */
2153 if (IS_GEN6(dev))
2154 ring->write_tail = gen6_bsd_ring_write_tail;
2155 ring->flush = gen6_bsd_ring_flush;
2156 ring->add_request = gen6_add_request;
2157 ring->get_seqno = gen6_ring_get_seqno;
2158 ring->set_seqno = ring_set_seqno;
2159 if (INTEL_INFO(dev)->gen >= 8) {
2160 ring->irq_enable_mask =
2161 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
2162 ring->irq_get = gen8_ring_get_irq;
2163 ring->irq_put = gen8_ring_put_irq;
2164 ring->dispatch_execbuffer =
2165 gen8_ring_dispatch_execbuffer;
2166 } else {
2167 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
2168 ring->irq_get = gen6_ring_get_irq;
2169 ring->irq_put = gen6_ring_put_irq;
2170 ring->dispatch_execbuffer =
2171 gen6_ring_dispatch_execbuffer;
2172 }
2173 ring->semaphore.sync_to = gen6_ring_sync;
2174 ring->semaphore.signal = gen6_signal;
2175 /*
2176 * The current semaphore is only applied on pre-gen8 platform.
2177 * And there is no VCS2 ring on the pre-gen8 platform. So the
2178 * semaphore between VCS and VCS2 is initialized as INVALID.
2179 * Gen8 will initialize the sema between VCS2 and VCS later.
2180 */
2181 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
2182 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
2183 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
2184 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
2185 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2186 ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
2187 ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
2188 ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
2189 ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
2190 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2191 } else {
2192 ring->mmio_base = BSD_RING_BASE;
2193 ring->flush = bsd_ring_flush;
2194 ring->add_request = i9xx_add_request;
2195 ring->get_seqno = ring_get_seqno;
2196 ring->set_seqno = ring_set_seqno;
2197 if (IS_GEN5(dev)) {
2198 ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
2199 ring->irq_get = gen5_ring_get_irq;
2200 ring->irq_put = gen5_ring_put_irq;
2201 } else {
2202 ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
2203 ring->irq_get = i9xx_ring_get_irq;
2204 ring->irq_put = i9xx_ring_put_irq;
2205 }
2206 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
2207 }
2208 ring->init = init_ring_common;
2209
2210 return intel_init_ring_buffer(dev, ring);
2211 }
2212
2213 /**
2214 * Initialize the second BSD ring for Broadwell GT3.
2215 * It is noted that this only exists on Broadwell GT3.
2216 */
2217 int intel_init_bsd2_ring_buffer(struct drm_device *dev)
2218 {
2219 struct drm_i915_private *dev_priv = dev->dev_private;
2220 struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
2221
2222 if ((INTEL_INFO(dev)->gen != 8)) {
2223 DRM_ERROR("No dual-BSD ring on non-BDW machine\n");
2224 return -EINVAL;
2225 }
2226
2227 ring->name = "bsd2 ring";
2228 ring->id = VCS2;
2229
2230 ring->write_tail = ring_write_tail;
2231 ring->mmio_base = GEN8_BSD2_RING_BASE;
2232 ring->flush = gen6_bsd_ring_flush;
2233 ring->add_request = gen6_add_request;
2234 ring->get_seqno = gen6_ring_get_seqno;
2235 ring->set_seqno = ring_set_seqno;
2236 ring->irq_enable_mask =
2237 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
2238 ring->irq_get = gen8_ring_get_irq;
2239 ring->irq_put = gen8_ring_put_irq;
2240 ring->dispatch_execbuffer =
2241 gen8_ring_dispatch_execbuffer;
2242 ring->semaphore.sync_to = gen6_ring_sync;
2243 ring->semaphore.signal = gen6_signal;
2244 /*
2245 * The current semaphore is only applied on the pre-gen8. And there
2246 * is no bsd2 ring on the pre-gen8. So now the semaphore_register
2247 * between VCS2 and other ring is initialized as invalid.
2248 * Gen8 will initialize the sema between VCS2 and other ring later.
2249 */
2250 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
2251 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
2252 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
2253 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
2254 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2255 ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
2256 ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
2257 ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
2258 ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
2259 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2260
2261 ring->init = init_ring_common;
2262
2263 return intel_init_ring_buffer(dev, ring);
2264 }
2265
2266 int intel_init_blt_ring_buffer(struct drm_device *dev)
2267 {
2268 struct drm_i915_private *dev_priv = dev->dev_private;
2269 struct intel_engine_cs *ring = &dev_priv->ring[BCS];
2270
2271 ring->name = "blitter ring";
2272 ring->id = BCS;
2273
2274 ring->mmio_base = BLT_RING_BASE;
2275 ring->write_tail = ring_write_tail;
2276 ring->flush = gen6_ring_flush;
2277 ring->add_request = gen6_add_request;
2278 ring->get_seqno = gen6_ring_get_seqno;
2279 ring->set_seqno = ring_set_seqno;
2280 if (INTEL_INFO(dev)->gen >= 8) {
2281 ring->irq_enable_mask =
2282 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
2283 ring->irq_get = gen8_ring_get_irq;
2284 ring->irq_put = gen8_ring_put_irq;
2285 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2286 } else {
2287 ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
2288 ring->irq_get = gen6_ring_get_irq;
2289 ring->irq_put = gen6_ring_put_irq;
2290 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2291 }
2292 ring->semaphore.sync_to = gen6_ring_sync;
2293 ring->semaphore.signal = gen6_signal;
2294 /*
2295 * The current semaphore is only applied on pre-gen8 platform. And
2296 * there is no VCS2 ring on the pre-gen8 platform. So the semaphore
2297 * between BCS and VCS2 is initialized as INVALID.
2298 * Gen8 will initialize the sema between BCS and VCS2 later.
2299 */
2300 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
2301 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
2302 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
2303 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
2304 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2305 ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
2306 ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
2307 ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
2308 ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
2309 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2310 ring->init = init_ring_common;
2311
2312 return intel_init_ring_buffer(dev, ring);
2313 }
2314
2315 int intel_init_vebox_ring_buffer(struct drm_device *dev)
2316 {
2317 struct drm_i915_private *dev_priv = dev->dev_private;
2318 struct intel_engine_cs *ring = &dev_priv->ring[VECS];
2319
2320 ring->name = "video enhancement ring";
2321 ring->id = VECS;
2322
2323 ring->mmio_base = VEBOX_RING_BASE;
2324 ring->write_tail = ring_write_tail;
2325 ring->flush = gen6_ring_flush;
2326 ring->add_request = gen6_add_request;
2327 ring->get_seqno = gen6_ring_get_seqno;
2328 ring->set_seqno = ring_set_seqno;
2329
2330 if (INTEL_INFO(dev)->gen >= 8) {
2331 ring->irq_enable_mask =
2332 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
2333 ring->irq_get = gen8_ring_get_irq;
2334 ring->irq_put = gen8_ring_put_irq;
2335 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2336 } else {
2337 ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
2338 ring->irq_get = hsw_vebox_get_irq;
2339 ring->irq_put = hsw_vebox_put_irq;
2340 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2341 }
2342 ring->semaphore.sync_to = gen6_ring_sync;
2343 ring->semaphore.signal = gen6_signal;
2344 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
2345 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
2346 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
2347 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
2348 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2349 ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
2350 ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
2351 ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
2352 ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
2353 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2354 ring->init = init_ring_common;
2355
2356 return intel_init_ring_buffer(dev, ring);
2357 }
2358
2359 int
2360 intel_ring_flush_all_caches(struct intel_engine_cs *ring)
2361 {
2362 int ret;
2363
2364 if (!ring->gpu_caches_dirty)
2365 return 0;
2366
2367 ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
2368 if (ret)
2369 return ret;
2370
2371 trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
2372
2373 ring->gpu_caches_dirty = false;
2374 return 0;
2375 }
2376
2377 int
2378 intel_ring_invalidate_all_caches(struct intel_engine_cs *ring)
2379 {
2380 uint32_t flush_domains;
2381 int ret;
2382
2383 flush_domains = 0;
2384 if (ring->gpu_caches_dirty)
2385 flush_domains = I915_GEM_GPU_DOMAINS;
2386
2387 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
2388 if (ret)
2389 return ret;
2390
2391 trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
2392
2393 ring->gpu_caches_dirty = false;
2394 return 0;
2395 }
2396
2397 void
2398 intel_stop_ring_buffer(struct intel_engine_cs *ring)
2399 {
2400 int ret;
2401
2402 if (!intel_ring_initialized(ring))
2403 return;
2404
2405 ret = intel_ring_idle(ring);
2406 if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
2407 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
2408 ring->name, ret);
2409
2410 stop_ring(ring);
2411 }
Linux kernel - Deliverables
This page took 0.111352 seconds and 6 git commands to generate.