projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'drm-init-cleanup' of git://people.freedesktop.org/~danvet/drm into...
[deliverable/linux.git] / drivers / gpu / drm / i915 / intel_ringbuffer.c
1 /*
2 * Copyright © 2008-2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Zou Nan hai <nanhai.zou@intel.com>
26 * Xiang Hai hao<haihao.xiang@intel.com>
27 *
28 */
29
30 #include <drm/drmP.h>
31 #include "i915_drv.h"
32 #include <drm/i915_drm.h>
33 #include "i915_trace.h"
34 #include "intel_drv.h"
35
36 static inline int ring_space(struct intel_ring_buffer *ring)
37 {
38 int space = (ring->head & HEAD_ADDR) - (ring->tail + I915_RING_FREE_SPACE);
39 if (space < 0)
40 space += ring->size;
41 return space;
42 }
43
44 static bool intel_ring_stopped(struct intel_ring_buffer *ring)
45 {
46 struct drm_i915_private *dev_priv = ring->dev->dev_private;
47 return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);
48 }
49
50 void __intel_ring_advance(struct intel_ring_buffer *ring)
51 {
52 ring->tail &= ring->size - 1;
53 if (intel_ring_stopped(ring))
54 return;
55 ring->write_tail(ring, ring->tail);
56 }
57
58 static int
59 gen2_render_ring_flush(struct intel_ring_buffer *ring,
60 u32 invalidate_domains,
61 u32 flush_domains)
62 {
63 u32 cmd;
64 int ret;
65
66 cmd = MI_FLUSH;
67 if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
68 cmd |= MI_NO_WRITE_FLUSH;
69
70 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
71 cmd |= MI_READ_FLUSH;
72
73 ret = intel_ring_begin(ring, 2);
74 if (ret)
75 return ret;
76
77 intel_ring_emit(ring, cmd);
78 intel_ring_emit(ring, MI_NOOP);
79 intel_ring_advance(ring);
80
81 return 0;
82 }
83
84 static int
85 gen4_render_ring_flush(struct intel_ring_buffer *ring,
86 u32 invalidate_domains,
87 u32 flush_domains)
88 {
89 struct drm_device *dev = ring->dev;
90 u32 cmd;
91 int ret;
92
93 /*
94 * read/write caches:
95 *
96 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
97 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
98 * also flushed at 2d versus 3d pipeline switches.
99 *
100 * read-only caches:
101 *
102 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
103 * MI_READ_FLUSH is set, and is always flushed on 965.
104 *
105 * I915_GEM_DOMAIN_COMMAND may not exist?
106 *
107 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
108 * invalidated when MI_EXE_FLUSH is set.
109 *
110 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
111 * invalidated with every MI_FLUSH.
112 *
113 * TLBs:
114 *
115 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
116 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
117 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
118 * are flushed at any MI_FLUSH.
119 */
120
121 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
122 if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
123 cmd &= ~MI_NO_WRITE_FLUSH;
124 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
125 cmd |= MI_EXE_FLUSH;
126
127 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
128 (IS_G4X(dev) || IS_GEN5(dev)))
129 cmd |= MI_INVALIDATE_ISP;
130
131 ret = intel_ring_begin(ring, 2);
132 if (ret)
133 return ret;
134
135 intel_ring_emit(ring, cmd);
136 intel_ring_emit(ring, MI_NOOP);
137 intel_ring_advance(ring);
138
139 return 0;
140 }
141
142 /**
143 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
144 * implementing two workarounds on gen6. From section 1.4.7.1
145 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
146 *
147 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
148 * produced by non-pipelined state commands), software needs to first
149 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
150 * 0.
151 *
152 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
153 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
154 *
155 * And the workaround for these two requires this workaround first:
156 *
157 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
158 * BEFORE the pipe-control with a post-sync op and no write-cache
159 * flushes.
160 *
161 * And this last workaround is tricky because of the requirements on
162 * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
163 * volume 2 part 1:
164 *
165 * "1 of the following must also be set:
166 * - Render Target Cache Flush Enable ([12] of DW1)
167 * - Depth Cache Flush Enable ([0] of DW1)
168 * - Stall at Pixel Scoreboard ([1] of DW1)
169 * - Depth Stall ([13] of DW1)
170 * - Post-Sync Operation ([13] of DW1)
171 * - Notify Enable ([8] of DW1)"
172 *
173 * The cache flushes require the workaround flush that triggered this
174 * one, so we can't use it. Depth stall would trigger the same.
175 * Post-sync nonzero is what triggered this second workaround, so we
176 * can't use that one either. Notify enable is IRQs, which aren't
177 * really our business. That leaves only stall at scoreboard.
178 */
179 static int
180 intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
181 {
182 u32 scratch_addr = ring->scratch.gtt_offset + 128;
183 int ret;
184
185
186 ret = intel_ring_begin(ring, 6);
187 if (ret)
188 return ret;
189
190 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
191 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
192 PIPE_CONTROL_STALL_AT_SCOREBOARD);
193 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
194 intel_ring_emit(ring, 0); /* low dword */
195 intel_ring_emit(ring, 0); /* high dword */
196 intel_ring_emit(ring, MI_NOOP);
197 intel_ring_advance(ring);
198
199 ret = intel_ring_begin(ring, 6);
200 if (ret)
201 return ret;
202
203 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
204 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
205 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
206 intel_ring_emit(ring, 0);
207 intel_ring_emit(ring, 0);
208 intel_ring_emit(ring, MI_NOOP);
209 intel_ring_advance(ring);
210
211 return 0;
212 }
213
214 static int
215 gen6_render_ring_flush(struct intel_ring_buffer *ring,
216 u32 invalidate_domains, u32 flush_domains)
217 {
218 u32 flags = 0;
219 u32 scratch_addr = ring->scratch.gtt_offset + 128;
220 int ret;
221
222 /* Force SNB workarounds for PIPE_CONTROL flushes */
223 ret = intel_emit_post_sync_nonzero_flush(ring);
224 if (ret)
225 return ret;
226
227 /* Just flush everything. Experiments have shown that reducing the
228 * number of bits based on the write domains has little performance
229 * impact.
230 */
231 if (flush_domains) {
232 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
233 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
234 /*
235 * Ensure that any following seqno writes only happen
236 * when the render cache is indeed flushed.
237 */
238 flags |= PIPE_CONTROL_CS_STALL;
239 }
240 if (invalidate_domains) {
241 flags |= PIPE_CONTROL_TLB_INVALIDATE;
242 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
243 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
244 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
245 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
246 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
247 /*
248 * TLB invalidate requires a post-sync write.
249 */
250 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
251 }
252
253 ret = intel_ring_begin(ring, 4);
254 if (ret)
255 return ret;
256
257 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
258 intel_ring_emit(ring, flags);
259 intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
260 intel_ring_emit(ring, 0);
261 intel_ring_advance(ring);
262
263 return 0;
264 }
265
266 static int
267 gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
268 {
269 int ret;
270
271 ret = intel_ring_begin(ring, 4);
272 if (ret)
273 return ret;
274
275 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
276 intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
277 PIPE_CONTROL_STALL_AT_SCOREBOARD);
278 intel_ring_emit(ring, 0);
279 intel_ring_emit(ring, 0);
280 intel_ring_advance(ring);
281
282 return 0;
283 }
284
285 static int gen7_ring_fbc_flush(struct intel_ring_buffer *ring, u32 value)
286 {
287 int ret;
288
289 if (!ring->fbc_dirty)
290 return 0;
291
292 ret = intel_ring_begin(ring, 6);
293 if (ret)
294 return ret;
295 /* WaFbcNukeOn3DBlt:ivb/hsw */
296 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
297 intel_ring_emit(ring, MSG_FBC_REND_STATE);
298 intel_ring_emit(ring, value);
299 intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT);
300 intel_ring_emit(ring, MSG_FBC_REND_STATE);
301 intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
302 intel_ring_advance(ring);
303
304 ring->fbc_dirty = false;
305 return 0;
306 }
307
308 static int
309 gen7_render_ring_flush(struct intel_ring_buffer *ring,
310 u32 invalidate_domains, u32 flush_domains)
311 {
312 u32 flags = 0;
313 u32 scratch_addr = ring->scratch.gtt_offset + 128;
314 int ret;
315
316 /*
317 * Ensure that any following seqno writes only happen when the render
318 * cache is indeed flushed.
319 *
320 * Workaround: 4th PIPE_CONTROL command (except the ones with only
321 * read-cache invalidate bits set) must have the CS_STALL bit set. We
322 * don't try to be clever and just set it unconditionally.
323 */
324 flags |= PIPE_CONTROL_CS_STALL;
325
326 /* Just flush everything. Experiments have shown that reducing the
327 * number of bits based on the write domains has little performance
328 * impact.
329 */
330 if (flush_domains) {
331 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
332 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
333 }
334 if (invalidate_domains) {
335 flags |= PIPE_CONTROL_TLB_INVALIDATE;
336 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
337 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
338 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
339 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
340 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
341 /*
342 * TLB invalidate requires a post-sync write.
343 */
344 flags |= PIPE_CONTROL_QW_WRITE;
345 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
346
347 /* Workaround: we must issue a pipe_control with CS-stall bit
348 * set before a pipe_control command that has the state cache
349 * invalidate bit set. */
350 gen7_render_ring_cs_stall_wa(ring);
351 }
352
353 ret = intel_ring_begin(ring, 4);
354 if (ret)
355 return ret;
356
357 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
358 intel_ring_emit(ring, flags);
359 intel_ring_emit(ring, scratch_addr);
360 intel_ring_emit(ring, 0);
361 intel_ring_advance(ring);
362
363 if (!invalidate_domains && flush_domains)
364 return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
365
366 return 0;
367 }
368
369 static int
370 gen8_render_ring_flush(struct intel_ring_buffer *ring,
371 u32 invalidate_domains, u32 flush_domains)
372 {
373 u32 flags = 0;
374 u32 scratch_addr = ring->scratch.gtt_offset + 128;
375 int ret;
376
377 flags |= PIPE_CONTROL_CS_STALL;
378
379 if (flush_domains) {
380 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
381 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
382 }
383 if (invalidate_domains) {
384 flags |= PIPE_CONTROL_TLB_INVALIDATE;
385 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
386 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
387 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
388 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
389 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
390 flags |= PIPE_CONTROL_QW_WRITE;
391 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
392 }
393
394 ret = intel_ring_begin(ring, 6);
395 if (ret)
396 return ret;
397
398 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
399 intel_ring_emit(ring, flags);
400 intel_ring_emit(ring, scratch_addr);
401 intel_ring_emit(ring, 0);
402 intel_ring_emit(ring, 0);
403 intel_ring_emit(ring, 0);
404 intel_ring_advance(ring);
405
406 return 0;
407
408 }
409
410 static void ring_write_tail(struct intel_ring_buffer *ring,
411 u32 value)
412 {
413 struct drm_i915_private *dev_priv = ring->dev->dev_private;
414 I915_WRITE_TAIL(ring, value);
415 }
416
417 u64 intel_ring_get_active_head(struct intel_ring_buffer *ring)
418 {
419 struct drm_i915_private *dev_priv = ring->dev->dev_private;
420 u64 acthd;
421
422 if (INTEL_INFO(ring->dev)->gen >= 8)
423 acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),
424 RING_ACTHD_UDW(ring->mmio_base));
425 else if (INTEL_INFO(ring->dev)->gen >= 4)
426 acthd = I915_READ(RING_ACTHD(ring->mmio_base));
427 else
428 acthd = I915_READ(ACTHD);
429
430 return acthd;
431 }
432
433 static void ring_setup_phys_status_page(struct intel_ring_buffer *ring)
434 {
435 struct drm_i915_private *dev_priv = ring->dev->dev_private;
436 u32 addr;
437
438 addr = dev_priv->status_page_dmah->busaddr;
439 if (INTEL_INFO(ring->dev)->gen >= 4)
440 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
441 I915_WRITE(HWS_PGA, addr);
442 }
443
444 static bool stop_ring(struct intel_ring_buffer *ring)
445 {
446 struct drm_i915_private *dev_priv = to_i915(ring->dev);
447
448 if (!IS_GEN2(ring->dev)) {
449 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
450 if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
451 DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
452 return false;
453 }
454 }
455
456 I915_WRITE_CTL(ring, 0);
457 I915_WRITE_HEAD(ring, 0);
458 ring->write_tail(ring, 0);
459
460 if (!IS_GEN2(ring->dev)) {
461 (void)I915_READ_CTL(ring);
462 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
463 }
464
465 return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0;
466 }
467
468 static int init_ring_common(struct intel_ring_buffer *ring)
469 {
470 struct drm_device *dev = ring->dev;
471 struct drm_i915_private *dev_priv = dev->dev_private;
472 struct drm_i915_gem_object *obj = ring->obj;
473 int ret = 0;
474
475 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
476
477 if (!stop_ring(ring)) {
478 /* G45 ring initialization often fails to reset head to zero */
479 DRM_DEBUG_KMS("%s head not reset to zero "
480 "ctl %08x head %08x tail %08x start %08x\n",
481 ring->name,
482 I915_READ_CTL(ring),
483 I915_READ_HEAD(ring),
484 I915_READ_TAIL(ring),
485 I915_READ_START(ring));
486
487 if (!stop_ring(ring)) {
488 DRM_ERROR("failed to set %s head to zero "
489 "ctl %08x head %08x tail %08x start %08x\n",
490 ring->name,
491 I915_READ_CTL(ring),
492 I915_READ_HEAD(ring),
493 I915_READ_TAIL(ring),
494 I915_READ_START(ring));
495 ret = -EIO;
496 goto out;
497 }
498 }
499
500 if (I915_NEED_GFX_HWS(dev))
501 intel_ring_setup_status_page(ring);
502 else
503 ring_setup_phys_status_page(ring);
504
505 /* Initialize the ring. This must happen _after_ we've cleared the ring
506 * registers with the above sequence (the readback of the HEAD registers
507 * also enforces ordering), otherwise the hw might lose the new ring
508 * register values. */
509 I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
510 I915_WRITE_CTL(ring,
511 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
512 | RING_VALID);
513
514 /* If the head is still not zero, the ring is dead */
515 if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
516 I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
517 (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
518 DRM_ERROR("%s initialization failed "
519 "ctl %08x head %08x tail %08x start %08x\n",
520 ring->name,
521 I915_READ_CTL(ring),
522 I915_READ_HEAD(ring),
523 I915_READ_TAIL(ring),
524 I915_READ_START(ring));
525 ret = -EIO;
526 goto out;
527 }
528
529 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
530 i915_kernel_lost_context(ring->dev);
531 else {
532 ring->head = I915_READ_HEAD(ring);
533 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
534 ring->space = ring_space(ring);
535 ring->last_retired_head = -1;
536 }
537
538 memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
539
540 out:
541 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
542
543 return ret;
544 }
545
546 static int
547 init_pipe_control(struct intel_ring_buffer *ring)
548 {
549 int ret;
550
551 if (ring->scratch.obj)
552 return 0;
553
554 ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
555 if (ring->scratch.obj == NULL) {
556 DRM_ERROR("Failed to allocate seqno page\n");
557 ret = -ENOMEM;
558 goto err;
559 }
560
561 ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
562 if (ret)
563 goto err_unref;
564
565 ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0);
566 if (ret)
567 goto err_unref;
568
569 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
570 ring->scratch.cpu_page = kmap(sg_page(ring->scratch.obj->pages->sgl));
571 if (ring->scratch.cpu_page == NULL) {
572 ret = -ENOMEM;
573 goto err_unpin;
574 }
575
576 DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
577 ring->name, ring->scratch.gtt_offset);
578 return 0;
579
580 err_unpin:
581 i915_gem_object_ggtt_unpin(ring->scratch.obj);
582 err_unref:
583 drm_gem_object_unreference(&ring->scratch.obj->base);
584 err:
585 return ret;
586 }
587
588 static int init_render_ring(struct intel_ring_buffer *ring)
589 {
590 struct drm_device *dev = ring->dev;
591 struct drm_i915_private *dev_priv = dev->dev_private;
592 int ret = init_ring_common(ring);
593
594 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
595 if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
596 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
597
598 /* We need to disable the AsyncFlip performance optimisations in order
599 * to use MI_WAIT_FOR_EVENT within the CS. It should already be
600 * programmed to '1' on all products.
601 *
602 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw
603 */
604 if (INTEL_INFO(dev)->gen >= 6)
605 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
606
607 /* Required for the hardware to program scanline values for waiting */
608 /* WaEnableFlushTlbInvalidationMode:snb */
609 if (INTEL_INFO(dev)->gen == 6)
610 I915_WRITE(GFX_MODE,
611 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
612
613 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
614 if (IS_GEN7(dev))
615 I915_WRITE(GFX_MODE_GEN7,
616 _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
617 _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
618
619 if (INTEL_INFO(dev)->gen >= 5) {
620 ret = init_pipe_control(ring);
621 if (ret)
622 return ret;
623 }
624
625 if (IS_GEN6(dev)) {
626 /* From the Sandybridge PRM, volume 1 part 3, page 24:
627 * "If this bit is set, STCunit will have LRA as replacement
628 * policy. [...] This bit must be reset. LRA replacement
629 * policy is not supported."
630 */
631 I915_WRITE(CACHE_MODE_0,
632 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
633 }
634
635 if (INTEL_INFO(dev)->gen >= 6)
636 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
637
638 if (HAS_L3_DPF(dev))
639 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
640
641 return ret;
642 }
643
644 static void render_ring_cleanup(struct intel_ring_buffer *ring)
645 {
646 struct drm_device *dev = ring->dev;
647
648 if (ring->scratch.obj == NULL)
649 return;
650
651 if (INTEL_INFO(dev)->gen >= 5) {
652 kunmap(sg_page(ring->scratch.obj->pages->sgl));
653 i915_gem_object_ggtt_unpin(ring->scratch.obj);
654 }
655
656 drm_gem_object_unreference(&ring->scratch.obj->base);
657 ring->scratch.obj = NULL;
658 }
659
660 static void
661 update_mboxes(struct intel_ring_buffer *ring,
662 u32 mmio_offset)
663 {
664 /* NB: In order to be able to do semaphore MBOX updates for varying number
665 * of rings, it's easiest if we round up each individual update to a
666 * multiple of 2 (since ring updates must always be a multiple of 2)
667 * even though the actual update only requires 3 dwords.
668 */
669 #define MBOX_UPDATE_DWORDS 4
670 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
671 intel_ring_emit(ring, mmio_offset);
672 intel_ring_emit(ring, ring->outstanding_lazy_seqno);
673 intel_ring_emit(ring, MI_NOOP);
674 }
675
676 /**
677 * gen6_add_request - Update the semaphore mailbox registers
678 *
679 * @ring - ring that is adding a request
680 * @seqno - return seqno stuck into the ring
681 *
682 * Update the mailbox registers in the *other* rings with the current seqno.
683 * This acts like a signal in the canonical semaphore.
684 */
685 static int
686 gen6_add_request(struct intel_ring_buffer *ring)
687 {
688 struct drm_device *dev = ring->dev;
689 struct drm_i915_private *dev_priv = dev->dev_private;
690 struct intel_ring_buffer *useless;
691 int i, ret, num_dwords = 4;
692
693 if (i915_semaphore_is_enabled(dev))
694 num_dwords += ((I915_NUM_RINGS-1) * MBOX_UPDATE_DWORDS);
695 #undef MBOX_UPDATE_DWORDS
696
697 ret = intel_ring_begin(ring, num_dwords);
698 if (ret)
699 return ret;
700
701 if (i915_semaphore_is_enabled(dev)) {
702 for_each_ring(useless, dev_priv, i) {
703 u32 mbox_reg = ring->signal_mbox[i];
704 if (mbox_reg != GEN6_NOSYNC)
705 update_mboxes(ring, mbox_reg);
706 }
707 }
708
709 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
710 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
711 intel_ring_emit(ring, ring->outstanding_lazy_seqno);
712 intel_ring_emit(ring, MI_USER_INTERRUPT);
713 __intel_ring_advance(ring);
714
715 return 0;
716 }
717
718 static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
719 u32 seqno)
720 {
721 struct drm_i915_private *dev_priv = dev->dev_private;
722 return dev_priv->last_seqno < seqno;
723 }
724
725 /**
726 * intel_ring_sync - sync the waiter to the signaller on seqno
727 *
728 * @waiter - ring that is waiting
729 * @signaller - ring which has, or will signal
730 * @seqno - seqno which the waiter will block on
731 */
732 static int
733 gen6_ring_sync(struct intel_ring_buffer *waiter,
734 struct intel_ring_buffer *signaller,
735 u32 seqno)
736 {
737 int ret;
738 u32 dw1 = MI_SEMAPHORE_MBOX |
739 MI_SEMAPHORE_COMPARE |
740 MI_SEMAPHORE_REGISTER;
741
742 /* Throughout all of the GEM code, seqno passed implies our current
743 * seqno is >= the last seqno executed. However for hardware the
744 * comparison is strictly greater than.
745 */
746 seqno -= 1;
747
748 WARN_ON(signaller->semaphore_register[waiter->id] ==
749 MI_SEMAPHORE_SYNC_INVALID);
750
751 ret = intel_ring_begin(waiter, 4);
752 if (ret)
753 return ret;
754
755 /* If seqno wrap happened, omit the wait with no-ops */
756 if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
757 intel_ring_emit(waiter,
758 dw1 |
759 signaller->semaphore_register[waiter->id]);
760 intel_ring_emit(waiter, seqno);
761 intel_ring_emit(waiter, 0);
762 intel_ring_emit(waiter, MI_NOOP);
763 } else {
764 intel_ring_emit(waiter, MI_NOOP);
765 intel_ring_emit(waiter, MI_NOOP);
766 intel_ring_emit(waiter, MI_NOOP);
767 intel_ring_emit(waiter, MI_NOOP);
768 }
769 intel_ring_advance(waiter);
770
771 return 0;
772 }
773
774 #define PIPE_CONTROL_FLUSH(ring__, addr__) \
775 do { \
776 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \
777 PIPE_CONTROL_DEPTH_STALL); \
778 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
779 intel_ring_emit(ring__, 0); \
780 intel_ring_emit(ring__, 0); \
781 } while (0)
782
783 static int
784 pc_render_add_request(struct intel_ring_buffer *ring)
785 {
786 u32 scratch_addr = ring->scratch.gtt_offset + 128;
787 int ret;
788
789 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
790 * incoherent with writes to memory, i.e. completely fubar,
791 * so we need to use PIPE_NOTIFY instead.
792 *
793 * However, we also need to workaround the qword write
794 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
795 * memory before requesting an interrupt.
796 */
797 ret = intel_ring_begin(ring, 32);
798 if (ret)
799 return ret;
800
801 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
802 PIPE_CONTROL_WRITE_FLUSH |
803 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
804 intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
805 intel_ring_emit(ring, ring->outstanding_lazy_seqno);
806 intel_ring_emit(ring, 0);
807 PIPE_CONTROL_FLUSH(ring, scratch_addr);
808 scratch_addr += 128; /* write to separate cachelines */
809 PIPE_CONTROL_FLUSH(ring, scratch_addr);
810 scratch_addr += 128;
811 PIPE_CONTROL_FLUSH(ring, scratch_addr);
812 scratch_addr += 128;
813 PIPE_CONTROL_FLUSH(ring, scratch_addr);
814 scratch_addr += 128;
815 PIPE_CONTROL_FLUSH(ring, scratch_addr);
816 scratch_addr += 128;
817 PIPE_CONTROL_FLUSH(ring, scratch_addr);
818
819 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
820 PIPE_CONTROL_WRITE_FLUSH |
821 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
822 PIPE_CONTROL_NOTIFY);
823 intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
824 intel_ring_emit(ring, ring->outstanding_lazy_seqno);
825 intel_ring_emit(ring, 0);
826 __intel_ring_advance(ring);
827
828 return 0;
829 }
830
831 static u32
832 gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
833 {
834 /* Workaround to force correct ordering between irq and seqno writes on
835 * ivb (and maybe also on snb) by reading from a CS register (like
836 * ACTHD) before reading the status page. */
837 if (!lazy_coherency) {
838 struct drm_i915_private *dev_priv = ring->dev->dev_private;
839 POSTING_READ(RING_ACTHD(ring->mmio_base));
840 }
841
842 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
843 }
844
845 static u32
846 ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
847 {
848 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
849 }
850
851 static void
852 ring_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
853 {
854 intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
855 }
856
857 static u32
858 pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
859 {
860 return ring->scratch.cpu_page[0];
861 }
862
863 static void
864 pc_render_set_seqno(struct intel_ring_buffer *ring, u32 seqno)
865 {
866 ring->scratch.cpu_page[0] = seqno;
867 }
868
869 static bool
870 gen5_ring_get_irq(struct intel_ring_buffer *ring)
871 {
872 struct drm_device *dev = ring->dev;
873 struct drm_i915_private *dev_priv = dev->dev_private;
874 unsigned long flags;
875
876 if (!dev->irq_enabled)
877 return false;
878
879 spin_lock_irqsave(&dev_priv->irq_lock, flags);
880 if (ring->irq_refcount++ == 0)
881 ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask);
882 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
883
884 return true;
885 }
886
887 static void
888 gen5_ring_put_irq(struct intel_ring_buffer *ring)
889 {
890 struct drm_device *dev = ring->dev;
891 struct drm_i915_private *dev_priv = dev->dev_private;
892 unsigned long flags;
893
894 spin_lock_irqsave(&dev_priv->irq_lock, flags);
895 if (--ring->irq_refcount == 0)
896 ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask);
897 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
898 }
899
900 static bool
901 i9xx_ring_get_irq(struct intel_ring_buffer *ring)
902 {
903 struct drm_device *dev = ring->dev;
904 struct drm_i915_private *dev_priv = dev->dev_private;
905 unsigned long flags;
906
907 if (!dev->irq_enabled)
908 return false;
909
910 spin_lock_irqsave(&dev_priv->irq_lock, flags);
911 if (ring->irq_refcount++ == 0) {
912 dev_priv->irq_mask &= ~ring->irq_enable_mask;
913 I915_WRITE(IMR, dev_priv->irq_mask);
914 POSTING_READ(IMR);
915 }
916 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
917
918 return true;
919 }
920
921 static void
922 i9xx_ring_put_irq(struct intel_ring_buffer *ring)
923 {
924 struct drm_device *dev = ring->dev;
925 struct drm_i915_private *dev_priv = dev->dev_private;
926 unsigned long flags;
927
928 spin_lock_irqsave(&dev_priv->irq_lock, flags);
929 if (--ring->irq_refcount == 0) {
930 dev_priv->irq_mask |= ring->irq_enable_mask;
931 I915_WRITE(IMR, dev_priv->irq_mask);
932 POSTING_READ(IMR);
933 }
934 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
935 }
936
937 static bool
938 i8xx_ring_get_irq(struct intel_ring_buffer *ring)
939 {
940 struct drm_device *dev = ring->dev;
941 struct drm_i915_private *dev_priv = dev->dev_private;
942 unsigned long flags;
943
944 if (!dev->irq_enabled)
945 return false;
946
947 spin_lock_irqsave(&dev_priv->irq_lock, flags);
948 if (ring->irq_refcount++ == 0) {
949 dev_priv->irq_mask &= ~ring->irq_enable_mask;
950 I915_WRITE16(IMR, dev_priv->irq_mask);
951 POSTING_READ16(IMR);
952 }
953 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
954
955 return true;
956 }
957
958 static void
959 i8xx_ring_put_irq(struct intel_ring_buffer *ring)
960 {
961 struct drm_device *dev = ring->dev;
962 struct drm_i915_private *dev_priv = dev->dev_private;
963 unsigned long flags;
964
965 spin_lock_irqsave(&dev_priv->irq_lock, flags);
966 if (--ring->irq_refcount == 0) {
967 dev_priv->irq_mask |= ring->irq_enable_mask;
968 I915_WRITE16(IMR, dev_priv->irq_mask);
969 POSTING_READ16(IMR);
970 }
971 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
972 }
973
974 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
975 {
976 struct drm_device *dev = ring->dev;
977 struct drm_i915_private *dev_priv = ring->dev->dev_private;
978 u32 mmio = 0;
979
980 /* The ring status page addresses are no longer next to the rest of
981 * the ring registers as of gen7.
982 */
983 if (IS_GEN7(dev)) {
984 switch (ring->id) {
985 case RCS:
986 mmio = RENDER_HWS_PGA_GEN7;
987 break;
988 case BCS:
989 mmio = BLT_HWS_PGA_GEN7;
990 break;
991 case VCS:
992 mmio = BSD_HWS_PGA_GEN7;
993 break;
994 case VECS:
995 mmio = VEBOX_HWS_PGA_GEN7;
996 break;
997 }
998 } else if (IS_GEN6(ring->dev)) {
999 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
1000 } else {
1001 /* XXX: gen8 returns to sanity */
1002 mmio = RING_HWS_PGA(ring->mmio_base);
1003 }
1004
1005 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
1006 POSTING_READ(mmio);
1007
1008 /*
1009 * Flush the TLB for this page
1010 *
1011 * FIXME: These two bits have disappeared on gen8, so a question
1012 * arises: do we still need this and if so how should we go about
1013 * invalidating the TLB?
1014 */
1015 if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
1016 u32 reg = RING_INSTPM(ring->mmio_base);
1017
1018 /* ring should be idle before issuing a sync flush*/
1019 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
1020
1021 I915_WRITE(reg,
1022 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
1023 INSTPM_SYNC_FLUSH));
1024 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
1025 1000))
1026 DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
1027 ring->name);
1028 }
1029 }
1030
1031 static int
1032 bsd_ring_flush(struct intel_ring_buffer *ring,
1033 u32 invalidate_domains,
1034 u32 flush_domains)
1035 {
1036 int ret;
1037
1038 ret = intel_ring_begin(ring, 2);
1039 if (ret)
1040 return ret;
1041
1042 intel_ring_emit(ring, MI_FLUSH);
1043 intel_ring_emit(ring, MI_NOOP);
1044 intel_ring_advance(ring);
1045 return 0;
1046 }
1047
1048 static int
1049 i9xx_add_request(struct intel_ring_buffer *ring)
1050 {
1051 int ret;
1052
1053 ret = intel_ring_begin(ring, 4);
1054 if (ret)
1055 return ret;
1056
1057 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1058 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1059 intel_ring_emit(ring, ring->outstanding_lazy_seqno);
1060 intel_ring_emit(ring, MI_USER_INTERRUPT);
1061 __intel_ring_advance(ring);
1062
1063 return 0;
1064 }
1065
1066 static bool
1067 gen6_ring_get_irq(struct intel_ring_buffer *ring)
1068 {
1069 struct drm_device *dev = ring->dev;
1070 struct drm_i915_private *dev_priv = dev->dev_private;
1071 unsigned long flags;
1072
1073 if (!dev->irq_enabled)
1074 return false;
1075
1076 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1077 if (ring->irq_refcount++ == 0) {
1078 if (HAS_L3_DPF(dev) && ring->id == RCS)
1079 I915_WRITE_IMR(ring,
1080 ~(ring->irq_enable_mask |
1081 GT_PARITY_ERROR(dev)));
1082 else
1083 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1084 ilk_enable_gt_irq(dev_priv, ring->irq_enable_mask);
1085 }
1086 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1087
1088 return true;
1089 }
1090
1091 static void
1092 gen6_ring_put_irq(struct intel_ring_buffer *ring)
1093 {
1094 struct drm_device *dev = ring->dev;
1095 struct drm_i915_private *dev_priv = dev->dev_private;
1096 unsigned long flags;
1097
1098 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1099 if (--ring->irq_refcount == 0) {
1100 if (HAS_L3_DPF(dev) && ring->id == RCS)
1101 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
1102 else
1103 I915_WRITE_IMR(ring, ~0);
1104 ilk_disable_gt_irq(dev_priv, ring->irq_enable_mask);
1105 }
1106 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1107 }
1108
1109 static bool
1110 hsw_vebox_get_irq(struct intel_ring_buffer *ring)
1111 {
1112 struct drm_device *dev = ring->dev;
1113 struct drm_i915_private *dev_priv = dev->dev_private;
1114 unsigned long flags;
1115
1116 if (!dev->irq_enabled)
1117 return false;
1118
1119 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1120 if (ring->irq_refcount++ == 0) {
1121 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1122 snb_enable_pm_irq(dev_priv, ring->irq_enable_mask);
1123 }
1124 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1125
1126 return true;
1127 }
1128
1129 static void
1130 hsw_vebox_put_irq(struct intel_ring_buffer *ring)
1131 {
1132 struct drm_device *dev = ring->dev;
1133 struct drm_i915_private *dev_priv = dev->dev_private;
1134 unsigned long flags;
1135
1136 if (!dev->irq_enabled)
1137 return;
1138
1139 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1140 if (--ring->irq_refcount == 0) {
1141 I915_WRITE_IMR(ring, ~0);
1142 snb_disable_pm_irq(dev_priv, ring->irq_enable_mask);
1143 }
1144 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1145 }
1146
1147 static bool
1148 gen8_ring_get_irq(struct intel_ring_buffer *ring)
1149 {
1150 struct drm_device *dev = ring->dev;
1151 struct drm_i915_private *dev_priv = dev->dev_private;
1152 unsigned long flags;
1153
1154 if (!dev->irq_enabled)
1155 return false;
1156
1157 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1158 if (ring->irq_refcount++ == 0) {
1159 if (HAS_L3_DPF(dev) && ring->id == RCS) {
1160 I915_WRITE_IMR(ring,
1161 ~(ring->irq_enable_mask |
1162 GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
1163 } else {
1164 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1165 }
1166 POSTING_READ(RING_IMR(ring->mmio_base));
1167 }
1168 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1169
1170 return true;
1171 }
1172
1173 static void
1174 gen8_ring_put_irq(struct intel_ring_buffer *ring)
1175 {
1176 struct drm_device *dev = ring->dev;
1177 struct drm_i915_private *dev_priv = dev->dev_private;
1178 unsigned long flags;
1179
1180 spin_lock_irqsave(&dev_priv->irq_lock, flags);
1181 if (--ring->irq_refcount == 0) {
1182 if (HAS_L3_DPF(dev) && ring->id == RCS) {
1183 I915_WRITE_IMR(ring,
1184 ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
1185 } else {
1186 I915_WRITE_IMR(ring, ~0);
1187 }
1188 POSTING_READ(RING_IMR(ring->mmio_base));
1189 }
1190 spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1191 }
1192
1193 static int
1194 i965_dispatch_execbuffer(struct intel_ring_buffer *ring,
1195 u32 offset, u32 length,
1196 unsigned flags)
1197 {
1198 int ret;
1199
1200 ret = intel_ring_begin(ring, 2);
1201 if (ret)
1202 return ret;
1203
1204 intel_ring_emit(ring,
1205 MI_BATCH_BUFFER_START |
1206 MI_BATCH_GTT |
1207 (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
1208 intel_ring_emit(ring, offset);
1209 intel_ring_advance(ring);
1210
1211 return 0;
1212 }
1213
1214 /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
1215 #define I830_BATCH_LIMIT (256*1024)
1216 static int
1217 i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
1218 u32 offset, u32 len,
1219 unsigned flags)
1220 {
1221 int ret;
1222
1223 if (flags & I915_DISPATCH_PINNED) {
1224 ret = intel_ring_begin(ring, 4);
1225 if (ret)
1226 return ret;
1227
1228 intel_ring_emit(ring, MI_BATCH_BUFFER);
1229 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1230 intel_ring_emit(ring, offset + len - 8);
1231 intel_ring_emit(ring, MI_NOOP);
1232 intel_ring_advance(ring);
1233 } else {
1234 u32 cs_offset = ring->scratch.gtt_offset;
1235
1236 if (len > I830_BATCH_LIMIT)
1237 return -ENOSPC;
1238
1239 ret = intel_ring_begin(ring, 9+3);
1240 if (ret)
1241 return ret;
1242 /* Blit the batch (which has now all relocs applied) to the stable batch
1243 * scratch bo area (so that the CS never stumbles over its tlb
1244 * invalidation bug) ... */
1245 intel_ring_emit(ring, XY_SRC_COPY_BLT_CMD |
1246 XY_SRC_COPY_BLT_WRITE_ALPHA |
1247 XY_SRC_COPY_BLT_WRITE_RGB);
1248 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_GXCOPY | 4096);
1249 intel_ring_emit(ring, 0);
1250 intel_ring_emit(ring, (DIV_ROUND_UP(len, 4096) << 16) | 1024);
1251 intel_ring_emit(ring, cs_offset);
1252 intel_ring_emit(ring, 0);
1253 intel_ring_emit(ring, 4096);
1254 intel_ring_emit(ring, offset);
1255 intel_ring_emit(ring, MI_FLUSH);
1256
1257 /* ... and execute it. */
1258 intel_ring_emit(ring, MI_BATCH_BUFFER);
1259 intel_ring_emit(ring, cs_offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1260 intel_ring_emit(ring, cs_offset + len - 8);
1261 intel_ring_advance(ring);
1262 }
1263
1264 return 0;
1265 }
1266
1267 static int
1268 i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
1269 u32 offset, u32 len,
1270 unsigned flags)
1271 {
1272 int ret;
1273
1274 ret = intel_ring_begin(ring, 2);
1275 if (ret)
1276 return ret;
1277
1278 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
1279 intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1280 intel_ring_advance(ring);
1281
1282 return 0;
1283 }
1284
1285 static void cleanup_status_page(struct intel_ring_buffer *ring)
1286 {
1287 struct drm_i915_gem_object *obj;
1288
1289 obj = ring->status_page.obj;
1290 if (obj == NULL)
1291 return;
1292
1293 kunmap(sg_page(obj->pages->sgl));
1294 i915_gem_object_ggtt_unpin(obj);
1295 drm_gem_object_unreference(&obj->base);
1296 ring->status_page.obj = NULL;
1297 }
1298
1299 static int init_status_page(struct intel_ring_buffer *ring)
1300 {
1301 struct drm_device *dev = ring->dev;
1302 struct drm_i915_gem_object *obj;
1303 int ret;
1304
1305 obj = i915_gem_alloc_object(dev, 4096);
1306 if (obj == NULL) {
1307 DRM_ERROR("Failed to allocate status page\n");
1308 ret = -ENOMEM;
1309 goto err;
1310 }
1311
1312 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
1313 if (ret)
1314 goto err_unref;
1315
1316 ret = i915_gem_obj_ggtt_pin(obj, 4096, 0);
1317 if (ret)
1318 goto err_unref;
1319
1320 ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
1321 ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
1322 if (ring->status_page.page_addr == NULL) {
1323 ret = -ENOMEM;
1324 goto err_unpin;
1325 }
1326 ring->status_page.obj = obj;
1327 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1328
1329 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
1330 ring->name, ring->status_page.gfx_addr);
1331
1332 return 0;
1333
1334 err_unpin:
1335 i915_gem_object_ggtt_unpin(obj);
1336 err_unref:
1337 drm_gem_object_unreference(&obj->base);
1338 err:
1339 return ret;
1340 }
1341
1342 static int init_phys_status_page(struct intel_ring_buffer *ring)
1343 {
1344 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1345
1346 if (!dev_priv->status_page_dmah) {
1347 dev_priv->status_page_dmah =
1348 drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
1349 if (!dev_priv->status_page_dmah)
1350 return -ENOMEM;
1351 }
1352
1353 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1354 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1355
1356 return 0;
1357 }
1358
1359 static int intel_init_ring_buffer(struct drm_device *dev,
1360 struct intel_ring_buffer *ring)
1361 {
1362 struct drm_i915_gem_object *obj;
1363 struct drm_i915_private *dev_priv = dev->dev_private;
1364 int ret;
1365
1366 ring->dev = dev;
1367 INIT_LIST_HEAD(&ring->active_list);
1368 INIT_LIST_HEAD(&ring->request_list);
1369 ring->size = 32 * PAGE_SIZE;
1370 memset(ring->sync_seqno, 0, sizeof(ring->sync_seqno));
1371
1372 init_waitqueue_head(&ring->irq_queue);
1373
1374 if (I915_NEED_GFX_HWS(dev)) {
1375 ret = init_status_page(ring);
1376 if (ret)
1377 return ret;
1378 } else {
1379 BUG_ON(ring->id != RCS);
1380 ret = init_phys_status_page(ring);
1381 if (ret)
1382 return ret;
1383 }
1384
1385 obj = NULL;
1386 if (!HAS_LLC(dev))
1387 obj = i915_gem_object_create_stolen(dev, ring->size);
1388 if (obj == NULL)
1389 obj = i915_gem_alloc_object(dev, ring->size);
1390 if (obj == NULL) {
1391 DRM_ERROR("Failed to allocate ringbuffer\n");
1392 ret = -ENOMEM;
1393 goto err_hws;
1394 }
1395
1396 ring->obj = obj;
1397
1398 ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
1399 if (ret)
1400 goto err_unref;
1401
1402 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1403 if (ret)
1404 goto err_unpin;
1405
1406 ring->virtual_start =
1407 ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj),
1408 ring->size);
1409 if (ring->virtual_start == NULL) {
1410 DRM_ERROR("Failed to map ringbuffer.\n");
1411 ret = -EINVAL;
1412 goto err_unpin;
1413 }
1414
1415 ret = ring->init(ring);
1416 if (ret)
1417 goto err_unmap;
1418
1419 /* Workaround an erratum on the i830 which causes a hang if
1420 * the TAIL pointer points to within the last 2 cachelines
1421 * of the buffer.
1422 */
1423 ring->effective_size = ring->size;
1424 if (IS_I830(ring->dev) || IS_845G(ring->dev))
1425 ring->effective_size -= 128;
1426
1427 i915_cmd_parser_init_ring(ring);
1428
1429 return 0;
1430
1431 err_unmap:
1432 iounmap(ring->virtual_start);
1433 err_unpin:
1434 i915_gem_object_ggtt_unpin(obj);
1435 err_unref:
1436 drm_gem_object_unreference(&obj->base);
1437 ring->obj = NULL;
1438 err_hws:
1439 cleanup_status_page(ring);
1440 return ret;
1441 }
1442
1443 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
1444 {
1445 struct drm_i915_private *dev_priv;
1446 int ret;
1447
1448 if (ring->obj == NULL)
1449 return;
1450
1451 /* Disable the ring buffer. The ring must be idle at this point */
1452 dev_priv = ring->dev->dev_private;
1453 ret = intel_ring_idle(ring);
1454 if (ret && !i915_reset_in_progress(&dev_priv->gpu_error))
1455 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
1456 ring->name, ret);
1457
1458 I915_WRITE_CTL(ring, 0);
1459
1460 iounmap(ring->virtual_start);
1461
1462 i915_gem_object_ggtt_unpin(ring->obj);
1463 drm_gem_object_unreference(&ring->obj->base);
1464 ring->obj = NULL;
1465 ring->preallocated_lazy_request = NULL;
1466 ring->outstanding_lazy_seqno = 0;
1467
1468 if (ring->cleanup)
1469 ring->cleanup(ring);
1470
1471 cleanup_status_page(ring);
1472 }
1473
1474 static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
1475 {
1476 struct drm_i915_gem_request *request;
1477 u32 seqno = 0, tail;
1478 int ret;
1479
1480 if (ring->last_retired_head != -1) {
1481 ring->head = ring->last_retired_head;
1482 ring->last_retired_head = -1;
1483
1484 ring->space = ring_space(ring);
1485 if (ring->space >= n)
1486 return 0;
1487 }
1488
1489 list_for_each_entry(request, &ring->request_list, list) {
1490 int space;
1491
1492 if (request->tail == -1)
1493 continue;
1494
1495 space = request->tail - (ring->tail + I915_RING_FREE_SPACE);
1496 if (space < 0)
1497 space += ring->size;
1498 if (space >= n) {
1499 seqno = request->seqno;
1500 tail = request->tail;
1501 break;
1502 }
1503
1504 /* Consume this request in case we need more space than
1505 * is available and so need to prevent a race between
1506 * updating last_retired_head and direct reads of
1507 * I915_RING_HEAD. It also provides a nice sanity check.
1508 */
1509 request->tail = -1;
1510 }
1511
1512 if (seqno == 0)
1513 return -ENOSPC;
1514
1515 ret = i915_wait_seqno(ring, seqno);
1516 if (ret)
1517 return ret;
1518
1519 ring->head = tail;
1520 ring->space = ring_space(ring);
1521 if (WARN_ON(ring->space < n))
1522 return -ENOSPC;
1523
1524 return 0;
1525 }
1526
1527 static int ring_wait_for_space(struct intel_ring_buffer *ring, int n)
1528 {
1529 struct drm_device *dev = ring->dev;
1530 struct drm_i915_private *dev_priv = dev->dev_private;
1531 unsigned long end;
1532 int ret;
1533
1534 ret = intel_ring_wait_request(ring, n);
1535 if (ret != -ENOSPC)
1536 return ret;
1537
1538 /* force the tail write in case we have been skipping them */
1539 __intel_ring_advance(ring);
1540
1541 trace_i915_ring_wait_begin(ring);
1542 /* With GEM the hangcheck timer should kick us out of the loop,
1543 * leaving it early runs the risk of corrupting GEM state (due
1544 * to running on almost untested codepaths). But on resume
1545 * timers don't work yet, so prevent a complete hang in that
1546 * case by choosing an insanely large timeout. */
1547 end = jiffies + 60 * HZ;
1548
1549 do {
1550 ring->head = I915_READ_HEAD(ring);
1551 ring->space = ring_space(ring);
1552 if (ring->space >= n) {
1553 trace_i915_ring_wait_end(ring);
1554 return 0;
1555 }
1556
1557 if (!drm_core_check_feature(dev, DRIVER_MODESET) &&
1558 dev->primary->master) {
1559 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1560 if (master_priv->sarea_priv)
1561 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1562 }
1563
1564 msleep(1);
1565
1566 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
1567 dev_priv->mm.interruptible);
1568 if (ret)
1569 return ret;
1570 } while (!time_after(jiffies, end));
1571 trace_i915_ring_wait_end(ring);
1572 return -EBUSY;
1573 }
1574
1575 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
1576 {
1577 uint32_t __iomem *virt;
1578 int rem = ring->size - ring->tail;
1579
1580 if (ring->space < rem) {
1581 int ret = ring_wait_for_space(ring, rem);
1582 if (ret)
1583 return ret;
1584 }
1585
1586 virt = ring->virtual_start + ring->tail;
1587 rem /= 4;
1588 while (rem--)
1589 iowrite32(MI_NOOP, virt++);
1590
1591 ring->tail = 0;
1592 ring->space = ring_space(ring);
1593
1594 return 0;
1595 }
1596
1597 int intel_ring_idle(struct intel_ring_buffer *ring)
1598 {
1599 u32 seqno;
1600 int ret;
1601
1602 /* We need to add any requests required to flush the objects and ring */
1603 if (ring->outstanding_lazy_seqno) {
1604 ret = i915_add_request(ring, NULL);
1605 if (ret)
1606 return ret;
1607 }
1608
1609 /* Wait upon the last request to be completed */
1610 if (list_empty(&ring->request_list))
1611 return 0;
1612
1613 seqno = list_entry(ring->request_list.prev,
1614 struct drm_i915_gem_request,
1615 list)->seqno;
1616
1617 return i915_wait_seqno(ring, seqno);
1618 }
1619
1620 static int
1621 intel_ring_alloc_seqno(struct intel_ring_buffer *ring)
1622 {
1623 if (ring->outstanding_lazy_seqno)
1624 return 0;
1625
1626 if (ring->preallocated_lazy_request == NULL) {
1627 struct drm_i915_gem_request *request;
1628
1629 request = kmalloc(sizeof(*request), GFP_KERNEL);
1630 if (request == NULL)
1631 return -ENOMEM;
1632
1633 ring->preallocated_lazy_request = request;
1634 }
1635
1636 return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
1637 }
1638
1639 static int __intel_ring_prepare(struct intel_ring_buffer *ring,
1640 int bytes)
1641 {
1642 int ret;
1643
1644 if (unlikely(ring->tail + bytes > ring->effective_size)) {
1645 ret = intel_wrap_ring_buffer(ring);
1646 if (unlikely(ret))
1647 return ret;
1648 }
1649
1650 if (unlikely(ring->space < bytes)) {
1651 ret = ring_wait_for_space(ring, bytes);
1652 if (unlikely(ret))
1653 return ret;
1654 }
1655
1656 return 0;
1657 }
1658
1659 int intel_ring_begin(struct intel_ring_buffer *ring,
1660 int num_dwords)
1661 {
1662 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1663 int ret;
1664
1665 ret = i915_gem_check_wedge(&dev_priv->gpu_error,
1666 dev_priv->mm.interruptible);
1667 if (ret)
1668 return ret;
1669
1670 ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
1671 if (ret)
1672 return ret;
1673
1674 /* Preallocate the olr before touching the ring */
1675 ret = intel_ring_alloc_seqno(ring);
1676 if (ret)
1677 return ret;
1678
1679 ring->space -= num_dwords * sizeof(uint32_t);
1680 return 0;
1681 }
1682
1683 /* Align the ring tail to a cacheline boundary */
1684 int intel_ring_cacheline_align(struct intel_ring_buffer *ring)
1685 {
1686 int num_dwords = (64 - (ring->tail & 63)) / sizeof(uint32_t);
1687 int ret;
1688
1689 if (num_dwords == 0)
1690 return 0;
1691
1692 ret = intel_ring_begin(ring, num_dwords);
1693 if (ret)
1694 return ret;
1695
1696 while (num_dwords--)
1697 intel_ring_emit(ring, MI_NOOP);
1698
1699 intel_ring_advance(ring);
1700
1701 return 0;
1702 }
1703
1704 void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno)
1705 {
1706 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1707
1708 BUG_ON(ring->outstanding_lazy_seqno);
1709
1710 if (INTEL_INFO(ring->dev)->gen >= 6) {
1711 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
1712 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
1713 if (HAS_VEBOX(ring->dev))
1714 I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
1715 }
1716
1717 ring->set_seqno(ring, seqno);
1718 ring->hangcheck.seqno = seqno;
1719 }
1720
1721 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1722 u32 value)
1723 {
1724 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1725
1726 /* Every tail move must follow the sequence below */
1727
1728 /* Disable notification that the ring is IDLE. The GT
1729 * will then assume that it is busy and bring it out of rc6.
1730 */
1731 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1732 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1733
1734 /* Clear the context id. Here be magic! */
1735 I915_WRITE64(GEN6_BSD_RNCID, 0x0);
1736
1737 /* Wait for the ring not to be idle, i.e. for it to wake up. */
1738 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1739 GEN6_BSD_SLEEP_INDICATOR) == 0,
1740 50))
1741 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
1742
1743 /* Now that the ring is fully powered up, update the tail */
1744 I915_WRITE_TAIL(ring, value);
1745 POSTING_READ(RING_TAIL(ring->mmio_base));
1746
1747 /* Let the ring send IDLE messages to the GT again,
1748 * and so let it sleep to conserve power when idle.
1749 */
1750 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1751 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1752 }
1753
1754 static int gen6_bsd_ring_flush(struct intel_ring_buffer *ring,
1755 u32 invalidate, u32 flush)
1756 {
1757 uint32_t cmd;
1758 int ret;
1759
1760 ret = intel_ring_begin(ring, 4);
1761 if (ret)
1762 return ret;
1763
1764 cmd = MI_FLUSH_DW;
1765 if (INTEL_INFO(ring->dev)->gen >= 8)
1766 cmd += 1;
1767 /*
1768 * Bspec vol 1c.5 - video engine command streamer:
1769 * "If ENABLED, all TLBs will be invalidated once the flush
1770 * operation is complete. This bit is only valid when the
1771 * Post-Sync Operation field is a value of 1h or 3h."
1772 */
1773 if (invalidate & I915_GEM_GPU_DOMAINS)
1774 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
1775 MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
1776 intel_ring_emit(ring, cmd);
1777 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
1778 if (INTEL_INFO(ring->dev)->gen >= 8) {
1779 intel_ring_emit(ring, 0); /* upper addr */
1780 intel_ring_emit(ring, 0); /* value */
1781 } else {
1782 intel_ring_emit(ring, 0);
1783 intel_ring_emit(ring, MI_NOOP);
1784 }
1785 intel_ring_advance(ring);
1786 return 0;
1787 }
1788
1789 static int
1790 gen8_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1791 u32 offset, u32 len,
1792 unsigned flags)
1793 {
1794 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1795 bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL &&
1796 !(flags & I915_DISPATCH_SECURE);
1797 int ret;
1798
1799 ret = intel_ring_begin(ring, 4);
1800 if (ret)
1801 return ret;
1802
1803 /* FIXME(BDW): Address space and security selectors. */
1804 intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
1805 intel_ring_emit(ring, offset);
1806 intel_ring_emit(ring, 0);
1807 intel_ring_emit(ring, MI_NOOP);
1808 intel_ring_advance(ring);
1809
1810 return 0;
1811 }
1812
1813 static int
1814 hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1815 u32 offset, u32 len,
1816 unsigned flags)
1817 {
1818 int ret;
1819
1820 ret = intel_ring_begin(ring, 2);
1821 if (ret)
1822 return ret;
1823
1824 intel_ring_emit(ring,
1825 MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW |
1826 (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
1827 /* bit0-7 is the length on GEN6+ */
1828 intel_ring_emit(ring, offset);
1829 intel_ring_advance(ring);
1830
1831 return 0;
1832 }
1833
1834 static int
1835 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1836 u32 offset, u32 len,
1837 unsigned flags)
1838 {
1839 int ret;
1840
1841 ret = intel_ring_begin(ring, 2);
1842 if (ret)
1843 return ret;
1844
1845 intel_ring_emit(ring,
1846 MI_BATCH_BUFFER_START |
1847 (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
1848 /* bit0-7 is the length on GEN6+ */
1849 intel_ring_emit(ring, offset);
1850 intel_ring_advance(ring);
1851
1852 return 0;
1853 }
1854
1855 /* Blitter support (SandyBridge+) */
1856
1857 static int gen6_ring_flush(struct intel_ring_buffer *ring,
1858 u32 invalidate, u32 flush)
1859 {
1860 struct drm_device *dev = ring->dev;
1861 uint32_t cmd;
1862 int ret;
1863
1864 ret = intel_ring_begin(ring, 4);
1865 if (ret)
1866 return ret;
1867
1868 cmd = MI_FLUSH_DW;
1869 if (INTEL_INFO(ring->dev)->gen >= 8)
1870 cmd += 1;
1871 /*
1872 * Bspec vol 1c.3 - blitter engine command streamer:
1873 * "If ENABLED, all TLBs will be invalidated once the flush
1874 * operation is complete. This bit is only valid when the
1875 * Post-Sync Operation field is a value of 1h or 3h."
1876 */
1877 if (invalidate & I915_GEM_DOMAIN_RENDER)
1878 cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
1879 MI_FLUSH_DW_OP_STOREDW;
1880 intel_ring_emit(ring, cmd);
1881 intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
1882 if (INTEL_INFO(ring->dev)->gen >= 8) {
1883 intel_ring_emit(ring, 0); /* upper addr */
1884 intel_ring_emit(ring, 0); /* value */
1885 } else {
1886 intel_ring_emit(ring, 0);
1887 intel_ring_emit(ring, MI_NOOP);
1888 }
1889 intel_ring_advance(ring);
1890
1891 if (IS_GEN7(dev) && !invalidate && flush)
1892 return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN);
1893
1894 return 0;
1895 }
1896
1897 int intel_init_render_ring_buffer(struct drm_device *dev)
1898 {
1899 struct drm_i915_private *dev_priv = dev->dev_private;
1900 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1901
1902 ring->name = "render ring";
1903 ring->id = RCS;
1904 ring->mmio_base = RENDER_RING_BASE;
1905
1906 if (INTEL_INFO(dev)->gen >= 6) {
1907 ring->add_request = gen6_add_request;
1908 ring->flush = gen7_render_ring_flush;
1909 if (INTEL_INFO(dev)->gen == 6)
1910 ring->flush = gen6_render_ring_flush;
1911 if (INTEL_INFO(dev)->gen >= 8) {
1912 ring->flush = gen8_render_ring_flush;
1913 ring->irq_get = gen8_ring_get_irq;
1914 ring->irq_put = gen8_ring_put_irq;
1915 } else {
1916 ring->irq_get = gen6_ring_get_irq;
1917 ring->irq_put = gen6_ring_put_irq;
1918 }
1919 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
1920 ring->get_seqno = gen6_ring_get_seqno;
1921 ring->set_seqno = ring_set_seqno;
1922 ring->sync_to = gen6_ring_sync;
1923 ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_INVALID;
1924 ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_RV;
1925 ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_RB;
1926 ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_RVE;
1927 ring->signal_mbox[RCS] = GEN6_NOSYNC;
1928 ring->signal_mbox[VCS] = GEN6_VRSYNC;
1929 ring->signal_mbox[BCS] = GEN6_BRSYNC;
1930 ring->signal_mbox[VECS] = GEN6_VERSYNC;
1931 } else if (IS_GEN5(dev)) {
1932 ring->add_request = pc_render_add_request;
1933 ring->flush = gen4_render_ring_flush;
1934 ring->get_seqno = pc_render_get_seqno;
1935 ring->set_seqno = pc_render_set_seqno;
1936 ring->irq_get = gen5_ring_get_irq;
1937 ring->irq_put = gen5_ring_put_irq;
1938 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
1939 GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
1940 } else {
1941 ring->add_request = i9xx_add_request;
1942 if (INTEL_INFO(dev)->gen < 4)
1943 ring->flush = gen2_render_ring_flush;
1944 else
1945 ring->flush = gen4_render_ring_flush;
1946 ring->get_seqno = ring_get_seqno;
1947 ring->set_seqno = ring_set_seqno;
1948 if (IS_GEN2(dev)) {
1949 ring->irq_get = i8xx_ring_get_irq;
1950 ring->irq_put = i8xx_ring_put_irq;
1951 } else {
1952 ring->irq_get = i9xx_ring_get_irq;
1953 ring->irq_put = i9xx_ring_put_irq;
1954 }
1955 ring->irq_enable_mask = I915_USER_INTERRUPT;
1956 }
1957 ring->write_tail = ring_write_tail;
1958 if (IS_HASWELL(dev))
1959 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
1960 else if (IS_GEN8(dev))
1961 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
1962 else if (INTEL_INFO(dev)->gen >= 6)
1963 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1964 else if (INTEL_INFO(dev)->gen >= 4)
1965 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1966 else if (IS_I830(dev) || IS_845G(dev))
1967 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1968 else
1969 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1970 ring->init = init_render_ring;
1971 ring->cleanup = render_ring_cleanup;
1972
1973 /* Workaround batchbuffer to combat CS tlb bug. */
1974 if (HAS_BROKEN_CS_TLB(dev)) {
1975 struct drm_i915_gem_object *obj;
1976 int ret;
1977
1978 obj = i915_gem_alloc_object(dev, I830_BATCH_LIMIT);
1979 if (obj == NULL) {
1980 DRM_ERROR("Failed to allocate batch bo\n");
1981 return -ENOMEM;
1982 }
1983
1984 ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
1985 if (ret != 0) {
1986 drm_gem_object_unreference(&obj->base);
1987 DRM_ERROR("Failed to ping batch bo\n");
1988 return ret;
1989 }
1990
1991 ring->scratch.obj = obj;
1992 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
1993 }
1994
1995 return intel_init_ring_buffer(dev, ring);
1996 }
1997
1998 int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1999 {
2000 struct drm_i915_private *dev_priv = dev->dev_private;
2001 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
2002 int ret;
2003
2004 ring->name = "render ring";
2005 ring->id = RCS;
2006 ring->mmio_base = RENDER_RING_BASE;
2007
2008 if (INTEL_INFO(dev)->gen >= 6) {
2009 /* non-kms not supported on gen6+ */
2010 return -ENODEV;
2011 }
2012
2013 /* Note: gem is not supported on gen5/ilk without kms (the corresponding
2014 * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
2015 * the special gen5 functions. */
2016 ring->add_request = i9xx_add_request;
2017 if (INTEL_INFO(dev)->gen < 4)
2018 ring->flush = gen2_render_ring_flush;
2019 else
2020 ring->flush = gen4_render_ring_flush;
2021 ring->get_seqno = ring_get_seqno;
2022 ring->set_seqno = ring_set_seqno;
2023 if (IS_GEN2(dev)) {
2024 ring->irq_get = i8xx_ring_get_irq;
2025 ring->irq_put = i8xx_ring_put_irq;
2026 } else {
2027 ring->irq_get = i9xx_ring_get_irq;
2028 ring->irq_put = i9xx_ring_put_irq;
2029 }
2030 ring->irq_enable_mask = I915_USER_INTERRUPT;
2031 ring->write_tail = ring_write_tail;
2032 if (INTEL_INFO(dev)->gen >= 4)
2033 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
2034 else if (IS_I830(dev) || IS_845G(dev))
2035 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
2036 else
2037 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
2038 ring->init = init_render_ring;
2039 ring->cleanup = render_ring_cleanup;
2040
2041 ring->dev = dev;
2042 INIT_LIST_HEAD(&ring->active_list);
2043 INIT_LIST_HEAD(&ring->request_list);
2044
2045 ring->size = size;
2046 ring->effective_size = ring->size;
2047 if (IS_I830(ring->dev) || IS_845G(ring->dev))
2048 ring->effective_size -= 128;
2049
2050 ring->virtual_start = ioremap_wc(start, size);
2051 if (ring->virtual_start == NULL) {
2052 DRM_ERROR("can not ioremap virtual address for"
2053 " ring buffer\n");
2054 return -ENOMEM;
2055 }
2056
2057 if (!I915_NEED_GFX_HWS(dev)) {
2058 ret = init_phys_status_page(ring);
2059 if (ret)
2060 return ret;
2061 }
2062
2063 return 0;
2064 }
2065
2066 int intel_init_bsd_ring_buffer(struct drm_device *dev)
2067 {
2068 struct drm_i915_private *dev_priv = dev->dev_private;
2069 struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
2070
2071 ring->name = "bsd ring";
2072 ring->id = VCS;
2073
2074 ring->write_tail = ring_write_tail;
2075 if (INTEL_INFO(dev)->gen >= 6) {
2076 ring->mmio_base = GEN6_BSD_RING_BASE;
2077 /* gen6 bsd needs a special wa for tail updates */
2078 if (IS_GEN6(dev))
2079 ring->write_tail = gen6_bsd_ring_write_tail;
2080 ring->flush = gen6_bsd_ring_flush;
2081 ring->add_request = gen6_add_request;
2082 ring->get_seqno = gen6_ring_get_seqno;
2083 ring->set_seqno = ring_set_seqno;
2084 if (INTEL_INFO(dev)->gen >= 8) {
2085 ring->irq_enable_mask =
2086 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
2087 ring->irq_get = gen8_ring_get_irq;
2088 ring->irq_put = gen8_ring_put_irq;
2089 ring->dispatch_execbuffer =
2090 gen8_ring_dispatch_execbuffer;
2091 } else {
2092 ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
2093 ring->irq_get = gen6_ring_get_irq;
2094 ring->irq_put = gen6_ring_put_irq;
2095 ring->dispatch_execbuffer =
2096 gen6_ring_dispatch_execbuffer;
2097 }
2098 ring->sync_to = gen6_ring_sync;
2099 ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VR;
2100 ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_INVALID;
2101 ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VB;
2102 ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_VVE;
2103 ring->signal_mbox[RCS] = GEN6_RVSYNC;
2104 ring->signal_mbox[VCS] = GEN6_NOSYNC;
2105 ring->signal_mbox[BCS] = GEN6_BVSYNC;
2106 ring->signal_mbox[VECS] = GEN6_VEVSYNC;
2107 } else {
2108 ring->mmio_base = BSD_RING_BASE;
2109 ring->flush = bsd_ring_flush;
2110 ring->add_request = i9xx_add_request;
2111 ring->get_seqno = ring_get_seqno;
2112 ring->set_seqno = ring_set_seqno;
2113 if (IS_GEN5(dev)) {
2114 ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
2115 ring->irq_get = gen5_ring_get_irq;
2116 ring->irq_put = gen5_ring_put_irq;
2117 } else {
2118 ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
2119 ring->irq_get = i9xx_ring_get_irq;
2120 ring->irq_put = i9xx_ring_put_irq;
2121 }
2122 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
2123 }
2124 ring->init = init_ring_common;
2125
2126 return intel_init_ring_buffer(dev, ring);
2127 }
2128
2129 int intel_init_blt_ring_buffer(struct drm_device *dev)
2130 {
2131 struct drm_i915_private *dev_priv = dev->dev_private;
2132 struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
2133
2134 ring->name = "blitter ring";
2135 ring->id = BCS;
2136
2137 ring->mmio_base = BLT_RING_BASE;
2138 ring->write_tail = ring_write_tail;
2139 ring->flush = gen6_ring_flush;
2140 ring->add_request = gen6_add_request;
2141 ring->get_seqno = gen6_ring_get_seqno;
2142 ring->set_seqno = ring_set_seqno;
2143 if (INTEL_INFO(dev)->gen >= 8) {
2144 ring->irq_enable_mask =
2145 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
2146 ring->irq_get = gen8_ring_get_irq;
2147 ring->irq_put = gen8_ring_put_irq;
2148 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2149 } else {
2150 ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
2151 ring->irq_get = gen6_ring_get_irq;
2152 ring->irq_put = gen6_ring_put_irq;
2153 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2154 }
2155 ring->sync_to = gen6_ring_sync;
2156 ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_BR;
2157 ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_BV;
2158 ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_INVALID;
2159 ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_BVE;
2160 ring->signal_mbox[RCS] = GEN6_RBSYNC;
2161 ring->signal_mbox[VCS] = GEN6_VBSYNC;
2162 ring->signal_mbox[BCS] = GEN6_NOSYNC;
2163 ring->signal_mbox[VECS] = GEN6_VEBSYNC;
2164 ring->init = init_ring_common;
2165
2166 return intel_init_ring_buffer(dev, ring);
2167 }
2168
2169 int intel_init_vebox_ring_buffer(struct drm_device *dev)
2170 {
2171 struct drm_i915_private *dev_priv = dev->dev_private;
2172 struct intel_ring_buffer *ring = &dev_priv->ring[VECS];
2173
2174 ring->name = "video enhancement ring";
2175 ring->id = VECS;
2176
2177 ring->mmio_base = VEBOX_RING_BASE;
2178 ring->write_tail = ring_write_tail;
2179 ring->flush = gen6_ring_flush;
2180 ring->add_request = gen6_add_request;
2181 ring->get_seqno = gen6_ring_get_seqno;
2182 ring->set_seqno = ring_set_seqno;
2183
2184 if (INTEL_INFO(dev)->gen >= 8) {
2185 ring->irq_enable_mask =
2186 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
2187 ring->irq_get = gen8_ring_get_irq;
2188 ring->irq_put = gen8_ring_put_irq;
2189 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2190 } else {
2191 ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
2192 ring->irq_get = hsw_vebox_get_irq;
2193 ring->irq_put = hsw_vebox_put_irq;
2194 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2195 }
2196 ring->sync_to = gen6_ring_sync;
2197 ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VER;
2198 ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_VEV;
2199 ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VEB;
2200 ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_INVALID;
2201 ring->signal_mbox[RCS] = GEN6_RVESYNC;
2202 ring->signal_mbox[VCS] = GEN6_VVESYNC;
2203 ring->signal_mbox[BCS] = GEN6_BVESYNC;
2204 ring->signal_mbox[VECS] = GEN6_NOSYNC;
2205 ring->init = init_ring_common;
2206
2207 return intel_init_ring_buffer(dev, ring);
2208 }
2209
2210 int
2211 intel_ring_flush_all_caches(struct intel_ring_buffer *ring)
2212 {
2213 int ret;
2214
2215 if (!ring->gpu_caches_dirty)
2216 return 0;
2217
2218 ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
2219 if (ret)
2220 return ret;
2221
2222 trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
2223
2224 ring->gpu_caches_dirty = false;
2225 return 0;
2226 }
2227
2228 int
2229 intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring)
2230 {
2231 uint32_t flush_domains;
2232 int ret;
2233
2234 flush_domains = 0;
2235 if (ring->gpu_caches_dirty)
2236 flush_domains = I915_GEM_GPU_DOMAINS;
2237
2238 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
2239 if (ret)
2240 return ret;
2241
2242 trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
2243
2244 ring->gpu_caches_dirty = false;
2245 return 0;
2246 }
Linux kernel - Deliverables
This page took 0.100943 seconds and 6 git commands to generate.