projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/i915/dp: convert to encoder disable/enable
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem_execbuffer.c
1 /*
2 * Copyright © 2008,2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Chris Wilson <chris@chris-wilson.co.uk>
26 *
27 */
28
29 #include "drmP.h"
30 #include "drm.h"
31 #include "i915_drm.h"
32 #include "i915_drv.h"
33 #include "i915_trace.h"
34 #include "intel_drv.h"
35 #include <linux/dma_remapping.h>
36
37 struct eb_objects {
38 int and;
39 struct hlist_head buckets[0];
40 };
41
42 static struct eb_objects *
43 eb_create(int size)
44 {
45 struct eb_objects *eb;
46 int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
47 while (count > size)
48 count >>= 1;
49 eb = kzalloc(count*sizeof(struct hlist_head) +
50 sizeof(struct eb_objects),
51 GFP_KERNEL);
52 if (eb == NULL)
53 return eb;
54
55 eb->and = count - 1;
56 return eb;
57 }
58
59 static void
60 eb_reset(struct eb_objects *eb)
61 {
62 memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
63 }
64
65 static void
66 eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj)
67 {
68 hlist_add_head(&obj->exec_node,
69 &eb->buckets[obj->exec_handle & eb->and]);
70 }
71
72 static struct drm_i915_gem_object *
73 eb_get_object(struct eb_objects *eb, unsigned long handle)
74 {
75 struct hlist_head *head;
76 struct hlist_node *node;
77 struct drm_i915_gem_object *obj;
78
79 head = &eb->buckets[handle & eb->and];
80 hlist_for_each(node, head) {
81 obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
82 if (obj->exec_handle == handle)
83 return obj;
84 }
85
86 return NULL;
87 }
88
89 static void
90 eb_destroy(struct eb_objects *eb)
91 {
92 kfree(eb);
93 }
94
95 static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
96 {
97 return (obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
98 obj->cache_level != I915_CACHE_NONE);
99 }
100
101 static int
102 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
103 struct eb_objects *eb,
104 struct drm_i915_gem_relocation_entry *reloc)
105 {
106 struct drm_device *dev = obj->base.dev;
107 struct drm_gem_object *target_obj;
108 struct drm_i915_gem_object *target_i915_obj;
109 uint32_t target_offset;
110 int ret = -EINVAL;
111
112 /* we've already hold a reference to all valid objects */
113 target_obj = &eb_get_object(eb, reloc->target_handle)->base;
114 if (unlikely(target_obj == NULL))
115 return -ENOENT;
116
117 target_i915_obj = to_intel_bo(target_obj);
118 target_offset = target_i915_obj->gtt_offset;
119
120 /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
121 * pipe_control writes because the gpu doesn't properly redirect them
122 * through the ppgtt for non_secure batchbuffers. */
123 if (unlikely(IS_GEN6(dev) &&
124 reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
125 !target_i915_obj->has_global_gtt_mapping)) {
126 i915_gem_gtt_bind_object(target_i915_obj,
127 target_i915_obj->cache_level);
128 }
129
130 /* The target buffer should have appeared before us in the
131 * exec_object list, so it should have a GTT space bound by now.
132 */
133 if (unlikely(target_offset == 0)) {
134 DRM_DEBUG("No GTT space found for object %d\n",
135 reloc->target_handle);
136 return ret;
137 }
138
139 /* Validate that the target is in a valid r/w GPU domain */
140 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
141 DRM_DEBUG("reloc with multiple write domains: "
142 "obj %p target %d offset %d "
143 "read %08x write %08x",
144 obj, reloc->target_handle,
145 (int) reloc->offset,
146 reloc->read_domains,
147 reloc->write_domain);
148 return ret;
149 }
150 if (unlikely((reloc->write_domain | reloc->read_domains)
151 & ~I915_GEM_GPU_DOMAINS)) {
152 DRM_DEBUG("reloc with read/write non-GPU domains: "
153 "obj %p target %d offset %d "
154 "read %08x write %08x",
155 obj, reloc->target_handle,
156 (int) reloc->offset,
157 reloc->read_domains,
158 reloc->write_domain);
159 return ret;
160 }
161 if (unlikely(reloc->write_domain && target_obj->pending_write_domain &&
162 reloc->write_domain != target_obj->pending_write_domain)) {
163 DRM_DEBUG("Write domain conflict: "
164 "obj %p target %d offset %d "
165 "new %08x old %08x\n",
166 obj, reloc->target_handle,
167 (int) reloc->offset,
168 reloc->write_domain,
169 target_obj->pending_write_domain);
170 return ret;
171 }
172
173 target_obj->pending_read_domains |= reloc->read_domains;
174 target_obj->pending_write_domain |= reloc->write_domain;
175
176 /* If the relocation already has the right value in it, no
177 * more work needs to be done.
178 */
179 if (target_offset == reloc->presumed_offset)
180 return 0;
181
182 /* Check that the relocation address is valid... */
183 if (unlikely(reloc->offset > obj->base.size - 4)) {
184 DRM_DEBUG("Relocation beyond object bounds: "
185 "obj %p target %d offset %d size %d.\n",
186 obj, reloc->target_handle,
187 (int) reloc->offset,
188 (int) obj->base.size);
189 return ret;
190 }
191 if (unlikely(reloc->offset & 3)) {
192 DRM_DEBUG("Relocation not 4-byte aligned: "
193 "obj %p target %d offset %d.\n",
194 obj, reloc->target_handle,
195 (int) reloc->offset);
196 return ret;
197 }
198
199 /* We can't wait for rendering with pagefaults disabled */
200 if (obj->active && in_atomic())
201 return -EFAULT;
202
203 reloc->delta += target_offset;
204 if (use_cpu_reloc(obj)) {
205 uint32_t page_offset = reloc->offset & ~PAGE_MASK;
206 char *vaddr;
207
208 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
209 if (ret)
210 return ret;
211
212 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
213 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
214 kunmap_atomic(vaddr);
215 } else {
216 struct drm_i915_private *dev_priv = dev->dev_private;
217 uint32_t __iomem *reloc_entry;
218 void __iomem *reloc_page;
219
220 ret = i915_gem_object_set_to_gtt_domain(obj, true);
221 if (ret)
222 return ret;
223
224 ret = i915_gem_object_put_fence(obj);
225 if (ret)
226 return ret;
227
228 /* Map the page containing the relocation we're going to perform. */
229 reloc->offset += obj->gtt_offset;
230 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
231 reloc->offset & PAGE_MASK);
232 reloc_entry = (uint32_t __iomem *)
233 (reloc_page + (reloc->offset & ~PAGE_MASK));
234 iowrite32(reloc->delta, reloc_entry);
235 io_mapping_unmap_atomic(reloc_page);
236 }
237
238 /* and update the user's relocation entry */
239 reloc->presumed_offset = target_offset;
240
241 return 0;
242 }
243
244 static int
245 i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
246 struct eb_objects *eb)
247 {
248 #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
249 struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
250 struct drm_i915_gem_relocation_entry __user *user_relocs;
251 struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
252 int remain, ret;
253
254 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
255
256 remain = entry->relocation_count;
257 while (remain) {
258 struct drm_i915_gem_relocation_entry *r = stack_reloc;
259 int count = remain;
260 if (count > ARRAY_SIZE(stack_reloc))
261 count = ARRAY_SIZE(stack_reloc);
262 remain -= count;
263
264 if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0])))
265 return -EFAULT;
266
267 do {
268 u64 offset = r->presumed_offset;
269
270 ret = i915_gem_execbuffer_relocate_entry(obj, eb, r);
271 if (ret)
272 return ret;
273
274 if (r->presumed_offset != offset &&
275 __copy_to_user_inatomic(&user_relocs->presumed_offset,
276 &r->presumed_offset,
277 sizeof(r->presumed_offset))) {
278 return -EFAULT;
279 }
280
281 user_relocs++;
282 r++;
283 } while (--count);
284 }
285
286 return 0;
287 #undef N_RELOC
288 }
289
290 static int
291 i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
292 struct eb_objects *eb,
293 struct drm_i915_gem_relocation_entry *relocs)
294 {
295 const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
296 int i, ret;
297
298 for (i = 0; i < entry->relocation_count; i++) {
299 ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]);
300 if (ret)
301 return ret;
302 }
303
304 return 0;
305 }
306
307 static int
308 i915_gem_execbuffer_relocate(struct drm_device *dev,
309 struct eb_objects *eb,
310 struct list_head *objects)
311 {
312 struct drm_i915_gem_object *obj;
313 int ret = 0;
314
315 /* This is the fast path and we cannot handle a pagefault whilst
316 * holding the struct mutex lest the user pass in the relocations
317 * contained within a mmaped bo. For in such a case we, the page
318 * fault handler would call i915_gem_fault() and we would try to
319 * acquire the struct mutex again. Obviously this is bad and so
320 * lockdep complains vehemently.
321 */
322 pagefault_disable();
323 list_for_each_entry(obj, objects, exec_list) {
324 ret = i915_gem_execbuffer_relocate_object(obj, eb);
325 if (ret)
326 break;
327 }
328 pagefault_enable();
329
330 return ret;
331 }
332
333 #define __EXEC_OBJECT_HAS_FENCE (1<<31)
334
335 static int
336 need_reloc_mappable(struct drm_i915_gem_object *obj)
337 {
338 struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
339 return entry->relocation_count && !use_cpu_reloc(obj);
340 }
341
342 static int
343 pin_and_fence_object(struct drm_i915_gem_object *obj,
344 struct intel_ring_buffer *ring)
345 {
346 struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
347 bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
348 bool need_fence, need_mappable;
349 int ret;
350
351 need_fence =
352 has_fenced_gpu_access &&
353 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
354 obj->tiling_mode != I915_TILING_NONE;
355 need_mappable = need_fence || need_reloc_mappable(obj);
356
357 ret = i915_gem_object_pin(obj, entry->alignment, need_mappable);
358 if (ret)
359 return ret;
360
361 if (has_fenced_gpu_access) {
362 if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
363 ret = i915_gem_object_get_fence(obj);
364 if (ret)
365 goto err_unpin;
366
367 if (i915_gem_object_pin_fence(obj))
368 entry->flags |= __EXEC_OBJECT_HAS_FENCE;
369
370 obj->pending_fenced_gpu_access = true;
371 }
372 }
373
374 entry->offset = obj->gtt_offset;
375 return 0;
376
377 err_unpin:
378 i915_gem_object_unpin(obj);
379 return ret;
380 }
381
382 static int
383 i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
384 struct drm_file *file,
385 struct list_head *objects)
386 {
387 drm_i915_private_t *dev_priv = ring->dev->dev_private;
388 struct drm_i915_gem_object *obj;
389 int ret, retry;
390 bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
391 struct list_head ordered_objects;
392
393 INIT_LIST_HEAD(&ordered_objects);
394 while (!list_empty(objects)) {
395 struct drm_i915_gem_exec_object2 *entry;
396 bool need_fence, need_mappable;
397
398 obj = list_first_entry(objects,
399 struct drm_i915_gem_object,
400 exec_list);
401 entry = obj->exec_entry;
402
403 need_fence =
404 has_fenced_gpu_access &&
405 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
406 obj->tiling_mode != I915_TILING_NONE;
407 need_mappable = need_fence || need_reloc_mappable(obj);
408
409 if (need_mappable)
410 list_move(&obj->exec_list, &ordered_objects);
411 else
412 list_move_tail(&obj->exec_list, &ordered_objects);
413
414 obj->base.pending_read_domains = 0;
415 obj->base.pending_write_domain = 0;
416 obj->pending_fenced_gpu_access = false;
417 }
418 list_splice(&ordered_objects, objects);
419
420 /* Attempt to pin all of the buffers into the GTT.
421 * This is done in 3 phases:
422 *
423 * 1a. Unbind all objects that do not match the GTT constraints for
424 * the execbuffer (fenceable, mappable, alignment etc).
425 * 1b. Increment pin count for already bound objects.
426 * 2. Bind new objects.
427 * 3. Decrement pin count.
428 *
429 * This avoid unnecessary unbinding of later objects in order to makr
430 * room for the earlier objects *unless* we need to defragment.
431 */
432 retry = 0;
433 do {
434 ret = 0;
435
436 /* Unbind any ill-fitting objects or pin. */
437 list_for_each_entry(obj, objects, exec_list) {
438 struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
439 bool need_fence, need_mappable;
440
441 if (!obj->gtt_space)
442 continue;
443
444 need_fence =
445 has_fenced_gpu_access &&
446 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
447 obj->tiling_mode != I915_TILING_NONE;
448 need_mappable = need_fence || need_reloc_mappable(obj);
449
450 if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) ||
451 (need_mappable && !obj->map_and_fenceable))
452 ret = i915_gem_object_unbind(obj);
453 else
454 ret = pin_and_fence_object(obj, ring);
455 if (ret)
456 goto err;
457 }
458
459 /* Bind fresh objects */
460 list_for_each_entry(obj, objects, exec_list) {
461 if (obj->gtt_space)
462 continue;
463
464 ret = pin_and_fence_object(obj, ring);
465 if (ret) {
466 int ret_ignore;
467
468 /* This can potentially raise a harmless
469 * -EINVAL if we failed to bind in the above
470 * call. It cannot raise -EINTR since we know
471 * that the bo is freshly bound and so will
472 * not need to be flushed or waited upon.
473 */
474 ret_ignore = i915_gem_object_unbind(obj);
475 (void)ret_ignore;
476 WARN_ON(obj->gtt_space);
477 break;
478 }
479 }
480
481 /* Decrement pin count for bound objects */
482 list_for_each_entry(obj, objects, exec_list) {
483 struct drm_i915_gem_exec_object2 *entry;
484
485 if (!obj->gtt_space)
486 continue;
487
488 entry = obj->exec_entry;
489 if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
490 i915_gem_object_unpin_fence(obj);
491 entry->flags &= ~__EXEC_OBJECT_HAS_FENCE;
492 }
493
494 i915_gem_object_unpin(obj);
495
496 /* ... and ensure ppgtt mapping exist if needed. */
497 if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) {
498 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
499 obj, obj->cache_level);
500
501 obj->has_aliasing_ppgtt_mapping = 1;
502 }
503 }
504
505 if (ret != -ENOSPC || retry > 1)
506 return ret;
507
508 /* First attempt, just clear anything that is purgeable.
509 * Second attempt, clear the entire GTT.
510 */
511 ret = i915_gem_evict_everything(ring->dev, retry == 0);
512 if (ret)
513 return ret;
514
515 retry++;
516 } while (1);
517
518 err:
519 list_for_each_entry_continue_reverse(obj, objects, exec_list) {
520 struct drm_i915_gem_exec_object2 *entry;
521
522 if (!obj->gtt_space)
523 continue;
524
525 entry = obj->exec_entry;
526 if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
527 i915_gem_object_unpin_fence(obj);
528 entry->flags &= ~__EXEC_OBJECT_HAS_FENCE;
529 }
530
531 i915_gem_object_unpin(obj);
532 }
533
534 return ret;
535 }
536
537 static int
538 i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
539 struct drm_file *file,
540 struct intel_ring_buffer *ring,
541 struct list_head *objects,
542 struct eb_objects *eb,
543 struct drm_i915_gem_exec_object2 *exec,
544 int count)
545 {
546 struct drm_i915_gem_relocation_entry *reloc;
547 struct drm_i915_gem_object *obj;
548 int *reloc_offset;
549 int i, total, ret;
550
551 /* We may process another execbuffer during the unlock... */
552 while (!list_empty(objects)) {
553 obj = list_first_entry(objects,
554 struct drm_i915_gem_object,
555 exec_list);
556 list_del_init(&obj->exec_list);
557 drm_gem_object_unreference(&obj->base);
558 }
559
560 mutex_unlock(&dev->struct_mutex);
561
562 total = 0;
563 for (i = 0; i < count; i++)
564 total += exec[i].relocation_count;
565
566 reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
567 reloc = drm_malloc_ab(total, sizeof(*reloc));
568 if (reloc == NULL || reloc_offset == NULL) {
569 drm_free_large(reloc);
570 drm_free_large(reloc_offset);
571 mutex_lock(&dev->struct_mutex);
572 return -ENOMEM;
573 }
574
575 total = 0;
576 for (i = 0; i < count; i++) {
577 struct drm_i915_gem_relocation_entry __user *user_relocs;
578
579 user_relocs = (void __user *)(uintptr_t)exec[i].relocs_ptr;
580
581 if (copy_from_user(reloc+total, user_relocs,
582 exec[i].relocation_count * sizeof(*reloc))) {
583 ret = -EFAULT;
584 mutex_lock(&dev->struct_mutex);
585 goto err;
586 }
587
588 reloc_offset[i] = total;
589 total += exec[i].relocation_count;
590 }
591
592 ret = i915_mutex_lock_interruptible(dev);
593 if (ret) {
594 mutex_lock(&dev->struct_mutex);
595 goto err;
596 }
597
598 /* reacquire the objects */
599 eb_reset(eb);
600 for (i = 0; i < count; i++) {
601 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
602 exec[i].handle));
603 if (&obj->base == NULL) {
604 DRM_DEBUG("Invalid object handle %d at index %d\n",
605 exec[i].handle, i);
606 ret = -ENOENT;
607 goto err;
608 }
609
610 list_add_tail(&obj->exec_list, objects);
611 obj->exec_handle = exec[i].handle;
612 obj->exec_entry = &exec[i];
613 eb_add_object(eb, obj);
614 }
615
616 ret = i915_gem_execbuffer_reserve(ring, file, objects);
617 if (ret)
618 goto err;
619
620 list_for_each_entry(obj, objects, exec_list) {
621 int offset = obj->exec_entry - exec;
622 ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
623 reloc + reloc_offset[offset]);
624 if (ret)
625 goto err;
626 }
627
628 /* Leave the user relocations as are, this is the painfully slow path,
629 * and we want to avoid the complication of dropping the lock whilst
630 * having buffers reserved in the aperture and so causing spurious
631 * ENOSPC for random operations.
632 */
633
634 err:
635 drm_free_large(reloc);
636 drm_free_large(reloc_offset);
637 return ret;
638 }
639
640 static int
641 i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, u32 flips)
642 {
643 u32 plane, flip_mask;
644 int ret;
645
646 /* Check for any pending flips. As we only maintain a flip queue depth
647 * of 1, we can simply insert a WAIT for the next display flip prior
648 * to executing the batch and avoid stalling the CPU.
649 */
650
651 for (plane = 0; flips >> plane; plane++) {
652 if (((flips >> plane) & 1) == 0)
653 continue;
654
655 if (plane)
656 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
657 else
658 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
659
660 ret = intel_ring_begin(ring, 2);
661 if (ret)
662 return ret;
663
664 intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
665 intel_ring_emit(ring, MI_NOOP);
666 intel_ring_advance(ring);
667 }
668
669 return 0;
670 }
671
672 static int
673 i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
674 struct list_head *objects)
675 {
676 struct drm_i915_gem_object *obj;
677 uint32_t flush_domains = 0;
678 uint32_t flips = 0;
679 int ret;
680
681 list_for_each_entry(obj, objects, exec_list) {
682 ret = i915_gem_object_sync(obj, ring);
683 if (ret)
684 return ret;
685
686 if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
687 i915_gem_clflush_object(obj);
688
689 if (obj->base.pending_write_domain)
690 flips |= atomic_read(&obj->pending_flip);
691
692 flush_domains |= obj->base.write_domain;
693 }
694
695 if (flips) {
696 ret = i915_gem_execbuffer_wait_for_flips(ring, flips);
697 if (ret)
698 return ret;
699 }
700
701 if (flush_domains & I915_GEM_DOMAIN_CPU)
702 intel_gtt_chipset_flush();
703
704 if (flush_domains & I915_GEM_DOMAIN_GTT)
705 wmb();
706
707 /* Unconditionally invalidate gpu caches and ensure that we do flush
708 * any residual writes from the previous batch.
709 */
710 return intel_ring_invalidate_all_caches(ring);
711 }
712
713 static bool
714 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
715 {
716 return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
717 }
718
719 static int
720 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
721 int count)
722 {
723 int i;
724
725 for (i = 0; i < count; i++) {
726 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
727 int length; /* limited by fault_in_pages_readable() */
728
729 /* First check for malicious input causing overflow */
730 if (exec[i].relocation_count >
731 INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
732 return -EINVAL;
733
734 length = exec[i].relocation_count *
735 sizeof(struct drm_i915_gem_relocation_entry);
736 if (!access_ok(VERIFY_READ, ptr, length))
737 return -EFAULT;
738
739 /* we may also need to update the presumed offsets */
740 if (!access_ok(VERIFY_WRITE, ptr, length))
741 return -EFAULT;
742
743 if (fault_in_multipages_readable(ptr, length))
744 return -EFAULT;
745 }
746
747 return 0;
748 }
749
750 static void
751 i915_gem_execbuffer_move_to_active(struct list_head *objects,
752 struct intel_ring_buffer *ring,
753 u32 seqno)
754 {
755 struct drm_i915_gem_object *obj;
756
757 list_for_each_entry(obj, objects, exec_list) {
758 u32 old_read = obj->base.read_domains;
759 u32 old_write = obj->base.write_domain;
760
761 obj->base.read_domains = obj->base.pending_read_domains;
762 obj->base.write_domain = obj->base.pending_write_domain;
763 obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
764
765 i915_gem_object_move_to_active(obj, ring, seqno);
766 if (obj->base.write_domain) {
767 obj->dirty = 1;
768 obj->last_write_seqno = seqno;
769 if (obj->pin_count) /* check for potential scanout */
770 intel_mark_fb_busy(obj);
771 }
772
773 trace_i915_gem_object_change_domain(obj, old_read, old_write);
774 }
775 }
776
777 static void
778 i915_gem_execbuffer_retire_commands(struct drm_device *dev,
779 struct drm_file *file,
780 struct intel_ring_buffer *ring)
781 {
782 /* Unconditionally force add_request to emit a full flush. */
783 ring->gpu_caches_dirty = true;
784
785 /* Add a breadcrumb for the completion of the batch buffer */
786 (void)i915_add_request(ring, file, NULL);
787 }
788
789 static int
790 i915_reset_gen7_sol_offsets(struct drm_device *dev,
791 struct intel_ring_buffer *ring)
792 {
793 drm_i915_private_t *dev_priv = dev->dev_private;
794 int ret, i;
795
796 if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS])
797 return 0;
798
799 ret = intel_ring_begin(ring, 4 * 3);
800 if (ret)
801 return ret;
802
803 for (i = 0; i < 4; i++) {
804 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
805 intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
806 intel_ring_emit(ring, 0);
807 }
808
809 intel_ring_advance(ring);
810
811 return 0;
812 }
813
814 static int
815 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
816 struct drm_file *file,
817 struct drm_i915_gem_execbuffer2 *args,
818 struct drm_i915_gem_exec_object2 *exec)
819 {
820 drm_i915_private_t *dev_priv = dev->dev_private;
821 struct list_head objects;
822 struct eb_objects *eb;
823 struct drm_i915_gem_object *batch_obj;
824 struct drm_clip_rect *cliprects = NULL;
825 struct intel_ring_buffer *ring;
826 u32 ctx_id = i915_execbuffer2_get_context_id(*args);
827 u32 exec_start, exec_len;
828 u32 seqno;
829 u32 mask;
830 int ret, mode, i;
831
832 if (!i915_gem_check_execbuffer(args)) {
833 DRM_DEBUG("execbuf with invalid offset/length\n");
834 return -EINVAL;
835 }
836
837 ret = validate_exec_list(exec, args->buffer_count);
838 if (ret)
839 return ret;
840
841 switch (args->flags & I915_EXEC_RING_MASK) {
842 case I915_EXEC_DEFAULT:
843 case I915_EXEC_RENDER:
844 ring = &dev_priv->ring[RCS];
845 break;
846 case I915_EXEC_BSD:
847 ring = &dev_priv->ring[VCS];
848 if (ctx_id != 0) {
849 DRM_DEBUG("Ring %s doesn't support contexts\n",
850 ring->name);
851 return -EPERM;
852 }
853 break;
854 case I915_EXEC_BLT:
855 ring = &dev_priv->ring[BCS];
856 if (ctx_id != 0) {
857 DRM_DEBUG("Ring %s doesn't support contexts\n",
858 ring->name);
859 return -EPERM;
860 }
861 break;
862 default:
863 DRM_DEBUG("execbuf with unknown ring: %d\n",
864 (int)(args->flags & I915_EXEC_RING_MASK));
865 return -EINVAL;
866 }
867 if (!intel_ring_initialized(ring)) {
868 DRM_DEBUG("execbuf with invalid ring: %d\n",
869 (int)(args->flags & I915_EXEC_RING_MASK));
870 return -EINVAL;
871 }
872
873 mode = args->flags & I915_EXEC_CONSTANTS_MASK;
874 mask = I915_EXEC_CONSTANTS_MASK;
875 switch (mode) {
876 case I915_EXEC_CONSTANTS_REL_GENERAL:
877 case I915_EXEC_CONSTANTS_ABSOLUTE:
878 case I915_EXEC_CONSTANTS_REL_SURFACE:
879 if (ring == &dev_priv->ring[RCS] &&
880 mode != dev_priv->relative_constants_mode) {
881 if (INTEL_INFO(dev)->gen < 4)
882 return -EINVAL;
883
884 if (INTEL_INFO(dev)->gen > 5 &&
885 mode == I915_EXEC_CONSTANTS_REL_SURFACE)
886 return -EINVAL;
887
888 /* The HW changed the meaning on this bit on gen6 */
889 if (INTEL_INFO(dev)->gen >= 6)
890 mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
891 }
892 break;
893 default:
894 DRM_DEBUG("execbuf with unknown constants: %d\n", mode);
895 return -EINVAL;
896 }
897
898 if (args->buffer_count < 1) {
899 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
900 return -EINVAL;
901 }
902
903 if (args->num_cliprects != 0) {
904 if (ring != &dev_priv->ring[RCS]) {
905 DRM_DEBUG("clip rectangles are only valid with the render ring\n");
906 return -EINVAL;
907 }
908
909 if (INTEL_INFO(dev)->gen >= 5) {
910 DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
911 return -EINVAL;
912 }
913
914 if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) {
915 DRM_DEBUG("execbuf with %u cliprects\n",
916 args->num_cliprects);
917 return -EINVAL;
918 }
919
920 cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
921 GFP_KERNEL);
922 if (cliprects == NULL) {
923 ret = -ENOMEM;
924 goto pre_mutex_err;
925 }
926
927 if (copy_from_user(cliprects,
928 (struct drm_clip_rect __user *)(uintptr_t)
929 args->cliprects_ptr,
930 sizeof(*cliprects)*args->num_cliprects)) {
931 ret = -EFAULT;
932 goto pre_mutex_err;
933 }
934 }
935
936 ret = i915_mutex_lock_interruptible(dev);
937 if (ret)
938 goto pre_mutex_err;
939
940 if (dev_priv->mm.suspended) {
941 mutex_unlock(&dev->struct_mutex);
942 ret = -EBUSY;
943 goto pre_mutex_err;
944 }
945
946 eb = eb_create(args->buffer_count);
947 if (eb == NULL) {
948 mutex_unlock(&dev->struct_mutex);
949 ret = -ENOMEM;
950 goto pre_mutex_err;
951 }
952
953 /* Look up object handles */
954 INIT_LIST_HEAD(&objects);
955 for (i = 0; i < args->buffer_count; i++) {
956 struct drm_i915_gem_object *obj;
957
958 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
959 exec[i].handle));
960 if (&obj->base == NULL) {
961 DRM_DEBUG("Invalid object handle %d at index %d\n",
962 exec[i].handle, i);
963 /* prevent error path from reading uninitialized data */
964 ret = -ENOENT;
965 goto err;
966 }
967
968 if (!list_empty(&obj->exec_list)) {
969 DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
970 obj, exec[i].handle, i);
971 ret = -EINVAL;
972 goto err;
973 }
974
975 list_add_tail(&obj->exec_list, &objects);
976 obj->exec_handle = exec[i].handle;
977 obj->exec_entry = &exec[i];
978 eb_add_object(eb, obj);
979 }
980
981 /* take note of the batch buffer before we might reorder the lists */
982 batch_obj = list_entry(objects.prev,
983 struct drm_i915_gem_object,
984 exec_list);
985
986 /* Move the objects en-masse into the GTT, evicting if necessary. */
987 ret = i915_gem_execbuffer_reserve(ring, file, &objects);
988 if (ret)
989 goto err;
990
991 /* The objects are in their final locations, apply the relocations. */
992 ret = i915_gem_execbuffer_relocate(dev, eb, &objects);
993 if (ret) {
994 if (ret == -EFAULT) {
995 ret = i915_gem_execbuffer_relocate_slow(dev, file, ring,
996 &objects, eb,
997 exec,
998 args->buffer_count);
999 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1000 }
1001 if (ret)
1002 goto err;
1003 }
1004
1005 /* Set the pending read domains for the batch buffer to COMMAND */
1006 if (batch_obj->base.pending_write_domain) {
1007 DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
1008 ret = -EINVAL;
1009 goto err;
1010 }
1011 batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1012
1013 ret = i915_gem_execbuffer_move_to_gpu(ring, &objects);
1014 if (ret)
1015 goto err;
1016
1017 seqno = i915_gem_next_request_seqno(ring);
1018 for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++) {
1019 if (seqno < ring->sync_seqno[i]) {
1020 /* The GPU can not handle its semaphore value wrapping,
1021 * so every billion or so execbuffers, we need to stall
1022 * the GPU in order to reset the counters.
1023 */
1024 ret = i915_gpu_idle(dev);
1025 if (ret)
1026 goto err;
1027 i915_gem_retire_requests(dev);
1028
1029 BUG_ON(ring->sync_seqno[i]);
1030 }
1031 }
1032
1033 ret = i915_switch_context(ring, file, ctx_id);
1034 if (ret)
1035 goto err;
1036
1037 if (ring == &dev_priv->ring[RCS] &&
1038 mode != dev_priv->relative_constants_mode) {
1039 ret = intel_ring_begin(ring, 4);
1040 if (ret)
1041 goto err;
1042
1043 intel_ring_emit(ring, MI_NOOP);
1044 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1045 intel_ring_emit(ring, INSTPM);
1046 intel_ring_emit(ring, mask << 16 | mode);
1047 intel_ring_advance(ring);
1048
1049 dev_priv->relative_constants_mode = mode;
1050 }
1051
1052 if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
1053 ret = i915_reset_gen7_sol_offsets(dev, ring);
1054 if (ret)
1055 goto err;
1056 }
1057
1058 trace_i915_gem_ring_dispatch(ring, seqno);
1059
1060 exec_start = batch_obj->gtt_offset + args->batch_start_offset;
1061 exec_len = args->batch_len;
1062 if (cliprects) {
1063 for (i = 0; i < args->num_cliprects; i++) {
1064 ret = i915_emit_box(dev, &cliprects[i],
1065 args->DR1, args->DR4);
1066 if (ret)
1067 goto err;
1068
1069 ret = ring->dispatch_execbuffer(ring,
1070 exec_start, exec_len);
1071 if (ret)
1072 goto err;
1073 }
1074 } else {
1075 ret = ring->dispatch_execbuffer(ring, exec_start, exec_len);
1076 if (ret)
1077 goto err;
1078 }
1079
1080 i915_gem_execbuffer_move_to_active(&objects, ring, seqno);
1081 i915_gem_execbuffer_retire_commands(dev, file, ring);
1082
1083 err:
1084 eb_destroy(eb);
1085 while (!list_empty(&objects)) {
1086 struct drm_i915_gem_object *obj;
1087
1088 obj = list_first_entry(&objects,
1089 struct drm_i915_gem_object,
1090 exec_list);
1091 list_del_init(&obj->exec_list);
1092 drm_gem_object_unreference(&obj->base);
1093 }
1094
1095 mutex_unlock(&dev->struct_mutex);
1096
1097 pre_mutex_err:
1098 kfree(cliprects);
1099 return ret;
1100 }
1101
1102 /*
1103 * Legacy execbuffer just creates an exec2 list from the original exec object
1104 * list array and passes it to the real function.
1105 */
1106 int
1107 i915_gem_execbuffer(struct drm_device *dev, void *data,
1108 struct drm_file *file)
1109 {
1110 struct drm_i915_gem_execbuffer *args = data;
1111 struct drm_i915_gem_execbuffer2 exec2;
1112 struct drm_i915_gem_exec_object *exec_list = NULL;
1113 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1114 int ret, i;
1115
1116 if (args->buffer_count < 1) {
1117 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1118 return -EINVAL;
1119 }
1120
1121 /* Copy in the exec list from userland */
1122 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
1123 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1124 if (exec_list == NULL || exec2_list == NULL) {
1125 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1126 args->buffer_count);
1127 drm_free_large(exec_list);
1128 drm_free_large(exec2_list);
1129 return -ENOMEM;
1130 }
1131 ret = copy_from_user(exec_list,
1132 (struct drm_i915_relocation_entry __user *)
1133 (uintptr_t) args->buffers_ptr,
1134 sizeof(*exec_list) * args->buffer_count);
1135 if (ret != 0) {
1136 DRM_DEBUG("copy %d exec entries failed %d\n",
1137 args->buffer_count, ret);
1138 drm_free_large(exec_list);
1139 drm_free_large(exec2_list);
1140 return -EFAULT;
1141 }
1142
1143 for (i = 0; i < args->buffer_count; i++) {
1144 exec2_list[i].handle = exec_list[i].handle;
1145 exec2_list[i].relocation_count = exec_list[i].relocation_count;
1146 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1147 exec2_list[i].alignment = exec_list[i].alignment;
1148 exec2_list[i].offset = exec_list[i].offset;
1149 if (INTEL_INFO(dev)->gen < 4)
1150 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1151 else
1152 exec2_list[i].flags = 0;
1153 }
1154
1155 exec2.buffers_ptr = args->buffers_ptr;
1156 exec2.buffer_count = args->buffer_count;
1157 exec2.batch_start_offset = args->batch_start_offset;
1158 exec2.batch_len = args->batch_len;
1159 exec2.DR1 = args->DR1;
1160 exec2.DR4 = args->DR4;
1161 exec2.num_cliprects = args->num_cliprects;
1162 exec2.cliprects_ptr = args->cliprects_ptr;
1163 exec2.flags = I915_EXEC_RENDER;
1164 i915_execbuffer2_set_context_id(exec2, 0);
1165
1166 ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1167 if (!ret) {
1168 /* Copy the new buffer offsets back to the user's exec list. */
1169 for (i = 0; i < args->buffer_count; i++)
1170 exec_list[i].offset = exec2_list[i].offset;
1171 /* ... and back out to userspace */
1172 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1173 (uintptr_t) args->buffers_ptr,
1174 exec_list,
1175 sizeof(*exec_list) * args->buffer_count);
1176 if (ret) {
1177 ret = -EFAULT;
1178 DRM_DEBUG("failed to copy %d exec entries "
1179 "back to user (%d)\n",
1180 args->buffer_count, ret);
1181 }
1182 }
1183
1184 drm_free_large(exec_list);
1185 drm_free_large(exec2_list);
1186 return ret;
1187 }
1188
1189 int
1190 i915_gem_execbuffer2(struct drm_device *dev, void *data,
1191 struct drm_file *file)
1192 {
1193 struct drm_i915_gem_execbuffer2 *args = data;
1194 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1195 int ret;
1196
1197 if (args->buffer_count < 1 ||
1198 args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
1199 DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
1200 return -EINVAL;
1201 }
1202
1203 exec2_list = kmalloc(sizeof(*exec2_list)*args->buffer_count,
1204 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
1205 if (exec2_list == NULL)
1206 exec2_list = drm_malloc_ab(sizeof(*exec2_list),
1207 args->buffer_count);
1208 if (exec2_list == NULL) {
1209 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1210 args->buffer_count);
1211 return -ENOMEM;
1212 }
1213 ret = copy_from_user(exec2_list,
1214 (struct drm_i915_relocation_entry __user *)
1215 (uintptr_t) args->buffers_ptr,
1216 sizeof(*exec2_list) * args->buffer_count);
1217 if (ret != 0) {
1218 DRM_DEBUG("copy %d exec entries failed %d\n",
1219 args->buffer_count, ret);
1220 drm_free_large(exec2_list);
1221 return -EFAULT;
1222 }
1223
1224 ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1225 if (!ret) {
1226 /* Copy the new buffer offsets back to the user's exec list. */
1227 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1228 (uintptr_t) args->buffers_ptr,
1229 exec2_list,
1230 sizeof(*exec2_list) * args->buffer_count);
1231 if (ret) {
1232 ret = -EFAULT;
1233 DRM_DEBUG("failed to copy %d exec entries "
1234 "back to user (%d)\n",
1235 args->buffer_count, ret);
1236 }
1237 }
1238
1239 drm_free_large(exec2_list);
1240 return ret;
1241 }
Linux kernel - Deliverables
This page took 0.079494 seconds and 5 git commands to generate.