projects / deliverable / linux.git / blob
? search:


e8e194fa2e6593e8f9fc2fc8d5ead40f247cfbb4
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem_execbuffer.c
1 /*
2 * Copyright © 2008,2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Chris Wilson <chris@chris-wilson.co.uk>
26 *
27 */
28
29 #include <drm/drmP.h>
30 #include <drm/i915_drm.h>
31 #include "i915_drv.h"
32 #include "i915_trace.h"
33 #include "intel_drv.h"
34 #include "intel_frontbuffer.h"
35 #include <linux/dma_remapping.h>
36 #include <linux/uaccess.h>
37
38 #define __EXEC_OBJECT_HAS_PIN (1<<31)
39 #define __EXEC_OBJECT_HAS_FENCE (1<<30)
40 #define __EXEC_OBJECT_NEEDS_MAP (1<<29)
41 #define __EXEC_OBJECT_NEEDS_BIAS (1<<28)
42 #define __EXEC_OBJECT_INTERNAL_FLAGS (0xf<<28) /* all of the above */
43
44 #define BATCH_OFFSET_BIAS (256*1024)
45
46 struct i915_execbuffer_params {
47 struct drm_device *dev;
48 struct drm_file *file;
49 struct i915_vma *batch;
50 u32 dispatch_flags;
51 u32 args_batch_start_offset;
52 struct intel_engine_cs *engine;
53 struct i915_gem_context *ctx;
54 struct drm_i915_gem_request *request;
55 };
56
57 struct eb_vmas {
58 struct list_head vmas;
59 int and;
60 union {
61 struct i915_vma *lut[0];
62 struct hlist_head buckets[0];
63 };
64 };
65
66 static struct eb_vmas *
67 eb_create(struct drm_i915_gem_execbuffer2 *args)
68 {
69 struct eb_vmas *eb = NULL;
70
71 if (args->flags & I915_EXEC_HANDLE_LUT) {
72 unsigned size = args->buffer_count;
73 size *= sizeof(struct i915_vma *);
74 size += sizeof(struct eb_vmas);
75 eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
76 }
77
78 if (eb == NULL) {
79 unsigned size = args->buffer_count;
80 unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
81 BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
82 while (count > 2*size)
83 count >>= 1;
84 eb = kzalloc(count*sizeof(struct hlist_head) +
85 sizeof(struct eb_vmas),
86 GFP_TEMPORARY);
87 if (eb == NULL)
88 return eb;
89
90 eb->and = count - 1;
91 } else
92 eb->and = -args->buffer_count;
93
94 INIT_LIST_HEAD(&eb->vmas);
95 return eb;
96 }
97
98 static void
99 eb_reset(struct eb_vmas *eb)
100 {
101 if (eb->and >= 0)
102 memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
103 }
104
105 static struct i915_vma *
106 eb_get_batch(struct eb_vmas *eb)
107 {
108 struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list);
109
110 /*
111 * SNA is doing fancy tricks with compressing batch buffers, which leads
112 * to negative relocation deltas. Usually that works out ok since the
113 * relocate address is still positive, except when the batch is placed
114 * very low in the GTT. Ensure this doesn't happen.
115 *
116 * Note that actual hangs have only been observed on gen7, but for
117 * paranoia do it everywhere.
118 */
119 if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
120 vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
121
122 return vma;
123 }
124
125 static int
126 eb_lookup_vmas(struct eb_vmas *eb,
127 struct drm_i915_gem_exec_object2 *exec,
128 const struct drm_i915_gem_execbuffer2 *args,
129 struct i915_address_space *vm,
130 struct drm_file *file)
131 {
132 struct drm_i915_gem_object *obj;
133 struct list_head objects;
134 int i, ret;
135
136 INIT_LIST_HEAD(&objects);
137 spin_lock(&file->table_lock);
138 /* Grab a reference to the object and release the lock so we can lookup
139 * or create the VMA without using GFP_ATOMIC */
140 for (i = 0; i < args->buffer_count; i++) {
141 obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
142 if (obj == NULL) {
143 spin_unlock(&file->table_lock);
144 DRM_DEBUG("Invalid object handle %d at index %d\n",
145 exec[i].handle, i);
146 ret = -ENOENT;
147 goto err;
148 }
149
150 if (!list_empty(&obj->obj_exec_link)) {
151 spin_unlock(&file->table_lock);
152 DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
153 obj, exec[i].handle, i);
154 ret = -EINVAL;
155 goto err;
156 }
157
158 i915_gem_object_get(obj);
159 list_add_tail(&obj->obj_exec_link, &objects);
160 }
161 spin_unlock(&file->table_lock);
162
163 i = 0;
164 while (!list_empty(&objects)) {
165 struct i915_vma *vma;
166
167 obj = list_first_entry(&objects,
168 struct drm_i915_gem_object,
169 obj_exec_link);
170
171 /*
172 * NOTE: We can leak any vmas created here when something fails
173 * later on. But that's no issue since vma_unbind can deal with
174 * vmas which are not actually bound. And since only
175 * lookup_or_create exists as an interface to get at the vma
176 * from the (obj, vm) we don't run the risk of creating
177 * duplicated vmas for the same vm.
178 */
179 vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
180 if (IS_ERR(vma)) {
181 DRM_DEBUG("Failed to lookup VMA\n");
182 ret = PTR_ERR(vma);
183 goto err;
184 }
185
186 /* Transfer ownership from the objects list to the vmas list. */
187 list_add_tail(&vma->exec_list, &eb->vmas);
188 list_del_init(&obj->obj_exec_link);
189
190 vma->exec_entry = &exec[i];
191 if (eb->and < 0) {
192 eb->lut[i] = vma;
193 } else {
194 uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
195 vma->exec_handle = handle;
196 hlist_add_head(&vma->exec_node,
197 &eb->buckets[handle & eb->and]);
198 }
199 ++i;
200 }
201
202 return 0;
203
204
205 err:
206 while (!list_empty(&objects)) {
207 obj = list_first_entry(&objects,
208 struct drm_i915_gem_object,
209 obj_exec_link);
210 list_del_init(&obj->obj_exec_link);
211 i915_gem_object_put(obj);
212 }
213 /*
214 * Objects already transfered to the vmas list will be unreferenced by
215 * eb_destroy.
216 */
217
218 return ret;
219 }
220
221 static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle)
222 {
223 if (eb->and < 0) {
224 if (handle >= -eb->and)
225 return NULL;
226 return eb->lut[handle];
227 } else {
228 struct hlist_head *head;
229 struct i915_vma *vma;
230
231 head = &eb->buckets[handle & eb->and];
232 hlist_for_each_entry(vma, head, exec_node) {
233 if (vma->exec_handle == handle)
234 return vma;
235 }
236 return NULL;
237 }
238 }
239
240 static void
241 i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
242 {
243 struct drm_i915_gem_exec_object2 *entry;
244 struct drm_i915_gem_object *obj = vma->obj;
245
246 if (!drm_mm_node_allocated(&vma->node))
247 return;
248
249 entry = vma->exec_entry;
250
251 if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
252 i915_gem_object_unpin_fence(obj);
253
254 if (entry->flags & __EXEC_OBJECT_HAS_PIN)
255 __i915_vma_unpin(vma);
256
257 entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
258 }
259
260 static void eb_destroy(struct eb_vmas *eb)
261 {
262 while (!list_empty(&eb->vmas)) {
263 struct i915_vma *vma;
264
265 vma = list_first_entry(&eb->vmas,
266 struct i915_vma,
267 exec_list);
268 list_del_init(&vma->exec_list);
269 i915_gem_execbuffer_unreserve_vma(vma);
270 i915_gem_object_put(vma->obj);
271 }
272 kfree(eb);
273 }
274
275 static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
276 {
277 return (HAS_LLC(obj->base.dev) ||
278 obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
279 obj->cache_level != I915_CACHE_NONE);
280 }
281
282 /* Used to convert any address to canonical form.
283 * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
284 * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
285 * addresses to be in a canonical form:
286 * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
287 * canonical form [63:48] == [47]."
288 */
289 #define GEN8_HIGH_ADDRESS_BIT 47
290 static inline uint64_t gen8_canonical_addr(uint64_t address)
291 {
292 return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
293 }
294
295 static inline uint64_t gen8_noncanonical_addr(uint64_t address)
296 {
297 return address & ((1ULL << (GEN8_HIGH_ADDRESS_BIT + 1)) - 1);
298 }
299
300 static inline uint64_t
301 relocation_target(struct drm_i915_gem_relocation_entry *reloc,
302 uint64_t target_offset)
303 {
304 return gen8_canonical_addr((int)reloc->delta + target_offset);
305 }
306
307 static int
308 relocate_entry_cpu(struct drm_i915_gem_object *obj,
309 struct drm_i915_gem_relocation_entry *reloc,
310 uint64_t target_offset)
311 {
312 struct drm_device *dev = obj->base.dev;
313 uint32_t page_offset = offset_in_page(reloc->offset);
314 uint64_t delta = relocation_target(reloc, target_offset);
315 char *vaddr;
316 int ret;
317
318 ret = i915_gem_object_set_to_cpu_domain(obj, true);
319 if (ret)
320 return ret;
321
322 vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
323 reloc->offset >> PAGE_SHIFT));
324 *(uint32_t *)(vaddr + page_offset) = lower_32_bits(delta);
325
326 if (INTEL_INFO(dev)->gen >= 8) {
327 page_offset = offset_in_page(page_offset + sizeof(uint32_t));
328
329 if (page_offset == 0) {
330 kunmap_atomic(vaddr);
331 vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
332 (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
333 }
334
335 *(uint32_t *)(vaddr + page_offset) = upper_32_bits(delta);
336 }
337
338 kunmap_atomic(vaddr);
339
340 return 0;
341 }
342
343 static int
344 relocate_entry_gtt(struct drm_i915_gem_object *obj,
345 struct drm_i915_gem_relocation_entry *reloc,
346 uint64_t target_offset)
347 {
348 struct drm_device *dev = obj->base.dev;
349 struct drm_i915_private *dev_priv = to_i915(dev);
350 struct i915_ggtt *ggtt = &dev_priv->ggtt;
351 uint64_t delta = relocation_target(reloc, target_offset);
352 uint64_t offset;
353 void __iomem *reloc_page;
354 int ret;
355
356 ret = i915_gem_object_set_to_gtt_domain(obj, true);
357 if (ret)
358 return ret;
359
360 ret = i915_gem_object_put_fence(obj);
361 if (ret)
362 return ret;
363
364 /* Map the page containing the relocation we're going to perform. */
365 offset = i915_gem_obj_ggtt_offset(obj);
366 offset += reloc->offset;
367 reloc_page = io_mapping_map_atomic_wc(ggtt->mappable,
368 offset & PAGE_MASK);
369 iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));
370
371 if (INTEL_INFO(dev)->gen >= 8) {
372 offset += sizeof(uint32_t);
373
374 if (offset_in_page(offset) == 0) {
375 io_mapping_unmap_atomic(reloc_page);
376 reloc_page =
377 io_mapping_map_atomic_wc(ggtt->mappable,
378 offset);
379 }
380
381 iowrite32(upper_32_bits(delta),
382 reloc_page + offset_in_page(offset));
383 }
384
385 io_mapping_unmap_atomic(reloc_page);
386
387 return 0;
388 }
389
390 static void
391 clflush_write32(void *addr, uint32_t value)
392 {
393 /* This is not a fast path, so KISS. */
394 drm_clflush_virt_range(addr, sizeof(uint32_t));
395 *(uint32_t *)addr = value;
396 drm_clflush_virt_range(addr, sizeof(uint32_t));
397 }
398
399 static int
400 relocate_entry_clflush(struct drm_i915_gem_object *obj,
401 struct drm_i915_gem_relocation_entry *reloc,
402 uint64_t target_offset)
403 {
404 struct drm_device *dev = obj->base.dev;
405 uint32_t page_offset = offset_in_page(reloc->offset);
406 uint64_t delta = relocation_target(reloc, target_offset);
407 char *vaddr;
408 int ret;
409
410 ret = i915_gem_object_set_to_gtt_domain(obj, true);
411 if (ret)
412 return ret;
413
414 vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
415 reloc->offset >> PAGE_SHIFT));
416 clflush_write32(vaddr + page_offset, lower_32_bits(delta));
417
418 if (INTEL_INFO(dev)->gen >= 8) {
419 page_offset = offset_in_page(page_offset + sizeof(uint32_t));
420
421 if (page_offset == 0) {
422 kunmap_atomic(vaddr);
423 vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
424 (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
425 }
426
427 clflush_write32(vaddr + page_offset, upper_32_bits(delta));
428 }
429
430 kunmap_atomic(vaddr);
431
432 return 0;
433 }
434
435 static bool object_is_idle(struct drm_i915_gem_object *obj)
436 {
437 unsigned long active = obj->active;
438 int idx;
439
440 for_each_active(active, idx) {
441 if (!i915_gem_active_is_idle(&obj->last_read[idx],
442 &obj->base.dev->struct_mutex))
443 return false;
444 }
445
446 return true;
447 }
448
449 static int
450 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
451 struct eb_vmas *eb,
452 struct drm_i915_gem_relocation_entry *reloc)
453 {
454 struct drm_device *dev = obj->base.dev;
455 struct drm_gem_object *target_obj;
456 struct drm_i915_gem_object *target_i915_obj;
457 struct i915_vma *target_vma;
458 uint64_t target_offset;
459 int ret;
460
461 /* we've already hold a reference to all valid objects */
462 target_vma = eb_get_vma(eb, reloc->target_handle);
463 if (unlikely(target_vma == NULL))
464 return -ENOENT;
465 target_i915_obj = target_vma->obj;
466 target_obj = &target_vma->obj->base;
467
468 target_offset = gen8_canonical_addr(target_vma->node.start);
469
470 /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
471 * pipe_control writes because the gpu doesn't properly redirect them
472 * through the ppgtt for non_secure batchbuffers. */
473 if (unlikely(IS_GEN6(dev) &&
474 reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
475 ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
476 PIN_GLOBAL);
477 if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!"))
478 return ret;
479 }
480
481 /* Validate that the target is in a valid r/w GPU domain */
482 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
483 DRM_DEBUG("reloc with multiple write domains: "
484 "obj %p target %d offset %d "
485 "read %08x write %08x",
486 obj, reloc->target_handle,
487 (int) reloc->offset,
488 reloc->read_domains,
489 reloc->write_domain);
490 return -EINVAL;
491 }
492 if (unlikely((reloc->write_domain | reloc->read_domains)
493 & ~I915_GEM_GPU_DOMAINS)) {
494 DRM_DEBUG("reloc with read/write non-GPU domains: "
495 "obj %p target %d offset %d "
496 "read %08x write %08x",
497 obj, reloc->target_handle,
498 (int) reloc->offset,
499 reloc->read_domains,
500 reloc->write_domain);
501 return -EINVAL;
502 }
503
504 target_obj->pending_read_domains |= reloc->read_domains;
505 target_obj->pending_write_domain |= reloc->write_domain;
506
507 /* If the relocation already has the right value in it, no
508 * more work needs to be done.
509 */
510 if (target_offset == reloc->presumed_offset)
511 return 0;
512
513 /* Check that the relocation address is valid... */
514 if (unlikely(reloc->offset >
515 obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {
516 DRM_DEBUG("Relocation beyond object bounds: "
517 "obj %p target %d offset %d size %d.\n",
518 obj, reloc->target_handle,
519 (int) reloc->offset,
520 (int) obj->base.size);
521 return -EINVAL;
522 }
523 if (unlikely(reloc->offset & 3)) {
524 DRM_DEBUG("Relocation not 4-byte aligned: "
525 "obj %p target %d offset %d.\n",
526 obj, reloc->target_handle,
527 (int) reloc->offset);
528 return -EINVAL;
529 }
530
531 /* We can't wait for rendering with pagefaults disabled */
532 if (pagefault_disabled() && !object_is_idle(obj))
533 return -EFAULT;
534
535 if (use_cpu_reloc(obj))
536 ret = relocate_entry_cpu(obj, reloc, target_offset);
537 else if (obj->map_and_fenceable)
538 ret = relocate_entry_gtt(obj, reloc, target_offset);
539 else if (static_cpu_has(X86_FEATURE_CLFLUSH))
540 ret = relocate_entry_clflush(obj, reloc, target_offset);
541 else {
542 WARN_ONCE(1, "Impossible case in relocation handling\n");
543 ret = -ENODEV;
544 }
545
546 if (ret)
547 return ret;
548
549 /* and update the user's relocation entry */
550 reloc->presumed_offset = target_offset;
551
552 return 0;
553 }
554
555 static int
556 i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
557 struct eb_vmas *eb)
558 {
559 #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
560 struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
561 struct drm_i915_gem_relocation_entry __user *user_relocs;
562 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
563 int remain, ret;
564
565 user_relocs = u64_to_user_ptr(entry->relocs_ptr);
566
567 remain = entry->relocation_count;
568 while (remain) {
569 struct drm_i915_gem_relocation_entry *r = stack_reloc;
570 int count = remain;
571 if (count > ARRAY_SIZE(stack_reloc))
572 count = ARRAY_SIZE(stack_reloc);
573 remain -= count;
574
575 if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0])))
576 return -EFAULT;
577
578 do {
579 u64 offset = r->presumed_offset;
580
581 ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);
582 if (ret)
583 return ret;
584
585 if (r->presumed_offset != offset &&
586 __put_user(r->presumed_offset, &user_relocs->presumed_offset)) {
587 return -EFAULT;
588 }
589
590 user_relocs++;
591 r++;
592 } while (--count);
593 }
594
595 return 0;
596 #undef N_RELOC
597 }
598
599 static int
600 i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
601 struct eb_vmas *eb,
602 struct drm_i915_gem_relocation_entry *relocs)
603 {
604 const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
605 int i, ret;
606
607 for (i = 0; i < entry->relocation_count; i++) {
608 ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);
609 if (ret)
610 return ret;
611 }
612
613 return 0;
614 }
615
616 static int
617 i915_gem_execbuffer_relocate(struct eb_vmas *eb)
618 {
619 struct i915_vma *vma;
620 int ret = 0;
621
622 /* This is the fast path and we cannot handle a pagefault whilst
623 * holding the struct mutex lest the user pass in the relocations
624 * contained within a mmaped bo. For in such a case we, the page
625 * fault handler would call i915_gem_fault() and we would try to
626 * acquire the struct mutex again. Obviously this is bad and so
627 * lockdep complains vehemently.
628 */
629 pagefault_disable();
630 list_for_each_entry(vma, &eb->vmas, exec_list) {
631 ret = i915_gem_execbuffer_relocate_vma(vma, eb);
632 if (ret)
633 break;
634 }
635 pagefault_enable();
636
637 return ret;
638 }
639
640 static bool only_mappable_for_reloc(unsigned int flags)
641 {
642 return (flags & (EXEC_OBJECT_NEEDS_FENCE | __EXEC_OBJECT_NEEDS_MAP)) ==
643 __EXEC_OBJECT_NEEDS_MAP;
644 }
645
646 static int
647 i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
648 struct intel_engine_cs *engine,
649 bool *need_reloc)
650 {
651 struct drm_i915_gem_object *obj = vma->obj;
652 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
653 uint64_t flags;
654 int ret;
655
656 flags = PIN_USER;
657 if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
658 flags |= PIN_GLOBAL;
659
660 if (!drm_mm_node_allocated(&vma->node)) {
661 /* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
662 * limit address to the first 4GBs for unflagged objects.
663 */
664 if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0)
665 flags |= PIN_ZONE_4G;
666 if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
667 flags |= PIN_GLOBAL | PIN_MAPPABLE;
668 if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
669 flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
670 if (entry->flags & EXEC_OBJECT_PINNED)
671 flags |= entry->offset | PIN_OFFSET_FIXED;
672 if ((flags & PIN_MAPPABLE) == 0)
673 flags |= PIN_HIGH;
674 }
675
676 ret = i915_vma_pin(vma,
677 entry->pad_to_size,
678 entry->alignment,
679 flags);
680 if ((ret == -ENOSPC || ret == -E2BIG) &&
681 only_mappable_for_reloc(entry->flags))
682 ret = i915_vma_pin(vma,
683 entry->pad_to_size,
684 entry->alignment,
685 flags & ~PIN_MAPPABLE);
686 if (ret)
687 return ret;
688
689 entry->flags |= __EXEC_OBJECT_HAS_PIN;
690
691 if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
692 ret = i915_gem_object_get_fence(obj);
693 if (ret)
694 return ret;
695
696 if (i915_gem_object_pin_fence(obj))
697 entry->flags |= __EXEC_OBJECT_HAS_FENCE;
698 }
699
700 if (entry->offset != vma->node.start) {
701 entry->offset = vma->node.start;
702 *need_reloc = true;
703 }
704
705 if (entry->flags & EXEC_OBJECT_WRITE) {
706 obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
707 obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
708 }
709
710 return 0;
711 }
712
713 static bool
714 need_reloc_mappable(struct i915_vma *vma)
715 {
716 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
717
718 if (entry->relocation_count == 0)
719 return false;
720
721 if (!i915_vma_is_ggtt(vma))
722 return false;
723
724 /* See also use_cpu_reloc() */
725 if (HAS_LLC(vma->obj->base.dev))
726 return false;
727
728 if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
729 return false;
730
731 return true;
732 }
733
734 static bool
735 eb_vma_misplaced(struct i915_vma *vma)
736 {
737 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
738 struct drm_i915_gem_object *obj = vma->obj;
739
740 WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
741 !i915_vma_is_ggtt(vma));
742
743 if (entry->alignment &&
744 vma->node.start & (entry->alignment - 1))
745 return true;
746
747 if (vma->node.size < entry->pad_to_size)
748 return true;
749
750 if (entry->flags & EXEC_OBJECT_PINNED &&
751 vma->node.start != entry->offset)
752 return true;
753
754 if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
755 vma->node.start < BATCH_OFFSET_BIAS)
756 return true;
757
758 /* avoid costly ping-pong once a batch bo ended up non-mappable */
759 if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
760 return !only_mappable_for_reloc(entry->flags);
761
762 if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
763 (vma->node.start + vma->node.size - 1) >> 32)
764 return true;
765
766 return false;
767 }
768
769 static int
770 i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
771 struct list_head *vmas,
772 struct i915_gem_context *ctx,
773 bool *need_relocs)
774 {
775 struct drm_i915_gem_object *obj;
776 struct i915_vma *vma;
777 struct i915_address_space *vm;
778 struct list_head ordered_vmas;
779 struct list_head pinned_vmas;
780 bool has_fenced_gpu_access = INTEL_GEN(engine->i915) < 4;
781 int retry;
782
783 vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
784
785 INIT_LIST_HEAD(&ordered_vmas);
786 INIT_LIST_HEAD(&pinned_vmas);
787 while (!list_empty(vmas)) {
788 struct drm_i915_gem_exec_object2 *entry;
789 bool need_fence, need_mappable;
790
791 vma = list_first_entry(vmas, struct i915_vma, exec_list);
792 obj = vma->obj;
793 entry = vma->exec_entry;
794
795 if (ctx->flags & CONTEXT_NO_ZEROMAP)
796 entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
797
798 if (!has_fenced_gpu_access)
799 entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
800 need_fence =
801 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
802 obj->tiling_mode != I915_TILING_NONE;
803 need_mappable = need_fence || need_reloc_mappable(vma);
804
805 if (entry->flags & EXEC_OBJECT_PINNED)
806 list_move_tail(&vma->exec_list, &pinned_vmas);
807 else if (need_mappable) {
808 entry->flags |= __EXEC_OBJECT_NEEDS_MAP;
809 list_move(&vma->exec_list, &ordered_vmas);
810 } else
811 list_move_tail(&vma->exec_list, &ordered_vmas);
812
813 obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
814 obj->base.pending_write_domain = 0;
815 }
816 list_splice(&ordered_vmas, vmas);
817 list_splice(&pinned_vmas, vmas);
818
819 /* Attempt to pin all of the buffers into the GTT.
820 * This is done in 3 phases:
821 *
822 * 1a. Unbind all objects that do not match the GTT constraints for
823 * the execbuffer (fenceable, mappable, alignment etc).
824 * 1b. Increment pin count for already bound objects.
825 * 2. Bind new objects.
826 * 3. Decrement pin count.
827 *
828 * This avoid unnecessary unbinding of later objects in order to make
829 * room for the earlier objects *unless* we need to defragment.
830 */
831 retry = 0;
832 do {
833 int ret = 0;
834
835 /* Unbind any ill-fitting objects or pin. */
836 list_for_each_entry(vma, vmas, exec_list) {
837 if (!drm_mm_node_allocated(&vma->node))
838 continue;
839
840 if (eb_vma_misplaced(vma))
841 ret = i915_vma_unbind(vma);
842 else
843 ret = i915_gem_execbuffer_reserve_vma(vma,
844 engine,
845 need_relocs);
846 if (ret)
847 goto err;
848 }
849
850 /* Bind fresh objects */
851 list_for_each_entry(vma, vmas, exec_list) {
852 if (drm_mm_node_allocated(&vma->node))
853 continue;
854
855 ret = i915_gem_execbuffer_reserve_vma(vma, engine,
856 need_relocs);
857 if (ret)
858 goto err;
859 }
860
861 err:
862 if (ret != -ENOSPC || retry++)
863 return ret;
864
865 /* Decrement pin count for bound objects */
866 list_for_each_entry(vma, vmas, exec_list)
867 i915_gem_execbuffer_unreserve_vma(vma);
868
869 ret = i915_gem_evict_vm(vm, true);
870 if (ret)
871 return ret;
872 } while (1);
873 }
874
875 static int
876 i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
877 struct drm_i915_gem_execbuffer2 *args,
878 struct drm_file *file,
879 struct intel_engine_cs *engine,
880 struct eb_vmas *eb,
881 struct drm_i915_gem_exec_object2 *exec,
882 struct i915_gem_context *ctx)
883 {
884 struct drm_i915_gem_relocation_entry *reloc;
885 struct i915_address_space *vm;
886 struct i915_vma *vma;
887 bool need_relocs;
888 int *reloc_offset;
889 int i, total, ret;
890 unsigned count = args->buffer_count;
891
892 vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
893
894 /* We may process another execbuffer during the unlock... */
895 while (!list_empty(&eb->vmas)) {
896 vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
897 list_del_init(&vma->exec_list);
898 i915_gem_execbuffer_unreserve_vma(vma);
899 i915_gem_object_put(vma->obj);
900 }
901
902 mutex_unlock(&dev->struct_mutex);
903
904 total = 0;
905 for (i = 0; i < count; i++)
906 total += exec[i].relocation_count;
907
908 reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
909 reloc = drm_malloc_ab(total, sizeof(*reloc));
910 if (reloc == NULL || reloc_offset == NULL) {
911 drm_free_large(reloc);
912 drm_free_large(reloc_offset);
913 mutex_lock(&dev->struct_mutex);
914 return -ENOMEM;
915 }
916
917 total = 0;
918 for (i = 0; i < count; i++) {
919 struct drm_i915_gem_relocation_entry __user *user_relocs;
920 u64 invalid_offset = (u64)-1;
921 int j;
922
923 user_relocs = u64_to_user_ptr(exec[i].relocs_ptr);
924
925 if (copy_from_user(reloc+total, user_relocs,
926 exec[i].relocation_count * sizeof(*reloc))) {
927 ret = -EFAULT;
928 mutex_lock(&dev->struct_mutex);
929 goto err;
930 }
931
932 /* As we do not update the known relocation offsets after
933 * relocating (due to the complexities in lock handling),
934 * we need to mark them as invalid now so that we force the
935 * relocation processing next time. Just in case the target
936 * object is evicted and then rebound into its old
937 * presumed_offset before the next execbuffer - if that
938 * happened we would make the mistake of assuming that the
939 * relocations were valid.
940 */
941 for (j = 0; j < exec[i].relocation_count; j++) {
942 if (__copy_to_user(&user_relocs[j].presumed_offset,
943 &invalid_offset,
944 sizeof(invalid_offset))) {
945 ret = -EFAULT;
946 mutex_lock(&dev->struct_mutex);
947 goto err;
948 }
949 }
950
951 reloc_offset[i] = total;
952 total += exec[i].relocation_count;
953 }
954
955 ret = i915_mutex_lock_interruptible(dev);
956 if (ret) {
957 mutex_lock(&dev->struct_mutex);
958 goto err;
959 }
960
961 /* reacquire the objects */
962 eb_reset(eb);
963 ret = eb_lookup_vmas(eb, exec, args, vm, file);
964 if (ret)
965 goto err;
966
967 need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
968 ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
969 &need_relocs);
970 if (ret)
971 goto err;
972
973 list_for_each_entry(vma, &eb->vmas, exec_list) {
974 int offset = vma->exec_entry - exec;
975 ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,
976 reloc + reloc_offset[offset]);
977 if (ret)
978 goto err;
979 }
980
981 /* Leave the user relocations as are, this is the painfully slow path,
982 * and we want to avoid the complication of dropping the lock whilst
983 * having buffers reserved in the aperture and so causing spurious
984 * ENOSPC for random operations.
985 */
986
987 err:
988 drm_free_large(reloc);
989 drm_free_large(reloc_offset);
990 return ret;
991 }
992
993 static int
994 i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
995 struct list_head *vmas)
996 {
997 const unsigned other_rings = ~intel_engine_flag(req->engine);
998 struct i915_vma *vma;
999 uint32_t flush_domains = 0;
1000 bool flush_chipset = false;
1001 int ret;
1002
1003 list_for_each_entry(vma, vmas, exec_list) {
1004 struct drm_i915_gem_object *obj = vma->obj;
1005
1006 if (obj->active & other_rings) {
1007 ret = i915_gem_object_sync(obj, req);
1008 if (ret)
1009 return ret;
1010 }
1011
1012 if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
1013 flush_chipset |= i915_gem_clflush_object(obj, false);
1014
1015 flush_domains |= obj->base.write_domain;
1016 }
1017
1018 if (flush_chipset)
1019 i915_gem_chipset_flush(req->engine->i915);
1020
1021 if (flush_domains & I915_GEM_DOMAIN_GTT)
1022 wmb();
1023
1024 /* Unconditionally invalidate GPU caches and TLBs. */
1025 return req->engine->emit_flush(req, EMIT_INVALIDATE);
1026 }
1027
1028 static bool
1029 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
1030 {
1031 if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
1032 return false;
1033
1034 /* Kernel clipping was a DRI1 misfeature */
1035 if (exec->num_cliprects || exec->cliprects_ptr)
1036 return false;
1037
1038 if (exec->DR4 == 0xffffffff) {
1039 DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
1040 exec->DR4 = 0;
1041 }
1042 if (exec->DR1 || exec->DR4)
1043 return false;
1044
1045 if ((exec->batch_start_offset | exec->batch_len) & 0x7)
1046 return false;
1047
1048 return true;
1049 }
1050
1051 static int
1052 validate_exec_list(struct drm_device *dev,
1053 struct drm_i915_gem_exec_object2 *exec,
1054 int count)
1055 {
1056 unsigned relocs_total = 0;
1057 unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
1058 unsigned invalid_flags;
1059 int i;
1060
1061 /* INTERNAL flags must not overlap with external ones */
1062 BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & ~__EXEC_OBJECT_UNKNOWN_FLAGS);
1063
1064 invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
1065 if (USES_FULL_PPGTT(dev))
1066 invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
1067
1068 for (i = 0; i < count; i++) {
1069 char __user *ptr = u64_to_user_ptr(exec[i].relocs_ptr);
1070 int length; /* limited by fault_in_pages_readable() */
1071
1072 if (exec[i].flags & invalid_flags)
1073 return -EINVAL;
1074
1075 /* Offset can be used as input (EXEC_OBJECT_PINNED), reject
1076 * any non-page-aligned or non-canonical addresses.
1077 */
1078 if (exec[i].flags & EXEC_OBJECT_PINNED) {
1079 if (exec[i].offset !=
1080 gen8_canonical_addr(exec[i].offset & PAGE_MASK))
1081 return -EINVAL;
1082
1083 /* From drm_mm perspective address space is continuous,
1084 * so from this point we're always using non-canonical
1085 * form internally.
1086 */
1087 exec[i].offset = gen8_noncanonical_addr(exec[i].offset);
1088 }
1089
1090 if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
1091 return -EINVAL;
1092
1093 /* pad_to_size was once a reserved field, so sanitize it */
1094 if (exec[i].flags & EXEC_OBJECT_PAD_TO_SIZE) {
1095 if (offset_in_page(exec[i].pad_to_size))
1096 return -EINVAL;
1097 } else {
1098 exec[i].pad_to_size = 0;
1099 }
1100
1101 /* First check for malicious input causing overflow in
1102 * the worst case where we need to allocate the entire
1103 * relocation tree as a single array.
1104 */
1105 if (exec[i].relocation_count > relocs_max - relocs_total)
1106 return -EINVAL;
1107 relocs_total += exec[i].relocation_count;
1108
1109 length = exec[i].relocation_count *
1110 sizeof(struct drm_i915_gem_relocation_entry);
1111 /*
1112 * We must check that the entire relocation array is safe
1113 * to read, but since we may need to update the presumed
1114 * offsets during execution, check for full write access.
1115 */
1116 if (!access_ok(VERIFY_WRITE, ptr, length))
1117 return -EFAULT;
1118
1119 if (likely(!i915.prefault_disable)) {
1120 if (fault_in_multipages_readable(ptr, length))
1121 return -EFAULT;
1122 }
1123 }
1124
1125 return 0;
1126 }
1127
1128 static struct i915_gem_context *
1129 i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
1130 struct intel_engine_cs *engine, const u32 ctx_id)
1131 {
1132 struct i915_gem_context *ctx = NULL;
1133 struct i915_ctx_hang_stats *hs;
1134
1135 if (engine->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)
1136 return ERR_PTR(-EINVAL);
1137
1138 ctx = i915_gem_context_lookup(file->driver_priv, ctx_id);
1139 if (IS_ERR(ctx))
1140 return ctx;
1141
1142 hs = &ctx->hang_stats;
1143 if (hs->banned) {
1144 DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
1145 return ERR_PTR(-EIO);
1146 }
1147
1148 return ctx;
1149 }
1150
1151 void i915_vma_move_to_active(struct i915_vma *vma,
1152 struct drm_i915_gem_request *req,
1153 unsigned int flags)
1154 {
1155 struct drm_i915_gem_object *obj = vma->obj;
1156 const unsigned int idx = req->engine->id;
1157
1158 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1159
1160 obj->dirty = 1; /* be paranoid */
1161
1162 /* Add a reference if we're newly entering the active list.
1163 * The order in which we add operations to the retirement queue is
1164 * vital here: mark_active adds to the start of the callback list,
1165 * such that subsequent callbacks are called first. Therefore we
1166 * add the active reference first and queue for it to be dropped
1167 * *last*.
1168 */
1169 if (obj->active == 0)
1170 i915_gem_object_get(obj);
1171 obj->active |= 1 << idx;
1172 i915_gem_active_set(&obj->last_read[idx], req);
1173
1174 if (flags & EXEC_OBJECT_WRITE) {
1175 i915_gem_active_set(&obj->last_write, req);
1176
1177 intel_fb_obj_invalidate(obj, ORIGIN_CS);
1178
1179 /* update for the implicit flush after a batch */
1180 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1181 }
1182
1183 if (flags & EXEC_OBJECT_NEEDS_FENCE) {
1184 i915_gem_active_set(&obj->last_fence, req);
1185 if (flags & __EXEC_OBJECT_HAS_FENCE) {
1186 struct drm_i915_private *dev_priv = req->i915;
1187
1188 list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
1189 &dev_priv->mm.fence_list);
1190 }
1191 }
1192
1193 i915_vma_set_active(vma, idx);
1194 i915_gem_active_set(&vma->last_read[idx], req);
1195 list_move_tail(&vma->vm_link, &vma->vm->active_list);
1196 }
1197
1198 static void
1199 i915_gem_execbuffer_move_to_active(struct list_head *vmas,
1200 struct drm_i915_gem_request *req)
1201 {
1202 struct i915_vma *vma;
1203
1204 list_for_each_entry(vma, vmas, exec_list) {
1205 struct drm_i915_gem_object *obj = vma->obj;
1206 u32 old_read = obj->base.read_domains;
1207 u32 old_write = obj->base.write_domain;
1208
1209 obj->base.write_domain = obj->base.pending_write_domain;
1210 if (obj->base.write_domain)
1211 vma->exec_entry->flags |= EXEC_OBJECT_WRITE;
1212 else
1213 obj->base.pending_read_domains |= obj->base.read_domains;
1214 obj->base.read_domains = obj->base.pending_read_domains;
1215
1216 i915_vma_move_to_active(vma, req, vma->exec_entry->flags);
1217 trace_i915_gem_object_change_domain(obj, old_read, old_write);
1218 }
1219 }
1220
1221 static int
1222 i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
1223 {
1224 struct intel_ring *ring = req->ring;
1225 int ret, i;
1226
1227 if (!IS_GEN7(req->i915) || req->engine->id != RCS) {
1228 DRM_DEBUG("sol reset is gen7/rcs only\n");
1229 return -EINVAL;
1230 }
1231
1232 ret = intel_ring_begin(req, 4 * 3);
1233 if (ret)
1234 return ret;
1235
1236 for (i = 0; i < 4; i++) {
1237 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1238 intel_ring_emit_reg(ring, GEN7_SO_WRITE_OFFSET(i));
1239 intel_ring_emit(ring, 0);
1240 }
1241
1242 intel_ring_advance(ring);
1243
1244 return 0;
1245 }
1246
1247 static struct i915_vma*
1248 i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
1249 struct drm_i915_gem_exec_object2 *shadow_exec_entry,
1250 struct drm_i915_gem_object *batch_obj,
1251 struct eb_vmas *eb,
1252 u32 batch_start_offset,
1253 u32 batch_len,
1254 bool is_master)
1255 {
1256 struct drm_i915_gem_object *shadow_batch_obj;
1257 struct i915_vma *vma;
1258 int ret;
1259
1260 shadow_batch_obj = i915_gem_batch_pool_get(&engine->batch_pool,
1261 PAGE_ALIGN(batch_len));
1262 if (IS_ERR(shadow_batch_obj))
1263 return ERR_CAST(shadow_batch_obj);
1264
1265 ret = intel_engine_cmd_parser(engine,
1266 batch_obj,
1267 shadow_batch_obj,
1268 batch_start_offset,
1269 batch_len,
1270 is_master);
1271 if (ret)
1272 goto err;
1273
1274 ret = i915_gem_object_ggtt_pin(shadow_batch_obj, NULL, 0, 0, 0);
1275 if (ret)
1276 goto err;
1277
1278 i915_gem_object_unpin_pages(shadow_batch_obj);
1279
1280 memset(shadow_exec_entry, 0, sizeof(*shadow_exec_entry));
1281
1282 vma = i915_gem_obj_to_ggtt(shadow_batch_obj);
1283 vma->exec_entry = shadow_exec_entry;
1284 vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
1285 i915_gem_object_get(shadow_batch_obj);
1286 list_add_tail(&vma->exec_list, &eb->vmas);
1287
1288 return vma;
1289
1290 err:
1291 i915_gem_object_unpin_pages(shadow_batch_obj);
1292 if (ret == -EACCES) /* unhandled chained batch */
1293 return NULL;
1294 else
1295 return ERR_PTR(ret);
1296 }
1297
1298 static int
1299 execbuf_submit(struct i915_execbuffer_params *params,
1300 struct drm_i915_gem_execbuffer2 *args,
1301 struct list_head *vmas)
1302 {
1303 struct drm_i915_private *dev_priv = params->request->i915;
1304 u64 exec_start, exec_len;
1305 int instp_mode;
1306 u32 instp_mask;
1307 int ret;
1308
1309 ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas);
1310 if (ret)
1311 return ret;
1312
1313 ret = i915_switch_context(params->request);
1314 if (ret)
1315 return ret;
1316
1317 instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
1318 instp_mask = I915_EXEC_CONSTANTS_MASK;
1319 switch (instp_mode) {
1320 case I915_EXEC_CONSTANTS_REL_GENERAL:
1321 case I915_EXEC_CONSTANTS_ABSOLUTE:
1322 case I915_EXEC_CONSTANTS_REL_SURFACE:
1323 if (instp_mode != 0 && params->engine->id != RCS) {
1324 DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
1325 return -EINVAL;
1326 }
1327
1328 if (instp_mode != dev_priv->relative_constants_mode) {
1329 if (INTEL_INFO(dev_priv)->gen < 4) {
1330 DRM_DEBUG("no rel constants on pre-gen4\n");
1331 return -EINVAL;
1332 }
1333
1334 if (INTEL_INFO(dev_priv)->gen > 5 &&
1335 instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
1336 DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
1337 return -EINVAL;
1338 }
1339
1340 /* The HW changed the meaning on this bit on gen6 */
1341 if (INTEL_INFO(dev_priv)->gen >= 6)
1342 instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
1343 }
1344 break;
1345 default:
1346 DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
1347 return -EINVAL;
1348 }
1349
1350 if (params->engine->id == RCS &&
1351 instp_mode != dev_priv->relative_constants_mode) {
1352 struct intel_ring *ring = params->request->ring;
1353
1354 ret = intel_ring_begin(params->request, 4);
1355 if (ret)
1356 return ret;
1357
1358 intel_ring_emit(ring, MI_NOOP);
1359 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1360 intel_ring_emit_reg(ring, INSTPM);
1361 intel_ring_emit(ring, instp_mask << 16 | instp_mode);
1362 intel_ring_advance(ring);
1363
1364 dev_priv->relative_constants_mode = instp_mode;
1365 }
1366
1367 if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
1368 ret = i915_reset_gen7_sol_offsets(params->request);
1369 if (ret)
1370 return ret;
1371 }
1372
1373 exec_len = args->batch_len;
1374 exec_start = params->batch->node.start +
1375 params->args_batch_start_offset;
1376
1377 if (exec_len == 0)
1378 exec_len = params->batch->size;
1379
1380 ret = params->engine->emit_bb_start(params->request,
1381 exec_start, exec_len,
1382 params->dispatch_flags);
1383 if (ret)
1384 return ret;
1385
1386 trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
1387
1388 i915_gem_execbuffer_move_to_active(vmas, params->request);
1389
1390 return 0;
1391 }
1392
1393 /**
1394 * Find one BSD ring to dispatch the corresponding BSD command.
1395 * The engine index is returned.
1396 */
1397 static unsigned int
1398 gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
1399 struct drm_file *file)
1400 {
1401 struct drm_i915_file_private *file_priv = file->driver_priv;
1402
1403 /* Check whether the file_priv has already selected one ring. */
1404 if ((int)file_priv->bsd_engine < 0) {
1405 /* If not, use the ping-pong mechanism to select one. */
1406 mutex_lock(&dev_priv->drm.struct_mutex);
1407 file_priv->bsd_engine = dev_priv->mm.bsd_engine_dispatch_index;
1408 dev_priv->mm.bsd_engine_dispatch_index ^= 1;
1409 mutex_unlock(&dev_priv->drm.struct_mutex);
1410 }
1411
1412 return file_priv->bsd_engine;
1413 }
1414
1415 #define I915_USER_RINGS (4)
1416
1417 static const enum intel_engine_id user_ring_map[I915_USER_RINGS + 1] = {
1418 [I915_EXEC_DEFAULT] = RCS,
1419 [I915_EXEC_RENDER] = RCS,
1420 [I915_EXEC_BLT] = BCS,
1421 [I915_EXEC_BSD] = VCS,
1422 [I915_EXEC_VEBOX] = VECS
1423 };
1424
1425 static struct intel_engine_cs *
1426 eb_select_engine(struct drm_i915_private *dev_priv,
1427 struct drm_file *file,
1428 struct drm_i915_gem_execbuffer2 *args)
1429 {
1430 unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;
1431 struct intel_engine_cs *engine;
1432
1433 if (user_ring_id > I915_USER_RINGS) {
1434 DRM_DEBUG("execbuf with unknown ring: %u\n", user_ring_id);
1435 return NULL;
1436 }
1437
1438 if ((user_ring_id != I915_EXEC_BSD) &&
1439 ((args->flags & I915_EXEC_BSD_MASK) != 0)) {
1440 DRM_DEBUG("execbuf with non bsd ring but with invalid "
1441 "bsd dispatch flags: %d\n", (int)(args->flags));
1442 return NULL;
1443 }
1444
1445 if (user_ring_id == I915_EXEC_BSD && HAS_BSD2(dev_priv)) {
1446 unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
1447
1448 if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
1449 bsd_idx = gen8_dispatch_bsd_engine(dev_priv, file);
1450 } else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
1451 bsd_idx <= I915_EXEC_BSD_RING2) {
1452 bsd_idx >>= I915_EXEC_BSD_SHIFT;
1453 bsd_idx--;
1454 } else {
1455 DRM_DEBUG("execbuf with unknown bsd ring: %u\n",
1456 bsd_idx);
1457 return NULL;
1458 }
1459
1460 engine = &dev_priv->engine[_VCS(bsd_idx)];
1461 } else {
1462 engine = &dev_priv->engine[user_ring_map[user_ring_id]];
1463 }
1464
1465 if (!intel_engine_initialized(engine)) {
1466 DRM_DEBUG("execbuf with invalid ring: %u\n", user_ring_id);
1467 return NULL;
1468 }
1469
1470 return engine;
1471 }
1472
1473 static int
1474 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1475 struct drm_file *file,
1476 struct drm_i915_gem_execbuffer2 *args,
1477 struct drm_i915_gem_exec_object2 *exec)
1478 {
1479 struct drm_i915_private *dev_priv = to_i915(dev);
1480 struct i915_ggtt *ggtt = &dev_priv->ggtt;
1481 struct eb_vmas *eb;
1482 struct drm_i915_gem_exec_object2 shadow_exec_entry;
1483 struct intel_engine_cs *engine;
1484 struct i915_gem_context *ctx;
1485 struct i915_address_space *vm;
1486 struct i915_execbuffer_params params_master; /* XXX: will be removed later */
1487 struct i915_execbuffer_params *params = &params_master;
1488 const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
1489 u32 dispatch_flags;
1490 int ret;
1491 bool need_relocs;
1492
1493 if (!i915_gem_check_execbuffer(args))
1494 return -EINVAL;
1495
1496 ret = validate_exec_list(dev, exec, args->buffer_count);
1497 if (ret)
1498 return ret;
1499
1500 dispatch_flags = 0;
1501 if (args->flags & I915_EXEC_SECURE) {
1502 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
1503 return -EPERM;
1504
1505 dispatch_flags |= I915_DISPATCH_SECURE;
1506 }
1507 if (args->flags & I915_EXEC_IS_PINNED)
1508 dispatch_flags |= I915_DISPATCH_PINNED;
1509
1510 engine = eb_select_engine(dev_priv, file, args);
1511 if (!engine)
1512 return -EINVAL;
1513
1514 if (args->buffer_count < 1) {
1515 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1516 return -EINVAL;
1517 }
1518
1519 if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
1520 if (!HAS_RESOURCE_STREAMER(dev)) {
1521 DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
1522 return -EINVAL;
1523 }
1524 if (engine->id != RCS) {
1525 DRM_DEBUG("RS is not available on %s\n",
1526 engine->name);
1527 return -EINVAL;
1528 }
1529
1530 dispatch_flags |= I915_DISPATCH_RS;
1531 }
1532
1533 /* Take a local wakeref for preparing to dispatch the execbuf as
1534 * we expect to access the hardware fairly frequently in the
1535 * process. Upon first dispatch, we acquire another prolonged
1536 * wakeref that we hold until the GPU has been idle for at least
1537 * 100ms.
1538 */
1539 intel_runtime_pm_get(dev_priv);
1540
1541 ret = i915_mutex_lock_interruptible(dev);
1542 if (ret)
1543 goto pre_mutex_err;
1544
1545 ctx = i915_gem_validate_context(dev, file, engine, ctx_id);
1546 if (IS_ERR(ctx)) {
1547 mutex_unlock(&dev->struct_mutex);
1548 ret = PTR_ERR(ctx);
1549 goto pre_mutex_err;
1550 }
1551
1552 i915_gem_context_get(ctx);
1553
1554 if (ctx->ppgtt)
1555 vm = &ctx->ppgtt->base;
1556 else
1557 vm = &ggtt->base;
1558
1559 memset(&params_master, 0x00, sizeof(params_master));
1560
1561 eb = eb_create(args);
1562 if (eb == NULL) {
1563 i915_gem_context_put(ctx);
1564 mutex_unlock(&dev->struct_mutex);
1565 ret = -ENOMEM;
1566 goto pre_mutex_err;
1567 }
1568
1569 /* Look up object handles */
1570 ret = eb_lookup_vmas(eb, exec, args, vm, file);
1571 if (ret)
1572 goto err;
1573
1574 /* take note of the batch buffer before we might reorder the lists */
1575 params->batch = eb_get_batch(eb);
1576
1577 /* Move the objects en-masse into the GTT, evicting if necessary. */
1578 need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
1579 ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
1580 &need_relocs);
1581 if (ret)
1582 goto err;
1583
1584 /* The objects are in their final locations, apply the relocations. */
1585 if (need_relocs)
1586 ret = i915_gem_execbuffer_relocate(eb);
1587 if (ret) {
1588 if (ret == -EFAULT) {
1589 ret = i915_gem_execbuffer_relocate_slow(dev, args, file,
1590 engine,
1591 eb, exec, ctx);
1592 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1593 }
1594 if (ret)
1595 goto err;
1596 }
1597
1598 /* Set the pending read domains for the batch buffer to COMMAND */
1599 if (params->batch->obj->base.pending_write_domain) {
1600 DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
1601 ret = -EINVAL;
1602 goto err;
1603 }
1604
1605 params->args_batch_start_offset = args->batch_start_offset;
1606 if (intel_engine_needs_cmd_parser(engine) && args->batch_len) {
1607 struct i915_vma *vma;
1608
1609 vma = i915_gem_execbuffer_parse(engine, &shadow_exec_entry,
1610 params->batch->obj,
1611 eb,
1612 args->batch_start_offset,
1613 args->batch_len,
1614 drm_is_current_master(file));
1615 if (IS_ERR(vma)) {
1616 ret = PTR_ERR(vma);
1617 goto err;
1618 }
1619
1620 if (vma) {
1621 /*
1622 * Batch parsed and accepted:
1623 *
1624 * Set the DISPATCH_SECURE bit to remove the NON_SECURE
1625 * bit from MI_BATCH_BUFFER_START commands issued in
1626 * the dispatch_execbuffer implementations. We
1627 * specifically don't want that set on batches the
1628 * command parser has accepted.
1629 */
1630 dispatch_flags |= I915_DISPATCH_SECURE;
1631 params->args_batch_start_offset = 0;
1632 params->batch = vma;
1633 }
1634 }
1635
1636 params->batch->obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1637
1638 /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
1639 * batch" bit. Hence we need to pin secure batches into the global gtt.
1640 * hsw should have this fixed, but bdw mucks it up again. */
1641 if (dispatch_flags & I915_DISPATCH_SECURE) {
1642 struct drm_i915_gem_object *obj = params->batch->obj;
1643
1644 /*
1645 * So on first glance it looks freaky that we pin the batch here
1646 * outside of the reservation loop. But:
1647 * - The batch is already pinned into the relevant ppgtt, so we
1648 * already have the backing storage fully allocated.
1649 * - No other BO uses the global gtt (well contexts, but meh),
1650 * so we don't really have issues with multiple objects not
1651 * fitting due to fragmentation.
1652 * So this is actually safe.
1653 */
1654 ret = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
1655 if (ret)
1656 goto err;
1657
1658 params->batch = i915_gem_obj_to_ggtt(obj);
1659 }
1660
1661 /* Allocate a request for this batch buffer nice and early. */
1662 params->request = i915_gem_request_alloc(engine, ctx);
1663 if (IS_ERR(params->request)) {
1664 ret = PTR_ERR(params->request);
1665 goto err_batch_unpin;
1666 }
1667
1668 ret = i915_gem_request_add_to_client(params->request, file);
1669 if (ret)
1670 goto err_request;
1671
1672 /*
1673 * Save assorted stuff away to pass through to *_submission().
1674 * NB: This data should be 'persistent' and not local as it will
1675 * kept around beyond the duration of the IOCTL once the GPU
1676 * scheduler arrives.
1677 */
1678 params->dev = dev;
1679 params->file = file;
1680 params->engine = engine;
1681 params->dispatch_flags = dispatch_flags;
1682 params->ctx = ctx;
1683
1684 ret = execbuf_submit(params, args, &eb->vmas);
1685 err_request:
1686 __i915_add_request(params->request, params->batch->obj, ret == 0);
1687
1688 err_batch_unpin:
1689 /*
1690 * FIXME: We crucially rely upon the active tracking for the (ppgtt)
1691 * batch vma for correctness. For less ugly and less fragility this
1692 * needs to be adjusted to also track the ggtt batch vma properly as
1693 * active.
1694 */
1695 if (dispatch_flags & I915_DISPATCH_SECURE)
1696 i915_vma_unpin(params->batch);
1697 err:
1698 /* the request owns the ref now */
1699 i915_gem_context_put(ctx);
1700 eb_destroy(eb);
1701
1702 mutex_unlock(&dev->struct_mutex);
1703
1704 pre_mutex_err:
1705 /* intel_gpu_busy should also get a ref, so it will free when the device
1706 * is really idle. */
1707 intel_runtime_pm_put(dev_priv);
1708 return ret;
1709 }
1710
1711 /*
1712 * Legacy execbuffer just creates an exec2 list from the original exec object
1713 * list array and passes it to the real function.
1714 */
1715 int
1716 i915_gem_execbuffer(struct drm_device *dev, void *data,
1717 struct drm_file *file)
1718 {
1719 struct drm_i915_gem_execbuffer *args = data;
1720 struct drm_i915_gem_execbuffer2 exec2;
1721 struct drm_i915_gem_exec_object *exec_list = NULL;
1722 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1723 int ret, i;
1724
1725 if (args->buffer_count < 1) {
1726 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1727 return -EINVAL;
1728 }
1729
1730 /* Copy in the exec list from userland */
1731 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
1732 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1733 if (exec_list == NULL || exec2_list == NULL) {
1734 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1735 args->buffer_count);
1736 drm_free_large(exec_list);
1737 drm_free_large(exec2_list);
1738 return -ENOMEM;
1739 }
1740 ret = copy_from_user(exec_list,
1741 u64_to_user_ptr(args->buffers_ptr),
1742 sizeof(*exec_list) * args->buffer_count);
1743 if (ret != 0) {
1744 DRM_DEBUG("copy %d exec entries failed %d\n",
1745 args->buffer_count, ret);
1746 drm_free_large(exec_list);
1747 drm_free_large(exec2_list);
1748 return -EFAULT;
1749 }
1750
1751 for (i = 0; i < args->buffer_count; i++) {
1752 exec2_list[i].handle = exec_list[i].handle;
1753 exec2_list[i].relocation_count = exec_list[i].relocation_count;
1754 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1755 exec2_list[i].alignment = exec_list[i].alignment;
1756 exec2_list[i].offset = exec_list[i].offset;
1757 if (INTEL_INFO(dev)->gen < 4)
1758 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1759 else
1760 exec2_list[i].flags = 0;
1761 }
1762
1763 exec2.buffers_ptr = args->buffers_ptr;
1764 exec2.buffer_count = args->buffer_count;
1765 exec2.batch_start_offset = args->batch_start_offset;
1766 exec2.batch_len = args->batch_len;
1767 exec2.DR1 = args->DR1;
1768 exec2.DR4 = args->DR4;
1769 exec2.num_cliprects = args->num_cliprects;
1770 exec2.cliprects_ptr = args->cliprects_ptr;
1771 exec2.flags = I915_EXEC_RENDER;
1772 i915_execbuffer2_set_context_id(exec2, 0);
1773
1774 ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1775 if (!ret) {
1776 struct drm_i915_gem_exec_object __user *user_exec_list =
1777 u64_to_user_ptr(args->buffers_ptr);
1778
1779 /* Copy the new buffer offsets back to the user's exec list. */
1780 for (i = 0; i < args->buffer_count; i++) {
1781 exec2_list[i].offset =
1782 gen8_canonical_addr(exec2_list[i].offset);
1783 ret = __copy_to_user(&user_exec_list[i].offset,
1784 &exec2_list[i].offset,
1785 sizeof(user_exec_list[i].offset));
1786 if (ret) {
1787 ret = -EFAULT;
1788 DRM_DEBUG("failed to copy %d exec entries "
1789 "back to user (%d)\n",
1790 args->buffer_count, ret);
1791 break;
1792 }
1793 }
1794 }
1795
1796 drm_free_large(exec_list);
1797 drm_free_large(exec2_list);
1798 return ret;
1799 }
1800
1801 int
1802 i915_gem_execbuffer2(struct drm_device *dev, void *data,
1803 struct drm_file *file)
1804 {
1805 struct drm_i915_gem_execbuffer2 *args = data;
1806 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1807 int ret;
1808
1809 if (args->buffer_count < 1 ||
1810 args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
1811 DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
1812 return -EINVAL;
1813 }
1814
1815 if (args->rsvd2 != 0) {
1816 DRM_DEBUG("dirty rvsd2 field\n");
1817 return -EINVAL;
1818 }
1819
1820 exec2_list = drm_malloc_gfp(args->buffer_count,
1821 sizeof(*exec2_list),
1822 GFP_TEMPORARY);
1823 if (exec2_list == NULL) {
1824 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1825 args->buffer_count);
1826 return -ENOMEM;
1827 }
1828 ret = copy_from_user(exec2_list,
1829 u64_to_user_ptr(args->buffers_ptr),
1830 sizeof(*exec2_list) * args->buffer_count);
1831 if (ret != 0) {
1832 DRM_DEBUG("copy %d exec entries failed %d\n",
1833 args->buffer_count, ret);
1834 drm_free_large(exec2_list);
1835 return -EFAULT;
1836 }
1837
1838 ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1839 if (!ret) {
1840 /* Copy the new buffer offsets back to the user's exec list. */
1841 struct drm_i915_gem_exec_object2 __user *user_exec_list =
1842 u64_to_user_ptr(args->buffers_ptr);
1843 int i;
1844
1845 for (i = 0; i < args->buffer_count; i++) {
1846 exec2_list[i].offset =
1847 gen8_canonical_addr(exec2_list[i].offset);
1848 ret = __copy_to_user(&user_exec_list[i].offset,
1849 &exec2_list[i].offset,
1850 sizeof(user_exec_list[i].offset));
1851 if (ret) {
1852 ret = -EFAULT;
1853 DRM_DEBUG("failed to copy %d exec entries "
1854 "back to user\n",
1855 args->buffer_count);
1856 break;
1857 }
1858 }
1859 }
1860
1861 drm_free_large(exec2_list);
1862 return ret;
1863 }
Linux kernel - Deliverables
This page took 0.123681 seconds and 5 git commands to generate.