projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Enable SandyBridge blitter ring
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
5a0e3ad6 34#include <linux/slab.h>
673a394b 35#include <linux/swap.h>
79e53945 36#include <linux/pci.h>
f8f235e5 37#include <linux/intel-gtt.h>
673a394b 38
0108a3ed 39static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
ba3d8d74
DV		 40
41static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
42 bool pipelined);
e47c68e9
EA		 43static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
44static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
e47c68e9
EA		 45static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
46 int write);
47static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
48 uint64_t offset,
49 uint64_t size);
50static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
2cf34d7b
CW		 51static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
52 bool interruptible);
de151cf6
JB		 53static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
54 unsigned alignment);
de151cf6 55static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
71acb5eb
DA		 56static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
57 struct drm_i915_gem_pwrite *args,
58 struct drm_file *file_priv);
be72615b 59static void i915_gem_free_object_tail(struct drm_gem_object *obj);
673a394b 60
5cdf5881
CW		 61static int
62i915_gem_object_get_pages(struct drm_gem_object *obj,
63 gfp_t gfpmask);
64
65static void
66i915_gem_object_put_pages(struct drm_gem_object *obj);
67
31169714
CW		 68static LIST_HEAD(shrink_list);
69static DEFINE_SPINLOCK(shrink_list_lock);
70
73aa808f
CW		 71/* some bookkeeping */
72static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
73 size_t size)
74{
75 dev_priv->mm.object_count++;
76 dev_priv->mm.object_memory += size;
77}
78
79static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
80 size_t size)
81{
82 dev_priv->mm.object_count--;
83 dev_priv->mm.object_memory -= size;
84}
85
86static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
87 size_t size)
88{
89 dev_priv->mm.gtt_count++;
90 dev_priv->mm.gtt_memory += size;
91}
92
93static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
94 size_t size)
95{
96 dev_priv->mm.gtt_count--;
97 dev_priv->mm.gtt_memory -= size;
98}
99
100static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
101 size_t size)
102{
103 dev_priv->mm.pin_count++;
104 dev_priv->mm.pin_memory += size;
105}
106
107static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
108 size_t size)
109{
110 dev_priv->mm.pin_count--;
111 dev_priv->mm.pin_memory -= size;
112}
113
30dbf0c0
CW		 114int
115i915_gem_check_is_wedged(struct drm_device *dev)
116{
117 struct drm_i915_private *dev_priv = dev->dev_private;
118 struct completion *x = &dev_priv->error_completion;
119 unsigned long flags;
120 int ret;
121
122 if (!atomic_read(&dev_priv->mm.wedged))
123 return 0;
124
125 ret = wait_for_completion_interruptible(x);
126 if (ret)
127 return ret;
128
129 /* Success, we reset the GPU! */
130 if (!atomic_read(&dev_priv->mm.wedged))
131 return 0;
132
133 /* GPU is hung, bump the completion count to account for
134 * the token we just consumed so that we never hit zero and
135 * end up waiting upon a subsequent completion event that
136 * will never happen.
137 */
138 spin_lock_irqsave(&x->wait.lock, flags);
139 x->done++;
140 spin_unlock_irqrestore(&x->wait.lock, flags);
141 return -EIO;
142}
143
76c1dec1
CW		 144static int i915_mutex_lock_interruptible(struct drm_device *dev)
145{
146 struct drm_i915_private *dev_priv = dev->dev_private;
147 int ret;
148
149 ret = i915_gem_check_is_wedged(dev);
150 if (ret)
151 return ret;
152
153 ret = mutex_lock_interruptible(&dev->struct_mutex);
154 if (ret)
155 return ret;
156
157 if (atomic_read(&dev_priv->mm.wedged)) {
158 mutex_unlock(&dev->struct_mutex);
159 return -EAGAIN;
160 }
161
23bc5982 162 WARN_ON(i915_verify_lists(dev));
76c1dec1
CW		 163 return 0;
164}
30dbf0c0 165
7d1c4804
CW		 166static inline bool
167i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
168{
169 return obj_priv->gtt_space &&
170 !obj_priv->active &&
171 obj_priv->pin_count == 0;
172}
173
73aa808f
CW		 174int i915_gem_do_init(struct drm_device *dev,
175 unsigned long start,
79e53945 176 unsigned long end)
673a394b
EA		 177{
178 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 179
79e53945
JB		 180 if (start >= end ||
181 (start & (PAGE_SIZE - 1)) != 0 ||
182 (end & (PAGE_SIZE - 1)) != 0) {
673a394b
EA		 183 return -EINVAL;
184 }
185
79e53945
JB		 186 drm_mm_init(&dev_priv->mm.gtt_space, start,
187 end - start);
673a394b 188
73aa808f 189 dev_priv->mm.gtt_total = end - start;
79e53945
JB		 190
191 return 0;
192}
673a394b 193
79e53945
JB		 194int
195i915_gem_init_ioctl(struct drm_device *dev, void *data,
196 struct drm_file *file_priv)
197{
198 struct drm_i915_gem_init *args = data;
199 int ret;
200
201 mutex_lock(&dev->struct_mutex);
202 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
673a394b
EA		 203 mutex_unlock(&dev->struct_mutex);
204
79e53945 205 return ret;
673a394b
EA		 206}
207
5a125c3c
EA		 208int
209i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
210 struct drm_file *file_priv)
211{
73aa808f 212 struct drm_i915_private *dev_priv = dev->dev_private;
5a125c3c 213 struct drm_i915_gem_get_aperture *args = data;
5a125c3c
EA		 214
215 if (!(dev->driver->driver_features & DRIVER_GEM))
216 return -ENODEV;
217
73aa808f
CW		 218 mutex_lock(&dev->struct_mutex);
219 args->aper_size = dev_priv->mm.gtt_total;
220 args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
221 mutex_unlock(&dev->struct_mutex);
5a125c3c
EA		 222
223 return 0;
224}
225
673a394b
EA		 226
227/**
228 * Creates a new mm object and returns a handle to it.
229 */
230int
231i915_gem_create_ioctl(struct drm_device *dev, void *data,
232 struct drm_file *file_priv)
233{
234 struct drm_i915_gem_create *args = data;
235 struct drm_gem_object *obj;
a1a2d1d3
PP		 236 int ret;
237 u32 handle;
673a394b
EA		 238
239 args->size = roundup(args->size, PAGE_SIZE);
240
241 /* Allocate the new object */
ac52bc56 242 obj = i915_gem_alloc_object(dev, args->size);
673a394b
EA		 243 if (obj == NULL)
244 return -ENOMEM;
245
246 ret = drm_gem_handle_create(file_priv, obj, &handle);
1dfd9754 247 if (ret) {
202f2fef
CW		 248 drm_gem_object_release(obj);
249 i915_gem_info_remove_obj(dev->dev_private, obj->size);
250 kfree(obj);
673a394b 251 return ret;
1dfd9754 252 }
673a394b 253
202f2fef
CW		 254 /* drop reference from allocate - handle holds it now */
255 drm_gem_object_unreference(obj);
256 trace_i915_gem_object_create(obj);
257
1dfd9754 258 args->handle = handle;
673a394b
EA		 259 return 0;
260}
261
eb01459f
EA		 262static inline int
263fast_shmem_read(struct page **pages,
264 loff_t page_base, int page_offset,
265 char __user *data,
266 int length)
267{
b5e4feb6 268 char *vaddr;
4f27b75d 269 int ret;
eb01459f
EA		 270
271 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
4f27b75d 272 ret = __copy_to_user_inatomic(data, vaddr + page_offset, length);
eb01459f
EA		 273 kunmap_atomic(vaddr, KM_USER0);
274
4f27b75d 275 return ret;
eb01459f
EA		 276}
277
280b713b
EA		 278static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
279{
280 drm_i915_private_t *dev_priv = obj->dev->dev_private;
23010e43 281 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
280b713b
EA		 282
283 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
284 obj_priv->tiling_mode != I915_TILING_NONE;
285}
286
99a03df5 287static inline void
40123c1f
EA		 288slow_shmem_copy(struct page *dst_page,
289 int dst_offset,
290 struct page *src_page,
291 int src_offset,
292 int length)
293{
294 char *dst_vaddr, *src_vaddr;
295
99a03df5
CW		 296 dst_vaddr = kmap(dst_page);
297 src_vaddr = kmap(src_page);
40123c1f
EA		 298
299 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
300
99a03df5
CW		 301 kunmap(src_page);
302 kunmap(dst_page);
40123c1f
EA		 303}
304
99a03df5 305static inline void
280b713b
EA		 306slow_shmem_bit17_copy(struct page *gpu_page,
307 int gpu_offset,
308 struct page *cpu_page,
309 int cpu_offset,
310 int length,
311 int is_read)
312{
313 char *gpu_vaddr, *cpu_vaddr;
314
315 /* Use the unswizzled path if this page isn't affected. */
316 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
317 if (is_read)
318 return slow_shmem_copy(cpu_page, cpu_offset,
319 gpu_page, gpu_offset, length);
320 else
321 return slow_shmem_copy(gpu_page, gpu_offset,
322 cpu_page, cpu_offset, length);
323 }
324
99a03df5
CW		 325 gpu_vaddr = kmap(gpu_page);
326 cpu_vaddr = kmap(cpu_page);
280b713b
EA		 327
328 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
329 * XORing with the other bits (A9 for Y, A9 and A10 for X)
330 */
331 while (length > 0) {
332 int cacheline_end = ALIGN(gpu_offset + 1, 64);
333 int this_length = min(cacheline_end - gpu_offset, length);
334 int swizzled_gpu_offset = gpu_offset ^ 64;
335
336 if (is_read) {
337 memcpy(cpu_vaddr + cpu_offset,
338 gpu_vaddr + swizzled_gpu_offset,
339 this_length);
340 } else {
341 memcpy(gpu_vaddr + swizzled_gpu_offset,
342 cpu_vaddr + cpu_offset,
343 this_length);
344 }
345 cpu_offset += this_length;
346 gpu_offset += this_length;
347 length -= this_length;
348 }
349
99a03df5
CW		 350 kunmap(cpu_page);
351 kunmap(gpu_page);
280b713b
EA		 352}
353
eb01459f
EA		 354/**
355 * This is the fast shmem pread path, which attempts to copy_from_user directly
356 * from the backing pages of the object to the user's address space. On a
357 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
358 */
359static int
360i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
361 struct drm_i915_gem_pread *args,
362 struct drm_file *file_priv)
363{
23010e43 364 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
eb01459f
EA		 365 ssize_t remain;
366 loff_t offset, page_base;
367 char __user *user_data;
368 int page_offset, page_length;
eb01459f
EA		 369
370 user_data = (char __user *) (uintptr_t) args->data_ptr;
371 remain = args->size;
372
23010e43 373 obj_priv = to_intel_bo(obj);
eb01459f
EA		 374 offset = args->offset;
375
376 while (remain > 0) {
377 /* Operation in this page
378 *
379 * page_base = page offset within aperture
380 * page_offset = offset within page
381 * page_length = bytes to copy for this page
382 */
383 page_base = (offset & ~(PAGE_SIZE-1));
384 page_offset = offset & (PAGE_SIZE-1);
385 page_length = remain;
386 if ((page_offset + remain) > PAGE_SIZE)
387 page_length = PAGE_SIZE - page_offset;
388
4f27b75d
CW		 389 if (fast_shmem_read(obj_priv->pages,
390 page_base, page_offset,
391 user_data, page_length))
392 return -EFAULT;
eb01459f
EA		 393
394 remain -= page_length;
395 user_data += page_length;
396 offset += page_length;
397 }
398
4f27b75d 399 return 0;
eb01459f
EA		 400}
401
07f73f69
CW		 402static int
403i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
404{
405 int ret;
406
4bdadb97 407 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
07f73f69
CW		 408
409 /* If we've insufficient memory to map in the pages, attempt
410 * to make some space by throwing out some old buffers.
411 */
412 if (ret == -ENOMEM) {
413 struct drm_device *dev = obj->dev;
07f73f69 414
0108a3ed
DV		 415 ret = i915_gem_evict_something(dev, obj->size,
416 i915_gem_get_gtt_alignment(obj));
07f73f69
CW		 417 if (ret)
418 return ret;
419
4bdadb97 420 ret = i915_gem_object_get_pages(obj, 0);
07f73f69
CW		 421 }
422
423 return ret;
424}
425
eb01459f
EA		 426/**
427 * This is the fallback shmem pread path, which allocates temporary storage
428 * in kernel space to copy_to_user into outside of the struct_mutex, so we
429 * can copy out of the object's backing pages while holding the struct mutex
430 * and not take page faults.
431 */
432static int
433i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
434 struct drm_i915_gem_pread *args,
435 struct drm_file *file_priv)
436{
23010e43 437 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
eb01459f
EA		 438 struct mm_struct *mm = current->mm;
439 struct page **user_pages;
440 ssize_t remain;
441 loff_t offset, pinned_pages, i;
442 loff_t first_data_page, last_data_page, num_pages;
443 int shmem_page_index, shmem_page_offset;
444 int data_page_index, data_page_offset;
445 int page_length;
446 int ret;
447 uint64_t data_ptr = args->data_ptr;
280b713b 448 int do_bit17_swizzling;
eb01459f
EA		 449
450 remain = args->size;
451
452 /* Pin the user pages containing the data. We can't fault while
453 * holding the struct mutex, yet we want to hold it while
454 * dereferencing the user data.
455 */
456 first_data_page = data_ptr / PAGE_SIZE;
457 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
458 num_pages = last_data_page - first_data_page + 1;
459
4f27b75d 460 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
eb01459f
EA		 461 if (user_pages == NULL)
462 return -ENOMEM;
463
4f27b75d 464 mutex_unlock(&dev->struct_mutex);
eb01459f
EA		 465 down_read(&mm->mmap_sem);
466 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 467 num_pages, 1, 0, user_pages, NULL);
eb01459f 468 up_read(&mm->mmap_sem);
4f27b75d 469 mutex_lock(&dev->struct_mutex);
eb01459f
EA		 470 if (pinned_pages < num_pages) {
471 ret = -EFAULT;
4f27b75d 472 goto out;
eb01459f
EA		 473 }
474
4f27b75d
CW		 475 ret = i915_gem_object_set_cpu_read_domain_range(obj,
476 args->offset,
477 args->size);
07f73f69 478 if (ret)
4f27b75d 479 goto out;
eb01459f 480
4f27b75d 481 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
eb01459f 482
23010e43 483 obj_priv = to_intel_bo(obj);
eb01459f
EA		 484 offset = args->offset;
485
486 while (remain > 0) {
487 /* Operation in this page
488 *
489 * shmem_page_index = page number within shmem file
490 * shmem_page_offset = offset within page in shmem file
491 * data_page_index = page number in get_user_pages return
492 * data_page_offset = offset with data_page_index page.
493 * page_length = bytes to copy for this page
494 */
495 shmem_page_index = offset / PAGE_SIZE;
496 shmem_page_offset = offset & ~PAGE_MASK;
497 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
498 data_page_offset = data_ptr & ~PAGE_MASK;
499
500 page_length = remain;
501 if ((shmem_page_offset + page_length) > PAGE_SIZE)
502 page_length = PAGE_SIZE - shmem_page_offset;
503 if ((data_page_offset + page_length) > PAGE_SIZE)
504 page_length = PAGE_SIZE - data_page_offset;
505
280b713b 506 if (do_bit17_swizzling) {
99a03df5 507 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
280b713b 508 shmem_page_offset,
99a03df5
CW		 509 user_pages[data_page_index],
510 data_page_offset,
511 page_length,
512 1);
513 } else {
514 slow_shmem_copy(user_pages[data_page_index],
515 data_page_offset,
516 obj_priv->pages[shmem_page_index],
517 shmem_page_offset,
518 page_length);
280b713b 519 }
eb01459f
EA		 520
521 remain -= page_length;
522 data_ptr += page_length;
523 offset += page_length;
524 }
525
4f27b75d 526out:
eb01459f
EA		 527 for (i = 0; i < pinned_pages; i++) {
528 SetPageDirty(user_pages[i]);
529 page_cache_release(user_pages[i]);
530 }
8e7d2b2c 531 drm_free_large(user_pages);
eb01459f
EA		 532
533 return ret;
534}
535
673a394b
EA		 536/**
537 * Reads data from the object referenced by handle.
538 *
539 * On error, the contents of *data are undefined.
540 */
541int
542i915_gem_pread_ioctl(struct drm_device *dev, void *data,
543 struct drm_file *file_priv)
544{
545 struct drm_i915_gem_pread *args = data;
546 struct drm_gem_object *obj;
547 struct drm_i915_gem_object *obj_priv;
35b62a89 548 int ret = 0;
673a394b 549
4f27b75d 550 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 551 if (ret)
4f27b75d 552 return ret;
1d7cfea1
CW		 553
554 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
555 if (obj == NULL) {
556 ret = -ENOENT;
557 goto unlock;
4f27b75d 558 }
1d7cfea1 559 obj_priv = to_intel_bo(obj);
4f27b75d 560
7dcd2499
CW		 561 /* Bounds check source. */
562 if (args->offset > obj->size || args->size > obj->size - args->offset) {
ce9d419d 563 ret = -EINVAL;
35b62a89 564 goto out;
ce9d419d
CW		 565 }
566
35b62a89
CW		 567 if (args->size == 0)
568 goto out;
569
ce9d419d
CW		 570 if (!access_ok(VERIFY_WRITE,
571 (char __user *)(uintptr_t)args->data_ptr,
572 args->size)) {
573 ret = -EFAULT;
35b62a89 574 goto out;
673a394b
EA		 575 }
576
b5e4feb6
CW		 577 ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
578 args->size);
579 if (ret) {
580 ret = -EFAULT;
581 goto out;
582 }
583
4f27b75d
CW		 584 ret = i915_gem_object_get_pages_or_evict(obj);
585 if (ret)
586 goto out;
587
588 ret = i915_gem_object_set_cpu_read_domain_range(obj,
589 args->offset,
590 args->size);
591 if (ret)
592 goto out_put;
593
594 ret = -EFAULT;
595 if (!i915_gem_object_needs_bit17_swizzle(obj))
280b713b 596 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
4f27b75d
CW		 597 if (ret == -EFAULT)
598 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
673a394b 599
4f27b75d
CW		 600out_put:
601 i915_gem_object_put_pages(obj);
35b62a89 602out:
4f27b75d 603 drm_gem_object_unreference(obj);
1d7cfea1 604unlock:
4f27b75d 605 mutex_unlock(&dev->struct_mutex);
eb01459f 606 return ret;
673a394b
EA		 607}
608
0839ccb8
KP		 609/* This is the fast write path which cannot handle
610 * page faults in the source data
9b7530cc 611 */
0839ccb8
KP		 612
613static inline int
614fast_user_write(struct io_mapping *mapping,
615 loff_t page_base, int page_offset,
616 char __user *user_data,
617 int length)
9b7530cc 618{
9b7530cc 619 char *vaddr_atomic;
0839ccb8 620 unsigned long unwritten;
9b7530cc 621
fca3ec01 622 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
0839ccb8
KP		 623 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
624 user_data, length);
fca3ec01 625 io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
fbd5a26d 626 return unwritten;
0839ccb8
KP		 627}
628
629/* Here's the write path which can sleep for
630 * page faults
631 */
632
ab34c226 633static inline void
3de09aa3
EA		 634slow_kernel_write(struct io_mapping *mapping,
635 loff_t gtt_base, int gtt_offset,
636 struct page *user_page, int user_offset,
637 int length)
0839ccb8 638{
ab34c226
CW		 639 char __iomem *dst_vaddr;
640 char *src_vaddr;
0839ccb8 641
ab34c226
CW		 642 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
643 src_vaddr = kmap(user_page);
644
645 memcpy_toio(dst_vaddr + gtt_offset,
646 src_vaddr + user_offset,
647 length);
648
649 kunmap(user_page);
650 io_mapping_unmap(dst_vaddr);
9b7530cc
LT		 651}
652
40123c1f
EA		 653static inline int
654fast_shmem_write(struct page **pages,
655 loff_t page_base, int page_offset,
656 char __user *data,
657 int length)
658{
b5e4feb6 659 char *vaddr;
fbd5a26d 660 int ret;
40123c1f
EA		 661
662 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
fbd5a26d 663 ret = __copy_from_user_inatomic(vaddr + page_offset, data, length);
40123c1f
EA		 664 kunmap_atomic(vaddr, KM_USER0);
665
fbd5a26d 666 return ret;
40123c1f
EA		 667}
668
3de09aa3
EA		 669/**
670 * This is the fast pwrite path, where we copy the data directly from the
671 * user into the GTT, uncached.
672 */
673a394b 673static int
3de09aa3
EA		 674i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
675 struct drm_i915_gem_pwrite *args,
676 struct drm_file *file_priv)
673a394b 677{
23010e43 678 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
0839ccb8 679 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 680 ssize_t remain;
0839ccb8 681 loff_t offset, page_base;
673a394b 682 char __user *user_data;
0839ccb8 683 int page_offset, page_length;
673a394b
EA		 684
685 user_data = (char __user *) (uintptr_t) args->data_ptr;
686 remain = args->size;
673a394b 687
23010e43 688 obj_priv = to_intel_bo(obj);
673a394b 689 offset = obj_priv->gtt_offset + args->offset;
673a394b
EA		 690
691 while (remain > 0) {
692 /* Operation in this page
693 *
0839ccb8
KP		 694 * page_base = page offset within aperture
695 * page_offset = offset within page
696 * page_length = bytes to copy for this page
673a394b 697 */
0839ccb8
KP		 698 page_base = (offset & ~(PAGE_SIZE-1));
699 page_offset = offset & (PAGE_SIZE-1);
700 page_length = remain;
701 if ((page_offset + remain) > PAGE_SIZE)
702 page_length = PAGE_SIZE - page_offset;
703
0839ccb8 704 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 705 * source page isn't available. Return the error and we'll
706 * retry in the slow path.
0839ccb8 707 */
fbd5a26d
CW		 708 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
709 page_offset, user_data, page_length))
710
711 return -EFAULT;
673a394b 712
0839ccb8
KP		 713 remain -= page_length;
714 user_data += page_length;
715 offset += page_length;
673a394b 716 }
673a394b 717
fbd5a26d 718 return 0;
673a394b
EA		 719}
720
3de09aa3
EA		 721/**
722 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
723 * the memory and maps it using kmap_atomic for copying.
724 *
725 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
726 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
727 */
3043c60c 728static int
3de09aa3
EA		 729i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
730 struct drm_i915_gem_pwrite *args,
731 struct drm_file *file_priv)
673a394b 732{
23010e43 733 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3de09aa3
EA		 734 drm_i915_private_t *dev_priv = dev->dev_private;
735 ssize_t remain;
736 loff_t gtt_page_base, offset;
737 loff_t first_data_page, last_data_page, num_pages;
738 loff_t pinned_pages, i;
739 struct page **user_pages;
740 struct mm_struct *mm = current->mm;
741 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 742 int ret;
3de09aa3
EA		 743 uint64_t data_ptr = args->data_ptr;
744
745 remain = args->size;
746
747 /* Pin the user pages containing the data. We can't fault while
748 * holding the struct mutex, and all of the pwrite implementations
749 * want to hold it while dereferencing the user data.
750 */
751 first_data_page = data_ptr / PAGE_SIZE;
752 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
753 num_pages = last_data_page - first_data_page + 1;
754
fbd5a26d 755 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
3de09aa3
EA		 756 if (user_pages == NULL)
757 return -ENOMEM;
758
fbd5a26d 759 mutex_unlock(&dev->struct_mutex);
3de09aa3
EA		 760 down_read(&mm->mmap_sem);
761 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
762 num_pages, 0, 0, user_pages, NULL);
763 up_read(&mm->mmap_sem);
fbd5a26d 764 mutex_lock(&dev->struct_mutex);
3de09aa3
EA		 765 if (pinned_pages < num_pages) {
766 ret = -EFAULT;
767 goto out_unpin_pages;
768 }
673a394b 769
3de09aa3
EA		 770 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
771 if (ret)
fbd5a26d 772 goto out_unpin_pages;
3de09aa3 773
23010e43 774 obj_priv = to_intel_bo(obj);
3de09aa3
EA		 775 offset = obj_priv->gtt_offset + args->offset;
776
777 while (remain > 0) {
778 /* Operation in this page
779 *
780 * gtt_page_base = page offset within aperture
781 * gtt_page_offset = offset within page in aperture
782 * data_page_index = page number in get_user_pages return
783 * data_page_offset = offset with data_page_index page.
784 * page_length = bytes to copy for this page
785 */
786 gtt_page_base = offset & PAGE_MASK;
787 gtt_page_offset = offset & ~PAGE_MASK;
788 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
789 data_page_offset = data_ptr & ~PAGE_MASK;
790
791 page_length = remain;
792 if ((gtt_page_offset + page_length) > PAGE_SIZE)
793 page_length = PAGE_SIZE - gtt_page_offset;
794 if ((data_page_offset + page_length) > PAGE_SIZE)
795 page_length = PAGE_SIZE - data_page_offset;
796
ab34c226
CW		 797 slow_kernel_write(dev_priv->mm.gtt_mapping,
798 gtt_page_base, gtt_page_offset,
799 user_pages[data_page_index],
800 data_page_offset,
801 page_length);
3de09aa3
EA		 802
803 remain -= page_length;
804 offset += page_length;
805 data_ptr += page_length;
806 }
807
3de09aa3
EA		 808out_unpin_pages:
809 for (i = 0; i < pinned_pages; i++)
810 page_cache_release(user_pages[i]);
8e7d2b2c 811 drm_free_large(user_pages);
3de09aa3
EA		 812
813 return ret;
814}
815
40123c1f
EA		 816/**
817 * This is the fast shmem pwrite path, which attempts to directly
818 * copy_from_user into the kmapped pages backing the object.
819 */
3043c60c 820static int
40123c1f
EA		 821i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
822 struct drm_i915_gem_pwrite *args,
823 struct drm_file *file_priv)
673a394b 824{
23010e43 825 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
40123c1f
EA		 826 ssize_t remain;
827 loff_t offset, page_base;
828 char __user *user_data;
829 int page_offset, page_length;
40123c1f
EA		 830
831 user_data = (char __user *) (uintptr_t) args->data_ptr;
832 remain = args->size;
673a394b 833
23010e43 834 obj_priv = to_intel_bo(obj);
40123c1f
EA		 835 offset = args->offset;
836 obj_priv->dirty = 1;
837
838 while (remain > 0) {
839 /* Operation in this page
840 *
841 * page_base = page offset within aperture
842 * page_offset = offset within page
843 * page_length = bytes to copy for this page
844 */
845 page_base = (offset & ~(PAGE_SIZE-1));
846 page_offset = offset & (PAGE_SIZE-1);
847 page_length = remain;
848 if ((page_offset + remain) > PAGE_SIZE)
849 page_length = PAGE_SIZE - page_offset;
850
fbd5a26d 851 if (fast_shmem_write(obj_priv->pages,
40123c1f 852 page_base, page_offset,
fbd5a26d
CW		 853 user_data, page_length))
854 return -EFAULT;
40123c1f
EA		 855
856 remain -= page_length;
857 user_data += page_length;
858 offset += page_length;
859 }
860
fbd5a26d 861 return 0;
40123c1f
EA		 862}
863
864/**
865 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
866 * the memory and maps it using kmap_atomic for copying.
867 *
868 * This avoids taking mmap_sem for faulting on the user's address while the
869 * struct_mutex is held.
870 */
871static int
872i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
873 struct drm_i915_gem_pwrite *args,
874 struct drm_file *file_priv)
875{
23010e43 876 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
40123c1f
EA		 877 struct mm_struct *mm = current->mm;
878 struct page **user_pages;
879 ssize_t remain;
880 loff_t offset, pinned_pages, i;
881 loff_t first_data_page, last_data_page, num_pages;
882 int shmem_page_index, shmem_page_offset;
883 int data_page_index, data_page_offset;
884 int page_length;
885 int ret;
886 uint64_t data_ptr = args->data_ptr;
280b713b 887 int do_bit17_swizzling;
40123c1f
EA		 888
889 remain = args->size;
890
891 /* Pin the user pages containing the data. We can't fault while
892 * holding the struct mutex, and all of the pwrite implementations
893 * want to hold it while dereferencing the user data.
894 */
895 first_data_page = data_ptr / PAGE_SIZE;
896 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
897 num_pages = last_data_page - first_data_page + 1;
898
4f27b75d 899 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
40123c1f
EA		 900 if (user_pages == NULL)
901 return -ENOMEM;
902
fbd5a26d 903 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 904 down_read(&mm->mmap_sem);
905 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
906 num_pages, 0, 0, user_pages, NULL);
907 up_read(&mm->mmap_sem);
fbd5a26d 908 mutex_lock(&dev->struct_mutex);
40123c1f
EA		 909 if (pinned_pages < num_pages) {
910 ret = -EFAULT;
fbd5a26d 911 goto out;
673a394b
EA		 912 }
913
fbd5a26d 914 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
07f73f69 915 if (ret)
fbd5a26d 916 goto out;
40123c1f 917
fbd5a26d 918 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
40123c1f 919
23010e43 920 obj_priv = to_intel_bo(obj);
673a394b 921 offset = args->offset;
40123c1f 922 obj_priv->dirty = 1;
673a394b 923
40123c1f
EA		 924 while (remain > 0) {
925 /* Operation in this page
926 *
927 * shmem_page_index = page number within shmem file
928 * shmem_page_offset = offset within page in shmem file
929 * data_page_index = page number in get_user_pages return
930 * data_page_offset = offset with data_page_index page.
931 * page_length = bytes to copy for this page
932 */
933 shmem_page_index = offset / PAGE_SIZE;
934 shmem_page_offset = offset & ~PAGE_MASK;
935 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
936 data_page_offset = data_ptr & ~PAGE_MASK;
937
938 page_length = remain;
939 if ((shmem_page_offset + page_length) > PAGE_SIZE)
940 page_length = PAGE_SIZE - shmem_page_offset;
941 if ((data_page_offset + page_length) > PAGE_SIZE)
942 page_length = PAGE_SIZE - data_page_offset;
943
280b713b 944 if (do_bit17_swizzling) {
99a03df5 945 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
280b713b
EA		 946 shmem_page_offset,
947 user_pages[data_page_index],
948 data_page_offset,
99a03df5
CW		 949 page_length,
950 0);
951 } else {
952 slow_shmem_copy(obj_priv->pages[shmem_page_index],
953 shmem_page_offset,
954 user_pages[data_page_index],
955 data_page_offset,
956 page_length);
280b713b 957 }
40123c1f
EA		 958
959 remain -= page_length;
960 data_ptr += page_length;
961 offset += page_length;
673a394b
EA		 962 }
963
fbd5a26d 964out:
40123c1f
EA		 965 for (i = 0; i < pinned_pages; i++)
966 page_cache_release(user_pages[i]);
8e7d2b2c 967 drm_free_large(user_pages);
673a394b 968
40123c1f 969 return ret;
673a394b
EA		 970}
971
972/**
973 * Writes data to the object referenced by handle.
974 *
975 * On error, the contents of the buffer that were to be modified are undefined.
976 */
977int
978i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
fbd5a26d 979 struct drm_file *file)
673a394b
EA		 980{
981 struct drm_i915_gem_pwrite *args = data;
982 struct drm_gem_object *obj;
983 struct drm_i915_gem_object *obj_priv;
984 int ret = 0;
985
fbd5a26d 986 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 987 if (ret)
fbd5a26d 988 return ret;
1d7cfea1
CW		 989
990 obj = drm_gem_object_lookup(dev, file, args->handle);
991 if (obj == NULL) {
992 ret = -ENOENT;
993 goto unlock;
fbd5a26d 994 }
1d7cfea1
CW		 995 obj_priv = to_intel_bo(obj);
996
fbd5a26d 997
7dcd2499
CW		 998 /* Bounds check destination. */
999 if (args->offset > obj->size || args->size > obj->size - args->offset) {
ce9d419d 1000 ret = -EINVAL;
35b62a89 1001 goto out;
ce9d419d
CW		 1002 }
1003
35b62a89
CW		 1004 if (args->size == 0)
1005 goto out;
1006
ce9d419d
CW		 1007 if (!access_ok(VERIFY_READ,
1008 (char __user *)(uintptr_t)args->data_ptr,
1009 args->size)) {
1010 ret = -EFAULT;
35b62a89 1011 goto out;
673a394b
EA		 1012 }
1013
b5e4feb6
CW		 1014 ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
1015 args->size);
1016 if (ret) {
1017 ret = -EFAULT;
1018 goto out;
1019 }
1020
673a394b
EA		 1021 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1022 * it would end up going through the fenced access, and we'll get
1023 * different detiling behavior between reading and writing.
1024 * pread/pwrite currently are reading and writing from the CPU
1025 * perspective, requiring manual detiling by the client.
1026 */
71acb5eb 1027 if (obj_priv->phys_obj)
fbd5a26d 1028 ret = i915_gem_phys_pwrite(dev, obj, args, file);
71acb5eb 1029 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
5cdf5881 1030 obj_priv->gtt_space &&
9b8c4a0b 1031 obj->write_domain != I915_GEM_DOMAIN_CPU) {
fbd5a26d
CW		 1032 ret = i915_gem_object_pin(obj, 0);
1033 if (ret)
1034 goto out;
1035
1036 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1037 if (ret)
1038 goto out_unpin;
1039
1040 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1041 if (ret == -EFAULT)
1042 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1043
1044out_unpin:
1045 i915_gem_object_unpin(obj);
40123c1f 1046 } else {
fbd5a26d
CW		 1047 ret = i915_gem_object_get_pages_or_evict(obj);
1048 if (ret)
1049 goto out;
673a394b 1050
fbd5a26d
CW		 1051 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1052 if (ret)
1053 goto out_put;
1054
1055 ret = -EFAULT;
1056 if (!i915_gem_object_needs_bit17_swizzle(obj))
1057 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1058 if (ret == -EFAULT)
1059 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1060
1061out_put:
1062 i915_gem_object_put_pages(obj);
1063 }
673a394b 1064
35b62a89 1065out:
fbd5a26d 1066 drm_gem_object_unreference(obj);
1d7cfea1 1067unlock:
fbd5a26d 1068 mutex_unlock(&dev->struct_mutex);
673a394b
EA		 1069 return ret;
1070}
1071
1072/**
2ef7eeaa
EA		 1073 * Called when user space prepares to use an object with the CPU, either
1074 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 1075 */
1076int
1077i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1078 struct drm_file *file_priv)
1079{
a09ba7fa 1080 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 1081 struct drm_i915_gem_set_domain *args = data;
1082 struct drm_gem_object *obj;
652c393a 1083 struct drm_i915_gem_object *obj_priv;
2ef7eeaa
EA		 1084 uint32_t read_domains = args->read_domains;
1085 uint32_t write_domain = args->write_domain;
673a394b
EA		 1086 int ret;
1087
1088 if (!(dev->driver->driver_features & DRIVER_GEM))
1089 return -ENODEV;
1090
2ef7eeaa 1091 /* Only handle setting domains to types used by the CPU. */
21d509e3 1092 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1093 return -EINVAL;
1094
21d509e3 1095 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1096 return -EINVAL;
1097
1098 /* Having something in the write domain implies it's in the read
1099 * domain, and only that read domain. Enforce that in the request.
1100 */
1101 if (write_domain != 0 && read_domains != write_domain)
1102 return -EINVAL;
1103
76c1dec1 1104 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1105 if (ret)
76c1dec1 1106 return ret;
1d7cfea1
CW		 1107
1108 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1109 if (obj == NULL) {
1110 ret = -ENOENT;
1111 goto unlock;
76c1dec1 1112 }
1d7cfea1 1113 obj_priv = to_intel_bo(obj);
652c393a
JB		 1114
1115 intel_mark_busy(dev, obj);
1116
2ef7eeaa
EA		 1117 if (read_domains & I915_GEM_DOMAIN_GTT) {
1118 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392 1119
a09ba7fa
EA		 1120 /* Update the LRU on the fence for the CPU access that's
1121 * about to occur.
1122 */
1123 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
007cc8ac
DV		 1124 struct drm_i915_fence_reg *reg =
1125 &dev_priv->fence_regs[obj_priv->fence_reg];
1126 list_move_tail(&reg->lru_list,
a09ba7fa
EA		 1127 &dev_priv->mm.fence_list);
1128 }
1129
02354392
EA		 1130 /* Silently promote "you're not bound, there was nothing to do"
1131 * to success, since the client was just asking us to
1132 * make sure everything was done.
1133 */
1134 if (ret == -EINVAL)
1135 ret = 0;
2ef7eeaa 1136 } else {
e47c68e9 1137 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1138 }
1139
7d1c4804
CW		 1140 /* Maintain LRU order of "inactive" objects */
1141 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
69dc4987 1142 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
7d1c4804 1143
673a394b 1144 drm_gem_object_unreference(obj);
1d7cfea1 1145unlock:
673a394b
EA		 1146 mutex_unlock(&dev->struct_mutex);
1147 return ret;
1148}
1149
1150/**
1151 * Called when user space has done writes to this buffer
1152 */
1153int
1154i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1155 struct drm_file *file_priv)
1156{
1157 struct drm_i915_gem_sw_finish *args = data;
1158 struct drm_gem_object *obj;
673a394b
EA		 1159 int ret = 0;
1160
1161 if (!(dev->driver->driver_features & DRIVER_GEM))
1162 return -ENODEV;
1163
76c1dec1 1164 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1165 if (ret)
76c1dec1 1166 return ret;
1d7cfea1
CW		 1167
1168 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1169 if (obj == NULL) {
1170 ret = -ENOENT;
1171 goto unlock;
673a394b
EA		 1172 }
1173
673a394b 1174 /* Pinned buffers may be scanout, so flush the cache */
3d2a812a 1175 if (to_intel_bo(obj)->pin_count)
e47c68e9
EA		 1176 i915_gem_object_flush_cpu_write_domain(obj);
1177
673a394b 1178 drm_gem_object_unreference(obj);
1d7cfea1 1179unlock:
673a394b
EA		 1180 mutex_unlock(&dev->struct_mutex);
1181 return ret;
1182}
1183
1184/**
1185 * Maps the contents of an object, returning the address it is mapped
1186 * into.
1187 *
1188 * While the mapping holds a reference on the contents of the object, it doesn't
1189 * imply a ref on the object itself.
1190 */
1191int
1192i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1193 struct drm_file *file_priv)
1194{
1195 struct drm_i915_gem_mmap *args = data;
1196 struct drm_gem_object *obj;
1197 loff_t offset;
1198 unsigned long addr;
1199
1200 if (!(dev->driver->driver_features & DRIVER_GEM))
1201 return -ENODEV;
1202
1203 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1204 if (obj == NULL)
bf79cb91 1205 return -ENOENT;
673a394b
EA		 1206
1207 offset = args->offset;
1208
1209 down_write(&current->mm->mmap_sem);
1210 addr = do_mmap(obj->filp, 0, args->size,
1211 PROT_READ | PROT_WRITE, MAP_SHARED,
1212 args->offset);
1213 up_write(&current->mm->mmap_sem);
bc9025bd 1214 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1215 if (IS_ERR((void *)addr))
1216 return addr;
1217
1218 args->addr_ptr = (uint64_t) addr;
1219
1220 return 0;
1221}
1222
de151cf6
JB		 1223/**
1224 * i915_gem_fault - fault a page into the GTT
1225 * vma: VMA in question
1226 * vmf: fault info
1227 *
1228 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1229 * from userspace. The fault handler takes care of binding the object to
1230 * the GTT (if needed), allocating and programming a fence register (again,
1231 * only if needed based on whether the old reg is still valid or the object
1232 * is tiled) and inserting a new PTE into the faulting process.
1233 *
1234 * Note that the faulting process may involve evicting existing objects
1235 * from the GTT and/or fence registers to make room. So performance may
1236 * suffer if the GTT working set is large or there are few fence registers
1237 * left.
1238 */
1239int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1240{
1241 struct drm_gem_object *obj = vma->vm_private_data;
1242 struct drm_device *dev = obj->dev;
7d1c4804 1243 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1244 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 1245 pgoff_t page_offset;
1246 unsigned long pfn;
1247 int ret = 0;
0f973f27 1248 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1249
1250 /* We don't use vmf->pgoff since that has the fake offset */
1251 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1252 PAGE_SHIFT;
1253
1254 /* Now bind it into the GTT if needed */
1255 mutex_lock(&dev->struct_mutex);
1256 if (!obj_priv->gtt_space) {
e67b8ce1 1257 ret = i915_gem_object_bind_to_gtt(obj, 0);
c715089f
CW		 1258 if (ret)
1259 goto unlock;
07f4f3e8 1260
07f4f3e8 1261 ret = i915_gem_object_set_to_gtt_domain(obj, write);
c715089f
CW		 1262 if (ret)
1263 goto unlock;
de151cf6
JB		 1264 }
1265
1266 /* Need a new fence register? */
a09ba7fa 1267 if (obj_priv->tiling_mode != I915_TILING_NONE) {
2cf34d7b 1268 ret = i915_gem_object_get_fence_reg(obj, true);
c715089f
CW		 1269 if (ret)
1270 goto unlock;
d9ddcb96 1271 }
de151cf6 1272
7d1c4804 1273 if (i915_gem_object_is_inactive(obj_priv))
69dc4987 1274 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
7d1c4804 1275
de151cf6
JB		 1276 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1277 page_offset;
1278
1279 /* Finally, remap it using the new GTT offset */
1280 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1281unlock:
de151cf6
JB		 1282 mutex_unlock(&dev->struct_mutex);
1283
1284 switch (ret) {
c715089f
CW		 1285 case 0:
1286 case -ERESTARTSYS:
1287 return VM_FAULT_NOPAGE;
de151cf6
JB		 1288 case -ENOMEM:
1289 case -EAGAIN:
1290 return VM_FAULT_OOM;
de151cf6 1291 default:
c715089f 1292 return VM_FAULT_SIGBUS;
de151cf6
JB		 1293 }
1294}
1295
1296/**
1297 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1298 * @obj: obj in question
1299 *
1300 * GEM memory mapping works by handing back to userspace a fake mmap offset
1301 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1302 * up the object based on the offset and sets up the various memory mapping
1303 * structures.
1304 *
1305 * This routine allocates and attaches a fake offset for @obj.
1306 */
1307static int
1308i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1309{
1310 struct drm_device *dev = obj->dev;
1311 struct drm_gem_mm *mm = dev->mm_private;
23010e43 1312 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 1313 struct drm_map_list *list;
f77d390c 1314 struct drm_local_map *map;
de151cf6
JB		 1315 int ret = 0;
1316
1317 /* Set the object up for mmap'ing */
1318 list = &obj->map_list;
9a298b2a 1319 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
de151cf6
JB		 1320 if (!list->map)
1321 return -ENOMEM;
1322
1323 map = list->map;
1324 map->type = _DRM_GEM;
1325 map->size = obj->size;
1326 map->handle = obj;
1327
1328 /* Get a DRM GEM mmap offset allocated... */
1329 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1330 obj->size / PAGE_SIZE, 0, 0);
1331 if (!list->file_offset_node) {
1332 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
9e0ae534 1333 ret = -ENOSPC;
de151cf6
JB		 1334 goto out_free_list;
1335 }
1336
1337 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1338 obj->size / PAGE_SIZE, 0);
1339 if (!list->file_offset_node) {
1340 ret = -ENOMEM;
1341 goto out_free_list;
1342 }
1343
1344 list->hash.key = list->file_offset_node->start;
9e0ae534
CW		 1345 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1346 if (ret) {
de151cf6
JB		 1347 DRM_ERROR("failed to add to map hash\n");
1348 goto out_free_mm;
1349 }
1350
1351 /* By now we should be all set, any drm_mmap request on the offset
1352 * below will get to our mmap & fault handler */
1353 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1354
1355 return 0;
1356
1357out_free_mm:
1358 drm_mm_put_block(list->file_offset_node);
1359out_free_list:
9a298b2a 1360 kfree(list->map);
de151cf6
JB		 1361
1362 return ret;
1363}
1364
901782b2
CW		 1365/**
1366 * i915_gem_release_mmap - remove physical page mappings
1367 * @obj: obj in question
1368 *
af901ca1 1369 * Preserve the reservation of the mmapping with the DRM core code, but
901782b2
CW		 1370 * relinquish ownership of the pages back to the system.
1371 *
1372 * It is vital that we remove the page mapping if we have mapped a tiled
1373 * object through the GTT and then lose the fence register due to
1374 * resource pressure. Similarly if the object has been moved out of the
1375 * aperture, than pages mapped into userspace must be revoked. Removing the
1376 * mapping will then trigger a page fault on the next user access, allowing
1377 * fixup by i915_gem_fault().
1378 */
d05ca301 1379void
901782b2
CW		 1380i915_gem_release_mmap(struct drm_gem_object *obj)
1381{
1382 struct drm_device *dev = obj->dev;
23010e43 1383 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
901782b2
CW		 1384
1385 if (dev->dev_mapping)
1386 unmap_mapping_range(dev->dev_mapping,
1387 obj_priv->mmap_offset, obj->size, 1);
1388}
1389
ab00b3e5
JB		 1390static void
1391i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1392{
1393 struct drm_device *dev = obj->dev;
23010e43 1394 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ab00b3e5
JB		 1395 struct drm_gem_mm *mm = dev->mm_private;
1396 struct drm_map_list *list;
1397
1398 list = &obj->map_list;
1399 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1400
1401 if (list->file_offset_node) {
1402 drm_mm_put_block(list->file_offset_node);
1403 list->file_offset_node = NULL;
1404 }
1405
1406 if (list->map) {
9a298b2a 1407 kfree(list->map);
ab00b3e5
JB		 1408 list->map = NULL;
1409 }
1410
1411 obj_priv->mmap_offset = 0;
1412}
1413
de151cf6
JB		 1414/**
1415 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1416 * @obj: object to check
1417 *
1418 * Return the required GTT alignment for an object, taking into account
1419 * potential fence register mapping if needed.
1420 */
1421static uint32_t
1422i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1423{
1424 struct drm_device *dev = obj->dev;
23010e43 1425 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 1426 int start, i;
1427
1428 /*
1429 * Minimum alignment is 4k (GTT page size), but might be greater
1430 * if a fence register is needed for the object.
1431 */
a6c45cf0 1432 if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
de151cf6
JB		 1433 return 4096;
1434
1435 /*
1436 * Previous chips need to be aligned to the size of the smallest
1437 * fence register that can contain the object.
1438 */
a6c45cf0 1439 if (INTEL_INFO(dev)->gen == 3)
de151cf6
JB		 1440 start = 1024*1024;
1441 else
1442 start = 512*1024;
1443
1444 for (i = start; i < obj->size; i <<= 1)
1445 ;
1446
1447 return i;
1448}
1449
1450/**
1451 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1452 * @dev: DRM device
1453 * @data: GTT mapping ioctl data
1454 * @file_priv: GEM object info
1455 *
1456 * Simply returns the fake offset to userspace so it can mmap it.
1457 * The mmap call will end up in drm_gem_mmap(), which will set things
1458 * up so we can get faults in the handler above.
1459 *
1460 * The fault handler will take care of binding the object into the GTT
1461 * (since it may have been evicted to make room for something), allocating
1462 * a fence register, and mapping the appropriate aperture address into
1463 * userspace.
1464 */
1465int
1466i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1467 struct drm_file *file_priv)
1468{
1469 struct drm_i915_gem_mmap_gtt *args = data;
de151cf6
JB		 1470 struct drm_gem_object *obj;
1471 struct drm_i915_gem_object *obj_priv;
1472 int ret;
1473
1474 if (!(dev->driver->driver_features & DRIVER_GEM))
1475 return -ENODEV;
1476
76c1dec1 1477 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1478 if (ret)
76c1dec1 1479 return ret;
de151cf6 1480
1d7cfea1
CW		 1481 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1482 if (obj == NULL) {
1483 ret = -ENOENT;
1484 goto unlock;
1485 }
23010e43 1486 obj_priv = to_intel_bo(obj);
de151cf6 1487
ab18282d
CW		 1488 if (obj_priv->madv != I915_MADV_WILLNEED) {
1489 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1d7cfea1
CW		 1490 ret = -EINVAL;
1491 goto out;
ab18282d
CW		 1492 }
1493
de151cf6
JB		 1494 if (!obj_priv->mmap_offset) {
1495 ret = i915_gem_create_mmap_offset(obj);
1d7cfea1
CW		 1496 if (ret)
1497 goto out;
de151cf6
JB		 1498 }
1499
1500 args->offset = obj_priv->mmap_offset;
1501
de151cf6
JB		 1502 /*
1503 * Pull it into the GTT so that we have a page list (makes the
1504 * initial fault faster and any subsequent flushing possible).
1505 */
1506 if (!obj_priv->agp_mem) {
e67b8ce1 1507 ret = i915_gem_object_bind_to_gtt(obj, 0);
1d7cfea1
CW		 1508 if (ret)
1509 goto out;
de151cf6
JB		 1510 }
1511
1d7cfea1 1512out:
de151cf6 1513 drm_gem_object_unreference(obj);
1d7cfea1 1514unlock:
de151cf6 1515 mutex_unlock(&dev->struct_mutex);
1d7cfea1 1516 return ret;
de151cf6
JB		 1517}
1518
5cdf5881 1519static void
856fa198 1520i915_gem_object_put_pages(struct drm_gem_object *obj)
673a394b 1521{
23010e43 1522 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 1523 int page_count = obj->size / PAGE_SIZE;
1524 int i;
1525
856fa198 1526 BUG_ON(obj_priv->pages_refcount == 0);
bb6baf76 1527 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
673a394b 1528
856fa198
EA		 1529 if (--obj_priv->pages_refcount != 0)
1530 return;
673a394b 1531
280b713b
EA		 1532 if (obj_priv->tiling_mode != I915_TILING_NONE)
1533 i915_gem_object_save_bit_17_swizzle(obj);
1534
3ef94daa 1535 if (obj_priv->madv == I915_MADV_DONTNEED)
13a05fd9 1536 obj_priv->dirty = 0;
3ef94daa
CW		 1537
1538 for (i = 0; i < page_count; i++) {
3ef94daa
CW		 1539 if (obj_priv->dirty)
1540 set_page_dirty(obj_priv->pages[i]);
1541
1542 if (obj_priv->madv == I915_MADV_WILLNEED)
856fa198 1543 mark_page_accessed(obj_priv->pages[i]);
3ef94daa
CW		 1544
1545 page_cache_release(obj_priv->pages[i]);
1546 }
673a394b
EA		 1547 obj_priv->dirty = 0;
1548
8e7d2b2c 1549 drm_free_large(obj_priv->pages);
856fa198 1550 obj_priv->pages = NULL;
673a394b
EA		 1551}
1552
a56ba56c
CW		 1553static uint32_t
1554i915_gem_next_request_seqno(struct drm_device *dev,
1555 struct intel_ring_buffer *ring)
1556{
1557 drm_i915_private_t *dev_priv = dev->dev_private;
1558
1559 ring->outstanding_lazy_request = true;
1560 return dev_priv->next_seqno;
1561}
1562
673a394b 1563static void
617dbe27 1564i915_gem_object_move_to_active(struct drm_gem_object *obj,
852835f3 1565 struct intel_ring_buffer *ring)
673a394b 1566{
a56ba56c 1567 struct drm_device *dev = obj->dev;
69dc4987 1568 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 1569 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
a56ba56c 1570 uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
617dbe27 1571
852835f3
ZN		 1572 BUG_ON(ring == NULL);
1573 obj_priv->ring = ring;
673a394b
EA		 1574
1575 /* Add a reference if we're newly entering the active list. */
1576 if (!obj_priv->active) {
1577 drm_gem_object_reference(obj);
1578 obj_priv->active = 1;
1579 }
e35a41de 1580
673a394b 1581 /* Move from whatever list we were on to the tail of execution. */
69dc4987
CW		 1582 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list);
1583 list_move_tail(&obj_priv->ring_list, &ring->active_list);
a56ba56c 1584 obj_priv->last_rendering_seqno = seqno;
673a394b
EA		 1585}
1586
ce44b0ea
EA		 1587static void
1588i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1589{
1590 struct drm_device *dev = obj->dev;
1591 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1592 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ce44b0ea
EA		 1593
1594 BUG_ON(!obj_priv->active);
69dc4987
CW		 1595 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list);
1596 list_del_init(&obj_priv->ring_list);
ce44b0ea
EA		 1597 obj_priv->last_rendering_seqno = 0;
1598}
673a394b 1599
963b4836
CW		 1600/* Immediately discard the backing storage */
1601static void
1602i915_gem_object_truncate(struct drm_gem_object *obj)
1603{
23010e43 1604 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
bb6baf76 1605 struct inode *inode;
963b4836 1606
ae9fed6b
CW		 1607 /* Our goal here is to return as much of the memory as
1608 * is possible back to the system as we are called from OOM.
1609 * To do this we must instruct the shmfs to drop all of its
1610 * backing pages, *now*. Here we mirror the actions taken
1611 * when by shmem_delete_inode() to release the backing store.
1612 */
bb6baf76 1613 inode = obj->filp->f_path.dentry->d_inode;
ae9fed6b
CW		 1614 truncate_inode_pages(inode->i_mapping, 0);
1615 if (inode->i_op->truncate_range)
1616 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
bb6baf76
CW		 1617
1618 obj_priv->madv = __I915_MADV_PURGED;
963b4836
CW		 1619}
1620
1621static inline int
1622i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1623{
1624 return obj_priv->madv == I915_MADV_DONTNEED;
1625}
1626
673a394b
EA		 1627static void
1628i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1629{
1630 struct drm_device *dev = obj->dev;
1631 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1632 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 1633
673a394b 1634 if (obj_priv->pin_count != 0)
69dc4987 1635 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
673a394b 1636 else
69dc4987
CW		 1637 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1638 list_del_init(&obj_priv->ring_list);
673a394b 1639
99fcb766
DV		 1640 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1641
ce44b0ea 1642 obj_priv->last_rendering_seqno = 0;
852835f3 1643 obj_priv->ring = NULL;
673a394b
EA		 1644 if (obj_priv->active) {
1645 obj_priv->active = 0;
1646 drm_gem_object_unreference(obj);
1647 }
23bc5982 1648 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 1649}
1650
9220434a 1651static void
63560396 1652i915_gem_process_flushing_list(struct drm_device *dev,
8a1a49f9 1653 uint32_t flush_domains,
852835f3 1654 struct intel_ring_buffer *ring)
63560396
DV		 1655{
1656 drm_i915_private_t *dev_priv = dev->dev_private;
1657 struct drm_i915_gem_object *obj_priv, *next;
1658
1659 list_for_each_entry_safe(obj_priv, next,
1660 &dev_priv->mm.gpu_write_list,
1661 gpu_write_list) {
a8089e84 1662 struct drm_gem_object *obj = &obj_priv->base;
63560396 1663
2b6efaa4
CW		 1664 if (obj->write_domain & flush_domains &&
1665 obj_priv->ring == ring) {
63560396
DV		 1666 uint32_t old_write_domain = obj->write_domain;
1667
1668 obj->write_domain = 0;
1669 list_del_init(&obj_priv->gpu_write_list);
617dbe27 1670 i915_gem_object_move_to_active(obj, ring);
63560396
DV		 1671
1672 /* update the fence lru list */
007cc8ac
DV		 1673 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1674 struct drm_i915_fence_reg *reg =
1675 &dev_priv->fence_regs[obj_priv->fence_reg];
1676 list_move_tail(&reg->lru_list,
63560396 1677 &dev_priv->mm.fence_list);
007cc8ac 1678 }
63560396
DV		 1679
1680 trace_i915_gem_object_change_domain(obj,
1681 obj->read_domains,
1682 old_write_domain);
1683 }
1684 }
1685}
8187a2b7 1686
5a5a0c64 1687uint32_t
8a1a49f9 1688i915_add_request(struct drm_device *dev,
f787a5f5 1689 struct drm_file *file,
8dc5d147 1690 struct drm_i915_gem_request *request,
8a1a49f9 1691 struct intel_ring_buffer *ring)
673a394b
EA		 1692{
1693 drm_i915_private_t *dev_priv = dev->dev_private;
f787a5f5 1694 struct drm_i915_file_private *file_priv = NULL;
673a394b
EA		 1695 uint32_t seqno;
1696 int was_empty;
673a394b 1697
f787a5f5
CW		 1698 if (file != NULL)
1699 file_priv = file->driver_priv;
b962442e 1700
8dc5d147
CW		 1701 if (request == NULL) {
1702 request = kzalloc(sizeof(*request), GFP_KERNEL);
1703 if (request == NULL)
1704 return 0;
1705 }
673a394b 1706
f787a5f5 1707 seqno = ring->add_request(dev, ring, 0);
a56ba56c 1708 ring->outstanding_lazy_request = false;
673a394b
EA		 1709
1710 request->seqno = seqno;
852835f3 1711 request->ring = ring;
673a394b 1712 request->emitted_jiffies = jiffies;
852835f3
ZN		 1713 was_empty = list_empty(&ring->request_list);
1714 list_add_tail(&request->list, &ring->request_list);
1715
f787a5f5 1716 if (file_priv) {
1c25595f 1717 spin_lock(&file_priv->mm.lock);
f787a5f5 1718 request->file_priv = file_priv;
b962442e 1719 list_add_tail(&request->client_list,
f787a5f5 1720 &file_priv->mm.request_list);
1c25595f 1721 spin_unlock(&file_priv->mm.lock);
b962442e 1722 }
673a394b 1723
f65d9421 1724 if (!dev_priv->mm.suspended) {
b3b079db
CW		 1725 mod_timer(&dev_priv->hangcheck_timer,
1726 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
f65d9421 1727 if (was_empty)
b3b079db
CW		 1728 queue_delayed_work(dev_priv->wq,
1729 &dev_priv->mm.retire_work, HZ);
f65d9421 1730 }
673a394b
EA		 1731 return seqno;
1732}
1733
1734/**
1735 * Command execution barrier
1736 *
1737 * Ensures that all commands in the ring are finished
1738 * before signalling the CPU
1739 */
8a1a49f9 1740static void
852835f3 1741i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
673a394b 1742{
673a394b 1743 uint32_t flush_domains = 0;
673a394b
EA		 1744
1745 /* The sampler always gets flushed on i965 (sigh) */
a6c45cf0 1746 if (INTEL_INFO(dev)->gen >= 4)
673a394b 1747 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
852835f3
ZN		 1748
1749 ring->flush(dev, ring,
1750 I915_GEM_DOMAIN_COMMAND, flush_domains);
673a394b
EA		 1751}
1752
f787a5f5
CW		 1753static inline void
1754i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
673a394b 1755{
1c25595f
CW		 1756 struct drm_i915_file_private *file_priv = request->file_priv;
1757
1758 if (!file_priv)
1759 return;
1760
1761 spin_lock(&file_priv->mm.lock);
1762 list_del(&request->client_list);
1763 request->file_priv = NULL;
1764 spin_unlock(&file_priv->mm.lock);
673a394b
EA		 1765}
1766
dfaae392
CW		 1767static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1768 struct intel_ring_buffer *ring)
9375e446 1769{
dfaae392
CW		 1770 while (!list_empty(&ring->request_list)) {
1771 struct drm_i915_gem_request *request;
9375e446 1772
dfaae392
CW		 1773 request = list_first_entry(&ring->request_list,
1774 struct drm_i915_gem_request,
1775 list);
1776
1777 list_del(&request->list);
f787a5f5 1778 i915_gem_request_remove_from_client(request);
dfaae392
CW		 1779 kfree(request);
1780 }
1781
1782 while (!list_empty(&ring->active_list)) {
9375e446
CW		 1783 struct drm_i915_gem_object *obj_priv;
1784
dfaae392 1785 obj_priv = list_first_entry(&ring->active_list,
9375e446 1786 struct drm_i915_gem_object,
69dc4987 1787 ring_list);
9375e446
CW		 1788
1789 obj_priv->base.write_domain = 0;
dfaae392 1790 list_del_init(&obj_priv->gpu_write_list);
9375e446
CW		 1791 i915_gem_object_move_to_inactive(&obj_priv->base);
1792 }
1793}
1794
069efc1d 1795void i915_gem_reset(struct drm_device *dev)
77f01230
CW		 1796{
1797 struct drm_i915_private *dev_priv = dev->dev_private;
1798 struct drm_i915_gem_object *obj_priv;
069efc1d 1799 int i;
77f01230 1800
dfaae392 1801 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
87acb0a5 1802 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
549f7365 1803 i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
dfaae392
CW		 1804
1805 /* Remove anything from the flushing lists. The GPU cache is likely
1806 * to be lost on reset along with the data, so simply move the
1807 * lost bo to the inactive list.
1808 */
1809 while (!list_empty(&dev_priv->mm.flushing_list)) {
1810 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1811 struct drm_i915_gem_object,
69dc4987 1812 mm_list);
dfaae392
CW		 1813
1814 obj_priv->base.write_domain = 0;
1815 list_del_init(&obj_priv->gpu_write_list);
1816 i915_gem_object_move_to_inactive(&obj_priv->base);
1817 }
1818
1819 /* Move everything out of the GPU domains to ensure we do any
1820 * necessary invalidation upon reuse.
1821 */
77f01230
CW		 1822 list_for_each_entry(obj_priv,
1823 &dev_priv->mm.inactive_list,
69dc4987 1824 mm_list)
77f01230
CW		 1825 {
1826 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1827 }
069efc1d
CW		 1828
1829 /* The fence registers are invalidated so clear them out */
1830 for (i = 0; i < 16; i++) {
1831 struct drm_i915_fence_reg *reg;
1832
1833 reg = &dev_priv->fence_regs[i];
1834 if (!reg->obj)
1835 continue;
1836
1837 i915_gem_clear_fence_reg(reg->obj);
1838 }
77f01230
CW		 1839}
1840
673a394b
EA		 1841/**
1842 * This function clears the request list as sequence numbers are passed.
1843 */
b09a1fec
CW		 1844static void
1845i915_gem_retire_requests_ring(struct drm_device *dev,
1846 struct intel_ring_buffer *ring)
673a394b
EA		 1847{
1848 drm_i915_private_t *dev_priv = dev->dev_private;
1849 uint32_t seqno;
1850
b84d5f0c
CW		 1851 if (!ring->status_page.page_addr ||
1852 list_empty(&ring->request_list))
6c0594a3
KW		 1853 return;
1854
23bc5982
CW		 1855 WARN_ON(i915_verify_lists(dev));
1856
f787a5f5 1857 seqno = ring->get_seqno(dev, ring);
852835f3 1858 while (!list_empty(&ring->request_list)) {
673a394b 1859 struct drm_i915_gem_request *request;
673a394b 1860
852835f3 1861 request = list_first_entry(&ring->request_list,
673a394b
EA		 1862 struct drm_i915_gem_request,
1863 list);
673a394b 1864
dfaae392 1865 if (!i915_seqno_passed(seqno, request->seqno))
b84d5f0c
CW		 1866 break;
1867
1868 trace_i915_gem_request_retire(dev, request->seqno);
1869
1870 list_del(&request->list);
f787a5f5 1871 i915_gem_request_remove_from_client(request);
b84d5f0c
CW		 1872 kfree(request);
1873 }
1874
1875 /* Move any buffers on the active list that are no longer referenced
1876 * by the ringbuffer to the flushing/inactive lists as appropriate.
1877 */
1878 while (!list_empty(&ring->active_list)) {
1879 struct drm_gem_object *obj;
1880 struct drm_i915_gem_object *obj_priv;
1881
1882 obj_priv = list_first_entry(&ring->active_list,
1883 struct drm_i915_gem_object,
69dc4987 1884 ring_list);
673a394b 1885
dfaae392 1886 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
673a394b 1887 break;
b84d5f0c
CW		 1888
1889 obj = &obj_priv->base;
b84d5f0c
CW		 1890 if (obj->write_domain != 0)
1891 i915_gem_object_move_to_flushing(obj);
1892 else
1893 i915_gem_object_move_to_inactive(obj);
673a394b 1894 }
9d34e5db
CW		 1895
1896 if (unlikely (dev_priv->trace_irq_seqno &&
1897 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
8187a2b7 1898 ring->user_irq_put(dev, ring);
9d34e5db
CW		 1899 dev_priv->trace_irq_seqno = 0;
1900 }
23bc5982
CW		 1901
1902 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 1903}
1904
b09a1fec
CW		 1905void
1906i915_gem_retire_requests(struct drm_device *dev)
1907{
1908 drm_i915_private_t *dev_priv = dev->dev_private;
1909
be72615b
CW		 1910 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1911 struct drm_i915_gem_object *obj_priv, *tmp;
1912
1913 /* We must be careful that during unbind() we do not
1914 * accidentally infinitely recurse into retire requests.
1915 * Currently:
1916 * retire -> free -> unbind -> wait -> retire_ring
1917 */
1918 list_for_each_entry_safe(obj_priv, tmp,
1919 &dev_priv->mm.deferred_free_list,
69dc4987 1920 mm_list)
be72615b
CW		 1921 i915_gem_free_object_tail(&obj_priv->base);
1922 }
1923
b09a1fec 1924 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
87acb0a5 1925 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
549f7365 1926 i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
b09a1fec
CW		 1927}
1928
75ef9da2 1929static void
673a394b
EA		 1930i915_gem_retire_work_handler(struct work_struct *work)
1931{
1932 drm_i915_private_t *dev_priv;
1933 struct drm_device *dev;
1934
1935 dev_priv = container_of(work, drm_i915_private_t,
1936 mm.retire_work.work);
1937 dev = dev_priv->dev;
1938
891b48cf
CW		 1939 /* Come back later if the device is busy... */
1940 if (!mutex_trylock(&dev->struct_mutex)) {
1941 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1942 return;
1943 }
1944
b09a1fec 1945 i915_gem_retire_requests(dev);
d1b851fc 1946
6dbe2772 1947 if (!dev_priv->mm.suspended &&
d1b851fc 1948 (!list_empty(&dev_priv->render_ring.request_list) ||
549f7365
CW		 1949 !list_empty(&dev_priv->bsd_ring.request_list) ||
1950 !list_empty(&dev_priv->blt_ring.request_list)))
9c9fe1f8 1951 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
673a394b
EA		 1952 mutex_unlock(&dev->struct_mutex);
1953}
1954
5a5a0c64 1955int
852835f3 1956i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
8a1a49f9 1957 bool interruptible, struct intel_ring_buffer *ring)
673a394b
EA		 1958{
1959 drm_i915_private_t *dev_priv = dev->dev_private;
802c7eb6 1960 u32 ier;
673a394b
EA		 1961 int ret = 0;
1962
1963 BUG_ON(seqno == 0);
1964
30dbf0c0
CW		 1965 if (atomic_read(&dev_priv->mm.wedged))
1966 return -EAGAIN;
1967
a56ba56c 1968 if (ring->outstanding_lazy_request) {
8dc5d147 1969 seqno = i915_add_request(dev, NULL, NULL, ring);
e35a41de
DV		 1970 if (seqno == 0)
1971 return -ENOMEM;
1972 }
a56ba56c 1973 BUG_ON(seqno == dev_priv->next_seqno);
e35a41de 1974
f787a5f5 1975 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
bad720ff 1976 if (HAS_PCH_SPLIT(dev))
036a4a7d
ZW		 1977 ier = I915_READ(DEIER) | I915_READ(GTIER);
1978 else
1979 ier = I915_READ(IER);
802c7eb6
JB		 1980 if (!ier) {
1981 DRM_ERROR("something (likely vbetool) disabled "
1982 "interrupts, re-enabling\n");
1983 i915_driver_irq_preinstall(dev);
1984 i915_driver_irq_postinstall(dev);
1985 }
1986
1c5d22f7
CW		 1987 trace_i915_gem_request_wait_begin(dev, seqno);
1988
852835f3 1989 ring->waiting_gem_seqno = seqno;
8187a2b7 1990 ring->user_irq_get(dev, ring);
48764bf4 1991 if (interruptible)
852835f3
ZN		 1992 ret = wait_event_interruptible(ring->irq_queue,
1993 i915_seqno_passed(
f787a5f5 1994 ring->get_seqno(dev, ring), seqno)
852835f3 1995 || atomic_read(&dev_priv->mm.wedged));
48764bf4 1996 else
852835f3
ZN		 1997 wait_event(ring->irq_queue,
1998 i915_seqno_passed(
f787a5f5 1999 ring->get_seqno(dev, ring), seqno)
852835f3 2000 || atomic_read(&dev_priv->mm.wedged));
48764bf4 2001
8187a2b7 2002 ring->user_irq_put(dev, ring);
852835f3 2003 ring->waiting_gem_seqno = 0;
1c5d22f7
CW		 2004
2005 trace_i915_gem_request_wait_end(dev, seqno);
673a394b 2006 }
ba1234d1 2007 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0 2008 ret = -EAGAIN;
673a394b
EA		 2009
2010 if (ret && ret != -ERESTARTSYS)
8bff917c 2011 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
f787a5f5 2012 __func__, ret, seqno, ring->get_seqno(dev, ring),
8bff917c 2013 dev_priv->next_seqno);
673a394b
EA		 2014
2015 /* Directly dispatch request retiring. While we have the work queue
2016 * to handle this, the waiter on a request often wants an associated
2017 * buffer to have made it to the inactive list, and we would need
2018 * a separate wait queue to handle that.
2019 */
2020 if (ret == 0)
b09a1fec 2021 i915_gem_retire_requests_ring(dev, ring);
673a394b
EA		 2022
2023 return ret;
2024}
2025
48764bf4
DV		 2026/**
2027 * Waits for a sequence number to be signaled, and cleans up the
2028 * request and object lists appropriately for that event.
2029 */
2030static int
852835f3 2031i915_wait_request(struct drm_device *dev, uint32_t seqno,
a56ba56c 2032 struct intel_ring_buffer *ring)
48764bf4 2033{
852835f3 2034 return i915_do_wait_request(dev, seqno, 1, ring);
48764bf4
DV		 2035}
2036
20f0cd55 2037static void
9220434a 2038i915_gem_flush_ring(struct drm_device *dev,
c78ec30b 2039 struct drm_file *file_priv,
9220434a
CW		 2040 struct intel_ring_buffer *ring,
2041 uint32_t invalidate_domains,
2042 uint32_t flush_domains)
2043{
2044 ring->flush(dev, ring, invalidate_domains, flush_domains);
2045 i915_gem_process_flushing_list(dev, flush_domains, ring);
2046}
2047
8187a2b7
ZN		 2048static void
2049i915_gem_flush(struct drm_device *dev,
c78ec30b 2050 struct drm_file *file_priv,
8187a2b7 2051 uint32_t invalidate_domains,
9220434a
CW		 2052 uint32_t flush_domains,
2053 uint32_t flush_rings)
8187a2b7
ZN		 2054{
2055 drm_i915_private_t *dev_priv = dev->dev_private;
8bff917c 2056
8187a2b7
ZN		 2057 if (flush_domains & I915_GEM_DOMAIN_CPU)
2058 drm_agp_chipset_flush(dev);
8bff917c 2059
9220434a
CW		 2060 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2061 if (flush_rings & RING_RENDER)
c78ec30b 2062 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 2063 &dev_priv->render_ring,
2064 invalidate_domains, flush_domains);
2065 if (flush_rings & RING_BSD)
c78ec30b 2066 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 2067 &dev_priv->bsd_ring,
2068 invalidate_domains, flush_domains);
549f7365
CW		 2069 if (flush_rings & RING_BLT)
2070 i915_gem_flush_ring(dev, file_priv,
2071 &dev_priv->blt_ring,
2072 invalidate_domains, flush_domains);
9220434a 2073 }
8187a2b7
ZN		 2074}
2075
673a394b
EA		 2076/**
2077 * Ensures that all rendering to the object has completed and the object is
2078 * safe to unbind from the GTT or access from the CPU.
2079 */
2080static int
2cf34d7b
CW		 2081i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2082 bool interruptible)
673a394b
EA		 2083{
2084 struct drm_device *dev = obj->dev;
23010e43 2085 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2086 int ret;
2087
e47c68e9
EA		 2088 /* This function only exists to support waiting for existing rendering,
2089 * not for emitting required flushes.
673a394b 2090 */
e47c68e9 2091 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 2092
2093 /* If there is rendering queued on the buffer being evicted, wait for
2094 * it.
2095 */
2096 if (obj_priv->active) {
2cf34d7b
CW		 2097 ret = i915_do_wait_request(dev,
2098 obj_priv->last_rendering_seqno,
2099 interruptible,
2100 obj_priv->ring);
2101 if (ret)
673a394b
EA		 2102 return ret;
2103 }
2104
2105 return 0;
2106}
2107
2108/**
2109 * Unbinds an object from the GTT aperture.
2110 */
0f973f27 2111int
673a394b
EA		 2112i915_gem_object_unbind(struct drm_gem_object *obj)
2113{
2114 struct drm_device *dev = obj->dev;
73aa808f 2115 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2116 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2117 int ret = 0;
2118
673a394b
EA		 2119 if (obj_priv->gtt_space == NULL)
2120 return 0;
2121
2122 if (obj_priv->pin_count != 0) {
2123 DRM_ERROR("Attempting to unbind pinned buffer\n");
2124 return -EINVAL;
2125 }
2126
5323fd04
EA		 2127 /* blow away mappings if mapped through GTT */
2128 i915_gem_release_mmap(obj);
2129
673a394b
EA		 2130 /* Move the object to the CPU domain to ensure that
2131 * any possible CPU writes while it's not in the GTT
2132 * are flushed when we go to remap it. This will
2133 * also ensure that all pending GPU writes are finished
2134 * before we unbind.
2135 */
e47c68e9 2136 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
8dc1775d 2137 if (ret == -ERESTARTSYS)
673a394b 2138 return ret;
8dc1775d
CW		 2139 /* Continue on if we fail due to EIO, the GPU is hung so we
2140 * should be safe and we need to cleanup or else we might
2141 * cause memory corruption through use-after-free.
2142 */
812ed492
CW		 2143 if (ret) {
2144 i915_gem_clflush_object(obj);
2145 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
2146 }
673a394b 2147
96b47b65
DV		 2148 /* release the fence reg _after_ flushing */
2149 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2150 i915_gem_clear_fence_reg(obj);
2151
73aa808f
CW		 2152 drm_unbind_agp(obj_priv->agp_mem);
2153 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
673a394b 2154
856fa198 2155 i915_gem_object_put_pages(obj);
a32808c0 2156 BUG_ON(obj_priv->pages_refcount);
673a394b 2157
73aa808f 2158 i915_gem_info_remove_gtt(dev_priv, obj->size);
69dc4987 2159 list_del_init(&obj_priv->mm_list);
673a394b 2160
73aa808f
CW		 2161 drm_mm_put_block(obj_priv->gtt_space);
2162 obj_priv->gtt_space = NULL;
9af90d19 2163 obj_priv->gtt_offset = 0;
73aa808f 2164
963b4836
CW		 2165 if (i915_gem_object_is_purgeable(obj_priv))
2166 i915_gem_object_truncate(obj);
2167
1c5d22f7
CW		 2168 trace_i915_gem_object_unbind(obj);
2169
8dc1775d 2170 return ret;
673a394b
EA		 2171}
2172
a56ba56c
CW		 2173static int i915_ring_idle(struct drm_device *dev,
2174 struct intel_ring_buffer *ring)
2175{
2176 i915_gem_flush_ring(dev, NULL, ring,
2177 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2178 return i915_wait_request(dev,
2179 i915_gem_next_request_seqno(dev, ring),
2180 ring);
2181}
2182
b47eb4a2 2183int
4df2faf4
DV		 2184i915_gpu_idle(struct drm_device *dev)
2185{
2186 drm_i915_private_t *dev_priv = dev->dev_private;
2187 bool lists_empty;
852835f3 2188 int ret;
4df2faf4 2189
d1b851fc
ZN		 2190 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2191 list_empty(&dev_priv->render_ring.active_list) &&
549f7365
CW		 2192 list_empty(&dev_priv->bsd_ring.active_list) &&
2193 list_empty(&dev_priv->blt_ring.active_list));
4df2faf4
DV		 2194 if (lists_empty)
2195 return 0;
2196
2197 /* Flush everything onto the inactive list. */
a56ba56c 2198 ret = i915_ring_idle(dev, &dev_priv->render_ring);
8a1a49f9
DV		 2199 if (ret)
2200 return ret;
d1b851fc 2201
87acb0a5
CW		 2202 ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2203 if (ret)
2204 return ret;
d1b851fc 2205
549f7365
CW		 2206 ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2207 if (ret)
2208 return ret;
2209
8a1a49f9 2210 return 0;
4df2faf4
DV		 2211}
2212
5cdf5881 2213static int
4bdadb97
CW		 2214i915_gem_object_get_pages(struct drm_gem_object *obj,
2215 gfp_t gfpmask)
673a394b 2216{
23010e43 2217 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2218 int page_count, i;
2219 struct address_space *mapping;
2220 struct inode *inode;
2221 struct page *page;
673a394b 2222
778c3544
DV		 2223 BUG_ON(obj_priv->pages_refcount
2224 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2225
856fa198 2226 if (obj_priv->pages_refcount++ != 0)
673a394b
EA		 2227 return 0;
2228
2229 /* Get the list of pages out of our struct file. They'll be pinned
2230 * at this point until we release them.
2231 */
2232 page_count = obj->size / PAGE_SIZE;
856fa198 2233 BUG_ON(obj_priv->pages != NULL);
8e7d2b2c 2234 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
856fa198 2235 if (obj_priv->pages == NULL) {
856fa198 2236 obj_priv->pages_refcount--;
673a394b
EA		 2237 return -ENOMEM;
2238 }
2239
2240 inode = obj->filp->f_path.dentry->d_inode;
2241 mapping = inode->i_mapping;
2242 for (i = 0; i < page_count; i++) {
4bdadb97 2243 page = read_cache_page_gfp(mapping, i,
985b823b 2244 GFP_HIGHUSER |
4bdadb97 2245 __GFP_COLD |
cd9f040d 2246 __GFP_RECLAIMABLE |
4bdadb97 2247 gfpmask);
1f2b1013
CW		 2248 if (IS_ERR(page))
2249 goto err_pages;
2250
856fa198 2251 obj_priv->pages[i] = page;
673a394b 2252 }
280b713b
EA		 2253
2254 if (obj_priv->tiling_mode != I915_TILING_NONE)
2255 i915_gem_object_do_bit_17_swizzle(obj);
2256
673a394b 2257 return 0;
1f2b1013
CW		 2258
2259err_pages:
2260 while (i--)
2261 page_cache_release(obj_priv->pages[i]);
2262
2263 drm_free_large(obj_priv->pages);
2264 obj_priv->pages = NULL;
2265 obj_priv->pages_refcount--;
2266 return PTR_ERR(page);
673a394b
EA		 2267}
2268
4e901fdc
EA		 2269static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2270{
2271 struct drm_gem_object *obj = reg->obj;
2272 struct drm_device *dev = obj->dev;
2273 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2274 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4e901fdc
EA		 2275 int regnum = obj_priv->fence_reg;
2276 uint64_t val;
2277
2278 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2279 0xfffff000) << 32;
2280 val |= obj_priv->gtt_offset & 0xfffff000;
2281 val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2282 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2283
2284 if (obj_priv->tiling_mode == I915_TILING_Y)
2285 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2286 val |= I965_FENCE_REG_VALID;
2287
2288 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2289}
2290
de151cf6
JB		 2291static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2292{
2293 struct drm_gem_object *obj = reg->obj;
2294 struct drm_device *dev = obj->dev;
2295 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2296 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 2297 int regnum = obj_priv->fence_reg;
2298 uint64_t val;
2299
2300 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2301 0xfffff000) << 32;
2302 val |= obj_priv->gtt_offset & 0xfffff000;
2303 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2304 if (obj_priv->tiling_mode == I915_TILING_Y)
2305 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2306 val |= I965_FENCE_REG_VALID;
2307
2308 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2309}
2310
2311static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2312{
2313 struct drm_gem_object *obj = reg->obj;
2314 struct drm_device *dev = obj->dev;
2315 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2316 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2317 int regnum = obj_priv->fence_reg;
0f973f27 2318 int tile_width;
dc529a4f 2319 uint32_t fence_reg, val;
de151cf6
JB		 2320 uint32_t pitch_val;
2321
2322 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2323 (obj_priv->gtt_offset & (obj->size - 1))) {
f06da264 2324 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
0f973f27 2325 __func__, obj_priv->gtt_offset, obj->size);
de151cf6
JB		 2326 return;
2327 }
2328
0f973f27
JB		 2329 if (obj_priv->tiling_mode == I915_TILING_Y &&
2330 HAS_128_BYTE_Y_TILING(dev))
2331 tile_width = 128;
de151cf6 2332 else
0f973f27
JB		 2333 tile_width = 512;
2334
2335 /* Note: pitch better be a power of two tile widths */
2336 pitch_val = obj_priv->stride / tile_width;
2337 pitch_val = ffs(pitch_val) - 1;
de151cf6 2338
c36a2a6d
DV		 2339 if (obj_priv->tiling_mode == I915_TILING_Y &&
2340 HAS_128_BYTE_Y_TILING(dev))
2341 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2342 else
2343 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2344
de151cf6
JB		 2345 val = obj_priv->gtt_offset;
2346 if (obj_priv->tiling_mode == I915_TILING_Y)
2347 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2348 val |= I915_FENCE_SIZE_BITS(obj->size);
2349 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2350 val |= I830_FENCE_REG_VALID;
2351
dc529a4f
EA		 2352 if (regnum < 8)
2353 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2354 else
2355 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2356 I915_WRITE(fence_reg, val);
de151cf6
JB		 2357}
2358
2359static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2360{
2361 struct drm_gem_object *obj = reg->obj;
2362 struct drm_device *dev = obj->dev;
2363 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2364 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 2365 int regnum = obj_priv->fence_reg;
2366 uint32_t val;
2367 uint32_t pitch_val;
8d7773a3 2368 uint32_t fence_size_bits;
de151cf6 2369
8d7773a3 2370 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
de151cf6 2371 (obj_priv->gtt_offset & (obj->size - 1))) {
8d7773a3 2372 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
0f973f27 2373 __func__, obj_priv->gtt_offset);
de151cf6
JB		 2374 return;
2375 }
2376
e76a16de
EA		 2377 pitch_val = obj_priv->stride / 128;
2378 pitch_val = ffs(pitch_val) - 1;
2379 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2380
de151cf6
JB		 2381 val = obj_priv->gtt_offset;
2382 if (obj_priv->tiling_mode == I915_TILING_Y)
2383 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
8d7773a3
DV		 2384 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2385 WARN_ON(fence_size_bits & ~0x00000f00);
2386 val |= fence_size_bits;
de151cf6
JB		 2387 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2388 val |= I830_FENCE_REG_VALID;
2389
2390 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
de151cf6
JB		 2391}
2392
2cf34d7b
CW		 2393static int i915_find_fence_reg(struct drm_device *dev,
2394 bool interruptible)
ae3db24a
DV		 2395{
2396 struct drm_i915_fence_reg *reg = NULL;
2397 struct drm_i915_gem_object *obj_priv = NULL;
2398 struct drm_i915_private *dev_priv = dev->dev_private;
2399 struct drm_gem_object *obj = NULL;
2400 int i, avail, ret;
2401
2402 /* First try to find a free reg */
2403 avail = 0;
2404 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2405 reg = &dev_priv->fence_regs[i];
2406 if (!reg->obj)
2407 return i;
2408
23010e43 2409 obj_priv = to_intel_bo(reg->obj);
ae3db24a
DV		 2410 if (!obj_priv->pin_count)
2411 avail++;
2412 }
2413
2414 if (avail == 0)
2415 return -ENOSPC;
2416
2417 /* None available, try to steal one or wait for a user to finish */
2418 i = I915_FENCE_REG_NONE;
007cc8ac
DV		 2419 list_for_each_entry(reg, &dev_priv->mm.fence_list,
2420 lru_list) {
2421 obj = reg->obj;
2422 obj_priv = to_intel_bo(obj);
ae3db24a
DV		 2423
2424 if (obj_priv->pin_count)
2425 continue;
2426
2427 /* found one! */
2428 i = obj_priv->fence_reg;
2429 break;
2430 }
2431
2432 BUG_ON(i == I915_FENCE_REG_NONE);
2433
2434 /* We only have a reference on obj from the active list. put_fence_reg
2435 * might drop that one, causing a use-after-free in it. So hold a
2436 * private reference to obj like the other callers of put_fence_reg
2437 * (set_tiling ioctl) do. */
2438 drm_gem_object_reference(obj);
2cf34d7b 2439 ret = i915_gem_object_put_fence_reg(obj, interruptible);
ae3db24a
DV		 2440 drm_gem_object_unreference(obj);
2441 if (ret != 0)
2442 return ret;
2443
2444 return i;
2445}
2446
de151cf6
JB		 2447/**
2448 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2449 * @obj: object to map through a fence reg
2450 *
2451 * When mapping objects through the GTT, userspace wants to be able to write
2452 * to them without having to worry about swizzling if the object is tiled.
2453 *
2454 * This function walks the fence regs looking for a free one for @obj,
2455 * stealing one if it can't find any.
2456 *
2457 * It then sets up the reg based on the object's properties: address, pitch
2458 * and tiling format.
2459 */
8c4b8c3f 2460int
2cf34d7b
CW		 2461i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2462 bool interruptible)
de151cf6
JB		 2463{
2464 struct drm_device *dev = obj->dev;
79e53945 2465 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2466 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2467 struct drm_i915_fence_reg *reg = NULL;
ae3db24a 2468 int ret;
de151cf6 2469
a09ba7fa
EA		 2470 /* Just update our place in the LRU if our fence is getting used. */
2471 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
007cc8ac
DV		 2472 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2473 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
a09ba7fa
EA		 2474 return 0;
2475 }
2476
de151cf6
JB		 2477 switch (obj_priv->tiling_mode) {
2478 case I915_TILING_NONE:
2479 WARN(1, "allocating a fence for non-tiled object?\n");
2480 break;
2481 case I915_TILING_X:
0f973f27
JB		 2482 if (!obj_priv->stride)
2483 return -EINVAL;
2484 WARN((obj_priv->stride & (512 - 1)),
2485 "object 0x%08x is X tiled but has non-512B pitch\n",
2486 obj_priv->gtt_offset);
de151cf6
JB		 2487 break;
2488 case I915_TILING_Y:
0f973f27
JB		 2489 if (!obj_priv->stride)
2490 return -EINVAL;
2491 WARN((obj_priv->stride & (128 - 1)),
2492 "object 0x%08x is Y tiled but has non-128B pitch\n",
2493 obj_priv->gtt_offset);
de151cf6
JB		 2494 break;
2495 }
2496
2cf34d7b 2497 ret = i915_find_fence_reg(dev, interruptible);
ae3db24a
DV		 2498 if (ret < 0)
2499 return ret;
de151cf6 2500
ae3db24a
DV		 2501 obj_priv->fence_reg = ret;
2502 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
007cc8ac 2503 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
a09ba7fa 2504
de151cf6
JB		 2505 reg->obj = obj;
2506
e259befd
CW		 2507 switch (INTEL_INFO(dev)->gen) {
2508 case 6:
4e901fdc 2509 sandybridge_write_fence_reg(reg);
e259befd
CW		 2510 break;
2511 case 5:
2512 case 4:
de151cf6 2513 i965_write_fence_reg(reg);
e259befd
CW		 2514 break;
2515 case 3:
de151cf6 2516 i915_write_fence_reg(reg);
e259befd
CW		 2517 break;
2518 case 2:
de151cf6 2519 i830_write_fence_reg(reg);
e259befd
CW		 2520 break;
2521 }
d9ddcb96 2522
ae3db24a
DV		 2523 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2524 obj_priv->tiling_mode);
1c5d22f7 2525
d9ddcb96 2526 return 0;
de151cf6
JB		 2527}
2528
2529/**
2530 * i915_gem_clear_fence_reg - clear out fence register info
2531 * @obj: object to clear
2532 *
2533 * Zeroes out the fence register itself and clears out the associated
2534 * data structures in dev_priv and obj_priv.
2535 */
2536static void
2537i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2538{
2539 struct drm_device *dev = obj->dev;
79e53945 2540 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2541 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
007cc8ac
DV		 2542 struct drm_i915_fence_reg *reg =
2543 &dev_priv->fence_regs[obj_priv->fence_reg];
e259befd 2544 uint32_t fence_reg;
de151cf6 2545
e259befd
CW		 2546 switch (INTEL_INFO(dev)->gen) {
2547 case 6:
4e901fdc
EA		 2548 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2549 (obj_priv->fence_reg * 8), 0);
e259befd
CW		 2550 break;
2551 case 5:
2552 case 4:
de151cf6 2553 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
e259befd
CW		 2554 break;
2555 case 3:
9b74f734 2556 if (obj_priv->fence_reg >= 8)
e259befd 2557 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
dc529a4f 2558 else
e259befd
CW		 2559 case 2:
2560 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
dc529a4f
EA		 2561
2562 I915_WRITE(fence_reg, 0);
e259befd 2563 break;
dc529a4f 2564 }
de151cf6 2565
007cc8ac 2566 reg->obj = NULL;
de151cf6 2567 obj_priv->fence_reg = I915_FENCE_REG_NONE;
007cc8ac 2568 list_del_init(&reg->lru_list);
de151cf6
JB		 2569}
2570
52dc7d32
CW		 2571/**
2572 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2573 * to the buffer to finish, and then resets the fence register.
2574 * @obj: tiled object holding a fence register.
2cf34d7b 2575 * @bool: whether the wait upon the fence is interruptible
52dc7d32
CW		 2576 *
2577 * Zeroes out the fence register itself and clears out the associated
2578 * data structures in dev_priv and obj_priv.
2579 */
2580int
2cf34d7b
CW		 2581i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2582 bool interruptible)
52dc7d32
CW		 2583{
2584 struct drm_device *dev = obj->dev;
53640e1d 2585 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2586 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
53640e1d 2587 struct drm_i915_fence_reg *reg;
52dc7d32
CW		 2588
2589 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2590 return 0;
2591
10ae9bd2
DV		 2592 /* If we've changed tiling, GTT-mappings of the object
2593 * need to re-fault to ensure that the correct fence register
2594 * setup is in place.
2595 */
2596 i915_gem_release_mmap(obj);
2597
52dc7d32
CW		 2598 /* On the i915, GPU access to tiled buffers is via a fence,
2599 * therefore we must wait for any outstanding access to complete
2600 * before clearing the fence.
2601 */
53640e1d
CW		 2602 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2603 if (reg->gpu) {
52dc7d32
CW		 2604 int ret;
2605
2cf34d7b 2606 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
0bc23aad
CW		 2607 if (ret)
2608 return ret;
2609
2cf34d7b 2610 ret = i915_gem_object_wait_rendering(obj, interruptible);
0bc23aad 2611 if (ret)
52dc7d32 2612 return ret;
53640e1d
CW		 2613
2614 reg->gpu = false;
52dc7d32
CW		 2615 }
2616
4a726612 2617 i915_gem_object_flush_gtt_write_domain(obj);
0bc23aad 2618 i915_gem_clear_fence_reg(obj);
52dc7d32
CW		 2619
2620 return 0;
2621}
2622
673a394b
EA		 2623/**
2624 * Finds free space in the GTT aperture and binds the object there.
2625 */
2626static int
2627i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2628{
2629 struct drm_device *dev = obj->dev;
2630 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2631 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 2632 struct drm_mm_node *free_space;
4bdadb97 2633 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
07f73f69 2634 int ret;
673a394b 2635
bb6baf76 2636 if (obj_priv->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2637 DRM_ERROR("Attempting to bind a purgeable object\n");
2638 return -EINVAL;
2639 }
2640
673a394b 2641 if (alignment == 0)
0f973f27 2642 alignment = i915_gem_get_gtt_alignment(obj);
8d7773a3 2643 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
673a394b
EA		 2644 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2645 return -EINVAL;
2646 }
2647
654fc607
CW		 2648 /* If the object is bigger than the entire aperture, reject it early
2649 * before evicting everything in a vain attempt to find space.
2650 */
73aa808f 2651 if (obj->size > dev_priv->mm.gtt_total) {
654fc607
CW		 2652 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2653 return -E2BIG;
2654 }
2655
673a394b
EA		 2656 search_free:
2657 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2658 obj->size, alignment, 0);
9af90d19 2659 if (free_space != NULL)
673a394b
EA		 2660 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2661 alignment);
673a394b
EA		 2662 if (obj_priv->gtt_space == NULL) {
2663 /* If the gtt is empty and we're still having trouble
2664 * fitting our object in, we're out of memory.
2665 */
0108a3ed 2666 ret = i915_gem_evict_something(dev, obj->size, alignment);
9731129c 2667 if (ret)
673a394b 2668 return ret;
9731129c 2669
673a394b
EA		 2670 goto search_free;
2671 }
2672
4bdadb97 2673 ret = i915_gem_object_get_pages(obj, gfpmask);
673a394b
EA		 2674 if (ret) {
2675 drm_mm_put_block(obj_priv->gtt_space);
2676 obj_priv->gtt_space = NULL;
07f73f69
CW		 2677
2678 if (ret == -ENOMEM) {
2679 /* first try to clear up some space from the GTT */
0108a3ed
DV		 2680 ret = i915_gem_evict_something(dev, obj->size,
2681 alignment);
07f73f69 2682 if (ret) {
07f73f69 2683 /* now try to shrink everyone else */
4bdadb97
CW		 2684 if (gfpmask) {
2685 gfpmask = 0;
2686 goto search_free;
07f73f69
CW		 2687 }
2688
2689 return ret;
2690 }
2691
2692 goto search_free;
2693 }
2694
673a394b
EA		 2695 return ret;
2696 }
2697
673a394b
EA		 2698 /* Create an AGP memory structure pointing at our pages, and bind it
2699 * into the GTT.
2700 */
2701 obj_priv->agp_mem = drm_agp_bind_pages(dev,
856fa198 2702 obj_priv->pages,
07f73f69 2703 obj->size >> PAGE_SHIFT,
9af90d19 2704 obj_priv->gtt_space->start,
ba1eb1d8 2705 obj_priv->agp_type);
673a394b 2706 if (obj_priv->agp_mem == NULL) {
856fa198 2707 i915_gem_object_put_pages(obj);
673a394b
EA		 2708 drm_mm_put_block(obj_priv->gtt_space);
2709 obj_priv->gtt_space = NULL;
07f73f69 2710
0108a3ed 2711 ret = i915_gem_evict_something(dev, obj->size, alignment);
9731129c 2712 if (ret)
07f73f69 2713 return ret;
07f73f69
CW		 2714
2715 goto search_free;
673a394b 2716 }
673a394b 2717
bf1a1092 2718 /* keep track of bounds object by adding it to the inactive list */
69dc4987 2719 list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
73aa808f 2720 i915_gem_info_add_gtt(dev_priv, obj->size);
bf1a1092 2721
673a394b
EA		 2722 /* Assert that the object is not currently in any GPU domain. As it
2723 * wasn't in the GTT, there shouldn't be any way it could have been in
2724 * a GPU cache
2725 */
21d509e3
CW		 2726 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2727 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2728
9af90d19 2729 obj_priv->gtt_offset = obj_priv->gtt_space->start;
1c5d22f7
CW		 2730 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2731
673a394b
EA		 2732 return 0;
2733}
2734
2735void
2736i915_gem_clflush_object(struct drm_gem_object *obj)
2737{
23010e43 2738 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2739
2740 /* If we don't have a page list set up, then we're not pinned
2741 * to GPU, and we can ignore the cache flush because it'll happen
2742 * again at bind time.
2743 */
856fa198 2744 if (obj_priv->pages == NULL)
673a394b
EA		 2745 return;
2746
1c5d22f7 2747 trace_i915_gem_object_clflush(obj);
cfa16a0d 2748
856fa198 2749 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
673a394b
EA		 2750}
2751
e47c68e9 2752/** Flushes any GPU write domain for the object if it's dirty. */
2dafb1e0 2753static int
ba3d8d74
DV		 2754i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2755 bool pipelined)
e47c68e9
EA		 2756{
2757 struct drm_device *dev = obj->dev;
1c5d22f7 2758 uint32_t old_write_domain;
e47c68e9
EA		 2759
2760 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2dafb1e0 2761 return 0;
e47c68e9
EA		 2762
2763 /* Queue the GPU write cache flushing we need. */
1c5d22f7 2764 old_write_domain = obj->write_domain;
c78ec30b 2765 i915_gem_flush_ring(dev, NULL,
9220434a
CW		 2766 to_intel_bo(obj)->ring,
2767 0, obj->write_domain);
48b956c5 2768 BUG_ON(obj->write_domain);
1c5d22f7
CW		 2769
2770 trace_i915_gem_object_change_domain(obj,
2771 obj->read_domains,
2772 old_write_domain);
ba3d8d74
DV		 2773
2774 if (pipelined)
2775 return 0;
2776
2cf34d7b 2777 return i915_gem_object_wait_rendering(obj, true);
e47c68e9
EA		 2778}
2779
2780/** Flushes the GTT write domain for the object if it's dirty. */
2781static void
2782i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2783{
1c5d22f7
CW		 2784 uint32_t old_write_domain;
2785
e47c68e9
EA		 2786 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2787 return;
2788
2789 /* No actual flushing is required for the GTT write domain. Writes
2790 * to it immediately go to main memory as far as we know, so there's
2791 * no chipset flush. It also doesn't land in render cache.
2792 */
1c5d22f7 2793 old_write_domain = obj->write_domain;
e47c68e9 2794 obj->write_domain = 0;
1c5d22f7
CW		 2795
2796 trace_i915_gem_object_change_domain(obj,
2797 obj->read_domains,
2798 old_write_domain);
e47c68e9
EA		 2799}
2800
2801/** Flushes the CPU write domain for the object if it's dirty. */
2802static void
2803i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2804{
2805 struct drm_device *dev = obj->dev;
1c5d22f7 2806 uint32_t old_write_domain;
e47c68e9
EA		 2807
2808 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2809 return;
2810
2811 i915_gem_clflush_object(obj);
2812 drm_agp_chipset_flush(dev);
1c5d22f7 2813 old_write_domain = obj->write_domain;
e47c68e9 2814 obj->write_domain = 0;
1c5d22f7
CW		 2815
2816 trace_i915_gem_object_change_domain(obj,
2817 obj->read_domains,
2818 old_write_domain);
e47c68e9
EA		 2819}
2820
2ef7eeaa
EA		 2821/**
2822 * Moves a single object to the GTT read, and possibly write domain.
2823 *
2824 * This function returns when the move is complete, including waiting on
2825 * flushes to occur.
2826 */
79e53945 2827int
2ef7eeaa
EA		 2828i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2829{
23010e43 2830 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 2831 uint32_t old_write_domain, old_read_domains;
e47c68e9 2832 int ret;
2ef7eeaa 2833
02354392
EA		 2834 /* Not valid to be called on unbound objects. */
2835 if (obj_priv->gtt_space == NULL)
2836 return -EINVAL;
2837
ba3d8d74 2838 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9
EA		 2839 if (ret != 0)
2840 return ret;
2841
7213342d 2842 i915_gem_object_flush_cpu_write_domain(obj);
1c5d22f7 2843
ba3d8d74 2844 if (write) {
2cf34d7b 2845 ret = i915_gem_object_wait_rendering(obj, true);
ba3d8d74
DV		 2846 if (ret)
2847 return ret;
ba3d8d74 2848 }
2ef7eeaa 2849
7213342d
CW		 2850 old_write_domain = obj->write_domain;
2851 old_read_domains = obj->read_domains;
2ef7eeaa 2852
e47c68e9
EA		 2853 /* It should now be out of any other write domains, and we can update
2854 * the domain values for our changes.
2855 */
2856 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2857 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2858 if (write) {
7213342d 2859 obj->read_domains = I915_GEM_DOMAIN_GTT;
e47c68e9
EA		 2860 obj->write_domain = I915_GEM_DOMAIN_GTT;
2861 obj_priv->dirty = 1;
2ef7eeaa
EA		 2862 }
2863
1c5d22f7
CW		 2864 trace_i915_gem_object_change_domain(obj,
2865 old_read_domains,
2866 old_write_domain);
2867
e47c68e9
EA		 2868 return 0;
2869}
2870
b9241ea3
ZW		 2871/*
2872 * Prepare buffer for display plane. Use uninterruptible for possible flush
2873 * wait, as in modesetting process we're not supposed to be interrupted.
2874 */
2875int
48b956c5
CW		 2876i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2877 bool pipelined)
b9241ea3 2878{
23010e43 2879 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ba3d8d74 2880 uint32_t old_read_domains;
b9241ea3
ZW		 2881 int ret;
2882
2883 /* Not valid to be called on unbound objects. */
2884 if (obj_priv->gtt_space == NULL)
2885 return -EINVAL;
2886
ced270fa 2887 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
48b956c5 2888 if (ret)
e35a41de 2889 return ret;
b9241ea3 2890
ced270fa
CW		 2891 /* Currently, we are always called from an non-interruptible context. */
2892 if (!pipelined) {
2893 ret = i915_gem_object_wait_rendering(obj, false);
2894 if (ret)
2895 return ret;
2896 }
2897
b118c1e3
CW		 2898 i915_gem_object_flush_cpu_write_domain(obj);
2899
b9241ea3 2900 old_read_domains = obj->read_domains;
c78ec30b 2901 obj->read_domains |= I915_GEM_DOMAIN_GTT;
b9241ea3
ZW		 2902
2903 trace_i915_gem_object_change_domain(obj,
2904 old_read_domains,
ba3d8d74 2905 obj->write_domain);
b9241ea3
ZW		 2906
2907 return 0;
2908}
2909
e47c68e9
EA		 2910/**
2911 * Moves a single object to the CPU read, and possibly write domain.
2912 *
2913 * This function returns when the move is complete, including waiting on
2914 * flushes to occur.
2915 */
2916static int
2917i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2918{
1c5d22f7 2919 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 2920 int ret;
2921
ba3d8d74 2922 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9
EA		 2923 if (ret != 0)
2924 return ret;
2ef7eeaa 2925
e47c68e9 2926 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 2927
e47c68e9
EA		 2928 /* If we have a partially-valid cache of the object in the CPU,
2929 * finish invalidating it and free the per-page flags.
2ef7eeaa 2930 */
e47c68e9 2931 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 2932
7213342d 2933 if (write) {
2cf34d7b 2934 ret = i915_gem_object_wait_rendering(obj, true);
7213342d
CW		 2935 if (ret)
2936 return ret;
2937 }
2938
1c5d22f7
CW		 2939 old_write_domain = obj->write_domain;
2940 old_read_domains = obj->read_domains;
2941
e47c68e9
EA		 2942 /* Flush the CPU cache if it's still invalid. */
2943 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 2944 i915_gem_clflush_object(obj);
2ef7eeaa 2945
e47c68e9 2946 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 2947 }
2948
2949 /* It should now be out of any other write domains, and we can update
2950 * the domain values for our changes.
2951 */
e47c68e9
EA		 2952 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2953
2954 /* If we're writing through the CPU, then the GPU read domains will
2955 * need to be invalidated at next use.
2956 */
2957 if (write) {
c78ec30b 2958 obj->read_domains = I915_GEM_DOMAIN_CPU;
e47c68e9
EA		 2959 obj->write_domain = I915_GEM_DOMAIN_CPU;
2960 }
2ef7eeaa 2961
1c5d22f7
CW		 2962 trace_i915_gem_object_change_domain(obj,
2963 old_read_domains,
2964 old_write_domain);
2965
2ef7eeaa
EA		 2966 return 0;
2967}
2968
673a394b
EA		 2969/*
2970 * Set the next domain for the specified object. This
2971 * may not actually perform the necessary flushing/invaliding though,
2972 * as that may want to be batched with other set_domain operations
2973 *
2974 * This is (we hope) the only really tricky part of gem. The goal
2975 * is fairly simple -- track which caches hold bits of the object
2976 * and make sure they remain coherent. A few concrete examples may
2977 * help to explain how it works. For shorthand, we use the notation
2978 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2979 * a pair of read and write domain masks.
2980 *
2981 * Case 1: the batch buffer
2982 *
2983 * 1. Allocated
2984 * 2. Written by CPU
2985 * 3. Mapped to GTT
2986 * 4. Read by GPU
2987 * 5. Unmapped from GTT
2988 * 6. Freed
2989 *
2990 * Let's take these a step at a time
2991 *
2992 * 1. Allocated
2993 * Pages allocated from the kernel may still have
2994 * cache contents, so we set them to (CPU, CPU) always.
2995 * 2. Written by CPU (using pwrite)
2996 * The pwrite function calls set_domain (CPU, CPU) and
2997 * this function does nothing (as nothing changes)
2998 * 3. Mapped by GTT
2999 * This function asserts that the object is not
3000 * currently in any GPU-based read or write domains
3001 * 4. Read by GPU
3002 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
3003 * As write_domain is zero, this function adds in the
3004 * current read domains (CPU+COMMAND, 0).
3005 * flush_domains is set to CPU.
3006 * invalidate_domains is set to COMMAND
3007 * clflush is run to get data out of the CPU caches
3008 * then i915_dev_set_domain calls i915_gem_flush to
3009 * emit an MI_FLUSH and drm_agp_chipset_flush
3010 * 5. Unmapped from GTT
3011 * i915_gem_object_unbind calls set_domain (CPU, CPU)
3012 * flush_domains and invalidate_domains end up both zero
3013 * so no flushing/invalidating happens
3014 * 6. Freed
3015 * yay, done
3016 *
3017 * Case 2: The shared render buffer
3018 *
3019 * 1. Allocated
3020 * 2. Mapped to GTT
3021 * 3. Read/written by GPU
3022 * 4. set_domain to (CPU,CPU)
3023 * 5. Read/written by CPU
3024 * 6. Read/written by GPU
3025 *
3026 * 1. Allocated
3027 * Same as last example, (CPU, CPU)
3028 * 2. Mapped to GTT
3029 * Nothing changes (assertions find that it is not in the GPU)
3030 * 3. Read/written by GPU
3031 * execbuffer calls set_domain (RENDER, RENDER)
3032 * flush_domains gets CPU
3033 * invalidate_domains gets GPU
3034 * clflush (obj)
3035 * MI_FLUSH and drm_agp_chipset_flush
3036 * 4. set_domain (CPU, CPU)
3037 * flush_domains gets GPU
3038 * invalidate_domains gets CPU
3039 * wait_rendering (obj) to make sure all drawing is complete.
3040 * This will include an MI_FLUSH to get the data from GPU
3041 * to memory
3042 * clflush (obj) to invalidate the CPU cache
3043 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3044 * 5. Read/written by CPU
3045 * cache lines are loaded and dirtied
3046 * 6. Read written by GPU
3047 * Same as last GPU access
3048 *
3049 * Case 3: The constant buffer
3050 *
3051 * 1. Allocated
3052 * 2. Written by CPU
3053 * 3. Read by GPU
3054 * 4. Updated (written) by CPU again
3055 * 5. Read by GPU
3056 *
3057 * 1. Allocated
3058 * (CPU, CPU)
3059 * 2. Written by CPU
3060 * (CPU, CPU)
3061 * 3. Read by GPU
3062 * (CPU+RENDER, 0)
3063 * flush_domains = CPU
3064 * invalidate_domains = RENDER
3065 * clflush (obj)
3066 * MI_FLUSH
3067 * drm_agp_chipset_flush
3068 * 4. Updated (written) by CPU again
3069 * (CPU, CPU)
3070 * flush_domains = 0 (no previous write domain)
3071 * invalidate_domains = 0 (no new read domains)
3072 * 5. Read by GPU
3073 * (CPU+RENDER, 0)
3074 * flush_domains = CPU
3075 * invalidate_domains = RENDER
3076 * clflush (obj)
3077 * MI_FLUSH
3078 * drm_agp_chipset_flush
3079 */
c0d90829 3080static void
8b0e378a 3081i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
673a394b
EA		 3082{
3083 struct drm_device *dev = obj->dev;
9220434a 3084 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 3085 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 3086 uint32_t invalidate_domains = 0;
3087 uint32_t flush_domains = 0;
1c5d22f7 3088 uint32_t old_read_domains;
e47c68e9 3089
652c393a
JB		 3090 intel_mark_busy(dev, obj);
3091
673a394b
EA		 3092 /*
3093 * If the object isn't moving to a new write domain,
3094 * let the object stay in multiple read domains
3095 */
8b0e378a
EA		 3096 if (obj->pending_write_domain == 0)
3097 obj->pending_read_domains |= obj->read_domains;
673a394b
EA		 3098 else
3099 obj_priv->dirty = 1;
3100
3101 /*
3102 * Flush the current write domain if
3103 * the new read domains don't match. Invalidate
3104 * any read domains which differ from the old
3105 * write domain
3106 */
8b0e378a
EA		 3107 if (obj->write_domain &&
3108 obj->write_domain != obj->pending_read_domains) {
673a394b 3109 flush_domains |= obj->write_domain;
8b0e378a
EA		 3110 invalidate_domains |=
3111 obj->pending_read_domains & ~obj->write_domain;
673a394b
EA		 3112 }
3113 /*
3114 * Invalidate any read caches which may have
3115 * stale data. That is, any new read domains.
3116 */
8b0e378a 3117 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3d2a812a 3118 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
673a394b 3119 i915_gem_clflush_object(obj);
673a394b 3120
1c5d22f7
CW		 3121 old_read_domains = obj->read_domains;
3122
efbeed96
EA		 3123 /* The actual obj->write_domain will be updated with
3124 * pending_write_domain after we emit the accumulated flush for all
3125 * of our domain changes in execbuffers (which clears objects'
3126 * write_domains). So if we have a current write domain that we
3127 * aren't changing, set pending_write_domain to that.
3128 */
3129 if (flush_domains == 0 && obj->pending_write_domain == 0)
3130 obj->pending_write_domain = obj->write_domain;
8b0e378a 3131 obj->read_domains = obj->pending_read_domains;
673a394b
EA		 3132
3133 dev->invalidate_domains |= invalidate_domains;
3134 dev->flush_domains |= flush_domains;
9220434a
CW		 3135 if (obj_priv->ring)
3136 dev_priv->mm.flush_rings |= obj_priv->ring->id;
1c5d22f7
CW		 3137
3138 trace_i915_gem_object_change_domain(obj,
3139 old_read_domains,
3140 obj->write_domain);
673a394b
EA		 3141}
3142
3143/**
e47c68e9 3144 * Moves the object from a partially CPU read to a full one.
673a394b 3145 *
e47c68e9
EA		 3146 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3147 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 3148 */
e47c68e9
EA		 3149static void
3150i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
673a394b 3151{
23010e43 3152 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 3153
e47c68e9
EA		 3154 if (!obj_priv->page_cpu_valid)
3155 return;
3156
3157 /* If we're partially in the CPU read domain, finish moving it in.
3158 */
3159 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3160 int i;
3161
3162 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3163 if (obj_priv->page_cpu_valid[i])
3164 continue;
856fa198 3165 drm_clflush_pages(obj_priv->pages + i, 1);
e47c68e9 3166 }
e47c68e9
EA		 3167 }
3168
3169 /* Free the page_cpu_valid mappings which are now stale, whether
3170 * or not we've got I915_GEM_DOMAIN_CPU.
3171 */
9a298b2a 3172 kfree(obj_priv->page_cpu_valid);
e47c68e9
EA		 3173 obj_priv->page_cpu_valid = NULL;
3174}
3175
3176/**
3177 * Set the CPU read domain on a range of the object.
3178 *
3179 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3180 * not entirely valid. The page_cpu_valid member of the object flags which
3181 * pages have been flushed, and will be respected by
3182 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3183 * of the whole object.
3184 *
3185 * This function returns when the move is complete, including waiting on
3186 * flushes to occur.
3187 */
3188static int
3189i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3190 uint64_t offset, uint64_t size)
3191{
23010e43 3192 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 3193 uint32_t old_read_domains;
e47c68e9 3194 int i, ret;
673a394b 3195
e47c68e9
EA		 3196 if (offset == 0 && size == obj->size)
3197 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 3198
ba3d8d74 3199 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9 3200 if (ret != 0)
6a47baa6 3201 return ret;
e47c68e9
EA		 3202 i915_gem_object_flush_gtt_write_domain(obj);
3203
3204 /* If we're already fully in the CPU read domain, we're done. */
3205 if (obj_priv->page_cpu_valid == NULL &&
3206 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3207 return 0;
673a394b 3208
e47c68e9
EA		 3209 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3210 * newly adding I915_GEM_DOMAIN_CPU
3211 */
673a394b 3212 if (obj_priv->page_cpu_valid == NULL) {
9a298b2a
EA		 3213 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3214 GFP_KERNEL);
e47c68e9
EA		 3215 if (obj_priv->page_cpu_valid == NULL)
3216 return -ENOMEM;
3217 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3218 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
673a394b
EA		 3219
3220 /* Flush the cache on any pages that are still invalid from the CPU's
3221 * perspective.
3222 */
e47c68e9
EA		 3223 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3224 i++) {
673a394b
EA		 3225 if (obj_priv->page_cpu_valid[i])
3226 continue;
3227
856fa198 3228 drm_clflush_pages(obj_priv->pages + i, 1);
673a394b
EA		 3229
3230 obj_priv->page_cpu_valid[i] = 1;
3231 }
3232
e47c68e9
EA		 3233 /* It should now be out of any other write domains, and we can update
3234 * the domain values for our changes.
3235 */
3236 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3237
1c5d22f7 3238 old_read_domains = obj->read_domains;
e47c68e9
EA		 3239 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3240
1c5d22f7
CW		 3241 trace_i915_gem_object_change_domain(obj,
3242 old_read_domains,
3243 obj->write_domain);
3244
673a394b
EA		 3245 return 0;
3246}
3247
673a394b
EA		 3248/**
3249 * Pin an object to the GTT and evaluate the relocations landing in it.
3250 */
3251static int
9af90d19
CW		 3252i915_gem_execbuffer_relocate(struct drm_i915_gem_object *obj,
3253 struct drm_file *file_priv,
3254 struct drm_i915_gem_exec_object2 *entry)
673a394b 3255{
9af90d19 3256 struct drm_device *dev = obj->base.dev;
0839ccb8 3257 drm_i915_private_t *dev_priv = dev->dev_private;
2549d6c2 3258 struct drm_i915_gem_relocation_entry __user *user_relocs;
9af90d19
CW		 3259 struct drm_gem_object *target_obj = NULL;
3260 uint32_t target_handle = 0;
3261 int i, ret = 0;
53640e1d 3262
2549d6c2 3263 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
673a394b 3264 for (i = 0; i < entry->relocation_count; i++) {
2549d6c2 3265 struct drm_i915_gem_relocation_entry reloc;
9af90d19
CW		 3266 uint32_t target_offset;
3267
3268 if (__copy_from_user_inatomic(&reloc,
3269 user_relocs+i,
3270 sizeof(reloc))) {
3271 ret = -EFAULT;
3272 break;
2549d6c2
CW		 3273 }
3274
9af90d19
CW		 3275 if (reloc.target_handle != target_handle) {
3276 drm_gem_object_unreference(target_obj);
3277
3278 target_obj = drm_gem_object_lookup(dev, file_priv,
3279 reloc.target_handle);
3280 if (target_obj == NULL) {
3281 ret = -ENOENT;
3282 break;
3283 }
3284
3285 target_handle = reloc.target_handle;
673a394b 3286 }
9af90d19 3287 target_offset = to_intel_bo(target_obj)->gtt_offset;
673a394b 3288
8542a0bb
CW		 3289#if WATCH_RELOC
3290 DRM_INFO("%s: obj %p offset %08x target %d "
3291 "read %08x write %08x gtt %08x "
3292 "presumed %08x delta %08x\n",
3293 __func__,
3294 obj,
2549d6c2
CW		 3295 (int) reloc.offset,
3296 (int) reloc.target_handle,
3297 (int) reloc.read_domains,
3298 (int) reloc.write_domain,
9af90d19 3299 (int) target_offset,
2549d6c2
CW		 3300 (int) reloc.presumed_offset,
3301 reloc.delta);
8542a0bb
CW		 3302#endif
3303
673a394b
EA		 3304 /* The target buffer should have appeared before us in the
3305 * exec_object list, so it should have a GTT space bound by now.
3306 */
9af90d19 3307 if (target_offset == 0) {
673a394b 3308 DRM_ERROR("No GTT space found for object %d\n",
2549d6c2 3309 reloc.target_handle);
9af90d19
CW		 3310 ret = -EINVAL;
3311 break;
673a394b
EA		 3312 }
3313
8542a0bb 3314 /* Validate that the target is in a valid r/w GPU domain */
2549d6c2 3315 if (reloc.write_domain & (reloc.write_domain - 1)) {
16edd550
DV		 3316 DRM_ERROR("reloc with multiple write domains: "
3317 "obj %p target %d offset %d "
3318 "read %08x write %08x",
2549d6c2
CW		 3319 obj, reloc.target_handle,
3320 (int) reloc.offset,
3321 reloc.read_domains,
3322 reloc.write_domain);
9af90d19
CW		 3323 ret = -EINVAL;
3324 break;
16edd550 3325 }
2549d6c2
CW		 3326 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
3327 reloc.read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 3328 DRM_ERROR("reloc with read/write CPU domains: "
3329 "obj %p target %d offset %d "
3330 "read %08x write %08x",
2549d6c2
CW		 3331 obj, reloc.target_handle,
3332 (int) reloc.offset,
3333 reloc.read_domains,
3334 reloc.write_domain);
9af90d19
CW		 3335 ret = -EINVAL;
3336 break;
e47c68e9 3337 }
2549d6c2
CW		 3338 if (reloc.write_domain && target_obj->pending_write_domain &&
3339 reloc.write_domain != target_obj->pending_write_domain) {
673a394b
EA		 3340 DRM_ERROR("Write domain conflict: "
3341 "obj %p target %d offset %d "
3342 "new %08x old %08x\n",
2549d6c2
CW		 3343 obj, reloc.target_handle,
3344 (int) reloc.offset,
3345 reloc.write_domain,
673a394b 3346 target_obj->pending_write_domain);
9af90d19
CW		 3347 ret = -EINVAL;
3348 break;
673a394b
EA		 3349 }
3350
2549d6c2 3351 target_obj->pending_read_domains |= reloc.read_domains;
9af90d19 3352 target_obj->pending_write_domain = reloc.write_domain;
673a394b
EA		 3353
3354 /* If the relocation already has the right value in it, no
3355 * more work needs to be done.
3356 */
9af90d19 3357 if (target_offset == reloc.presumed_offset)
673a394b 3358 continue;
673a394b 3359
8542a0bb 3360 /* Check that the relocation address is valid... */
9af90d19 3361 if (reloc.offset > obj->base.size - 4) {
8542a0bb
CW		 3362 DRM_ERROR("Relocation beyond object bounds: "
3363 "obj %p target %d offset %d size %d.\n",
2549d6c2 3364 obj, reloc.target_handle,
9af90d19
CW		 3365 (int) reloc.offset, (int) obj->base.size);
3366 ret = -EINVAL;
3367 break;
8542a0bb 3368 }
2549d6c2 3369 if (reloc.offset & 3) {
8542a0bb
CW		 3370 DRM_ERROR("Relocation not 4-byte aligned: "
3371 "obj %p target %d offset %d.\n",
2549d6c2
CW		 3372 obj, reloc.target_handle,
3373 (int) reloc.offset);
9af90d19
CW		 3374 ret = -EINVAL;
3375 break;
8542a0bb
CW		 3376 }
3377
3378 /* and points to somewhere within the target object. */
2549d6c2 3379 if (reloc.delta >= target_obj->size) {
8542a0bb
CW		 3380 DRM_ERROR("Relocation beyond target object bounds: "
3381 "obj %p target %d delta %d size %d.\n",
2549d6c2
CW		 3382 obj, reloc.target_handle,
3383 (int) reloc.delta, (int) target_obj->size);
9af90d19
CW		 3384 ret = -EINVAL;
3385 break;
8542a0bb
CW		 3386 }
3387
9af90d19
CW		 3388 reloc.delta += target_offset;
3389 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
f0c43d9b
CW		 3390 uint32_t page_offset = reloc.offset & ~PAGE_MASK;
3391 char *vaddr;
673a394b 3392
9af90d19 3393 vaddr = kmap_atomic(obj->pages[reloc.offset >> PAGE_SHIFT], KM_USER0);
f0c43d9b
CW		 3394 *(uint32_t *)(vaddr + page_offset) = reloc.delta;
3395 kunmap_atomic(vaddr, KM_USER0);
3396 } else {
3397 uint32_t __iomem *reloc_entry;
3398 void __iomem *reloc_page;
f0c43d9b 3399
9af90d19
CW		 3400 ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3401 if (ret)
3402 break;
f0c43d9b
CW		 3403
3404 /* Map the page containing the relocation we're going to perform. */
9af90d19 3405 reloc.offset += obj->gtt_offset;
f0c43d9b
CW		 3406 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3407 reloc.offset & PAGE_MASK,
3408 KM_USER0);
3409 reloc_entry = (uint32_t __iomem *)
3410 (reloc_page + (reloc.offset & ~PAGE_MASK));
3411 iowrite32(reloc.delta, reloc_entry);
3412 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3413 }
b5dc608c
CW		 3414
3415 /* and update the user's relocation entry */
3416 reloc.presumed_offset = target_offset;
3417 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3418 &reloc.presumed_offset,
3419 sizeof(reloc.presumed_offset))) {
3420 ret = -EFAULT;
3421 break;
3422 }
9af90d19
CW		 3423 }
3424
3425 drm_gem_object_unreference(target_obj);
3426 return ret;
3427}
3428
3429static int
3430i915_gem_execbuffer_pin(struct drm_device *dev,
3431 struct drm_file *file,
3432 struct drm_gem_object **object_list,
3433 struct drm_i915_gem_exec_object2 *exec_list,
3434 int count)
3435{
3436 struct drm_i915_private *dev_priv = dev->dev_private;
3437 int ret, i, retry;
3438
3439 /* attempt to pin all of the buffers into the GTT */
3440 for (retry = 0; retry < 2; retry++) {
3441 ret = 0;
3442 for (i = 0; i < count; i++) {
3443 struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3444 struct drm_i915_gem_object *obj= to_intel_bo(object_list[i]);
3445 bool need_fence =
3446 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3447 obj->tiling_mode != I915_TILING_NONE;
3448
3449 /* Check fence reg constraints and rebind if necessary */
3450 if (need_fence &&
3451 !i915_gem_object_fence_offset_ok(&obj->base,
3452 obj->tiling_mode)) {
3453 ret = i915_gem_object_unbind(&obj->base);
3454 if (ret)
3455 break;
3456 }
3457
3458 ret = i915_gem_object_pin(&obj->base, entry->alignment);
3459 if (ret)
3460 break;
3461
3462 /*
3463 * Pre-965 chips need a fence register set up in order
3464 * to properly handle blits to/from tiled surfaces.
3465 */
3466 if (need_fence) {
3467 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3468 if (ret) {
3469 i915_gem_object_unpin(&obj->base);
3470 break;
3471 }
3472
3473 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3474 }
3475
3476 entry->offset = obj->gtt_offset;
3477 }
3478
3479 while (i--)
3480 i915_gem_object_unpin(object_list[i]);
3481
3482 if (ret == 0)
3483 break;
673a394b 3484
9af90d19
CW		 3485 if (ret != -ENOSPC || retry)
3486 return ret;
3487
3488 ret = i915_gem_evict_everything(dev);
3489 if (ret)
3490 return ret;
673a394b
EA		 3491 }
3492
673a394b
EA		 3493 return 0;
3494}
3495
673a394b
EA		 3496/* Throttle our rendering by waiting until the ring has completed our requests
3497 * emitted over 20 msec ago.
3498 *
b962442e
EA		 3499 * Note that if we were to use the current jiffies each time around the loop,
3500 * we wouldn't escape the function with any frames outstanding if the time to
3501 * render a frame was over 20ms.
3502 *
673a394b
EA		 3503 * This should get us reasonable parallelism between CPU and GPU but also
3504 * relatively low latency when blocking on a particular request to finish.
3505 */
3506static int
f787a5f5 3507i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
673a394b 3508{
f787a5f5
CW		 3509 struct drm_i915_private *dev_priv = dev->dev_private;
3510 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e 3511 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
f787a5f5
CW		 3512 struct drm_i915_gem_request *request;
3513 struct intel_ring_buffer *ring = NULL;
3514 u32 seqno = 0;
3515 int ret;
673a394b 3516
1c25595f 3517 spin_lock(&file_priv->mm.lock);
f787a5f5 3518 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
b962442e
EA		 3519 if (time_after_eq(request->emitted_jiffies, recent_enough))
3520 break;
3521
f787a5f5
CW		 3522 ring = request->ring;
3523 seqno = request->seqno;
b962442e 3524 }
1c25595f 3525 spin_unlock(&file_priv->mm.lock);
f787a5f5
CW		 3526
3527 if (seqno == 0)
3528 return 0;
3529
3530 ret = 0;
3531 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
3532 /* And wait for the seqno passing without holding any locks and
3533 * causing extra latency for others. This is safe as the irq
3534 * generation is designed to be run atomically and so is
3535 * lockless.
3536 */
3537 ring->user_irq_get(dev, ring);
3538 ret = wait_event_interruptible(ring->irq_queue,
3539 i915_seqno_passed(ring->get_seqno(dev, ring), seqno)
3540 || atomic_read(&dev_priv->mm.wedged));
3541 ring->user_irq_put(dev, ring);
3542
3543 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3544 ret = -EIO;
3545 }
3546
3547 if (ret == 0)
3548 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
b962442e 3549
673a394b
EA		 3550 return ret;
3551}
3552
40a5f0de 3553static int
2549d6c2
CW		 3554i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3555 uint64_t exec_offset)
40a5f0de 3556{
2549d6c2 3557 uint32_t exec_start, exec_len;
40a5f0de 3558
2549d6c2
CW		 3559 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3560 exec_len = (uint32_t) exec->batch_len;
40a5f0de 3561
2549d6c2
CW		 3562 if ((exec_start | exec_len) & 0x7)
3563 return -EINVAL;
40a5f0de 3564
2549d6c2
CW		 3565 if (!exec_start)
3566 return -EINVAL;
40a5f0de 3567
2bc43b5c 3568 return 0;
40a5f0de
EA		 3569}
3570
3571static int
2549d6c2
CW		 3572validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3573 int count)
40a5f0de 3574{
2549d6c2 3575 int i;
40a5f0de 3576
2549d6c2
CW		 3577 for (i = 0; i < count; i++) {
3578 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3579 size_t length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry);
2bc43b5c 3580
2549d6c2
CW		 3581 if (!access_ok(VERIFY_READ, ptr, length))
3582 return -EFAULT;
40a5f0de 3583
b5dc608c
CW		 3584 /* we may also need to update the presumed offsets */
3585 if (!access_ok(VERIFY_WRITE, ptr, length))
3586 return -EFAULT;
3587
2549d6c2
CW		 3588 if (fault_in_pages_readable(ptr, length))
3589 return -EFAULT;
40a5f0de
EA		 3590 }
3591
83d60795
CW		 3592 return 0;
3593}
3594
8dc5d147 3595static int
76446cac 3596i915_gem_do_execbuffer(struct drm_device *dev, void *data,
9af90d19 3597 struct drm_file *file,
76446cac
JB		 3598 struct drm_i915_gem_execbuffer2 *args,
3599 struct drm_i915_gem_exec_object2 *exec_list)
673a394b
EA		 3600{
3601 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3602 struct drm_gem_object **object_list = NULL;
3603 struct drm_gem_object *batch_obj;
b70d11da 3604 struct drm_i915_gem_object *obj_priv;
201361a5 3605 struct drm_clip_rect *cliprects = NULL;
8dc5d147 3606 struct drm_i915_gem_request *request = NULL;
9af90d19 3607 int ret, i, flips;
673a394b 3608 uint64_t exec_offset;
673a394b 3609
852835f3
ZN		 3610 struct intel_ring_buffer *ring = NULL;
3611
30dbf0c0
CW		 3612 ret = i915_gem_check_is_wedged(dev);
3613 if (ret)
3614 return ret;
3615
2549d6c2
CW		 3616 ret = validate_exec_list(exec_list, args->buffer_count);
3617 if (ret)
3618 return ret;
3619
673a394b
EA		 3620#if WATCH_EXEC
3621 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3622 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3623#endif
549f7365
CW		 3624 switch (args->flags & I915_EXEC_RING_MASK) {
3625 case I915_EXEC_DEFAULT:
3626 case I915_EXEC_RENDER:
3627 ring = &dev_priv->render_ring;
3628 break;
3629 case I915_EXEC_BSD:
d1b851fc 3630 if (!HAS_BSD(dev)) {
549f7365 3631 DRM_ERROR("execbuf with invalid ring (BSD)\n");
d1b851fc
ZN		 3632 return -EINVAL;
3633 }
3634 ring = &dev_priv->bsd_ring;
549f7365
CW		 3635 break;
3636 case I915_EXEC_BLT:
3637 if (!HAS_BLT(dev)) {
3638 DRM_ERROR("execbuf with invalid ring (BLT)\n");
3639 return -EINVAL;
3640 }
3641 ring = &dev_priv->blt_ring;
3642 break;
3643 default:
3644 DRM_ERROR("execbuf with unknown ring: %d\n",
3645 (int)(args->flags & I915_EXEC_RING_MASK));
3646 return -EINVAL;
d1b851fc
ZN		 3647 }
3648
4f481ed2
EA		 3649 if (args->buffer_count < 1) {
3650 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3651 return -EINVAL;
3652 }
c8e0f93a 3653 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
76446cac
JB		 3654 if (object_list == NULL) {
3655 DRM_ERROR("Failed to allocate object list for %d buffers\n",
673a394b
EA		 3656 args->buffer_count);
3657 ret = -ENOMEM;
3658 goto pre_mutex_err;
3659 }
673a394b 3660
201361a5 3661 if (args->num_cliprects != 0) {
9a298b2a
EA		 3662 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3663 GFP_KERNEL);
a40e8d31
OA		 3664 if (cliprects == NULL) {
3665 ret = -ENOMEM;
201361a5 3666 goto pre_mutex_err;
a40e8d31 3667 }
201361a5
EA		 3668
3669 ret = copy_from_user(cliprects,
3670 (struct drm_clip_rect __user *)
3671 (uintptr_t) args->cliprects_ptr,
3672 sizeof(*cliprects) * args->num_cliprects);
3673 if (ret != 0) {
3674 DRM_ERROR("copy %d cliprects failed: %d\n",
3675 args->num_cliprects, ret);
c877cdce 3676 ret = -EFAULT;
201361a5
EA		 3677 goto pre_mutex_err;
3678 }
3679 }
3680
8dc5d147
CW		 3681 request = kzalloc(sizeof(*request), GFP_KERNEL);
3682 if (request == NULL) {
3683 ret = -ENOMEM;
3684 goto pre_mutex_err;
3685 }
3686
76c1dec1
CW		 3687 ret = i915_mutex_lock_interruptible(dev);
3688 if (ret)
3689 goto pre_mutex_err;
673a394b 3690
673a394b 3691 if (dev_priv->mm.suspended) {
673a394b 3692 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3693 ret = -EBUSY;
3694 goto pre_mutex_err;
673a394b
EA		 3695 }
3696
ac94a962 3697 /* Look up object handles */
673a394b 3698 for (i = 0; i < args->buffer_count; i++) {
9af90d19 3699 object_list[i] = drm_gem_object_lookup(dev, file,
673a394b
EA		 3700 exec_list[i].handle);
3701 if (object_list[i] == NULL) {
3702 DRM_ERROR("Invalid object handle %d at index %d\n",
3703 exec_list[i].handle, i);
0ce907f8
CW		 3704 /* prevent error path from reading uninitialized data */
3705 args->buffer_count = i + 1;
bf79cb91 3706 ret = -ENOENT;
673a394b
EA		 3707 goto err;
3708 }
b70d11da 3709
23010e43 3710 obj_priv = to_intel_bo(object_list[i]);
b70d11da
KH		 3711 if (obj_priv->in_execbuffer) {
3712 DRM_ERROR("Object %p appears more than once in object list\n",
3713 object_list[i]);
0ce907f8
CW		 3714 /* prevent error path from reading uninitialized data */
3715 args->buffer_count = i + 1;
bf79cb91 3716 ret = -EINVAL;
b70d11da
KH		 3717 goto err;
3718 }
3719 obj_priv->in_execbuffer = true;
ac94a962 3720 }
673a394b 3721
9af90d19
CW		 3722 /* Move the objects en-masse into the GTT, evicting if necessary. */
3723 ret = i915_gem_execbuffer_pin(dev, file,
3724 object_list, exec_list,
3725 args->buffer_count);
3726 if (ret)
3727 goto err;
ac94a962 3728
9af90d19
CW		 3729 /* The objects are in their final locations, apply the relocations. */
3730 for (i = 0; i < args->buffer_count; i++) {
3731 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3732 obj->base.pending_read_domains = 0;
3733 obj->base.pending_write_domain = 0;
3734 ret = i915_gem_execbuffer_relocate(obj, file, &exec_list[i]);
3735 if (ret)
ac94a962 3736 goto err;
673a394b
EA		 3737 }
3738
3739 /* Set the pending read domains for the batch buffer to COMMAND */
3740 batch_obj = object_list[args->buffer_count-1];
5f26a2c7
CW		 3741 if (batch_obj->pending_write_domain) {
3742 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3743 ret = -EINVAL;
3744 goto err;
3745 }
3746 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
673a394b 3747
9af90d19
CW		 3748 /* Sanity check the batch buffer */
3749 exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3750 ret = i915_gem_check_execbuffer(args, exec_offset);
83d60795
CW		 3751 if (ret != 0) {
3752 DRM_ERROR("execbuf with invalid offset/length\n");
3753 goto err;
3754 }
3755
646f0f6e
KP		 3756 /* Zero the global flush/invalidate flags. These
3757 * will be modified as new domains are computed
3758 * for each object
3759 */
3760 dev->invalidate_domains = 0;
3761 dev->flush_domains = 0;
9220434a 3762 dev_priv->mm.flush_rings = 0;
646f0f6e 3763
673a394b
EA		 3764 for (i = 0; i < args->buffer_count; i++) {
3765 struct drm_gem_object *obj = object_list[i];
673a394b 3766
646f0f6e 3767 /* Compute new gpu domains and update invalidate/flush */
8b0e378a 3768 i915_gem_object_set_to_gpu_domain(obj);
673a394b
EA		 3769 }
3770
646f0f6e
KP		 3771 if (dev->invalidate_domains | dev->flush_domains) {
3772#if WATCH_EXEC
3773 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3774 __func__,
3775 dev->invalidate_domains,
3776 dev->flush_domains);
3777#endif
9af90d19 3778 i915_gem_flush(dev, file,
646f0f6e 3779 dev->invalidate_domains,
9220434a
CW		 3780 dev->flush_domains,
3781 dev_priv->mm.flush_rings);
a6910434
DV		 3782 }
3783
efbeed96
EA		 3784 for (i = 0; i < args->buffer_count; i++) {
3785 struct drm_gem_object *obj = object_list[i];
23010e43 3786 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 3787 uint32_t old_write_domain = obj->write_domain;
efbeed96
EA		 3788
3789 obj->write_domain = obj->pending_write_domain;
99fcb766
DV		 3790 if (obj->write_domain)
3791 list_move_tail(&obj_priv->gpu_write_list,
3792 &dev_priv->mm.gpu_write_list);
99fcb766 3793
1c5d22f7
CW		 3794 trace_i915_gem_object_change_domain(obj,
3795 obj->read_domains,
3796 old_write_domain);
efbeed96
EA		 3797 }
3798
673a394b
EA		 3799#if WATCH_COHERENCY
3800 for (i = 0; i < args->buffer_count; i++) {
3801 i915_gem_object_check_coherency(object_list[i],
3802 exec_list[i].handle);
3803 }
3804#endif
3805
673a394b 3806#if WATCH_EXEC
6911a9b8 3807 i915_gem_dump_object(batch_obj,
673a394b
EA		 3808 args->batch_len,
3809 __func__,
3810 ~0);
3811#endif
3812
e59f2bac
CW		 3813 /* Check for any pending flips. As we only maintain a flip queue depth
3814 * of 1, we can simply insert a WAIT for the next display flip prior
3815 * to executing the batch and avoid stalling the CPU.
3816 */
3817 flips = 0;
3818 for (i = 0; i < args->buffer_count; i++) {
3819 if (object_list[i]->write_domain)
3820 flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3821 }
3822 if (flips) {
3823 int plane, flip_mask;
3824
3825 for (plane = 0; flips >> plane; plane++) {
3826 if (((flips >> plane) & 1) == 0)
3827 continue;
3828
3829 if (plane)
3830 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3831 else
3832 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3833
3834 intel_ring_begin(dev, ring, 2);
3835 intel_ring_emit(dev, ring,
3836 MI_WAIT_FOR_EVENT | flip_mask);
3837 intel_ring_emit(dev, ring, MI_NOOP);
3838 intel_ring_advance(dev, ring);
3839 }
3840 }
3841
673a394b 3842 /* Exec the batchbuffer */
852835f3 3843 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
e59f2bac 3844 cliprects, exec_offset);
673a394b
EA		 3845 if (ret) {
3846 DRM_ERROR("dispatch failed %d\n", ret);
3847 goto err;
3848 }
3849
3850 /*
3851 * Ensure that the commands in the batch buffer are
3852 * finished before the interrupt fires
3853 */
8a1a49f9 3854 i915_retire_commands(dev, ring);
673a394b 3855
617dbe27
DV		 3856 for (i = 0; i < args->buffer_count; i++) {
3857 struct drm_gem_object *obj = object_list[i];
3858 obj_priv = to_intel_bo(obj);
3859
3860 i915_gem_object_move_to_active(obj, ring);
617dbe27 3861 }
a56ba56c 3862
9af90d19 3863 i915_add_request(dev, file, request, ring);
8dc5d147 3864 request = NULL;
673a394b 3865
673a394b 3866err:
b70d11da
KH		 3867 for (i = 0; i < args->buffer_count; i++) {
3868 if (object_list[i]) {
23010e43 3869 obj_priv = to_intel_bo(object_list[i]);
b70d11da
KH		 3870 obj_priv->in_execbuffer = false;
3871 }
aad87dff 3872 drm_gem_object_unreference(object_list[i]);
b70d11da 3873 }
673a394b 3874
673a394b
EA		 3875 mutex_unlock(&dev->struct_mutex);
3876
93533c29 3877pre_mutex_err:
8e7d2b2c 3878 drm_free_large(object_list);
9a298b2a 3879 kfree(cliprects);
8dc5d147 3880 kfree(request);
673a394b
EA		 3881
3882 return ret;
3883}
3884
76446cac
JB		 3885/*
3886 * Legacy execbuffer just creates an exec2 list from the original exec object
3887 * list array and passes it to the real function.
3888 */
3889int
3890i915_gem_execbuffer(struct drm_device *dev, void *data,
3891 struct drm_file *file_priv)
3892{
3893 struct drm_i915_gem_execbuffer *args = data;
3894 struct drm_i915_gem_execbuffer2 exec2;
3895 struct drm_i915_gem_exec_object *exec_list = NULL;
3896 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3897 int ret, i;
3898
3899#if WATCH_EXEC
3900 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3901 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3902#endif
3903
3904 if (args->buffer_count < 1) {
3905 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3906 return -EINVAL;
3907 }
3908
3909 /* Copy in the exec list from userland */
3910 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3911 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3912 if (exec_list == NULL || exec2_list == NULL) {
3913 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3914 args->buffer_count);
3915 drm_free_large(exec_list);
3916 drm_free_large(exec2_list);
3917 return -ENOMEM;
3918 }
3919 ret = copy_from_user(exec_list,
3920 (struct drm_i915_relocation_entry __user *)
3921 (uintptr_t) args->buffers_ptr,
3922 sizeof(*exec_list) * args->buffer_count);
3923 if (ret != 0) {
3924 DRM_ERROR("copy %d exec entries failed %d\n",
3925 args->buffer_count, ret);
3926 drm_free_large(exec_list);
3927 drm_free_large(exec2_list);
3928 return -EFAULT;
3929 }
3930
3931 for (i = 0; i < args->buffer_count; i++) {
3932 exec2_list[i].handle = exec_list[i].handle;
3933 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3934 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3935 exec2_list[i].alignment = exec_list[i].alignment;
3936 exec2_list[i].offset = exec_list[i].offset;
a6c45cf0 3937 if (INTEL_INFO(dev)->gen < 4)
76446cac
JB		 3938 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3939 else
3940 exec2_list[i].flags = 0;
3941 }
3942
3943 exec2.buffers_ptr = args->buffers_ptr;
3944 exec2.buffer_count = args->buffer_count;
3945 exec2.batch_start_offset = args->batch_start_offset;
3946 exec2.batch_len = args->batch_len;
3947 exec2.DR1 = args->DR1;
3948 exec2.DR4 = args->DR4;
3949 exec2.num_cliprects = args->num_cliprects;
3950 exec2.cliprects_ptr = args->cliprects_ptr;
852835f3 3951 exec2.flags = I915_EXEC_RENDER;
76446cac
JB		 3952
3953 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3954 if (!ret) {
3955 /* Copy the new buffer offsets back to the user's exec list. */
3956 for (i = 0; i < args->buffer_count; i++)
3957 exec_list[i].offset = exec2_list[i].offset;
3958 /* ... and back out to userspace */
3959 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3960 (uintptr_t) args->buffers_ptr,
3961 exec_list,
3962 sizeof(*exec_list) * args->buffer_count);
3963 if (ret) {
3964 ret = -EFAULT;
3965 DRM_ERROR("failed to copy %d exec entries "
3966 "back to user (%d)\n",
3967 args->buffer_count, ret);
3968 }
76446cac
JB		 3969 }
3970
3971 drm_free_large(exec_list);
3972 drm_free_large(exec2_list);
3973 return ret;
3974}
3975
3976int
3977i915_gem_execbuffer2(struct drm_device *dev, void *data,
3978 struct drm_file *file_priv)
3979{
3980 struct drm_i915_gem_execbuffer2 *args = data;
3981 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3982 int ret;
3983
3984#if WATCH_EXEC
3985 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3986 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3987#endif
3988
3989 if (args->buffer_count < 1) {
3990 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
3991 return -EINVAL;
3992 }
3993
3994 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3995 if (exec2_list == NULL) {
3996 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3997 args->buffer_count);
3998 return -ENOMEM;
3999 }
4000 ret = copy_from_user(exec2_list,
4001 (struct drm_i915_relocation_entry __user *)
4002 (uintptr_t) args->buffers_ptr,
4003 sizeof(*exec2_list) * args->buffer_count);
4004 if (ret != 0) {
4005 DRM_ERROR("copy %d exec entries failed %d\n",
4006 args->buffer_count, ret);
4007 drm_free_large(exec2_list);
4008 return -EFAULT;
4009 }
4010
4011 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4012 if (!ret) {
4013 /* Copy the new buffer offsets back to the user's exec list. */
4014 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4015 (uintptr_t) args->buffers_ptr,
4016 exec2_list,
4017 sizeof(*exec2_list) * args->buffer_count);
4018 if (ret) {
4019 ret = -EFAULT;
4020 DRM_ERROR("failed to copy %d exec entries "
4021 "back to user (%d)\n",
4022 args->buffer_count, ret);
4023 }
4024 }
4025
4026 drm_free_large(exec2_list);
4027 return ret;
4028}
4029
673a394b
EA		 4030int
4031i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4032{
4033 struct drm_device *dev = obj->dev;
f13d3f73 4034 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 4035 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 4036 int ret;
4037
778c3544 4038 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
23bc5982 4039 WARN_ON(i915_verify_lists(dev));
ac0c6b5a
CW		 4040
4041 if (obj_priv->gtt_space != NULL) {
4042 if (alignment == 0)
4043 alignment = i915_gem_get_gtt_alignment(obj);
4044 if (obj_priv->gtt_offset & (alignment - 1)) {
ae7d49d8
CW		 4045 WARN(obj_priv->pin_count,
4046 "bo is already pinned with incorrect alignment:"
4047 " offset=%x, req.alignment=%x\n",
4048 obj_priv->gtt_offset, alignment);
ac0c6b5a
CW		 4049 ret = i915_gem_object_unbind(obj);
4050 if (ret)
4051 return ret;
4052 }
4053 }
4054
673a394b
EA		 4055 if (obj_priv->gtt_space == NULL) {
4056 ret = i915_gem_object_bind_to_gtt(obj, alignment);
9731129c 4057 if (ret)
673a394b 4058 return ret;
22c344e9 4059 }
76446cac 4060
673a394b
EA		 4061 obj_priv->pin_count++;
4062
4063 /* If the object is not active and not pending a flush,
4064 * remove it from the inactive list
4065 */
4066 if (obj_priv->pin_count == 1) {
73aa808f 4067 i915_gem_info_add_pin(dev_priv, obj->size);
f13d3f73 4068 if (!obj_priv->active)
69dc4987 4069 list_move_tail(&obj_priv->mm_list,
f13d3f73 4070 &dev_priv->mm.pinned_list);
673a394b 4071 }
673a394b 4072
23bc5982 4073 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 4074 return 0;
4075}
4076
4077void
4078i915_gem_object_unpin(struct drm_gem_object *obj)
4079{
4080 struct drm_device *dev = obj->dev;
4081 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 4082 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 4083
23bc5982 4084 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 4085 obj_priv->pin_count--;
4086 BUG_ON(obj_priv->pin_count < 0);
4087 BUG_ON(obj_priv->gtt_space == NULL);
4088
4089 /* If the object is no longer pinned, and is
4090 * neither active nor being flushed, then stick it on
4091 * the inactive list
4092 */
4093 if (obj_priv->pin_count == 0) {
f13d3f73 4094 if (!obj_priv->active)
69dc4987 4095 list_move_tail(&obj_priv->mm_list,
673a394b 4096 &dev_priv->mm.inactive_list);
73aa808f 4097 i915_gem_info_remove_pin(dev_priv, obj->size);
673a394b 4098 }
23bc5982 4099 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 4100}
4101
4102int
4103i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4104 struct drm_file *file_priv)
4105{
4106 struct drm_i915_gem_pin *args = data;
4107 struct drm_gem_object *obj;
4108 struct drm_i915_gem_object *obj_priv;
4109 int ret;
4110
1d7cfea1
CW		 4111 ret = i915_mutex_lock_interruptible(dev);
4112 if (ret)
4113 return ret;
4114
673a394b
EA		 4115 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4116 if (obj == NULL) {
1d7cfea1
CW		 4117 ret = -ENOENT;
4118 goto unlock;
673a394b 4119 }
23010e43 4120 obj_priv = to_intel_bo(obj);
673a394b 4121
bb6baf76
CW		 4122 if (obj_priv->madv != I915_MADV_WILLNEED) {
4123 DRM_ERROR("Attempting to pin a purgeable buffer\n");
1d7cfea1
CW		 4124 ret = -EINVAL;
4125 goto out;
3ef94daa
CW		 4126 }
4127
79e53945
JB		 4128 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4129 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4130 args->handle);
1d7cfea1
CW		 4131 ret = -EINVAL;
4132 goto out;
79e53945
JB		 4133 }
4134
4135 obj_priv->user_pin_count++;
4136 obj_priv->pin_filp = file_priv;
4137 if (obj_priv->user_pin_count == 1) {
4138 ret = i915_gem_object_pin(obj, args->alignment);
1d7cfea1
CW		 4139 if (ret)
4140 goto out;
673a394b
EA		 4141 }
4142
4143 /* XXX - flush the CPU caches for pinned objects
4144 * as the X server doesn't manage domains yet
4145 */
e47c68e9 4146 i915_gem_object_flush_cpu_write_domain(obj);
673a394b 4147 args->offset = obj_priv->gtt_offset;
1d7cfea1 4148out:
673a394b 4149 drm_gem_object_unreference(obj);
1d7cfea1 4150unlock:
673a394b 4151 mutex_unlock(&dev->struct_mutex);
1d7cfea1 4152 return ret;
673a394b
EA		 4153}
4154
4155int
4156i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4157 struct drm_file *file_priv)
4158{
4159 struct drm_i915_gem_pin *args = data;
4160 struct drm_gem_object *obj;
79e53945 4161 struct drm_i915_gem_object *obj_priv;
76c1dec1 4162 int ret;
673a394b 4163
1d7cfea1
CW		 4164 ret = i915_mutex_lock_interruptible(dev);
4165 if (ret)
4166 return ret;
4167
673a394b
EA		 4168 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4169 if (obj == NULL) {
1d7cfea1
CW		 4170 ret = -ENOENT;
4171 goto unlock;
673a394b 4172 }
23010e43 4173 obj_priv = to_intel_bo(obj);
76c1dec1 4174
79e53945
JB		 4175 if (obj_priv->pin_filp != file_priv) {
4176 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4177 args->handle);
1d7cfea1
CW		 4178 ret = -EINVAL;
4179 goto out;
79e53945
JB		 4180 }
4181 obj_priv->user_pin_count--;
4182 if (obj_priv->user_pin_count == 0) {
4183 obj_priv->pin_filp = NULL;
4184 i915_gem_object_unpin(obj);
4185 }
673a394b 4186
1d7cfea1 4187out:
673a394b 4188 drm_gem_object_unreference(obj);
1d7cfea1 4189unlock:
673a394b 4190 mutex_unlock(&dev->struct_mutex);
1d7cfea1 4191 return ret;
673a394b
EA		 4192}
4193
4194int
4195i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4196 struct drm_file *file_priv)
4197{
4198 struct drm_i915_gem_busy *args = data;
4199 struct drm_gem_object *obj;
4200 struct drm_i915_gem_object *obj_priv;
30dbf0c0
CW		 4201 int ret;
4202
76c1dec1 4203 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 4204 if (ret)
76c1dec1 4205 return ret;
1d7cfea1
CW		 4206
4207 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4208 if (obj == NULL) {
4209 ret = -ENOENT;
4210 goto unlock;
30dbf0c0 4211 }
1d7cfea1 4212 obj_priv = to_intel_bo(obj);
30dbf0c0 4213
0be555b6
CW		 4214 /* Count all active objects as busy, even if they are currently not used
4215 * by the gpu. Users of this interface expect objects to eventually
4216 * become non-busy without any further actions, therefore emit any
4217 * necessary flushes here.
c4de0a5d 4218 */
0be555b6
CW		 4219 args->busy = obj_priv->active;
4220 if (args->busy) {
4221 /* Unconditionally flush objects, even when the gpu still uses this
4222 * object. Userspace calling this function indicates that it wants to
4223 * use this buffer rather sooner than later, so issuing the required
4224 * flush earlier is beneficial.
4225 */
c78ec30b
CW		 4226 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4227 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 4228 obj_priv->ring,
4229 0, obj->write_domain);
0be555b6
CW		 4230
4231 /* Update the active list for the hardware's current position.
4232 * Otherwise this only updates on a delayed timer or when irqs
4233 * are actually unmasked, and our working set ends up being
4234 * larger than required.
4235 */
4236 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4237
4238 args->busy = obj_priv->active;
4239 }
673a394b
EA		 4240
4241 drm_gem_object_unreference(obj);
1d7cfea1 4242unlock:
673a394b 4243 mutex_unlock(&dev->struct_mutex);
1d7cfea1 4244 return ret;
673a394b
EA		 4245}
4246
4247int
4248i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4249 struct drm_file *file_priv)
4250{
4251 return i915_gem_ring_throttle(dev, file_priv);
4252}
4253
3ef94daa
CW		 4254int
4255i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4256 struct drm_file *file_priv)
4257{
4258 struct drm_i915_gem_madvise *args = data;
4259 struct drm_gem_object *obj;
4260 struct drm_i915_gem_object *obj_priv;
76c1dec1 4261 int ret;
3ef94daa
CW		 4262
4263 switch (args->madv) {
4264 case I915_MADV_DONTNEED:
4265 case I915_MADV_WILLNEED:
4266 break;
4267 default:
4268 return -EINVAL;
4269 }
4270
1d7cfea1
CW		 4271 ret = i915_mutex_lock_interruptible(dev);
4272 if (ret)
4273 return ret;
4274
3ef94daa
CW		 4275 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4276 if (obj == NULL) {
1d7cfea1
CW		 4277 ret = -ENOENT;
4278 goto unlock;
3ef94daa 4279 }
23010e43 4280 obj_priv = to_intel_bo(obj);
3ef94daa
CW		 4281
4282 if (obj_priv->pin_count) {
1d7cfea1
CW		 4283 ret = -EINVAL;
4284 goto out;
3ef94daa
CW		 4285 }
4286
bb6baf76
CW		 4287 if (obj_priv->madv != __I915_MADV_PURGED)
4288 obj_priv->madv = args->madv;
3ef94daa 4289
2d7ef395
CW		 4290 /* if the object is no longer bound, discard its backing storage */
4291 if (i915_gem_object_is_purgeable(obj_priv) &&
4292 obj_priv->gtt_space == NULL)
4293 i915_gem_object_truncate(obj);
4294
bb6baf76
CW		 4295 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4296
1d7cfea1 4297out:
3ef94daa 4298 drm_gem_object_unreference(obj);
1d7cfea1 4299unlock:
3ef94daa 4300 mutex_unlock(&dev->struct_mutex);
1d7cfea1 4301 return ret;
3ef94daa
CW		 4302}
4303
ac52bc56
DV		 4304struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4305 size_t size)
4306{
73aa808f 4307 struct drm_i915_private *dev_priv = dev->dev_private;
c397b908 4308 struct drm_i915_gem_object *obj;
ac52bc56 4309
c397b908
DV		 4310 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4311 if (obj == NULL)
4312 return NULL;
673a394b 4313
c397b908
DV		 4314 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4315 kfree(obj);
4316 return NULL;
4317 }
673a394b 4318
73aa808f
CW		 4319 i915_gem_info_add_obj(dev_priv, size);
4320
c397b908
DV		 4321 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4322 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
673a394b 4323
c397b908 4324 obj->agp_type = AGP_USER_MEMORY;
62b8b215 4325 obj->base.driver_private = NULL;
c397b908 4326 obj->fence_reg = I915_FENCE_REG_NONE;
69dc4987
CW		 4327 INIT_LIST_HEAD(&obj->mm_list);
4328 INIT_LIST_HEAD(&obj->ring_list);
c397b908 4329 INIT_LIST_HEAD(&obj->gpu_write_list);
c397b908 4330 obj->madv = I915_MADV_WILLNEED;
de151cf6 4331
c397b908
DV		 4332 return &obj->base;
4333}
4334
4335int i915_gem_init_object(struct drm_gem_object *obj)
4336{
4337 BUG();
de151cf6 4338
673a394b
EA		 4339 return 0;
4340}
4341
be72615b 4342static void i915_gem_free_object_tail(struct drm_gem_object *obj)
673a394b 4343{
de151cf6 4344 struct drm_device *dev = obj->dev;
be72615b 4345 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 4346 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
be72615b 4347 int ret;
673a394b 4348
be72615b
CW		 4349 ret = i915_gem_object_unbind(obj);
4350 if (ret == -ERESTARTSYS) {
69dc4987 4351 list_move(&obj_priv->mm_list,
be72615b
CW		 4352 &dev_priv->mm.deferred_free_list);
4353 return;
4354 }
673a394b 4355
7e616158
CW		 4356 if (obj_priv->mmap_offset)
4357 i915_gem_free_mmap_offset(obj);
de151cf6 4358
c397b908 4359 drm_gem_object_release(obj);
73aa808f 4360 i915_gem_info_remove_obj(dev_priv, obj->size);
c397b908 4361
9a298b2a 4362 kfree(obj_priv->page_cpu_valid);
280b713b 4363 kfree(obj_priv->bit_17);
c397b908 4364 kfree(obj_priv);
673a394b
EA		 4365}
4366
be72615b
CW		 4367void i915_gem_free_object(struct drm_gem_object *obj)
4368{
4369 struct drm_device *dev = obj->dev;
4370 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4371
4372 trace_i915_gem_object_destroy(obj);
4373
4374 while (obj_priv->pin_count > 0)
4375 i915_gem_object_unpin(obj);
4376
4377 if (obj_priv->phys_obj)
4378 i915_gem_detach_phys_object(dev, obj);
4379
4380 i915_gem_free_object_tail(obj);
4381}
4382
29105ccc
CW		 4383int
4384i915_gem_idle(struct drm_device *dev)
4385{
4386 drm_i915_private_t *dev_priv = dev->dev_private;
4387 int ret;
28dfe52a 4388
29105ccc 4389 mutex_lock(&dev->struct_mutex);
1c5d22f7 4390
87acb0a5 4391 if (dev_priv->mm.suspended) {
29105ccc
CW		 4392 mutex_unlock(&dev->struct_mutex);
4393 return 0;
28dfe52a
EA		 4394 }
4395
29105ccc 4396 ret = i915_gpu_idle(dev);
6dbe2772
KP		 4397 if (ret) {
4398 mutex_unlock(&dev->struct_mutex);
673a394b 4399 return ret;
6dbe2772 4400 }
673a394b 4401
29105ccc
CW		 4402 /* Under UMS, be paranoid and evict. */
4403 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
b47eb4a2 4404 ret = i915_gem_evict_inactive(dev);
29105ccc
CW		 4405 if (ret) {
4406 mutex_unlock(&dev->struct_mutex);
4407 return ret;
4408 }
4409 }
4410
4411 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4412 * We need to replace this with a semaphore, or something.
4413 * And not confound mm.suspended!
4414 */
4415 dev_priv->mm.suspended = 1;
bc0c7f14 4416 del_timer_sync(&dev_priv->hangcheck_timer);
29105ccc
CW		 4417
4418 i915_kernel_lost_context(dev);
6dbe2772 4419 i915_gem_cleanup_ringbuffer(dev);
29105ccc 4420
6dbe2772
KP		 4421 mutex_unlock(&dev->struct_mutex);
4422
29105ccc
CW		 4423 /* Cancel the retire work handler, which should be idle now. */
4424 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4425
673a394b
EA		 4426 return 0;
4427}
4428
e552eb70
JB		 4429/*
4430 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4431 * over cache flushing.
4432 */
8187a2b7 4433static int
e552eb70
JB		 4434i915_gem_init_pipe_control(struct drm_device *dev)
4435{
4436 drm_i915_private_t *dev_priv = dev->dev_private;
4437 struct drm_gem_object *obj;
4438 struct drm_i915_gem_object *obj_priv;
4439 int ret;
4440
34dc4d44 4441 obj = i915_gem_alloc_object(dev, 4096);
e552eb70
JB		 4442 if (obj == NULL) {
4443 DRM_ERROR("Failed to allocate seqno page\n");
4444 ret = -ENOMEM;
4445 goto err;
4446 }
4447 obj_priv = to_intel_bo(obj);
4448 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4449
4450 ret = i915_gem_object_pin(obj, 4096);
4451 if (ret)
4452 goto err_unref;
4453
4454 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4455 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4456 if (dev_priv->seqno_page == NULL)
4457 goto err_unpin;
4458
4459 dev_priv->seqno_obj = obj;
4460 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4461
4462 return 0;
4463
4464err_unpin:
4465 i915_gem_object_unpin(obj);
4466err_unref:
4467 drm_gem_object_unreference(obj);
4468err:
4469 return ret;
4470}
4471
8187a2b7
ZN		 4472
4473static void
e552eb70
JB		 4474i915_gem_cleanup_pipe_control(struct drm_device *dev)
4475{
4476 drm_i915_private_t *dev_priv = dev->dev_private;
4477 struct drm_gem_object *obj;
4478 struct drm_i915_gem_object *obj_priv;
4479
4480 obj = dev_priv->seqno_obj;
4481 obj_priv = to_intel_bo(obj);
4482 kunmap(obj_priv->pages[0]);
4483 i915_gem_object_unpin(obj);
4484 drm_gem_object_unreference(obj);
4485 dev_priv->seqno_obj = NULL;
4486
4487 dev_priv->seqno_page = NULL;
673a394b
EA		 4488}
4489
8187a2b7
ZN		 4490int
4491i915_gem_init_ringbuffer(struct drm_device *dev)
4492{
4493 drm_i915_private_t *dev_priv = dev->dev_private;
4494 int ret;
68f95ba9 4495
8187a2b7
ZN		 4496 if (HAS_PIPE_CONTROL(dev)) {
4497 ret = i915_gem_init_pipe_control(dev);
4498 if (ret)
4499 return ret;
4500 }
68f95ba9 4501
5c1143bb 4502 ret = intel_init_render_ring_buffer(dev);
68f95ba9
CW		 4503 if (ret)
4504 goto cleanup_pipe_control;
4505
4506 if (HAS_BSD(dev)) {
5c1143bb 4507 ret = intel_init_bsd_ring_buffer(dev);
68f95ba9
CW		 4508 if (ret)
4509 goto cleanup_render_ring;
d1b851fc 4510 }
68f95ba9 4511
549f7365
CW		 4512 if (HAS_BLT(dev)) {
4513 ret = intel_init_blt_ring_buffer(dev);
4514 if (ret)
4515 goto cleanup_bsd_ring;
4516 }
4517
6f392d54
CW		 4518 dev_priv->next_seqno = 1;
4519
68f95ba9
CW		 4520 return 0;
4521
549f7365
CW		 4522cleanup_bsd_ring:
4523 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
68f95ba9
CW		 4524cleanup_render_ring:
4525 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4526cleanup_pipe_control:
4527 if (HAS_PIPE_CONTROL(dev))
4528 i915_gem_cleanup_pipe_control(dev);
8187a2b7
ZN		 4529 return ret;
4530}
4531
4532void
4533i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4534{
4535 drm_i915_private_t *dev_priv = dev->dev_private;
4536
4537 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
87acb0a5 4538 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
549f7365 4539 intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring);
8187a2b7
ZN		 4540 if (HAS_PIPE_CONTROL(dev))
4541 i915_gem_cleanup_pipe_control(dev);
4542}
4543
673a394b
EA		 4544int
4545i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4546 struct drm_file *file_priv)
4547{
4548 drm_i915_private_t *dev_priv = dev->dev_private;
4549 int ret;
4550
79e53945
JB		 4551 if (drm_core_check_feature(dev, DRIVER_MODESET))
4552 return 0;
4553
ba1234d1 4554 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 4555 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 4556 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 4557 }
4558
673a394b 4559 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 4560 dev_priv->mm.suspended = 0;
4561
4562 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 4563 if (ret != 0) {
4564 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 4565 return ret;
d816f6ac 4566 }
9bb2d6f9 4567
69dc4987 4568 BUG_ON(!list_empty(&dev_priv->mm.active_list));
852835f3 4569 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
87acb0a5 4570 BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
549f7365 4571 BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
673a394b
EA		 4572 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4573 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
852835f3 4574 BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
87acb0a5 4575 BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
549f7365 4576 BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
673a394b 4577 mutex_unlock(&dev->struct_mutex);
dbb19d30 4578
5f35308b
CW		 4579 ret = drm_irq_install(dev);
4580 if (ret)
4581 goto cleanup_ringbuffer;
dbb19d30 4582
673a394b 4583 return 0;
5f35308b
CW		 4584
4585cleanup_ringbuffer:
4586 mutex_lock(&dev->struct_mutex);
4587 i915_gem_cleanup_ringbuffer(dev);
4588 dev_priv->mm.suspended = 1;
4589 mutex_unlock(&dev->struct_mutex);
4590
4591 return ret;
673a394b
EA		 4592}
4593
4594int
4595i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4596 struct drm_file *file_priv)
4597{
79e53945
JB		 4598 if (drm_core_check_feature(dev, DRIVER_MODESET))
4599 return 0;
4600
dbb19d30 4601 drm_irq_uninstall(dev);
e6890f6f 4602 return i915_gem_idle(dev);
673a394b
EA		 4603}
4604
4605void
4606i915_gem_lastclose(struct drm_device *dev)
4607{
4608 int ret;
673a394b 4609
e806b495
EA		 4610 if (drm_core_check_feature(dev, DRIVER_MODESET))
4611 return;
4612
6dbe2772
KP		 4613 ret = i915_gem_idle(dev);
4614 if (ret)
4615 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 4616}
4617
4618void
4619i915_gem_load(struct drm_device *dev)
4620{
b5aa8a0f 4621 int i;
673a394b
EA		 4622 drm_i915_private_t *dev_priv = dev->dev_private;
4623
69dc4987 4624 INIT_LIST_HEAD(&dev_priv->mm.active_list);
673a394b 4625 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
99fcb766 4626 INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
673a394b 4627 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
f13d3f73 4628 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
a09ba7fa 4629 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
be72615b 4630 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
852835f3
ZN		 4631 INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4632 INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
87acb0a5
CW		 4633 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4634 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
549f7365
CW		 4635 INIT_LIST_HEAD(&dev_priv->blt_ring.active_list);
4636 INIT_LIST_HEAD(&dev_priv->blt_ring.request_list);
007cc8ac
DV		 4637 for (i = 0; i < 16; i++)
4638 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
673a394b
EA		 4639 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4640 i915_gem_retire_work_handler);
30dbf0c0 4641 init_completion(&dev_priv->error_completion);
31169714
CW		 4642 spin_lock(&shrink_list_lock);
4643 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4644 spin_unlock(&shrink_list_lock);
4645
94400120
DA		 4646 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4647 if (IS_GEN3(dev)) {
4648 u32 tmp = I915_READ(MI_ARB_STATE);
4649 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4650 /* arb state is a masked write, so set bit + bit in mask */
4651 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4652 I915_WRITE(MI_ARB_STATE, tmp);
4653 }
4654 }
4655
de151cf6 4656 /* Old X drivers will take 0-2 for front, back, depth buffers */
b397c836
EA		 4657 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4658 dev_priv->fence_reg_start = 3;
de151cf6 4659
a6c45cf0 4660 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 4661 dev_priv->num_fence_regs = 16;
4662 else
4663 dev_priv->num_fence_regs = 8;
4664
b5aa8a0f 4665 /* Initialize fence registers to zero */
a6c45cf0
CW		 4666 switch (INTEL_INFO(dev)->gen) {
4667 case 6:
4668 for (i = 0; i < 16; i++)
4669 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4670 break;
4671 case 5:
4672 case 4:
b5aa8a0f
GH		 4673 for (i = 0; i < 16; i++)
4674 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
a6c45cf0
CW		 4675 break;
4676 case 3:
b5aa8a0f
GH		 4677 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4678 for (i = 0; i < 8; i++)
4679 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
a6c45cf0
CW		 4680 case 2:
4681 for (i = 0; i < 8; i++)
4682 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4683 break;
b5aa8a0f 4684 }
673a394b 4685 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 4686 init_waitqueue_head(&dev_priv->pending_flip_queue);
673a394b 4687}
71acb5eb
DA		 4688
4689/*
4690 * Create a physically contiguous memory object for this object
4691 * e.g. for cursor + overlay regs
4692 */
995b6762
CW		 4693static int i915_gem_init_phys_object(struct drm_device *dev,
4694 int id, int size, int align)
71acb5eb
DA		 4695{
4696 drm_i915_private_t *dev_priv = dev->dev_private;
4697 struct drm_i915_gem_phys_object *phys_obj;
4698 int ret;
4699
4700 if (dev_priv->mm.phys_objs[id - 1] || !size)
4701 return 0;
4702
9a298b2a 4703 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 4704 if (!phys_obj)
4705 return -ENOMEM;
4706
4707 phys_obj->id = id;
4708
6eeefaf3 4709 phys_obj->handle = drm_pci_alloc(dev, size, align);
71acb5eb
DA		 4710 if (!phys_obj->handle) {
4711 ret = -ENOMEM;
4712 goto kfree_obj;
4713 }
4714#ifdef CONFIG_X86
4715 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4716#endif
4717
4718 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4719
4720 return 0;
4721kfree_obj:
9a298b2a 4722 kfree(phys_obj);
71acb5eb
DA		 4723 return ret;
4724}
4725
995b6762 4726static void i915_gem_free_phys_object(struct drm_device *dev, int id)
71acb5eb
DA		 4727{
4728 drm_i915_private_t *dev_priv = dev->dev_private;
4729 struct drm_i915_gem_phys_object *phys_obj;
4730
4731 if (!dev_priv->mm.phys_objs[id - 1])
4732 return;
4733
4734 phys_obj = dev_priv->mm.phys_objs[id - 1];
4735 if (phys_obj->cur_obj) {
4736 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4737 }
4738
4739#ifdef CONFIG_X86
4740 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4741#endif
4742 drm_pci_free(dev, phys_obj->handle);
4743 kfree(phys_obj);
4744 dev_priv->mm.phys_objs[id - 1] = NULL;
4745}
4746
4747void i915_gem_free_all_phys_object(struct drm_device *dev)
4748{
4749 int i;
4750
260883c8 4751 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 4752 i915_gem_free_phys_object(dev, i);
4753}
4754
4755void i915_gem_detach_phys_object(struct drm_device *dev,
4756 struct drm_gem_object *obj)
4757{
4758 struct drm_i915_gem_object *obj_priv;
4759 int i;
4760 int ret;
4761 int page_count;
4762
23010e43 4763 obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4764 if (!obj_priv->phys_obj)
4765 return;
4766
4bdadb97 4767 ret = i915_gem_object_get_pages(obj, 0);
71acb5eb
DA		 4768 if (ret)
4769 goto out;
4770
4771 page_count = obj->size / PAGE_SIZE;
4772
4773 for (i = 0; i < page_count; i++) {
856fa198 4774 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4775 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4776
4777 memcpy(dst, src, PAGE_SIZE);
4778 kunmap_atomic(dst, KM_USER0);
4779 }
856fa198 4780 drm_clflush_pages(obj_priv->pages, page_count);
71acb5eb 4781 drm_agp_chipset_flush(dev);
d78b47b9
CW		 4782
4783 i915_gem_object_put_pages(obj);
71acb5eb
DA		 4784out:
4785 obj_priv->phys_obj->cur_obj = NULL;
4786 obj_priv->phys_obj = NULL;
4787}
4788
4789int
4790i915_gem_attach_phys_object(struct drm_device *dev,
6eeefaf3
CW		 4791 struct drm_gem_object *obj,
4792 int id,
4793 int align)
71acb5eb
DA		 4794{
4795 drm_i915_private_t *dev_priv = dev->dev_private;
4796 struct drm_i915_gem_object *obj_priv;
4797 int ret = 0;
4798 int page_count;
4799 int i;
4800
4801 if (id > I915_MAX_PHYS_OBJECT)
4802 return -EINVAL;
4803
23010e43 4804 obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4805
4806 if (obj_priv->phys_obj) {
4807 if (obj_priv->phys_obj->id == id)
4808 return 0;
4809 i915_gem_detach_phys_object(dev, obj);
4810 }
4811
71acb5eb
DA		 4812 /* create a new object */
4813 if (!dev_priv->mm.phys_objs[id - 1]) {
4814 ret = i915_gem_init_phys_object(dev, id,
6eeefaf3 4815 obj->size, align);
71acb5eb 4816 if (ret) {
aeb565df 4817 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
71acb5eb
DA		 4818 goto out;
4819 }
4820 }
4821
4822 /* bind to the object */
4823 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4824 obj_priv->phys_obj->cur_obj = obj;
4825
4bdadb97 4826 ret = i915_gem_object_get_pages(obj, 0);
71acb5eb
DA		 4827 if (ret) {
4828 DRM_ERROR("failed to get page list\n");
4829 goto out;
4830 }
4831
4832 page_count = obj->size / PAGE_SIZE;
4833
4834 for (i = 0; i < page_count; i++) {
856fa198 4835 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4836 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4837
4838 memcpy(dst, src, PAGE_SIZE);
4839 kunmap_atomic(src, KM_USER0);
4840 }
4841
d78b47b9
CW		 4842 i915_gem_object_put_pages(obj);
4843
71acb5eb
DA		 4844 return 0;
4845out:
4846 return ret;
4847}
4848
4849static int
4850i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4851 struct drm_i915_gem_pwrite *args,
4852 struct drm_file *file_priv)
4853{
23010e43 4854 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4855 void *obj_addr;
4856 int ret;
4857 char __user *user_data;
4858
4859 user_data = (char __user *) (uintptr_t) args->data_ptr;
4860 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4861
44d98a61 4862 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
71acb5eb
DA		 4863 ret = copy_from_user(obj_addr, user_data, args->size);
4864 if (ret)
4865 return -EFAULT;
4866
4867 drm_agp_chipset_flush(dev);
4868 return 0;
4869}
b962442e 4870
f787a5f5 4871void i915_gem_release(struct drm_device *dev, struct drm_file *file)
b962442e 4872{
f787a5f5 4873 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e
EA		 4874
4875 /* Clean up our request list when the client is going away, so that
4876 * later retire_requests won't dereference our soon-to-be-gone
4877 * file_priv.
4878 */
1c25595f 4879 spin_lock(&file_priv->mm.lock);
f787a5f5
CW		 4880 while (!list_empty(&file_priv->mm.request_list)) {
4881 struct drm_i915_gem_request *request;
4882
4883 request = list_first_entry(&file_priv->mm.request_list,
4884 struct drm_i915_gem_request,
4885 client_list);
4886 list_del(&request->client_list);
4887 request->file_priv = NULL;
4888 }
1c25595f 4889 spin_unlock(&file_priv->mm.lock);
b962442e 4890}
31169714 4891
1637ef41
CW		 4892static int
4893i915_gpu_is_active(struct drm_device *dev)
4894{
4895 drm_i915_private_t *dev_priv = dev->dev_private;
4896 int lists_empty;
4897
1637ef41 4898 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
87acb0a5 4899 list_empty(&dev_priv->render_ring.active_list) &&
549f7365
CW		 4900 list_empty(&dev_priv->bsd_ring.active_list) &&
4901 list_empty(&dev_priv->blt_ring.active_list);
1637ef41
CW		 4902
4903 return !lists_empty;
4904}
4905
31169714 4906static int
7f8275d0 4907i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
31169714
CW		 4908{
4909 drm_i915_private_t *dev_priv, *next_dev;
4910 struct drm_i915_gem_object *obj_priv, *next_obj;
4911 int cnt = 0;
4912 int would_deadlock = 1;
4913
4914 /* "fast-path" to count number of available objects */
4915 if (nr_to_scan == 0) {
4916 spin_lock(&shrink_list_lock);
4917 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4918 struct drm_device *dev = dev_priv->dev;
4919
4920 if (mutex_trylock(&dev->struct_mutex)) {
4921 list_for_each_entry(obj_priv,
4922 &dev_priv->mm.inactive_list,
69dc4987 4923 mm_list)
31169714
CW		 4924 cnt++;
4925 mutex_unlock(&dev->struct_mutex);
4926 }
4927 }
4928 spin_unlock(&shrink_list_lock);
4929
4930 return (cnt / 100) * sysctl_vfs_cache_pressure;
4931 }
4932
4933 spin_lock(&shrink_list_lock);
4934
1637ef41 4935rescan:
31169714
CW		 4936 /* first scan for clean buffers */
4937 list_for_each_entry_safe(dev_priv, next_dev,
4938 &shrink_list, mm.shrink_list) {
4939 struct drm_device *dev = dev_priv->dev;
4940
4941 if (! mutex_trylock(&dev->struct_mutex))
4942 continue;
4943
4944 spin_unlock(&shrink_list_lock);
b09a1fec 4945 i915_gem_retire_requests(dev);
31169714
CW		 4946
4947 list_for_each_entry_safe(obj_priv, next_obj,
4948 &dev_priv->mm.inactive_list,
69dc4987 4949 mm_list) {
31169714 4950 if (i915_gem_object_is_purgeable(obj_priv)) {
a8089e84 4951 i915_gem_object_unbind(&obj_priv->base);
31169714
CW		 4952 if (--nr_to_scan <= 0)
4953 break;
4954 }
4955 }
4956
4957 spin_lock(&shrink_list_lock);
4958 mutex_unlock(&dev->struct_mutex);
4959
963b4836
CW		 4960 would_deadlock = 0;
4961
31169714
CW		 4962 if (nr_to_scan <= 0)
4963 break;
4964 }
4965
4966 /* second pass, evict/count anything still on the inactive list */
4967 list_for_each_entry_safe(dev_priv, next_dev,
4968 &shrink_list, mm.shrink_list) {
4969 struct drm_device *dev = dev_priv->dev;
4970
4971 if (! mutex_trylock(&dev->struct_mutex))
4972 continue;
4973
4974 spin_unlock(&shrink_list_lock);
4975
4976 list_for_each_entry_safe(obj_priv, next_obj,
4977 &dev_priv->mm.inactive_list,
69dc4987 4978 mm_list) {
31169714 4979 if (nr_to_scan > 0) {
a8089e84 4980 i915_gem_object_unbind(&obj_priv->base);
31169714
CW		 4981 nr_to_scan--;
4982 } else
4983 cnt++;
4984 }
4985
4986 spin_lock(&shrink_list_lock);
4987 mutex_unlock(&dev->struct_mutex);
4988
4989 would_deadlock = 0;
4990 }
4991
1637ef41
CW		 4992 if (nr_to_scan) {
4993 int active = 0;
4994
4995 /*
4996 * We are desperate for pages, so as a last resort, wait
4997 * for the GPU to finish and discard whatever we can.
4998 * This has a dramatic impact to reduce the number of
4999 * OOM-killer events whilst running the GPU aggressively.
5000 */
5001 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5002 struct drm_device *dev = dev_priv->dev;
5003
5004 if (!mutex_trylock(&dev->struct_mutex))
5005 continue;
5006
5007 spin_unlock(&shrink_list_lock);
5008
5009 if (i915_gpu_is_active(dev)) {
5010 i915_gpu_idle(dev);
5011 active++;
5012 }
5013
5014 spin_lock(&shrink_list_lock);
5015 mutex_unlock(&dev->struct_mutex);
5016 }
5017
5018 if (active)
5019 goto rescan;
5020 }
5021
31169714
CW		 5022 spin_unlock(&shrink_list_lock);
5023
5024 if (would_deadlock)
5025 return -1;
5026 else if (cnt > 0)
5027 return (cnt / 100) * sysctl_vfs_cache_pressure;
5028 else
5029 return 0;
5030}
5031
5032static struct shrinker shrinker = {
5033 .shrink = i915_gem_shrink,
5034 .seeks = DEFAULT_SEEKS,
5035};
5036
5037__init void
5038i915_gem_shrinker_init(void)
5039{
5040 register_shrinker(&shrinker);
5041}
5042
5043__exit void
5044i915_gem_shrinker_exit(void)
5045{
5046 unregister_shrinker(&shrinker);
5047}
Linux kernel - Deliverables
This page took 0.846635 seconds and 5 git commands to generate.