projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Recognise non-VGA display devices
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
5a0e3ad6 34#include <linux/slab.h>
673a394b 35#include <linux/swap.h>
79e53945 36#include <linux/pci.h>
673a394b 37
88241785 38static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
05394f39
CW		 39static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
40static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
88241785
CW		 41static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
42 bool write);
43static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
44 uint64_t offset,
45 uint64_t size);
05394f39 46static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
88241785
CW		 47static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
48 unsigned alignment,
49 bool map_and_fenceable);
d9e86c0e
CW		 50static void i915_gem_clear_fence_reg(struct drm_device *dev,
51 struct drm_i915_fence_reg *reg);
05394f39
CW		 52static int i915_gem_phys_pwrite(struct drm_device *dev,
53 struct drm_i915_gem_object *obj,
71acb5eb 54 struct drm_i915_gem_pwrite *args,
05394f39
CW		 55 struct drm_file *file);
56static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
673a394b 57
17250b71
CW		 58static int i915_gem_inactive_shrink(struct shrinker *shrinker,
59 int nr_to_scan,
60 gfp_t gfp_mask);
61
31169714 62
73aa808f
CW		 63/* some bookkeeping */
64static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
65 size_t size)
66{
67 dev_priv->mm.object_count++;
68 dev_priv->mm.object_memory += size;
69}
70
71static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
72 size_t size)
73{
74 dev_priv->mm.object_count--;
75 dev_priv->mm.object_memory -= size;
76}
77
30dbf0c0
CW		 78int
79i915_gem_check_is_wedged(struct drm_device *dev)
80{
81 struct drm_i915_private *dev_priv = dev->dev_private;
82 struct completion *x = &dev_priv->error_completion;
83 unsigned long flags;
84 int ret;
85
86 if (!atomic_read(&dev_priv->mm.wedged))
87 return 0;
88
89 ret = wait_for_completion_interruptible(x);
90 if (ret)
91 return ret;
92
93 /* Success, we reset the GPU! */
94 if (!atomic_read(&dev_priv->mm.wedged))
95 return 0;
96
97 /* GPU is hung, bump the completion count to account for
98 * the token we just consumed so that we never hit zero and
99 * end up waiting upon a subsequent completion event that
100 * will never happen.
101 */
102 spin_lock_irqsave(&x->wait.lock, flags);
103 x->done++;
104 spin_unlock_irqrestore(&x->wait.lock, flags);
105 return -EIO;
106}
107
54cf91dc 108int i915_mutex_lock_interruptible(struct drm_device *dev)
76c1dec1
CW		 109{
110 struct drm_i915_private *dev_priv = dev->dev_private;
111 int ret;
112
113 ret = i915_gem_check_is_wedged(dev);
114 if (ret)
115 return ret;
116
117 ret = mutex_lock_interruptible(&dev->struct_mutex);
118 if (ret)
119 return ret;
120
121 if (atomic_read(&dev_priv->mm.wedged)) {
122 mutex_unlock(&dev->struct_mutex);
123 return -EAGAIN;
124 }
125
23bc5982 126 WARN_ON(i915_verify_lists(dev));
76c1dec1
CW		 127 return 0;
128}
30dbf0c0 129
7d1c4804 130static inline bool
05394f39 131i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
7d1c4804 132{
05394f39 133 return obj->gtt_space && !obj->active && obj->pin_count == 0;
7d1c4804
CW		 134}
135
2021746e
CW		 136void i915_gem_do_init(struct drm_device *dev,
137 unsigned long start,
138 unsigned long mappable_end,
139 unsigned long end)
673a394b
EA		 140{
141 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 142
79e53945
JB		 143 drm_mm_init(&dev_priv->mm.gtt_space, start,
144 end - start);
673a394b 145
73aa808f 146 dev_priv->mm.gtt_total = end - start;
fb7d516a 147 dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
53984635 148 dev_priv->mm.gtt_mappable_end = mappable_end;
79e53945 149}
673a394b 150
79e53945
JB		 151int
152i915_gem_init_ioctl(struct drm_device *dev, void *data,
05394f39 153 struct drm_file *file)
79e53945
JB		 154{
155 struct drm_i915_gem_init *args = data;
2021746e
CW		 156
157 if (args->gtt_start >= args->gtt_end ||
158 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
159 return -EINVAL;
79e53945
JB		 160
161 mutex_lock(&dev->struct_mutex);
2021746e 162 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
673a394b
EA		 163 mutex_unlock(&dev->struct_mutex);
164
2021746e 165 return 0;
673a394b
EA		 166}
167
5a125c3c
EA		 168int
169i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
05394f39 170 struct drm_file *file)
5a125c3c 171{
73aa808f 172 struct drm_i915_private *dev_priv = dev->dev_private;
5a125c3c 173 struct drm_i915_gem_get_aperture *args = data;
6299f992
CW		 174 struct drm_i915_gem_object *obj;
175 size_t pinned;
5a125c3c
EA		 176
177 if (!(dev->driver->driver_features & DRIVER_GEM))
178 return -ENODEV;
179
6299f992 180 pinned = 0;
73aa808f 181 mutex_lock(&dev->struct_mutex);
6299f992
CW		 182 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
183 pinned += obj->gtt_space->size;
73aa808f 184 mutex_unlock(&dev->struct_mutex);
5a125c3c 185
6299f992
CW		 186 args->aper_size = dev_priv->mm.gtt_total;
187 args->aper_available_size = args->aper_size -pinned;
188
5a125c3c
EA		 189 return 0;
190}
191
673a394b
EA		 192/**
193 * Creates a new mm object and returns a handle to it.
194 */
195int
196i915_gem_create_ioctl(struct drm_device *dev, void *data,
05394f39 197 struct drm_file *file)
673a394b
EA		 198{
199 struct drm_i915_gem_create *args = data;
05394f39 200 struct drm_i915_gem_object *obj;
a1a2d1d3
PP		 201 int ret;
202 u32 handle;
673a394b
EA		 203
204 args->size = roundup(args->size, PAGE_SIZE);
205
206 /* Allocate the new object */
ac52bc56 207 obj = i915_gem_alloc_object(dev, args->size);
673a394b
EA		 208 if (obj == NULL)
209 return -ENOMEM;
210
05394f39 211 ret = drm_gem_handle_create(file, &obj->base, &handle);
1dfd9754 212 if (ret) {
05394f39
CW		 213 drm_gem_object_release(&obj->base);
214 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
202f2fef 215 kfree(obj);
673a394b 216 return ret;
1dfd9754 217 }
673a394b 218
202f2fef 219 /* drop reference from allocate - handle holds it now */
05394f39 220 drm_gem_object_unreference(&obj->base);
202f2fef
CW		 221 trace_i915_gem_object_create(obj);
222
1dfd9754 223 args->handle = handle;
673a394b
EA		 224 return 0;
225}
226
05394f39 227static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
280b713b 228{
05394f39 229 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
280b713b
EA		 230
231 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
05394f39 232 obj->tiling_mode != I915_TILING_NONE;
280b713b
EA		 233}
234
99a03df5 235static inline void
40123c1f
EA		 236slow_shmem_copy(struct page *dst_page,
237 int dst_offset,
238 struct page *src_page,
239 int src_offset,
240 int length)
241{
242 char *dst_vaddr, *src_vaddr;
243
99a03df5
CW		 244 dst_vaddr = kmap(dst_page);
245 src_vaddr = kmap(src_page);
40123c1f
EA		 246
247 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
248
99a03df5
CW		 249 kunmap(src_page);
250 kunmap(dst_page);
40123c1f
EA		 251}
252
99a03df5 253static inline void
280b713b
EA		 254slow_shmem_bit17_copy(struct page *gpu_page,
255 int gpu_offset,
256 struct page *cpu_page,
257 int cpu_offset,
258 int length,
259 int is_read)
260{
261 char *gpu_vaddr, *cpu_vaddr;
262
263 /* Use the unswizzled path if this page isn't affected. */
264 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
265 if (is_read)
266 return slow_shmem_copy(cpu_page, cpu_offset,
267 gpu_page, gpu_offset, length);
268 else
269 return slow_shmem_copy(gpu_page, gpu_offset,
270 cpu_page, cpu_offset, length);
271 }
272
99a03df5
CW		 273 gpu_vaddr = kmap(gpu_page);
274 cpu_vaddr = kmap(cpu_page);
280b713b
EA		 275
276 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
277 * XORing with the other bits (A9 for Y, A9 and A10 for X)
278 */
279 while (length > 0) {
280 int cacheline_end = ALIGN(gpu_offset + 1, 64);
281 int this_length = min(cacheline_end - gpu_offset, length);
282 int swizzled_gpu_offset = gpu_offset ^ 64;
283
284 if (is_read) {
285 memcpy(cpu_vaddr + cpu_offset,
286 gpu_vaddr + swizzled_gpu_offset,
287 this_length);
288 } else {
289 memcpy(gpu_vaddr + swizzled_gpu_offset,
290 cpu_vaddr + cpu_offset,
291 this_length);
292 }
293 cpu_offset += this_length;
294 gpu_offset += this_length;
295 length -= this_length;
296 }
297
99a03df5
CW		 298 kunmap(cpu_page);
299 kunmap(gpu_page);
280b713b
EA		 300}
301
eb01459f
EA		 302/**
303 * This is the fast shmem pread path, which attempts to copy_from_user directly
304 * from the backing pages of the object to the user's address space. On a
305 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
306 */
307static int
05394f39
CW		 308i915_gem_shmem_pread_fast(struct drm_device *dev,
309 struct drm_i915_gem_object *obj,
eb01459f 310 struct drm_i915_gem_pread *args,
05394f39 311 struct drm_file *file)
eb01459f 312{
05394f39 313 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
eb01459f 314 ssize_t remain;
e5281ccd 315 loff_t offset;
eb01459f
EA		 316 char __user *user_data;
317 int page_offset, page_length;
eb01459f
EA		 318
319 user_data = (char __user *) (uintptr_t) args->data_ptr;
320 remain = args->size;
321
eb01459f
EA		 322 offset = args->offset;
323
324 while (remain > 0) {
e5281ccd
CW		 325 struct page *page;
326 char *vaddr;
327 int ret;
328
eb01459f
EA		 329 /* Operation in this page
330 *
eb01459f
EA		 331 * page_offset = offset within page
332 * page_length = bytes to copy for this page
333 */
eb01459f
EA		 334 page_offset = offset & (PAGE_SIZE-1);
335 page_length = remain;
336 if ((page_offset + remain) > PAGE_SIZE)
337 page_length = PAGE_SIZE - page_offset;
338
e5281ccd
CW		 339 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
340 GFP_HIGHUSER | __GFP_RECLAIMABLE);
341 if (IS_ERR(page))
342 return PTR_ERR(page);
343
344 vaddr = kmap_atomic(page);
345 ret = __copy_to_user_inatomic(user_data,
346 vaddr + page_offset,
347 page_length);
348 kunmap_atomic(vaddr);
349
350 mark_page_accessed(page);
351 page_cache_release(page);
352 if (ret)
4f27b75d 353 return -EFAULT;
eb01459f
EA		 354
355 remain -= page_length;
356 user_data += page_length;
357 offset += page_length;
358 }
359
4f27b75d 360 return 0;
eb01459f
EA		 361}
362
363/**
364 * This is the fallback shmem pread path, which allocates temporary storage
365 * in kernel space to copy_to_user into outside of the struct_mutex, so we
366 * can copy out of the object's backing pages while holding the struct mutex
367 * and not take page faults.
368 */
369static int
05394f39
CW		 370i915_gem_shmem_pread_slow(struct drm_device *dev,
371 struct drm_i915_gem_object *obj,
eb01459f 372 struct drm_i915_gem_pread *args,
05394f39 373 struct drm_file *file)
eb01459f 374{
05394f39 375 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
eb01459f
EA		 376 struct mm_struct *mm = current->mm;
377 struct page **user_pages;
378 ssize_t remain;
379 loff_t offset, pinned_pages, i;
380 loff_t first_data_page, last_data_page, num_pages;
e5281ccd
CW		 381 int shmem_page_offset;
382 int data_page_index, data_page_offset;
eb01459f
EA		 383 int page_length;
384 int ret;
385 uint64_t data_ptr = args->data_ptr;
280b713b 386 int do_bit17_swizzling;
eb01459f
EA		 387
388 remain = args->size;
389
390 /* Pin the user pages containing the data. We can't fault while
391 * holding the struct mutex, yet we want to hold it while
392 * dereferencing the user data.
393 */
394 first_data_page = data_ptr / PAGE_SIZE;
395 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
396 num_pages = last_data_page - first_data_page + 1;
397
4f27b75d 398 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
eb01459f
EA		 399 if (user_pages == NULL)
400 return -ENOMEM;
401
4f27b75d 402 mutex_unlock(&dev->struct_mutex);
eb01459f
EA		 403 down_read(&mm->mmap_sem);
404 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 405 num_pages, 1, 0, user_pages, NULL);
eb01459f 406 up_read(&mm->mmap_sem);
4f27b75d 407 mutex_lock(&dev->struct_mutex);
eb01459f
EA		 408 if (pinned_pages < num_pages) {
409 ret = -EFAULT;
4f27b75d 410 goto out;
eb01459f
EA		 411 }
412
4f27b75d
CW		 413 ret = i915_gem_object_set_cpu_read_domain_range(obj,
414 args->offset,
415 args->size);
07f73f69 416 if (ret)
4f27b75d 417 goto out;
eb01459f 418
4f27b75d 419 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
eb01459f 420
eb01459f
EA		 421 offset = args->offset;
422
423 while (remain > 0) {
e5281ccd
CW		 424 struct page *page;
425
eb01459f
EA		 426 /* Operation in this page
427 *
eb01459f
EA		 428 * shmem_page_offset = offset within page in shmem file
429 * data_page_index = page number in get_user_pages return
430 * data_page_offset = offset with data_page_index page.
431 * page_length = bytes to copy for this page
432 */
eb01459f
EA		 433 shmem_page_offset = offset & ~PAGE_MASK;
434 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
435 data_page_offset = data_ptr & ~PAGE_MASK;
436
437 page_length = remain;
438 if ((shmem_page_offset + page_length) > PAGE_SIZE)
439 page_length = PAGE_SIZE - shmem_page_offset;
440 if ((data_page_offset + page_length) > PAGE_SIZE)
441 page_length = PAGE_SIZE - data_page_offset;
442
e5281ccd
CW		 443 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
444 GFP_HIGHUSER | __GFP_RECLAIMABLE);
445 if (IS_ERR(page))
446 return PTR_ERR(page);
447
280b713b 448 if (do_bit17_swizzling) {
e5281ccd 449 slow_shmem_bit17_copy(page,
280b713b 450 shmem_page_offset,
99a03df5
CW		 451 user_pages[data_page_index],
452 data_page_offset,
453 page_length,
454 1);
455 } else {
456 slow_shmem_copy(user_pages[data_page_index],
457 data_page_offset,
e5281ccd 458 page,
99a03df5
CW		 459 shmem_page_offset,
460 page_length);
280b713b 461 }
eb01459f 462
e5281ccd
CW		 463 mark_page_accessed(page);
464 page_cache_release(page);
465
eb01459f
EA		 466 remain -= page_length;
467 data_ptr += page_length;
468 offset += page_length;
469 }
470
4f27b75d 471out:
eb01459f
EA		 472 for (i = 0; i < pinned_pages; i++) {
473 SetPageDirty(user_pages[i]);
e5281ccd 474 mark_page_accessed(user_pages[i]);
eb01459f
EA		 475 page_cache_release(user_pages[i]);
476 }
8e7d2b2c 477 drm_free_large(user_pages);
eb01459f
EA		 478
479 return ret;
480}
481
673a394b
EA		 482/**
483 * Reads data from the object referenced by handle.
484 *
485 * On error, the contents of *data are undefined.
486 */
487int
488i915_gem_pread_ioctl(struct drm_device *dev, void *data,
05394f39 489 struct drm_file *file)
673a394b
EA		 490{
491 struct drm_i915_gem_pread *args = data;
05394f39 492 struct drm_i915_gem_object *obj;
35b62a89 493 int ret = 0;
673a394b 494
51311d0a
CW		 495 if (args->size == 0)
496 return 0;
497
498 if (!access_ok(VERIFY_WRITE,
499 (char __user *)(uintptr_t)args->data_ptr,
500 args->size))
501 return -EFAULT;
502
503 ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
504 args->size);
505 if (ret)
506 return -EFAULT;
507
4f27b75d 508 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 509 if (ret)
4f27b75d 510 return ret;
673a394b 511
05394f39 512 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1d7cfea1
CW		 513 if (obj == NULL) {
514 ret = -ENOENT;
515 goto unlock;
4f27b75d 516 }
673a394b 517
7dcd2499 518 /* Bounds check source. */
05394f39
CW		 519 if (args->offset > obj->base.size ||
520 args->size > obj->base.size - args->offset) {
ce9d419d 521 ret = -EINVAL;
35b62a89 522 goto out;
ce9d419d
CW		 523 }
524
4f27b75d
CW		 525 ret = i915_gem_object_set_cpu_read_domain_range(obj,
526 args->offset,
527 args->size);
528 if (ret)
e5281ccd 529 goto out;
4f27b75d
CW		 530
531 ret = -EFAULT;
532 if (!i915_gem_object_needs_bit17_swizzle(obj))
05394f39 533 ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
4f27b75d 534 if (ret == -EFAULT)
05394f39 535 ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
673a394b 536
35b62a89 537out:
05394f39 538 drm_gem_object_unreference(&obj->base);
1d7cfea1 539unlock:
4f27b75d 540 mutex_unlock(&dev->struct_mutex);
eb01459f 541 return ret;
673a394b
EA		 542}
543
0839ccb8
KP		 544/* This is the fast write path which cannot handle
545 * page faults in the source data
9b7530cc 546 */
0839ccb8
KP		 547
548static inline int
549fast_user_write(struct io_mapping *mapping,
550 loff_t page_base, int page_offset,
551 char __user *user_data,
552 int length)
9b7530cc 553{
9b7530cc 554 char *vaddr_atomic;
0839ccb8 555 unsigned long unwritten;
9b7530cc 556
3e4d3af5 557 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
0839ccb8
KP		 558 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
559 user_data, length);
3e4d3af5 560 io_mapping_unmap_atomic(vaddr_atomic);
fbd5a26d 561 return unwritten;
0839ccb8
KP		 562}
563
564/* Here's the write path which can sleep for
565 * page faults
566 */
567
ab34c226 568static inline void
3de09aa3
EA		 569slow_kernel_write(struct io_mapping *mapping,
570 loff_t gtt_base, int gtt_offset,
571 struct page *user_page, int user_offset,
572 int length)
0839ccb8 573{
ab34c226
CW		 574 char __iomem *dst_vaddr;
575 char *src_vaddr;
0839ccb8 576
ab34c226
CW		 577 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
578 src_vaddr = kmap(user_page);
579
580 memcpy_toio(dst_vaddr + gtt_offset,
581 src_vaddr + user_offset,
582 length);
583
584 kunmap(user_page);
585 io_mapping_unmap(dst_vaddr);
9b7530cc
LT		 586}
587
3de09aa3
EA		 588/**
589 * This is the fast pwrite path, where we copy the data directly from the
590 * user into the GTT, uncached.
591 */
673a394b 592static int
05394f39
CW		 593i915_gem_gtt_pwrite_fast(struct drm_device *dev,
594 struct drm_i915_gem_object *obj,
3de09aa3 595 struct drm_i915_gem_pwrite *args,
05394f39 596 struct drm_file *file)
673a394b 597{
0839ccb8 598 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 599 ssize_t remain;
0839ccb8 600 loff_t offset, page_base;
673a394b 601 char __user *user_data;
0839ccb8 602 int page_offset, page_length;
673a394b
EA		 603
604 user_data = (char __user *) (uintptr_t) args->data_ptr;
605 remain = args->size;
673a394b 606
05394f39 607 offset = obj->gtt_offset + args->offset;
673a394b
EA		 608
609 while (remain > 0) {
610 /* Operation in this page
611 *
0839ccb8
KP		 612 * page_base = page offset within aperture
613 * page_offset = offset within page
614 * page_length = bytes to copy for this page
673a394b 615 */
0839ccb8
KP		 616 page_base = (offset & ~(PAGE_SIZE-1));
617 page_offset = offset & (PAGE_SIZE-1);
618 page_length = remain;
619 if ((page_offset + remain) > PAGE_SIZE)
620 page_length = PAGE_SIZE - page_offset;
621
0839ccb8 622 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 623 * source page isn't available. Return the error and we'll
624 * retry in the slow path.
0839ccb8 625 */
fbd5a26d
CW		 626 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
627 page_offset, user_data, page_length))
628
629 return -EFAULT;
673a394b 630
0839ccb8
KP		 631 remain -= page_length;
632 user_data += page_length;
633 offset += page_length;
673a394b 634 }
673a394b 635
fbd5a26d 636 return 0;
673a394b
EA		 637}
638
3de09aa3
EA		 639/**
640 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
641 * the memory and maps it using kmap_atomic for copying.
642 *
643 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
644 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
645 */
3043c60c 646static int
05394f39
CW		 647i915_gem_gtt_pwrite_slow(struct drm_device *dev,
648 struct drm_i915_gem_object *obj,
3de09aa3 649 struct drm_i915_gem_pwrite *args,
05394f39 650 struct drm_file *file)
673a394b 651{
3de09aa3
EA		 652 drm_i915_private_t *dev_priv = dev->dev_private;
653 ssize_t remain;
654 loff_t gtt_page_base, offset;
655 loff_t first_data_page, last_data_page, num_pages;
656 loff_t pinned_pages, i;
657 struct page **user_pages;
658 struct mm_struct *mm = current->mm;
659 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 660 int ret;
3de09aa3
EA		 661 uint64_t data_ptr = args->data_ptr;
662
663 remain = args->size;
664
665 /* Pin the user pages containing the data. We can't fault while
666 * holding the struct mutex, and all of the pwrite implementations
667 * want to hold it while dereferencing the user data.
668 */
669 first_data_page = data_ptr / PAGE_SIZE;
670 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
671 num_pages = last_data_page - first_data_page + 1;
672
fbd5a26d 673 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
3de09aa3
EA		 674 if (user_pages == NULL)
675 return -ENOMEM;
676
fbd5a26d 677 mutex_unlock(&dev->struct_mutex);
3de09aa3
EA		 678 down_read(&mm->mmap_sem);
679 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
680 num_pages, 0, 0, user_pages, NULL);
681 up_read(&mm->mmap_sem);
fbd5a26d 682 mutex_lock(&dev->struct_mutex);
3de09aa3
EA		 683 if (pinned_pages < num_pages) {
684 ret = -EFAULT;
685 goto out_unpin_pages;
686 }
673a394b 687
d9e86c0e
CW		 688 ret = i915_gem_object_set_to_gtt_domain(obj, true);
689 if (ret)
690 goto out_unpin_pages;
691
692 ret = i915_gem_object_put_fence(obj);
3de09aa3 693 if (ret)
fbd5a26d 694 goto out_unpin_pages;
3de09aa3 695
05394f39 696 offset = obj->gtt_offset + args->offset;
3de09aa3
EA		 697
698 while (remain > 0) {
699 /* Operation in this page
700 *
701 * gtt_page_base = page offset within aperture
702 * gtt_page_offset = offset within page in aperture
703 * data_page_index = page number in get_user_pages return
704 * data_page_offset = offset with data_page_index page.
705 * page_length = bytes to copy for this page
706 */
707 gtt_page_base = offset & PAGE_MASK;
708 gtt_page_offset = offset & ~PAGE_MASK;
709 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
710 data_page_offset = data_ptr & ~PAGE_MASK;
711
712 page_length = remain;
713 if ((gtt_page_offset + page_length) > PAGE_SIZE)
714 page_length = PAGE_SIZE - gtt_page_offset;
715 if ((data_page_offset + page_length) > PAGE_SIZE)
716 page_length = PAGE_SIZE - data_page_offset;
717
ab34c226
CW		 718 slow_kernel_write(dev_priv->mm.gtt_mapping,
719 gtt_page_base, gtt_page_offset,
720 user_pages[data_page_index],
721 data_page_offset,
722 page_length);
3de09aa3
EA		 723
724 remain -= page_length;
725 offset += page_length;
726 data_ptr += page_length;
727 }
728
3de09aa3
EA		 729out_unpin_pages:
730 for (i = 0; i < pinned_pages; i++)
731 page_cache_release(user_pages[i]);
8e7d2b2c 732 drm_free_large(user_pages);
3de09aa3
EA		 733
734 return ret;
735}
736
40123c1f
EA		 737/**
738 * This is the fast shmem pwrite path, which attempts to directly
739 * copy_from_user into the kmapped pages backing the object.
740 */
3043c60c 741static int
05394f39
CW		 742i915_gem_shmem_pwrite_fast(struct drm_device *dev,
743 struct drm_i915_gem_object *obj,
40123c1f 744 struct drm_i915_gem_pwrite *args,
05394f39 745 struct drm_file *file)
673a394b 746{
05394f39 747 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
40123c1f 748 ssize_t remain;
e5281ccd 749 loff_t offset;
40123c1f
EA		 750 char __user *user_data;
751 int page_offset, page_length;
40123c1f
EA		 752
753 user_data = (char __user *) (uintptr_t) args->data_ptr;
754 remain = args->size;
673a394b 755
40123c1f 756 offset = args->offset;
05394f39 757 obj->dirty = 1;
40123c1f
EA		 758
759 while (remain > 0) {
e5281ccd
CW		 760 struct page *page;
761 char *vaddr;
762 int ret;
763
40123c1f
EA		 764 /* Operation in this page
765 *
40123c1f
EA		 766 * page_offset = offset within page
767 * page_length = bytes to copy for this page
768 */
40123c1f
EA		 769 page_offset = offset & (PAGE_SIZE-1);
770 page_length = remain;
771 if ((page_offset + remain) > PAGE_SIZE)
772 page_length = PAGE_SIZE - page_offset;
773
e5281ccd
CW		 774 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
775 GFP_HIGHUSER | __GFP_RECLAIMABLE);
776 if (IS_ERR(page))
777 return PTR_ERR(page);
778
779 vaddr = kmap_atomic(page, KM_USER0);
780 ret = __copy_from_user_inatomic(vaddr + page_offset,
781 user_data,
782 page_length);
783 kunmap_atomic(vaddr, KM_USER0);
784
785 set_page_dirty(page);
786 mark_page_accessed(page);
787 page_cache_release(page);
788
789 /* If we get a fault while copying data, then (presumably) our
790 * source page isn't available. Return the error and we'll
791 * retry in the slow path.
792 */
793 if (ret)
fbd5a26d 794 return -EFAULT;
40123c1f
EA		 795
796 remain -= page_length;
797 user_data += page_length;
798 offset += page_length;
799 }
800
fbd5a26d 801 return 0;
40123c1f
EA		 802}
803
804/**
805 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
806 * the memory and maps it using kmap_atomic for copying.
807 *
808 * This avoids taking mmap_sem for faulting on the user's address while the
809 * struct_mutex is held.
810 */
811static int
05394f39
CW		 812i915_gem_shmem_pwrite_slow(struct drm_device *dev,
813 struct drm_i915_gem_object *obj,
40123c1f 814 struct drm_i915_gem_pwrite *args,
05394f39 815 struct drm_file *file)
40123c1f 816{
05394f39 817 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
40123c1f
EA		 818 struct mm_struct *mm = current->mm;
819 struct page **user_pages;
820 ssize_t remain;
821 loff_t offset, pinned_pages, i;
822 loff_t first_data_page, last_data_page, num_pages;
e5281ccd 823 int shmem_page_offset;
40123c1f
EA		 824 int data_page_index, data_page_offset;
825 int page_length;
826 int ret;
827 uint64_t data_ptr = args->data_ptr;
280b713b 828 int do_bit17_swizzling;
40123c1f
EA		 829
830 remain = args->size;
831
832 /* Pin the user pages containing the data. We can't fault while
833 * holding the struct mutex, and all of the pwrite implementations
834 * want to hold it while dereferencing the user data.
835 */
836 first_data_page = data_ptr / PAGE_SIZE;
837 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
838 num_pages = last_data_page - first_data_page + 1;
839
4f27b75d 840 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
40123c1f
EA		 841 if (user_pages == NULL)
842 return -ENOMEM;
843
fbd5a26d 844 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 845 down_read(&mm->mmap_sem);
846 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
847 num_pages, 0, 0, user_pages, NULL);
848 up_read(&mm->mmap_sem);
fbd5a26d 849 mutex_lock(&dev->struct_mutex);
40123c1f
EA		 850 if (pinned_pages < num_pages) {
851 ret = -EFAULT;
fbd5a26d 852 goto out;
673a394b
EA		 853 }
854
fbd5a26d 855 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
07f73f69 856 if (ret)
fbd5a26d 857 goto out;
40123c1f 858
fbd5a26d 859 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
40123c1f 860
673a394b 861 offset = args->offset;
05394f39 862 obj->dirty = 1;
673a394b 863
40123c1f 864 while (remain > 0) {
e5281ccd
CW		 865 struct page *page;
866
40123c1f
EA		 867 /* Operation in this page
868 *
40123c1f
EA		 869 * shmem_page_offset = offset within page in shmem file
870 * data_page_index = page number in get_user_pages return
871 * data_page_offset = offset with data_page_index page.
872 * page_length = bytes to copy for this page
873 */
40123c1f
EA		 874 shmem_page_offset = offset & ~PAGE_MASK;
875 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
876 data_page_offset = data_ptr & ~PAGE_MASK;
877
878 page_length = remain;
879 if ((shmem_page_offset + page_length) > PAGE_SIZE)
880 page_length = PAGE_SIZE - shmem_page_offset;
881 if ((data_page_offset + page_length) > PAGE_SIZE)
882 page_length = PAGE_SIZE - data_page_offset;
883
e5281ccd
CW		 884 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
885 GFP_HIGHUSER | __GFP_RECLAIMABLE);
886 if (IS_ERR(page)) {
887 ret = PTR_ERR(page);
888 goto out;
889 }
890
280b713b 891 if (do_bit17_swizzling) {
e5281ccd 892 slow_shmem_bit17_copy(page,
280b713b
EA		 893 shmem_page_offset,
894 user_pages[data_page_index],
895 data_page_offset,
99a03df5
CW		 896 page_length,
897 0);
898 } else {
e5281ccd 899 slow_shmem_copy(page,
99a03df5
CW		 900 shmem_page_offset,
901 user_pages[data_page_index],
902 data_page_offset,
903 page_length);
280b713b 904 }
40123c1f 905
e5281ccd
CW		 906 set_page_dirty(page);
907 mark_page_accessed(page);
908 page_cache_release(page);
909
40123c1f
EA		 910 remain -= page_length;
911 data_ptr += page_length;
912 offset += page_length;
673a394b
EA		 913 }
914
fbd5a26d 915out:
40123c1f
EA		 916 for (i = 0; i < pinned_pages; i++)
917 page_cache_release(user_pages[i]);
8e7d2b2c 918 drm_free_large(user_pages);
673a394b 919
40123c1f 920 return ret;
673a394b
EA		 921}
922
923/**
924 * Writes data to the object referenced by handle.
925 *
926 * On error, the contents of the buffer that were to be modified are undefined.
927 */
928int
929i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
fbd5a26d 930 struct drm_file *file)
673a394b
EA		 931{
932 struct drm_i915_gem_pwrite *args = data;
05394f39 933 struct drm_i915_gem_object *obj;
51311d0a
CW		 934 int ret;
935
936 if (args->size == 0)
937 return 0;
938
939 if (!access_ok(VERIFY_READ,
940 (char __user *)(uintptr_t)args->data_ptr,
941 args->size))
942 return -EFAULT;
943
944 ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
945 args->size);
946 if (ret)
947 return -EFAULT;
673a394b 948
fbd5a26d 949 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 950 if (ret)
fbd5a26d 951 return ret;
1d7cfea1 952
05394f39 953 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1d7cfea1
CW		 954 if (obj == NULL) {
955 ret = -ENOENT;
956 goto unlock;
fbd5a26d 957 }
673a394b 958
7dcd2499 959 /* Bounds check destination. */
05394f39
CW		 960 if (args->offset > obj->base.size ||
961 args->size > obj->base.size - args->offset) {
ce9d419d 962 ret = -EINVAL;
35b62a89 963 goto out;
ce9d419d
CW		 964 }
965
673a394b
EA		 966 /* We can only do the GTT pwrite on untiled buffers, as otherwise
967 * it would end up going through the fenced access, and we'll get
968 * different detiling behavior between reading and writing.
969 * pread/pwrite currently are reading and writing from the CPU
970 * perspective, requiring manual detiling by the client.
971 */
05394f39 972 if (obj->phys_obj)
fbd5a26d 973 ret = i915_gem_phys_pwrite(dev, obj, args, file);
d9e86c0e 974 else if (obj->gtt_space &&
05394f39 975 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
75e9e915 976 ret = i915_gem_object_pin(obj, 0, true);
fbd5a26d
CW		 977 if (ret)
978 goto out;
979
d9e86c0e
CW		 980 ret = i915_gem_object_set_to_gtt_domain(obj, true);
981 if (ret)
982 goto out_unpin;
983
984 ret = i915_gem_object_put_fence(obj);
fbd5a26d
CW		 985 if (ret)
986 goto out_unpin;
987
988 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
989 if (ret == -EFAULT)
990 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
991
992out_unpin:
993 i915_gem_object_unpin(obj);
40123c1f 994 } else {
fbd5a26d
CW		 995 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
996 if (ret)
e5281ccd 997 goto out;
673a394b 998
fbd5a26d
CW		 999 ret = -EFAULT;
1000 if (!i915_gem_object_needs_bit17_swizzle(obj))
1001 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1002 if (ret == -EFAULT)
1003 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
fbd5a26d 1004 }
673a394b 1005
35b62a89 1006out:
05394f39 1007 drm_gem_object_unreference(&obj->base);
1d7cfea1 1008unlock:
fbd5a26d 1009 mutex_unlock(&dev->struct_mutex);
673a394b
EA		 1010 return ret;
1011}
1012
1013/**
2ef7eeaa
EA		 1014 * Called when user space prepares to use an object with the CPU, either
1015 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 1016 */
1017int
1018i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
05394f39 1019 struct drm_file *file)
673a394b
EA		 1020{
1021 struct drm_i915_gem_set_domain *args = data;
05394f39 1022 struct drm_i915_gem_object *obj;
2ef7eeaa
EA		 1023 uint32_t read_domains = args->read_domains;
1024 uint32_t write_domain = args->write_domain;
673a394b
EA		 1025 int ret;
1026
1027 if (!(dev->driver->driver_features & DRIVER_GEM))
1028 return -ENODEV;
1029
2ef7eeaa 1030 /* Only handle setting domains to types used by the CPU. */
21d509e3 1031 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1032 return -EINVAL;
1033
21d509e3 1034 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1035 return -EINVAL;
1036
1037 /* Having something in the write domain implies it's in the read
1038 * domain, and only that read domain. Enforce that in the request.
1039 */
1040 if (write_domain != 0 && read_domains != write_domain)
1041 return -EINVAL;
1042
76c1dec1 1043 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1044 if (ret)
76c1dec1 1045 return ret;
1d7cfea1 1046
05394f39 1047 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1d7cfea1
CW		 1048 if (obj == NULL) {
1049 ret = -ENOENT;
1050 goto unlock;
76c1dec1 1051 }
673a394b 1052
2ef7eeaa
EA		 1053 if (read_domains & I915_GEM_DOMAIN_GTT) {
1054 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392
EA		 1055
1056 /* Silently promote "you're not bound, there was nothing to do"
1057 * to success, since the client was just asking us to
1058 * make sure everything was done.
1059 */
1060 if (ret == -EINVAL)
1061 ret = 0;
2ef7eeaa 1062 } else {
e47c68e9 1063 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1064 }
1065
05394f39 1066 drm_gem_object_unreference(&obj->base);
1d7cfea1 1067unlock:
673a394b
EA		 1068 mutex_unlock(&dev->struct_mutex);
1069 return ret;
1070}
1071
1072/**
1073 * Called when user space has done writes to this buffer
1074 */
1075int
1076i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
05394f39 1077 struct drm_file *file)
673a394b
EA		 1078{
1079 struct drm_i915_gem_sw_finish *args = data;
05394f39 1080 struct drm_i915_gem_object *obj;
673a394b
EA		 1081 int ret = 0;
1082
1083 if (!(dev->driver->driver_features & DRIVER_GEM))
1084 return -ENODEV;
1085
76c1dec1 1086 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1087 if (ret)
76c1dec1 1088 return ret;
1d7cfea1 1089
05394f39 1090 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
673a394b 1091 if (obj == NULL) {
1d7cfea1
CW		 1092 ret = -ENOENT;
1093 goto unlock;
673a394b
EA		 1094 }
1095
673a394b 1096 /* Pinned buffers may be scanout, so flush the cache */
05394f39 1097 if (obj->pin_count)
e47c68e9
EA		 1098 i915_gem_object_flush_cpu_write_domain(obj);
1099
05394f39 1100 drm_gem_object_unreference(&obj->base);
1d7cfea1 1101unlock:
673a394b
EA		 1102 mutex_unlock(&dev->struct_mutex);
1103 return ret;
1104}
1105
1106/**
1107 * Maps the contents of an object, returning the address it is mapped
1108 * into.
1109 *
1110 * While the mapping holds a reference on the contents of the object, it doesn't
1111 * imply a ref on the object itself.
1112 */
1113int
1114i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
05394f39 1115 struct drm_file *file)
673a394b 1116{
da761a6e 1117 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 1118 struct drm_i915_gem_mmap *args = data;
1119 struct drm_gem_object *obj;
1120 loff_t offset;
1121 unsigned long addr;
1122
1123 if (!(dev->driver->driver_features & DRIVER_GEM))
1124 return -ENODEV;
1125
05394f39 1126 obj = drm_gem_object_lookup(dev, file, args->handle);
673a394b 1127 if (obj == NULL)
bf79cb91 1128 return -ENOENT;
673a394b 1129
da761a6e
CW		 1130 if (obj->size > dev_priv->mm.gtt_mappable_end) {
1131 drm_gem_object_unreference_unlocked(obj);
1132 return -E2BIG;
1133 }
1134
673a394b
EA		 1135 offset = args->offset;
1136
1137 down_write(&current->mm->mmap_sem);
1138 addr = do_mmap(obj->filp, 0, args->size,
1139 PROT_READ | PROT_WRITE, MAP_SHARED,
1140 args->offset);
1141 up_write(&current->mm->mmap_sem);
bc9025bd 1142 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1143 if (IS_ERR((void *)addr))
1144 return addr;
1145
1146 args->addr_ptr = (uint64_t) addr;
1147
1148 return 0;
1149}
1150
de151cf6
JB		 1151/**
1152 * i915_gem_fault - fault a page into the GTT
1153 * vma: VMA in question
1154 * vmf: fault info
1155 *
1156 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1157 * from userspace. The fault handler takes care of binding the object to
1158 * the GTT (if needed), allocating and programming a fence register (again,
1159 * only if needed based on whether the old reg is still valid or the object
1160 * is tiled) and inserting a new PTE into the faulting process.
1161 *
1162 * Note that the faulting process may involve evicting existing objects
1163 * from the GTT and/or fence registers to make room. So performance may
1164 * suffer if the GTT working set is large or there are few fence registers
1165 * left.
1166 */
1167int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1168{
05394f39
CW		 1169 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1170 struct drm_device *dev = obj->base.dev;
7d1c4804 1171 drm_i915_private_t *dev_priv = dev->dev_private;
de151cf6
JB		 1172 pgoff_t page_offset;
1173 unsigned long pfn;
1174 int ret = 0;
0f973f27 1175 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1176
1177 /* We don't use vmf->pgoff since that has the fake offset */
1178 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1179 PAGE_SHIFT;
1180
1181 /* Now bind it into the GTT if needed */
1182 mutex_lock(&dev->struct_mutex);
a00b10c3 1183
919926ae
CW		 1184 if (!obj->map_and_fenceable) {
1185 ret = i915_gem_object_unbind(obj);
1186 if (ret)
1187 goto unlock;
a00b10c3 1188 }
05394f39 1189 if (!obj->gtt_space) {
75e9e915 1190 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
c715089f
CW		 1191 if (ret)
1192 goto unlock;
de151cf6
JB		 1193 }
1194
4a684a41
CW		 1195 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1196 if (ret)
1197 goto unlock;
1198
d9e86c0e
CW		 1199 if (obj->tiling_mode == I915_TILING_NONE)
1200 ret = i915_gem_object_put_fence(obj);
1201 else
1202 ret = i915_gem_object_get_fence(obj, NULL, true);
1203 if (ret)
1204 goto unlock;
de151cf6 1205
05394f39
CW		 1206 if (i915_gem_object_is_inactive(obj))
1207 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
7d1c4804 1208
6299f992
CW		 1209 obj->fault_mappable = true;
1210
05394f39 1211 pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
de151cf6
JB		 1212 page_offset;
1213
1214 /* Finally, remap it using the new GTT offset */
1215 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1216unlock:
de151cf6
JB		 1217 mutex_unlock(&dev->struct_mutex);
1218
1219 switch (ret) {
045e769a
CW		 1220 case -EAGAIN:
1221 set_need_resched();
c715089f
CW		 1222 case 0:
1223 case -ERESTARTSYS:
1224 return VM_FAULT_NOPAGE;
de151cf6 1225 case -ENOMEM:
de151cf6 1226 return VM_FAULT_OOM;
de151cf6 1227 default:
c715089f 1228 return VM_FAULT_SIGBUS;
de151cf6
JB		 1229 }
1230}
1231
1232/**
1233 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1234 * @obj: obj in question
1235 *
1236 * GEM memory mapping works by handing back to userspace a fake mmap offset
1237 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1238 * up the object based on the offset and sets up the various memory mapping
1239 * structures.
1240 *
1241 * This routine allocates and attaches a fake offset for @obj.
1242 */
1243static int
05394f39 1244i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
de151cf6 1245{
05394f39 1246 struct drm_device *dev = obj->base.dev;
de151cf6 1247 struct drm_gem_mm *mm = dev->mm_private;
de151cf6 1248 struct drm_map_list *list;
f77d390c 1249 struct drm_local_map *map;
de151cf6
JB		 1250 int ret = 0;
1251
1252 /* Set the object up for mmap'ing */
05394f39 1253 list = &obj->base.map_list;
9a298b2a 1254 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
de151cf6
JB		 1255 if (!list->map)
1256 return -ENOMEM;
1257
1258 map = list->map;
1259 map->type = _DRM_GEM;
05394f39 1260 map->size = obj->base.size;
de151cf6
JB		 1261 map->handle = obj;
1262
1263 /* Get a DRM GEM mmap offset allocated... */
1264 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
05394f39
CW		 1265 obj->base.size / PAGE_SIZE,
1266 0, 0);
de151cf6 1267 if (!list->file_offset_node) {
05394f39
CW		 1268 DRM_ERROR("failed to allocate offset for bo %d\n",
1269 obj->base.name);
9e0ae534 1270 ret = -ENOSPC;
de151cf6
JB		 1271 goto out_free_list;
1272 }
1273
1274 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
05394f39
CW		 1275 obj->base.size / PAGE_SIZE,
1276 0);
de151cf6
JB		 1277 if (!list->file_offset_node) {
1278 ret = -ENOMEM;
1279 goto out_free_list;
1280 }
1281
1282 list->hash.key = list->file_offset_node->start;
9e0ae534
CW		 1283 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1284 if (ret) {
de151cf6
JB		 1285 DRM_ERROR("failed to add to map hash\n");
1286 goto out_free_mm;
1287 }
1288
de151cf6
JB		 1289 return 0;
1290
1291out_free_mm:
1292 drm_mm_put_block(list->file_offset_node);
1293out_free_list:
9a298b2a 1294 kfree(list->map);
39a01d1f 1295 list->map = NULL;
de151cf6
JB		 1296
1297 return ret;
1298}
1299
901782b2
CW		 1300/**
1301 * i915_gem_release_mmap - remove physical page mappings
1302 * @obj: obj in question
1303 *
af901ca1 1304 * Preserve the reservation of the mmapping with the DRM core code, but
901782b2
CW		 1305 * relinquish ownership of the pages back to the system.
1306 *
1307 * It is vital that we remove the page mapping if we have mapped a tiled
1308 * object through the GTT and then lose the fence register due to
1309 * resource pressure. Similarly if the object has been moved out of the
1310 * aperture, than pages mapped into userspace must be revoked. Removing the
1311 * mapping will then trigger a page fault on the next user access, allowing
1312 * fixup by i915_gem_fault().
1313 */
d05ca301 1314void
05394f39 1315i915_gem_release_mmap(struct drm_i915_gem_object *obj)
901782b2 1316{
6299f992
CW		 1317 if (!obj->fault_mappable)
1318 return;
901782b2 1319
6299f992
CW		 1320 unmap_mapping_range(obj->base.dev->dev_mapping,
1321 (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1322 obj->base.size, 1);
fb7d516a 1323
6299f992 1324 obj->fault_mappable = false;
901782b2
CW		 1325}
1326
ab00b3e5 1327static void
05394f39 1328i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
ab00b3e5 1329{
05394f39 1330 struct drm_device *dev = obj->base.dev;
ab00b3e5 1331 struct drm_gem_mm *mm = dev->mm_private;
05394f39 1332 struct drm_map_list *list = &obj->base.map_list;
ab00b3e5 1333
ab00b3e5 1334 drm_ht_remove_item(&mm->offset_hash, &list->hash);
39a01d1f
CW		 1335 drm_mm_put_block(list->file_offset_node);
1336 kfree(list->map);
1337 list->map = NULL;
ab00b3e5
JB		 1338}
1339
92b88aeb
CW		 1340static uint32_t
1341i915_gem_get_gtt_size(struct drm_i915_gem_object *obj)
1342{
1343 struct drm_device *dev = obj->base.dev;
1344 uint32_t size;
1345
1346 if (INTEL_INFO(dev)->gen >= 4 ||
1347 obj->tiling_mode == I915_TILING_NONE)
1348 return obj->base.size;
1349
1350 /* Previous chips need a power-of-two fence region when tiling */
1351 if (INTEL_INFO(dev)->gen == 3)
1352 size = 1024*1024;
1353 else
1354 size = 512*1024;
1355
1356 while (size < obj->base.size)
1357 size <<= 1;
1358
1359 return size;
1360}
1361
de151cf6
JB		 1362/**
1363 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1364 * @obj: object to check
1365 *
1366 * Return the required GTT alignment for an object, taking into account
5e783301 1367 * potential fence register mapping.
de151cf6
JB		 1368 */
1369static uint32_t
05394f39 1370i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj)
de151cf6 1371{
05394f39 1372 struct drm_device *dev = obj->base.dev;
de151cf6
JB		 1373
1374 /*
1375 * Minimum alignment is 4k (GTT page size), but might be greater
1376 * if a fence register is needed for the object.
1377 */
a00b10c3 1378 if (INTEL_INFO(dev)->gen >= 4 ||
05394f39 1379 obj->tiling_mode == I915_TILING_NONE)
de151cf6
JB		 1380 return 4096;
1381
a00b10c3
CW		 1382 /*
1383 * Previous chips need to be aligned to the size of the smallest
1384 * fence register that can contain the object.
1385 */
05394f39 1386 return i915_gem_get_gtt_size(obj);
a00b10c3
CW		 1387}
1388
5e783301
DV		 1389/**
1390 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1391 * unfenced object
1392 * @obj: object to check
1393 *
1394 * Return the required GTT alignment for an object, only taking into account
1395 * unfenced tiled surface requirements.
1396 */
1397static uint32_t
05394f39 1398i915_gem_get_unfenced_gtt_alignment(struct drm_i915_gem_object *obj)
5e783301 1399{
05394f39 1400 struct drm_device *dev = obj->base.dev;
5e783301
DV		 1401 int tile_height;
1402
1403 /*
1404 * Minimum alignment is 4k (GTT page size) for sane hw.
1405 */
1406 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
05394f39 1407 obj->tiling_mode == I915_TILING_NONE)
5e783301
DV		 1408 return 4096;
1409
1410 /*
1411 * Older chips need unfenced tiled buffers to be aligned to the left
1412 * edge of an even tile row (where tile rows are counted as if the bo is
1413 * placed in a fenced gtt region).
1414 */
1415 if (IS_GEN2(dev) ||
05394f39 1416 (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
5e783301
DV		 1417 tile_height = 32;
1418 else
1419 tile_height = 8;
1420
05394f39 1421 return tile_height * obj->stride * 2;
5e783301
DV		 1422}
1423
de151cf6
JB		 1424/**
1425 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1426 * @dev: DRM device
1427 * @data: GTT mapping ioctl data
05394f39 1428 * @file: GEM object info
de151cf6
JB		 1429 *
1430 * Simply returns the fake offset to userspace so it can mmap it.
1431 * The mmap call will end up in drm_gem_mmap(), which will set things
1432 * up so we can get faults in the handler above.
1433 *
1434 * The fault handler will take care of binding the object into the GTT
1435 * (since it may have been evicted to make room for something), allocating
1436 * a fence register, and mapping the appropriate aperture address into
1437 * userspace.
1438 */
1439int
1440i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
05394f39 1441 struct drm_file *file)
de151cf6 1442{
da761a6e 1443 struct drm_i915_private *dev_priv = dev->dev_private;
de151cf6 1444 struct drm_i915_gem_mmap_gtt *args = data;
05394f39 1445 struct drm_i915_gem_object *obj;
de151cf6
JB		 1446 int ret;
1447
1448 if (!(dev->driver->driver_features & DRIVER_GEM))
1449 return -ENODEV;
1450
76c1dec1 1451 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1452 if (ret)
76c1dec1 1453 return ret;
de151cf6 1454
05394f39 1455 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1d7cfea1
CW		 1456 if (obj == NULL) {
1457 ret = -ENOENT;
1458 goto unlock;
1459 }
de151cf6 1460
05394f39 1461 if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
da761a6e
CW		 1462 ret = -E2BIG;
1463 goto unlock;
1464 }
1465
05394f39 1466 if (obj->madv != I915_MADV_WILLNEED) {
ab18282d 1467 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1d7cfea1
CW		 1468 ret = -EINVAL;
1469 goto out;
ab18282d
CW		 1470 }
1471
05394f39 1472 if (!obj->base.map_list.map) {
de151cf6 1473 ret = i915_gem_create_mmap_offset(obj);
1d7cfea1
CW		 1474 if (ret)
1475 goto out;
de151cf6
JB		 1476 }
1477
05394f39 1478 args->offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
de151cf6 1479
1d7cfea1 1480out:
05394f39 1481 drm_gem_object_unreference(&obj->base);
1d7cfea1 1482unlock:
de151cf6 1483 mutex_unlock(&dev->struct_mutex);
1d7cfea1 1484 return ret;
de151cf6
JB		 1485}
1486
e5281ccd 1487static int
05394f39 1488i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
e5281ccd
CW		 1489 gfp_t gfpmask)
1490{
e5281ccd
CW		 1491 int page_count, i;
1492 struct address_space *mapping;
1493 struct inode *inode;
1494 struct page *page;
1495
1496 /* Get the list of pages out of our struct file. They'll be pinned
1497 * at this point until we release them.
1498 */
05394f39
CW		 1499 page_count = obj->base.size / PAGE_SIZE;
1500 BUG_ON(obj->pages != NULL);
1501 obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1502 if (obj->pages == NULL)
e5281ccd
CW		 1503 return -ENOMEM;
1504
05394f39 1505 inode = obj->base.filp->f_path.dentry->d_inode;
e5281ccd
CW		 1506 mapping = inode->i_mapping;
1507 for (i = 0; i < page_count; i++) {
1508 page = read_cache_page_gfp(mapping, i,
1509 GFP_HIGHUSER |
1510 __GFP_COLD |
1511 __GFP_RECLAIMABLE |
1512 gfpmask);
1513 if (IS_ERR(page))
1514 goto err_pages;
1515
05394f39 1516 obj->pages[i] = page;
e5281ccd
CW		 1517 }
1518
05394f39 1519 if (obj->tiling_mode != I915_TILING_NONE)
e5281ccd
CW		 1520 i915_gem_object_do_bit_17_swizzle(obj);
1521
1522 return 0;
1523
1524err_pages:
1525 while (i--)
05394f39 1526 page_cache_release(obj->pages[i]);
e5281ccd 1527
05394f39
CW		 1528 drm_free_large(obj->pages);
1529 obj->pages = NULL;
e5281ccd
CW		 1530 return PTR_ERR(page);
1531}
1532
5cdf5881 1533static void
05394f39 1534i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
673a394b 1535{
05394f39 1536 int page_count = obj->base.size / PAGE_SIZE;
673a394b
EA		 1537 int i;
1538
05394f39 1539 BUG_ON(obj->madv == __I915_MADV_PURGED);
673a394b 1540
05394f39 1541 if (obj->tiling_mode != I915_TILING_NONE)
280b713b
EA		 1542 i915_gem_object_save_bit_17_swizzle(obj);
1543
05394f39
CW		 1544 if (obj->madv == I915_MADV_DONTNEED)
1545 obj->dirty = 0;
3ef94daa
CW		 1546
1547 for (i = 0; i < page_count; i++) {
05394f39
CW		 1548 if (obj->dirty)
1549 set_page_dirty(obj->pages[i]);
3ef94daa 1550
05394f39
CW		 1551 if (obj->madv == I915_MADV_WILLNEED)
1552 mark_page_accessed(obj->pages[i]);
3ef94daa 1553
05394f39 1554 page_cache_release(obj->pages[i]);
3ef94daa 1555 }
05394f39 1556 obj->dirty = 0;
673a394b 1557
05394f39
CW		 1558 drm_free_large(obj->pages);
1559 obj->pages = NULL;
673a394b
EA		 1560}
1561
54cf91dc 1562void
05394f39 1563i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1ec14ad3
CW		 1564 struct intel_ring_buffer *ring,
1565 u32 seqno)
673a394b 1566{
05394f39 1567 struct drm_device *dev = obj->base.dev;
69dc4987 1568 struct drm_i915_private *dev_priv = dev->dev_private;
617dbe27 1569
852835f3 1570 BUG_ON(ring == NULL);
05394f39 1571 obj->ring = ring;
673a394b
EA		 1572
1573 /* Add a reference if we're newly entering the active list. */
05394f39
CW		 1574 if (!obj->active) {
1575 drm_gem_object_reference(&obj->base);
1576 obj->active = 1;
673a394b 1577 }
e35a41de 1578
673a394b 1579 /* Move from whatever list we were on to the tail of execution. */
05394f39
CW		 1580 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1581 list_move_tail(&obj->ring_list, &ring->active_list);
caea7476 1582
05394f39 1583 obj->last_rendering_seqno = seqno;
caea7476
CW		 1584 if (obj->fenced_gpu_access) {
1585 struct drm_i915_fence_reg *reg;
1586
1587 BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
1588
1589 obj->last_fenced_seqno = seqno;
1590 obj->last_fenced_ring = ring;
1591
1592 reg = &dev_priv->fence_regs[obj->fence_reg];
1593 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1594 }
1595}
1596
1597static void
1598i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1599{
1600 list_del_init(&obj->ring_list);
1601 obj->last_rendering_seqno = 0;
673a394b
EA		 1602}
1603
ce44b0ea 1604static void
05394f39 1605i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
ce44b0ea 1606{
05394f39 1607 struct drm_device *dev = obj->base.dev;
ce44b0ea 1608 drm_i915_private_t *dev_priv = dev->dev_private;
ce44b0ea 1609
05394f39
CW		 1610 BUG_ON(!obj->active);
1611 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
caea7476
CW		 1612
1613 i915_gem_object_move_off_active(obj);
1614}
1615
1616static void
1617i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1618{
1619 struct drm_device *dev = obj->base.dev;
1620 struct drm_i915_private *dev_priv = dev->dev_private;
1621
1622 if (obj->pin_count != 0)
1623 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1624 else
1625 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1626
1627 BUG_ON(!list_empty(&obj->gpu_write_list));
1628 BUG_ON(!obj->active);
1629 obj->ring = NULL;
1630
1631 i915_gem_object_move_off_active(obj);
1632 obj->fenced_gpu_access = false;
caea7476
CW		 1633
1634 obj->active = 0;
87ca9c8a 1635 obj->pending_gpu_write = false;
caea7476
CW		 1636 drm_gem_object_unreference(&obj->base);
1637
1638 WARN_ON(i915_verify_lists(dev));
ce44b0ea 1639}
673a394b 1640
963b4836
CW		 1641/* Immediately discard the backing storage */
1642static void
05394f39 1643i915_gem_object_truncate(struct drm_i915_gem_object *obj)
963b4836 1644{
bb6baf76 1645 struct inode *inode;
963b4836 1646
ae9fed6b
CW		 1647 /* Our goal here is to return as much of the memory as
1648 * is possible back to the system as we are called from OOM.
1649 * To do this we must instruct the shmfs to drop all of its
1650 * backing pages, *now*. Here we mirror the actions taken
1651 * when by shmem_delete_inode() to release the backing store.
1652 */
05394f39 1653 inode = obj->base.filp->f_path.dentry->d_inode;
ae9fed6b
CW		 1654 truncate_inode_pages(inode->i_mapping, 0);
1655 if (inode->i_op->truncate_range)
1656 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
bb6baf76 1657
05394f39 1658 obj->madv = __I915_MADV_PURGED;
963b4836
CW		 1659}
1660
1661static inline int
05394f39 1662i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
963b4836 1663{
05394f39 1664 return obj->madv == I915_MADV_DONTNEED;
963b4836
CW		 1665}
1666
63560396
DV		 1667static void
1668i915_gem_process_flushing_list(struct drm_device *dev,
8a1a49f9 1669 uint32_t flush_domains,
852835f3 1670 struct intel_ring_buffer *ring)
63560396 1671{
05394f39 1672 struct drm_i915_gem_object *obj, *next;
63560396 1673
05394f39 1674 list_for_each_entry_safe(obj, next,
64193406 1675 &ring->gpu_write_list,
63560396 1676 gpu_write_list) {
05394f39
CW		 1677 if (obj->base.write_domain & flush_domains) {
1678 uint32_t old_write_domain = obj->base.write_domain;
63560396 1679
05394f39
CW		 1680 obj->base.write_domain = 0;
1681 list_del_init(&obj->gpu_write_list);
1ec14ad3
CW		 1682 i915_gem_object_move_to_active(obj, ring,
1683 i915_gem_next_request_seqno(dev, ring));
63560396 1684
63560396 1685 trace_i915_gem_object_change_domain(obj,
05394f39 1686 obj->base.read_domains,
63560396
DV		 1687 old_write_domain);
1688 }
1689 }
1690}
8187a2b7 1691
3cce469c 1692int
8a1a49f9 1693i915_add_request(struct drm_device *dev,
f787a5f5 1694 struct drm_file *file,
8dc5d147 1695 struct drm_i915_gem_request *request,
8a1a49f9 1696 struct intel_ring_buffer *ring)
673a394b
EA		 1697{
1698 drm_i915_private_t *dev_priv = dev->dev_private;
f787a5f5 1699 struct drm_i915_file_private *file_priv = NULL;
673a394b
EA		 1700 uint32_t seqno;
1701 int was_empty;
3cce469c
CW		 1702 int ret;
1703
1704 BUG_ON(request == NULL);
673a394b 1705
f787a5f5
CW		 1706 if (file != NULL)
1707 file_priv = file->driver_priv;
b962442e 1708
3cce469c
CW		 1709 ret = ring->add_request(ring, &seqno);
1710 if (ret)
1711 return ret;
673a394b 1712
a56ba56c 1713 ring->outstanding_lazy_request = false;
673a394b
EA		 1714
1715 request->seqno = seqno;
852835f3 1716 request->ring = ring;
673a394b 1717 request->emitted_jiffies = jiffies;
852835f3
ZN		 1718 was_empty = list_empty(&ring->request_list);
1719 list_add_tail(&request->list, &ring->request_list);
1720
f787a5f5 1721 if (file_priv) {
1c25595f 1722 spin_lock(&file_priv->mm.lock);
f787a5f5 1723 request->file_priv = file_priv;
b962442e 1724 list_add_tail(&request->client_list,
f787a5f5 1725 &file_priv->mm.request_list);
1c25595f 1726 spin_unlock(&file_priv->mm.lock);
b962442e 1727 }
673a394b 1728
f65d9421 1729 if (!dev_priv->mm.suspended) {
b3b079db
CW		 1730 mod_timer(&dev_priv->hangcheck_timer,
1731 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
f65d9421 1732 if (was_empty)
b3b079db
CW		 1733 queue_delayed_work(dev_priv->wq,
1734 &dev_priv->mm.retire_work, HZ);
f65d9421 1735 }
3cce469c 1736 return 0;
673a394b
EA		 1737}
1738
f787a5f5
CW		 1739static inline void
1740i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
673a394b 1741{
1c25595f 1742 struct drm_i915_file_private *file_priv = request->file_priv;
673a394b 1743
1c25595f
CW		 1744 if (!file_priv)
1745 return;
1c5d22f7 1746
1c25595f
CW		 1747 spin_lock(&file_priv->mm.lock);
1748 list_del(&request->client_list);
1749 request->file_priv = NULL;
1750 spin_unlock(&file_priv->mm.lock);
673a394b 1751}
673a394b 1752
dfaae392
CW		 1753static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1754 struct intel_ring_buffer *ring)
9375e446 1755{
dfaae392
CW		 1756 while (!list_empty(&ring->request_list)) {
1757 struct drm_i915_gem_request *request;
673a394b 1758
dfaae392
CW		 1759 request = list_first_entry(&ring->request_list,
1760 struct drm_i915_gem_request,
1761 list);
de151cf6 1762
dfaae392 1763 list_del(&request->list);
f787a5f5 1764 i915_gem_request_remove_from_client(request);
dfaae392
CW		 1765 kfree(request);
1766 }
673a394b 1767
dfaae392 1768 while (!list_empty(&ring->active_list)) {
05394f39 1769 struct drm_i915_gem_object *obj;
9375e446 1770
05394f39
CW		 1771 obj = list_first_entry(&ring->active_list,
1772 struct drm_i915_gem_object,
1773 ring_list);
9375e446 1774
05394f39
CW		 1775 obj->base.write_domain = 0;
1776 list_del_init(&obj->gpu_write_list);
1777 i915_gem_object_move_to_inactive(obj);
673a394b
EA		 1778 }
1779}
1780
312817a3
CW		 1781static void i915_gem_reset_fences(struct drm_device *dev)
1782{
1783 struct drm_i915_private *dev_priv = dev->dev_private;
1784 int i;
1785
1786 for (i = 0; i < 16; i++) {
1787 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
7d2cb39c
CW		 1788 struct drm_i915_gem_object *obj = reg->obj;
1789
1790 if (!obj)
1791 continue;
1792
1793 if (obj->tiling_mode)
1794 i915_gem_release_mmap(obj);
1795
d9e86c0e
CW		 1796 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1797 reg->obj->fenced_gpu_access = false;
1798 reg->obj->last_fenced_seqno = 0;
1799 reg->obj->last_fenced_ring = NULL;
1800 i915_gem_clear_fence_reg(dev, reg);
312817a3
CW		 1801 }
1802}
1803
069efc1d 1804void i915_gem_reset(struct drm_device *dev)
673a394b 1805{
77f01230 1806 struct drm_i915_private *dev_priv = dev->dev_private;
05394f39 1807 struct drm_i915_gem_object *obj;
1ec14ad3 1808 int i;
673a394b 1809
1ec14ad3
CW		 1810 for (i = 0; i < I915_NUM_RINGS; i++)
1811 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
dfaae392
CW		 1812
1813 /* Remove anything from the flushing lists. The GPU cache is likely
1814 * to be lost on reset along with the data, so simply move the
1815 * lost bo to the inactive list.
1816 */
1817 while (!list_empty(&dev_priv->mm.flushing_list)) {
05394f39
CW		 1818 obj= list_first_entry(&dev_priv->mm.flushing_list,
1819 struct drm_i915_gem_object,
1820 mm_list);
dfaae392 1821
05394f39
CW		 1822 obj->base.write_domain = 0;
1823 list_del_init(&obj->gpu_write_list);
1824 i915_gem_object_move_to_inactive(obj);
dfaae392
CW		 1825 }
1826
1827 /* Move everything out of the GPU domains to ensure we do any
1828 * necessary invalidation upon reuse.
1829 */
05394f39 1830 list_for_each_entry(obj,
77f01230 1831 &dev_priv->mm.inactive_list,
69dc4987 1832 mm_list)
77f01230 1833 {
05394f39 1834 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
77f01230 1835 }
069efc1d
CW		 1836
1837 /* The fence registers are invalidated so clear them out */
312817a3 1838 i915_gem_reset_fences(dev);
673a394b
EA		 1839}
1840
1841/**
1842 * This function clears the request list as sequence numbers are passed.
1843 */
b09a1fec
CW		 1844static void
1845i915_gem_retire_requests_ring(struct drm_device *dev,
1846 struct intel_ring_buffer *ring)
673a394b
EA		 1847{
1848 drm_i915_private_t *dev_priv = dev->dev_private;
1849 uint32_t seqno;
1ec14ad3 1850 int i;
673a394b 1851
b84d5f0c
CW		 1852 if (!ring->status_page.page_addr ||
1853 list_empty(&ring->request_list))
6c0594a3
KW		 1854 return;
1855
23bc5982 1856 WARN_ON(i915_verify_lists(dev));
673a394b 1857
78501eac 1858 seqno = ring->get_seqno(ring);
1ec14ad3 1859
076e2c0e 1860 for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
1ec14ad3
CW		 1861 if (seqno >= ring->sync_seqno[i])
1862 ring->sync_seqno[i] = 0;
1863
852835f3 1864 while (!list_empty(&ring->request_list)) {
673a394b 1865 struct drm_i915_gem_request *request;
673a394b 1866
852835f3 1867 request = list_first_entry(&ring->request_list,
673a394b
EA		 1868 struct drm_i915_gem_request,
1869 list);
673a394b 1870
dfaae392 1871 if (!i915_seqno_passed(seqno, request->seqno))
b84d5f0c
CW		 1872 break;
1873
1874 trace_i915_gem_request_retire(dev, request->seqno);
1875
1876 list_del(&request->list);
f787a5f5 1877 i915_gem_request_remove_from_client(request);
b84d5f0c
CW		 1878 kfree(request);
1879 }
673a394b 1880
b84d5f0c
CW		 1881 /* Move any buffers on the active list that are no longer referenced
1882 * by the ringbuffer to the flushing/inactive lists as appropriate.
1883 */
1884 while (!list_empty(&ring->active_list)) {
05394f39 1885 struct drm_i915_gem_object *obj;
b84d5f0c 1886
05394f39
CW		 1887 obj= list_first_entry(&ring->active_list,
1888 struct drm_i915_gem_object,
1889 ring_list);
673a394b 1890
05394f39 1891 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
673a394b 1892 break;
b84d5f0c 1893
05394f39 1894 if (obj->base.write_domain != 0)
b84d5f0c
CW		 1895 i915_gem_object_move_to_flushing(obj);
1896 else
1897 i915_gem_object_move_to_inactive(obj);
673a394b 1898 }
9d34e5db
CW		 1899
1900 if (unlikely (dev_priv->trace_irq_seqno &&
1901 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1ec14ad3 1902 ring->irq_put(ring);
9d34e5db
CW		 1903 dev_priv->trace_irq_seqno = 0;
1904 }
23bc5982
CW		 1905
1906 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 1907}
1908
b09a1fec
CW		 1909void
1910i915_gem_retire_requests(struct drm_device *dev)
1911{
1912 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 1913 int i;
b09a1fec 1914
be72615b 1915 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
05394f39 1916 struct drm_i915_gem_object *obj, *next;
be72615b
CW		 1917
1918 /* We must be careful that during unbind() we do not
1919 * accidentally infinitely recurse into retire requests.
1920 * Currently:
1921 * retire -> free -> unbind -> wait -> retire_ring
1922 */
05394f39 1923 list_for_each_entry_safe(obj, next,
be72615b 1924 &dev_priv->mm.deferred_free_list,
69dc4987 1925 mm_list)
05394f39 1926 i915_gem_free_object_tail(obj);
be72615b
CW		 1927 }
1928
1ec14ad3
CW		 1929 for (i = 0; i < I915_NUM_RINGS; i++)
1930 i915_gem_retire_requests_ring(dev, &dev_priv->ring[i]);
b09a1fec
CW		 1931}
1932
75ef9da2 1933static void
673a394b
EA		 1934i915_gem_retire_work_handler(struct work_struct *work)
1935{
1936 drm_i915_private_t *dev_priv;
1937 struct drm_device *dev;
0a58705b
CW		 1938 bool idle;
1939 int i;
673a394b
EA		 1940
1941 dev_priv = container_of(work, drm_i915_private_t,
1942 mm.retire_work.work);
1943 dev = dev_priv->dev;
1944
891b48cf
CW		 1945 /* Come back later if the device is busy... */
1946 if (!mutex_trylock(&dev->struct_mutex)) {
1947 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1948 return;
1949 }
1950
b09a1fec 1951 i915_gem_retire_requests(dev);
d1b851fc 1952
0a58705b
CW		 1953 /* Send a periodic flush down the ring so we don't hold onto GEM
1954 * objects indefinitely.
1955 */
1956 idle = true;
1957 for (i = 0; i < I915_NUM_RINGS; i++) {
1958 struct intel_ring_buffer *ring = &dev_priv->ring[i];
1959
1960 if (!list_empty(&ring->gpu_write_list)) {
1961 struct drm_i915_gem_request *request;
1962 int ret;
1963
1964 ret = i915_gem_flush_ring(dev, ring, 0,
1965 I915_GEM_GPU_DOMAINS);
1966 request = kzalloc(sizeof(*request), GFP_KERNEL);
1967 if (ret || request == NULL ||
1968 i915_add_request(dev, NULL, request, ring))
1969 kfree(request);
1970 }
1971
1972 idle &= list_empty(&ring->request_list);
1973 }
1974
1975 if (!dev_priv->mm.suspended && !idle)
9c9fe1f8 1976 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
0a58705b 1977
673a394b
EA		 1978 mutex_unlock(&dev->struct_mutex);
1979}
1980
5a5a0c64 1981int
852835f3 1982i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
8a1a49f9 1983 bool interruptible, struct intel_ring_buffer *ring)
673a394b
EA		 1984{
1985 drm_i915_private_t *dev_priv = dev->dev_private;
802c7eb6 1986 u32 ier;
673a394b
EA		 1987 int ret = 0;
1988
1989 BUG_ON(seqno == 0);
1990
ba1234d1 1991 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0
CW		 1992 return -EAGAIN;
1993
5d97eb69 1994 if (seqno == ring->outstanding_lazy_request) {
3cce469c
CW		 1995 struct drm_i915_gem_request *request;
1996
1997 request = kzalloc(sizeof(*request), GFP_KERNEL);
1998 if (request == NULL)
e35a41de 1999 return -ENOMEM;
3cce469c
CW		 2000
2001 ret = i915_add_request(dev, NULL, request, ring);
2002 if (ret) {
2003 kfree(request);
2004 return ret;
2005 }
2006
2007 seqno = request->seqno;
e35a41de 2008 }
ffed1d09 2009
78501eac 2010 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
bad720ff 2011 if (HAS_PCH_SPLIT(dev))
036a4a7d
ZW		 2012 ier = I915_READ(DEIER) | I915_READ(GTIER);
2013 else
2014 ier = I915_READ(IER);
802c7eb6
JB		 2015 if (!ier) {
2016 DRM_ERROR("something (likely vbetool) disabled "
2017 "interrupts, re-enabling\n");
2018 i915_driver_irq_preinstall(dev);
2019 i915_driver_irq_postinstall(dev);
2020 }
2021
1c5d22f7
CW		 2022 trace_i915_gem_request_wait_begin(dev, seqno);
2023
b2223497 2024 ring->waiting_seqno = seqno;
b13c2b96
CW		 2025 if (ring->irq_get(ring)) {
2026 if (interruptible)
2027 ret = wait_event_interruptible(ring->irq_queue,
2028 i915_seqno_passed(ring->get_seqno(ring), seqno)
2029 || atomic_read(&dev_priv->mm.wedged));
2030 else
2031 wait_event(ring->irq_queue,
2032 i915_seqno_passed(ring->get_seqno(ring), seqno)
2033 || atomic_read(&dev_priv->mm.wedged));
2034
2035 ring->irq_put(ring);
b5ba177d
CW		 2036 } else if (wait_for(i915_seqno_passed(ring->get_seqno(ring),
2037 seqno) ||
2038 atomic_read(&dev_priv->mm.wedged), 3000))
2039 ret = -EBUSY;
b2223497 2040 ring->waiting_seqno = 0;
1c5d22f7
CW		 2041
2042 trace_i915_gem_request_wait_end(dev, seqno);
673a394b 2043 }
ba1234d1 2044 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0 2045 ret = -EAGAIN;
673a394b
EA		 2046
2047 if (ret && ret != -ERESTARTSYS)
8bff917c 2048 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
78501eac 2049 __func__, ret, seqno, ring->get_seqno(ring),
8bff917c 2050 dev_priv->next_seqno);
673a394b
EA		 2051
2052 /* Directly dispatch request retiring. While we have the work queue
2053 * to handle this, the waiter on a request often wants an associated
2054 * buffer to have made it to the inactive list, and we would need
2055 * a separate wait queue to handle that.
2056 */
2057 if (ret == 0)
b09a1fec 2058 i915_gem_retire_requests_ring(dev, ring);
673a394b
EA		 2059
2060 return ret;
2061}
2062
48764bf4
DV		 2063/**
2064 * Waits for a sequence number to be signaled, and cleans up the
2065 * request and object lists appropriately for that event.
2066 */
2067static int
852835f3 2068i915_wait_request(struct drm_device *dev, uint32_t seqno,
a56ba56c 2069 struct intel_ring_buffer *ring)
48764bf4 2070{
852835f3 2071 return i915_do_wait_request(dev, seqno, 1, ring);
48764bf4
DV		 2072}
2073
673a394b
EA		 2074/**
2075 * Ensures that all rendering to the object has completed and the object is
2076 * safe to unbind from the GTT or access from the CPU.
2077 */
54cf91dc 2078int
05394f39 2079i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
2cf34d7b 2080 bool interruptible)
673a394b 2081{
05394f39 2082 struct drm_device *dev = obj->base.dev;
673a394b
EA		 2083 int ret;
2084
e47c68e9
EA		 2085 /* This function only exists to support waiting for existing rendering,
2086 * not for emitting required flushes.
673a394b 2087 */
05394f39 2088 BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 2089
2090 /* If there is rendering queued on the buffer being evicted, wait for
2091 * it.
2092 */
05394f39 2093 if (obj->active) {
2cf34d7b 2094 ret = i915_do_wait_request(dev,
05394f39 2095 obj->last_rendering_seqno,
2cf34d7b 2096 interruptible,
05394f39 2097 obj->ring);
2cf34d7b 2098 if (ret)
673a394b
EA		 2099 return ret;
2100 }
2101
2102 return 0;
2103}
2104
2105/**
2106 * Unbinds an object from the GTT aperture.
2107 */
0f973f27 2108int
05394f39 2109i915_gem_object_unbind(struct drm_i915_gem_object *obj)
673a394b 2110{
673a394b
EA		 2111 int ret = 0;
2112
05394f39 2113 if (obj->gtt_space == NULL)
673a394b
EA		 2114 return 0;
2115
05394f39 2116 if (obj->pin_count != 0) {
673a394b
EA		 2117 DRM_ERROR("Attempting to unbind pinned buffer\n");
2118 return -EINVAL;
2119 }
2120
5323fd04
EA		 2121 /* blow away mappings if mapped through GTT */
2122 i915_gem_release_mmap(obj);
2123
673a394b
EA		 2124 /* Move the object to the CPU domain to ensure that
2125 * any possible CPU writes while it's not in the GTT
2126 * are flushed when we go to remap it. This will
2127 * also ensure that all pending GPU writes are finished
2128 * before we unbind.
2129 */
e47c68e9 2130 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
8dc1775d 2131 if (ret == -ERESTARTSYS)
673a394b 2132 return ret;
8dc1775d
CW		 2133 /* Continue on if we fail due to EIO, the GPU is hung so we
2134 * should be safe and we need to cleanup or else we might
2135 * cause memory corruption through use-after-free.
2136 */
812ed492
CW		 2137 if (ret) {
2138 i915_gem_clflush_object(obj);
05394f39 2139 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
812ed492 2140 }
673a394b 2141
96b47b65 2142 /* release the fence reg _after_ flushing */
d9e86c0e
CW		 2143 ret = i915_gem_object_put_fence(obj);
2144 if (ret == -ERESTARTSYS)
2145 return ret;
96b47b65 2146
7c2e6fdf 2147 i915_gem_gtt_unbind_object(obj);
e5281ccd 2148 i915_gem_object_put_pages_gtt(obj);
673a394b 2149
6299f992 2150 list_del_init(&obj->gtt_list);
05394f39 2151 list_del_init(&obj->mm_list);
75e9e915 2152 /* Avoid an unnecessary call to unbind on rebind. */
05394f39 2153 obj->map_and_fenceable = true;
673a394b 2154
05394f39
CW		 2155 drm_mm_put_block(obj->gtt_space);
2156 obj->gtt_space = NULL;
2157 obj->gtt_offset = 0;
673a394b 2158
05394f39 2159 if (i915_gem_object_is_purgeable(obj))
963b4836
CW		 2160 i915_gem_object_truncate(obj);
2161
1c5d22f7
CW		 2162 trace_i915_gem_object_unbind(obj);
2163
8dc1775d 2164 return ret;
673a394b
EA		 2165}
2166
88241785 2167int
54cf91dc
CW		 2168i915_gem_flush_ring(struct drm_device *dev,
2169 struct intel_ring_buffer *ring,
2170 uint32_t invalidate_domains,
2171 uint32_t flush_domains)
2172{
88241785
CW		 2173 int ret;
2174
2175 ret = ring->flush(ring, invalidate_domains, flush_domains);
2176 if (ret)
2177 return ret;
2178
2179 i915_gem_process_flushing_list(dev, flush_domains, ring);
2180 return 0;
54cf91dc
CW		 2181}
2182
a56ba56c
CW		 2183static int i915_ring_idle(struct drm_device *dev,
2184 struct intel_ring_buffer *ring)
2185{
88241785
CW		 2186 int ret;
2187
395b70be 2188 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
64193406
CW		 2189 return 0;
2190
88241785
CW		 2191 if (!list_empty(&ring->gpu_write_list)) {
2192 ret = i915_gem_flush_ring(dev, ring,
0ac74c6b 2193 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
88241785
CW		 2194 if (ret)
2195 return ret;
2196 }
2197
a56ba56c
CW		 2198 return i915_wait_request(dev,
2199 i915_gem_next_request_seqno(dev, ring),
2200 ring);
2201}
2202
b47eb4a2 2203int
4df2faf4
DV		 2204i915_gpu_idle(struct drm_device *dev)
2205{
2206 drm_i915_private_t *dev_priv = dev->dev_private;
2207 bool lists_empty;
1ec14ad3 2208 int ret, i;
4df2faf4 2209
d1b851fc 2210 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
395b70be 2211 list_empty(&dev_priv->mm.active_list));
4df2faf4
DV		 2212 if (lists_empty)
2213 return 0;
2214
2215 /* Flush everything onto the inactive list. */
1ec14ad3
CW		 2216 for (i = 0; i < I915_NUM_RINGS; i++) {
2217 ret = i915_ring_idle(dev, &dev_priv->ring[i]);
2218 if (ret)
2219 return ret;
2220 }
4df2faf4 2221
8a1a49f9 2222 return 0;
4df2faf4
DV		 2223}
2224
c6642782
DV		 2225static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2226 struct intel_ring_buffer *pipelined)
4e901fdc 2227{
05394f39 2228 struct drm_device *dev = obj->base.dev;
4e901fdc 2229 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 2230 u32 size = obj->gtt_space->size;
2231 int regnum = obj->fence_reg;
4e901fdc
EA		 2232 uint64_t val;
2233
05394f39 2234 val = (uint64_t)((obj->gtt_offset + size - 4096) &
c6642782 2235 0xfffff000) << 32;
05394f39
CW		 2236 val |= obj->gtt_offset & 0xfffff000;
2237 val |= (uint64_t)((obj->stride / 128) - 1) <<
4e901fdc
EA		 2238 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2239
05394f39 2240 if (obj->tiling_mode == I915_TILING_Y)
4e901fdc
EA		 2241 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2242 val |= I965_FENCE_REG_VALID;
2243
c6642782
DV		 2244 if (pipelined) {
2245 int ret = intel_ring_begin(pipelined, 6);
2246 if (ret)
2247 return ret;
2248
2249 intel_ring_emit(pipelined, MI_NOOP);
2250 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2251 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2252 intel_ring_emit(pipelined, (u32)val);
2253 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2254 intel_ring_emit(pipelined, (u32)(val >> 32));
2255 intel_ring_advance(pipelined);
2256 } else
2257 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2258
2259 return 0;
4e901fdc
EA		 2260}
2261
c6642782
DV		 2262static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
2263 struct intel_ring_buffer *pipelined)
de151cf6 2264{
05394f39 2265 struct drm_device *dev = obj->base.dev;
de151cf6 2266 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 2267 u32 size = obj->gtt_space->size;
2268 int regnum = obj->fence_reg;
de151cf6
JB		 2269 uint64_t val;
2270
05394f39 2271 val = (uint64_t)((obj->gtt_offset + size - 4096) &
de151cf6 2272 0xfffff000) << 32;
05394f39
CW		 2273 val |= obj->gtt_offset & 0xfffff000;
2274 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2275 if (obj->tiling_mode == I915_TILING_Y)
de151cf6
JB		 2276 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2277 val |= I965_FENCE_REG_VALID;
2278
c6642782
DV		 2279 if (pipelined) {
2280 int ret = intel_ring_begin(pipelined, 6);
2281 if (ret)
2282 return ret;
2283
2284 intel_ring_emit(pipelined, MI_NOOP);
2285 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2286 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2287 intel_ring_emit(pipelined, (u32)val);
2288 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2289 intel_ring_emit(pipelined, (u32)(val >> 32));
2290 intel_ring_advance(pipelined);
2291 } else
2292 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2293
2294 return 0;
de151cf6
JB		 2295}
2296
c6642782
DV		 2297static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
2298 struct intel_ring_buffer *pipelined)
de151cf6 2299{
05394f39 2300 struct drm_device *dev = obj->base.dev;
de151cf6 2301 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39 2302 u32 size = obj->gtt_space->size;
c6642782 2303 u32 fence_reg, val, pitch_val;
0f973f27 2304 int tile_width;
de151cf6 2305
c6642782
DV		 2306 if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2307 (size & -size) != size ||
2308 (obj->gtt_offset & (size - 1)),
2309 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2310 obj->gtt_offset, obj->map_and_fenceable, size))
2311 return -EINVAL;
de151cf6 2312
c6642782 2313 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
0f973f27 2314 tile_width = 128;
de151cf6 2315 else
0f973f27
JB		 2316 tile_width = 512;
2317
2318 /* Note: pitch better be a power of two tile widths */
05394f39 2319 pitch_val = obj->stride / tile_width;
0f973f27 2320 pitch_val = ffs(pitch_val) - 1;
de151cf6 2321
05394f39
CW		 2322 val = obj->gtt_offset;
2323 if (obj->tiling_mode == I915_TILING_Y)
de151cf6 2324 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
a00b10c3 2325 val |= I915_FENCE_SIZE_BITS(size);
de151cf6
JB		 2326 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2327 val |= I830_FENCE_REG_VALID;
2328
05394f39 2329 fence_reg = obj->fence_reg;
a00b10c3
CW		 2330 if (fence_reg < 8)
2331 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
dc529a4f 2332 else
a00b10c3 2333 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
c6642782
DV		 2334
2335 if (pipelined) {
2336 int ret = intel_ring_begin(pipelined, 4);
2337 if (ret)
2338 return ret;
2339
2340 intel_ring_emit(pipelined, MI_NOOP);
2341 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2342 intel_ring_emit(pipelined, fence_reg);
2343 intel_ring_emit(pipelined, val);
2344 intel_ring_advance(pipelined);
2345 } else
2346 I915_WRITE(fence_reg, val);
2347
2348 return 0;
de151cf6
JB		 2349}
2350
c6642782
DV		 2351static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
2352 struct intel_ring_buffer *pipelined)
de151cf6 2353{
05394f39 2354 struct drm_device *dev = obj->base.dev;
de151cf6 2355 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 2356 u32 size = obj->gtt_space->size;
2357 int regnum = obj->fence_reg;
de151cf6
JB		 2358 uint32_t val;
2359 uint32_t pitch_val;
2360
c6642782
DV		 2361 if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2362 (size & -size) != size ||
2363 (obj->gtt_offset & (size - 1)),
2364 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2365 obj->gtt_offset, size))
2366 return -EINVAL;
de151cf6 2367
05394f39 2368 pitch_val = obj->stride / 128;
e76a16de 2369 pitch_val = ffs(pitch_val) - 1;
e76a16de 2370
05394f39
CW		 2371 val = obj->gtt_offset;
2372 if (obj->tiling_mode == I915_TILING_Y)
de151cf6 2373 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
c6642782 2374 val |= I830_FENCE_SIZE_BITS(size);
de151cf6
JB		 2375 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2376 val |= I830_FENCE_REG_VALID;
2377
c6642782
DV		 2378 if (pipelined) {
2379 int ret = intel_ring_begin(pipelined, 4);
2380 if (ret)
2381 return ret;
2382
2383 intel_ring_emit(pipelined, MI_NOOP);
2384 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2385 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2386 intel_ring_emit(pipelined, val);
2387 intel_ring_advance(pipelined);
2388 } else
2389 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2390
2391 return 0;
de151cf6
JB		 2392}
2393
d9e86c0e
CW		 2394static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2395{
2396 return i915_seqno_passed(ring->get_seqno(ring), seqno);
2397}
2398
2399static int
2400i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2401 struct intel_ring_buffer *pipelined,
2402 bool interruptible)
2403{
2404 int ret;
2405
2406 if (obj->fenced_gpu_access) {
88241785
CW		 2407 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2408 ret = i915_gem_flush_ring(obj->base.dev,
2409 obj->last_fenced_ring,
2410 0, obj->base.write_domain);
2411 if (ret)
2412 return ret;
2413 }
d9e86c0e
CW		 2414
2415 obj->fenced_gpu_access = false;
2416 }
2417
2418 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
2419 if (!ring_passed_seqno(obj->last_fenced_ring,
2420 obj->last_fenced_seqno)) {
2421 ret = i915_do_wait_request(obj->base.dev,
2422 obj->last_fenced_seqno,
2423 interruptible,
2424 obj->last_fenced_ring);
2425 if (ret)
2426 return ret;
2427 }
2428
2429 obj->last_fenced_seqno = 0;
2430 obj->last_fenced_ring = NULL;
2431 }
2432
63256ec5
CW		 2433 /* Ensure that all CPU reads are completed before installing a fence
2434 * and all writes before removing the fence.
2435 */
2436 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2437 mb();
2438
d9e86c0e
CW		 2439 return 0;
2440}
2441
2442int
2443i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2444{
2445 int ret;
2446
2447 if (obj->tiling_mode)
2448 i915_gem_release_mmap(obj);
2449
2450 ret = i915_gem_object_flush_fence(obj, NULL, true);
2451 if (ret)
2452 return ret;
2453
2454 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2455 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2456 i915_gem_clear_fence_reg(obj->base.dev,
2457 &dev_priv->fence_regs[obj->fence_reg]);
2458
2459 obj->fence_reg = I915_FENCE_REG_NONE;
2460 }
2461
2462 return 0;
2463}
2464
2465static struct drm_i915_fence_reg *
2466i915_find_fence_reg(struct drm_device *dev,
2467 struct intel_ring_buffer *pipelined)
ae3db24a 2468{
ae3db24a 2469 struct drm_i915_private *dev_priv = dev->dev_private;
d9e86c0e
CW		 2470 struct drm_i915_fence_reg *reg, *first, *avail;
2471 int i;
ae3db24a
DV		 2472
2473 /* First try to find a free reg */
d9e86c0e 2474 avail = NULL;
ae3db24a
DV		 2475 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2476 reg = &dev_priv->fence_regs[i];
2477 if (!reg->obj)
d9e86c0e 2478 return reg;
ae3db24a 2479
05394f39 2480 if (!reg->obj->pin_count)
d9e86c0e 2481 avail = reg;
ae3db24a
DV		 2482 }
2483
d9e86c0e
CW		 2484 if (avail == NULL)
2485 return NULL;
ae3db24a
DV		 2486
2487 /* None available, try to steal one or wait for a user to finish */
d9e86c0e
CW		 2488 avail = first = NULL;
2489 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2490 if (reg->obj->pin_count)
ae3db24a
DV		 2491 continue;
2492
d9e86c0e
CW		 2493 if (first == NULL)
2494 first = reg;
2495
2496 if (!pipelined ||
2497 !reg->obj->last_fenced_ring ||
2498 reg->obj->last_fenced_ring == pipelined) {
2499 avail = reg;
2500 break;
2501 }
ae3db24a
DV		 2502 }
2503
d9e86c0e
CW		 2504 if (avail == NULL)
2505 avail = first;
ae3db24a 2506
a00b10c3 2507 return avail;
ae3db24a
DV		 2508}
2509
de151cf6 2510/**
d9e86c0e 2511 * i915_gem_object_get_fence - set up a fence reg for an object
de151cf6 2512 * @obj: object to map through a fence reg
d9e86c0e
CW		 2513 * @pipelined: ring on which to queue the change, or NULL for CPU access
2514 * @interruptible: must we wait uninterruptibly for the register to retire?
de151cf6
JB		 2515 *
2516 * When mapping objects through the GTT, userspace wants to be able to write
2517 * to them without having to worry about swizzling if the object is tiled.
2518 *
2519 * This function walks the fence regs looking for a free one for @obj,
2520 * stealing one if it can't find any.
2521 *
2522 * It then sets up the reg based on the object's properties: address, pitch
2523 * and tiling format.
2524 */
8c4b8c3f 2525int
d9e86c0e
CW		 2526i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
2527 struct intel_ring_buffer *pipelined,
2528 bool interruptible)
de151cf6 2529{
05394f39 2530 struct drm_device *dev = obj->base.dev;
79e53945 2531 struct drm_i915_private *dev_priv = dev->dev_private;
d9e86c0e 2532 struct drm_i915_fence_reg *reg;
ae3db24a 2533 int ret;
de151cf6 2534
6bda10d1
CW		 2535 /* XXX disable pipelining. There are bugs. Shocking. */
2536 pipelined = NULL;
2537
d9e86c0e 2538 /* Just update our place in the LRU if our fence is getting reused. */
05394f39
CW		 2539 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2540 reg = &dev_priv->fence_regs[obj->fence_reg];
007cc8ac 2541 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
d9e86c0e
CW		 2542
2543 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2544 pipelined = NULL;
2545
2546 if (!pipelined) {
2547 if (reg->setup_seqno) {
2548 if (!ring_passed_seqno(obj->last_fenced_ring,
2549 reg->setup_seqno)) {
2550 ret = i915_do_wait_request(obj->base.dev,
2551 reg->setup_seqno,
2552 interruptible,
2553 obj->last_fenced_ring);
2554 if (ret)
2555 return ret;
2556 }
2557
2558 reg->setup_seqno = 0;
2559 }
2560 } else if (obj->last_fenced_ring &&
2561 obj->last_fenced_ring != pipelined) {
2562 ret = i915_gem_object_flush_fence(obj,
2563 pipelined,
2564 interruptible);
2565 if (ret)
2566 return ret;
2567 } else if (obj->tiling_changed) {
2568 if (obj->fenced_gpu_access) {
88241785
CW		 2569 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2570 ret = i915_gem_flush_ring(obj->base.dev, obj->ring,
2571 0, obj->base.write_domain);
2572 if (ret)
2573 return ret;
2574 }
d9e86c0e
CW		 2575
2576 obj->fenced_gpu_access = false;
2577 }
2578 }
2579
2580 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2581 pipelined = NULL;
2582 BUG_ON(!pipelined && reg->setup_seqno);
2583
2584 if (obj->tiling_changed) {
2585 if (pipelined) {
2586 reg->setup_seqno =
2587 i915_gem_next_request_seqno(dev, pipelined);
2588 obj->last_fenced_seqno = reg->setup_seqno;
2589 obj->last_fenced_ring = pipelined;
2590 }
2591 goto update;
2592 }
2593
a09ba7fa
EA		 2594 return 0;
2595 }
2596
d9e86c0e
CW		 2597 reg = i915_find_fence_reg(dev, pipelined);
2598 if (reg == NULL)
2599 return -ENOSPC;
de151cf6 2600
d9e86c0e
CW		 2601 ret = i915_gem_object_flush_fence(obj, pipelined, interruptible);
2602 if (ret)
ae3db24a 2603 return ret;
de151cf6 2604
d9e86c0e
CW		 2605 if (reg->obj) {
2606 struct drm_i915_gem_object *old = reg->obj;
2607
2608 drm_gem_object_reference(&old->base);
2609
2610 if (old->tiling_mode)
2611 i915_gem_release_mmap(old);
2612
d9e86c0e 2613 ret = i915_gem_object_flush_fence(old,
6bda10d1 2614 pipelined,
d9e86c0e
CW		 2615 interruptible);
2616 if (ret) {
2617 drm_gem_object_unreference(&old->base);
2618 return ret;
2619 }
2620
2621 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
2622 pipelined = NULL;
2623
2624 old->fence_reg = I915_FENCE_REG_NONE;
2625 old->last_fenced_ring = pipelined;
2626 old->last_fenced_seqno =
2627 pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0;
2628
2629 drm_gem_object_unreference(&old->base);
2630 } else if (obj->last_fenced_seqno == 0)
2631 pipelined = NULL;
a09ba7fa 2632
de151cf6 2633 reg->obj = obj;
d9e86c0e
CW		 2634 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2635 obj->fence_reg = reg - dev_priv->fence_regs;
2636 obj->last_fenced_ring = pipelined;
de151cf6 2637
d9e86c0e
CW		 2638 reg->setup_seqno =
2639 pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0;
2640 obj->last_fenced_seqno = reg->setup_seqno;
2641
2642update:
2643 obj->tiling_changed = false;
e259befd
CW		 2644 switch (INTEL_INFO(dev)->gen) {
2645 case 6:
c6642782 2646 ret = sandybridge_write_fence_reg(obj, pipelined);
e259befd
CW		 2647 break;
2648 case 5:
2649 case 4:
c6642782 2650 ret = i965_write_fence_reg(obj, pipelined);
e259befd
CW		 2651 break;
2652 case 3:
c6642782 2653 ret = i915_write_fence_reg(obj, pipelined);
e259befd
CW		 2654 break;
2655 case 2:
c6642782 2656 ret = i830_write_fence_reg(obj, pipelined);
e259befd
CW		 2657 break;
2658 }
d9ddcb96 2659
c6642782 2660 return ret;
de151cf6
JB		 2661}
2662
2663/**
2664 * i915_gem_clear_fence_reg - clear out fence register info
2665 * @obj: object to clear
2666 *
2667 * Zeroes out the fence register itself and clears out the associated
05394f39 2668 * data structures in dev_priv and obj.
de151cf6
JB		 2669 */
2670static void
d9e86c0e
CW		 2671i915_gem_clear_fence_reg(struct drm_device *dev,
2672 struct drm_i915_fence_reg *reg)
de151cf6 2673{
79e53945 2674 drm_i915_private_t *dev_priv = dev->dev_private;
d9e86c0e 2675 uint32_t fence_reg = reg - dev_priv->fence_regs;
de151cf6 2676
e259befd
CW		 2677 switch (INTEL_INFO(dev)->gen) {
2678 case 6:
d9e86c0e 2679 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
e259befd
CW		 2680 break;
2681 case 5:
2682 case 4:
d9e86c0e 2683 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
e259befd
CW		 2684 break;
2685 case 3:
d9e86c0e
CW		 2686 if (fence_reg >= 8)
2687 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
dc529a4f 2688 else
e259befd 2689 case 2:
d9e86c0e 2690 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
dc529a4f
EA		 2691
2692 I915_WRITE(fence_reg, 0);
e259befd 2693 break;
dc529a4f 2694 }
de151cf6 2695
007cc8ac 2696 list_del_init(&reg->lru_list);
d9e86c0e
CW		 2697 reg->obj = NULL;
2698 reg->setup_seqno = 0;
52dc7d32
CW		 2699}
2700
673a394b
EA		 2701/**
2702 * Finds free space in the GTT aperture and binds the object there.
2703 */
2704static int
05394f39 2705i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
920afa77 2706 unsigned alignment,
75e9e915 2707 bool map_and_fenceable)
673a394b 2708{
05394f39 2709 struct drm_device *dev = obj->base.dev;
673a394b 2710 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 2711 struct drm_mm_node *free_space;
a00b10c3 2712 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
5e783301 2713 u32 size, fence_size, fence_alignment, unfenced_alignment;
75e9e915 2714 bool mappable, fenceable;
07f73f69 2715 int ret;
673a394b 2716
05394f39 2717 if (obj->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2718 DRM_ERROR("Attempting to bind a purgeable object\n");
2719 return -EINVAL;
2720 }
2721
05394f39
CW		 2722 fence_size = i915_gem_get_gtt_size(obj);
2723 fence_alignment = i915_gem_get_gtt_alignment(obj);
2724 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(obj);
a00b10c3 2725
673a394b 2726 if (alignment == 0)
5e783301
DV		 2727 alignment = map_and_fenceable ? fence_alignment :
2728 unfenced_alignment;
75e9e915 2729 if (map_and_fenceable && alignment & (fence_alignment - 1)) {
673a394b
EA		 2730 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2731 return -EINVAL;
2732 }
2733
05394f39 2734 size = map_and_fenceable ? fence_size : obj->base.size;
a00b10c3 2735
654fc607
CW		 2736 /* If the object is bigger than the entire aperture, reject it early
2737 * before evicting everything in a vain attempt to find space.
2738 */
05394f39 2739 if (obj->base.size >
75e9e915 2740 (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
654fc607
CW		 2741 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2742 return -E2BIG;
2743 }
2744
673a394b 2745 search_free:
75e9e915 2746 if (map_and_fenceable)
920afa77
DV		 2747 free_space =
2748 drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
a00b10c3 2749 size, alignment, 0,
920afa77
DV		 2750 dev_priv->mm.gtt_mappable_end,
2751 0);
2752 else
2753 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
a00b10c3 2754 size, alignment, 0);
920afa77
DV		 2755
2756 if (free_space != NULL) {
75e9e915 2757 if (map_and_fenceable)
05394f39 2758 obj->gtt_space =
920afa77 2759 drm_mm_get_block_range_generic(free_space,
a00b10c3 2760 size, alignment, 0,
920afa77
DV		 2761 dev_priv->mm.gtt_mappable_end,
2762 0);
2763 else
05394f39 2764 obj->gtt_space =
a00b10c3 2765 drm_mm_get_block(free_space, size, alignment);
920afa77 2766 }
05394f39 2767 if (obj->gtt_space == NULL) {
673a394b
EA		 2768 /* If the gtt is empty and we're still having trouble
2769 * fitting our object in, we're out of memory.
2770 */
75e9e915
DV		 2771 ret = i915_gem_evict_something(dev, size, alignment,
2772 map_and_fenceable);
9731129c 2773 if (ret)
673a394b 2774 return ret;
9731129c 2775
673a394b
EA		 2776 goto search_free;
2777 }
2778
e5281ccd 2779 ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
673a394b 2780 if (ret) {
05394f39
CW		 2781 drm_mm_put_block(obj->gtt_space);
2782 obj->gtt_space = NULL;
07f73f69
CW		 2783
2784 if (ret == -ENOMEM) {
809b6334
CW		 2785 /* first try to reclaim some memory by clearing the GTT */
2786 ret = i915_gem_evict_everything(dev, false);
07f73f69 2787 if (ret) {
07f73f69 2788 /* now try to shrink everyone else */
4bdadb97
CW		 2789 if (gfpmask) {
2790 gfpmask = 0;
2791 goto search_free;
07f73f69
CW		 2792 }
2793
809b6334 2794 return -ENOMEM;
07f73f69
CW		 2795 }
2796
2797 goto search_free;
2798 }
2799
673a394b
EA		 2800 return ret;
2801 }
2802
7c2e6fdf
DV		 2803 ret = i915_gem_gtt_bind_object(obj);
2804 if (ret) {
e5281ccd 2805 i915_gem_object_put_pages_gtt(obj);
05394f39
CW		 2806 drm_mm_put_block(obj->gtt_space);
2807 obj->gtt_space = NULL;
07f73f69 2808
809b6334 2809 if (i915_gem_evict_everything(dev, false))
07f73f69 2810 return ret;
07f73f69
CW		 2811
2812 goto search_free;
673a394b 2813 }
673a394b 2814
6299f992 2815 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
05394f39 2816 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
bf1a1092 2817
673a394b
EA		 2818 /* Assert that the object is not currently in any GPU domain. As it
2819 * wasn't in the GTT, there shouldn't be any way it could have been in
2820 * a GPU cache
2821 */
05394f39
CW		 2822 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2823 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2824
6299f992 2825 obj->gtt_offset = obj->gtt_space->start;
1c5d22f7 2826
75e9e915 2827 fenceable =
05394f39
CW		 2828 obj->gtt_space->size == fence_size &&
2829 (obj->gtt_space->start & (fence_alignment -1)) == 0;
a00b10c3 2830
75e9e915 2831 mappable =
05394f39 2832 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
a00b10c3 2833
05394f39 2834 obj->map_and_fenceable = mappable && fenceable;
75e9e915 2835
6299f992 2836 trace_i915_gem_object_bind(obj, obj->gtt_offset, map_and_fenceable);
673a394b
EA		 2837 return 0;
2838}
2839
2840void
05394f39 2841i915_gem_clflush_object(struct drm_i915_gem_object *obj)
673a394b 2842{
673a394b
EA		 2843 /* If we don't have a page list set up, then we're not pinned
2844 * to GPU, and we can ignore the cache flush because it'll happen
2845 * again at bind time.
2846 */
05394f39 2847 if (obj->pages == NULL)
673a394b
EA		 2848 return;
2849
1c5d22f7 2850 trace_i915_gem_object_clflush(obj);
cfa16a0d 2851
05394f39 2852 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
673a394b
EA		 2853}
2854
e47c68e9 2855/** Flushes any GPU write domain for the object if it's dirty. */
88241785 2856static int
3619df03 2857i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2858{
05394f39 2859 struct drm_device *dev = obj->base.dev;
e47c68e9 2860
05394f39 2861 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
88241785 2862 return 0;
e47c68e9
EA		 2863
2864 /* Queue the GPU write cache flushing we need. */
88241785 2865 return i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain);
e47c68e9
EA		 2866}
2867
2868/** Flushes the GTT write domain for the object if it's dirty. */
2869static void
05394f39 2870i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2871{
1c5d22f7
CW		 2872 uint32_t old_write_domain;
2873
05394f39 2874 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
e47c68e9
EA		 2875 return;
2876
63256ec5 2877 /* No actual flushing is required for the GTT write domain. Writes
e47c68e9
EA		 2878 * to it immediately go to main memory as far as we know, so there's
2879 * no chipset flush. It also doesn't land in render cache.
63256ec5
CW		 2880 *
2881 * However, we do have to enforce the order so that all writes through
2882 * the GTT land before any writes to the device, such as updates to
2883 * the GATT itself.
e47c68e9 2884 */
63256ec5
CW		 2885 wmb();
2886
4a684a41
CW		 2887 i915_gem_release_mmap(obj);
2888
05394f39
CW		 2889 old_write_domain = obj->base.write_domain;
2890 obj->base.write_domain = 0;
1c5d22f7
CW		 2891
2892 trace_i915_gem_object_change_domain(obj,
05394f39 2893 obj->base.read_domains,
1c5d22f7 2894 old_write_domain);
e47c68e9
EA		 2895}
2896
2897/** Flushes the CPU write domain for the object if it's dirty. */
2898static void
05394f39 2899i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2900{
1c5d22f7 2901 uint32_t old_write_domain;
e47c68e9 2902
05394f39 2903 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
e47c68e9
EA		 2904 return;
2905
2906 i915_gem_clflush_object(obj);
40ce6575 2907 intel_gtt_chipset_flush();
05394f39
CW		 2908 old_write_domain = obj->base.write_domain;
2909 obj->base.write_domain = 0;
1c5d22f7
CW		 2910
2911 trace_i915_gem_object_change_domain(obj,
05394f39 2912 obj->base.read_domains,
1c5d22f7 2913 old_write_domain);
e47c68e9
EA		 2914}
2915
2ef7eeaa
EA		 2916/**
2917 * Moves a single object to the GTT read, and possibly write domain.
2918 *
2919 * This function returns when the move is complete, including waiting on
2920 * flushes to occur.
2921 */
79e53945 2922int
2021746e 2923i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2ef7eeaa 2924{
1c5d22f7 2925 uint32_t old_write_domain, old_read_domains;
e47c68e9 2926 int ret;
2ef7eeaa 2927
02354392 2928 /* Not valid to be called on unbound objects. */
05394f39 2929 if (obj->gtt_space == NULL)
02354392
EA		 2930 return -EINVAL;
2931
88241785
CW		 2932 ret = i915_gem_object_flush_gpu_write_domain(obj);
2933 if (ret)
2934 return ret;
2935
87ca9c8a
CW		 2936 if (obj->pending_gpu_write || write) {
2937 ret = i915_gem_object_wait_rendering(obj, true);
2938 if (ret)
2939 return ret;
2940 }
2dafb1e0 2941
7213342d 2942 i915_gem_object_flush_cpu_write_domain(obj);
1c5d22f7 2943
05394f39
CW		 2944 old_write_domain = obj->base.write_domain;
2945 old_read_domains = obj->base.read_domains;
1c5d22f7 2946
e47c68e9
EA		 2947 /* It should now be out of any other write domains, and we can update
2948 * the domain values for our changes.
2949 */
05394f39
CW		 2950 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2951 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
e47c68e9 2952 if (write) {
05394f39
CW		 2953 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2954 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2955 obj->dirty = 1;
2ef7eeaa
EA		 2956 }
2957
1c5d22f7
CW		 2958 trace_i915_gem_object_change_domain(obj,
2959 old_read_domains,
2960 old_write_domain);
2961
e47c68e9
EA		 2962 return 0;
2963}
2964
b9241ea3
ZW		 2965/*
2966 * Prepare buffer for display plane. Use uninterruptible for possible flush
2967 * wait, as in modesetting process we're not supposed to be interrupted.
2968 */
2969int
05394f39 2970i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
919926ae 2971 struct intel_ring_buffer *pipelined)
b9241ea3 2972{
ba3d8d74 2973 uint32_t old_read_domains;
b9241ea3
ZW		 2974 int ret;
2975
2976 /* Not valid to be called on unbound objects. */
05394f39 2977 if (obj->gtt_space == NULL)
b9241ea3
ZW		 2978 return -EINVAL;
2979
88241785
CW		 2980 ret = i915_gem_object_flush_gpu_write_domain(obj);
2981 if (ret)
2982 return ret;
2983
b9241ea3 2984
ced270fa 2985 /* Currently, we are always called from an non-interruptible context. */
0be73284 2986 if (pipelined != obj->ring) {
ced270fa
CW		 2987 ret = i915_gem_object_wait_rendering(obj, false);
2988 if (ret)
b9241ea3
ZW		 2989 return ret;
2990 }
2991
b118c1e3
CW		 2992 i915_gem_object_flush_cpu_write_domain(obj);
2993
05394f39
CW		 2994 old_read_domains = obj->base.read_domains;
2995 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
b9241ea3
ZW		 2996
2997 trace_i915_gem_object_change_domain(obj,
2998 old_read_domains,
05394f39 2999 obj->base.write_domain);
b9241ea3
ZW		 3000
3001 return 0;
3002}
3003
85345517
CW		 3004int
3005i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
3006 bool interruptible)
3007{
88241785
CW		 3008 int ret;
3009
85345517
CW		 3010 if (!obj->active)
3011 return 0;
3012
88241785
CW		 3013 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3014 ret = i915_gem_flush_ring(obj->base.dev, obj->ring,
3015 0, obj->base.write_domain);
3016 if (ret)
3017 return ret;
3018 }
85345517 3019
05394f39 3020 return i915_gem_object_wait_rendering(obj, interruptible);
85345517
CW		 3021}
3022
e47c68e9
EA		 3023/**
3024 * Moves a single object to the CPU read, and possibly write domain.
3025 *
3026 * This function returns when the move is complete, including waiting on
3027 * flushes to occur.
3028 */
3029static int
919926ae 3030i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
e47c68e9 3031{
1c5d22f7 3032 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 3033 int ret;
3034
88241785
CW		 3035 ret = i915_gem_object_flush_gpu_write_domain(obj);
3036 if (ret)
3037 return ret;
3038
de18a29e
DV		 3039 ret = i915_gem_object_wait_rendering(obj, true);
3040 if (ret)
e47c68e9 3041 return ret;
2ef7eeaa 3042
e47c68e9 3043 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 3044
e47c68e9
EA		 3045 /* If we have a partially-valid cache of the object in the CPU,
3046 * finish invalidating it and free the per-page flags.
2ef7eeaa 3047 */
e47c68e9 3048 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 3049
05394f39
CW		 3050 old_write_domain = obj->base.write_domain;
3051 old_read_domains = obj->base.read_domains;
1c5d22f7 3052
e47c68e9 3053 /* Flush the CPU cache if it's still invalid. */
05394f39 3054 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 3055 i915_gem_clflush_object(obj);
2ef7eeaa 3056
05394f39 3057 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 3058 }
3059
3060 /* It should now be out of any other write domains, and we can update
3061 * the domain values for our changes.
3062 */
05394f39 3063 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
e47c68e9
EA		 3064
3065 /* If we're writing through the CPU, then the GPU read domains will
3066 * need to be invalidated at next use.
3067 */
3068 if (write) {
05394f39
CW		 3069 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3070 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
e47c68e9 3071 }
2ef7eeaa 3072
1c5d22f7
CW		 3073 trace_i915_gem_object_change_domain(obj,
3074 old_read_domains,
3075 old_write_domain);
3076
2ef7eeaa
EA		 3077 return 0;
3078}
3079
673a394b 3080/**
e47c68e9 3081 * Moves the object from a partially CPU read to a full one.
673a394b 3082 *
e47c68e9
EA		 3083 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3084 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 3085 */
e47c68e9 3086static void
05394f39 3087i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
673a394b 3088{
05394f39 3089 if (!obj->page_cpu_valid)
e47c68e9
EA		 3090 return;
3091
3092 /* If we're partially in the CPU read domain, finish moving it in.
3093 */
05394f39 3094 if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 3095 int i;
3096
05394f39
CW		 3097 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
3098 if (obj->page_cpu_valid[i])
e47c68e9 3099 continue;
05394f39 3100 drm_clflush_pages(obj->pages + i, 1);
e47c68e9 3101 }
e47c68e9
EA		 3102 }
3103
3104 /* Free the page_cpu_valid mappings which are now stale, whether
3105 * or not we've got I915_GEM_DOMAIN_CPU.
3106 */
05394f39
CW		 3107 kfree(obj->page_cpu_valid);
3108 obj->page_cpu_valid = NULL;
e47c68e9
EA		 3109}
3110
3111/**
3112 * Set the CPU read domain on a range of the object.
3113 *
3114 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3115 * not entirely valid. The page_cpu_valid member of the object flags which
3116 * pages have been flushed, and will be respected by
3117 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3118 * of the whole object.
3119 *
3120 * This function returns when the move is complete, including waiting on
3121 * flushes to occur.
3122 */
3123static int
05394f39 3124i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
e47c68e9
EA		 3125 uint64_t offset, uint64_t size)
3126{
1c5d22f7 3127 uint32_t old_read_domains;
e47c68e9 3128 int i, ret;
673a394b 3129
05394f39 3130 if (offset == 0 && size == obj->base.size)
e47c68e9 3131 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 3132
88241785
CW		 3133 ret = i915_gem_object_flush_gpu_write_domain(obj);
3134 if (ret)
3135 return ret;
3136
de18a29e
DV		 3137 ret = i915_gem_object_wait_rendering(obj, true);
3138 if (ret)
6a47baa6 3139 return ret;
de18a29e 3140
e47c68e9
EA		 3141 i915_gem_object_flush_gtt_write_domain(obj);
3142
3143 /* If we're already fully in the CPU read domain, we're done. */
05394f39
CW		 3144 if (obj->page_cpu_valid == NULL &&
3145 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
e47c68e9 3146 return 0;
673a394b 3147
e47c68e9
EA		 3148 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3149 * newly adding I915_GEM_DOMAIN_CPU
3150 */
05394f39
CW		 3151 if (obj->page_cpu_valid == NULL) {
3152 obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
3153 GFP_KERNEL);
3154 if (obj->page_cpu_valid == NULL)
e47c68e9 3155 return -ENOMEM;
05394f39
CW		 3156 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3157 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
673a394b
EA		 3158
3159 /* Flush the cache on any pages that are still invalid from the CPU's
3160 * perspective.
3161 */
e47c68e9
EA		 3162 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3163 i++) {
05394f39 3164 if (obj->page_cpu_valid[i])
673a394b
EA		 3165 continue;
3166
05394f39 3167 drm_clflush_pages(obj->pages + i, 1);
673a394b 3168
05394f39 3169 obj->page_cpu_valid[i] = 1;
673a394b
EA		 3170 }
3171
e47c68e9
EA		 3172 /* It should now be out of any other write domains, and we can update
3173 * the domain values for our changes.
3174 */
05394f39 3175 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
e47c68e9 3176
05394f39
CW		 3177 old_read_domains = obj->base.read_domains;
3178 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
e47c68e9 3179
1c5d22f7
CW		 3180 trace_i915_gem_object_change_domain(obj,
3181 old_read_domains,
05394f39 3182 obj->base.write_domain);
1c5d22f7 3183
673a394b
EA		 3184 return 0;
3185}
3186
673a394b
EA		 3187/* Throttle our rendering by waiting until the ring has completed our requests
3188 * emitted over 20 msec ago.
3189 *
b962442e
EA		 3190 * Note that if we were to use the current jiffies each time around the loop,
3191 * we wouldn't escape the function with any frames outstanding if the time to
3192 * render a frame was over 20ms.
3193 *
673a394b
EA		 3194 * This should get us reasonable parallelism between CPU and GPU but also
3195 * relatively low latency when blocking on a particular request to finish.
3196 */
40a5f0de 3197static int
f787a5f5 3198i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
40a5f0de 3199{
f787a5f5
CW		 3200 struct drm_i915_private *dev_priv = dev->dev_private;
3201 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e 3202 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
f787a5f5
CW		 3203 struct drm_i915_gem_request *request;
3204 struct intel_ring_buffer *ring = NULL;
3205 u32 seqno = 0;
3206 int ret;
93533c29 3207
1c25595f 3208 spin_lock(&file_priv->mm.lock);
f787a5f5 3209 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
b962442e
EA		 3210 if (time_after_eq(request->emitted_jiffies, recent_enough))
3211 break;
40a5f0de 3212
f787a5f5
CW		 3213 ring = request->ring;
3214 seqno = request->seqno;
b962442e 3215 }
1c25595f 3216 spin_unlock(&file_priv->mm.lock);
40a5f0de 3217
f787a5f5
CW		 3218 if (seqno == 0)
3219 return 0;
2bc43b5c 3220
f787a5f5 3221 ret = 0;
78501eac 3222 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
f787a5f5
CW		 3223 /* And wait for the seqno passing without holding any locks and
3224 * causing extra latency for others. This is safe as the irq
3225 * generation is designed to be run atomically and so is
3226 * lockless.
3227 */
b13c2b96
CW		 3228 if (ring->irq_get(ring)) {
3229 ret = wait_event_interruptible(ring->irq_queue,
3230 i915_seqno_passed(ring->get_seqno(ring), seqno)
3231 || atomic_read(&dev_priv->mm.wedged));
3232 ring->irq_put(ring);
40a5f0de 3233
b13c2b96
CW		 3234 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3235 ret = -EIO;
3236 }
40a5f0de
EA		 3237 }
3238
f787a5f5
CW		 3239 if (ret == 0)
3240 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
40a5f0de
EA		 3241
3242 return ret;
3243}
3244
673a394b 3245int
05394f39
CW		 3246i915_gem_object_pin(struct drm_i915_gem_object *obj,
3247 uint32_t alignment,
75e9e915 3248 bool map_and_fenceable)
673a394b 3249{
05394f39 3250 struct drm_device *dev = obj->base.dev;
f13d3f73 3251 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 3252 int ret;
3253
05394f39 3254 BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
23bc5982 3255 WARN_ON(i915_verify_lists(dev));
ac0c6b5a 3256
05394f39
CW		 3257 if (obj->gtt_space != NULL) {
3258 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3259 (map_and_fenceable && !obj->map_and_fenceable)) {
3260 WARN(obj->pin_count,
ae7d49d8 3261 "bo is already pinned with incorrect alignment:"
75e9e915
DV		 3262 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3263 " obj->map_and_fenceable=%d\n",
05394f39 3264 obj->gtt_offset, alignment,
75e9e915 3265 map_and_fenceable,
05394f39 3266 obj->map_and_fenceable);
ac0c6b5a
CW		 3267 ret = i915_gem_object_unbind(obj);
3268 if (ret)
3269 return ret;
3270 }
3271 }
3272
05394f39 3273 if (obj->gtt_space == NULL) {
a00b10c3 3274 ret = i915_gem_object_bind_to_gtt(obj, alignment,
75e9e915 3275 map_and_fenceable);
9731129c 3276 if (ret)
673a394b 3277 return ret;
22c344e9 3278 }
76446cac 3279
05394f39 3280 if (obj->pin_count++ == 0) {
05394f39
CW		 3281 if (!obj->active)
3282 list_move_tail(&obj->mm_list,
f13d3f73 3283 &dev_priv->mm.pinned_list);
673a394b 3284 }
6299f992 3285 obj->pin_mappable |= map_and_fenceable;
673a394b 3286
23bc5982 3287 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 3288 return 0;
3289}
3290
3291void
05394f39 3292i915_gem_object_unpin(struct drm_i915_gem_object *obj)
673a394b 3293{
05394f39 3294 struct drm_device *dev = obj->base.dev;
673a394b 3295 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 3296
23bc5982 3297 WARN_ON(i915_verify_lists(dev));
05394f39
CW		 3298 BUG_ON(obj->pin_count == 0);
3299 BUG_ON(obj->gtt_space == NULL);
673a394b 3300
05394f39
CW		 3301 if (--obj->pin_count == 0) {
3302 if (!obj->active)
3303 list_move_tail(&obj->mm_list,
673a394b 3304 &dev_priv->mm.inactive_list);
6299f992 3305 obj->pin_mappable = false;
673a394b 3306 }
23bc5982 3307 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 3308}
3309
3310int
3311i915_gem_pin_ioctl(struct drm_device *dev, void *data,
05394f39 3312 struct drm_file *file)
673a394b
EA		 3313{
3314 struct drm_i915_gem_pin *args = data;
05394f39 3315 struct drm_i915_gem_object *obj;
673a394b
EA		 3316 int ret;
3317
1d7cfea1
CW		 3318 ret = i915_mutex_lock_interruptible(dev);
3319 if (ret)
3320 return ret;
673a394b 3321
05394f39 3322 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
673a394b 3323 if (obj == NULL) {
1d7cfea1
CW		 3324 ret = -ENOENT;
3325 goto unlock;
673a394b 3326 }
673a394b 3327
05394f39 3328 if (obj->madv != I915_MADV_WILLNEED) {
bb6baf76 3329 DRM_ERROR("Attempting to pin a purgeable buffer\n");
1d7cfea1
CW		 3330 ret = -EINVAL;
3331 goto out;
3ef94daa
CW		 3332 }
3333
05394f39 3334 if (obj->pin_filp != NULL && obj->pin_filp != file) {
79e53945
JB		 3335 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3336 args->handle);
1d7cfea1
CW		 3337 ret = -EINVAL;
3338 goto out;
79e53945
JB		 3339 }
3340
05394f39
CW		 3341 obj->user_pin_count++;
3342 obj->pin_filp = file;
3343 if (obj->user_pin_count == 1) {
75e9e915 3344 ret = i915_gem_object_pin(obj, args->alignment, true);
1d7cfea1
CW		 3345 if (ret)
3346 goto out;
673a394b
EA		 3347 }
3348
3349 /* XXX - flush the CPU caches for pinned objects
3350 * as the X server doesn't manage domains yet
3351 */
e47c68e9 3352 i915_gem_object_flush_cpu_write_domain(obj);
05394f39 3353 args->offset = obj->gtt_offset;
1d7cfea1 3354out:
05394f39 3355 drm_gem_object_unreference(&obj->base);
1d7cfea1 3356unlock:
673a394b 3357 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3358 return ret;
673a394b
EA		 3359}
3360
3361int
3362i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
05394f39 3363 struct drm_file *file)
673a394b
EA		 3364{
3365 struct drm_i915_gem_pin *args = data;
05394f39 3366 struct drm_i915_gem_object *obj;
76c1dec1 3367 int ret;
673a394b 3368
1d7cfea1
CW		 3369 ret = i915_mutex_lock_interruptible(dev);
3370 if (ret)
3371 return ret;
673a394b 3372
05394f39 3373 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
673a394b 3374 if (obj == NULL) {
1d7cfea1
CW		 3375 ret = -ENOENT;
3376 goto unlock;
673a394b 3377 }
76c1dec1 3378
05394f39 3379 if (obj->pin_filp != file) {
79e53945
JB		 3380 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3381 args->handle);
1d7cfea1
CW		 3382 ret = -EINVAL;
3383 goto out;
79e53945 3384 }
05394f39
CW		 3385 obj->user_pin_count--;
3386 if (obj->user_pin_count == 0) {
3387 obj->pin_filp = NULL;
79e53945
JB		 3388 i915_gem_object_unpin(obj);
3389 }
673a394b 3390
1d7cfea1 3391out:
05394f39 3392 drm_gem_object_unreference(&obj->base);
1d7cfea1 3393unlock:
673a394b 3394 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3395 return ret;
673a394b
EA		 3396}
3397
3398int
3399i915_gem_busy_ioctl(struct drm_device *dev, void *data,
05394f39 3400 struct drm_file *file)
673a394b
EA		 3401{
3402 struct drm_i915_gem_busy *args = data;
05394f39 3403 struct drm_i915_gem_object *obj;
30dbf0c0
CW		 3404 int ret;
3405
76c1dec1 3406 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 3407 if (ret)
76c1dec1 3408 return ret;
673a394b 3409
05394f39 3410 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
673a394b 3411 if (obj == NULL) {
1d7cfea1
CW		 3412 ret = -ENOENT;
3413 goto unlock;
673a394b 3414 }
d1b851fc 3415
0be555b6
CW		 3416 /* Count all active objects as busy, even if they are currently not used
3417 * by the gpu. Users of this interface expect objects to eventually
3418 * become non-busy without any further actions, therefore emit any
3419 * necessary flushes here.
c4de0a5d 3420 */
05394f39 3421 args->busy = obj->active;
0be555b6
CW		 3422 if (args->busy) {
3423 /* Unconditionally flush objects, even when the gpu still uses this
3424 * object. Userspace calling this function indicates that it wants to
3425 * use this buffer rather sooner than later, so issuing the required
3426 * flush earlier is beneficial.
3427 */
1a1c6976 3428 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
88241785
CW		 3429 ret = i915_gem_flush_ring(dev, obj->ring,
3430 0, obj->base.write_domain);
1a1c6976
CW		 3431 } else if (obj->ring->outstanding_lazy_request ==
3432 obj->last_rendering_seqno) {
3433 struct drm_i915_gem_request *request;
3434
7a194876
CW		 3435 /* This ring is not being cleared by active usage,
3436 * so emit a request to do so.
3437 */
1a1c6976
CW		 3438 request = kzalloc(sizeof(*request), GFP_KERNEL);
3439 if (request)
3440 ret = i915_add_request(dev,
3441 NULL, request,
3442 obj->ring);
3443 else
7a194876
CW		 3444 ret = -ENOMEM;
3445 }
0be555b6
CW		 3446
3447 /* Update the active list for the hardware's current position.
3448 * Otherwise this only updates on a delayed timer or when irqs
3449 * are actually unmasked, and our working set ends up being
3450 * larger than required.
3451 */
05394f39 3452 i915_gem_retire_requests_ring(dev, obj->ring);
0be555b6 3453
05394f39 3454 args->busy = obj->active;
0be555b6 3455 }
673a394b 3456
05394f39 3457 drm_gem_object_unreference(&obj->base);
1d7cfea1 3458unlock:
673a394b 3459 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3460 return ret;
673a394b
EA		 3461}
3462
3463int
3464i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3465 struct drm_file *file_priv)
3466{
3467 return i915_gem_ring_throttle(dev, file_priv);
3468}
3469
3ef94daa
CW		 3470int
3471i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3472 struct drm_file *file_priv)
3473{
3474 struct drm_i915_gem_madvise *args = data;
05394f39 3475 struct drm_i915_gem_object *obj;
76c1dec1 3476 int ret;
3ef94daa
CW		 3477
3478 switch (args->madv) {
3479 case I915_MADV_DONTNEED:
3480 case I915_MADV_WILLNEED:
3481 break;
3482 default:
3483 return -EINVAL;
3484 }
3485
1d7cfea1
CW		 3486 ret = i915_mutex_lock_interruptible(dev);
3487 if (ret)
3488 return ret;
3489
05394f39 3490 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3ef94daa 3491 if (obj == NULL) {
1d7cfea1
CW		 3492 ret = -ENOENT;
3493 goto unlock;
3ef94daa 3494 }
3ef94daa 3495
05394f39 3496 if (obj->pin_count) {
1d7cfea1
CW		 3497 ret = -EINVAL;
3498 goto out;
3ef94daa
CW		 3499 }
3500
05394f39
CW		 3501 if (obj->madv != __I915_MADV_PURGED)
3502 obj->madv = args->madv;
3ef94daa 3503
2d7ef395 3504 /* if the object is no longer bound, discard its backing storage */
05394f39
CW		 3505 if (i915_gem_object_is_purgeable(obj) &&
3506 obj->gtt_space == NULL)
2d7ef395
CW		 3507 i915_gem_object_truncate(obj);
3508
05394f39 3509 args->retained = obj->madv != __I915_MADV_PURGED;
bb6baf76 3510
1d7cfea1 3511out:
05394f39 3512 drm_gem_object_unreference(&obj->base);
1d7cfea1 3513unlock:
3ef94daa 3514 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3515 return ret;
3ef94daa
CW		 3516}
3517
05394f39
CW		 3518struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3519 size_t size)
ac52bc56 3520{
73aa808f 3521 struct drm_i915_private *dev_priv = dev->dev_private;
c397b908 3522 struct drm_i915_gem_object *obj;
ac52bc56 3523
c397b908
DV		 3524 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3525 if (obj == NULL)
3526 return NULL;
673a394b 3527
c397b908
DV		 3528 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3529 kfree(obj);
3530 return NULL;
3531 }
673a394b 3532
73aa808f
CW		 3533 i915_gem_info_add_obj(dev_priv, size);
3534
c397b908
DV		 3535 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3536 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
673a394b 3537
c397b908 3538 obj->agp_type = AGP_USER_MEMORY;
62b8b215 3539 obj->base.driver_private = NULL;
c397b908 3540 obj->fence_reg = I915_FENCE_REG_NONE;
69dc4987 3541 INIT_LIST_HEAD(&obj->mm_list);
93a37f20 3542 INIT_LIST_HEAD(&obj->gtt_list);
69dc4987 3543 INIT_LIST_HEAD(&obj->ring_list);
432e58ed 3544 INIT_LIST_HEAD(&obj->exec_list);
c397b908 3545 INIT_LIST_HEAD(&obj->gpu_write_list);
c397b908 3546 obj->madv = I915_MADV_WILLNEED;
75e9e915
DV		 3547 /* Avoid an unnecessary call to unbind on the first bind. */
3548 obj->map_and_fenceable = true;
de151cf6 3549
05394f39 3550 return obj;
c397b908
DV		 3551}
3552
3553int i915_gem_init_object(struct drm_gem_object *obj)
3554{
3555 BUG();
de151cf6 3556
673a394b
EA		 3557 return 0;
3558}
3559
05394f39 3560static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
673a394b 3561{
05394f39 3562 struct drm_device *dev = obj->base.dev;
be72615b 3563 drm_i915_private_t *dev_priv = dev->dev_private;
be72615b 3564 int ret;
673a394b 3565
be72615b
CW		 3566 ret = i915_gem_object_unbind(obj);
3567 if (ret == -ERESTARTSYS) {
05394f39 3568 list_move(&obj->mm_list,
be72615b
CW		 3569 &dev_priv->mm.deferred_free_list);
3570 return;
3571 }
673a394b 3572
05394f39 3573 if (obj->base.map_list.map)
7e616158 3574 i915_gem_free_mmap_offset(obj);
de151cf6 3575
05394f39
CW		 3576 drm_gem_object_release(&obj->base);
3577 i915_gem_info_remove_obj(dev_priv, obj->base.size);
c397b908 3578
05394f39
CW		 3579 kfree(obj->page_cpu_valid);
3580 kfree(obj->bit_17);
3581 kfree(obj);
673a394b
EA		 3582}
3583
05394f39 3584void i915_gem_free_object(struct drm_gem_object *gem_obj)
be72615b 3585{
05394f39
CW		 3586 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3587 struct drm_device *dev = obj->base.dev;
be72615b
CW		 3588
3589 trace_i915_gem_object_destroy(obj);
3590
05394f39 3591 while (obj->pin_count > 0)
be72615b
CW		 3592 i915_gem_object_unpin(obj);
3593
05394f39 3594 if (obj->phys_obj)
be72615b
CW		 3595 i915_gem_detach_phys_object(dev, obj);
3596
3597 i915_gem_free_object_tail(obj);
3598}
3599
29105ccc
CW		 3600int
3601i915_gem_idle(struct drm_device *dev)
3602{
3603 drm_i915_private_t *dev_priv = dev->dev_private;
3604 int ret;
28dfe52a 3605
29105ccc 3606 mutex_lock(&dev->struct_mutex);
1c5d22f7 3607
87acb0a5 3608 if (dev_priv->mm.suspended) {
29105ccc
CW		 3609 mutex_unlock(&dev->struct_mutex);
3610 return 0;
28dfe52a
EA		 3611 }
3612
29105ccc 3613 ret = i915_gpu_idle(dev);
6dbe2772
KP		 3614 if (ret) {
3615 mutex_unlock(&dev->struct_mutex);
673a394b 3616 return ret;
6dbe2772 3617 }
673a394b 3618
29105ccc
CW		 3619 /* Under UMS, be paranoid and evict. */
3620 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
5eac3ab4 3621 ret = i915_gem_evict_inactive(dev, false);
29105ccc
CW		 3622 if (ret) {
3623 mutex_unlock(&dev->struct_mutex);
3624 return ret;
3625 }
3626 }
3627
312817a3
CW		 3628 i915_gem_reset_fences(dev);
3629
29105ccc
CW		 3630 /* Hack! Don't let anybody do execbuf while we don't control the chip.
3631 * We need to replace this with a semaphore, or something.
3632 * And not confound mm.suspended!
3633 */
3634 dev_priv->mm.suspended = 1;
bc0c7f14 3635 del_timer_sync(&dev_priv->hangcheck_timer);
29105ccc
CW		 3636
3637 i915_kernel_lost_context(dev);
6dbe2772 3638 i915_gem_cleanup_ringbuffer(dev);
29105ccc 3639
6dbe2772
KP		 3640 mutex_unlock(&dev->struct_mutex);
3641
29105ccc
CW		 3642 /* Cancel the retire work handler, which should be idle now. */
3643 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3644
673a394b
EA		 3645 return 0;
3646}
3647
8187a2b7
ZN		 3648int
3649i915_gem_init_ringbuffer(struct drm_device *dev)
3650{
3651 drm_i915_private_t *dev_priv = dev->dev_private;
3652 int ret;
68f95ba9 3653
5c1143bb 3654 ret = intel_init_render_ring_buffer(dev);
68f95ba9 3655 if (ret)
b6913e4b 3656 return ret;
68f95ba9
CW		 3657
3658 if (HAS_BSD(dev)) {
5c1143bb 3659 ret = intel_init_bsd_ring_buffer(dev);
68f95ba9
CW		 3660 if (ret)
3661 goto cleanup_render_ring;
d1b851fc 3662 }
68f95ba9 3663
549f7365
CW		 3664 if (HAS_BLT(dev)) {
3665 ret = intel_init_blt_ring_buffer(dev);
3666 if (ret)
3667 goto cleanup_bsd_ring;
3668 }
3669
6f392d54
CW		 3670 dev_priv->next_seqno = 1;
3671
68f95ba9
CW		 3672 return 0;
3673
549f7365 3674cleanup_bsd_ring:
1ec14ad3 3675 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
68f95ba9 3676cleanup_render_ring:
1ec14ad3 3677 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
8187a2b7
ZN		 3678 return ret;
3679}
3680
3681void
3682i915_gem_cleanup_ringbuffer(struct drm_device *dev)
3683{
3684 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 3685 int i;
8187a2b7 3686
1ec14ad3
CW		 3687 for (i = 0; i < I915_NUM_RINGS; i++)
3688 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
8187a2b7
ZN		 3689}
3690
673a394b
EA		 3691int
3692i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
3693 struct drm_file *file_priv)
3694{
3695 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 3696 int ret, i;
673a394b 3697
79e53945
JB		 3698 if (drm_core_check_feature(dev, DRIVER_MODESET))
3699 return 0;
3700
ba1234d1 3701 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 3702 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 3703 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 3704 }
3705
673a394b 3706 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 3707 dev_priv->mm.suspended = 0;
3708
3709 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 3710 if (ret != 0) {
3711 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 3712 return ret;
d816f6ac 3713 }
9bb2d6f9 3714
69dc4987 3715 BUG_ON(!list_empty(&dev_priv->mm.active_list));
673a394b
EA		 3716 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
3717 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
1ec14ad3
CW		 3718 for (i = 0; i < I915_NUM_RINGS; i++) {
3719 BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
3720 BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
3721 }
673a394b 3722 mutex_unlock(&dev->struct_mutex);
dbb19d30 3723
5f35308b
CW		 3724 ret = drm_irq_install(dev);
3725 if (ret)
3726 goto cleanup_ringbuffer;
dbb19d30 3727
673a394b 3728 return 0;
5f35308b
CW		 3729
3730cleanup_ringbuffer:
3731 mutex_lock(&dev->struct_mutex);
3732 i915_gem_cleanup_ringbuffer(dev);
3733 dev_priv->mm.suspended = 1;
3734 mutex_unlock(&dev->struct_mutex);
3735
3736 return ret;
673a394b
EA		 3737}
3738
3739int
3740i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
3741 struct drm_file *file_priv)
3742{
79e53945
JB		 3743 if (drm_core_check_feature(dev, DRIVER_MODESET))
3744 return 0;
3745
dbb19d30 3746 drm_irq_uninstall(dev);
e6890f6f 3747 return i915_gem_idle(dev);
673a394b
EA		 3748}
3749
3750void
3751i915_gem_lastclose(struct drm_device *dev)
3752{
3753 int ret;
673a394b 3754
e806b495
EA		 3755 if (drm_core_check_feature(dev, DRIVER_MODESET))
3756 return;
3757
6dbe2772
KP		 3758 ret = i915_gem_idle(dev);
3759 if (ret)
3760 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 3761}
3762
64193406
CW		 3763static void
3764init_ring_lists(struct intel_ring_buffer *ring)
3765{
3766 INIT_LIST_HEAD(&ring->active_list);
3767 INIT_LIST_HEAD(&ring->request_list);
3768 INIT_LIST_HEAD(&ring->gpu_write_list);
3769}
3770
673a394b
EA		 3771void
3772i915_gem_load(struct drm_device *dev)
3773{
b5aa8a0f 3774 int i;
673a394b
EA		 3775 drm_i915_private_t *dev_priv = dev->dev_private;
3776
69dc4987 3777 INIT_LIST_HEAD(&dev_priv->mm.active_list);
673a394b
EA		 3778 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3779 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
f13d3f73 3780 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
a09ba7fa 3781 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
be72615b 3782 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
93a37f20 3783 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
1ec14ad3
CW		 3784 for (i = 0; i < I915_NUM_RINGS; i++)
3785 init_ring_lists(&dev_priv->ring[i]);
007cc8ac
DV		 3786 for (i = 0; i < 16; i++)
3787 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
673a394b
EA		 3788 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3789 i915_gem_retire_work_handler);
30dbf0c0 3790 init_completion(&dev_priv->error_completion);
31169714 3791
94400120
DA		 3792 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3793 if (IS_GEN3(dev)) {
3794 u32 tmp = I915_READ(MI_ARB_STATE);
3795 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
3796 /* arb state is a masked write, so set bit + bit in mask */
3797 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
3798 I915_WRITE(MI_ARB_STATE, tmp);
3799 }
3800 }
3801
72bfa19c
CW		 3802 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3803
de151cf6 3804 /* Old X drivers will take 0-2 for front, back, depth buffers */
b397c836
EA		 3805 if (!drm_core_check_feature(dev, DRIVER_MODESET))
3806 dev_priv->fence_reg_start = 3;
de151cf6 3807
a6c45cf0 3808 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 3809 dev_priv->num_fence_regs = 16;
3810 else
3811 dev_priv->num_fence_regs = 8;
3812
b5aa8a0f 3813 /* Initialize fence registers to zero */
a6c45cf0
CW		 3814 switch (INTEL_INFO(dev)->gen) {
3815 case 6:
3816 for (i = 0; i < 16; i++)
3817 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
3818 break;
3819 case 5:
3820 case 4:
b5aa8a0f
GH		 3821 for (i = 0; i < 16; i++)
3822 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
a6c45cf0
CW		 3823 break;
3824 case 3:
b5aa8a0f
GH		 3825 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3826 for (i = 0; i < 8; i++)
3827 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
a6c45cf0
CW		 3828 case 2:
3829 for (i = 0; i < 8; i++)
3830 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
3831 break;
b5aa8a0f 3832 }
673a394b 3833 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 3834 init_waitqueue_head(&dev_priv->pending_flip_queue);
17250b71
CW		 3835
3836 dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3837 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3838 register_shrinker(&dev_priv->mm.inactive_shrinker);
673a394b 3839}
71acb5eb
DA		 3840
3841/*
3842 * Create a physically contiguous memory object for this object
3843 * e.g. for cursor + overlay regs
3844 */
995b6762
CW		 3845static int i915_gem_init_phys_object(struct drm_device *dev,
3846 int id, int size, int align)
71acb5eb
DA		 3847{
3848 drm_i915_private_t *dev_priv = dev->dev_private;
3849 struct drm_i915_gem_phys_object *phys_obj;
3850 int ret;
3851
3852 if (dev_priv->mm.phys_objs[id - 1] || !size)
3853 return 0;
3854
9a298b2a 3855 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 3856 if (!phys_obj)
3857 return -ENOMEM;
3858
3859 phys_obj->id = id;
3860
6eeefaf3 3861 phys_obj->handle = drm_pci_alloc(dev, size, align);
71acb5eb
DA		 3862 if (!phys_obj->handle) {
3863 ret = -ENOMEM;
3864 goto kfree_obj;
3865 }
3866#ifdef CONFIG_X86
3867 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3868#endif
3869
3870 dev_priv->mm.phys_objs[id - 1] = phys_obj;
3871
3872 return 0;
3873kfree_obj:
9a298b2a 3874 kfree(phys_obj);
71acb5eb
DA		 3875 return ret;
3876}
3877
995b6762 3878static void i915_gem_free_phys_object(struct drm_device *dev, int id)
71acb5eb
DA		 3879{
3880 drm_i915_private_t *dev_priv = dev->dev_private;
3881 struct drm_i915_gem_phys_object *phys_obj;
3882
3883 if (!dev_priv->mm.phys_objs[id - 1])
3884 return;
3885
3886 phys_obj = dev_priv->mm.phys_objs[id - 1];
3887 if (phys_obj->cur_obj) {
3888 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3889 }
3890
3891#ifdef CONFIG_X86
3892 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3893#endif
3894 drm_pci_free(dev, phys_obj->handle);
3895 kfree(phys_obj);
3896 dev_priv->mm.phys_objs[id - 1] = NULL;
3897}
3898
3899void i915_gem_free_all_phys_object(struct drm_device *dev)
3900{
3901 int i;
3902
260883c8 3903 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 3904 i915_gem_free_phys_object(dev, i);
3905}
3906
3907void i915_gem_detach_phys_object(struct drm_device *dev,
05394f39 3908 struct drm_i915_gem_object *obj)
71acb5eb 3909{
05394f39 3910 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
e5281ccd 3911 char *vaddr;
71acb5eb 3912 int i;
71acb5eb
DA		 3913 int page_count;
3914
05394f39 3915 if (!obj->phys_obj)
71acb5eb 3916 return;
05394f39 3917 vaddr = obj->phys_obj->handle->vaddr;
71acb5eb 3918
05394f39 3919 page_count = obj->base.size / PAGE_SIZE;
71acb5eb 3920 for (i = 0; i < page_count; i++) {
e5281ccd
CW		 3921 struct page *page = read_cache_page_gfp(mapping, i,
3922 GFP_HIGHUSER | __GFP_RECLAIMABLE);
3923 if (!IS_ERR(page)) {
3924 char *dst = kmap_atomic(page);
3925 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
3926 kunmap_atomic(dst);
3927
3928 drm_clflush_pages(&page, 1);
3929
3930 set_page_dirty(page);
3931 mark_page_accessed(page);
3932 page_cache_release(page);
3933 }
71acb5eb 3934 }
40ce6575 3935 intel_gtt_chipset_flush();
d78b47b9 3936
05394f39
CW		 3937 obj->phys_obj->cur_obj = NULL;
3938 obj->phys_obj = NULL;
71acb5eb
DA		 3939}
3940
3941int
3942i915_gem_attach_phys_object(struct drm_device *dev,
05394f39 3943 struct drm_i915_gem_object *obj,
6eeefaf3
CW		 3944 int id,
3945 int align)
71acb5eb 3946{
05394f39 3947 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
71acb5eb 3948 drm_i915_private_t *dev_priv = dev->dev_private;
71acb5eb
DA		 3949 int ret = 0;
3950 int page_count;
3951 int i;
3952
3953 if (id > I915_MAX_PHYS_OBJECT)
3954 return -EINVAL;
3955
05394f39
CW		 3956 if (obj->phys_obj) {
3957 if (obj->phys_obj->id == id)
71acb5eb
DA		 3958 return 0;
3959 i915_gem_detach_phys_object(dev, obj);
3960 }
3961
71acb5eb
DA		 3962 /* create a new object */
3963 if (!dev_priv->mm.phys_objs[id - 1]) {
3964 ret = i915_gem_init_phys_object(dev, id,
05394f39 3965 obj->base.size, align);
71acb5eb 3966 if (ret) {
05394f39
CW		 3967 DRM_ERROR("failed to init phys object %d size: %zu\n",
3968 id, obj->base.size);
e5281ccd 3969 return ret;
71acb5eb
DA		 3970 }
3971 }
3972
3973 /* bind to the object */
05394f39
CW		 3974 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3975 obj->phys_obj->cur_obj = obj;
71acb5eb 3976
05394f39 3977 page_count = obj->base.size / PAGE_SIZE;
71acb5eb
DA		 3978
3979 for (i = 0; i < page_count; i++) {
e5281ccd
CW		 3980 struct page *page;
3981 char *dst, *src;
3982
3983 page = read_cache_page_gfp(mapping, i,
3984 GFP_HIGHUSER | __GFP_RECLAIMABLE);
3985 if (IS_ERR(page))
3986 return PTR_ERR(page);
71acb5eb 3987
ff75b9bc 3988 src = kmap_atomic(page);
05394f39 3989 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
71acb5eb 3990 memcpy(dst, src, PAGE_SIZE);
3e4d3af5 3991 kunmap_atomic(src);
71acb5eb 3992
e5281ccd
CW		 3993 mark_page_accessed(page);
3994 page_cache_release(page);
3995 }
d78b47b9 3996
71acb5eb 3997 return 0;
71acb5eb
DA		 3998}
3999
4000static int
05394f39
CW		 4001i915_gem_phys_pwrite(struct drm_device *dev,
4002 struct drm_i915_gem_object *obj,
71acb5eb
DA		 4003 struct drm_i915_gem_pwrite *args,
4004 struct drm_file *file_priv)
4005{
05394f39 4006 void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
b47b30cc 4007 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
71acb5eb 4008
b47b30cc
CW		 4009 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4010 unsigned long unwritten;
4011
4012 /* The physical object once assigned is fixed for the lifetime
4013 * of the obj, so we can safely drop the lock and continue
4014 * to access vaddr.
4015 */
4016 mutex_unlock(&dev->struct_mutex);
4017 unwritten = copy_from_user(vaddr, user_data, args->size);
4018 mutex_lock(&dev->struct_mutex);
4019 if (unwritten)
4020 return -EFAULT;
4021 }
71acb5eb 4022
40ce6575 4023 intel_gtt_chipset_flush();
71acb5eb
DA		 4024 return 0;
4025}
b962442e 4026
f787a5f5 4027void i915_gem_release(struct drm_device *dev, struct drm_file *file)
b962442e 4028{
f787a5f5 4029 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e
EA		 4030
4031 /* Clean up our request list when the client is going away, so that
4032 * later retire_requests won't dereference our soon-to-be-gone
4033 * file_priv.
4034 */
1c25595f 4035 spin_lock(&file_priv->mm.lock);
f787a5f5
CW		 4036 while (!list_empty(&file_priv->mm.request_list)) {
4037 struct drm_i915_gem_request *request;
4038
4039 request = list_first_entry(&file_priv->mm.request_list,
4040 struct drm_i915_gem_request,
4041 client_list);
4042 list_del(&request->client_list);
4043 request->file_priv = NULL;
4044 }
1c25595f 4045 spin_unlock(&file_priv->mm.lock);
b962442e 4046}
31169714 4047
1637ef41
CW		 4048static int
4049i915_gpu_is_active(struct drm_device *dev)
4050{
4051 drm_i915_private_t *dev_priv = dev->dev_private;
4052 int lists_empty;
4053
1637ef41 4054 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
17250b71 4055 list_empty(&dev_priv->mm.active_list);
1637ef41
CW		 4056
4057 return !lists_empty;
4058}
4059
31169714 4060static int
17250b71
CW		 4061i915_gem_inactive_shrink(struct shrinker *shrinker,
4062 int nr_to_scan,
4063 gfp_t gfp_mask)
31169714 4064{
17250b71
CW		 4065 struct drm_i915_private *dev_priv =
4066 container_of(shrinker,
4067 struct drm_i915_private,
4068 mm.inactive_shrinker);
4069 struct drm_device *dev = dev_priv->dev;
4070 struct drm_i915_gem_object *obj, *next;
4071 int cnt;
4072
4073 if (!mutex_trylock(&dev->struct_mutex))
bbe2e11a 4074 return 0;
31169714
CW		 4075
4076 /* "fast-path" to count number of available objects */
4077 if (nr_to_scan == 0) {
17250b71
CW		 4078 cnt = 0;
4079 list_for_each_entry(obj,
4080 &dev_priv->mm.inactive_list,
4081 mm_list)
4082 cnt++;
4083 mutex_unlock(&dev->struct_mutex);
4084 return cnt / 100 * sysctl_vfs_cache_pressure;
31169714
CW		 4085 }
4086
1637ef41 4087rescan:
31169714 4088 /* first scan for clean buffers */
17250b71 4089 i915_gem_retire_requests(dev);
31169714 4090
17250b71
CW		 4091 list_for_each_entry_safe(obj, next,
4092 &dev_priv->mm.inactive_list,
4093 mm_list) {
4094 if (i915_gem_object_is_purgeable(obj)) {
2021746e
CW		 4095 if (i915_gem_object_unbind(obj) == 0 &&
4096 --nr_to_scan == 0)
17250b71 4097 break;
31169714 4098 }
31169714
CW		 4099 }
4100
4101 /* second pass, evict/count anything still on the inactive list */
17250b71
CW		 4102 cnt = 0;
4103 list_for_each_entry_safe(obj, next,
4104 &dev_priv->mm.inactive_list,
4105 mm_list) {
2021746e
CW		 4106 if (nr_to_scan &&
4107 i915_gem_object_unbind(obj) == 0)
17250b71 4108 nr_to_scan--;
2021746e 4109 else
17250b71
CW		 4110 cnt++;
4111 }
4112
4113 if (nr_to_scan && i915_gpu_is_active(dev)) {
1637ef41
CW		 4114 /*
4115 * We are desperate for pages, so as a last resort, wait
4116 * for the GPU to finish and discard whatever we can.
4117 * This has a dramatic impact to reduce the number of
4118 * OOM-killer events whilst running the GPU aggressively.
4119 */
17250b71 4120 if (i915_gpu_idle(dev) == 0)
1637ef41
CW		 4121 goto rescan;
4122 }
17250b71
CW		 4123 mutex_unlock(&dev->struct_mutex);
4124 return cnt / 100 * sysctl_vfs_cache_pressure;
31169714 4125}
Linux kernel - Deliverables
This page took 0.619758 seconds and 5 git commands to generate.