projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Introduce i915_gem_object_finish_gpu()
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
5a0e3ad6 34#include <linux/slab.h>
673a394b 35#include <linux/swap.h>
79e53945 36#include <linux/pci.h>
673a394b 37
88241785 38static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
05394f39
CW		 39static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
40static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
88241785
CW		 41static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
42 bool write);
43static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
44 uint64_t offset,
45 uint64_t size);
05394f39 46static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
88241785
CW		 47static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
48 unsigned alignment,
49 bool map_and_fenceable);
d9e86c0e
CW		 50static void i915_gem_clear_fence_reg(struct drm_device *dev,
51 struct drm_i915_fence_reg *reg);
05394f39
CW		 52static int i915_gem_phys_pwrite(struct drm_device *dev,
53 struct drm_i915_gem_object *obj,
71acb5eb 54 struct drm_i915_gem_pwrite *args,
05394f39
CW		 55 struct drm_file *file);
56static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
673a394b 57
17250b71 58static int i915_gem_inactive_shrink(struct shrinker *shrinker,
1495f230 59 struct shrink_control *sc);
31169714 60
73aa808f
CW		 61/* some bookkeeping */
62static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
63 size_t size)
64{
65 dev_priv->mm.object_count++;
66 dev_priv->mm.object_memory += size;
67}
68
69static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
70 size_t size)
71{
72 dev_priv->mm.object_count--;
73 dev_priv->mm.object_memory -= size;
74}
75
21dd3734
CW		 76static int
77i915_gem_wait_for_error(struct drm_device *dev)
30dbf0c0
CW		 78{
79 struct drm_i915_private *dev_priv = dev->dev_private;
80 struct completion *x = &dev_priv->error_completion;
81 unsigned long flags;
82 int ret;
83
84 if (!atomic_read(&dev_priv->mm.wedged))
85 return 0;
86
87 ret = wait_for_completion_interruptible(x);
88 if (ret)
89 return ret;
90
21dd3734
CW		 91 if (atomic_read(&dev_priv->mm.wedged)) {
92 /* GPU is hung, bump the completion count to account for
93 * the token we just consumed so that we never hit zero and
94 * end up waiting upon a subsequent completion event that
95 * will never happen.
96 */
97 spin_lock_irqsave(&x->wait.lock, flags);
98 x->done++;
99 spin_unlock_irqrestore(&x->wait.lock, flags);
100 }
101 return 0;
30dbf0c0
CW		 102}
103
54cf91dc 104int i915_mutex_lock_interruptible(struct drm_device *dev)
76c1dec1 105{
76c1dec1
CW		 106 int ret;
107
21dd3734 108 ret = i915_gem_wait_for_error(dev);
76c1dec1
CW		 109 if (ret)
110 return ret;
111
112 ret = mutex_lock_interruptible(&dev->struct_mutex);
113 if (ret)
114 return ret;
115
23bc5982 116 WARN_ON(i915_verify_lists(dev));
76c1dec1
CW		 117 return 0;
118}
30dbf0c0 119
7d1c4804 120static inline bool
05394f39 121i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
7d1c4804 122{
05394f39 123 return obj->gtt_space && !obj->active && obj->pin_count == 0;
7d1c4804
CW		 124}
125
2021746e
CW		 126void i915_gem_do_init(struct drm_device *dev,
127 unsigned long start,
128 unsigned long mappable_end,
129 unsigned long end)
673a394b
EA		 130{
131 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 132
bee4a186 133 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
673a394b 134
bee4a186
CW		 135 dev_priv->mm.gtt_start = start;
136 dev_priv->mm.gtt_mappable_end = mappable_end;
137 dev_priv->mm.gtt_end = end;
73aa808f 138 dev_priv->mm.gtt_total = end - start;
fb7d516a 139 dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
bee4a186
CW		 140
141 /* Take over this portion of the GTT */
142 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
79e53945 143}
673a394b 144
79e53945
JB		 145int
146i915_gem_init_ioctl(struct drm_device *dev, void *data,
05394f39 147 struct drm_file *file)
79e53945
JB		 148{
149 struct drm_i915_gem_init *args = data;
2021746e
CW		 150
151 if (args->gtt_start >= args->gtt_end ||
152 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
153 return -EINVAL;
79e53945
JB		 154
155 mutex_lock(&dev->struct_mutex);
2021746e 156 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
673a394b
EA		 157 mutex_unlock(&dev->struct_mutex);
158
2021746e 159 return 0;
673a394b
EA		 160}
161
5a125c3c
EA		 162int
163i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
05394f39 164 struct drm_file *file)
5a125c3c 165{
73aa808f 166 struct drm_i915_private *dev_priv = dev->dev_private;
5a125c3c 167 struct drm_i915_gem_get_aperture *args = data;
6299f992
CW		 168 struct drm_i915_gem_object *obj;
169 size_t pinned;
5a125c3c
EA		 170
171 if (!(dev->driver->driver_features & DRIVER_GEM))
172 return -ENODEV;
173
6299f992 174 pinned = 0;
73aa808f 175 mutex_lock(&dev->struct_mutex);
6299f992
CW		 176 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
177 pinned += obj->gtt_space->size;
73aa808f 178 mutex_unlock(&dev->struct_mutex);
5a125c3c 179
6299f992
CW		 180 args->aper_size = dev_priv->mm.gtt_total;
181 args->aper_available_size = args->aper_size -pinned;
182
5a125c3c
EA		 183 return 0;
184}
185
ff72145b
DA		 186static int
187i915_gem_create(struct drm_file *file,
188 struct drm_device *dev,
189 uint64_t size,
190 uint32_t *handle_p)
673a394b 191{
05394f39 192 struct drm_i915_gem_object *obj;
a1a2d1d3
PP		 193 int ret;
194 u32 handle;
673a394b 195
ff72145b 196 size = roundup(size, PAGE_SIZE);
673a394b
EA		 197
198 /* Allocate the new object */
ff72145b 199 obj = i915_gem_alloc_object(dev, size);
673a394b
EA		 200 if (obj == NULL)
201 return -ENOMEM;
202
05394f39 203 ret = drm_gem_handle_create(file, &obj->base, &handle);
1dfd9754 204 if (ret) {
05394f39
CW		 205 drm_gem_object_release(&obj->base);
206 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
202f2fef 207 kfree(obj);
673a394b 208 return ret;
1dfd9754 209 }
673a394b 210
202f2fef 211 /* drop reference from allocate - handle holds it now */
05394f39 212 drm_gem_object_unreference(&obj->base);
202f2fef
CW		 213 trace_i915_gem_object_create(obj);
214
ff72145b 215 *handle_p = handle;
673a394b
EA		 216 return 0;
217}
218
ff72145b
DA		 219int
220i915_gem_dumb_create(struct drm_file *file,
221 struct drm_device *dev,
222 struct drm_mode_create_dumb *args)
223{
224 /* have to work out size/pitch and return them */
ed0291fd 225 args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
ff72145b
DA		 226 args->size = args->pitch * args->height;
227 return i915_gem_create(file, dev,
228 args->size, &args->handle);
229}
230
231int i915_gem_dumb_destroy(struct drm_file *file,
232 struct drm_device *dev,
233 uint32_t handle)
234{
235 return drm_gem_handle_delete(file, handle);
236}
237
238/**
239 * Creates a new mm object and returns a handle to it.
240 */
241int
242i915_gem_create_ioctl(struct drm_device *dev, void *data,
243 struct drm_file *file)
244{
245 struct drm_i915_gem_create *args = data;
246 return i915_gem_create(file, dev,
247 args->size, &args->handle);
248}
249
05394f39 250static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
280b713b 251{
05394f39 252 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
280b713b
EA		 253
254 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
05394f39 255 obj->tiling_mode != I915_TILING_NONE;
280b713b
EA		 256}
257
99a03df5 258static inline void
40123c1f
EA		 259slow_shmem_copy(struct page *dst_page,
260 int dst_offset,
261 struct page *src_page,
262 int src_offset,
263 int length)
264{
265 char *dst_vaddr, *src_vaddr;
266
99a03df5
CW		 267 dst_vaddr = kmap(dst_page);
268 src_vaddr = kmap(src_page);
40123c1f
EA		 269
270 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
271
99a03df5
CW		 272 kunmap(src_page);
273 kunmap(dst_page);
40123c1f
EA		 274}
275
99a03df5 276static inline void
280b713b
EA		 277slow_shmem_bit17_copy(struct page *gpu_page,
278 int gpu_offset,
279 struct page *cpu_page,
280 int cpu_offset,
281 int length,
282 int is_read)
283{
284 char *gpu_vaddr, *cpu_vaddr;
285
286 /* Use the unswizzled path if this page isn't affected. */
287 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
288 if (is_read)
289 return slow_shmem_copy(cpu_page, cpu_offset,
290 gpu_page, gpu_offset, length);
291 else
292 return slow_shmem_copy(gpu_page, gpu_offset,
293 cpu_page, cpu_offset, length);
294 }
295
99a03df5
CW		 296 gpu_vaddr = kmap(gpu_page);
297 cpu_vaddr = kmap(cpu_page);
280b713b
EA		 298
299 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
300 * XORing with the other bits (A9 for Y, A9 and A10 for X)
301 */
302 while (length > 0) {
303 int cacheline_end = ALIGN(gpu_offset + 1, 64);
304 int this_length = min(cacheline_end - gpu_offset, length);
305 int swizzled_gpu_offset = gpu_offset ^ 64;
306
307 if (is_read) {
308 memcpy(cpu_vaddr + cpu_offset,
309 gpu_vaddr + swizzled_gpu_offset,
310 this_length);
311 } else {
312 memcpy(gpu_vaddr + swizzled_gpu_offset,
313 cpu_vaddr + cpu_offset,
314 this_length);
315 }
316 cpu_offset += this_length;
317 gpu_offset += this_length;
318 length -= this_length;
319 }
320
99a03df5
CW		 321 kunmap(cpu_page);
322 kunmap(gpu_page);
280b713b
EA		 323}
324
eb01459f
EA		 325/**
326 * This is the fast shmem pread path, which attempts to copy_from_user directly
327 * from the backing pages of the object to the user's address space. On a
328 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
329 */
330static int
05394f39
CW		 331i915_gem_shmem_pread_fast(struct drm_device *dev,
332 struct drm_i915_gem_object *obj,
eb01459f 333 struct drm_i915_gem_pread *args,
05394f39 334 struct drm_file *file)
eb01459f 335{
05394f39 336 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
eb01459f 337 ssize_t remain;
e5281ccd 338 loff_t offset;
eb01459f
EA		 339 char __user *user_data;
340 int page_offset, page_length;
eb01459f
EA		 341
342 user_data = (char __user *) (uintptr_t) args->data_ptr;
343 remain = args->size;
344
eb01459f
EA		 345 offset = args->offset;
346
347 while (remain > 0) {
e5281ccd
CW		 348 struct page *page;
349 char *vaddr;
350 int ret;
351
eb01459f
EA		 352 /* Operation in this page
353 *
eb01459f
EA		 354 * page_offset = offset within page
355 * page_length = bytes to copy for this page
356 */
c8cbbb8b 357 page_offset = offset_in_page(offset);
eb01459f
EA		 358 page_length = remain;
359 if ((page_offset + remain) > PAGE_SIZE)
360 page_length = PAGE_SIZE - page_offset;
361
e5281ccd
CW		 362 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
363 GFP_HIGHUSER | __GFP_RECLAIMABLE);
364 if (IS_ERR(page))
365 return PTR_ERR(page);
366
367 vaddr = kmap_atomic(page);
368 ret = __copy_to_user_inatomic(user_data,
369 vaddr + page_offset,
370 page_length);
371 kunmap_atomic(vaddr);
372
373 mark_page_accessed(page);
374 page_cache_release(page);
375 if (ret)
4f27b75d 376 return -EFAULT;
eb01459f
EA		 377
378 remain -= page_length;
379 user_data += page_length;
380 offset += page_length;
381 }
382
4f27b75d 383 return 0;
eb01459f
EA		 384}
385
386/**
387 * This is the fallback shmem pread path, which allocates temporary storage
388 * in kernel space to copy_to_user into outside of the struct_mutex, so we
389 * can copy out of the object's backing pages while holding the struct mutex
390 * and not take page faults.
391 */
392static int
05394f39
CW		 393i915_gem_shmem_pread_slow(struct drm_device *dev,
394 struct drm_i915_gem_object *obj,
eb01459f 395 struct drm_i915_gem_pread *args,
05394f39 396 struct drm_file *file)
eb01459f 397{
05394f39 398 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
eb01459f
EA		 399 struct mm_struct *mm = current->mm;
400 struct page **user_pages;
401 ssize_t remain;
402 loff_t offset, pinned_pages, i;
403 loff_t first_data_page, last_data_page, num_pages;
e5281ccd
CW		 404 int shmem_page_offset;
405 int data_page_index, data_page_offset;
eb01459f
EA		 406 int page_length;
407 int ret;
408 uint64_t data_ptr = args->data_ptr;
280b713b 409 int do_bit17_swizzling;
eb01459f
EA		 410
411 remain = args->size;
412
413 /* Pin the user pages containing the data. We can't fault while
414 * holding the struct mutex, yet we want to hold it while
415 * dereferencing the user data.
416 */
417 first_data_page = data_ptr / PAGE_SIZE;
418 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
419 num_pages = last_data_page - first_data_page + 1;
420
4f27b75d 421 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
eb01459f
EA		 422 if (user_pages == NULL)
423 return -ENOMEM;
424
4f27b75d 425 mutex_unlock(&dev->struct_mutex);
eb01459f
EA		 426 down_read(&mm->mmap_sem);
427 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 428 num_pages, 1, 0, user_pages, NULL);
eb01459f 429 up_read(&mm->mmap_sem);
4f27b75d 430 mutex_lock(&dev->struct_mutex);
eb01459f
EA		 431 if (pinned_pages < num_pages) {
432 ret = -EFAULT;
4f27b75d 433 goto out;
eb01459f
EA		 434 }
435
4f27b75d
CW		 436 ret = i915_gem_object_set_cpu_read_domain_range(obj,
437 args->offset,
438 args->size);
07f73f69 439 if (ret)
4f27b75d 440 goto out;
eb01459f 441
4f27b75d 442 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
eb01459f 443
eb01459f
EA		 444 offset = args->offset;
445
446 while (remain > 0) {
e5281ccd
CW		 447 struct page *page;
448
eb01459f
EA		 449 /* Operation in this page
450 *
eb01459f
EA		 451 * shmem_page_offset = offset within page in shmem file
452 * data_page_index = page number in get_user_pages return
453 * data_page_offset = offset with data_page_index page.
454 * page_length = bytes to copy for this page
455 */
c8cbbb8b 456 shmem_page_offset = offset_in_page(offset);
eb01459f 457 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
c8cbbb8b 458 data_page_offset = offset_in_page(data_ptr);
eb01459f
EA		 459
460 page_length = remain;
461 if ((shmem_page_offset + page_length) > PAGE_SIZE)
462 page_length = PAGE_SIZE - shmem_page_offset;
463 if ((data_page_offset + page_length) > PAGE_SIZE)
464 page_length = PAGE_SIZE - data_page_offset;
465
e5281ccd
CW		 466 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
467 GFP_HIGHUSER | __GFP_RECLAIMABLE);
468 if (IS_ERR(page))
469 return PTR_ERR(page);
470
280b713b 471 if (do_bit17_swizzling) {
e5281ccd 472 slow_shmem_bit17_copy(page,
280b713b 473 shmem_page_offset,
99a03df5
CW		 474 user_pages[data_page_index],
475 data_page_offset,
476 page_length,
477 1);
478 } else {
479 slow_shmem_copy(user_pages[data_page_index],
480 data_page_offset,
e5281ccd 481 page,
99a03df5
CW		 482 shmem_page_offset,
483 page_length);
280b713b 484 }
eb01459f 485
e5281ccd
CW		 486 mark_page_accessed(page);
487 page_cache_release(page);
488
eb01459f
EA		 489 remain -= page_length;
490 data_ptr += page_length;
491 offset += page_length;
492 }
493
4f27b75d 494out:
eb01459f
EA		 495 for (i = 0; i < pinned_pages; i++) {
496 SetPageDirty(user_pages[i]);
e5281ccd 497 mark_page_accessed(user_pages[i]);
eb01459f
EA		 498 page_cache_release(user_pages[i]);
499 }
8e7d2b2c 500 drm_free_large(user_pages);
eb01459f
EA		 501
502 return ret;
503}
504
673a394b
EA		 505/**
506 * Reads data from the object referenced by handle.
507 *
508 * On error, the contents of *data are undefined.
509 */
510int
511i915_gem_pread_ioctl(struct drm_device *dev, void *data,
05394f39 512 struct drm_file *file)
673a394b
EA		 513{
514 struct drm_i915_gem_pread *args = data;
05394f39 515 struct drm_i915_gem_object *obj;
35b62a89 516 int ret = 0;
673a394b 517
51311d0a
CW		 518 if (args->size == 0)
519 return 0;
520
521 if (!access_ok(VERIFY_WRITE,
522 (char __user *)(uintptr_t)args->data_ptr,
523 args->size))
524 return -EFAULT;
525
526 ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
527 args->size);
528 if (ret)
529 return -EFAULT;
530
4f27b75d 531 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 532 if (ret)
4f27b75d 533 return ret;
673a394b 534
05394f39 535 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 536 if (&obj->base == NULL) {
1d7cfea1
CW		 537 ret = -ENOENT;
538 goto unlock;
4f27b75d 539 }
673a394b 540
7dcd2499 541 /* Bounds check source. */
05394f39
CW		 542 if (args->offset > obj->base.size ||
543 args->size > obj->base.size - args->offset) {
ce9d419d 544 ret = -EINVAL;
35b62a89 545 goto out;
ce9d419d
CW		 546 }
547
db53a302
CW		 548 trace_i915_gem_object_pread(obj, args->offset, args->size);
549
4f27b75d
CW		 550 ret = i915_gem_object_set_cpu_read_domain_range(obj,
551 args->offset,
552 args->size);
553 if (ret)
e5281ccd 554 goto out;
4f27b75d
CW		 555
556 ret = -EFAULT;
557 if (!i915_gem_object_needs_bit17_swizzle(obj))
05394f39 558 ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
4f27b75d 559 if (ret == -EFAULT)
05394f39 560 ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
673a394b 561
35b62a89 562out:
05394f39 563 drm_gem_object_unreference(&obj->base);
1d7cfea1 564unlock:
4f27b75d 565 mutex_unlock(&dev->struct_mutex);
eb01459f 566 return ret;
673a394b
EA		 567}
568
0839ccb8
KP		 569/* This is the fast write path which cannot handle
570 * page faults in the source data
9b7530cc 571 */
0839ccb8
KP		 572
573static inline int
574fast_user_write(struct io_mapping *mapping,
575 loff_t page_base, int page_offset,
576 char __user *user_data,
577 int length)
9b7530cc 578{
9b7530cc 579 char *vaddr_atomic;
0839ccb8 580 unsigned long unwritten;
9b7530cc 581
3e4d3af5 582 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
0839ccb8
KP		 583 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
584 user_data, length);
3e4d3af5 585 io_mapping_unmap_atomic(vaddr_atomic);
fbd5a26d 586 return unwritten;
0839ccb8
KP		 587}
588
589/* Here's the write path which can sleep for
590 * page faults
591 */
592
ab34c226 593static inline void
3de09aa3
EA		 594slow_kernel_write(struct io_mapping *mapping,
595 loff_t gtt_base, int gtt_offset,
596 struct page *user_page, int user_offset,
597 int length)
0839ccb8 598{
ab34c226
CW		 599 char __iomem *dst_vaddr;
600 char *src_vaddr;
0839ccb8 601
ab34c226
CW		 602 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
603 src_vaddr = kmap(user_page);
604
605 memcpy_toio(dst_vaddr + gtt_offset,
606 src_vaddr + user_offset,
607 length);
608
609 kunmap(user_page);
610 io_mapping_unmap(dst_vaddr);
9b7530cc
LT		 611}
612
3de09aa3
EA		 613/**
614 * This is the fast pwrite path, where we copy the data directly from the
615 * user into the GTT, uncached.
616 */
673a394b 617static int
05394f39
CW		 618i915_gem_gtt_pwrite_fast(struct drm_device *dev,
619 struct drm_i915_gem_object *obj,
3de09aa3 620 struct drm_i915_gem_pwrite *args,
05394f39 621 struct drm_file *file)
673a394b 622{
0839ccb8 623 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 624 ssize_t remain;
0839ccb8 625 loff_t offset, page_base;
673a394b 626 char __user *user_data;
0839ccb8 627 int page_offset, page_length;
673a394b
EA		 628
629 user_data = (char __user *) (uintptr_t) args->data_ptr;
630 remain = args->size;
673a394b 631
05394f39 632 offset = obj->gtt_offset + args->offset;
673a394b
EA		 633
634 while (remain > 0) {
635 /* Operation in this page
636 *
0839ccb8
KP		 637 * page_base = page offset within aperture
638 * page_offset = offset within page
639 * page_length = bytes to copy for this page
673a394b 640 */
c8cbbb8b
CW		 641 page_base = offset & PAGE_MASK;
642 page_offset = offset_in_page(offset);
0839ccb8
KP		 643 page_length = remain;
644 if ((page_offset + remain) > PAGE_SIZE)
645 page_length = PAGE_SIZE - page_offset;
646
0839ccb8 647 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 648 * source page isn't available. Return the error and we'll
649 * retry in the slow path.
0839ccb8 650 */
fbd5a26d
CW		 651 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
652 page_offset, user_data, page_length))
fbd5a26d 653 return -EFAULT;
673a394b 654
0839ccb8
KP		 655 remain -= page_length;
656 user_data += page_length;
657 offset += page_length;
673a394b 658 }
673a394b 659
fbd5a26d 660 return 0;
673a394b
EA		 661}
662
3de09aa3
EA		 663/**
664 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
665 * the memory and maps it using kmap_atomic for copying.
666 *
667 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
668 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
669 */
3043c60c 670static int
05394f39
CW		 671i915_gem_gtt_pwrite_slow(struct drm_device *dev,
672 struct drm_i915_gem_object *obj,
3de09aa3 673 struct drm_i915_gem_pwrite *args,
05394f39 674 struct drm_file *file)
673a394b 675{
3de09aa3
EA		 676 drm_i915_private_t *dev_priv = dev->dev_private;
677 ssize_t remain;
678 loff_t gtt_page_base, offset;
679 loff_t first_data_page, last_data_page, num_pages;
680 loff_t pinned_pages, i;
681 struct page **user_pages;
682 struct mm_struct *mm = current->mm;
683 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 684 int ret;
3de09aa3
EA		 685 uint64_t data_ptr = args->data_ptr;
686
687 remain = args->size;
688
689 /* Pin the user pages containing the data. We can't fault while
690 * holding the struct mutex, and all of the pwrite implementations
691 * want to hold it while dereferencing the user data.
692 */
693 first_data_page = data_ptr / PAGE_SIZE;
694 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
695 num_pages = last_data_page - first_data_page + 1;
696
fbd5a26d 697 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
3de09aa3
EA		 698 if (user_pages == NULL)
699 return -ENOMEM;
700
fbd5a26d 701 mutex_unlock(&dev->struct_mutex);
3de09aa3
EA		 702 down_read(&mm->mmap_sem);
703 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
704 num_pages, 0, 0, user_pages, NULL);
705 up_read(&mm->mmap_sem);
fbd5a26d 706 mutex_lock(&dev->struct_mutex);
3de09aa3
EA		 707 if (pinned_pages < num_pages) {
708 ret = -EFAULT;
709 goto out_unpin_pages;
710 }
673a394b 711
d9e86c0e
CW		 712 ret = i915_gem_object_set_to_gtt_domain(obj, true);
713 if (ret)
714 goto out_unpin_pages;
715
716 ret = i915_gem_object_put_fence(obj);
3de09aa3 717 if (ret)
fbd5a26d 718 goto out_unpin_pages;
3de09aa3 719
05394f39 720 offset = obj->gtt_offset + args->offset;
3de09aa3
EA		 721
722 while (remain > 0) {
723 /* Operation in this page
724 *
725 * gtt_page_base = page offset within aperture
726 * gtt_page_offset = offset within page in aperture
727 * data_page_index = page number in get_user_pages return
728 * data_page_offset = offset with data_page_index page.
729 * page_length = bytes to copy for this page
730 */
731 gtt_page_base = offset & PAGE_MASK;
c8cbbb8b 732 gtt_page_offset = offset_in_page(offset);
3de09aa3 733 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
c8cbbb8b 734 data_page_offset = offset_in_page(data_ptr);
3de09aa3
EA		 735
736 page_length = remain;
737 if ((gtt_page_offset + page_length) > PAGE_SIZE)
738 page_length = PAGE_SIZE - gtt_page_offset;
739 if ((data_page_offset + page_length) > PAGE_SIZE)
740 page_length = PAGE_SIZE - data_page_offset;
741
ab34c226
CW		 742 slow_kernel_write(dev_priv->mm.gtt_mapping,
743 gtt_page_base, gtt_page_offset,
744 user_pages[data_page_index],
745 data_page_offset,
746 page_length);
3de09aa3
EA		 747
748 remain -= page_length;
749 offset += page_length;
750 data_ptr += page_length;
751 }
752
3de09aa3
EA		 753out_unpin_pages:
754 for (i = 0; i < pinned_pages; i++)
755 page_cache_release(user_pages[i]);
8e7d2b2c 756 drm_free_large(user_pages);
3de09aa3
EA		 757
758 return ret;
759}
760
40123c1f
EA		 761/**
762 * This is the fast shmem pwrite path, which attempts to directly
763 * copy_from_user into the kmapped pages backing the object.
764 */
3043c60c 765static int
05394f39
CW		 766i915_gem_shmem_pwrite_fast(struct drm_device *dev,
767 struct drm_i915_gem_object *obj,
40123c1f 768 struct drm_i915_gem_pwrite *args,
05394f39 769 struct drm_file *file)
673a394b 770{
05394f39 771 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
40123c1f 772 ssize_t remain;
e5281ccd 773 loff_t offset;
40123c1f
EA		 774 char __user *user_data;
775 int page_offset, page_length;
40123c1f
EA		 776
777 user_data = (char __user *) (uintptr_t) args->data_ptr;
778 remain = args->size;
673a394b 779
40123c1f 780 offset = args->offset;
05394f39 781 obj->dirty = 1;
40123c1f
EA		 782
783 while (remain > 0) {
e5281ccd
CW		 784 struct page *page;
785 char *vaddr;
786 int ret;
787
40123c1f
EA		 788 /* Operation in this page
789 *
40123c1f
EA		 790 * page_offset = offset within page
791 * page_length = bytes to copy for this page
792 */
c8cbbb8b 793 page_offset = offset_in_page(offset);
40123c1f
EA		 794 page_length = remain;
795 if ((page_offset + remain) > PAGE_SIZE)
796 page_length = PAGE_SIZE - page_offset;
797
e5281ccd
CW		 798 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
799 GFP_HIGHUSER | __GFP_RECLAIMABLE);
800 if (IS_ERR(page))
801 return PTR_ERR(page);
802
803 vaddr = kmap_atomic(page, KM_USER0);
804 ret = __copy_from_user_inatomic(vaddr + page_offset,
805 user_data,
806 page_length);
807 kunmap_atomic(vaddr, KM_USER0);
808
809 set_page_dirty(page);
810 mark_page_accessed(page);
811 page_cache_release(page);
812
813 /* If we get a fault while copying data, then (presumably) our
814 * source page isn't available. Return the error and we'll
815 * retry in the slow path.
816 */
817 if (ret)
fbd5a26d 818 return -EFAULT;
40123c1f
EA		 819
820 remain -= page_length;
821 user_data += page_length;
822 offset += page_length;
823 }
824
fbd5a26d 825 return 0;
40123c1f
EA		 826}
827
828/**
829 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
830 * the memory and maps it using kmap_atomic for copying.
831 *
832 * This avoids taking mmap_sem for faulting on the user's address while the
833 * struct_mutex is held.
834 */
835static int
05394f39
CW		 836i915_gem_shmem_pwrite_slow(struct drm_device *dev,
837 struct drm_i915_gem_object *obj,
40123c1f 838 struct drm_i915_gem_pwrite *args,
05394f39 839 struct drm_file *file)
40123c1f 840{
05394f39 841 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
40123c1f
EA		 842 struct mm_struct *mm = current->mm;
843 struct page **user_pages;
844 ssize_t remain;
845 loff_t offset, pinned_pages, i;
846 loff_t first_data_page, last_data_page, num_pages;
e5281ccd 847 int shmem_page_offset;
40123c1f
EA		 848 int data_page_index, data_page_offset;
849 int page_length;
850 int ret;
851 uint64_t data_ptr = args->data_ptr;
280b713b 852 int do_bit17_swizzling;
40123c1f
EA		 853
854 remain = args->size;
855
856 /* Pin the user pages containing the data. We can't fault while
857 * holding the struct mutex, and all of the pwrite implementations
858 * want to hold it while dereferencing the user data.
859 */
860 first_data_page = data_ptr / PAGE_SIZE;
861 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
862 num_pages = last_data_page - first_data_page + 1;
863
4f27b75d 864 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
40123c1f
EA		 865 if (user_pages == NULL)
866 return -ENOMEM;
867
fbd5a26d 868 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 869 down_read(&mm->mmap_sem);
870 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
871 num_pages, 0, 0, user_pages, NULL);
872 up_read(&mm->mmap_sem);
fbd5a26d 873 mutex_lock(&dev->struct_mutex);
40123c1f
EA		 874 if (pinned_pages < num_pages) {
875 ret = -EFAULT;
fbd5a26d 876 goto out;
673a394b
EA		 877 }
878
fbd5a26d 879 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
07f73f69 880 if (ret)
fbd5a26d 881 goto out;
40123c1f 882
fbd5a26d 883 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
40123c1f 884
673a394b 885 offset = args->offset;
05394f39 886 obj->dirty = 1;
673a394b 887
40123c1f 888 while (remain > 0) {
e5281ccd
CW		 889 struct page *page;
890
40123c1f
EA		 891 /* Operation in this page
892 *
40123c1f
EA		 893 * shmem_page_offset = offset within page in shmem file
894 * data_page_index = page number in get_user_pages return
895 * data_page_offset = offset with data_page_index page.
896 * page_length = bytes to copy for this page
897 */
c8cbbb8b 898 shmem_page_offset = offset_in_page(offset);
40123c1f 899 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
c8cbbb8b 900 data_page_offset = offset_in_page(data_ptr);
40123c1f
EA		 901
902 page_length = remain;
903 if ((shmem_page_offset + page_length) > PAGE_SIZE)
904 page_length = PAGE_SIZE - shmem_page_offset;
905 if ((data_page_offset + page_length) > PAGE_SIZE)
906 page_length = PAGE_SIZE - data_page_offset;
907
e5281ccd
CW		 908 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
909 GFP_HIGHUSER | __GFP_RECLAIMABLE);
910 if (IS_ERR(page)) {
911 ret = PTR_ERR(page);
912 goto out;
913 }
914
280b713b 915 if (do_bit17_swizzling) {
e5281ccd 916 slow_shmem_bit17_copy(page,
280b713b
EA		 917 shmem_page_offset,
918 user_pages[data_page_index],
919 data_page_offset,
99a03df5
CW		 920 page_length,
921 0);
922 } else {
e5281ccd 923 slow_shmem_copy(page,
99a03df5
CW		 924 shmem_page_offset,
925 user_pages[data_page_index],
926 data_page_offset,
927 page_length);
280b713b 928 }
40123c1f 929
e5281ccd
CW		 930 set_page_dirty(page);
931 mark_page_accessed(page);
932 page_cache_release(page);
933
40123c1f
EA		 934 remain -= page_length;
935 data_ptr += page_length;
936 offset += page_length;
673a394b
EA		 937 }
938
fbd5a26d 939out:
40123c1f
EA		 940 for (i = 0; i < pinned_pages; i++)
941 page_cache_release(user_pages[i]);
8e7d2b2c 942 drm_free_large(user_pages);
673a394b 943
40123c1f 944 return ret;
673a394b
EA		 945}
946
947/**
948 * Writes data to the object referenced by handle.
949 *
950 * On error, the contents of the buffer that were to be modified are undefined.
951 */
952int
953i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
fbd5a26d 954 struct drm_file *file)
673a394b
EA		 955{
956 struct drm_i915_gem_pwrite *args = data;
05394f39 957 struct drm_i915_gem_object *obj;
51311d0a
CW		 958 int ret;
959
960 if (args->size == 0)
961 return 0;
962
963 if (!access_ok(VERIFY_READ,
964 (char __user *)(uintptr_t)args->data_ptr,
965 args->size))
966 return -EFAULT;
967
968 ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
969 args->size);
970 if (ret)
971 return -EFAULT;
673a394b 972
fbd5a26d 973 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 974 if (ret)
fbd5a26d 975 return ret;
1d7cfea1 976
05394f39 977 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 978 if (&obj->base == NULL) {
1d7cfea1
CW		 979 ret = -ENOENT;
980 goto unlock;
fbd5a26d 981 }
673a394b 982
7dcd2499 983 /* Bounds check destination. */
05394f39
CW		 984 if (args->offset > obj->base.size ||
985 args->size > obj->base.size - args->offset) {
ce9d419d 986 ret = -EINVAL;
35b62a89 987 goto out;
ce9d419d
CW		 988 }
989
db53a302
CW		 990 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
991
673a394b
EA		 992 /* We can only do the GTT pwrite on untiled buffers, as otherwise
993 * it would end up going through the fenced access, and we'll get
994 * different detiling behavior between reading and writing.
995 * pread/pwrite currently are reading and writing from the CPU
996 * perspective, requiring manual detiling by the client.
997 */
05394f39 998 if (obj->phys_obj)
fbd5a26d 999 ret = i915_gem_phys_pwrite(dev, obj, args, file);
d9e86c0e 1000 else if (obj->gtt_space &&
05394f39 1001 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
75e9e915 1002 ret = i915_gem_object_pin(obj, 0, true);
fbd5a26d
CW		 1003 if (ret)
1004 goto out;
1005
d9e86c0e
CW		 1006 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1007 if (ret)
1008 goto out_unpin;
1009
1010 ret = i915_gem_object_put_fence(obj);
fbd5a26d
CW		 1011 if (ret)
1012 goto out_unpin;
1013
1014 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1015 if (ret == -EFAULT)
1016 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1017
1018out_unpin:
1019 i915_gem_object_unpin(obj);
40123c1f 1020 } else {
fbd5a26d
CW		 1021 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1022 if (ret)
e5281ccd 1023 goto out;
673a394b 1024
fbd5a26d
CW		 1025 ret = -EFAULT;
1026 if (!i915_gem_object_needs_bit17_swizzle(obj))
1027 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1028 if (ret == -EFAULT)
1029 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
fbd5a26d 1030 }
673a394b 1031
35b62a89 1032out:
05394f39 1033 drm_gem_object_unreference(&obj->base);
1d7cfea1 1034unlock:
fbd5a26d 1035 mutex_unlock(&dev->struct_mutex);
673a394b
EA		 1036 return ret;
1037}
1038
1039/**
2ef7eeaa
EA		 1040 * Called when user space prepares to use an object with the CPU, either
1041 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 1042 */
1043int
1044i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
05394f39 1045 struct drm_file *file)
673a394b
EA		 1046{
1047 struct drm_i915_gem_set_domain *args = data;
05394f39 1048 struct drm_i915_gem_object *obj;
2ef7eeaa
EA		 1049 uint32_t read_domains = args->read_domains;
1050 uint32_t write_domain = args->write_domain;
673a394b
EA		 1051 int ret;
1052
1053 if (!(dev->driver->driver_features & DRIVER_GEM))
1054 return -ENODEV;
1055
2ef7eeaa 1056 /* Only handle setting domains to types used by the CPU. */
21d509e3 1057 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1058 return -EINVAL;
1059
21d509e3 1060 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1061 return -EINVAL;
1062
1063 /* Having something in the write domain implies it's in the read
1064 * domain, and only that read domain. Enforce that in the request.
1065 */
1066 if (write_domain != 0 && read_domains != write_domain)
1067 return -EINVAL;
1068
76c1dec1 1069 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1070 if (ret)
76c1dec1 1071 return ret;
1d7cfea1 1072
05394f39 1073 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 1074 if (&obj->base == NULL) {
1d7cfea1
CW		 1075 ret = -ENOENT;
1076 goto unlock;
76c1dec1 1077 }
673a394b 1078
2ef7eeaa
EA		 1079 if (read_domains & I915_GEM_DOMAIN_GTT) {
1080 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392
EA		 1081
1082 /* Silently promote "you're not bound, there was nothing to do"
1083 * to success, since the client was just asking us to
1084 * make sure everything was done.
1085 */
1086 if (ret == -EINVAL)
1087 ret = 0;
2ef7eeaa 1088 } else {
e47c68e9 1089 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1090 }
1091
05394f39 1092 drm_gem_object_unreference(&obj->base);
1d7cfea1 1093unlock:
673a394b
EA		 1094 mutex_unlock(&dev->struct_mutex);
1095 return ret;
1096}
1097
1098/**
1099 * Called when user space has done writes to this buffer
1100 */
1101int
1102i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
05394f39 1103 struct drm_file *file)
673a394b
EA		 1104{
1105 struct drm_i915_gem_sw_finish *args = data;
05394f39 1106 struct drm_i915_gem_object *obj;
673a394b
EA		 1107 int ret = 0;
1108
1109 if (!(dev->driver->driver_features & DRIVER_GEM))
1110 return -ENODEV;
1111
76c1dec1 1112 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1113 if (ret)
76c1dec1 1114 return ret;
1d7cfea1 1115
05394f39 1116 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 1117 if (&obj->base == NULL) {
1d7cfea1
CW		 1118 ret = -ENOENT;
1119 goto unlock;
673a394b
EA		 1120 }
1121
673a394b 1122 /* Pinned buffers may be scanout, so flush the cache */
05394f39 1123 if (obj->pin_count)
e47c68e9
EA		 1124 i915_gem_object_flush_cpu_write_domain(obj);
1125
05394f39 1126 drm_gem_object_unreference(&obj->base);
1d7cfea1 1127unlock:
673a394b
EA		 1128 mutex_unlock(&dev->struct_mutex);
1129 return ret;
1130}
1131
1132/**
1133 * Maps the contents of an object, returning the address it is mapped
1134 * into.
1135 *
1136 * While the mapping holds a reference on the contents of the object, it doesn't
1137 * imply a ref on the object itself.
1138 */
1139int
1140i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
05394f39 1141 struct drm_file *file)
673a394b 1142{
da761a6e 1143 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 1144 struct drm_i915_gem_mmap *args = data;
1145 struct drm_gem_object *obj;
673a394b
EA		 1146 unsigned long addr;
1147
1148 if (!(dev->driver->driver_features & DRIVER_GEM))
1149 return -ENODEV;
1150
05394f39 1151 obj = drm_gem_object_lookup(dev, file, args->handle);
673a394b 1152 if (obj == NULL)
bf79cb91 1153 return -ENOENT;
673a394b 1154
da761a6e
CW		 1155 if (obj->size > dev_priv->mm.gtt_mappable_end) {
1156 drm_gem_object_unreference_unlocked(obj);
1157 return -E2BIG;
1158 }
1159
673a394b
EA		 1160 down_write(&current->mm->mmap_sem);
1161 addr = do_mmap(obj->filp, 0, args->size,
1162 PROT_READ | PROT_WRITE, MAP_SHARED,
1163 args->offset);
1164 up_write(&current->mm->mmap_sem);
bc9025bd 1165 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1166 if (IS_ERR((void *)addr))
1167 return addr;
1168
1169 args->addr_ptr = (uint64_t) addr;
1170
1171 return 0;
1172}
1173
de151cf6
JB		 1174/**
1175 * i915_gem_fault - fault a page into the GTT
1176 * vma: VMA in question
1177 * vmf: fault info
1178 *
1179 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1180 * from userspace. The fault handler takes care of binding the object to
1181 * the GTT (if needed), allocating and programming a fence register (again,
1182 * only if needed based on whether the old reg is still valid or the object
1183 * is tiled) and inserting a new PTE into the faulting process.
1184 *
1185 * Note that the faulting process may involve evicting existing objects
1186 * from the GTT and/or fence registers to make room. So performance may
1187 * suffer if the GTT working set is large or there are few fence registers
1188 * left.
1189 */
1190int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1191{
05394f39
CW		 1192 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1193 struct drm_device *dev = obj->base.dev;
7d1c4804 1194 drm_i915_private_t *dev_priv = dev->dev_private;
de151cf6
JB		 1195 pgoff_t page_offset;
1196 unsigned long pfn;
1197 int ret = 0;
0f973f27 1198 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1199
1200 /* We don't use vmf->pgoff since that has the fake offset */
1201 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1202 PAGE_SHIFT;
1203
d9bc7e9f
CW		 1204 ret = i915_mutex_lock_interruptible(dev);
1205 if (ret)
1206 goto out;
a00b10c3 1207
db53a302
CW		 1208 trace_i915_gem_object_fault(obj, page_offset, true, write);
1209
d9bc7e9f 1210 /* Now bind it into the GTT if needed */
919926ae
CW		 1211 if (!obj->map_and_fenceable) {
1212 ret = i915_gem_object_unbind(obj);
1213 if (ret)
1214 goto unlock;
a00b10c3 1215 }
05394f39 1216 if (!obj->gtt_space) {
75e9e915 1217 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
c715089f
CW		 1218 if (ret)
1219 goto unlock;
de151cf6
JB		 1220 }
1221
4a684a41
CW		 1222 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1223 if (ret)
1224 goto unlock;
1225
d9e86c0e
CW		 1226 if (obj->tiling_mode == I915_TILING_NONE)
1227 ret = i915_gem_object_put_fence(obj);
1228 else
ce453d81 1229 ret = i915_gem_object_get_fence(obj, NULL);
d9e86c0e
CW		 1230 if (ret)
1231 goto unlock;
de151cf6 1232
05394f39
CW		 1233 if (i915_gem_object_is_inactive(obj))
1234 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
7d1c4804 1235
6299f992
CW		 1236 obj->fault_mappable = true;
1237
05394f39 1238 pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
de151cf6
JB		 1239 page_offset;
1240
1241 /* Finally, remap it using the new GTT offset */
1242 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1243unlock:
de151cf6 1244 mutex_unlock(&dev->struct_mutex);
d9bc7e9f 1245out:
de151cf6 1246 switch (ret) {
d9bc7e9f 1247 case -EIO:
045e769a 1248 case -EAGAIN:
d9bc7e9f
CW		 1249 /* Give the error handler a chance to run and move the
1250 * objects off the GPU active list. Next time we service the
1251 * fault, we should be able to transition the page into the
1252 * GTT without touching the GPU (and so avoid further
1253 * EIO/EGAIN). If the GPU is wedged, then there is no issue
1254 * with coherency, just lost writes.
1255 */
045e769a 1256 set_need_resched();
c715089f
CW		 1257 case 0:
1258 case -ERESTARTSYS:
bed636ab 1259 case -EINTR:
c715089f 1260 return VM_FAULT_NOPAGE;
de151cf6 1261 case -ENOMEM:
de151cf6 1262 return VM_FAULT_OOM;
de151cf6 1263 default:
c715089f 1264 return VM_FAULT_SIGBUS;
de151cf6
JB		 1265 }
1266}
1267
1268/**
1269 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1270 * @obj: obj in question
1271 *
1272 * GEM memory mapping works by handing back to userspace a fake mmap offset
1273 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1274 * up the object based on the offset and sets up the various memory mapping
1275 * structures.
1276 *
1277 * This routine allocates and attaches a fake offset for @obj.
1278 */
1279static int
05394f39 1280i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
de151cf6 1281{
05394f39 1282 struct drm_device *dev = obj->base.dev;
de151cf6 1283 struct drm_gem_mm *mm = dev->mm_private;
de151cf6 1284 struct drm_map_list *list;
f77d390c 1285 struct drm_local_map *map;
de151cf6
JB		 1286 int ret = 0;
1287
1288 /* Set the object up for mmap'ing */
05394f39 1289 list = &obj->base.map_list;
9a298b2a 1290 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
de151cf6
JB		 1291 if (!list->map)
1292 return -ENOMEM;
1293
1294 map = list->map;
1295 map->type = _DRM_GEM;
05394f39 1296 map->size = obj->base.size;
de151cf6
JB		 1297 map->handle = obj;
1298
1299 /* Get a DRM GEM mmap offset allocated... */
1300 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
05394f39
CW		 1301 obj->base.size / PAGE_SIZE,
1302 0, 0);
de151cf6 1303 if (!list->file_offset_node) {
05394f39
CW		 1304 DRM_ERROR("failed to allocate offset for bo %d\n",
1305 obj->base.name);
9e0ae534 1306 ret = -ENOSPC;
de151cf6
JB		 1307 goto out_free_list;
1308 }
1309
1310 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
05394f39
CW		 1311 obj->base.size / PAGE_SIZE,
1312 0);
de151cf6
JB		 1313 if (!list->file_offset_node) {
1314 ret = -ENOMEM;
1315 goto out_free_list;
1316 }
1317
1318 list->hash.key = list->file_offset_node->start;
9e0ae534
CW		 1319 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1320 if (ret) {
de151cf6
JB		 1321 DRM_ERROR("failed to add to map hash\n");
1322 goto out_free_mm;
1323 }
1324
de151cf6
JB		 1325 return 0;
1326
1327out_free_mm:
1328 drm_mm_put_block(list->file_offset_node);
1329out_free_list:
9a298b2a 1330 kfree(list->map);
39a01d1f 1331 list->map = NULL;
de151cf6
JB		 1332
1333 return ret;
1334}
1335
901782b2
CW		 1336/**
1337 * i915_gem_release_mmap - remove physical page mappings
1338 * @obj: obj in question
1339 *
af901ca1 1340 * Preserve the reservation of the mmapping with the DRM core code, but
901782b2
CW		 1341 * relinquish ownership of the pages back to the system.
1342 *
1343 * It is vital that we remove the page mapping if we have mapped a tiled
1344 * object through the GTT and then lose the fence register due to
1345 * resource pressure. Similarly if the object has been moved out of the
1346 * aperture, than pages mapped into userspace must be revoked. Removing the
1347 * mapping will then trigger a page fault on the next user access, allowing
1348 * fixup by i915_gem_fault().
1349 */
d05ca301 1350void
05394f39 1351i915_gem_release_mmap(struct drm_i915_gem_object *obj)
901782b2 1352{
6299f992
CW		 1353 if (!obj->fault_mappable)
1354 return;
901782b2 1355
f6e47884
CW		 1356 if (obj->base.dev->dev_mapping)
1357 unmap_mapping_range(obj->base.dev->dev_mapping,
1358 (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1359 obj->base.size, 1);
fb7d516a 1360
6299f992 1361 obj->fault_mappable = false;
901782b2
CW		 1362}
1363
ab00b3e5 1364static void
05394f39 1365i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
ab00b3e5 1366{
05394f39 1367 struct drm_device *dev = obj->base.dev;
ab00b3e5 1368 struct drm_gem_mm *mm = dev->mm_private;
05394f39 1369 struct drm_map_list *list = &obj->base.map_list;
ab00b3e5 1370
ab00b3e5 1371 drm_ht_remove_item(&mm->offset_hash, &list->hash);
39a01d1f
CW		 1372 drm_mm_put_block(list->file_offset_node);
1373 kfree(list->map);
1374 list->map = NULL;
ab00b3e5
JB		 1375}
1376
92b88aeb
CW		 1377static uint32_t
1378i915_gem_get_gtt_size(struct drm_i915_gem_object *obj)
1379{
1380 struct drm_device *dev = obj->base.dev;
1381 uint32_t size;
1382
1383 if (INTEL_INFO(dev)->gen >= 4 ||
1384 obj->tiling_mode == I915_TILING_NONE)
1385 return obj->base.size;
1386
1387 /* Previous chips need a power-of-two fence region when tiling */
1388 if (INTEL_INFO(dev)->gen == 3)
1389 size = 1024*1024;
1390 else
1391 size = 512*1024;
1392
1393 while (size < obj->base.size)
1394 size <<= 1;
1395
1396 return size;
1397}
1398
de151cf6
JB		 1399/**
1400 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1401 * @obj: object to check
1402 *
1403 * Return the required GTT alignment for an object, taking into account
5e783301 1404 * potential fence register mapping.
de151cf6
JB		 1405 */
1406static uint32_t
05394f39 1407i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj)
de151cf6 1408{
05394f39 1409 struct drm_device *dev = obj->base.dev;
de151cf6
JB		 1410
1411 /*
1412 * Minimum alignment is 4k (GTT page size), but might be greater
1413 * if a fence register is needed for the object.
1414 */
a00b10c3 1415 if (INTEL_INFO(dev)->gen >= 4 ||
05394f39 1416 obj->tiling_mode == I915_TILING_NONE)
de151cf6
JB		 1417 return 4096;
1418
a00b10c3
CW		 1419 /*
1420 * Previous chips need to be aligned to the size of the smallest
1421 * fence register that can contain the object.
1422 */
05394f39 1423 return i915_gem_get_gtt_size(obj);
a00b10c3
CW		 1424}
1425
5e783301
DV		 1426/**
1427 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1428 * unfenced object
1429 * @obj: object to check
1430 *
1431 * Return the required GTT alignment for an object, only taking into account
1432 * unfenced tiled surface requirements.
1433 */
467cffba 1434uint32_t
05394f39 1435i915_gem_get_unfenced_gtt_alignment(struct drm_i915_gem_object *obj)
5e783301 1436{
05394f39 1437 struct drm_device *dev = obj->base.dev;
5e783301
DV		 1438 int tile_height;
1439
1440 /*
1441 * Minimum alignment is 4k (GTT page size) for sane hw.
1442 */
1443 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
05394f39 1444 obj->tiling_mode == I915_TILING_NONE)
5e783301
DV		 1445 return 4096;
1446
1447 /*
1448 * Older chips need unfenced tiled buffers to be aligned to the left
1449 * edge of an even tile row (where tile rows are counted as if the bo is
1450 * placed in a fenced gtt region).
1451 */
c8ebc2b0
DV		 1452 if (IS_GEN2(dev))
1453 tile_height = 16;
1454 else if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
5e783301
DV		 1455 tile_height = 32;
1456 else
1457 tile_height = 8;
1458
05394f39 1459 return tile_height * obj->stride * 2;
5e783301
DV		 1460}
1461
de151cf6 1462int
ff72145b
DA		 1463i915_gem_mmap_gtt(struct drm_file *file,
1464 struct drm_device *dev,
1465 uint32_t handle,
1466 uint64_t *offset)
de151cf6 1467{
da761a6e 1468 struct drm_i915_private *dev_priv = dev->dev_private;
05394f39 1469 struct drm_i915_gem_object *obj;
de151cf6
JB		 1470 int ret;
1471
1472 if (!(dev->driver->driver_features & DRIVER_GEM))
1473 return -ENODEV;
1474
76c1dec1 1475 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1476 if (ret)
76c1dec1 1477 return ret;
de151cf6 1478
ff72145b 1479 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
c8725226 1480 if (&obj->base == NULL) {
1d7cfea1
CW		 1481 ret = -ENOENT;
1482 goto unlock;
1483 }
de151cf6 1484
05394f39 1485 if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
da761a6e
CW		 1486 ret = -E2BIG;
1487 goto unlock;
1488 }
1489
05394f39 1490 if (obj->madv != I915_MADV_WILLNEED) {
ab18282d 1491 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1d7cfea1
CW		 1492 ret = -EINVAL;
1493 goto out;
ab18282d
CW		 1494 }
1495
05394f39 1496 if (!obj->base.map_list.map) {
de151cf6 1497 ret = i915_gem_create_mmap_offset(obj);
1d7cfea1
CW		 1498 if (ret)
1499 goto out;
de151cf6
JB		 1500 }
1501
ff72145b 1502 *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
de151cf6 1503
1d7cfea1 1504out:
05394f39 1505 drm_gem_object_unreference(&obj->base);
1d7cfea1 1506unlock:
de151cf6 1507 mutex_unlock(&dev->struct_mutex);
1d7cfea1 1508 return ret;
de151cf6
JB		 1509}
1510
ff72145b
DA		 1511/**
1512 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1513 * @dev: DRM device
1514 * @data: GTT mapping ioctl data
1515 * @file: GEM object info
1516 *
1517 * Simply returns the fake offset to userspace so it can mmap it.
1518 * The mmap call will end up in drm_gem_mmap(), which will set things
1519 * up so we can get faults in the handler above.
1520 *
1521 * The fault handler will take care of binding the object into the GTT
1522 * (since it may have been evicted to make room for something), allocating
1523 * a fence register, and mapping the appropriate aperture address into
1524 * userspace.
1525 */
1526int
1527i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1528 struct drm_file *file)
1529{
1530 struct drm_i915_gem_mmap_gtt *args = data;
1531
1532 if (!(dev->driver->driver_features & DRIVER_GEM))
1533 return -ENODEV;
1534
1535 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1536}
1537
1538
e5281ccd 1539static int
05394f39 1540i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
e5281ccd
CW		 1541 gfp_t gfpmask)
1542{
e5281ccd
CW		 1543 int page_count, i;
1544 struct address_space *mapping;
1545 struct inode *inode;
1546 struct page *page;
1547
1548 /* Get the list of pages out of our struct file. They'll be pinned
1549 * at this point until we release them.
1550 */
05394f39
CW		 1551 page_count = obj->base.size / PAGE_SIZE;
1552 BUG_ON(obj->pages != NULL);
1553 obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1554 if (obj->pages == NULL)
e5281ccd
CW		 1555 return -ENOMEM;
1556
05394f39 1557 inode = obj->base.filp->f_path.dentry->d_inode;
e5281ccd
CW		 1558 mapping = inode->i_mapping;
1559 for (i = 0; i < page_count; i++) {
1560 page = read_cache_page_gfp(mapping, i,
1561 GFP_HIGHUSER |
1562 __GFP_COLD |
1563 __GFP_RECLAIMABLE |
1564 gfpmask);
1565 if (IS_ERR(page))
1566 goto err_pages;
1567
05394f39 1568 obj->pages[i] = page;
e5281ccd
CW		 1569 }
1570
05394f39 1571 if (obj->tiling_mode != I915_TILING_NONE)
e5281ccd
CW		 1572 i915_gem_object_do_bit_17_swizzle(obj);
1573
1574 return 0;
1575
1576err_pages:
1577 while (i--)
05394f39 1578 page_cache_release(obj->pages[i]);
e5281ccd 1579
05394f39
CW		 1580 drm_free_large(obj->pages);
1581 obj->pages = NULL;
e5281ccd
CW		 1582 return PTR_ERR(page);
1583}
1584
5cdf5881 1585static void
05394f39 1586i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
673a394b 1587{
05394f39 1588 int page_count = obj->base.size / PAGE_SIZE;
673a394b
EA		 1589 int i;
1590
05394f39 1591 BUG_ON(obj->madv == __I915_MADV_PURGED);
673a394b 1592
05394f39 1593 if (obj->tiling_mode != I915_TILING_NONE)
280b713b
EA		 1594 i915_gem_object_save_bit_17_swizzle(obj);
1595
05394f39
CW		 1596 if (obj->madv == I915_MADV_DONTNEED)
1597 obj->dirty = 0;
3ef94daa
CW		 1598
1599 for (i = 0; i < page_count; i++) {
05394f39
CW		 1600 if (obj->dirty)
1601 set_page_dirty(obj->pages[i]);
3ef94daa 1602
05394f39
CW		 1603 if (obj->madv == I915_MADV_WILLNEED)
1604 mark_page_accessed(obj->pages[i]);
3ef94daa 1605
05394f39 1606 page_cache_release(obj->pages[i]);
3ef94daa 1607 }
05394f39 1608 obj->dirty = 0;
673a394b 1609
05394f39
CW		 1610 drm_free_large(obj->pages);
1611 obj->pages = NULL;
673a394b
EA		 1612}
1613
54cf91dc 1614void
05394f39 1615i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1ec14ad3
CW		 1616 struct intel_ring_buffer *ring,
1617 u32 seqno)
673a394b 1618{
05394f39 1619 struct drm_device *dev = obj->base.dev;
69dc4987 1620 struct drm_i915_private *dev_priv = dev->dev_private;
617dbe27 1621
852835f3 1622 BUG_ON(ring == NULL);
05394f39 1623 obj->ring = ring;
673a394b
EA		 1624
1625 /* Add a reference if we're newly entering the active list. */
05394f39
CW		 1626 if (!obj->active) {
1627 drm_gem_object_reference(&obj->base);
1628 obj->active = 1;
673a394b 1629 }
e35a41de 1630
673a394b 1631 /* Move from whatever list we were on to the tail of execution. */
05394f39
CW		 1632 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1633 list_move_tail(&obj->ring_list, &ring->active_list);
caea7476 1634
05394f39 1635 obj->last_rendering_seqno = seqno;
caea7476
CW		 1636 if (obj->fenced_gpu_access) {
1637 struct drm_i915_fence_reg *reg;
1638
1639 BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
1640
1641 obj->last_fenced_seqno = seqno;
1642 obj->last_fenced_ring = ring;
1643
1644 reg = &dev_priv->fence_regs[obj->fence_reg];
1645 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1646 }
1647}
1648
1649static void
1650i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1651{
1652 list_del_init(&obj->ring_list);
1653 obj->last_rendering_seqno = 0;
673a394b
EA		 1654}
1655
ce44b0ea 1656static void
05394f39 1657i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
ce44b0ea 1658{
05394f39 1659 struct drm_device *dev = obj->base.dev;
ce44b0ea 1660 drm_i915_private_t *dev_priv = dev->dev_private;
ce44b0ea 1661
05394f39
CW		 1662 BUG_ON(!obj->active);
1663 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
caea7476
CW		 1664
1665 i915_gem_object_move_off_active(obj);
1666}
1667
1668static void
1669i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1670{
1671 struct drm_device *dev = obj->base.dev;
1672 struct drm_i915_private *dev_priv = dev->dev_private;
1673
1674 if (obj->pin_count != 0)
1675 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1676 else
1677 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1678
1679 BUG_ON(!list_empty(&obj->gpu_write_list));
1680 BUG_ON(!obj->active);
1681 obj->ring = NULL;
1682
1683 i915_gem_object_move_off_active(obj);
1684 obj->fenced_gpu_access = false;
caea7476
CW		 1685
1686 obj->active = 0;
87ca9c8a 1687 obj->pending_gpu_write = false;
caea7476
CW		 1688 drm_gem_object_unreference(&obj->base);
1689
1690 WARN_ON(i915_verify_lists(dev));
ce44b0ea 1691}
673a394b 1692
963b4836
CW		 1693/* Immediately discard the backing storage */
1694static void
05394f39 1695i915_gem_object_truncate(struct drm_i915_gem_object *obj)
963b4836 1696{
bb6baf76 1697 struct inode *inode;
963b4836 1698
ae9fed6b
CW		 1699 /* Our goal here is to return as much of the memory as
1700 * is possible back to the system as we are called from OOM.
1701 * To do this we must instruct the shmfs to drop all of its
1702 * backing pages, *now*. Here we mirror the actions taken
1703 * when by shmem_delete_inode() to release the backing store.
1704 */
05394f39 1705 inode = obj->base.filp->f_path.dentry->d_inode;
ae9fed6b
CW		 1706 truncate_inode_pages(inode->i_mapping, 0);
1707 if (inode->i_op->truncate_range)
1708 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
bb6baf76 1709
05394f39 1710 obj->madv = __I915_MADV_PURGED;
963b4836
CW		 1711}
1712
1713static inline int
05394f39 1714i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
963b4836 1715{
05394f39 1716 return obj->madv == I915_MADV_DONTNEED;
963b4836
CW		 1717}
1718
63560396 1719static void
db53a302
CW		 1720i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
1721 uint32_t flush_domains)
63560396 1722{
05394f39 1723 struct drm_i915_gem_object *obj, *next;
63560396 1724
05394f39 1725 list_for_each_entry_safe(obj, next,
64193406 1726 &ring->gpu_write_list,
63560396 1727 gpu_write_list) {
05394f39
CW		 1728 if (obj->base.write_domain & flush_domains) {
1729 uint32_t old_write_domain = obj->base.write_domain;
63560396 1730
05394f39
CW		 1731 obj->base.write_domain = 0;
1732 list_del_init(&obj->gpu_write_list);
1ec14ad3 1733 i915_gem_object_move_to_active(obj, ring,
db53a302 1734 i915_gem_next_request_seqno(ring));
63560396 1735
63560396 1736 trace_i915_gem_object_change_domain(obj,
05394f39 1737 obj->base.read_domains,
63560396
DV		 1738 old_write_domain);
1739 }
1740 }
1741}
8187a2b7 1742
3cce469c 1743int
db53a302 1744i915_add_request(struct intel_ring_buffer *ring,
f787a5f5 1745 struct drm_file *file,
db53a302 1746 struct drm_i915_gem_request *request)
673a394b 1747{
db53a302 1748 drm_i915_private_t *dev_priv = ring->dev->dev_private;
673a394b
EA		 1749 uint32_t seqno;
1750 int was_empty;
3cce469c
CW		 1751 int ret;
1752
1753 BUG_ON(request == NULL);
673a394b 1754
3cce469c
CW		 1755 ret = ring->add_request(ring, &seqno);
1756 if (ret)
1757 return ret;
673a394b 1758
db53a302 1759 trace_i915_gem_request_add(ring, seqno);
673a394b
EA		 1760
1761 request->seqno = seqno;
852835f3 1762 request->ring = ring;
673a394b 1763 request->emitted_jiffies = jiffies;
852835f3
ZN		 1764 was_empty = list_empty(&ring->request_list);
1765 list_add_tail(&request->list, &ring->request_list);
1766
db53a302
CW		 1767 if (file) {
1768 struct drm_i915_file_private *file_priv = file->driver_priv;
1769
1c25595f 1770 spin_lock(&file_priv->mm.lock);
f787a5f5 1771 request->file_priv = file_priv;
b962442e 1772 list_add_tail(&request->client_list,
f787a5f5 1773 &file_priv->mm.request_list);
1c25595f 1774 spin_unlock(&file_priv->mm.lock);
b962442e 1775 }
673a394b 1776
db53a302
CW		 1777 ring->outstanding_lazy_request = false;
1778
f65d9421 1779 if (!dev_priv->mm.suspended) {
b3b079db
CW		 1780 mod_timer(&dev_priv->hangcheck_timer,
1781 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
f65d9421 1782 if (was_empty)
b3b079db
CW		 1783 queue_delayed_work(dev_priv->wq,
1784 &dev_priv->mm.retire_work, HZ);
f65d9421 1785 }
3cce469c 1786 return 0;
673a394b
EA		 1787}
1788
f787a5f5
CW		 1789static inline void
1790i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
673a394b 1791{
1c25595f 1792 struct drm_i915_file_private *file_priv = request->file_priv;
673a394b 1793
1c25595f
CW		 1794 if (!file_priv)
1795 return;
1c5d22f7 1796
1c25595f 1797 spin_lock(&file_priv->mm.lock);
09bfa517
HRK		 1798 if (request->file_priv) {
1799 list_del(&request->client_list);
1800 request->file_priv = NULL;
1801 }
1c25595f 1802 spin_unlock(&file_priv->mm.lock);
673a394b 1803}
673a394b 1804
dfaae392
CW		 1805static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1806 struct intel_ring_buffer *ring)
9375e446 1807{
dfaae392
CW		 1808 while (!list_empty(&ring->request_list)) {
1809 struct drm_i915_gem_request *request;
673a394b 1810
dfaae392
CW		 1811 request = list_first_entry(&ring->request_list,
1812 struct drm_i915_gem_request,
1813 list);
de151cf6 1814
dfaae392 1815 list_del(&request->list);
f787a5f5 1816 i915_gem_request_remove_from_client(request);
dfaae392
CW		 1817 kfree(request);
1818 }
673a394b 1819
dfaae392 1820 while (!list_empty(&ring->active_list)) {
05394f39 1821 struct drm_i915_gem_object *obj;
9375e446 1822
05394f39
CW		 1823 obj = list_first_entry(&ring->active_list,
1824 struct drm_i915_gem_object,
1825 ring_list);
9375e446 1826
05394f39
CW		 1827 obj->base.write_domain = 0;
1828 list_del_init(&obj->gpu_write_list);
1829 i915_gem_object_move_to_inactive(obj);
673a394b
EA		 1830 }
1831}
1832
312817a3
CW		 1833static void i915_gem_reset_fences(struct drm_device *dev)
1834{
1835 struct drm_i915_private *dev_priv = dev->dev_private;
1836 int i;
1837
1838 for (i = 0; i < 16; i++) {
1839 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
7d2cb39c
CW		 1840 struct drm_i915_gem_object *obj = reg->obj;
1841
1842 if (!obj)
1843 continue;
1844
1845 if (obj->tiling_mode)
1846 i915_gem_release_mmap(obj);
1847
d9e86c0e
CW		 1848 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1849 reg->obj->fenced_gpu_access = false;
1850 reg->obj->last_fenced_seqno = 0;
1851 reg->obj->last_fenced_ring = NULL;
1852 i915_gem_clear_fence_reg(dev, reg);
312817a3
CW		 1853 }
1854}
1855
069efc1d 1856void i915_gem_reset(struct drm_device *dev)
673a394b 1857{
77f01230 1858 struct drm_i915_private *dev_priv = dev->dev_private;
05394f39 1859 struct drm_i915_gem_object *obj;
1ec14ad3 1860 int i;
673a394b 1861
1ec14ad3
CW		 1862 for (i = 0; i < I915_NUM_RINGS; i++)
1863 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
dfaae392
CW		 1864
1865 /* Remove anything from the flushing lists. The GPU cache is likely
1866 * to be lost on reset along with the data, so simply move the
1867 * lost bo to the inactive list.
1868 */
1869 while (!list_empty(&dev_priv->mm.flushing_list)) {
05394f39
CW		 1870 obj= list_first_entry(&dev_priv->mm.flushing_list,
1871 struct drm_i915_gem_object,
1872 mm_list);
dfaae392 1873
05394f39
CW		 1874 obj->base.write_domain = 0;
1875 list_del_init(&obj->gpu_write_list);
1876 i915_gem_object_move_to_inactive(obj);
dfaae392
CW		 1877 }
1878
1879 /* Move everything out of the GPU domains to ensure we do any
1880 * necessary invalidation upon reuse.
1881 */
05394f39 1882 list_for_each_entry(obj,
77f01230 1883 &dev_priv->mm.inactive_list,
69dc4987 1884 mm_list)
77f01230 1885 {
05394f39 1886 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
77f01230 1887 }
069efc1d
CW		 1888
1889 /* The fence registers are invalidated so clear them out */
312817a3 1890 i915_gem_reset_fences(dev);
673a394b
EA		 1891}
1892
1893/**
1894 * This function clears the request list as sequence numbers are passed.
1895 */
b09a1fec 1896static void
db53a302 1897i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
673a394b 1898{
673a394b 1899 uint32_t seqno;
1ec14ad3 1900 int i;
673a394b 1901
db53a302 1902 if (list_empty(&ring->request_list))
6c0594a3
KW		 1903 return;
1904
db53a302 1905 WARN_ON(i915_verify_lists(ring->dev));
673a394b 1906
78501eac 1907 seqno = ring->get_seqno(ring);
1ec14ad3 1908
076e2c0e 1909 for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
1ec14ad3
CW		 1910 if (seqno >= ring->sync_seqno[i])
1911 ring->sync_seqno[i] = 0;
1912
852835f3 1913 while (!list_empty(&ring->request_list)) {
673a394b 1914 struct drm_i915_gem_request *request;
673a394b 1915
852835f3 1916 request = list_first_entry(&ring->request_list,
673a394b
EA		 1917 struct drm_i915_gem_request,
1918 list);
673a394b 1919
dfaae392 1920 if (!i915_seqno_passed(seqno, request->seqno))
b84d5f0c
CW		 1921 break;
1922
db53a302 1923 trace_i915_gem_request_retire(ring, request->seqno);
b84d5f0c
CW		 1924
1925 list_del(&request->list);
f787a5f5 1926 i915_gem_request_remove_from_client(request);
b84d5f0c
CW		 1927 kfree(request);
1928 }
673a394b 1929
b84d5f0c
CW		 1930 /* Move any buffers on the active list that are no longer referenced
1931 * by the ringbuffer to the flushing/inactive lists as appropriate.
1932 */
1933 while (!list_empty(&ring->active_list)) {
05394f39 1934 struct drm_i915_gem_object *obj;
b84d5f0c 1935
05394f39
CW		 1936 obj= list_first_entry(&ring->active_list,
1937 struct drm_i915_gem_object,
1938 ring_list);
673a394b 1939
05394f39 1940 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
673a394b 1941 break;
b84d5f0c 1942
05394f39 1943 if (obj->base.write_domain != 0)
b84d5f0c
CW		 1944 i915_gem_object_move_to_flushing(obj);
1945 else
1946 i915_gem_object_move_to_inactive(obj);
673a394b 1947 }
9d34e5db 1948
db53a302
CW		 1949 if (unlikely(ring->trace_irq_seqno &&
1950 i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1ec14ad3 1951 ring->irq_put(ring);
db53a302 1952 ring->trace_irq_seqno = 0;
9d34e5db 1953 }
23bc5982 1954
db53a302 1955 WARN_ON(i915_verify_lists(ring->dev));
673a394b
EA		 1956}
1957
b09a1fec
CW		 1958void
1959i915_gem_retire_requests(struct drm_device *dev)
1960{
1961 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 1962 int i;
b09a1fec 1963
be72615b 1964 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
05394f39 1965 struct drm_i915_gem_object *obj, *next;
be72615b
CW		 1966
1967 /* We must be careful that during unbind() we do not
1968 * accidentally infinitely recurse into retire requests.
1969 * Currently:
1970 * retire -> free -> unbind -> wait -> retire_ring
1971 */
05394f39 1972 list_for_each_entry_safe(obj, next,
be72615b 1973 &dev_priv->mm.deferred_free_list,
69dc4987 1974 mm_list)
05394f39 1975 i915_gem_free_object_tail(obj);
be72615b
CW		 1976 }
1977
1ec14ad3 1978 for (i = 0; i < I915_NUM_RINGS; i++)
db53a302 1979 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
b09a1fec
CW		 1980}
1981
75ef9da2 1982static void
673a394b
EA		 1983i915_gem_retire_work_handler(struct work_struct *work)
1984{
1985 drm_i915_private_t *dev_priv;
1986 struct drm_device *dev;
0a58705b
CW		 1987 bool idle;
1988 int i;
673a394b
EA		 1989
1990 dev_priv = container_of(work, drm_i915_private_t,
1991 mm.retire_work.work);
1992 dev = dev_priv->dev;
1993
891b48cf
CW		 1994 /* Come back later if the device is busy... */
1995 if (!mutex_trylock(&dev->struct_mutex)) {
1996 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1997 return;
1998 }
1999
b09a1fec 2000 i915_gem_retire_requests(dev);
d1b851fc 2001
0a58705b
CW		 2002 /* Send a periodic flush down the ring so we don't hold onto GEM
2003 * objects indefinitely.
2004 */
2005 idle = true;
2006 for (i = 0; i < I915_NUM_RINGS; i++) {
2007 struct intel_ring_buffer *ring = &dev_priv->ring[i];
2008
2009 if (!list_empty(&ring->gpu_write_list)) {
2010 struct drm_i915_gem_request *request;
2011 int ret;
2012
db53a302
CW		 2013 ret = i915_gem_flush_ring(ring,
2014 0, I915_GEM_GPU_DOMAINS);
0a58705b
CW		 2015 request = kzalloc(sizeof(*request), GFP_KERNEL);
2016 if (ret || request == NULL ||
db53a302 2017 i915_add_request(ring, NULL, request))
0a58705b
CW		 2018 kfree(request);
2019 }
2020
2021 idle &= list_empty(&ring->request_list);
2022 }
2023
2024 if (!dev_priv->mm.suspended && !idle)
9c9fe1f8 2025 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
0a58705b 2026
673a394b
EA		 2027 mutex_unlock(&dev->struct_mutex);
2028}
2029
db53a302
CW		 2030/**
2031 * Waits for a sequence number to be signaled, and cleans up the
2032 * request and object lists appropriately for that event.
2033 */
5a5a0c64 2034int
db53a302 2035i915_wait_request(struct intel_ring_buffer *ring,
ce453d81 2036 uint32_t seqno)
673a394b 2037{
db53a302 2038 drm_i915_private_t *dev_priv = ring->dev->dev_private;
802c7eb6 2039 u32 ier;
673a394b
EA		 2040 int ret = 0;
2041
2042 BUG_ON(seqno == 0);
2043
d9bc7e9f
CW		 2044 if (atomic_read(&dev_priv->mm.wedged)) {
2045 struct completion *x = &dev_priv->error_completion;
2046 bool recovery_complete;
2047 unsigned long flags;
2048
2049 /* Give the error handler a chance to run. */
2050 spin_lock_irqsave(&x->wait.lock, flags);
2051 recovery_complete = x->done > 0;
2052 spin_unlock_irqrestore(&x->wait.lock, flags);
2053
2054 return recovery_complete ? -EIO : -EAGAIN;
2055 }
30dbf0c0 2056
5d97eb69 2057 if (seqno == ring->outstanding_lazy_request) {
3cce469c
CW		 2058 struct drm_i915_gem_request *request;
2059
2060 request = kzalloc(sizeof(*request), GFP_KERNEL);
2061 if (request == NULL)
e35a41de 2062 return -ENOMEM;
3cce469c 2063
db53a302 2064 ret = i915_add_request(ring, NULL, request);
3cce469c
CW		 2065 if (ret) {
2066 kfree(request);
2067 return ret;
2068 }
2069
2070 seqno = request->seqno;
e35a41de 2071 }
ffed1d09 2072
78501eac 2073 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
db53a302 2074 if (HAS_PCH_SPLIT(ring->dev))
036a4a7d
ZW		 2075 ier = I915_READ(DEIER) | I915_READ(GTIER);
2076 else
2077 ier = I915_READ(IER);
802c7eb6
JB		 2078 if (!ier) {
2079 DRM_ERROR("something (likely vbetool) disabled "
2080 "interrupts, re-enabling\n");
db53a302
CW		 2081 i915_driver_irq_preinstall(ring->dev);
2082 i915_driver_irq_postinstall(ring->dev);
802c7eb6
JB		 2083 }
2084
db53a302 2085 trace_i915_gem_request_wait_begin(ring, seqno);
1c5d22f7 2086
b2223497 2087 ring->waiting_seqno = seqno;
b13c2b96 2088 if (ring->irq_get(ring)) {
ce453d81 2089 if (dev_priv->mm.interruptible)
b13c2b96
CW		 2090 ret = wait_event_interruptible(ring->irq_queue,
2091 i915_seqno_passed(ring->get_seqno(ring), seqno)
2092 || atomic_read(&dev_priv->mm.wedged));
2093 else
2094 wait_event(ring->irq_queue,
2095 i915_seqno_passed(ring->get_seqno(ring), seqno)
2096 || atomic_read(&dev_priv->mm.wedged));
2097
2098 ring->irq_put(ring);
b5ba177d
CW		 2099 } else if (wait_for(i915_seqno_passed(ring->get_seqno(ring),
2100 seqno) ||
2101 atomic_read(&dev_priv->mm.wedged), 3000))
2102 ret = -EBUSY;
b2223497 2103 ring->waiting_seqno = 0;
1c5d22f7 2104
db53a302 2105 trace_i915_gem_request_wait_end(ring, seqno);
673a394b 2106 }
ba1234d1 2107 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0 2108 ret = -EAGAIN;
673a394b
EA		 2109
2110 if (ret && ret != -ERESTARTSYS)
8bff917c 2111 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
78501eac 2112 __func__, ret, seqno, ring->get_seqno(ring),
8bff917c 2113 dev_priv->next_seqno);
673a394b
EA		 2114
2115 /* Directly dispatch request retiring. While we have the work queue
2116 * to handle this, the waiter on a request often wants an associated
2117 * buffer to have made it to the inactive list, and we would need
2118 * a separate wait queue to handle that.
2119 */
2120 if (ret == 0)
db53a302 2121 i915_gem_retire_requests_ring(ring);
673a394b
EA		 2122
2123 return ret;
2124}
2125
673a394b
EA		 2126/**
2127 * Ensures that all rendering to the object has completed and the object is
2128 * safe to unbind from the GTT or access from the CPU.
2129 */
54cf91dc 2130int
ce453d81 2131i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
673a394b 2132{
673a394b
EA		 2133 int ret;
2134
e47c68e9
EA		 2135 /* This function only exists to support waiting for existing rendering,
2136 * not for emitting required flushes.
673a394b 2137 */
05394f39 2138 BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 2139
2140 /* If there is rendering queued on the buffer being evicted, wait for
2141 * it.
2142 */
05394f39 2143 if (obj->active) {
ce453d81 2144 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
2cf34d7b 2145 if (ret)
673a394b
EA		 2146 return ret;
2147 }
2148
2149 return 0;
2150}
2151
2152/**
2153 * Unbinds an object from the GTT aperture.
2154 */
0f973f27 2155int
05394f39 2156i915_gem_object_unbind(struct drm_i915_gem_object *obj)
673a394b 2157{
673a394b
EA		 2158 int ret = 0;
2159
05394f39 2160 if (obj->gtt_space == NULL)
673a394b
EA		 2161 return 0;
2162
05394f39 2163 if (obj->pin_count != 0) {
673a394b
EA		 2164 DRM_ERROR("Attempting to unbind pinned buffer\n");
2165 return -EINVAL;
2166 }
2167
a8198eea
CW		 2168 ret = i915_gem_object_finish_gpu(obj);
2169 if (ret == -ERESTARTSYS)
2170 return ret;
2171 /* Continue on if we fail due to EIO, the GPU is hung so we
2172 * should be safe and we need to cleanup or else we might
2173 * cause memory corruption through use-after-free.
2174 */
2175
5323fd04
EA		 2176 /* blow away mappings if mapped through GTT */
2177 i915_gem_release_mmap(obj);
2178
673a394b
EA		 2179 /* Move the object to the CPU domain to ensure that
2180 * any possible CPU writes while it's not in the GTT
a8198eea 2181 * are flushed when we go to remap it.
673a394b 2182 */
a8198eea
CW		 2183 if (ret == 0)
2184 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
8dc1775d 2185 if (ret == -ERESTARTSYS)
673a394b 2186 return ret;
812ed492 2187 if (ret) {
a8198eea
CW		 2188 /* In the event of a disaster, abandon all caches and
2189 * hope for the best.
2190 */
812ed492 2191 i915_gem_clflush_object(obj);
05394f39 2192 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
812ed492 2193 }
673a394b 2194
96b47b65 2195 /* release the fence reg _after_ flushing */
d9e86c0e
CW		 2196 ret = i915_gem_object_put_fence(obj);
2197 if (ret == -ERESTARTSYS)
2198 return ret;
96b47b65 2199
db53a302
CW		 2200 trace_i915_gem_object_unbind(obj);
2201
7c2e6fdf 2202 i915_gem_gtt_unbind_object(obj);
e5281ccd 2203 i915_gem_object_put_pages_gtt(obj);
673a394b 2204
6299f992 2205 list_del_init(&obj->gtt_list);
05394f39 2206 list_del_init(&obj->mm_list);
75e9e915 2207 /* Avoid an unnecessary call to unbind on rebind. */
05394f39 2208 obj->map_and_fenceable = true;
673a394b 2209
05394f39
CW		 2210 drm_mm_put_block(obj->gtt_space);
2211 obj->gtt_space = NULL;
2212 obj->gtt_offset = 0;
673a394b 2213
05394f39 2214 if (i915_gem_object_is_purgeable(obj))
963b4836
CW		 2215 i915_gem_object_truncate(obj);
2216
8dc1775d 2217 return ret;
673a394b
EA		 2218}
2219
88241785 2220int
db53a302 2221i915_gem_flush_ring(struct intel_ring_buffer *ring,
54cf91dc
CW		 2222 uint32_t invalidate_domains,
2223 uint32_t flush_domains)
2224{
88241785
CW		 2225 int ret;
2226
36d527de
CW		 2227 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2228 return 0;
2229
db53a302
CW		 2230 trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
2231
88241785
CW		 2232 ret = ring->flush(ring, invalidate_domains, flush_domains);
2233 if (ret)
2234 return ret;
2235
36d527de
CW		 2236 if (flush_domains & I915_GEM_GPU_DOMAINS)
2237 i915_gem_process_flushing_list(ring, flush_domains);
2238
88241785 2239 return 0;
54cf91dc
CW		 2240}
2241
db53a302 2242static int i915_ring_idle(struct intel_ring_buffer *ring)
a56ba56c 2243{
88241785
CW		 2244 int ret;
2245
395b70be 2246 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
64193406
CW		 2247 return 0;
2248
88241785 2249 if (!list_empty(&ring->gpu_write_list)) {
db53a302 2250 ret = i915_gem_flush_ring(ring,
0ac74c6b 2251 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
88241785
CW		 2252 if (ret)
2253 return ret;
2254 }
2255
ce453d81 2256 return i915_wait_request(ring, i915_gem_next_request_seqno(ring));
a56ba56c
CW		 2257}
2258
b47eb4a2 2259int
4df2faf4
DV		 2260i915_gpu_idle(struct drm_device *dev)
2261{
2262 drm_i915_private_t *dev_priv = dev->dev_private;
2263 bool lists_empty;
1ec14ad3 2264 int ret, i;
4df2faf4 2265
d1b851fc 2266 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
395b70be 2267 list_empty(&dev_priv->mm.active_list));
4df2faf4
DV		 2268 if (lists_empty)
2269 return 0;
2270
2271 /* Flush everything onto the inactive list. */
1ec14ad3 2272 for (i = 0; i < I915_NUM_RINGS; i++) {
db53a302 2273 ret = i915_ring_idle(&dev_priv->ring[i]);
1ec14ad3
CW		 2274 if (ret)
2275 return ret;
2276 }
4df2faf4 2277
8a1a49f9 2278 return 0;
4df2faf4
DV		 2279}
2280
c6642782
DV		 2281static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2282 struct intel_ring_buffer *pipelined)
4e901fdc 2283{
05394f39 2284 struct drm_device *dev = obj->base.dev;
4e901fdc 2285 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 2286 u32 size = obj->gtt_space->size;
2287 int regnum = obj->fence_reg;
4e901fdc
EA		 2288 uint64_t val;
2289
05394f39 2290 val = (uint64_t)((obj->gtt_offset + size - 4096) &
c6642782 2291 0xfffff000) << 32;
05394f39
CW		 2292 val |= obj->gtt_offset & 0xfffff000;
2293 val |= (uint64_t)((obj->stride / 128) - 1) <<
4e901fdc
EA		 2294 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2295
05394f39 2296 if (obj->tiling_mode == I915_TILING_Y)
4e901fdc
EA		 2297 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2298 val |= I965_FENCE_REG_VALID;
2299
c6642782
DV		 2300 if (pipelined) {
2301 int ret = intel_ring_begin(pipelined, 6);
2302 if (ret)
2303 return ret;
2304
2305 intel_ring_emit(pipelined, MI_NOOP);
2306 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2307 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2308 intel_ring_emit(pipelined, (u32)val);
2309 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2310 intel_ring_emit(pipelined, (u32)(val >> 32));
2311 intel_ring_advance(pipelined);
2312 } else
2313 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2314
2315 return 0;
4e901fdc
EA		 2316}
2317
c6642782
DV		 2318static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
2319 struct intel_ring_buffer *pipelined)
de151cf6 2320{
05394f39 2321 struct drm_device *dev = obj->base.dev;
de151cf6 2322 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 2323 u32 size = obj->gtt_space->size;
2324 int regnum = obj->fence_reg;
de151cf6
JB		 2325 uint64_t val;
2326
05394f39 2327 val = (uint64_t)((obj->gtt_offset + size - 4096) &
de151cf6 2328 0xfffff000) << 32;
05394f39
CW		 2329 val |= obj->gtt_offset & 0xfffff000;
2330 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2331 if (obj->tiling_mode == I915_TILING_Y)
de151cf6
JB		 2332 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2333 val |= I965_FENCE_REG_VALID;
2334
c6642782
DV		 2335 if (pipelined) {
2336 int ret = intel_ring_begin(pipelined, 6);
2337 if (ret)
2338 return ret;
2339
2340 intel_ring_emit(pipelined, MI_NOOP);
2341 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2342 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2343 intel_ring_emit(pipelined, (u32)val);
2344 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2345 intel_ring_emit(pipelined, (u32)(val >> 32));
2346 intel_ring_advance(pipelined);
2347 } else
2348 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2349
2350 return 0;
de151cf6
JB		 2351}
2352
c6642782
DV		 2353static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
2354 struct intel_ring_buffer *pipelined)
de151cf6 2355{
05394f39 2356 struct drm_device *dev = obj->base.dev;
de151cf6 2357 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39 2358 u32 size = obj->gtt_space->size;
c6642782 2359 u32 fence_reg, val, pitch_val;
0f973f27 2360 int tile_width;
de151cf6 2361
c6642782
DV		 2362 if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2363 (size & -size) != size ||
2364 (obj->gtt_offset & (size - 1)),
2365 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2366 obj->gtt_offset, obj->map_and_fenceable, size))
2367 return -EINVAL;
de151cf6 2368
c6642782 2369 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
0f973f27 2370 tile_width = 128;
de151cf6 2371 else
0f973f27
JB		 2372 tile_width = 512;
2373
2374 /* Note: pitch better be a power of two tile widths */
05394f39 2375 pitch_val = obj->stride / tile_width;
0f973f27 2376 pitch_val = ffs(pitch_val) - 1;
de151cf6 2377
05394f39
CW		 2378 val = obj->gtt_offset;
2379 if (obj->tiling_mode == I915_TILING_Y)
de151cf6 2380 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
a00b10c3 2381 val |= I915_FENCE_SIZE_BITS(size);
de151cf6
JB		 2382 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2383 val |= I830_FENCE_REG_VALID;
2384
05394f39 2385 fence_reg = obj->fence_reg;
a00b10c3
CW		 2386 if (fence_reg < 8)
2387 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
dc529a4f 2388 else
a00b10c3 2389 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
c6642782
DV		 2390
2391 if (pipelined) {
2392 int ret = intel_ring_begin(pipelined, 4);
2393 if (ret)
2394 return ret;
2395
2396 intel_ring_emit(pipelined, MI_NOOP);
2397 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2398 intel_ring_emit(pipelined, fence_reg);
2399 intel_ring_emit(pipelined, val);
2400 intel_ring_advance(pipelined);
2401 } else
2402 I915_WRITE(fence_reg, val);
2403
2404 return 0;
de151cf6
JB		 2405}
2406
c6642782
DV		 2407static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
2408 struct intel_ring_buffer *pipelined)
de151cf6 2409{
05394f39 2410 struct drm_device *dev = obj->base.dev;
de151cf6 2411 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 2412 u32 size = obj->gtt_space->size;
2413 int regnum = obj->fence_reg;
de151cf6
JB		 2414 uint32_t val;
2415 uint32_t pitch_val;
2416
c6642782
DV		 2417 if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2418 (size & -size) != size ||
2419 (obj->gtt_offset & (size - 1)),
2420 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2421 obj->gtt_offset, size))
2422 return -EINVAL;
de151cf6 2423
05394f39 2424 pitch_val = obj->stride / 128;
e76a16de 2425 pitch_val = ffs(pitch_val) - 1;
e76a16de 2426
05394f39
CW		 2427 val = obj->gtt_offset;
2428 if (obj->tiling_mode == I915_TILING_Y)
de151cf6 2429 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
c6642782 2430 val |= I830_FENCE_SIZE_BITS(size);
de151cf6
JB		 2431 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2432 val |= I830_FENCE_REG_VALID;
2433
c6642782
DV		 2434 if (pipelined) {
2435 int ret = intel_ring_begin(pipelined, 4);
2436 if (ret)
2437 return ret;
2438
2439 intel_ring_emit(pipelined, MI_NOOP);
2440 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2441 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2442 intel_ring_emit(pipelined, val);
2443 intel_ring_advance(pipelined);
2444 } else
2445 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2446
2447 return 0;
de151cf6
JB		 2448}
2449
d9e86c0e
CW		 2450static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2451{
2452 return i915_seqno_passed(ring->get_seqno(ring), seqno);
2453}
2454
2455static int
2456i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
ce453d81 2457 struct intel_ring_buffer *pipelined)
d9e86c0e
CW		 2458{
2459 int ret;
2460
2461 if (obj->fenced_gpu_access) {
88241785 2462 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
db53a302 2463 ret = i915_gem_flush_ring(obj->last_fenced_ring,
88241785
CW		 2464 0, obj->base.write_domain);
2465 if (ret)
2466 return ret;
2467 }
d9e86c0e
CW		 2468
2469 obj->fenced_gpu_access = false;
2470 }
2471
2472 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
2473 if (!ring_passed_seqno(obj->last_fenced_ring,
2474 obj->last_fenced_seqno)) {
db53a302 2475 ret = i915_wait_request(obj->last_fenced_ring,
ce453d81 2476 obj->last_fenced_seqno);
d9e86c0e
CW		 2477 if (ret)
2478 return ret;
2479 }
2480
2481 obj->last_fenced_seqno = 0;
2482 obj->last_fenced_ring = NULL;
2483 }
2484
63256ec5
CW		 2485 /* Ensure that all CPU reads are completed before installing a fence
2486 * and all writes before removing the fence.
2487 */
2488 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2489 mb();
2490
d9e86c0e
CW		 2491 return 0;
2492}
2493
2494int
2495i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2496{
2497 int ret;
2498
2499 if (obj->tiling_mode)
2500 i915_gem_release_mmap(obj);
2501
ce453d81 2502 ret = i915_gem_object_flush_fence(obj, NULL);
d9e86c0e
CW		 2503 if (ret)
2504 return ret;
2505
2506 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2507 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2508 i915_gem_clear_fence_reg(obj->base.dev,
2509 &dev_priv->fence_regs[obj->fence_reg]);
2510
2511 obj->fence_reg = I915_FENCE_REG_NONE;
2512 }
2513
2514 return 0;
2515}
2516
2517static struct drm_i915_fence_reg *
2518i915_find_fence_reg(struct drm_device *dev,
2519 struct intel_ring_buffer *pipelined)
ae3db24a 2520{
ae3db24a 2521 struct drm_i915_private *dev_priv = dev->dev_private;
d9e86c0e
CW		 2522 struct drm_i915_fence_reg *reg, *first, *avail;
2523 int i;
ae3db24a
DV		 2524
2525 /* First try to find a free reg */
d9e86c0e 2526 avail = NULL;
ae3db24a
DV		 2527 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2528 reg = &dev_priv->fence_regs[i];
2529 if (!reg->obj)
d9e86c0e 2530 return reg;
ae3db24a 2531
05394f39 2532 if (!reg->obj->pin_count)
d9e86c0e 2533 avail = reg;
ae3db24a
DV		 2534 }
2535
d9e86c0e
CW		 2536 if (avail == NULL)
2537 return NULL;
ae3db24a
DV		 2538
2539 /* None available, try to steal one or wait for a user to finish */
d9e86c0e
CW		 2540 avail = first = NULL;
2541 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2542 if (reg->obj->pin_count)
ae3db24a
DV		 2543 continue;
2544
d9e86c0e
CW		 2545 if (first == NULL)
2546 first = reg;
2547
2548 if (!pipelined ||
2549 !reg->obj->last_fenced_ring ||
2550 reg->obj->last_fenced_ring == pipelined) {
2551 avail = reg;
2552 break;
2553 }
ae3db24a
DV		 2554 }
2555
d9e86c0e
CW		 2556 if (avail == NULL)
2557 avail = first;
ae3db24a 2558
a00b10c3 2559 return avail;
ae3db24a
DV		 2560}
2561
de151cf6 2562/**
d9e86c0e 2563 * i915_gem_object_get_fence - set up a fence reg for an object
de151cf6 2564 * @obj: object to map through a fence reg
d9e86c0e
CW		 2565 * @pipelined: ring on which to queue the change, or NULL for CPU access
2566 * @interruptible: must we wait uninterruptibly for the register to retire?
de151cf6
JB		 2567 *
2568 * When mapping objects through the GTT, userspace wants to be able to write
2569 * to them without having to worry about swizzling if the object is tiled.
2570 *
2571 * This function walks the fence regs looking for a free one for @obj,
2572 * stealing one if it can't find any.
2573 *
2574 * It then sets up the reg based on the object's properties: address, pitch
2575 * and tiling format.
2576 */
8c4b8c3f 2577int
d9e86c0e 2578i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
ce453d81 2579 struct intel_ring_buffer *pipelined)
de151cf6 2580{
05394f39 2581 struct drm_device *dev = obj->base.dev;
79e53945 2582 struct drm_i915_private *dev_priv = dev->dev_private;
d9e86c0e 2583 struct drm_i915_fence_reg *reg;
ae3db24a 2584 int ret;
de151cf6 2585
6bda10d1
CW		 2586 /* XXX disable pipelining. There are bugs. Shocking. */
2587 pipelined = NULL;
2588
d9e86c0e 2589 /* Just update our place in the LRU if our fence is getting reused. */
05394f39
CW		 2590 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2591 reg = &dev_priv->fence_regs[obj->fence_reg];
007cc8ac 2592 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
d9e86c0e 2593
29c5a587
CW		 2594 if (obj->tiling_changed) {
2595 ret = i915_gem_object_flush_fence(obj, pipelined);
2596 if (ret)
2597 return ret;
2598
2599 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2600 pipelined = NULL;
2601
2602 if (pipelined) {
2603 reg->setup_seqno =
2604 i915_gem_next_request_seqno(pipelined);
2605 obj->last_fenced_seqno = reg->setup_seqno;
2606 obj->last_fenced_ring = pipelined;
2607 }
2608
2609 goto update;
2610 }
d9e86c0e
CW		 2611
2612 if (!pipelined) {
2613 if (reg->setup_seqno) {
2614 if (!ring_passed_seqno(obj->last_fenced_ring,
2615 reg->setup_seqno)) {
db53a302 2616 ret = i915_wait_request(obj->last_fenced_ring,
ce453d81 2617 reg->setup_seqno);
d9e86c0e
CW		 2618 if (ret)
2619 return ret;
2620 }
2621
2622 reg->setup_seqno = 0;
2623 }
2624 } else if (obj->last_fenced_ring &&
2625 obj->last_fenced_ring != pipelined) {
ce453d81 2626 ret = i915_gem_object_flush_fence(obj, pipelined);
d9e86c0e
CW		 2627 if (ret)
2628 return ret;
d9e86c0e
CW		 2629 }
2630
a09ba7fa
EA		 2631 return 0;
2632 }
2633
d9e86c0e
CW		 2634 reg = i915_find_fence_reg(dev, pipelined);
2635 if (reg == NULL)
2636 return -ENOSPC;
de151cf6 2637
ce453d81 2638 ret = i915_gem_object_flush_fence(obj, pipelined);
d9e86c0e 2639 if (ret)
ae3db24a 2640 return ret;
de151cf6 2641
d9e86c0e
CW		 2642 if (reg->obj) {
2643 struct drm_i915_gem_object *old = reg->obj;
2644
2645 drm_gem_object_reference(&old->base);
2646
2647 if (old->tiling_mode)
2648 i915_gem_release_mmap(old);
2649
ce453d81 2650 ret = i915_gem_object_flush_fence(old, pipelined);
d9e86c0e
CW		 2651 if (ret) {
2652 drm_gem_object_unreference(&old->base);
2653 return ret;
2654 }
2655
2656 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
2657 pipelined = NULL;
2658
2659 old->fence_reg = I915_FENCE_REG_NONE;
2660 old->last_fenced_ring = pipelined;
2661 old->last_fenced_seqno =
db53a302 2662 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
d9e86c0e
CW		 2663
2664 drm_gem_object_unreference(&old->base);
2665 } else if (obj->last_fenced_seqno == 0)
2666 pipelined = NULL;
a09ba7fa 2667
de151cf6 2668 reg->obj = obj;
d9e86c0e
CW		 2669 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2670 obj->fence_reg = reg - dev_priv->fence_regs;
2671 obj->last_fenced_ring = pipelined;
de151cf6 2672
d9e86c0e 2673 reg->setup_seqno =
db53a302 2674 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
d9e86c0e
CW		 2675 obj->last_fenced_seqno = reg->setup_seqno;
2676
2677update:
2678 obj->tiling_changed = false;
e259befd 2679 switch (INTEL_INFO(dev)->gen) {
25aebfc3 2680 case 7:
e259befd 2681 case 6:
c6642782 2682 ret = sandybridge_write_fence_reg(obj, pipelined);
e259befd
CW		 2683 break;
2684 case 5:
2685 case 4:
c6642782 2686 ret = i965_write_fence_reg(obj, pipelined);
e259befd
CW		 2687 break;
2688 case 3:
c6642782 2689 ret = i915_write_fence_reg(obj, pipelined);
e259befd
CW		 2690 break;
2691 case 2:
c6642782 2692 ret = i830_write_fence_reg(obj, pipelined);
e259befd
CW		 2693 break;
2694 }
d9ddcb96 2695
c6642782 2696 return ret;
de151cf6
JB		 2697}
2698
2699/**
2700 * i915_gem_clear_fence_reg - clear out fence register info
2701 * @obj: object to clear
2702 *
2703 * Zeroes out the fence register itself and clears out the associated
05394f39 2704 * data structures in dev_priv and obj.
de151cf6
JB		 2705 */
2706static void
d9e86c0e
CW		 2707i915_gem_clear_fence_reg(struct drm_device *dev,
2708 struct drm_i915_fence_reg *reg)
de151cf6 2709{
79e53945 2710 drm_i915_private_t *dev_priv = dev->dev_private;
d9e86c0e 2711 uint32_t fence_reg = reg - dev_priv->fence_regs;
de151cf6 2712
e259befd 2713 switch (INTEL_INFO(dev)->gen) {
25aebfc3 2714 case 7:
e259befd 2715 case 6:
d9e86c0e 2716 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
e259befd
CW		 2717 break;
2718 case 5:
2719 case 4:
d9e86c0e 2720 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
e259befd
CW		 2721 break;
2722 case 3:
d9e86c0e
CW		 2723 if (fence_reg >= 8)
2724 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
dc529a4f 2725 else
e259befd 2726 case 2:
d9e86c0e 2727 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
dc529a4f
EA		 2728
2729 I915_WRITE(fence_reg, 0);
e259befd 2730 break;
dc529a4f 2731 }
de151cf6 2732
007cc8ac 2733 list_del_init(&reg->lru_list);
d9e86c0e
CW		 2734 reg->obj = NULL;
2735 reg->setup_seqno = 0;
52dc7d32
CW		 2736}
2737
673a394b
EA		 2738/**
2739 * Finds free space in the GTT aperture and binds the object there.
2740 */
2741static int
05394f39 2742i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
920afa77 2743 unsigned alignment,
75e9e915 2744 bool map_and_fenceable)
673a394b 2745{
05394f39 2746 struct drm_device *dev = obj->base.dev;
673a394b 2747 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 2748 struct drm_mm_node *free_space;
a00b10c3 2749 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
5e783301 2750 u32 size, fence_size, fence_alignment, unfenced_alignment;
75e9e915 2751 bool mappable, fenceable;
07f73f69 2752 int ret;
673a394b 2753
05394f39 2754 if (obj->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2755 DRM_ERROR("Attempting to bind a purgeable object\n");
2756 return -EINVAL;
2757 }
2758
05394f39
CW		 2759 fence_size = i915_gem_get_gtt_size(obj);
2760 fence_alignment = i915_gem_get_gtt_alignment(obj);
2761 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(obj);
a00b10c3 2762
673a394b 2763 if (alignment == 0)
5e783301
DV		 2764 alignment = map_and_fenceable ? fence_alignment :
2765 unfenced_alignment;
75e9e915 2766 if (map_and_fenceable && alignment & (fence_alignment - 1)) {
673a394b
EA		 2767 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2768 return -EINVAL;
2769 }
2770
05394f39 2771 size = map_and_fenceable ? fence_size : obj->base.size;
a00b10c3 2772
654fc607
CW		 2773 /* If the object is bigger than the entire aperture, reject it early
2774 * before evicting everything in a vain attempt to find space.
2775 */
05394f39 2776 if (obj->base.size >
75e9e915 2777 (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
654fc607
CW		 2778 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2779 return -E2BIG;
2780 }
2781
673a394b 2782 search_free:
75e9e915 2783 if (map_and_fenceable)
920afa77
DV		 2784 free_space =
2785 drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
a00b10c3 2786 size, alignment, 0,
920afa77
DV		 2787 dev_priv->mm.gtt_mappable_end,
2788 0);
2789 else
2790 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
a00b10c3 2791 size, alignment, 0);
920afa77
DV		 2792
2793 if (free_space != NULL) {
75e9e915 2794 if (map_and_fenceable)
05394f39 2795 obj->gtt_space =
920afa77 2796 drm_mm_get_block_range_generic(free_space,
a00b10c3 2797 size, alignment, 0,
920afa77
DV		 2798 dev_priv->mm.gtt_mappable_end,
2799 0);
2800 else
05394f39 2801 obj->gtt_space =
a00b10c3 2802 drm_mm_get_block(free_space, size, alignment);
920afa77 2803 }
05394f39 2804 if (obj->gtt_space == NULL) {
673a394b
EA		 2805 /* If the gtt is empty and we're still having trouble
2806 * fitting our object in, we're out of memory.
2807 */
75e9e915
DV		 2808 ret = i915_gem_evict_something(dev, size, alignment,
2809 map_and_fenceable);
9731129c 2810 if (ret)
673a394b 2811 return ret;
9731129c 2812
673a394b
EA		 2813 goto search_free;
2814 }
2815
e5281ccd 2816 ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
673a394b 2817 if (ret) {
05394f39
CW		 2818 drm_mm_put_block(obj->gtt_space);
2819 obj->gtt_space = NULL;
07f73f69
CW		 2820
2821 if (ret == -ENOMEM) {
809b6334
CW		 2822 /* first try to reclaim some memory by clearing the GTT */
2823 ret = i915_gem_evict_everything(dev, false);
07f73f69 2824 if (ret) {
07f73f69 2825 /* now try to shrink everyone else */
4bdadb97
CW		 2826 if (gfpmask) {
2827 gfpmask = 0;
2828 goto search_free;
07f73f69
CW		 2829 }
2830
809b6334 2831 return -ENOMEM;
07f73f69
CW		 2832 }
2833
2834 goto search_free;
2835 }
2836
673a394b
EA		 2837 return ret;
2838 }
2839
7c2e6fdf
DV		 2840 ret = i915_gem_gtt_bind_object(obj);
2841 if (ret) {
e5281ccd 2842 i915_gem_object_put_pages_gtt(obj);
05394f39
CW		 2843 drm_mm_put_block(obj->gtt_space);
2844 obj->gtt_space = NULL;
07f73f69 2845
809b6334 2846 if (i915_gem_evict_everything(dev, false))
07f73f69 2847 return ret;
07f73f69
CW		 2848
2849 goto search_free;
673a394b 2850 }
673a394b 2851
6299f992 2852 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
05394f39 2853 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
bf1a1092 2854
673a394b
EA		 2855 /* Assert that the object is not currently in any GPU domain. As it
2856 * wasn't in the GTT, there shouldn't be any way it could have been in
2857 * a GPU cache
2858 */
05394f39
CW		 2859 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2860 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2861
6299f992 2862 obj->gtt_offset = obj->gtt_space->start;
1c5d22f7 2863
75e9e915 2864 fenceable =
05394f39
CW		 2865 obj->gtt_space->size == fence_size &&
2866 (obj->gtt_space->start & (fence_alignment -1)) == 0;
a00b10c3 2867
75e9e915 2868 mappable =
05394f39 2869 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
a00b10c3 2870
05394f39 2871 obj->map_and_fenceable = mappable && fenceable;
75e9e915 2872
db53a302 2873 trace_i915_gem_object_bind(obj, map_and_fenceable);
673a394b
EA		 2874 return 0;
2875}
2876
2877void
05394f39 2878i915_gem_clflush_object(struct drm_i915_gem_object *obj)
673a394b 2879{
673a394b
EA		 2880 /* If we don't have a page list set up, then we're not pinned
2881 * to GPU, and we can ignore the cache flush because it'll happen
2882 * again at bind time.
2883 */
05394f39 2884 if (obj->pages == NULL)
673a394b
EA		 2885 return;
2886
9c23f7fc
CW		 2887 /* If the GPU is snooping the contents of the CPU cache,
2888 * we do not need to manually clear the CPU cache lines. However,
2889 * the caches are only snooped when the render cache is
2890 * flushed/invalidated. As we always have to emit invalidations
2891 * and flushes when moving into and out of the RENDER domain, correct
2892 * snooping behaviour occurs naturally as the result of our domain
2893 * tracking.
2894 */
2895 if (obj->cache_level != I915_CACHE_NONE)
2896 return;
2897
1c5d22f7 2898 trace_i915_gem_object_clflush(obj);
cfa16a0d 2899
05394f39 2900 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
673a394b
EA		 2901}
2902
e47c68e9 2903/** Flushes any GPU write domain for the object if it's dirty. */
88241785 2904static int
3619df03 2905i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2906{
05394f39 2907 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
88241785 2908 return 0;
e47c68e9
EA		 2909
2910 /* Queue the GPU write cache flushing we need. */
db53a302 2911 return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
e47c68e9
EA		 2912}
2913
2914/** Flushes the GTT write domain for the object if it's dirty. */
2915static void
05394f39 2916i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2917{
1c5d22f7
CW		 2918 uint32_t old_write_domain;
2919
05394f39 2920 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
e47c68e9
EA		 2921 return;
2922
63256ec5 2923 /* No actual flushing is required for the GTT write domain. Writes
e47c68e9
EA		 2924 * to it immediately go to main memory as far as we know, so there's
2925 * no chipset flush. It also doesn't land in render cache.
63256ec5
CW		 2926 *
2927 * However, we do have to enforce the order so that all writes through
2928 * the GTT land before any writes to the device, such as updates to
2929 * the GATT itself.
e47c68e9 2930 */
63256ec5
CW		 2931 wmb();
2932
4a684a41
CW		 2933 i915_gem_release_mmap(obj);
2934
05394f39
CW		 2935 old_write_domain = obj->base.write_domain;
2936 obj->base.write_domain = 0;
1c5d22f7
CW		 2937
2938 trace_i915_gem_object_change_domain(obj,
05394f39 2939 obj->base.read_domains,
1c5d22f7 2940 old_write_domain);
e47c68e9
EA		 2941}
2942
2943/** Flushes the CPU write domain for the object if it's dirty. */
2944static void
05394f39 2945i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2946{
1c5d22f7 2947 uint32_t old_write_domain;
e47c68e9 2948
05394f39 2949 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
e47c68e9
EA		 2950 return;
2951
2952 i915_gem_clflush_object(obj);
40ce6575 2953 intel_gtt_chipset_flush();
05394f39
CW		 2954 old_write_domain = obj->base.write_domain;
2955 obj->base.write_domain = 0;
1c5d22f7
CW		 2956
2957 trace_i915_gem_object_change_domain(obj,
05394f39 2958 obj->base.read_domains,
1c5d22f7 2959 old_write_domain);
e47c68e9
EA		 2960}
2961
2ef7eeaa
EA		 2962/**
2963 * Moves a single object to the GTT read, and possibly write domain.
2964 *
2965 * This function returns when the move is complete, including waiting on
2966 * flushes to occur.
2967 */
79e53945 2968int
2021746e 2969i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2ef7eeaa 2970{
1c5d22f7 2971 uint32_t old_write_domain, old_read_domains;
e47c68e9 2972 int ret;
2ef7eeaa 2973
02354392 2974 /* Not valid to be called on unbound objects. */
05394f39 2975 if (obj->gtt_space == NULL)
02354392
EA		 2976 return -EINVAL;
2977
8d7e3de1
CW		 2978 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2979 return 0;
2980
88241785
CW		 2981 ret = i915_gem_object_flush_gpu_write_domain(obj);
2982 if (ret)
2983 return ret;
2984
87ca9c8a 2985 if (obj->pending_gpu_write || write) {
ce453d81 2986 ret = i915_gem_object_wait_rendering(obj);
87ca9c8a
CW		 2987 if (ret)
2988 return ret;
2989 }
2dafb1e0 2990
7213342d 2991 i915_gem_object_flush_cpu_write_domain(obj);
1c5d22f7 2992
05394f39
CW		 2993 old_write_domain = obj->base.write_domain;
2994 old_read_domains = obj->base.read_domains;
1c5d22f7 2995
e47c68e9
EA		 2996 /* It should now be out of any other write domains, and we can update
2997 * the domain values for our changes.
2998 */
05394f39
CW		 2999 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3000 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
e47c68e9 3001 if (write) {
05394f39
CW		 3002 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3003 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3004 obj->dirty = 1;
2ef7eeaa
EA		 3005 }
3006
1c5d22f7
CW		 3007 trace_i915_gem_object_change_domain(obj,
3008 old_read_domains,
3009 old_write_domain);
3010
e47c68e9
EA		 3011 return 0;
3012}
3013
b9241ea3
ZW		 3014/*
3015 * Prepare buffer for display plane. Use uninterruptible for possible flush
3016 * wait, as in modesetting process we're not supposed to be interrupted.
3017 */
3018int
05394f39 3019i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
919926ae 3020 struct intel_ring_buffer *pipelined)
b9241ea3 3021{
ba3d8d74 3022 uint32_t old_read_domains;
b9241ea3
ZW		 3023 int ret;
3024
3025 /* Not valid to be called on unbound objects. */
05394f39 3026 if (obj->gtt_space == NULL)
b9241ea3
ZW		 3027 return -EINVAL;
3028
88241785
CW		 3029 ret = i915_gem_object_flush_gpu_write_domain(obj);
3030 if (ret)
3031 return ret;
3032
b9241ea3 3033
ced270fa 3034 /* Currently, we are always called from an non-interruptible context. */
0be73284 3035 if (pipelined != obj->ring) {
ce453d81 3036 ret = i915_gem_object_wait_rendering(obj);
ced270fa 3037 if (ret)
b9241ea3
ZW		 3038 return ret;
3039 }
3040
b118c1e3
CW		 3041 i915_gem_object_flush_cpu_write_domain(obj);
3042
05394f39
CW		 3043 old_read_domains = obj->base.read_domains;
3044 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
b9241ea3
ZW		 3045
3046 trace_i915_gem_object_change_domain(obj,
3047 old_read_domains,
05394f39 3048 obj->base.write_domain);
b9241ea3
ZW		 3049
3050 return 0;
3051}
3052
85345517 3053int
a8198eea 3054i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
85345517 3055{
88241785
CW		 3056 int ret;
3057
a8198eea 3058 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
85345517
CW		 3059 return 0;
3060
88241785 3061 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
db53a302 3062 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
88241785
CW		 3063 if (ret)
3064 return ret;
3065 }
85345517 3066
a8198eea
CW		 3067 /* Ensure that we invalidate the GPU's caches and TLBs. */
3068 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3069
ce453d81 3070 return i915_gem_object_wait_rendering(obj);
85345517
CW		 3071}
3072
e47c68e9
EA		 3073/**
3074 * Moves a single object to the CPU read, and possibly write domain.
3075 *
3076 * This function returns when the move is complete, including waiting on
3077 * flushes to occur.
3078 */
3079static int
919926ae 3080i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
e47c68e9 3081{
1c5d22f7 3082 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 3083 int ret;
3084
8d7e3de1
CW		 3085 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3086 return 0;
3087
88241785
CW		 3088 ret = i915_gem_object_flush_gpu_write_domain(obj);
3089 if (ret)
3090 return ret;
3091
ce453d81 3092 ret = i915_gem_object_wait_rendering(obj);
de18a29e 3093 if (ret)
e47c68e9 3094 return ret;
2ef7eeaa 3095
e47c68e9 3096 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 3097
e47c68e9
EA		 3098 /* If we have a partially-valid cache of the object in the CPU,
3099 * finish invalidating it and free the per-page flags.
2ef7eeaa 3100 */
e47c68e9 3101 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 3102
05394f39
CW		 3103 old_write_domain = obj->base.write_domain;
3104 old_read_domains = obj->base.read_domains;
1c5d22f7 3105
e47c68e9 3106 /* Flush the CPU cache if it's still invalid. */
05394f39 3107 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 3108 i915_gem_clflush_object(obj);
2ef7eeaa 3109
05394f39 3110 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 3111 }
3112
3113 /* It should now be out of any other write domains, and we can update
3114 * the domain values for our changes.
3115 */
05394f39 3116 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
e47c68e9
EA		 3117
3118 /* If we're writing through the CPU, then the GPU read domains will
3119 * need to be invalidated at next use.
3120 */
3121 if (write) {
05394f39
CW		 3122 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3123 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
e47c68e9 3124 }
2ef7eeaa 3125
1c5d22f7
CW		 3126 trace_i915_gem_object_change_domain(obj,
3127 old_read_domains,
3128 old_write_domain);
3129
2ef7eeaa
EA		 3130 return 0;
3131}
3132
673a394b 3133/**
e47c68e9 3134 * Moves the object from a partially CPU read to a full one.
673a394b 3135 *
e47c68e9
EA		 3136 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3137 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 3138 */
e47c68e9 3139static void
05394f39 3140i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
673a394b 3141{
05394f39 3142 if (!obj->page_cpu_valid)
e47c68e9
EA		 3143 return;
3144
3145 /* If we're partially in the CPU read domain, finish moving it in.
3146 */
05394f39 3147 if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 3148 int i;
3149
05394f39
CW		 3150 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
3151 if (obj->page_cpu_valid[i])
e47c68e9 3152 continue;
05394f39 3153 drm_clflush_pages(obj->pages + i, 1);
e47c68e9 3154 }
e47c68e9
EA		 3155 }
3156
3157 /* Free the page_cpu_valid mappings which are now stale, whether
3158 * or not we've got I915_GEM_DOMAIN_CPU.
3159 */
05394f39
CW		 3160 kfree(obj->page_cpu_valid);
3161 obj->page_cpu_valid = NULL;
e47c68e9
EA		 3162}
3163
3164/**
3165 * Set the CPU read domain on a range of the object.
3166 *
3167 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3168 * not entirely valid. The page_cpu_valid member of the object flags which
3169 * pages have been flushed, and will be respected by
3170 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3171 * of the whole object.
3172 *
3173 * This function returns when the move is complete, including waiting on
3174 * flushes to occur.
3175 */
3176static int
05394f39 3177i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
e47c68e9
EA		 3178 uint64_t offset, uint64_t size)
3179{
1c5d22f7 3180 uint32_t old_read_domains;
e47c68e9 3181 int i, ret;
673a394b 3182
05394f39 3183 if (offset == 0 && size == obj->base.size)
e47c68e9 3184 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 3185
88241785
CW		 3186 ret = i915_gem_object_flush_gpu_write_domain(obj);
3187 if (ret)
3188 return ret;
3189
ce453d81 3190 ret = i915_gem_object_wait_rendering(obj);
de18a29e 3191 if (ret)
6a47baa6 3192 return ret;
de18a29e 3193
e47c68e9
EA		 3194 i915_gem_object_flush_gtt_write_domain(obj);
3195
3196 /* If we're already fully in the CPU read domain, we're done. */
05394f39
CW		 3197 if (obj->page_cpu_valid == NULL &&
3198 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
e47c68e9 3199 return 0;
673a394b 3200
e47c68e9
EA		 3201 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3202 * newly adding I915_GEM_DOMAIN_CPU
3203 */
05394f39
CW		 3204 if (obj->page_cpu_valid == NULL) {
3205 obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
3206 GFP_KERNEL);
3207 if (obj->page_cpu_valid == NULL)
e47c68e9 3208 return -ENOMEM;
05394f39
CW		 3209 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3210 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
673a394b
EA		 3211
3212 /* Flush the cache on any pages that are still invalid from the CPU's
3213 * perspective.
3214 */
e47c68e9
EA		 3215 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3216 i++) {
05394f39 3217 if (obj->page_cpu_valid[i])
673a394b
EA		 3218 continue;
3219
05394f39 3220 drm_clflush_pages(obj->pages + i, 1);
673a394b 3221
05394f39 3222 obj->page_cpu_valid[i] = 1;
673a394b
EA		 3223 }
3224
e47c68e9
EA		 3225 /* It should now be out of any other write domains, and we can update
3226 * the domain values for our changes.
3227 */
05394f39 3228 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
e47c68e9 3229
05394f39
CW		 3230 old_read_domains = obj->base.read_domains;
3231 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
e47c68e9 3232
1c5d22f7
CW		 3233 trace_i915_gem_object_change_domain(obj,
3234 old_read_domains,
05394f39 3235 obj->base.write_domain);
1c5d22f7 3236
673a394b
EA		 3237 return 0;
3238}
3239
673a394b
EA		 3240/* Throttle our rendering by waiting until the ring has completed our requests
3241 * emitted over 20 msec ago.
3242 *
b962442e
EA		 3243 * Note that if we were to use the current jiffies each time around the loop,
3244 * we wouldn't escape the function with any frames outstanding if the time to
3245 * render a frame was over 20ms.
3246 *
673a394b
EA		 3247 * This should get us reasonable parallelism between CPU and GPU but also
3248 * relatively low latency when blocking on a particular request to finish.
3249 */
40a5f0de 3250static int
f787a5f5 3251i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
40a5f0de 3252{
f787a5f5
CW		 3253 struct drm_i915_private *dev_priv = dev->dev_private;
3254 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e 3255 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
f787a5f5
CW		 3256 struct drm_i915_gem_request *request;
3257 struct intel_ring_buffer *ring = NULL;
3258 u32 seqno = 0;
3259 int ret;
93533c29 3260
e110e8d6
CW		 3261 if (atomic_read(&dev_priv->mm.wedged))
3262 return -EIO;
3263
1c25595f 3264 spin_lock(&file_priv->mm.lock);
f787a5f5 3265 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
b962442e
EA		 3266 if (time_after_eq(request->emitted_jiffies, recent_enough))
3267 break;
40a5f0de 3268
f787a5f5
CW		 3269 ring = request->ring;
3270 seqno = request->seqno;
b962442e 3271 }
1c25595f 3272 spin_unlock(&file_priv->mm.lock);
40a5f0de 3273
f787a5f5
CW		 3274 if (seqno == 0)
3275 return 0;
2bc43b5c 3276
f787a5f5 3277 ret = 0;
78501eac 3278 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
f787a5f5
CW		 3279 /* And wait for the seqno passing without holding any locks and
3280 * causing extra latency for others. This is safe as the irq
3281 * generation is designed to be run atomically and so is
3282 * lockless.
3283 */
b13c2b96
CW		 3284 if (ring->irq_get(ring)) {
3285 ret = wait_event_interruptible(ring->irq_queue,
3286 i915_seqno_passed(ring->get_seqno(ring), seqno)
3287 || atomic_read(&dev_priv->mm.wedged));
3288 ring->irq_put(ring);
40a5f0de 3289
b13c2b96
CW		 3290 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3291 ret = -EIO;
3292 }
40a5f0de
EA		 3293 }
3294
f787a5f5
CW		 3295 if (ret == 0)
3296 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
40a5f0de
EA		 3297
3298 return ret;
3299}
3300
673a394b 3301int
05394f39
CW		 3302i915_gem_object_pin(struct drm_i915_gem_object *obj,
3303 uint32_t alignment,
75e9e915 3304 bool map_and_fenceable)
673a394b 3305{
05394f39 3306 struct drm_device *dev = obj->base.dev;
f13d3f73 3307 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 3308 int ret;
3309
05394f39 3310 BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
23bc5982 3311 WARN_ON(i915_verify_lists(dev));
ac0c6b5a 3312
05394f39
CW		 3313 if (obj->gtt_space != NULL) {
3314 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3315 (map_and_fenceable && !obj->map_and_fenceable)) {
3316 WARN(obj->pin_count,
ae7d49d8 3317 "bo is already pinned with incorrect alignment:"
75e9e915
DV		 3318 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3319 " obj->map_and_fenceable=%d\n",
05394f39 3320 obj->gtt_offset, alignment,
75e9e915 3321 map_and_fenceable,
05394f39 3322 obj->map_and_fenceable);
ac0c6b5a
CW		 3323 ret = i915_gem_object_unbind(obj);
3324 if (ret)
3325 return ret;
3326 }
3327 }
3328
05394f39 3329 if (obj->gtt_space == NULL) {
a00b10c3 3330 ret = i915_gem_object_bind_to_gtt(obj, alignment,
75e9e915 3331 map_and_fenceable);
9731129c 3332 if (ret)
673a394b 3333 return ret;
22c344e9 3334 }
76446cac 3335
05394f39 3336 if (obj->pin_count++ == 0) {
05394f39
CW		 3337 if (!obj->active)
3338 list_move_tail(&obj->mm_list,
f13d3f73 3339 &dev_priv->mm.pinned_list);
673a394b 3340 }
6299f992 3341 obj->pin_mappable |= map_and_fenceable;
673a394b 3342
23bc5982 3343 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 3344 return 0;
3345}
3346
3347void
05394f39 3348i915_gem_object_unpin(struct drm_i915_gem_object *obj)
673a394b 3349{
05394f39 3350 struct drm_device *dev = obj->base.dev;
673a394b 3351 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 3352
23bc5982 3353 WARN_ON(i915_verify_lists(dev));
05394f39
CW		 3354 BUG_ON(obj->pin_count == 0);
3355 BUG_ON(obj->gtt_space == NULL);
673a394b 3356
05394f39
CW		 3357 if (--obj->pin_count == 0) {
3358 if (!obj->active)
3359 list_move_tail(&obj->mm_list,
673a394b 3360 &dev_priv->mm.inactive_list);
6299f992 3361 obj->pin_mappable = false;
673a394b 3362 }
23bc5982 3363 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 3364}
3365
3366int
3367i915_gem_pin_ioctl(struct drm_device *dev, void *data,
05394f39 3368 struct drm_file *file)
673a394b
EA		 3369{
3370 struct drm_i915_gem_pin *args = data;
05394f39 3371 struct drm_i915_gem_object *obj;
673a394b
EA		 3372 int ret;
3373
1d7cfea1
CW		 3374 ret = i915_mutex_lock_interruptible(dev);
3375 if (ret)
3376 return ret;
673a394b 3377
05394f39 3378 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 3379 if (&obj->base == NULL) {
1d7cfea1
CW		 3380 ret = -ENOENT;
3381 goto unlock;
673a394b 3382 }
673a394b 3383
05394f39 3384 if (obj->madv != I915_MADV_WILLNEED) {
bb6baf76 3385 DRM_ERROR("Attempting to pin a purgeable buffer\n");
1d7cfea1
CW		 3386 ret = -EINVAL;
3387 goto out;
3ef94daa
CW		 3388 }
3389
05394f39 3390 if (obj->pin_filp != NULL && obj->pin_filp != file) {
79e53945
JB		 3391 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3392 args->handle);
1d7cfea1
CW		 3393 ret = -EINVAL;
3394 goto out;
79e53945
JB		 3395 }
3396
05394f39
CW		 3397 obj->user_pin_count++;
3398 obj->pin_filp = file;
3399 if (obj->user_pin_count == 1) {
75e9e915 3400 ret = i915_gem_object_pin(obj, args->alignment, true);
1d7cfea1
CW		 3401 if (ret)
3402 goto out;
673a394b
EA		 3403 }
3404
3405 /* XXX - flush the CPU caches for pinned objects
3406 * as the X server doesn't manage domains yet
3407 */
e47c68e9 3408 i915_gem_object_flush_cpu_write_domain(obj);
05394f39 3409 args->offset = obj->gtt_offset;
1d7cfea1 3410out:
05394f39 3411 drm_gem_object_unreference(&obj->base);
1d7cfea1 3412unlock:
673a394b 3413 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3414 return ret;
673a394b
EA		 3415}
3416
3417int
3418i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
05394f39 3419 struct drm_file *file)
673a394b
EA		 3420{
3421 struct drm_i915_gem_pin *args = data;
05394f39 3422 struct drm_i915_gem_object *obj;
76c1dec1 3423 int ret;
673a394b 3424
1d7cfea1
CW		 3425 ret = i915_mutex_lock_interruptible(dev);
3426 if (ret)
3427 return ret;
673a394b 3428
05394f39 3429 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 3430 if (&obj->base == NULL) {
1d7cfea1
CW		 3431 ret = -ENOENT;
3432 goto unlock;
673a394b 3433 }
76c1dec1 3434
05394f39 3435 if (obj->pin_filp != file) {
79e53945
JB		 3436 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3437 args->handle);
1d7cfea1
CW		 3438 ret = -EINVAL;
3439 goto out;
79e53945 3440 }
05394f39
CW		 3441 obj->user_pin_count--;
3442 if (obj->user_pin_count == 0) {
3443 obj->pin_filp = NULL;
79e53945
JB		 3444 i915_gem_object_unpin(obj);
3445 }
673a394b 3446
1d7cfea1 3447out:
05394f39 3448 drm_gem_object_unreference(&obj->base);
1d7cfea1 3449unlock:
673a394b 3450 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3451 return ret;
673a394b
EA		 3452}
3453
3454int
3455i915_gem_busy_ioctl(struct drm_device *dev, void *data,
05394f39 3456 struct drm_file *file)
673a394b
EA		 3457{
3458 struct drm_i915_gem_busy *args = data;
05394f39 3459 struct drm_i915_gem_object *obj;
30dbf0c0
CW		 3460 int ret;
3461
76c1dec1 3462 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 3463 if (ret)
76c1dec1 3464 return ret;
673a394b 3465
05394f39 3466 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
c8725226 3467 if (&obj->base == NULL) {
1d7cfea1
CW		 3468 ret = -ENOENT;
3469 goto unlock;
673a394b 3470 }
d1b851fc 3471
0be555b6
CW		 3472 /* Count all active objects as busy, even if they are currently not used
3473 * by the gpu. Users of this interface expect objects to eventually
3474 * become non-busy without any further actions, therefore emit any
3475 * necessary flushes here.
c4de0a5d 3476 */
05394f39 3477 args->busy = obj->active;
0be555b6
CW		 3478 if (args->busy) {
3479 /* Unconditionally flush objects, even when the gpu still uses this
3480 * object. Userspace calling this function indicates that it wants to
3481 * use this buffer rather sooner than later, so issuing the required
3482 * flush earlier is beneficial.
3483 */
1a1c6976 3484 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
db53a302 3485 ret = i915_gem_flush_ring(obj->ring,
88241785 3486 0, obj->base.write_domain);
1a1c6976
CW		 3487 } else if (obj->ring->outstanding_lazy_request ==
3488 obj->last_rendering_seqno) {
3489 struct drm_i915_gem_request *request;
3490
7a194876
CW		 3491 /* This ring is not being cleared by active usage,
3492 * so emit a request to do so.
3493 */
1a1c6976
CW		 3494 request = kzalloc(sizeof(*request), GFP_KERNEL);
3495 if (request)
db53a302 3496 ret = i915_add_request(obj->ring, NULL,request);
1a1c6976 3497 else
7a194876
CW		 3498 ret = -ENOMEM;
3499 }
0be555b6
CW		 3500
3501 /* Update the active list for the hardware's current position.
3502 * Otherwise this only updates on a delayed timer or when irqs
3503 * are actually unmasked, and our working set ends up being
3504 * larger than required.
3505 */
db53a302 3506 i915_gem_retire_requests_ring(obj->ring);
0be555b6 3507
05394f39 3508 args->busy = obj->active;
0be555b6 3509 }
673a394b 3510
05394f39 3511 drm_gem_object_unreference(&obj->base);
1d7cfea1 3512unlock:
673a394b 3513 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3514 return ret;
673a394b
EA		 3515}
3516
3517int
3518i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3519 struct drm_file *file_priv)
3520{
3521 return i915_gem_ring_throttle(dev, file_priv);
3522}
3523
3ef94daa
CW		 3524int
3525i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3526 struct drm_file *file_priv)
3527{
3528 struct drm_i915_gem_madvise *args = data;
05394f39 3529 struct drm_i915_gem_object *obj;
76c1dec1 3530 int ret;
3ef94daa
CW		 3531
3532 switch (args->madv) {
3533 case I915_MADV_DONTNEED:
3534 case I915_MADV_WILLNEED:
3535 break;
3536 default:
3537 return -EINVAL;
3538 }
3539
1d7cfea1
CW		 3540 ret = i915_mutex_lock_interruptible(dev);
3541 if (ret)
3542 return ret;
3543
05394f39 3544 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
c8725226 3545 if (&obj->base == NULL) {
1d7cfea1
CW		 3546 ret = -ENOENT;
3547 goto unlock;
3ef94daa 3548 }
3ef94daa 3549
05394f39 3550 if (obj->pin_count) {
1d7cfea1
CW		 3551 ret = -EINVAL;
3552 goto out;
3ef94daa
CW		 3553 }
3554
05394f39
CW		 3555 if (obj->madv != __I915_MADV_PURGED)
3556 obj->madv = args->madv;
3ef94daa 3557
2d7ef395 3558 /* if the object is no longer bound, discard its backing storage */
05394f39
CW		 3559 if (i915_gem_object_is_purgeable(obj) &&
3560 obj->gtt_space == NULL)
2d7ef395
CW		 3561 i915_gem_object_truncate(obj);
3562
05394f39 3563 args->retained = obj->madv != __I915_MADV_PURGED;
bb6baf76 3564
1d7cfea1 3565out:
05394f39 3566 drm_gem_object_unreference(&obj->base);
1d7cfea1 3567unlock:
3ef94daa 3568 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3569 return ret;
3ef94daa
CW		 3570}
3571
05394f39
CW		 3572struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3573 size_t size)
ac52bc56 3574{
73aa808f 3575 struct drm_i915_private *dev_priv = dev->dev_private;
c397b908 3576 struct drm_i915_gem_object *obj;
ac52bc56 3577
c397b908
DV		 3578 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3579 if (obj == NULL)
3580 return NULL;
673a394b 3581
c397b908
DV		 3582 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3583 kfree(obj);
3584 return NULL;
3585 }
673a394b 3586
73aa808f
CW		 3587 i915_gem_info_add_obj(dev_priv, size);
3588
c397b908
DV		 3589 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3590 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
673a394b 3591
93dfb40c 3592 obj->cache_level = I915_CACHE_NONE;
62b8b215 3593 obj->base.driver_private = NULL;
c397b908 3594 obj->fence_reg = I915_FENCE_REG_NONE;
69dc4987 3595 INIT_LIST_HEAD(&obj->mm_list);
93a37f20 3596 INIT_LIST_HEAD(&obj->gtt_list);
69dc4987 3597 INIT_LIST_HEAD(&obj->ring_list);
432e58ed 3598 INIT_LIST_HEAD(&obj->exec_list);
c397b908 3599 INIT_LIST_HEAD(&obj->gpu_write_list);
c397b908 3600 obj->madv = I915_MADV_WILLNEED;
75e9e915
DV		 3601 /* Avoid an unnecessary call to unbind on the first bind. */
3602 obj->map_and_fenceable = true;
de151cf6 3603
05394f39 3604 return obj;
c397b908
DV		 3605}
3606
3607int i915_gem_init_object(struct drm_gem_object *obj)
3608{
3609 BUG();
de151cf6 3610
673a394b
EA		 3611 return 0;
3612}
3613
05394f39 3614static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
673a394b 3615{
05394f39 3616 struct drm_device *dev = obj->base.dev;
be72615b 3617 drm_i915_private_t *dev_priv = dev->dev_private;
be72615b 3618 int ret;
673a394b 3619
be72615b
CW		 3620 ret = i915_gem_object_unbind(obj);
3621 if (ret == -ERESTARTSYS) {
05394f39 3622 list_move(&obj->mm_list,
be72615b
CW		 3623 &dev_priv->mm.deferred_free_list);
3624 return;
3625 }
673a394b 3626
26e12f89
CW		 3627 trace_i915_gem_object_destroy(obj);
3628
05394f39 3629 if (obj->base.map_list.map)
7e616158 3630 i915_gem_free_mmap_offset(obj);
de151cf6 3631
05394f39
CW		 3632 drm_gem_object_release(&obj->base);
3633 i915_gem_info_remove_obj(dev_priv, obj->base.size);
c397b908 3634
05394f39
CW		 3635 kfree(obj->page_cpu_valid);
3636 kfree(obj->bit_17);
3637 kfree(obj);
673a394b
EA		 3638}
3639
05394f39 3640void i915_gem_free_object(struct drm_gem_object *gem_obj)
be72615b 3641{
05394f39
CW		 3642 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3643 struct drm_device *dev = obj->base.dev;
be72615b 3644
05394f39 3645 while (obj->pin_count > 0)
be72615b
CW		 3646 i915_gem_object_unpin(obj);
3647
05394f39 3648 if (obj->phys_obj)
be72615b
CW		 3649 i915_gem_detach_phys_object(dev, obj);
3650
3651 i915_gem_free_object_tail(obj);
3652}
3653
29105ccc
CW		 3654int
3655i915_gem_idle(struct drm_device *dev)
3656{
3657 drm_i915_private_t *dev_priv = dev->dev_private;
3658 int ret;
28dfe52a 3659
29105ccc 3660 mutex_lock(&dev->struct_mutex);
1c5d22f7 3661
87acb0a5 3662 if (dev_priv->mm.suspended) {
29105ccc
CW		 3663 mutex_unlock(&dev->struct_mutex);
3664 return 0;
28dfe52a
EA		 3665 }
3666
29105ccc 3667 ret = i915_gpu_idle(dev);
6dbe2772
KP		 3668 if (ret) {
3669 mutex_unlock(&dev->struct_mutex);
673a394b 3670 return ret;
6dbe2772 3671 }
673a394b 3672
29105ccc
CW		 3673 /* Under UMS, be paranoid and evict. */
3674 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
5eac3ab4 3675 ret = i915_gem_evict_inactive(dev, false);
29105ccc
CW		 3676 if (ret) {
3677 mutex_unlock(&dev->struct_mutex);
3678 return ret;
3679 }
3680 }
3681
312817a3
CW		 3682 i915_gem_reset_fences(dev);
3683
29105ccc
CW		 3684 /* Hack! Don't let anybody do execbuf while we don't control the chip.
3685 * We need to replace this with a semaphore, or something.
3686 * And not confound mm.suspended!
3687 */
3688 dev_priv->mm.suspended = 1;
bc0c7f14 3689 del_timer_sync(&dev_priv->hangcheck_timer);
29105ccc
CW		 3690
3691 i915_kernel_lost_context(dev);
6dbe2772 3692 i915_gem_cleanup_ringbuffer(dev);
29105ccc 3693
6dbe2772
KP		 3694 mutex_unlock(&dev->struct_mutex);
3695
29105ccc
CW		 3696 /* Cancel the retire work handler, which should be idle now. */
3697 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3698
673a394b
EA		 3699 return 0;
3700}
3701
8187a2b7
ZN		 3702int
3703i915_gem_init_ringbuffer(struct drm_device *dev)
3704{
3705 drm_i915_private_t *dev_priv = dev->dev_private;
3706 int ret;
68f95ba9 3707
5c1143bb 3708 ret = intel_init_render_ring_buffer(dev);
68f95ba9 3709 if (ret)
b6913e4b 3710 return ret;
68f95ba9
CW		 3711
3712 if (HAS_BSD(dev)) {
5c1143bb 3713 ret = intel_init_bsd_ring_buffer(dev);
68f95ba9
CW		 3714 if (ret)
3715 goto cleanup_render_ring;
d1b851fc 3716 }
68f95ba9 3717
549f7365
CW		 3718 if (HAS_BLT(dev)) {
3719 ret = intel_init_blt_ring_buffer(dev);
3720 if (ret)
3721 goto cleanup_bsd_ring;
3722 }
3723
6f392d54
CW		 3724 dev_priv->next_seqno = 1;
3725
68f95ba9
CW		 3726 return 0;
3727
549f7365 3728cleanup_bsd_ring:
1ec14ad3 3729 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
68f95ba9 3730cleanup_render_ring:
1ec14ad3 3731 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
8187a2b7
ZN		 3732 return ret;
3733}
3734
3735void
3736i915_gem_cleanup_ringbuffer(struct drm_device *dev)
3737{
3738 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 3739 int i;
8187a2b7 3740
1ec14ad3
CW		 3741 for (i = 0; i < I915_NUM_RINGS; i++)
3742 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
8187a2b7
ZN		 3743}
3744
673a394b
EA		 3745int
3746i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
3747 struct drm_file *file_priv)
3748{
3749 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 3750 int ret, i;
673a394b 3751
79e53945
JB		 3752 if (drm_core_check_feature(dev, DRIVER_MODESET))
3753 return 0;
3754
ba1234d1 3755 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 3756 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 3757 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 3758 }
3759
673a394b 3760 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 3761 dev_priv->mm.suspended = 0;
3762
3763 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 3764 if (ret != 0) {
3765 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 3766 return ret;
d816f6ac 3767 }
9bb2d6f9 3768
69dc4987 3769 BUG_ON(!list_empty(&dev_priv->mm.active_list));
673a394b
EA		 3770 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
3771 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
1ec14ad3
CW		 3772 for (i = 0; i < I915_NUM_RINGS; i++) {
3773 BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
3774 BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
3775 }
673a394b 3776 mutex_unlock(&dev->struct_mutex);
dbb19d30 3777
5f35308b
CW		 3778 ret = drm_irq_install(dev);
3779 if (ret)
3780 goto cleanup_ringbuffer;
dbb19d30 3781
673a394b 3782 return 0;
5f35308b
CW		 3783
3784cleanup_ringbuffer:
3785 mutex_lock(&dev->struct_mutex);
3786 i915_gem_cleanup_ringbuffer(dev);
3787 dev_priv->mm.suspended = 1;
3788 mutex_unlock(&dev->struct_mutex);
3789
3790 return ret;
673a394b
EA		 3791}
3792
3793int
3794i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
3795 struct drm_file *file_priv)
3796{
79e53945
JB		 3797 if (drm_core_check_feature(dev, DRIVER_MODESET))
3798 return 0;
3799
dbb19d30 3800 drm_irq_uninstall(dev);
e6890f6f 3801 return i915_gem_idle(dev);
673a394b
EA		 3802}
3803
3804void
3805i915_gem_lastclose(struct drm_device *dev)
3806{
3807 int ret;
673a394b 3808
e806b495
EA		 3809 if (drm_core_check_feature(dev, DRIVER_MODESET))
3810 return;
3811
6dbe2772
KP		 3812 ret = i915_gem_idle(dev);
3813 if (ret)
3814 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 3815}
3816
64193406
CW		 3817static void
3818init_ring_lists(struct intel_ring_buffer *ring)
3819{
3820 INIT_LIST_HEAD(&ring->active_list);
3821 INIT_LIST_HEAD(&ring->request_list);
3822 INIT_LIST_HEAD(&ring->gpu_write_list);
3823}
3824
673a394b
EA		 3825void
3826i915_gem_load(struct drm_device *dev)
3827{
b5aa8a0f 3828 int i;
673a394b
EA		 3829 drm_i915_private_t *dev_priv = dev->dev_private;
3830
69dc4987 3831 INIT_LIST_HEAD(&dev_priv->mm.active_list);
673a394b
EA		 3832 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3833 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
f13d3f73 3834 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
a09ba7fa 3835 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
be72615b 3836 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
93a37f20 3837 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
1ec14ad3
CW		 3838 for (i = 0; i < I915_NUM_RINGS; i++)
3839 init_ring_lists(&dev_priv->ring[i]);
007cc8ac
DV		 3840 for (i = 0; i < 16; i++)
3841 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
673a394b
EA		 3842 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3843 i915_gem_retire_work_handler);
30dbf0c0 3844 init_completion(&dev_priv->error_completion);
31169714 3845
94400120
DA		 3846 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3847 if (IS_GEN3(dev)) {
3848 u32 tmp = I915_READ(MI_ARB_STATE);
3849 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
3850 /* arb state is a masked write, so set bit + bit in mask */
3851 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
3852 I915_WRITE(MI_ARB_STATE, tmp);
3853 }
3854 }
3855
72bfa19c
CW		 3856 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3857
de151cf6 3858 /* Old X drivers will take 0-2 for front, back, depth buffers */
b397c836
EA		 3859 if (!drm_core_check_feature(dev, DRIVER_MODESET))
3860 dev_priv->fence_reg_start = 3;
de151cf6 3861
a6c45cf0 3862 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 3863 dev_priv->num_fence_regs = 16;
3864 else
3865 dev_priv->num_fence_regs = 8;
3866
b5aa8a0f 3867 /* Initialize fence registers to zero */
10ed13e4
EA		 3868 for (i = 0; i < dev_priv->num_fence_regs; i++) {
3869 i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
b5aa8a0f 3870 }
10ed13e4 3871
673a394b 3872 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 3873 init_waitqueue_head(&dev_priv->pending_flip_queue);
17250b71 3874
ce453d81
CW		 3875 dev_priv->mm.interruptible = true;
3876
17250b71
CW		 3877 dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3878 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3879 register_shrinker(&dev_priv->mm.inactive_shrinker);
673a394b 3880}
71acb5eb
DA		 3881
3882/*
3883 * Create a physically contiguous memory object for this object
3884 * e.g. for cursor + overlay regs
3885 */
995b6762
CW		 3886static int i915_gem_init_phys_object(struct drm_device *dev,
3887 int id, int size, int align)
71acb5eb
DA		 3888{
3889 drm_i915_private_t *dev_priv = dev->dev_private;
3890 struct drm_i915_gem_phys_object *phys_obj;
3891 int ret;
3892
3893 if (dev_priv->mm.phys_objs[id - 1] || !size)
3894 return 0;
3895
9a298b2a 3896 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 3897 if (!phys_obj)
3898 return -ENOMEM;
3899
3900 phys_obj->id = id;
3901
6eeefaf3 3902 phys_obj->handle = drm_pci_alloc(dev, size, align);
71acb5eb
DA		 3903 if (!phys_obj->handle) {
3904 ret = -ENOMEM;
3905 goto kfree_obj;
3906 }
3907#ifdef CONFIG_X86
3908 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3909#endif
3910
3911 dev_priv->mm.phys_objs[id - 1] = phys_obj;
3912
3913 return 0;
3914kfree_obj:
9a298b2a 3915 kfree(phys_obj);
71acb5eb
DA		 3916 return ret;
3917}
3918
995b6762 3919static void i915_gem_free_phys_object(struct drm_device *dev, int id)
71acb5eb
DA		 3920{
3921 drm_i915_private_t *dev_priv = dev->dev_private;
3922 struct drm_i915_gem_phys_object *phys_obj;
3923
3924 if (!dev_priv->mm.phys_objs[id - 1])
3925 return;
3926
3927 phys_obj = dev_priv->mm.phys_objs[id - 1];
3928 if (phys_obj->cur_obj) {
3929 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3930 }
3931
3932#ifdef CONFIG_X86
3933 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3934#endif
3935 drm_pci_free(dev, phys_obj->handle);
3936 kfree(phys_obj);
3937 dev_priv->mm.phys_objs[id - 1] = NULL;
3938}
3939
3940void i915_gem_free_all_phys_object(struct drm_device *dev)
3941{
3942 int i;
3943
260883c8 3944 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 3945 i915_gem_free_phys_object(dev, i);
3946}
3947
3948void i915_gem_detach_phys_object(struct drm_device *dev,
05394f39 3949 struct drm_i915_gem_object *obj)
71acb5eb 3950{
05394f39 3951 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
e5281ccd 3952 char *vaddr;
71acb5eb 3953 int i;
71acb5eb
DA		 3954 int page_count;
3955
05394f39 3956 if (!obj->phys_obj)
71acb5eb 3957 return;
05394f39 3958 vaddr = obj->phys_obj->handle->vaddr;
71acb5eb 3959
05394f39 3960 page_count = obj->base.size / PAGE_SIZE;
71acb5eb 3961 for (i = 0; i < page_count; i++) {
e5281ccd
CW		 3962 struct page *page = read_cache_page_gfp(mapping, i,
3963 GFP_HIGHUSER | __GFP_RECLAIMABLE);
3964 if (!IS_ERR(page)) {
3965 char *dst = kmap_atomic(page);
3966 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
3967 kunmap_atomic(dst);
3968
3969 drm_clflush_pages(&page, 1);
3970
3971 set_page_dirty(page);
3972 mark_page_accessed(page);
3973 page_cache_release(page);
3974 }
71acb5eb 3975 }
40ce6575 3976 intel_gtt_chipset_flush();
d78b47b9 3977
05394f39
CW		 3978 obj->phys_obj->cur_obj = NULL;
3979 obj->phys_obj = NULL;
71acb5eb
DA		 3980}
3981
3982int
3983i915_gem_attach_phys_object(struct drm_device *dev,
05394f39 3984 struct drm_i915_gem_object *obj,
6eeefaf3
CW		 3985 int id,
3986 int align)
71acb5eb 3987{
05394f39 3988 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
71acb5eb 3989 drm_i915_private_t *dev_priv = dev->dev_private;
71acb5eb
DA		 3990 int ret = 0;
3991 int page_count;
3992 int i;
3993
3994 if (id > I915_MAX_PHYS_OBJECT)
3995 return -EINVAL;
3996
05394f39
CW		 3997 if (obj->phys_obj) {
3998 if (obj->phys_obj->id == id)
71acb5eb
DA		 3999 return 0;
4000 i915_gem_detach_phys_object(dev, obj);
4001 }
4002
71acb5eb
DA		 4003 /* create a new object */
4004 if (!dev_priv->mm.phys_objs[id - 1]) {
4005 ret = i915_gem_init_phys_object(dev, id,
05394f39 4006 obj->base.size, align);
71acb5eb 4007 if (ret) {
05394f39
CW		 4008 DRM_ERROR("failed to init phys object %d size: %zu\n",
4009 id, obj->base.size);
e5281ccd 4010 return ret;
71acb5eb
DA		 4011 }
4012 }
4013
4014 /* bind to the object */
05394f39
CW		 4015 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4016 obj->phys_obj->cur_obj = obj;
71acb5eb 4017
05394f39 4018 page_count = obj->base.size / PAGE_SIZE;
71acb5eb
DA		 4019
4020 for (i = 0; i < page_count; i++) {
e5281ccd
CW		 4021 struct page *page;
4022 char *dst, *src;
4023
4024 page = read_cache_page_gfp(mapping, i,
4025 GFP_HIGHUSER | __GFP_RECLAIMABLE);
4026 if (IS_ERR(page))
4027 return PTR_ERR(page);
71acb5eb 4028
ff75b9bc 4029 src = kmap_atomic(page);
05394f39 4030 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
71acb5eb 4031 memcpy(dst, src, PAGE_SIZE);
3e4d3af5 4032 kunmap_atomic(src);
71acb5eb 4033
e5281ccd
CW		 4034 mark_page_accessed(page);
4035 page_cache_release(page);
4036 }
d78b47b9 4037
71acb5eb 4038 return 0;
71acb5eb
DA		 4039}
4040
4041static int
05394f39
CW		 4042i915_gem_phys_pwrite(struct drm_device *dev,
4043 struct drm_i915_gem_object *obj,
71acb5eb
DA		 4044 struct drm_i915_gem_pwrite *args,
4045 struct drm_file *file_priv)
4046{
05394f39 4047 void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
b47b30cc 4048 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
71acb5eb 4049
b47b30cc
CW		 4050 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4051 unsigned long unwritten;
4052
4053 /* The physical object once assigned is fixed for the lifetime
4054 * of the obj, so we can safely drop the lock and continue
4055 * to access vaddr.
4056 */
4057 mutex_unlock(&dev->struct_mutex);
4058 unwritten = copy_from_user(vaddr, user_data, args->size);
4059 mutex_lock(&dev->struct_mutex);
4060 if (unwritten)
4061 return -EFAULT;
4062 }
71acb5eb 4063
40ce6575 4064 intel_gtt_chipset_flush();
71acb5eb
DA		 4065 return 0;
4066}
b962442e 4067
f787a5f5 4068void i915_gem_release(struct drm_device *dev, struct drm_file *file)
b962442e 4069{
f787a5f5 4070 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e
EA		 4071
4072 /* Clean up our request list when the client is going away, so that
4073 * later retire_requests won't dereference our soon-to-be-gone
4074 * file_priv.
4075 */
1c25595f 4076 spin_lock(&file_priv->mm.lock);
f787a5f5
CW		 4077 while (!list_empty(&file_priv->mm.request_list)) {
4078 struct drm_i915_gem_request *request;
4079
4080 request = list_first_entry(&file_priv->mm.request_list,
4081 struct drm_i915_gem_request,
4082 client_list);
4083 list_del(&request->client_list);
4084 request->file_priv = NULL;
4085 }
1c25595f 4086 spin_unlock(&file_priv->mm.lock);
b962442e 4087}
31169714 4088
1637ef41
CW		 4089static int
4090i915_gpu_is_active(struct drm_device *dev)
4091{
4092 drm_i915_private_t *dev_priv = dev->dev_private;
4093 int lists_empty;
4094
1637ef41 4095 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
17250b71 4096 list_empty(&dev_priv->mm.active_list);
1637ef41
CW		 4097
4098 return !lists_empty;
4099}
4100
31169714 4101static int
1495f230 4102i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
31169714 4103{
17250b71
CW		 4104 struct drm_i915_private *dev_priv =
4105 container_of(shrinker,
4106 struct drm_i915_private,
4107 mm.inactive_shrinker);
4108 struct drm_device *dev = dev_priv->dev;
4109 struct drm_i915_gem_object *obj, *next;
1495f230 4110 int nr_to_scan = sc->nr_to_scan;
17250b71
CW		 4111 int cnt;
4112
4113 if (!mutex_trylock(&dev->struct_mutex))
bbe2e11a 4114 return 0;
31169714
CW		 4115
4116 /* "fast-path" to count number of available objects */
4117 if (nr_to_scan == 0) {
17250b71
CW		 4118 cnt = 0;
4119 list_for_each_entry(obj,
4120 &dev_priv->mm.inactive_list,
4121 mm_list)
4122 cnt++;
4123 mutex_unlock(&dev->struct_mutex);
4124 return cnt / 100 * sysctl_vfs_cache_pressure;
31169714
CW		 4125 }
4126
1637ef41 4127rescan:
31169714 4128 /* first scan for clean buffers */
17250b71 4129 i915_gem_retire_requests(dev);
31169714 4130
17250b71
CW		 4131 list_for_each_entry_safe(obj, next,
4132 &dev_priv->mm.inactive_list,
4133 mm_list) {
4134 if (i915_gem_object_is_purgeable(obj)) {
2021746e
CW		 4135 if (i915_gem_object_unbind(obj) == 0 &&
4136 --nr_to_scan == 0)
17250b71 4137 break;
31169714 4138 }
31169714
CW		 4139 }
4140
4141 /* second pass, evict/count anything still on the inactive list */
17250b71
CW		 4142 cnt = 0;
4143 list_for_each_entry_safe(obj, next,
4144 &dev_priv->mm.inactive_list,
4145 mm_list) {
2021746e
CW		 4146 if (nr_to_scan &&
4147 i915_gem_object_unbind(obj) == 0)
17250b71 4148 nr_to_scan--;
2021746e 4149 else
17250b71
CW		 4150 cnt++;
4151 }
4152
4153 if (nr_to_scan && i915_gpu_is_active(dev)) {
1637ef41
CW		 4154 /*
4155 * We are desperate for pages, so as a last resort, wait
4156 * for the GPU to finish and discard whatever we can.
4157 * This has a dramatic impact to reduce the number of
4158 * OOM-killer events whilst running the GPU aggressively.
4159 */
17250b71 4160 if (i915_gpu_idle(dev) == 0)
1637ef41
CW		 4161 goto rescan;
4162 }
17250b71
CW		 4163 mutex_unlock(&dev->struct_mutex);
4164 return cnt / 100 * sysctl_vfs_cache_pressure;
31169714 4165}
Linux kernel - Deliverables
This page took 0.881347 seconds and 5 git commands to generate.