projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/nouveau: initial pass at moving to struct nvif_device
[deliverable/linux.git] / drivers / gpu / drm / nouveau / nouveau_bo.c
1 /*
2 * Copyright 2007 Dave Airlied
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 */
24 /*
25 * Authors: Dave Airlied <airlied@linux.ie>
26 * Ben Skeggs <darktama@iinet.net.au>
27 * Jeremy Kolb <jkolb@brandeis.edu>
28 */
29
30 #include <core/engine.h>
31 #include <linux/swiotlb.h>
32
33 #include <subdev/fb.h>
34 #include <subdev/vm.h>
35 #include <subdev/bar.h>
36
37 #include "nouveau_drm.h"
38 #include "nouveau_dma.h"
39 #include "nouveau_fence.h"
40
41 #include "nouveau_bo.h"
42 #include "nouveau_ttm.h"
43 #include "nouveau_gem.h"
44
45 /*
46 * NV10-NV40 tiling helpers
47 */
48
49 static void
50 nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg,
51 u32 addr, u32 size, u32 pitch, u32 flags)
52 {
53 struct nouveau_drm *drm = nouveau_drm(dev);
54 int i = reg - drm->tile.reg;
55 struct nouveau_fb *pfb = nvkm_fb(&drm->device);
56 struct nouveau_fb_tile *tile = &pfb->tile.region[i];
57 struct nouveau_engine *engine;
58
59 nouveau_fence_unref(&reg->fence);
60
61 if (tile->pitch)
62 pfb->tile.fini(pfb, i, tile);
63
64 if (pitch)
65 pfb->tile.init(pfb, i, addr, size, pitch, flags, tile);
66
67 pfb->tile.prog(pfb, i, tile);
68
69 if ((engine = nouveau_engine(pfb, NVDEV_ENGINE_GR)))
70 engine->tile_prog(engine, i);
71 if ((engine = nouveau_engine(pfb, NVDEV_ENGINE_MPEG)))
72 engine->tile_prog(engine, i);
73 }
74
75 static struct nouveau_drm_tile *
76 nv10_bo_get_tile_region(struct drm_device *dev, int i)
77 {
78 struct nouveau_drm *drm = nouveau_drm(dev);
79 struct nouveau_drm_tile *tile = &drm->tile.reg[i];
80
81 spin_lock(&drm->tile.lock);
82
83 if (!tile->used &&
84 (!tile->fence || nouveau_fence_done(tile->fence)))
85 tile->used = true;
86 else
87 tile = NULL;
88
89 spin_unlock(&drm->tile.lock);
90 return tile;
91 }
92
93 static void
94 nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile,
95 struct nouveau_fence *fence)
96 {
97 struct nouveau_drm *drm = nouveau_drm(dev);
98
99 if (tile) {
100 spin_lock(&drm->tile.lock);
101 tile->fence = nouveau_fence_ref(fence);
102 tile->used = false;
103 spin_unlock(&drm->tile.lock);
104 }
105 }
106
107 static struct nouveau_drm_tile *
108 nv10_bo_set_tiling(struct drm_device *dev, u32 addr,
109 u32 size, u32 pitch, u32 flags)
110 {
111 struct nouveau_drm *drm = nouveau_drm(dev);
112 struct nouveau_fb *pfb = nvkm_fb(&drm->device);
113 struct nouveau_drm_tile *tile, *found = NULL;
114 int i;
115
116 for (i = 0; i < pfb->tile.regions; i++) {
117 tile = nv10_bo_get_tile_region(dev, i);
118
119 if (pitch && !found) {
120 found = tile;
121 continue;
122
123 } else if (tile && pfb->tile.region[i].pitch) {
124 /* Kill an unused tile region. */
125 nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0);
126 }
127
128 nv10_bo_put_tile_region(dev, tile, NULL);
129 }
130
131 if (found)
132 nv10_bo_update_tile_region(dev, found, addr, size,
133 pitch, flags);
134 return found;
135 }
136
137 static void
138 nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
139 {
140 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
141 struct drm_device *dev = drm->dev;
142 struct nouveau_bo *nvbo = nouveau_bo(bo);
143
144 if (unlikely(nvbo->gem.filp))
145 DRM_ERROR("bo %p still attached to GEM object\n", bo);
146 WARN_ON(nvbo->pin_refcnt > 0);
147 nv10_bo_put_tile_region(dev, nvbo->tile, NULL);
148 kfree(nvbo);
149 }
150
151 static void
152 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
153 int *align, int *size)
154 {
155 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
156 struct nvif_device *device = &drm->device;
157
158 if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
159 if (nvbo->tile_mode) {
160 if (device->info.chipset >= 0x40) {
161 *align = 65536;
162 *size = roundup(*size, 64 * nvbo->tile_mode);
163
164 } else if (device->info.chipset >= 0x30) {
165 *align = 32768;
166 *size = roundup(*size, 64 * nvbo->tile_mode);
167
168 } else if (device->info.chipset >= 0x20) {
169 *align = 16384;
170 *size = roundup(*size, 64 * nvbo->tile_mode);
171
172 } else if (device->info.chipset >= 0x10) {
173 *align = 16384;
174 *size = roundup(*size, 32 * nvbo->tile_mode);
175 }
176 }
177 } else {
178 *size = roundup(*size, (1 << nvbo->page_shift));
179 *align = max((1 << nvbo->page_shift), *align);
180 }
181
182 *size = roundup(*size, PAGE_SIZE);
183 }
184
185 int
186 nouveau_bo_new(struct drm_device *dev, int size, int align,
187 uint32_t flags, uint32_t tile_mode, uint32_t tile_flags,
188 struct sg_table *sg,
189 struct nouveau_bo **pnvbo)
190 {
191 struct nouveau_drm *drm = nouveau_drm(dev);
192 struct nouveau_bo *nvbo;
193 size_t acc_size;
194 int ret;
195 int type = ttm_bo_type_device;
196 int lpg_shift = 12;
197 int max_size;
198
199 if (drm->client.vm)
200 lpg_shift = drm->client.vm->vmm->lpg_shift;
201 max_size = INT_MAX & ~((1 << lpg_shift) - 1);
202
203 if (size <= 0 || size > max_size) {
204 NV_WARN(drm, "skipped size %x\n", (u32)size);
205 return -EINVAL;
206 }
207
208 if (sg)
209 type = ttm_bo_type_sg;
210
211 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
212 if (!nvbo)
213 return -ENOMEM;
214 INIT_LIST_HEAD(&nvbo->head);
215 INIT_LIST_HEAD(&nvbo->entry);
216 INIT_LIST_HEAD(&nvbo->vma_list);
217 nvbo->tile_mode = tile_mode;
218 nvbo->tile_flags = tile_flags;
219 nvbo->bo.bdev = &drm->ttm.bdev;
220
221 nvbo->page_shift = 12;
222 if (drm->client.vm) {
223 if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024)
224 nvbo->page_shift = drm->client.vm->vmm->lpg_shift;
225 }
226
227 nouveau_bo_fixup_align(nvbo, flags, &align, &size);
228 nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
229 nouveau_bo_placement_set(nvbo, flags, 0);
230
231 acc_size = ttm_bo_dma_acc_size(&drm->ttm.bdev, size,
232 sizeof(struct nouveau_bo));
233
234 ret = ttm_bo_init(&drm->ttm.bdev, &nvbo->bo, size,
235 type, &nvbo->placement,
236 align >> PAGE_SHIFT, false, NULL, acc_size, sg,
237 nouveau_bo_del_ttm);
238 if (ret) {
239 /* ttm will call nouveau_bo_del_ttm if it fails.. */
240 return ret;
241 }
242
243 *pnvbo = nvbo;
244 return 0;
245 }
246
247 static void
248 set_placement_list(uint32_t *pl, unsigned *n, uint32_t type, uint32_t flags)
249 {
250 *n = 0;
251
252 if (type & TTM_PL_FLAG_VRAM)
253 pl[(*n)++] = TTM_PL_FLAG_VRAM | flags;
254 if (type & TTM_PL_FLAG_TT)
255 pl[(*n)++] = TTM_PL_FLAG_TT | flags;
256 if (type & TTM_PL_FLAG_SYSTEM)
257 pl[(*n)++] = TTM_PL_FLAG_SYSTEM | flags;
258 }
259
260 static void
261 set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
262 {
263 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
264 struct nouveau_fb *pfb = nvkm_fb(&drm->device);
265 u32 vram_pages = pfb->ram->size >> PAGE_SHIFT;
266
267 if (drm->device.info.family == NV_DEVICE_INFO_V0_CELSIUS &&
268 nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) &&
269 nvbo->bo.mem.num_pages < vram_pages / 4) {
270 /*
271 * Make sure that the color and depth buffers are handled
272 * by independent memory controller units. Up to a 9x
273 * speed up when alpha-blending and depth-test are enabled
274 * at the same time.
275 */
276 if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) {
277 nvbo->placement.fpfn = vram_pages / 2;
278 nvbo->placement.lpfn = ~0;
279 } else {
280 nvbo->placement.fpfn = 0;
281 nvbo->placement.lpfn = vram_pages / 2;
282 }
283 }
284 }
285
286 void
287 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
288 {
289 struct ttm_placement *pl = &nvbo->placement;
290 uint32_t flags = TTM_PL_MASK_CACHING |
291 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
292
293 pl->placement = nvbo->placements;
294 set_placement_list(nvbo->placements, &pl->num_placement,
295 type, flags);
296
297 pl->busy_placement = nvbo->busy_placements;
298 set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
299 type | busy, flags);
300
301 set_placement_range(nvbo, type);
302 }
303
304 int
305 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype)
306 {
307 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
308 struct ttm_buffer_object *bo = &nvbo->bo;
309 int ret;
310
311 ret = ttm_bo_reserve(bo, false, false, false, NULL);
312 if (ret)
313 goto out;
314
315 if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) {
316 NV_ERROR(drm, "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo,
317 1 << bo->mem.mem_type, memtype);
318 ret = -EINVAL;
319 goto out;
320 }
321
322 if (nvbo->pin_refcnt++)
323 goto out;
324
325 nouveau_bo_placement_set(nvbo, memtype, 0);
326
327 ret = nouveau_bo_validate(nvbo, false, false);
328 if (ret == 0) {
329 switch (bo->mem.mem_type) {
330 case TTM_PL_VRAM:
331 drm->gem.vram_available -= bo->mem.size;
332 break;
333 case TTM_PL_TT:
334 drm->gem.gart_available -= bo->mem.size;
335 break;
336 default:
337 break;
338 }
339 }
340 out:
341 ttm_bo_unreserve(bo);
342 return ret;
343 }
344
345 int
346 nouveau_bo_unpin(struct nouveau_bo *nvbo)
347 {
348 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
349 struct ttm_buffer_object *bo = &nvbo->bo;
350 int ret, ref;
351
352 ret = ttm_bo_reserve(bo, false, false, false, NULL);
353 if (ret)
354 return ret;
355
356 ref = --nvbo->pin_refcnt;
357 WARN_ON_ONCE(ref < 0);
358 if (ref)
359 goto out;
360
361 nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
362
363 ret = nouveau_bo_validate(nvbo, false, false);
364 if (ret == 0) {
365 switch (bo->mem.mem_type) {
366 case TTM_PL_VRAM:
367 drm->gem.vram_available += bo->mem.size;
368 break;
369 case TTM_PL_TT:
370 drm->gem.gart_available += bo->mem.size;
371 break;
372 default:
373 break;
374 }
375 }
376
377 out:
378 ttm_bo_unreserve(bo);
379 return ret;
380 }
381
382 int
383 nouveau_bo_map(struct nouveau_bo *nvbo)
384 {
385 int ret;
386
387 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL);
388 if (ret)
389 return ret;
390
391 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap);
392 ttm_bo_unreserve(&nvbo->bo);
393 return ret;
394 }
395
396 void
397 nouveau_bo_unmap(struct nouveau_bo *nvbo)
398 {
399 if (nvbo)
400 ttm_bo_kunmap(&nvbo->kmap);
401 }
402
403 int
404 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
405 bool no_wait_gpu)
406 {
407 int ret;
408
409 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement,
410 interruptible, no_wait_gpu);
411 if (ret)
412 return ret;
413
414 return 0;
415 }
416
417 u16
418 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index)
419 {
420 bool is_iomem;
421 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
422 mem = &mem[index];
423 if (is_iomem)
424 return ioread16_native((void __force __iomem *)mem);
425 else
426 return *mem;
427 }
428
429 void
430 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
431 {
432 bool is_iomem;
433 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
434 mem = &mem[index];
435 if (is_iomem)
436 iowrite16_native(val, (void __force __iomem *)mem);
437 else
438 *mem = val;
439 }
440
441 u32
442 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
443 {
444 bool is_iomem;
445 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
446 mem = &mem[index];
447 if (is_iomem)
448 return ioread32_native((void __force __iomem *)mem);
449 else
450 return *mem;
451 }
452
453 void
454 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
455 {
456 bool is_iomem;
457 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
458 mem = &mem[index];
459 if (is_iomem)
460 iowrite32_native(val, (void __force __iomem *)mem);
461 else
462 *mem = val;
463 }
464
465 static struct ttm_tt *
466 nouveau_ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size,
467 uint32_t page_flags, struct page *dummy_read)
468 {
469 #if __OS_HAS_AGP
470 struct nouveau_drm *drm = nouveau_bdev(bdev);
471 struct drm_device *dev = drm->dev;
472
473 if (drm->agp.stat == ENABLED) {
474 return ttm_agp_tt_create(bdev, dev->agp->bridge, size,
475 page_flags, dummy_read);
476 }
477 #endif
478
479 return nouveau_sgdma_create_ttm(bdev, size, page_flags, dummy_read);
480 }
481
482 static int
483 nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
484 {
485 /* We'll do this from user space. */
486 return 0;
487 }
488
489 static int
490 nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
491 struct ttm_mem_type_manager *man)
492 {
493 struct nouveau_drm *drm = nouveau_bdev(bdev);
494
495 switch (type) {
496 case TTM_PL_SYSTEM:
497 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
498 man->available_caching = TTM_PL_MASK_CACHING;
499 man->default_caching = TTM_PL_FLAG_CACHED;
500 break;
501 case TTM_PL_VRAM:
502 man->flags = TTM_MEMTYPE_FLAG_FIXED |
503 TTM_MEMTYPE_FLAG_MAPPABLE;
504 man->available_caching = TTM_PL_FLAG_UNCACHED |
505 TTM_PL_FLAG_WC;
506 man->default_caching = TTM_PL_FLAG_WC;
507
508 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
509 /* Some BARs do not support being ioremapped WC */
510 if (nvkm_bar(&drm->device)->iomap_uncached) {
511 man->available_caching = TTM_PL_FLAG_UNCACHED;
512 man->default_caching = TTM_PL_FLAG_UNCACHED;
513 }
514
515 man->func = &nouveau_vram_manager;
516 man->io_reserve_fastpath = false;
517 man->use_io_reserve_lru = true;
518 } else {
519 man->func = &ttm_bo_manager_func;
520 }
521 break;
522 case TTM_PL_TT:
523 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
524 man->func = &nouveau_gart_manager;
525 else
526 if (drm->agp.stat != ENABLED)
527 man->func = &nv04_gart_manager;
528 else
529 man->func = &ttm_bo_manager_func;
530
531 if (drm->agp.stat == ENABLED) {
532 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
533 man->available_caching = TTM_PL_FLAG_UNCACHED |
534 TTM_PL_FLAG_WC;
535 man->default_caching = TTM_PL_FLAG_WC;
536 } else {
537 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
538 TTM_MEMTYPE_FLAG_CMA;
539 man->available_caching = TTM_PL_MASK_CACHING;
540 man->default_caching = TTM_PL_FLAG_CACHED;
541 }
542
543 break;
544 default:
545 return -EINVAL;
546 }
547 return 0;
548 }
549
550 static void
551 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
552 {
553 struct nouveau_bo *nvbo = nouveau_bo(bo);
554
555 switch (bo->mem.mem_type) {
556 case TTM_PL_VRAM:
557 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT,
558 TTM_PL_FLAG_SYSTEM);
559 break;
560 default:
561 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0);
562 break;
563 }
564
565 *pl = nvbo->placement;
566 }
567
568
569 static int
570 nve0_bo_move_init(struct nouveau_channel *chan, u32 handle)
571 {
572 int ret = RING_SPACE(chan, 2);
573 if (ret == 0) {
574 BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
575 OUT_RING (chan, handle & 0x0000ffff);
576 FIRE_RING (chan);
577 }
578 return ret;
579 }
580
581 static int
582 nve0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
583 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
584 {
585 struct nouveau_mem *node = old_mem->mm_node;
586 int ret = RING_SPACE(chan, 10);
587 if (ret == 0) {
588 BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
589 OUT_RING (chan, upper_32_bits(node->vma[0].offset));
590 OUT_RING (chan, lower_32_bits(node->vma[0].offset));
591 OUT_RING (chan, upper_32_bits(node->vma[1].offset));
592 OUT_RING (chan, lower_32_bits(node->vma[1].offset));
593 OUT_RING (chan, PAGE_SIZE);
594 OUT_RING (chan, PAGE_SIZE);
595 OUT_RING (chan, PAGE_SIZE);
596 OUT_RING (chan, new_mem->num_pages);
597 BEGIN_IMC0(chan, NvSubCopy, 0x0300, 0x0386);
598 }
599 return ret;
600 }
601
602 static int
603 nvc0_bo_move_init(struct nouveau_channel *chan, u32 handle)
604 {
605 int ret = RING_SPACE(chan, 2);
606 if (ret == 0) {
607 BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
608 OUT_RING (chan, handle);
609 }
610 return ret;
611 }
612
613 static int
614 nvc0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
615 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
616 {
617 struct nouveau_mem *node = old_mem->mm_node;
618 u64 src_offset = node->vma[0].offset;
619 u64 dst_offset = node->vma[1].offset;
620 u32 page_count = new_mem->num_pages;
621 int ret;
622
623 page_count = new_mem->num_pages;
624 while (page_count) {
625 int line_count = (page_count > 8191) ? 8191 : page_count;
626
627 ret = RING_SPACE(chan, 11);
628 if (ret)
629 return ret;
630
631 BEGIN_NVC0(chan, NvSubCopy, 0x030c, 8);
632 OUT_RING (chan, upper_32_bits(src_offset));
633 OUT_RING (chan, lower_32_bits(src_offset));
634 OUT_RING (chan, upper_32_bits(dst_offset));
635 OUT_RING (chan, lower_32_bits(dst_offset));
636 OUT_RING (chan, PAGE_SIZE);
637 OUT_RING (chan, PAGE_SIZE);
638 OUT_RING (chan, PAGE_SIZE);
639 OUT_RING (chan, line_count);
640 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
641 OUT_RING (chan, 0x00000110);
642
643 page_count -= line_count;
644 src_offset += (PAGE_SIZE * line_count);
645 dst_offset += (PAGE_SIZE * line_count);
646 }
647
648 return 0;
649 }
650
651 static int
652 nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
653 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
654 {
655 struct nouveau_mem *node = old_mem->mm_node;
656 u64 src_offset = node->vma[0].offset;
657 u64 dst_offset = node->vma[1].offset;
658 u32 page_count = new_mem->num_pages;
659 int ret;
660
661 page_count = new_mem->num_pages;
662 while (page_count) {
663 int line_count = (page_count > 2047) ? 2047 : page_count;
664
665 ret = RING_SPACE(chan, 12);
666 if (ret)
667 return ret;
668
669 BEGIN_NVC0(chan, NvSubCopy, 0x0238, 2);
670 OUT_RING (chan, upper_32_bits(dst_offset));
671 OUT_RING (chan, lower_32_bits(dst_offset));
672 BEGIN_NVC0(chan, NvSubCopy, 0x030c, 6);
673 OUT_RING (chan, upper_32_bits(src_offset));
674 OUT_RING (chan, lower_32_bits(src_offset));
675 OUT_RING (chan, PAGE_SIZE); /* src_pitch */
676 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */
677 OUT_RING (chan, PAGE_SIZE); /* line_length */
678 OUT_RING (chan, line_count);
679 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
680 OUT_RING (chan, 0x00100110);
681
682 page_count -= line_count;
683 src_offset += (PAGE_SIZE * line_count);
684 dst_offset += (PAGE_SIZE * line_count);
685 }
686
687 return 0;
688 }
689
690 static int
691 nva3_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
692 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
693 {
694 struct nouveau_mem *node = old_mem->mm_node;
695 u64 src_offset = node->vma[0].offset;
696 u64 dst_offset = node->vma[1].offset;
697 u32 page_count = new_mem->num_pages;
698 int ret;
699
700 page_count = new_mem->num_pages;
701 while (page_count) {
702 int line_count = (page_count > 8191) ? 8191 : page_count;
703
704 ret = RING_SPACE(chan, 11);
705 if (ret)
706 return ret;
707
708 BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
709 OUT_RING (chan, upper_32_bits(src_offset));
710 OUT_RING (chan, lower_32_bits(src_offset));
711 OUT_RING (chan, upper_32_bits(dst_offset));
712 OUT_RING (chan, lower_32_bits(dst_offset));
713 OUT_RING (chan, PAGE_SIZE);
714 OUT_RING (chan, PAGE_SIZE);
715 OUT_RING (chan, PAGE_SIZE);
716 OUT_RING (chan, line_count);
717 BEGIN_NV04(chan, NvSubCopy, 0x0300, 1);
718 OUT_RING (chan, 0x00000110);
719
720 page_count -= line_count;
721 src_offset += (PAGE_SIZE * line_count);
722 dst_offset += (PAGE_SIZE * line_count);
723 }
724
725 return 0;
726 }
727
728 static int
729 nv98_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
730 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
731 {
732 struct nouveau_mem *node = old_mem->mm_node;
733 int ret = RING_SPACE(chan, 7);
734 if (ret == 0) {
735 BEGIN_NV04(chan, NvSubCopy, 0x0320, 6);
736 OUT_RING (chan, upper_32_bits(node->vma[0].offset));
737 OUT_RING (chan, lower_32_bits(node->vma[0].offset));
738 OUT_RING (chan, upper_32_bits(node->vma[1].offset));
739 OUT_RING (chan, lower_32_bits(node->vma[1].offset));
740 OUT_RING (chan, 0x00000000 /* COPY */);
741 OUT_RING (chan, new_mem->num_pages << PAGE_SHIFT);
742 }
743 return ret;
744 }
745
746 static int
747 nv84_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
748 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
749 {
750 struct nouveau_mem *node = old_mem->mm_node;
751 int ret = RING_SPACE(chan, 7);
752 if (ret == 0) {
753 BEGIN_NV04(chan, NvSubCopy, 0x0304, 6);
754 OUT_RING (chan, new_mem->num_pages << PAGE_SHIFT);
755 OUT_RING (chan, upper_32_bits(node->vma[0].offset));
756 OUT_RING (chan, lower_32_bits(node->vma[0].offset));
757 OUT_RING (chan, upper_32_bits(node->vma[1].offset));
758 OUT_RING (chan, lower_32_bits(node->vma[1].offset));
759 OUT_RING (chan, 0x00000000 /* MODE_COPY, QUERY_NONE */);
760 }
761 return ret;
762 }
763
764 static int
765 nv50_bo_move_init(struct nouveau_channel *chan, u32 handle)
766 {
767 int ret = RING_SPACE(chan, 6);
768 if (ret == 0) {
769 BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
770 OUT_RING (chan, handle);
771 BEGIN_NV04(chan, NvSubCopy, 0x0180, 3);
772 OUT_RING (chan, NvNotify0);
773 OUT_RING (chan, NvDmaFB);
774 OUT_RING (chan, NvDmaFB);
775 }
776
777 return ret;
778 }
779
780 static int
781 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
782 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
783 {
784 struct nouveau_mem *node = old_mem->mm_node;
785 u64 length = (new_mem->num_pages << PAGE_SHIFT);
786 u64 src_offset = node->vma[0].offset;
787 u64 dst_offset = node->vma[1].offset;
788 int src_tiled = !!node->memtype;
789 int dst_tiled = !!((struct nouveau_mem *)new_mem->mm_node)->memtype;
790 int ret;
791
792 while (length) {
793 u32 amount, stride, height;
794
795 ret = RING_SPACE(chan, 18 + 6 * (src_tiled + dst_tiled));
796 if (ret)
797 return ret;
798
799 amount = min(length, (u64)(4 * 1024 * 1024));
800 stride = 16 * 4;
801 height = amount / stride;
802
803 if (src_tiled) {
804 BEGIN_NV04(chan, NvSubCopy, 0x0200, 7);
805 OUT_RING (chan, 0);
806 OUT_RING (chan, 0);
807 OUT_RING (chan, stride);
808 OUT_RING (chan, height);
809 OUT_RING (chan, 1);
810 OUT_RING (chan, 0);
811 OUT_RING (chan, 0);
812 } else {
813 BEGIN_NV04(chan, NvSubCopy, 0x0200, 1);
814 OUT_RING (chan, 1);
815 }
816 if (dst_tiled) {
817 BEGIN_NV04(chan, NvSubCopy, 0x021c, 7);
818 OUT_RING (chan, 0);
819 OUT_RING (chan, 0);
820 OUT_RING (chan, stride);
821 OUT_RING (chan, height);
822 OUT_RING (chan, 1);
823 OUT_RING (chan, 0);
824 OUT_RING (chan, 0);
825 } else {
826 BEGIN_NV04(chan, NvSubCopy, 0x021c, 1);
827 OUT_RING (chan, 1);
828 }
829
830 BEGIN_NV04(chan, NvSubCopy, 0x0238, 2);
831 OUT_RING (chan, upper_32_bits(src_offset));
832 OUT_RING (chan, upper_32_bits(dst_offset));
833 BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
834 OUT_RING (chan, lower_32_bits(src_offset));
835 OUT_RING (chan, lower_32_bits(dst_offset));
836 OUT_RING (chan, stride);
837 OUT_RING (chan, stride);
838 OUT_RING (chan, stride);
839 OUT_RING (chan, height);
840 OUT_RING (chan, 0x00000101);
841 OUT_RING (chan, 0x00000000);
842 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
843 OUT_RING (chan, 0);
844
845 length -= amount;
846 src_offset += amount;
847 dst_offset += amount;
848 }
849
850 return 0;
851 }
852
853 static int
854 nv04_bo_move_init(struct nouveau_channel *chan, u32 handle)
855 {
856 int ret = RING_SPACE(chan, 4);
857 if (ret == 0) {
858 BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
859 OUT_RING (chan, handle);
860 BEGIN_NV04(chan, NvSubCopy, 0x0180, 1);
861 OUT_RING (chan, NvNotify0);
862 }
863
864 return ret;
865 }
866
867 static inline uint32_t
868 nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
869 struct nouveau_channel *chan, struct ttm_mem_reg *mem)
870 {
871 if (mem->mem_type == TTM_PL_TT)
872 return NvDmaTT;
873 return NvDmaFB;
874 }
875
876 static int
877 nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
878 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
879 {
880 u32 src_offset = old_mem->start << PAGE_SHIFT;
881 u32 dst_offset = new_mem->start << PAGE_SHIFT;
882 u32 page_count = new_mem->num_pages;
883 int ret;
884
885 ret = RING_SPACE(chan, 3);
886 if (ret)
887 return ret;
888
889 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
890 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));
891 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));
892
893 page_count = new_mem->num_pages;
894 while (page_count) {
895 int line_count = (page_count > 2047) ? 2047 : page_count;
896
897 ret = RING_SPACE(chan, 11);
898 if (ret)
899 return ret;
900
901 BEGIN_NV04(chan, NvSubCopy,
902 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
903 OUT_RING (chan, src_offset);
904 OUT_RING (chan, dst_offset);
905 OUT_RING (chan, PAGE_SIZE); /* src_pitch */
906 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */
907 OUT_RING (chan, PAGE_SIZE); /* line_length */
908 OUT_RING (chan, line_count);
909 OUT_RING (chan, 0x00000101);
910 OUT_RING (chan, 0x00000000);
911 BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
912 OUT_RING (chan, 0);
913
914 page_count -= line_count;
915 src_offset += (PAGE_SIZE * line_count);
916 dst_offset += (PAGE_SIZE * line_count);
917 }
918
919 return 0;
920 }
921
922 static int
923 nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo,
924 struct ttm_mem_reg *mem)
925 {
926 struct nouveau_mem *old_node = bo->mem.mm_node;
927 struct nouveau_mem *new_node = mem->mm_node;
928 u64 size = (u64)mem->num_pages << PAGE_SHIFT;
929 int ret;
930
931 ret = nouveau_vm_get(drm->client.vm, size, old_node->page_shift,
932 NV_MEM_ACCESS_RW, &old_node->vma[0]);
933 if (ret)
934 return ret;
935
936 ret = nouveau_vm_get(drm->client.vm, size, new_node->page_shift,
937 NV_MEM_ACCESS_RW, &old_node->vma[1]);
938 if (ret) {
939 nouveau_vm_put(&old_node->vma[0]);
940 return ret;
941 }
942
943 nouveau_vm_map(&old_node->vma[0], old_node);
944 nouveau_vm_map(&old_node->vma[1], new_node);
945 return 0;
946 }
947
948 static int
949 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
950 bool no_wait_gpu, struct ttm_mem_reg *new_mem)
951 {
952 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
953 struct nouveau_channel *chan = drm->ttm.chan;
954 struct nouveau_fence *fence;
955 int ret;
956
957 /* create temporary vmas for the transfer and attach them to the
958 * old nouveau_mem node, these will get cleaned up after ttm has
959 * destroyed the ttm_mem_reg
960 */
961 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
962 ret = nouveau_bo_move_prep(drm, bo, new_mem);
963 if (ret)
964 return ret;
965 }
966
967 mutex_lock_nested(&chan->cli->mutex, SINGLE_DEPTH_NESTING);
968 ret = nouveau_fence_sync(bo->sync_obj, chan);
969 if (ret == 0) {
970 ret = drm->ttm.move(chan, bo, &bo->mem, new_mem);
971 if (ret == 0) {
972 ret = nouveau_fence_new(chan, false, &fence);
973 if (ret == 0) {
974 ret = ttm_bo_move_accel_cleanup(bo, fence,
975 evict,
976 no_wait_gpu,
977 new_mem);
978 nouveau_fence_unref(&fence);
979 }
980 }
981 }
982 mutex_unlock(&chan->cli->mutex);
983 return ret;
984 }
985
986 void
987 nouveau_bo_move_init(struct nouveau_drm *drm)
988 {
989 static const struct {
990 const char *name;
991 int engine;
992 u32 oclass;
993 int (*exec)(struct nouveau_channel *,
994 struct ttm_buffer_object *,
995 struct ttm_mem_reg *, struct ttm_mem_reg *);
996 int (*init)(struct nouveau_channel *, u32 handle);
997 } _methods[] = {
998 { "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init },
999 { "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1000 { "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init },
1001 { "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init },
1002 { "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init },
1003 { "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init },
1004 { "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init },
1005 { "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init },
1006 { "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init },
1007 {},
1008 { "CRYPT", 0, 0x88b4, nv98_bo_move_exec, nv50_bo_move_init },
1009 }, *mthd = _methods;
1010 const char *name = "CPU";
1011 int ret;
1012
1013 do {
1014 struct nouveau_object *object;
1015 struct nouveau_channel *chan;
1016 u32 handle = (mthd->engine << 16) | mthd->oclass;
1017
1018 if (mthd->engine)
1019 chan = drm->cechan;
1020 else
1021 chan = drm->channel;
1022 if (chan == NULL)
1023 continue;
1024
1025 ret = nouveau_object_new(nv_object(drm), chan->handle, handle,
1026 mthd->oclass, NULL, 0, &object);
1027 if (ret == 0) {
1028 ret = mthd->init(chan, handle);
1029 if (ret) {
1030 nouveau_object_del(nv_object(drm),
1031 chan->handle, handle);
1032 continue;
1033 }
1034
1035 drm->ttm.move = mthd->exec;
1036 drm->ttm.chan = chan;
1037 name = mthd->name;
1038 break;
1039 }
1040 } while ((++mthd)->exec);
1041
1042 NV_INFO(drm, "MM: using %s for buffer copies\n", name);
1043 }
1044
1045 static int
1046 nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
1047 bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1048 {
1049 u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
1050 struct ttm_placement placement;
1051 struct ttm_mem_reg tmp_mem;
1052 int ret;
1053
1054 placement.fpfn = placement.lpfn = 0;
1055 placement.num_placement = placement.num_busy_placement = 1;
1056 placement.placement = placement.busy_placement = &placement_memtype;
1057
1058 tmp_mem = *new_mem;
1059 tmp_mem.mm_node = NULL;
1060 ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_gpu);
1061 if (ret)
1062 return ret;
1063
1064 ret = ttm_tt_bind(bo->ttm, &tmp_mem);
1065 if (ret)
1066 goto out;
1067
1068 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, &tmp_mem);
1069 if (ret)
1070 goto out;
1071
1072 ret = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem);
1073 out:
1074 ttm_bo_mem_put(bo, &tmp_mem);
1075 return ret;
1076 }
1077
1078 static int
1079 nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
1080 bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1081 {
1082 u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
1083 struct ttm_placement placement;
1084 struct ttm_mem_reg tmp_mem;
1085 int ret;
1086
1087 placement.fpfn = placement.lpfn = 0;
1088 placement.num_placement = placement.num_busy_placement = 1;
1089 placement.placement = placement.busy_placement = &placement_memtype;
1090
1091 tmp_mem = *new_mem;
1092 tmp_mem.mm_node = NULL;
1093 ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_gpu);
1094 if (ret)
1095 return ret;
1096
1097 ret = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem);
1098 if (ret)
1099 goto out;
1100
1101 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, new_mem);
1102 if (ret)
1103 goto out;
1104
1105 out:
1106 ttm_bo_mem_put(bo, &tmp_mem);
1107 return ret;
1108 }
1109
1110 static void
1111 nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
1112 {
1113 struct nouveau_bo *nvbo = nouveau_bo(bo);
1114 struct nouveau_vma *vma;
1115
1116 /* ttm can now (stupidly) pass the driver bos it didn't create... */
1117 if (bo->destroy != nouveau_bo_del_ttm)
1118 return;
1119
1120 list_for_each_entry(vma, &nvbo->vma_list, head) {
1121 if (new_mem && new_mem->mem_type != TTM_PL_SYSTEM &&
1122 (new_mem->mem_type == TTM_PL_VRAM ||
1123 nvbo->page_shift != vma->vm->vmm->lpg_shift)) {
1124 nouveau_vm_map(vma, new_mem->mm_node);
1125 } else {
1126 nouveau_vm_unmap(vma);
1127 }
1128 }
1129 }
1130
1131 static int
1132 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
1133 struct nouveau_drm_tile **new_tile)
1134 {
1135 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1136 struct drm_device *dev = drm->dev;
1137 struct nouveau_bo *nvbo = nouveau_bo(bo);
1138 u64 offset = new_mem->start << PAGE_SHIFT;
1139
1140 *new_tile = NULL;
1141 if (new_mem->mem_type != TTM_PL_VRAM)
1142 return 0;
1143
1144 if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
1145 *new_tile = nv10_bo_set_tiling(dev, offset, new_mem->size,
1146 nvbo->tile_mode,
1147 nvbo->tile_flags);
1148 }
1149
1150 return 0;
1151 }
1152
1153 static void
1154 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
1155 struct nouveau_drm_tile *new_tile,
1156 struct nouveau_drm_tile **old_tile)
1157 {
1158 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1159 struct drm_device *dev = drm->dev;
1160
1161 nv10_bo_put_tile_region(dev, *old_tile, bo->sync_obj);
1162 *old_tile = new_tile;
1163 }
1164
1165 static int
1166 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
1167 bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1168 {
1169 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1170 struct nouveau_bo *nvbo = nouveau_bo(bo);
1171 struct ttm_mem_reg *old_mem = &bo->mem;
1172 struct nouveau_drm_tile *new_tile = NULL;
1173 int ret = 0;
1174
1175 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1176 ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
1177 if (ret)
1178 return ret;
1179 }
1180
1181 /* Fake bo copy. */
1182 if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
1183 BUG_ON(bo->mem.mm_node != NULL);
1184 bo->mem = *new_mem;
1185 new_mem->mm_node = NULL;
1186 goto out;
1187 }
1188
1189 /* Hardware assisted copy. */
1190 if (drm->ttm.move) {
1191 if (new_mem->mem_type == TTM_PL_SYSTEM)
1192 ret = nouveau_bo_move_flipd(bo, evict, intr,
1193 no_wait_gpu, new_mem);
1194 else if (old_mem->mem_type == TTM_PL_SYSTEM)
1195 ret = nouveau_bo_move_flips(bo, evict, intr,
1196 no_wait_gpu, new_mem);
1197 else
1198 ret = nouveau_bo_move_m2mf(bo, evict, intr,
1199 no_wait_gpu, new_mem);
1200 if (!ret)
1201 goto out;
1202 }
1203
1204 /* Fallback to software copy. */
1205 spin_lock(&bo->bdev->fence_lock);
1206 ret = ttm_bo_wait(bo, true, intr, no_wait_gpu);
1207 spin_unlock(&bo->bdev->fence_lock);
1208 if (ret == 0)
1209 ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
1210
1211 out:
1212 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1213 if (ret)
1214 nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
1215 else
1216 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
1217 }
1218
1219 return ret;
1220 }
1221
1222 static int
1223 nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
1224 {
1225 struct nouveau_bo *nvbo = nouveau_bo(bo);
1226
1227 return drm_vma_node_verify_access(&nvbo->gem.vma_node, filp);
1228 }
1229
1230 static int
1231 nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1232 {
1233 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1234 struct nouveau_drm *drm = nouveau_bdev(bdev);
1235 struct nouveau_mem *node = mem->mm_node;
1236 struct drm_device *dev = drm->dev;
1237 int ret;
1238
1239 mem->bus.addr = NULL;
1240 mem->bus.offset = 0;
1241 mem->bus.size = mem->num_pages << PAGE_SHIFT;
1242 mem->bus.base = 0;
1243 mem->bus.is_iomem = false;
1244 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
1245 return -EINVAL;
1246 switch (mem->mem_type) {
1247 case TTM_PL_SYSTEM:
1248 /* System memory */
1249 return 0;
1250 case TTM_PL_TT:
1251 #if __OS_HAS_AGP
1252 if (drm->agp.stat == ENABLED) {
1253 mem->bus.offset = mem->start << PAGE_SHIFT;
1254 mem->bus.base = drm->agp.base;
1255 mem->bus.is_iomem = !dev->agp->cant_use_aperture;
1256 }
1257 #endif
1258 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA || !node->memtype)
1259 /* untiled */
1260 break;
1261 /* fallthrough, tiled memory */
1262 case TTM_PL_VRAM:
1263 mem->bus.offset = mem->start << PAGE_SHIFT;
1264 mem->bus.base = nv_device_resource_start(nvkm_device(&drm->device), 1);
1265 mem->bus.is_iomem = true;
1266 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
1267 struct nouveau_bar *bar = nvkm_bar(&drm->device);
1268
1269 ret = bar->umap(bar, node, NV_MEM_ACCESS_RW,
1270 &node->bar_vma);
1271 if (ret)
1272 return ret;
1273
1274 mem->bus.offset = node->bar_vma.offset;
1275 }
1276 break;
1277 default:
1278 return -EINVAL;
1279 }
1280 return 0;
1281 }
1282
1283 static void
1284 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1285 {
1286 struct nouveau_drm *drm = nouveau_bdev(bdev);
1287 struct nouveau_bar *bar = nvkm_bar(&drm->device);
1288 struct nouveau_mem *node = mem->mm_node;
1289
1290 if (!node->bar_vma.node)
1291 return;
1292
1293 bar->unmap(bar, &node->bar_vma);
1294 }
1295
1296 static int
1297 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
1298 {
1299 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1300 struct nouveau_bo *nvbo = nouveau_bo(bo);
1301 struct nvif_device *device = &drm->device;
1302 u32 mappable = nv_device_resource_len(nvkm_device(device), 1) >> PAGE_SHIFT;
1303 int ret;
1304
1305 /* as long as the bo isn't in vram, and isn't tiled, we've got
1306 * nothing to do here.
1307 */
1308 if (bo->mem.mem_type != TTM_PL_VRAM) {
1309 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA ||
1310 !nouveau_bo_tile_layout(nvbo))
1311 return 0;
1312
1313 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
1314 nouveau_bo_placement_set(nvbo, TTM_PL_TT, 0);
1315
1316 ret = nouveau_bo_validate(nvbo, false, false);
1317 if (ret)
1318 return ret;
1319 }
1320 return 0;
1321 }
1322
1323 /* make sure bo is in mappable vram */
1324 if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA ||
1325 bo->mem.start + bo->mem.num_pages < mappable)
1326 return 0;
1327
1328
1329 nvbo->placement.fpfn = 0;
1330 nvbo->placement.lpfn = mappable;
1331 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_VRAM, 0);
1332 return nouveau_bo_validate(nvbo, false, false);
1333 }
1334
1335 static int
1336 nouveau_ttm_tt_populate(struct ttm_tt *ttm)
1337 {
1338 struct ttm_dma_tt *ttm_dma = (void *)ttm;
1339 struct nouveau_drm *drm;
1340 struct nouveau_device *device;
1341 struct drm_device *dev;
1342 struct device *pdev;
1343 unsigned i;
1344 int r;
1345 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1346
1347 if (ttm->state != tt_unpopulated)
1348 return 0;
1349
1350 if (slave && ttm->sg) {
1351 /* make userspace faulting work */
1352 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1353 ttm_dma->dma_address, ttm->num_pages);
1354 ttm->state = tt_unbound;
1355 return 0;
1356 }
1357
1358 drm = nouveau_bdev(ttm->bdev);
1359 device = nvkm_device(&drm->device);
1360 dev = drm->dev;
1361 pdev = nv_device_base(device);
1362
1363 #if __OS_HAS_AGP
1364 if (drm->agp.stat == ENABLED) {
1365 return ttm_agp_tt_populate(ttm);
1366 }
1367 #endif
1368
1369 #ifdef CONFIG_SWIOTLB
1370 if (swiotlb_nr_tbl()) {
1371 return ttm_dma_populate((void *)ttm, dev->dev);
1372 }
1373 #endif
1374
1375 r = ttm_pool_populate(ttm);
1376 if (r) {
1377 return r;
1378 }
1379
1380 for (i = 0; i < ttm->num_pages; i++) {
1381 dma_addr_t addr;
1382
1383 addr = dma_map_page(pdev, ttm->pages[i], 0, PAGE_SIZE,
1384 DMA_BIDIRECTIONAL);
1385
1386 if (dma_mapping_error(pdev, addr)) {
1387 while (--i) {
1388 dma_unmap_page(pdev, ttm_dma->dma_address[i],
1389 PAGE_SIZE, DMA_BIDIRECTIONAL);
1390 ttm_dma->dma_address[i] = 0;
1391 }
1392 ttm_pool_unpopulate(ttm);
1393 return -EFAULT;
1394 }
1395
1396 ttm_dma->dma_address[i] = addr;
1397 }
1398 return 0;
1399 }
1400
1401 static void
1402 nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
1403 {
1404 struct ttm_dma_tt *ttm_dma = (void *)ttm;
1405 struct nouveau_drm *drm;
1406 struct nouveau_device *device;
1407 struct drm_device *dev;
1408 struct device *pdev;
1409 unsigned i;
1410 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1411
1412 if (slave)
1413 return;
1414
1415 drm = nouveau_bdev(ttm->bdev);
1416 device = nvkm_device(&drm->device);
1417 dev = drm->dev;
1418 pdev = nv_device_base(device);
1419
1420 #if __OS_HAS_AGP
1421 if (drm->agp.stat == ENABLED) {
1422 ttm_agp_tt_unpopulate(ttm);
1423 return;
1424 }
1425 #endif
1426
1427 #ifdef CONFIG_SWIOTLB
1428 if (swiotlb_nr_tbl()) {
1429 ttm_dma_unpopulate((void *)ttm, dev->dev);
1430 return;
1431 }
1432 #endif
1433
1434 for (i = 0; i < ttm->num_pages; i++) {
1435 if (ttm_dma->dma_address[i]) {
1436 dma_unmap_page(pdev, ttm_dma->dma_address[i], PAGE_SIZE,
1437 DMA_BIDIRECTIONAL);
1438 }
1439 }
1440
1441 ttm_pool_unpopulate(ttm);
1442 }
1443
1444 void
1445 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence)
1446 {
1447 struct nouveau_fence *new_fence = nouveau_fence_ref(fence);
1448 struct nouveau_fence *old_fence = NULL;
1449
1450 spin_lock(&nvbo->bo.bdev->fence_lock);
1451 old_fence = nvbo->bo.sync_obj;
1452 nvbo->bo.sync_obj = new_fence;
1453 spin_unlock(&nvbo->bo.bdev->fence_lock);
1454
1455 nouveau_fence_unref(&old_fence);
1456 }
1457
1458 static void
1459 nouveau_bo_fence_unref(void **sync_obj)
1460 {
1461 nouveau_fence_unref((struct nouveau_fence **)sync_obj);
1462 }
1463
1464 static void *
1465 nouveau_bo_fence_ref(void *sync_obj)
1466 {
1467 return nouveau_fence_ref(sync_obj);
1468 }
1469
1470 static bool
1471 nouveau_bo_fence_signalled(void *sync_obj)
1472 {
1473 return nouveau_fence_done(sync_obj);
1474 }
1475
1476 static int
1477 nouveau_bo_fence_wait(void *sync_obj, bool lazy, bool intr)
1478 {
1479 return nouveau_fence_wait(sync_obj, lazy, intr);
1480 }
1481
1482 static int
1483 nouveau_bo_fence_flush(void *sync_obj)
1484 {
1485 return 0;
1486 }
1487
1488 struct ttm_bo_driver nouveau_bo_driver = {
1489 .ttm_tt_create = &nouveau_ttm_tt_create,
1490 .ttm_tt_populate = &nouveau_ttm_tt_populate,
1491 .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
1492 .invalidate_caches = nouveau_bo_invalidate_caches,
1493 .init_mem_type = nouveau_bo_init_mem_type,
1494 .evict_flags = nouveau_bo_evict_flags,
1495 .move_notify = nouveau_bo_move_ntfy,
1496 .move = nouveau_bo_move,
1497 .verify_access = nouveau_bo_verify_access,
1498 .sync_obj_signaled = nouveau_bo_fence_signalled,
1499 .sync_obj_wait = nouveau_bo_fence_wait,
1500 .sync_obj_flush = nouveau_bo_fence_flush,
1501 .sync_obj_unref = nouveau_bo_fence_unref,
1502 .sync_obj_ref = nouveau_bo_fence_ref,
1503 .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
1504 .io_mem_reserve = &nouveau_ttm_io_mem_reserve,
1505 .io_mem_free = &nouveau_ttm_io_mem_free,
1506 };
1507
1508 struct nouveau_vma *
1509 nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nouveau_vm *vm)
1510 {
1511 struct nouveau_vma *vma;
1512 list_for_each_entry(vma, &nvbo->vma_list, head) {
1513 if (vma->vm == vm)
1514 return vma;
1515 }
1516
1517 return NULL;
1518 }
1519
1520 int
1521 nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nouveau_vm *vm,
1522 struct nouveau_vma *vma)
1523 {
1524 const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
1525 int ret;
1526
1527 ret = nouveau_vm_get(vm, size, nvbo->page_shift,
1528 NV_MEM_ACCESS_RW, vma);
1529 if (ret)
1530 return ret;
1531
1532 if ( nvbo->bo.mem.mem_type != TTM_PL_SYSTEM &&
1533 (nvbo->bo.mem.mem_type == TTM_PL_VRAM ||
1534 nvbo->page_shift != vma->vm->vmm->lpg_shift))
1535 nouveau_vm_map(vma, nvbo->bo.mem.mm_node);
1536
1537 list_add_tail(&vma->head, &nvbo->vma_list);
1538 vma->refcount = 1;
1539 return 0;
1540 }
1541
1542 void
1543 nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
1544 {
1545 if (vma->node) {
1546 if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM)
1547 nouveau_vm_unmap(vma);
1548 nouveau_vm_put(vma);
1549 list_del(&vma->head);
1550 }
1551 }
Linux kernel - Deliverables
This page took 0.074971 seconds and 6 git commands to generate.