1 /**************************************************************************
3 * Copyright © 2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 #include "vmwgfx_drv.h"
29 #include <drm/ttm/ttm_bo_driver.h>
30 #include <drm/ttm/ttm_placement.h>
31 #include <drm/ttm/ttm_page_alloc.h>
33 static uint32_t vram_placement_flags
= TTM_PL_FLAG_VRAM
|
36 static uint32_t vram_ne_placement_flags
= TTM_PL_FLAG_VRAM
|
40 static uint32_t sys_placement_flags
= TTM_PL_FLAG_SYSTEM
|
43 static uint32_t gmr_placement_flags
= VMW_PL_FLAG_GMR
|
46 static uint32_t gmr_ne_placement_flags
= VMW_PL_FLAG_GMR
|
50 struct ttm_placement vmw_vram_placement
= {
54 .placement
= &vram_placement_flags
,
55 .num_busy_placement
= 1,
56 .busy_placement
= &vram_placement_flags
59 static uint32_t vram_gmr_placement_flags
[] = {
60 TTM_PL_FLAG_VRAM
| TTM_PL_FLAG_CACHED
,
61 VMW_PL_FLAG_GMR
| TTM_PL_FLAG_CACHED
64 static uint32_t gmr_vram_placement_flags
[] = {
65 VMW_PL_FLAG_GMR
| TTM_PL_FLAG_CACHED
,
66 TTM_PL_FLAG_VRAM
| TTM_PL_FLAG_CACHED
69 struct ttm_placement vmw_vram_gmr_placement
= {
73 .placement
= vram_gmr_placement_flags
,
74 .num_busy_placement
= 1,
75 .busy_placement
= &gmr_placement_flags
78 static uint32_t vram_gmr_ne_placement_flags
[] = {
79 TTM_PL_FLAG_VRAM
| TTM_PL_FLAG_CACHED
| TTM_PL_FLAG_NO_EVICT
,
80 VMW_PL_FLAG_GMR
| TTM_PL_FLAG_CACHED
| TTM_PL_FLAG_NO_EVICT
83 struct ttm_placement vmw_vram_gmr_ne_placement
= {
87 .placement
= vram_gmr_ne_placement_flags
,
88 .num_busy_placement
= 1,
89 .busy_placement
= &gmr_ne_placement_flags
92 struct ttm_placement vmw_vram_sys_placement
= {
96 .placement
= &vram_placement_flags
,
97 .num_busy_placement
= 1,
98 .busy_placement
= &sys_placement_flags
101 struct ttm_placement vmw_vram_ne_placement
= {
105 .placement
= &vram_ne_placement_flags
,
106 .num_busy_placement
= 1,
107 .busy_placement
= &vram_ne_placement_flags
110 struct ttm_placement vmw_sys_placement
= {
114 .placement
= &sys_placement_flags
,
115 .num_busy_placement
= 1,
116 .busy_placement
= &sys_placement_flags
119 static uint32_t evictable_placement_flags
[] = {
120 TTM_PL_FLAG_SYSTEM
| TTM_PL_FLAG_CACHED
,
121 TTM_PL_FLAG_VRAM
| TTM_PL_FLAG_CACHED
,
122 VMW_PL_FLAG_GMR
| TTM_PL_FLAG_CACHED
125 struct ttm_placement vmw_evictable_placement
= {
129 .placement
= evictable_placement_flags
,
130 .num_busy_placement
= 1,
131 .busy_placement
= &sys_placement_flags
134 struct ttm_placement vmw_srf_placement
= {
138 .num_busy_placement
= 2,
139 .placement
= &gmr_placement_flags
,
140 .busy_placement
= gmr_vram_placement_flags
144 struct ttm_dma_tt dma_ttm
;
145 struct vmw_private
*dev_priv
;
148 struct vmw_sg_table vsgt
;
149 uint64_t sg_alloc_size
;
154 * Helper functions to advance a struct vmw_piter iterator.
156 * @viter: Pointer to the iterator.
158 * These functions return false if past the end of the list,
159 * true otherwise. Functions are selected depending on the current
162 static bool __vmw_piter_non_sg_next(struct vmw_piter
*viter
)
164 return ++(viter
->i
) < viter
->num_pages
;
167 static bool __vmw_piter_sg_next(struct vmw_piter
*viter
)
169 return __sg_page_iter_next(&viter
->iter
);
174 * Helper functions to return a pointer to the current page.
176 * @viter: Pointer to the iterator
178 * These functions return a pointer to the page currently
179 * pointed to by @viter. Functions are selected depending on the
180 * current mapping mode.
182 static struct page
*__vmw_piter_non_sg_page(struct vmw_piter
*viter
)
184 return viter
->pages
[viter
->i
];
187 static struct page
*__vmw_piter_sg_page(struct vmw_piter
*viter
)
189 return sg_page_iter_page(&viter
->iter
);
194 * Helper functions to return the DMA address of the current page.
196 * @viter: Pointer to the iterator
198 * These functions return the DMA address of the page currently
199 * pointed to by @viter. Functions are selected depending on the
200 * current mapping mode.
202 static dma_addr_t
__vmw_piter_phys_addr(struct vmw_piter
*viter
)
204 return page_to_phys(viter
->pages
[viter
->i
]);
207 static dma_addr_t
__vmw_piter_dma_addr(struct vmw_piter
*viter
)
209 return viter
->addrs
[viter
->i
];
212 static dma_addr_t
__vmw_piter_sg_addr(struct vmw_piter
*viter
)
214 return sg_page_iter_dma_address(&viter
->iter
);
219 * vmw_piter_start - Initialize a struct vmw_piter.
221 * @viter: Pointer to the iterator to initialize
222 * @vsgt: Pointer to a struct vmw_sg_table to initialize from
224 * Note that we're following the convention of __sg_page_iter_start, so that
225 * the iterator doesn't point to a valid page after initialization; it has
226 * to be advanced one step first.
228 void vmw_piter_start(struct vmw_piter
*viter
, const struct vmw_sg_table
*vsgt
,
229 unsigned long p_offset
)
231 viter
->i
= p_offset
- 1;
232 viter
->num_pages
= vsgt
->num_pages
;
233 switch (vsgt
->mode
) {
235 viter
->next
= &__vmw_piter_non_sg_next
;
236 viter
->dma_address
= &__vmw_piter_phys_addr
;
237 viter
->page
= &__vmw_piter_non_sg_page
;
238 viter
->pages
= vsgt
->pages
;
240 case vmw_dma_alloc_coherent
:
241 viter
->next
= &__vmw_piter_non_sg_next
;
242 viter
->dma_address
= &__vmw_piter_dma_addr
;
243 viter
->page
= &__vmw_piter_non_sg_page
;
244 viter
->addrs
= vsgt
->addrs
;
246 case vmw_dma_map_populate
:
247 case vmw_dma_map_bind
:
248 viter
->next
= &__vmw_piter_sg_next
;
249 viter
->dma_address
= &__vmw_piter_sg_addr
;
250 viter
->page
= &__vmw_piter_sg_page
;
251 __sg_page_iter_start(&viter
->iter
, vsgt
->sgt
->sgl
,
252 vsgt
->sgt
->orig_nents
, p_offset
);
260 * vmw_ttm_unmap_from_dma - unmap device addresses previsouly mapped for
263 * @vmw_tt: Pointer to a struct vmw_ttm_backend
265 * Used to free dma mappings previously mapped by vmw_ttm_map_for_dma.
267 static void vmw_ttm_unmap_from_dma(struct vmw_ttm_tt
*vmw_tt
)
269 struct device
*dev
= vmw_tt
->dev_priv
->dev
->dev
;
271 dma_unmap_sg(dev
, vmw_tt
->sgt
.sgl
, vmw_tt
->sgt
.nents
,
273 vmw_tt
->sgt
.nents
= vmw_tt
->sgt
.orig_nents
;
277 * vmw_ttm_map_for_dma - map TTM pages to get device addresses
279 * @vmw_tt: Pointer to a struct vmw_ttm_backend
281 * This function is used to get device addresses from the kernel DMA layer.
282 * However, it's violating the DMA API in that when this operation has been
283 * performed, it's illegal for the CPU to write to the pages without first
284 * unmapping the DMA mappings, or calling dma_sync_sg_for_cpu(). It is
285 * therefore only legal to call this function if we know that the function
286 * dma_sync_sg_for_cpu() is a NOP, and dma_sync_sg_for_device() is at most
287 * a CPU write buffer flush.
289 static int vmw_ttm_map_for_dma(struct vmw_ttm_tt
*vmw_tt
)
291 struct device
*dev
= vmw_tt
->dev_priv
->dev
->dev
;
294 ret
= dma_map_sg(dev
, vmw_tt
->sgt
.sgl
, vmw_tt
->sgt
.orig_nents
,
296 if (unlikely(ret
== 0))
299 vmw_tt
->sgt
.nents
= ret
;
305 * vmw_ttm_map_dma - Make sure TTM pages are visible to the device
307 * @vmw_tt: Pointer to a struct vmw_ttm_tt
309 * Select the correct function for and make sure the TTM pages are
310 * visible to the device. Allocate storage for the device mappings.
311 * If a mapping has already been performed, indicated by the storage
312 * pointer being non NULL, the function returns success.
314 static int vmw_ttm_map_dma(struct vmw_ttm_tt
*vmw_tt
)
316 struct vmw_private
*dev_priv
= vmw_tt
->dev_priv
;
317 struct ttm_mem_global
*glob
= vmw_mem_glob(dev_priv
);
318 struct vmw_sg_table
*vsgt
= &vmw_tt
->vsgt
;
319 struct vmw_piter iter
;
322 static size_t sgl_size
;
323 static size_t sgt_size
;
328 vsgt
->mode
= dev_priv
->map_mode
;
329 vsgt
->pages
= vmw_tt
->dma_ttm
.ttm
.pages
;
330 vsgt
->num_pages
= vmw_tt
->dma_ttm
.ttm
.num_pages
;
331 vsgt
->addrs
= vmw_tt
->dma_ttm
.dma_address
;
332 vsgt
->sgt
= &vmw_tt
->sgt
;
334 switch (dev_priv
->map_mode
) {
335 case vmw_dma_map_bind
:
336 case vmw_dma_map_populate
:
337 if (unlikely(!sgl_size
)) {
338 sgl_size
= ttm_round_pot(sizeof(struct scatterlist
));
339 sgt_size
= ttm_round_pot(sizeof(struct sg_table
));
341 vmw_tt
->sg_alloc_size
= sgt_size
+ sgl_size
* vsgt
->num_pages
;
342 ret
= ttm_mem_global_alloc(glob
, vmw_tt
->sg_alloc_size
, false,
344 if (unlikely(ret
!= 0))
347 ret
= sg_alloc_table_from_pages(&vmw_tt
->sgt
, vsgt
->pages
,
350 vsgt
->num_pages
<< PAGE_SHIFT
,
352 if (unlikely(ret
!= 0))
353 goto out_sg_alloc_fail
;
355 if (vsgt
->num_pages
> vmw_tt
->sgt
.nents
) {
356 uint64_t over_alloc
=
357 sgl_size
* (vsgt
->num_pages
-
360 ttm_mem_global_free(glob
, over_alloc
);
361 vmw_tt
->sg_alloc_size
-= over_alloc
;
364 ret
= vmw_ttm_map_for_dma(vmw_tt
);
365 if (unlikely(ret
!= 0))
373 old
= ~((dma_addr_t
) 0);
374 vmw_tt
->vsgt
.num_regions
= 0;
375 for (vmw_piter_start(&iter
, vsgt
, 0); vmw_piter_next(&iter
);) {
376 dma_addr_t cur
= vmw_piter_dma_addr(&iter
);
378 if (cur
!= old
+ PAGE_SIZE
)
379 vmw_tt
->vsgt
.num_regions
++;
383 vmw_tt
->mapped
= true;
387 sg_free_table(vmw_tt
->vsgt
.sgt
);
388 vmw_tt
->vsgt
.sgt
= NULL
;
390 ttm_mem_global_free(glob
, vmw_tt
->sg_alloc_size
);
395 * vmw_ttm_unmap_dma - Tear down any TTM page device mappings
397 * @vmw_tt: Pointer to a struct vmw_ttm_tt
399 * Tear down any previously set up device DMA mappings and free
400 * any storage space allocated for them. If there are no mappings set up,
401 * this function is a NOP.
403 static void vmw_ttm_unmap_dma(struct vmw_ttm_tt
*vmw_tt
)
405 struct vmw_private
*dev_priv
= vmw_tt
->dev_priv
;
407 if (!vmw_tt
->vsgt
.sgt
)
410 switch (dev_priv
->map_mode
) {
411 case vmw_dma_map_bind
:
412 case vmw_dma_map_populate
:
413 vmw_ttm_unmap_from_dma(vmw_tt
);
414 sg_free_table(vmw_tt
->vsgt
.sgt
);
415 vmw_tt
->vsgt
.sgt
= NULL
;
416 ttm_mem_global_free(vmw_mem_glob(dev_priv
),
417 vmw_tt
->sg_alloc_size
);
422 vmw_tt
->mapped
= false;
425 static int vmw_ttm_bind(struct ttm_tt
*ttm
, struct ttm_mem_reg
*bo_mem
)
427 struct vmw_ttm_tt
*vmw_be
=
428 container_of(ttm
, struct vmw_ttm_tt
, dma_ttm
.ttm
);
431 ret
= vmw_ttm_map_dma(vmw_be
);
432 if (unlikely(ret
!= 0))
435 vmw_be
->gmr_id
= bo_mem
->start
;
437 return vmw_gmr_bind(vmw_be
->dev_priv
, &vmw_be
->vsgt
,
438 ttm
->num_pages
, vmw_be
->gmr_id
);
441 static int vmw_ttm_unbind(struct ttm_tt
*ttm
)
443 struct vmw_ttm_tt
*vmw_be
=
444 container_of(ttm
, struct vmw_ttm_tt
, dma_ttm
.ttm
);
446 vmw_gmr_unbind(vmw_be
->dev_priv
, vmw_be
->gmr_id
);
448 if (vmw_be
->dev_priv
->map_mode
== vmw_dma_map_bind
)
449 vmw_ttm_unmap_dma(vmw_be
);
454 static void vmw_ttm_destroy(struct ttm_tt
*ttm
)
456 struct vmw_ttm_tt
*vmw_be
=
457 container_of(ttm
, struct vmw_ttm_tt
, dma_ttm
.ttm
);
459 vmw_ttm_unmap_dma(vmw_be
);
460 if (vmw_be
->dev_priv
->map_mode
== vmw_dma_alloc_coherent
)
461 ttm_dma_tt_fini(&vmw_be
->dma_ttm
);
467 static int vmw_ttm_populate(struct ttm_tt
*ttm
)
469 struct vmw_ttm_tt
*vmw_tt
=
470 container_of(ttm
, struct vmw_ttm_tt
, dma_ttm
.ttm
);
471 struct vmw_private
*dev_priv
= vmw_tt
->dev_priv
;
472 struct ttm_mem_global
*glob
= vmw_mem_glob(dev_priv
);
475 if (ttm
->state
!= tt_unpopulated
)
478 if (dev_priv
->map_mode
== vmw_dma_alloc_coherent
) {
480 ttm_round_pot(ttm
->num_pages
* sizeof(dma_addr_t
));
481 ret
= ttm_mem_global_alloc(glob
, size
, false, true);
482 if (unlikely(ret
!= 0))
485 ret
= ttm_dma_populate(&vmw_tt
->dma_ttm
, dev_priv
->dev
->dev
);
486 if (unlikely(ret
!= 0))
487 ttm_mem_global_free(glob
, size
);
489 ret
= ttm_pool_populate(ttm
);
494 static void vmw_ttm_unpopulate(struct ttm_tt
*ttm
)
496 struct vmw_ttm_tt
*vmw_tt
= container_of(ttm
, struct vmw_ttm_tt
,
498 struct vmw_private
*dev_priv
= vmw_tt
->dev_priv
;
499 struct ttm_mem_global
*glob
= vmw_mem_glob(dev_priv
);
501 vmw_ttm_unmap_dma(vmw_tt
);
502 if (dev_priv
->map_mode
== vmw_dma_alloc_coherent
) {
504 ttm_round_pot(ttm
->num_pages
* sizeof(dma_addr_t
));
506 ttm_dma_unpopulate(&vmw_tt
->dma_ttm
, dev_priv
->dev
->dev
);
507 ttm_mem_global_free(glob
, size
);
509 ttm_pool_unpopulate(ttm
);
512 static struct ttm_backend_func vmw_ttm_func
= {
513 .bind
= vmw_ttm_bind
,
514 .unbind
= vmw_ttm_unbind
,
515 .destroy
= vmw_ttm_destroy
,
518 struct ttm_tt
*vmw_ttm_tt_create(struct ttm_bo_device
*bdev
,
519 unsigned long size
, uint32_t page_flags
,
520 struct page
*dummy_read_page
)
522 struct vmw_ttm_tt
*vmw_be
;
525 vmw_be
= kzalloc(sizeof(*vmw_be
), GFP_KERNEL
);
529 vmw_be
->dma_ttm
.ttm
.func
= &vmw_ttm_func
;
530 vmw_be
->dev_priv
= container_of(bdev
, struct vmw_private
, bdev
);
532 if (vmw_be
->dev_priv
->map_mode
== vmw_dma_alloc_coherent
)
533 ret
= ttm_dma_tt_init(&vmw_be
->dma_ttm
, bdev
, size
, page_flags
,
536 ret
= ttm_tt_init(&vmw_be
->dma_ttm
.ttm
, bdev
, size
, page_flags
,
538 if (unlikely(ret
!= 0))
541 return &vmw_be
->dma_ttm
.ttm
;
547 int vmw_invalidate_caches(struct ttm_bo_device
*bdev
, uint32_t flags
)
552 int vmw_init_mem_type(struct ttm_bo_device
*bdev
, uint32_t type
,
553 struct ttm_mem_type_manager
*man
)
559 man
->flags
= TTM_MEMTYPE_FLAG_MAPPABLE
;
560 man
->available_caching
= TTM_PL_FLAG_CACHED
;
561 man
->default_caching
= TTM_PL_FLAG_CACHED
;
564 /* "On-card" video ram */
565 man
->func
= &ttm_bo_manager_func
;
567 man
->flags
= TTM_MEMTYPE_FLAG_FIXED
| TTM_MEMTYPE_FLAG_MAPPABLE
;
568 man
->available_caching
= TTM_PL_FLAG_CACHED
;
569 man
->default_caching
= TTM_PL_FLAG_CACHED
;
573 * "Guest Memory Regions" is an aperture like feature with
574 * one slot per bo. There is an upper limit of the number of
575 * slots as well as the bo size.
577 man
->func
= &vmw_gmrid_manager_func
;
579 man
->flags
= TTM_MEMTYPE_FLAG_CMA
| TTM_MEMTYPE_FLAG_MAPPABLE
;
580 man
->available_caching
= TTM_PL_FLAG_CACHED
;
581 man
->default_caching
= TTM_PL_FLAG_CACHED
;
584 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type
);
590 void vmw_evict_flags(struct ttm_buffer_object
*bo
,
591 struct ttm_placement
*placement
)
593 *placement
= vmw_sys_placement
;
596 static int vmw_verify_access(struct ttm_buffer_object
*bo
, struct file
*filp
)
598 struct ttm_object_file
*tfile
=
599 vmw_fpriv((struct drm_file
*)filp
->private_data
)->tfile
;
601 return vmw_user_dmabuf_verify_access(bo
, tfile
);
604 static int vmw_ttm_io_mem_reserve(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
)
606 struct ttm_mem_type_manager
*man
= &bdev
->man
[mem
->mem_type
];
607 struct vmw_private
*dev_priv
= container_of(bdev
, struct vmw_private
, bdev
);
609 mem
->bus
.addr
= NULL
;
610 mem
->bus
.is_iomem
= false;
612 mem
->bus
.size
= mem
->num_pages
<< PAGE_SHIFT
;
614 if (!(man
->flags
& TTM_MEMTYPE_FLAG_MAPPABLE
))
616 switch (mem
->mem_type
) {
621 mem
->bus
.offset
= mem
->start
<< PAGE_SHIFT
;
622 mem
->bus
.base
= dev_priv
->vram_start
;
623 mem
->bus
.is_iomem
= true;
631 static void vmw_ttm_io_mem_free(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
)
635 static int vmw_ttm_fault_reserve_notify(struct ttm_buffer_object
*bo
)
641 * FIXME: We're using the old vmware polling method to sync.
642 * Do this with fences instead.
645 static void *vmw_sync_obj_ref(void *sync_obj
)
649 vmw_fence_obj_reference((struct vmw_fence_obj
*) sync_obj
);
652 static void vmw_sync_obj_unref(void **sync_obj
)
654 vmw_fence_obj_unreference((struct vmw_fence_obj
**) sync_obj
);
657 static int vmw_sync_obj_flush(void *sync_obj
)
659 vmw_fence_obj_flush((struct vmw_fence_obj
*) sync_obj
);
663 static bool vmw_sync_obj_signaled(void *sync_obj
)
665 return vmw_fence_obj_signaled((struct vmw_fence_obj
*) sync_obj
,
666 DRM_VMW_FENCE_FLAG_EXEC
);
670 static int vmw_sync_obj_wait(void *sync_obj
, bool lazy
, bool interruptible
)
672 return vmw_fence_obj_wait((struct vmw_fence_obj
*) sync_obj
,
673 DRM_VMW_FENCE_FLAG_EXEC
,
675 VMW_FENCE_WAIT_TIMEOUT
);
678 struct ttm_bo_driver vmw_bo_driver
= {
679 .ttm_tt_create
= &vmw_ttm_tt_create
,
680 .ttm_tt_populate
= &vmw_ttm_populate
,
681 .ttm_tt_unpopulate
= &vmw_ttm_unpopulate
,
682 .invalidate_caches
= vmw_invalidate_caches
,
683 .init_mem_type
= vmw_init_mem_type
,
684 .evict_flags
= vmw_evict_flags
,
686 .verify_access
= vmw_verify_access
,
687 .sync_obj_signaled
= vmw_sync_obj_signaled
,
688 .sync_obj_wait
= vmw_sync_obj_wait
,
689 .sync_obj_flush
= vmw_sync_obj_flush
,
690 .sync_obj_unref
= vmw_sync_obj_unref
,
691 .sync_obj_ref
= vmw_sync_obj_ref
,
694 .fault_reserve_notify
= &vmw_ttm_fault_reserve_notify
,
695 .io_mem_reserve
= &vmw_ttm_io_mem_reserve
,
696 .io_mem_free
= &vmw_ttm_io_mem_free
,