1 /**************************************************************************
3 * Copyright © 2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 #include "vmwgfx_drv.h"
29 #include <drm/ttm/ttm_bo_driver.h>
30 #include <drm/ttm/ttm_placement.h>
31 #include <drm/ttm/ttm_page_alloc.h>
33 static uint32_t vram_placement_flags
= TTM_PL_FLAG_VRAM
|
36 static uint32_t vram_ne_placement_flags
= TTM_PL_FLAG_VRAM
|
40 static uint32_t sys_placement_flags
= TTM_PL_FLAG_SYSTEM
|
43 static uint32_t gmr_placement_flags
= VMW_PL_FLAG_GMR
|
46 static uint32_t gmr_ne_placement_flags
= VMW_PL_FLAG_GMR
|
50 struct ttm_placement vmw_vram_placement
= {
54 .placement
= &vram_placement_flags
,
55 .num_busy_placement
= 1,
56 .busy_placement
= &vram_placement_flags
59 static uint32_t vram_gmr_placement_flags
[] = {
60 TTM_PL_FLAG_VRAM
| TTM_PL_FLAG_CACHED
,
61 VMW_PL_FLAG_GMR
| TTM_PL_FLAG_CACHED
64 static uint32_t gmr_vram_placement_flags
[] = {
65 VMW_PL_FLAG_GMR
| TTM_PL_FLAG_CACHED
,
66 TTM_PL_FLAG_VRAM
| TTM_PL_FLAG_CACHED
69 struct ttm_placement vmw_vram_gmr_placement
= {
73 .placement
= vram_gmr_placement_flags
,
74 .num_busy_placement
= 1,
75 .busy_placement
= &gmr_placement_flags
78 static uint32_t vram_gmr_ne_placement_flags
[] = {
79 TTM_PL_FLAG_VRAM
| TTM_PL_FLAG_CACHED
| TTM_PL_FLAG_NO_EVICT
,
80 VMW_PL_FLAG_GMR
| TTM_PL_FLAG_CACHED
| TTM_PL_FLAG_NO_EVICT
83 struct ttm_placement vmw_vram_gmr_ne_placement
= {
87 .placement
= vram_gmr_ne_placement_flags
,
88 .num_busy_placement
= 1,
89 .busy_placement
= &gmr_ne_placement_flags
92 struct ttm_placement vmw_vram_sys_placement
= {
96 .placement
= &vram_placement_flags
,
97 .num_busy_placement
= 1,
98 .busy_placement
= &sys_placement_flags
101 struct ttm_placement vmw_vram_ne_placement
= {
105 .placement
= &vram_ne_placement_flags
,
106 .num_busy_placement
= 1,
107 .busy_placement
= &vram_ne_placement_flags
110 struct ttm_placement vmw_sys_placement
= {
114 .placement
= &sys_placement_flags
,
115 .num_busy_placement
= 1,
116 .busy_placement
= &sys_placement_flags
119 static uint32_t evictable_placement_flags
[] = {
120 TTM_PL_FLAG_SYSTEM
| TTM_PL_FLAG_CACHED
,
121 TTM_PL_FLAG_VRAM
| TTM_PL_FLAG_CACHED
,
122 VMW_PL_FLAG_GMR
| TTM_PL_FLAG_CACHED
125 struct ttm_placement vmw_evictable_placement
= {
129 .placement
= evictable_placement_flags
,
130 .num_busy_placement
= 1,
131 .busy_placement
= &sys_placement_flags
134 struct ttm_placement vmw_srf_placement
= {
138 .num_busy_placement
= 2,
139 .placement
= &gmr_placement_flags
,
140 .busy_placement
= gmr_vram_placement_flags
144 struct ttm_dma_tt dma_ttm
;
145 struct vmw_private
*dev_priv
;
148 struct vmw_sg_table vsgt
;
149 uint64_t sg_alloc_size
;
153 const size_t vmw_tt_size
= sizeof(struct vmw_ttm_tt
);
156 * Helper functions to advance a struct vmw_piter iterator.
158 * @viter: Pointer to the iterator.
160 * These functions return false if past the end of the list,
161 * true otherwise. Functions are selected depending on the current
164 static bool __vmw_piter_non_sg_next(struct vmw_piter
*viter
)
166 return ++(viter
->i
) < viter
->num_pages
;
169 static bool __vmw_piter_sg_next(struct vmw_piter
*viter
)
171 return __sg_page_iter_next(&viter
->iter
);
176 * Helper functions to return a pointer to the current page.
178 * @viter: Pointer to the iterator
180 * These functions return a pointer to the page currently
181 * pointed to by @viter. Functions are selected depending on the
182 * current mapping mode.
184 static struct page
*__vmw_piter_non_sg_page(struct vmw_piter
*viter
)
186 return viter
->pages
[viter
->i
];
189 static struct page
*__vmw_piter_sg_page(struct vmw_piter
*viter
)
191 return sg_page_iter_page(&viter
->iter
);
196 * Helper functions to return the DMA address of the current page.
198 * @viter: Pointer to the iterator
200 * These functions return the DMA address of the page currently
201 * pointed to by @viter. Functions are selected depending on the
202 * current mapping mode.
204 static dma_addr_t
__vmw_piter_phys_addr(struct vmw_piter
*viter
)
206 return page_to_phys(viter
->pages
[viter
->i
]);
209 static dma_addr_t
__vmw_piter_dma_addr(struct vmw_piter
*viter
)
211 return viter
->addrs
[viter
->i
];
214 static dma_addr_t
__vmw_piter_sg_addr(struct vmw_piter
*viter
)
216 return sg_page_iter_dma_address(&viter
->iter
);
221 * vmw_piter_start - Initialize a struct vmw_piter.
223 * @viter: Pointer to the iterator to initialize
224 * @vsgt: Pointer to a struct vmw_sg_table to initialize from
226 * Note that we're following the convention of __sg_page_iter_start, so that
227 * the iterator doesn't point to a valid page after initialization; it has
228 * to be advanced one step first.
230 void vmw_piter_start(struct vmw_piter
*viter
, const struct vmw_sg_table
*vsgt
,
231 unsigned long p_offset
)
233 viter
->i
= p_offset
- 1;
234 viter
->num_pages
= vsgt
->num_pages
;
235 switch (vsgt
->mode
) {
237 viter
->next
= &__vmw_piter_non_sg_next
;
238 viter
->dma_address
= &__vmw_piter_phys_addr
;
239 viter
->page
= &__vmw_piter_non_sg_page
;
240 viter
->pages
= vsgt
->pages
;
242 case vmw_dma_alloc_coherent
:
243 viter
->next
= &__vmw_piter_non_sg_next
;
244 viter
->dma_address
= &__vmw_piter_dma_addr
;
245 viter
->page
= &__vmw_piter_non_sg_page
;
246 viter
->addrs
= vsgt
->addrs
;
248 case vmw_dma_map_populate
:
249 case vmw_dma_map_bind
:
250 viter
->next
= &__vmw_piter_sg_next
;
251 viter
->dma_address
= &__vmw_piter_sg_addr
;
252 viter
->page
= &__vmw_piter_sg_page
;
253 __sg_page_iter_start(&viter
->iter
, vsgt
->sgt
->sgl
,
254 vsgt
->sgt
->orig_nents
, p_offset
);
262 * vmw_ttm_unmap_from_dma - unmap device addresses previsouly mapped for
265 * @vmw_tt: Pointer to a struct vmw_ttm_backend
267 * Used to free dma mappings previously mapped by vmw_ttm_map_for_dma.
269 static void vmw_ttm_unmap_from_dma(struct vmw_ttm_tt
*vmw_tt
)
271 struct device
*dev
= vmw_tt
->dev_priv
->dev
->dev
;
273 dma_unmap_sg(dev
, vmw_tt
->sgt
.sgl
, vmw_tt
->sgt
.nents
,
275 vmw_tt
->sgt
.nents
= vmw_tt
->sgt
.orig_nents
;
279 * vmw_ttm_map_for_dma - map TTM pages to get device addresses
281 * @vmw_tt: Pointer to a struct vmw_ttm_backend
283 * This function is used to get device addresses from the kernel DMA layer.
284 * However, it's violating the DMA API in that when this operation has been
285 * performed, it's illegal for the CPU to write to the pages without first
286 * unmapping the DMA mappings, or calling dma_sync_sg_for_cpu(). It is
287 * therefore only legal to call this function if we know that the function
288 * dma_sync_sg_for_cpu() is a NOP, and dma_sync_sg_for_device() is at most
289 * a CPU write buffer flush.
291 static int vmw_ttm_map_for_dma(struct vmw_ttm_tt
*vmw_tt
)
293 struct device
*dev
= vmw_tt
->dev_priv
->dev
->dev
;
296 ret
= dma_map_sg(dev
, vmw_tt
->sgt
.sgl
, vmw_tt
->sgt
.orig_nents
,
298 if (unlikely(ret
== 0))
301 vmw_tt
->sgt
.nents
= ret
;
307 * vmw_ttm_map_dma - Make sure TTM pages are visible to the device
309 * @vmw_tt: Pointer to a struct vmw_ttm_tt
311 * Select the correct function for and make sure the TTM pages are
312 * visible to the device. Allocate storage for the device mappings.
313 * If a mapping has already been performed, indicated by the storage
314 * pointer being non NULL, the function returns success.
316 static int vmw_ttm_map_dma(struct vmw_ttm_tt
*vmw_tt
)
318 struct vmw_private
*dev_priv
= vmw_tt
->dev_priv
;
319 struct ttm_mem_global
*glob
= vmw_mem_glob(dev_priv
);
320 struct vmw_sg_table
*vsgt
= &vmw_tt
->vsgt
;
321 struct vmw_piter iter
;
324 static size_t sgl_size
;
325 static size_t sgt_size
;
330 vsgt
->mode
= dev_priv
->map_mode
;
331 vsgt
->pages
= vmw_tt
->dma_ttm
.ttm
.pages
;
332 vsgt
->num_pages
= vmw_tt
->dma_ttm
.ttm
.num_pages
;
333 vsgt
->addrs
= vmw_tt
->dma_ttm
.dma_address
;
334 vsgt
->sgt
= &vmw_tt
->sgt
;
336 switch (dev_priv
->map_mode
) {
337 case vmw_dma_map_bind
:
338 case vmw_dma_map_populate
:
339 if (unlikely(!sgl_size
)) {
340 sgl_size
= ttm_round_pot(sizeof(struct scatterlist
));
341 sgt_size
= ttm_round_pot(sizeof(struct sg_table
));
343 vmw_tt
->sg_alloc_size
= sgt_size
+ sgl_size
* vsgt
->num_pages
;
344 ret
= ttm_mem_global_alloc(glob
, vmw_tt
->sg_alloc_size
, false,
346 if (unlikely(ret
!= 0))
349 ret
= sg_alloc_table_from_pages(&vmw_tt
->sgt
, vsgt
->pages
,
352 vsgt
->num_pages
<< PAGE_SHIFT
,
354 if (unlikely(ret
!= 0))
355 goto out_sg_alloc_fail
;
357 if (vsgt
->num_pages
> vmw_tt
->sgt
.nents
) {
358 uint64_t over_alloc
=
359 sgl_size
* (vsgt
->num_pages
-
362 ttm_mem_global_free(glob
, over_alloc
);
363 vmw_tt
->sg_alloc_size
-= over_alloc
;
366 ret
= vmw_ttm_map_for_dma(vmw_tt
);
367 if (unlikely(ret
!= 0))
375 old
= ~((dma_addr_t
) 0);
376 vmw_tt
->vsgt
.num_regions
= 0;
377 for (vmw_piter_start(&iter
, vsgt
, 0); vmw_piter_next(&iter
);) {
378 dma_addr_t cur
= vmw_piter_dma_addr(&iter
);
380 if (cur
!= old
+ PAGE_SIZE
)
381 vmw_tt
->vsgt
.num_regions
++;
385 vmw_tt
->mapped
= true;
389 sg_free_table(vmw_tt
->vsgt
.sgt
);
390 vmw_tt
->vsgt
.sgt
= NULL
;
392 ttm_mem_global_free(glob
, vmw_tt
->sg_alloc_size
);
397 * vmw_ttm_unmap_dma - Tear down any TTM page device mappings
399 * @vmw_tt: Pointer to a struct vmw_ttm_tt
401 * Tear down any previously set up device DMA mappings and free
402 * any storage space allocated for them. If there are no mappings set up,
403 * this function is a NOP.
405 static void vmw_ttm_unmap_dma(struct vmw_ttm_tt
*vmw_tt
)
407 struct vmw_private
*dev_priv
= vmw_tt
->dev_priv
;
409 if (!vmw_tt
->vsgt
.sgt
)
412 switch (dev_priv
->map_mode
) {
413 case vmw_dma_map_bind
:
414 case vmw_dma_map_populate
:
415 vmw_ttm_unmap_from_dma(vmw_tt
);
416 sg_free_table(vmw_tt
->vsgt
.sgt
);
417 vmw_tt
->vsgt
.sgt
= NULL
;
418 ttm_mem_global_free(vmw_mem_glob(dev_priv
),
419 vmw_tt
->sg_alloc_size
);
424 vmw_tt
->mapped
= false;
427 static int vmw_ttm_bind(struct ttm_tt
*ttm
, struct ttm_mem_reg
*bo_mem
)
429 struct vmw_ttm_tt
*vmw_be
=
430 container_of(ttm
, struct vmw_ttm_tt
, dma_ttm
.ttm
);
433 ret
= vmw_ttm_map_dma(vmw_be
);
434 if (unlikely(ret
!= 0))
437 vmw_be
->gmr_id
= bo_mem
->start
;
439 return vmw_gmr_bind(vmw_be
->dev_priv
, &vmw_be
->vsgt
,
440 ttm
->num_pages
, vmw_be
->gmr_id
);
443 static int vmw_ttm_unbind(struct ttm_tt
*ttm
)
445 struct vmw_ttm_tt
*vmw_be
=
446 container_of(ttm
, struct vmw_ttm_tt
, dma_ttm
.ttm
);
448 vmw_gmr_unbind(vmw_be
->dev_priv
, vmw_be
->gmr_id
);
450 if (vmw_be
->dev_priv
->map_mode
== vmw_dma_map_bind
)
451 vmw_ttm_unmap_dma(vmw_be
);
456 static void vmw_ttm_destroy(struct ttm_tt
*ttm
)
458 struct vmw_ttm_tt
*vmw_be
=
459 container_of(ttm
, struct vmw_ttm_tt
, dma_ttm
.ttm
);
461 vmw_ttm_unmap_dma(vmw_be
);
462 if (vmw_be
->dev_priv
->map_mode
== vmw_dma_alloc_coherent
)
463 ttm_dma_tt_fini(&vmw_be
->dma_ttm
);
469 static int vmw_ttm_populate(struct ttm_tt
*ttm
)
471 struct vmw_ttm_tt
*vmw_tt
=
472 container_of(ttm
, struct vmw_ttm_tt
, dma_ttm
.ttm
);
473 struct vmw_private
*dev_priv
= vmw_tt
->dev_priv
;
474 struct ttm_mem_global
*glob
= vmw_mem_glob(dev_priv
);
477 if (ttm
->state
!= tt_unpopulated
)
480 if (dev_priv
->map_mode
== vmw_dma_alloc_coherent
) {
482 ttm_round_pot(ttm
->num_pages
* sizeof(dma_addr_t
));
483 ret
= ttm_mem_global_alloc(glob
, size
, false, true);
484 if (unlikely(ret
!= 0))
487 ret
= ttm_dma_populate(&vmw_tt
->dma_ttm
, dev_priv
->dev
->dev
);
488 if (unlikely(ret
!= 0))
489 ttm_mem_global_free(glob
, size
);
491 ret
= ttm_pool_populate(ttm
);
496 static void vmw_ttm_unpopulate(struct ttm_tt
*ttm
)
498 struct vmw_ttm_tt
*vmw_tt
= container_of(ttm
, struct vmw_ttm_tt
,
500 struct vmw_private
*dev_priv
= vmw_tt
->dev_priv
;
501 struct ttm_mem_global
*glob
= vmw_mem_glob(dev_priv
);
503 vmw_ttm_unmap_dma(vmw_tt
);
504 if (dev_priv
->map_mode
== vmw_dma_alloc_coherent
) {
506 ttm_round_pot(ttm
->num_pages
* sizeof(dma_addr_t
));
508 ttm_dma_unpopulate(&vmw_tt
->dma_ttm
, dev_priv
->dev
->dev
);
509 ttm_mem_global_free(glob
, size
);
511 ttm_pool_unpopulate(ttm
);
514 static struct ttm_backend_func vmw_ttm_func
= {
515 .bind
= vmw_ttm_bind
,
516 .unbind
= vmw_ttm_unbind
,
517 .destroy
= vmw_ttm_destroy
,
520 struct ttm_tt
*vmw_ttm_tt_create(struct ttm_bo_device
*bdev
,
521 unsigned long size
, uint32_t page_flags
,
522 struct page
*dummy_read_page
)
524 struct vmw_ttm_tt
*vmw_be
;
527 vmw_be
= kzalloc(sizeof(*vmw_be
), GFP_KERNEL
);
531 vmw_be
->dma_ttm
.ttm
.func
= &vmw_ttm_func
;
532 vmw_be
->dev_priv
= container_of(bdev
, struct vmw_private
, bdev
);
534 if (vmw_be
->dev_priv
->map_mode
== vmw_dma_alloc_coherent
)
535 ret
= ttm_dma_tt_init(&vmw_be
->dma_ttm
, bdev
, size
, page_flags
,
538 ret
= ttm_tt_init(&vmw_be
->dma_ttm
.ttm
, bdev
, size
, page_flags
,
540 if (unlikely(ret
!= 0))
543 return &vmw_be
->dma_ttm
.ttm
;
549 int vmw_invalidate_caches(struct ttm_bo_device
*bdev
, uint32_t flags
)
554 int vmw_init_mem_type(struct ttm_bo_device
*bdev
, uint32_t type
,
555 struct ttm_mem_type_manager
*man
)
561 man
->flags
= TTM_MEMTYPE_FLAG_MAPPABLE
;
562 man
->available_caching
= TTM_PL_FLAG_CACHED
;
563 man
->default_caching
= TTM_PL_FLAG_CACHED
;
566 /* "On-card" video ram */
567 man
->func
= &ttm_bo_manager_func
;
569 man
->flags
= TTM_MEMTYPE_FLAG_FIXED
| TTM_MEMTYPE_FLAG_MAPPABLE
;
570 man
->available_caching
= TTM_PL_FLAG_CACHED
;
571 man
->default_caching
= TTM_PL_FLAG_CACHED
;
575 * "Guest Memory Regions" is an aperture like feature with
576 * one slot per bo. There is an upper limit of the number of
577 * slots as well as the bo size.
579 man
->func
= &vmw_gmrid_manager_func
;
581 man
->flags
= TTM_MEMTYPE_FLAG_CMA
| TTM_MEMTYPE_FLAG_MAPPABLE
;
582 man
->available_caching
= TTM_PL_FLAG_CACHED
;
583 man
->default_caching
= TTM_PL_FLAG_CACHED
;
586 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type
);
592 void vmw_evict_flags(struct ttm_buffer_object
*bo
,
593 struct ttm_placement
*placement
)
595 *placement
= vmw_sys_placement
;
598 static int vmw_verify_access(struct ttm_buffer_object
*bo
, struct file
*filp
)
600 struct ttm_object_file
*tfile
=
601 vmw_fpriv((struct drm_file
*)filp
->private_data
)->tfile
;
603 return vmw_user_dmabuf_verify_access(bo
, tfile
);
606 static int vmw_ttm_io_mem_reserve(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
)
608 struct ttm_mem_type_manager
*man
= &bdev
->man
[mem
->mem_type
];
609 struct vmw_private
*dev_priv
= container_of(bdev
, struct vmw_private
, bdev
);
611 mem
->bus
.addr
= NULL
;
612 mem
->bus
.is_iomem
= false;
614 mem
->bus
.size
= mem
->num_pages
<< PAGE_SHIFT
;
616 if (!(man
->flags
& TTM_MEMTYPE_FLAG_MAPPABLE
))
618 switch (mem
->mem_type
) {
623 mem
->bus
.offset
= mem
->start
<< PAGE_SHIFT
;
624 mem
->bus
.base
= dev_priv
->vram_start
;
625 mem
->bus
.is_iomem
= true;
633 static void vmw_ttm_io_mem_free(struct ttm_bo_device
*bdev
, struct ttm_mem_reg
*mem
)
637 static int vmw_ttm_fault_reserve_notify(struct ttm_buffer_object
*bo
)
643 * FIXME: We're using the old vmware polling method to sync.
644 * Do this with fences instead.
647 static void *vmw_sync_obj_ref(void *sync_obj
)
651 vmw_fence_obj_reference((struct vmw_fence_obj
*) sync_obj
);
654 static void vmw_sync_obj_unref(void **sync_obj
)
656 vmw_fence_obj_unreference((struct vmw_fence_obj
**) sync_obj
);
659 static int vmw_sync_obj_flush(void *sync_obj
)
661 vmw_fence_obj_flush((struct vmw_fence_obj
*) sync_obj
);
665 static bool vmw_sync_obj_signaled(void *sync_obj
)
667 return vmw_fence_obj_signaled((struct vmw_fence_obj
*) sync_obj
,
668 DRM_VMW_FENCE_FLAG_EXEC
);
672 static int vmw_sync_obj_wait(void *sync_obj
, bool lazy
, bool interruptible
)
674 return vmw_fence_obj_wait((struct vmw_fence_obj
*) sync_obj
,
675 DRM_VMW_FENCE_FLAG_EXEC
,
677 VMW_FENCE_WAIT_TIMEOUT
);
680 struct ttm_bo_driver vmw_bo_driver
= {
681 .ttm_tt_create
= &vmw_ttm_tt_create
,
682 .ttm_tt_populate
= &vmw_ttm_populate
,
683 .ttm_tt_unpopulate
= &vmw_ttm_unpopulate
,
684 .invalidate_caches
= vmw_invalidate_caches
,
685 .init_mem_type
= vmw_init_mem_type
,
686 .evict_flags
= vmw_evict_flags
,
688 .verify_access
= vmw_verify_access
,
689 .sync_obj_signaled
= vmw_sync_obj_signaled
,
690 .sync_obj_wait
= vmw_sync_obj_wait
,
691 .sync_obj_flush
= vmw_sync_obj_flush
,
692 .sync_obj_unref
= vmw_sync_obj_unref
,
693 .sync_obj_ref
= vmw_sync_obj_ref
,
696 .fault_reserve_notify
= &vmw_ttm_fault_reserve_notify
,
697 .io_mem_reserve
= &vmw_ttm_io_mem_reserve
,
698 .io_mem_free
= &vmw_ttm_io_mem_free
,