1 /**************************************************************************
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
31 #include <linux/sched.h>
32 #include <linux/highmem.h>
33 #include <linux/pagemap.h>
34 #include <linux/file.h>
35 #include <linux/swap.h>
36 #include <linux/slab.h>
37 #include "drm_cache.h"
38 #include "drm_mem_util.h"
39 #include "ttm/ttm_module.h"
40 #include "ttm/ttm_bo_driver.h"
41 #include "ttm/ttm_placement.h"
42 #include "ttm/ttm_page_alloc.h"
44 static int ttm_tt_swapin(struct ttm_tt
*ttm
);
47 * Allocates storage for pointers to the pages that back the ttm.
49 static void ttm_tt_alloc_page_directory(struct ttm_tt
*ttm
)
51 ttm
->pages
= drm_calloc_large(ttm
->num_pages
, sizeof(*ttm
->pages
));
54 static void ttm_tt_free_page_directory(struct ttm_tt
*ttm
)
56 drm_free_large(ttm
->pages
);
60 static void ttm_tt_free_user_pages(struct ttm_tt
*ttm
)
66 struct ttm_backend
*be
= ttm
->be
;
68 BUG_ON(!(ttm
->page_flags
& TTM_PAGE_FLAG_USER
));
69 write
= ((ttm
->page_flags
& TTM_PAGE_FLAG_WRITE
) != 0);
70 dirty
= ((ttm
->page_flags
& TTM_PAGE_FLAG_USER_DIRTY
) != 0);
75 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
80 if (page
== ttm
->dummy_read_page
) {
85 if (write
&& dirty
&& !PageReserved(page
))
86 set_page_dirty_lock(page
);
89 ttm_mem_global_free(ttm
->glob
->mem_glob
, PAGE_SIZE
);
92 ttm
->state
= tt_unpopulated
;
93 ttm
->first_himem_page
= ttm
->num_pages
;
94 ttm
->last_lomem_page
= -1;
97 static struct page
*__ttm_tt_get_page(struct ttm_tt
*ttm
, int index
)
101 struct ttm_mem_global
*mem_glob
= ttm
->glob
->mem_glob
;
104 while (NULL
== (p
= ttm
->pages
[index
])) {
108 ret
= ttm_get_pages(&h
, ttm
->page_flags
, ttm
->caching_state
, 1);
113 p
= list_first_entry(&h
, struct page
, lru
);
115 ret
= ttm_mem_global_alloc_page(mem_glob
, p
, false, false);
116 if (unlikely(ret
!= 0))
120 ttm
->pages
[--ttm
->first_himem_page
] = p
;
122 ttm
->pages
[++ttm
->last_lomem_page
] = p
;
130 struct page
*ttm_tt_get_page(struct ttm_tt
*ttm
, int index
)
134 if (unlikely(ttm
->page_flags
& TTM_PAGE_FLAG_SWAPPED
)) {
135 ret
= ttm_tt_swapin(ttm
);
136 if (unlikely(ret
!= 0))
139 return __ttm_tt_get_page(ttm
, index
);
142 int ttm_tt_populate(struct ttm_tt
*ttm
)
146 struct ttm_backend
*be
;
149 if (ttm
->state
!= tt_unpopulated
)
152 if (unlikely(ttm
->page_flags
& TTM_PAGE_FLAG_SWAPPED
)) {
153 ret
= ttm_tt_swapin(ttm
);
154 if (unlikely(ret
!= 0))
160 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
161 page
= __ttm_tt_get_page(ttm
, i
);
166 be
->func
->populate(be
, ttm
->num_pages
, ttm
->pages
,
167 ttm
->dummy_read_page
);
168 ttm
->state
= tt_unbound
;
171 EXPORT_SYMBOL(ttm_tt_populate
);
174 static inline int ttm_tt_set_page_caching(struct page
*p
,
175 enum ttm_caching_state c_old
,
176 enum ttm_caching_state c_new
)
183 if (c_old
!= tt_cached
) {
184 /* p isn't in the default caching state, set it to
185 * writeback first to free its current memtype. */
187 ret
= set_pages_wb(p
, 1);
193 ret
= set_memory_wc((unsigned long) page_address(p
), 1);
194 else if (c_new
== tt_uncached
)
195 ret
= set_pages_uc(p
, 1);
199 #else /* CONFIG_X86 */
200 static inline int ttm_tt_set_page_caching(struct page
*p
,
201 enum ttm_caching_state c_old
,
202 enum ttm_caching_state c_new
)
206 #endif /* CONFIG_X86 */
209 * Change caching policy for the linear kernel map
210 * for range of pages in a ttm.
213 static int ttm_tt_set_caching(struct ttm_tt
*ttm
,
214 enum ttm_caching_state c_state
)
217 struct page
*cur_page
;
220 if (ttm
->caching_state
== c_state
)
223 if (ttm
->state
== tt_unpopulated
) {
224 /* Change caching but don't populate */
225 ttm
->caching_state
= c_state
;
229 if (ttm
->caching_state
== tt_cached
)
230 drm_clflush_pages(ttm
->pages
, ttm
->num_pages
);
232 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
233 cur_page
= ttm
->pages
[i
];
234 if (likely(cur_page
!= NULL
)) {
235 ret
= ttm_tt_set_page_caching(cur_page
,
238 if (unlikely(ret
!= 0))
243 ttm
->caching_state
= c_state
;
248 for (j
= 0; j
< i
; ++j
) {
249 cur_page
= ttm
->pages
[j
];
250 if (likely(cur_page
!= NULL
)) {
251 (void)ttm_tt_set_page_caching(cur_page
, c_state
,
259 int ttm_tt_set_placement_caching(struct ttm_tt
*ttm
, uint32_t placement
)
261 enum ttm_caching_state state
;
263 if (placement
& TTM_PL_FLAG_WC
)
265 else if (placement
& TTM_PL_FLAG_UNCACHED
)
270 return ttm_tt_set_caching(ttm
, state
);
272 EXPORT_SYMBOL(ttm_tt_set_placement_caching
);
274 static void ttm_tt_free_alloced_pages(struct ttm_tt
*ttm
)
279 struct page
*cur_page
;
280 struct ttm_backend
*be
= ttm
->be
;
286 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
288 cur_page
= ttm
->pages
[i
];
289 ttm
->pages
[i
] = NULL
;
291 if (page_count(cur_page
) != 1)
292 printk(KERN_ERR TTM_PFX
293 "Erroneous page count. "
295 ttm_mem_global_free_page(ttm
->glob
->mem_glob
,
297 list_add(&cur_page
->lru
, &h
);
301 ttm_put_pages(&h
, count
, ttm
->page_flags
, ttm
->caching_state
);
302 ttm
->state
= tt_unpopulated
;
303 ttm
->first_himem_page
= ttm
->num_pages
;
304 ttm
->last_lomem_page
= -1;
307 void ttm_tt_destroy(struct ttm_tt
*ttm
)
309 struct ttm_backend
*be
;
311 if (unlikely(ttm
== NULL
))
315 if (likely(be
!= NULL
)) {
316 be
->func
->destroy(be
);
320 if (likely(ttm
->pages
!= NULL
)) {
321 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
)
322 ttm_tt_free_user_pages(ttm
);
324 ttm_tt_free_alloced_pages(ttm
);
326 ttm_tt_free_page_directory(ttm
);
329 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTANT_SWAP
) &&
331 fput(ttm
->swap_storage
);
336 int ttm_tt_set_user(struct ttm_tt
*ttm
,
337 struct task_struct
*tsk
,
338 unsigned long start
, unsigned long num_pages
)
340 struct mm_struct
*mm
= tsk
->mm
;
342 int write
= (ttm
->page_flags
& TTM_PAGE_FLAG_WRITE
) != 0;
343 struct ttm_mem_global
*mem_glob
= ttm
->glob
->mem_glob
;
345 BUG_ON(num_pages
!= ttm
->num_pages
);
346 BUG_ON((ttm
->page_flags
& TTM_PAGE_FLAG_USER
) == 0);
349 * Account user pages as lowmem pages for now.
352 ret
= ttm_mem_global_alloc(mem_glob
, num_pages
* PAGE_SIZE
,
354 if (unlikely(ret
!= 0))
357 down_read(&mm
->mmap_sem
);
358 ret
= get_user_pages(tsk
, mm
, start
, num_pages
,
359 write
, 0, ttm
->pages
, NULL
);
360 up_read(&mm
->mmap_sem
);
362 if (ret
!= num_pages
&& write
) {
363 ttm_tt_free_user_pages(ttm
);
364 ttm_mem_global_free(mem_glob
, num_pages
* PAGE_SIZE
);
370 ttm
->state
= tt_unbound
;
375 struct ttm_tt
*ttm_tt_create(struct ttm_bo_device
*bdev
, unsigned long size
,
376 uint32_t page_flags
, struct page
*dummy_read_page
)
378 struct ttm_bo_driver
*bo_driver
= bdev
->driver
;
384 ttm
= kzalloc(sizeof(*ttm
), GFP_KERNEL
);
388 ttm
->glob
= bdev
->glob
;
389 ttm
->num_pages
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
390 ttm
->first_himem_page
= ttm
->num_pages
;
391 ttm
->last_lomem_page
= -1;
392 ttm
->caching_state
= tt_cached
;
393 ttm
->page_flags
= page_flags
;
395 ttm
->dummy_read_page
= dummy_read_page
;
397 ttm_tt_alloc_page_directory(ttm
);
400 printk(KERN_ERR TTM_PFX
"Failed allocating page table\n");
403 ttm
->be
= bo_driver
->create_ttm_backend_entry(bdev
);
406 printk(KERN_ERR TTM_PFX
"Failed creating ttm backend entry\n");
409 ttm
->state
= tt_unpopulated
;
413 void ttm_tt_unbind(struct ttm_tt
*ttm
)
416 struct ttm_backend
*be
= ttm
->be
;
418 if (ttm
->state
== tt_bound
) {
419 ret
= be
->func
->unbind(be
);
421 ttm
->state
= tt_unbound
;
425 int ttm_tt_bind(struct ttm_tt
*ttm
, struct ttm_mem_reg
*bo_mem
)
428 struct ttm_backend
*be
;
433 if (ttm
->state
== tt_bound
)
438 ret
= ttm_tt_populate(ttm
);
442 ret
= be
->func
->bind(be
, bo_mem
);
444 printk(KERN_ERR TTM_PFX
"Couldn't bind backend.\n");
448 ttm
->state
= tt_bound
;
450 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
)
451 ttm
->page_flags
|= TTM_PAGE_FLAG_USER_DIRTY
;
454 EXPORT_SYMBOL(ttm_tt_bind
);
456 static int ttm_tt_swapin(struct ttm_tt
*ttm
)
458 struct address_space
*swap_space
;
459 struct file
*swap_storage
;
460 struct page
*from_page
;
461 struct page
*to_page
;
467 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
) {
468 ret
= ttm_tt_set_user(ttm
, ttm
->tsk
, ttm
->start
,
470 if (unlikely(ret
!= 0))
473 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SWAPPED
;
477 swap_storage
= ttm
->swap_storage
;
478 BUG_ON(swap_storage
== NULL
);
480 swap_space
= swap_storage
->f_path
.dentry
->d_inode
->i_mapping
;
482 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
483 from_page
= read_mapping_page(swap_space
, i
, NULL
);
484 if (IS_ERR(from_page
)) {
485 ret
= PTR_ERR(from_page
);
488 to_page
= __ttm_tt_get_page(ttm
, i
);
489 if (unlikely(to_page
== NULL
))
493 from_virtual
= kmap_atomic(from_page
, KM_USER0
);
494 to_virtual
= kmap_atomic(to_page
, KM_USER1
);
495 memcpy(to_virtual
, from_virtual
, PAGE_SIZE
);
496 kunmap_atomic(to_virtual
, KM_USER1
);
497 kunmap_atomic(from_virtual
, KM_USER0
);
499 page_cache_release(from_page
);
502 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTANT_SWAP
))
504 ttm
->swap_storage
= NULL
;
505 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SWAPPED
;
509 ttm_tt_free_alloced_pages(ttm
);
513 int ttm_tt_swapout(struct ttm_tt
*ttm
, struct file
*persistant_swap_storage
)
515 struct address_space
*swap_space
;
516 struct file
*swap_storage
;
517 struct page
*from_page
;
518 struct page
*to_page
;
524 BUG_ON(ttm
->state
!= tt_unbound
&& ttm
->state
!= tt_unpopulated
);
525 BUG_ON(ttm
->caching_state
!= tt_cached
);
528 * For user buffers, just unpin the pages, as there should be
532 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
) {
533 ttm_tt_free_user_pages(ttm
);
534 ttm
->page_flags
|= TTM_PAGE_FLAG_SWAPPED
;
535 ttm
->swap_storage
= NULL
;
539 if (!persistant_swap_storage
) {
540 swap_storage
= shmem_file_setup("ttm swap",
541 ttm
->num_pages
<< PAGE_SHIFT
,
543 if (unlikely(IS_ERR(swap_storage
))) {
544 printk(KERN_ERR
"Failed allocating swap storage.\n");
545 return PTR_ERR(swap_storage
);
548 swap_storage
= persistant_swap_storage
;
550 swap_space
= swap_storage
->f_path
.dentry
->d_inode
->i_mapping
;
552 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
553 from_page
= ttm
->pages
[i
];
554 if (unlikely(from_page
== NULL
))
556 to_page
= read_mapping_page(swap_space
, i
, NULL
);
557 if (unlikely(IS_ERR(to_page
))) {
558 ret
= PTR_ERR(to_page
);
562 from_virtual
= kmap_atomic(from_page
, KM_USER0
);
563 to_virtual
= kmap_atomic(to_page
, KM_USER1
);
564 memcpy(to_virtual
, from_virtual
, PAGE_SIZE
);
565 kunmap_atomic(to_virtual
, KM_USER1
);
566 kunmap_atomic(from_virtual
, KM_USER0
);
568 set_page_dirty(to_page
);
569 mark_page_accessed(to_page
);
570 page_cache_release(to_page
);
573 ttm_tt_free_alloced_pages(ttm
);
574 ttm
->swap_storage
= swap_storage
;
575 ttm
->page_flags
|= TTM_PAGE_FLAG_SWAPPED
;
576 if (persistant_swap_storage
)
577 ttm
->page_flags
|= TTM_PAGE_FLAG_PERSISTANT_SWAP
;
581 if (!persistant_swap_storage
)