1 /**************************************************************************
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
31 #include <linux/sched.h>
32 #include <linux/highmem.h>
33 #include <linux/pagemap.h>
34 #include <linux/file.h>
35 #include <linux/swap.h>
36 #include "drm_cache.h"
37 #include "drm_mem_util.h"
38 #include "ttm/ttm_module.h"
39 #include "ttm/ttm_bo_driver.h"
40 #include "ttm/ttm_placement.h"
41 #include "ttm/ttm_page_alloc.h"
43 static int ttm_tt_swapin(struct ttm_tt
*ttm
);
46 * Allocates storage for pointers to the pages that back the ttm.
48 static void ttm_tt_alloc_page_directory(struct ttm_tt
*ttm
)
50 ttm
->pages
= drm_calloc_large(ttm
->num_pages
, sizeof(*ttm
->pages
));
53 static void ttm_tt_free_page_directory(struct ttm_tt
*ttm
)
55 drm_free_large(ttm
->pages
);
59 static void ttm_tt_free_user_pages(struct ttm_tt
*ttm
)
65 struct ttm_backend
*be
= ttm
->be
;
67 BUG_ON(!(ttm
->page_flags
& TTM_PAGE_FLAG_USER
));
68 write
= ((ttm
->page_flags
& TTM_PAGE_FLAG_WRITE
) != 0);
69 dirty
= ((ttm
->page_flags
& TTM_PAGE_FLAG_USER_DIRTY
) != 0);
74 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
79 if (page
== ttm
->dummy_read_page
) {
84 if (write
&& dirty
&& !PageReserved(page
))
85 set_page_dirty_lock(page
);
88 ttm_mem_global_free(ttm
->glob
->mem_glob
, PAGE_SIZE
);
91 ttm
->state
= tt_unpopulated
;
92 ttm
->first_himem_page
= ttm
->num_pages
;
93 ttm
->last_lomem_page
= -1;
96 static struct page
*__ttm_tt_get_page(struct ttm_tt
*ttm
, int index
)
100 struct ttm_mem_global
*mem_glob
= ttm
->glob
->mem_glob
;
103 while (NULL
== (p
= ttm
->pages
[index
])) {
107 ret
= ttm_get_pages(&h
, ttm
->page_flags
, ttm
->caching_state
, 1);
112 p
= list_first_entry(&h
, struct page
, lru
);
114 ret
= ttm_mem_global_alloc_page(mem_glob
, p
, false, false);
115 if (unlikely(ret
!= 0))
119 ttm
->pages
[--ttm
->first_himem_page
] = p
;
121 ttm
->pages
[++ttm
->last_lomem_page
] = p
;
129 struct page
*ttm_tt_get_page(struct ttm_tt
*ttm
, int index
)
133 if (unlikely(ttm
->page_flags
& TTM_PAGE_FLAG_SWAPPED
)) {
134 ret
= ttm_tt_swapin(ttm
);
135 if (unlikely(ret
!= 0))
138 return __ttm_tt_get_page(ttm
, index
);
141 int ttm_tt_populate(struct ttm_tt
*ttm
)
145 struct ttm_backend
*be
;
148 if (ttm
->state
!= tt_unpopulated
)
151 if (unlikely(ttm
->page_flags
& TTM_PAGE_FLAG_SWAPPED
)) {
152 ret
= ttm_tt_swapin(ttm
);
153 if (unlikely(ret
!= 0))
159 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
160 page
= __ttm_tt_get_page(ttm
, i
);
165 be
->func
->populate(be
, ttm
->num_pages
, ttm
->pages
,
166 ttm
->dummy_read_page
);
167 ttm
->state
= tt_unbound
;
170 EXPORT_SYMBOL(ttm_tt_populate
);
173 static inline int ttm_tt_set_page_caching(struct page
*p
,
174 enum ttm_caching_state c_old
,
175 enum ttm_caching_state c_new
)
182 if (c_old
!= tt_cached
) {
183 /* p isn't in the default caching state, set it to
184 * writeback first to free its current memtype. */
186 ret
= set_pages_wb(p
, 1);
192 ret
= set_memory_wc((unsigned long) page_address(p
), 1);
193 else if (c_new
== tt_uncached
)
194 ret
= set_pages_uc(p
, 1);
198 #else /* CONFIG_X86 */
199 static inline int ttm_tt_set_page_caching(struct page
*p
,
200 enum ttm_caching_state c_old
,
201 enum ttm_caching_state c_new
)
205 #endif /* CONFIG_X86 */
208 * Change caching policy for the linear kernel map
209 * for range of pages in a ttm.
212 static int ttm_tt_set_caching(struct ttm_tt
*ttm
,
213 enum ttm_caching_state c_state
)
216 struct page
*cur_page
;
219 if (ttm
->caching_state
== c_state
)
222 if (ttm
->state
== tt_unpopulated
) {
223 /* Change caching but don't populate */
224 ttm
->caching_state
= c_state
;
228 if (ttm
->caching_state
== tt_cached
)
229 drm_clflush_pages(ttm
->pages
, ttm
->num_pages
);
231 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
232 cur_page
= ttm
->pages
[i
];
233 if (likely(cur_page
!= NULL
)) {
234 ret
= ttm_tt_set_page_caching(cur_page
,
237 if (unlikely(ret
!= 0))
242 ttm
->caching_state
= c_state
;
247 for (j
= 0; j
< i
; ++j
) {
248 cur_page
= ttm
->pages
[j
];
249 if (likely(cur_page
!= NULL
)) {
250 (void)ttm_tt_set_page_caching(cur_page
, c_state
,
258 int ttm_tt_set_placement_caching(struct ttm_tt
*ttm
, uint32_t placement
)
260 enum ttm_caching_state state
;
262 if (placement
& TTM_PL_FLAG_WC
)
264 else if (placement
& TTM_PL_FLAG_UNCACHED
)
269 return ttm_tt_set_caching(ttm
, state
);
271 EXPORT_SYMBOL(ttm_tt_set_placement_caching
);
273 static void ttm_tt_free_alloced_pages(struct ttm_tt
*ttm
)
278 struct page
*cur_page
;
279 struct ttm_backend
*be
= ttm
->be
;
285 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
287 cur_page
= ttm
->pages
[i
];
288 ttm
->pages
[i
] = NULL
;
290 if (page_count(cur_page
) != 1)
291 printk(KERN_ERR TTM_PFX
292 "Erroneous page count. "
294 ttm_mem_global_free_page(ttm
->glob
->mem_glob
,
296 list_add(&cur_page
->lru
, &h
);
300 ttm_put_pages(&h
, count
, ttm
->page_flags
, ttm
->caching_state
);
301 ttm
->state
= tt_unpopulated
;
302 ttm
->first_himem_page
= ttm
->num_pages
;
303 ttm
->last_lomem_page
= -1;
306 void ttm_tt_destroy(struct ttm_tt
*ttm
)
308 struct ttm_backend
*be
;
310 if (unlikely(ttm
== NULL
))
314 if (likely(be
!= NULL
)) {
315 be
->func
->destroy(be
);
319 if (likely(ttm
->pages
!= NULL
)) {
320 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
)
321 ttm_tt_free_user_pages(ttm
);
323 ttm_tt_free_alloced_pages(ttm
);
325 ttm_tt_free_page_directory(ttm
);
328 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTANT_SWAP
) &&
330 fput(ttm
->swap_storage
);
335 int ttm_tt_set_user(struct ttm_tt
*ttm
,
336 struct task_struct
*tsk
,
337 unsigned long start
, unsigned long num_pages
)
339 struct mm_struct
*mm
= tsk
->mm
;
341 int write
= (ttm
->page_flags
& TTM_PAGE_FLAG_WRITE
) != 0;
342 struct ttm_mem_global
*mem_glob
= ttm
->glob
->mem_glob
;
344 BUG_ON(num_pages
!= ttm
->num_pages
);
345 BUG_ON((ttm
->page_flags
& TTM_PAGE_FLAG_USER
) == 0);
348 * Account user pages as lowmem pages for now.
351 ret
= ttm_mem_global_alloc(mem_glob
, num_pages
* PAGE_SIZE
,
353 if (unlikely(ret
!= 0))
356 down_read(&mm
->mmap_sem
);
357 ret
= get_user_pages(tsk
, mm
, start
, num_pages
,
358 write
, 0, ttm
->pages
, NULL
);
359 up_read(&mm
->mmap_sem
);
361 if (ret
!= num_pages
&& write
) {
362 ttm_tt_free_user_pages(ttm
);
363 ttm_mem_global_free(mem_glob
, num_pages
* PAGE_SIZE
);
369 ttm
->state
= tt_unbound
;
374 struct ttm_tt
*ttm_tt_create(struct ttm_bo_device
*bdev
, unsigned long size
,
375 uint32_t page_flags
, struct page
*dummy_read_page
)
377 struct ttm_bo_driver
*bo_driver
= bdev
->driver
;
383 ttm
= kzalloc(sizeof(*ttm
), GFP_KERNEL
);
387 ttm
->glob
= bdev
->glob
;
388 ttm
->num_pages
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
389 ttm
->first_himem_page
= ttm
->num_pages
;
390 ttm
->last_lomem_page
= -1;
391 ttm
->caching_state
= tt_cached
;
392 ttm
->page_flags
= page_flags
;
394 ttm
->dummy_read_page
= dummy_read_page
;
396 ttm_tt_alloc_page_directory(ttm
);
399 printk(KERN_ERR TTM_PFX
"Failed allocating page table\n");
402 ttm
->be
= bo_driver
->create_ttm_backend_entry(bdev
);
405 printk(KERN_ERR TTM_PFX
"Failed creating ttm backend entry\n");
408 ttm
->state
= tt_unpopulated
;
412 void ttm_tt_unbind(struct ttm_tt
*ttm
)
415 struct ttm_backend
*be
= ttm
->be
;
417 if (ttm
->state
== tt_bound
) {
418 ret
= be
->func
->unbind(be
);
420 ttm
->state
= tt_unbound
;
424 int ttm_tt_bind(struct ttm_tt
*ttm
, struct ttm_mem_reg
*bo_mem
)
427 struct ttm_backend
*be
;
432 if (ttm
->state
== tt_bound
)
437 ret
= ttm_tt_populate(ttm
);
441 ret
= be
->func
->bind(be
, bo_mem
);
443 printk(KERN_ERR TTM_PFX
"Couldn't bind backend.\n");
447 ttm
->state
= tt_bound
;
449 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
)
450 ttm
->page_flags
|= TTM_PAGE_FLAG_USER_DIRTY
;
453 EXPORT_SYMBOL(ttm_tt_bind
);
455 static int ttm_tt_swapin(struct ttm_tt
*ttm
)
457 struct address_space
*swap_space
;
458 struct file
*swap_storage
;
459 struct page
*from_page
;
460 struct page
*to_page
;
466 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
) {
467 ret
= ttm_tt_set_user(ttm
, ttm
->tsk
, ttm
->start
,
469 if (unlikely(ret
!= 0))
472 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SWAPPED
;
476 swap_storage
= ttm
->swap_storage
;
477 BUG_ON(swap_storage
== NULL
);
479 swap_space
= swap_storage
->f_path
.dentry
->d_inode
->i_mapping
;
481 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
482 from_page
= read_mapping_page(swap_space
, i
, NULL
);
483 if (IS_ERR(from_page
)) {
484 ret
= PTR_ERR(from_page
);
487 to_page
= __ttm_tt_get_page(ttm
, i
);
488 if (unlikely(to_page
== NULL
))
492 from_virtual
= kmap_atomic(from_page
, KM_USER0
);
493 to_virtual
= kmap_atomic(to_page
, KM_USER1
);
494 memcpy(to_virtual
, from_virtual
, PAGE_SIZE
);
495 kunmap_atomic(to_virtual
, KM_USER1
);
496 kunmap_atomic(from_virtual
, KM_USER0
);
498 page_cache_release(from_page
);
501 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTANT_SWAP
))
503 ttm
->swap_storage
= NULL
;
504 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SWAPPED
;
508 ttm_tt_free_alloced_pages(ttm
);
512 int ttm_tt_swapout(struct ttm_tt
*ttm
, struct file
*persistant_swap_storage
)
514 struct address_space
*swap_space
;
515 struct file
*swap_storage
;
516 struct page
*from_page
;
517 struct page
*to_page
;
523 BUG_ON(ttm
->state
!= tt_unbound
&& ttm
->state
!= tt_unpopulated
);
524 BUG_ON(ttm
->caching_state
!= tt_cached
);
527 * For user buffers, just unpin the pages, as there should be
531 if (ttm
->page_flags
& TTM_PAGE_FLAG_USER
) {
532 ttm_tt_free_user_pages(ttm
);
533 ttm
->page_flags
|= TTM_PAGE_FLAG_SWAPPED
;
534 ttm
->swap_storage
= NULL
;
538 if (!persistant_swap_storage
) {
539 swap_storage
= shmem_file_setup("ttm swap",
540 ttm
->num_pages
<< PAGE_SHIFT
,
542 if (unlikely(IS_ERR(swap_storage
))) {
543 printk(KERN_ERR
"Failed allocating swap storage.\n");
544 return PTR_ERR(swap_storage
);
547 swap_storage
= persistant_swap_storage
;
549 swap_space
= swap_storage
->f_path
.dentry
->d_inode
->i_mapping
;
551 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
552 from_page
= ttm
->pages
[i
];
553 if (unlikely(from_page
== NULL
))
555 to_page
= read_mapping_page(swap_space
, i
, NULL
);
556 if (unlikely(IS_ERR(to_page
))) {
557 ret
= PTR_ERR(to_page
);
561 from_virtual
= kmap_atomic(from_page
, KM_USER0
);
562 to_virtual
= kmap_atomic(to_page
, KM_USER1
);
563 memcpy(to_virtual
, from_virtual
, PAGE_SIZE
);
564 kunmap_atomic(to_virtual
, KM_USER1
);
565 kunmap_atomic(from_virtual
, KM_USER0
);
567 set_page_dirty(to_page
);
568 mark_page_accessed(to_page
);
569 page_cache_release(to_page
);
572 ttm_tt_free_alloced_pages(ttm
);
573 ttm
->swap_storage
= swap_storage
;
574 ttm
->page_flags
|= TTM_PAGE_FLAG_SWAPPED
;
575 if (persistant_swap_storage
)
576 ttm
->page_flags
|= TTM_PAGE_FLAG_PERSISTANT_SWAP
;
580 if (!persistant_swap_storage
)