1 /**************************************************************************
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
31 #include <linux/sched.h>
32 #include <linux/highmem.h>
33 #include <linux/pagemap.h>
34 #include <linux/shmem_fs.h>
35 #include <linux/file.h>
36 #include <linux/swap.h>
37 #include <linux/slab.h>
38 #include <linux/export.h>
39 #include "drm_cache.h"
40 #include "drm_mem_util.h"
41 #include "ttm/ttm_module.h"
42 #include "ttm/ttm_bo_driver.h"
43 #include "ttm/ttm_placement.h"
44 #include "ttm/ttm_page_alloc.h"
47 * Allocates storage for pointers to the pages that back the ttm.
49 static void ttm_tt_alloc_page_directory(struct ttm_tt
*ttm
)
51 ttm
->pages
= drm_calloc_large(ttm
->num_pages
, sizeof(*ttm
->pages
));
52 ttm
->dma_address
= drm_calloc_large(ttm
->num_pages
,
53 sizeof(*ttm
->dma_address
));
56 static void ttm_tt_free_page_directory(struct ttm_tt
*ttm
)
58 drm_free_large(ttm
->pages
);
60 drm_free_large(ttm
->dma_address
);
61 ttm
->dma_address
= NULL
;
65 static inline int ttm_tt_set_page_caching(struct page
*p
,
66 enum ttm_caching_state c_old
,
67 enum ttm_caching_state c_new
)
74 if (c_old
!= tt_cached
) {
75 /* p isn't in the default caching state, set it to
76 * writeback first to free its current memtype. */
78 ret
= set_pages_wb(p
, 1);
84 ret
= set_memory_wc((unsigned long) page_address(p
), 1);
85 else if (c_new
== tt_uncached
)
86 ret
= set_pages_uc(p
, 1);
90 #else /* CONFIG_X86 */
91 static inline int ttm_tt_set_page_caching(struct page
*p
,
92 enum ttm_caching_state c_old
,
93 enum ttm_caching_state c_new
)
97 #endif /* CONFIG_X86 */
100 * Change caching policy for the linear kernel map
101 * for range of pages in a ttm.
104 static int ttm_tt_set_caching(struct ttm_tt
*ttm
,
105 enum ttm_caching_state c_state
)
108 struct page
*cur_page
;
111 if (ttm
->caching_state
== c_state
)
114 if (ttm
->state
== tt_unpopulated
) {
115 /* Change caching but don't populate */
116 ttm
->caching_state
= c_state
;
120 if (ttm
->caching_state
== tt_cached
)
121 drm_clflush_pages(ttm
->pages
, ttm
->num_pages
);
123 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
124 cur_page
= ttm
->pages
[i
];
125 if (likely(cur_page
!= NULL
)) {
126 ret
= ttm_tt_set_page_caching(cur_page
,
129 if (unlikely(ret
!= 0))
134 ttm
->caching_state
= c_state
;
139 for (j
= 0; j
< i
; ++j
) {
140 cur_page
= ttm
->pages
[j
];
141 if (likely(cur_page
!= NULL
)) {
142 (void)ttm_tt_set_page_caching(cur_page
, c_state
,
150 int ttm_tt_set_placement_caching(struct ttm_tt
*ttm
, uint32_t placement
)
152 enum ttm_caching_state state
;
154 if (placement
& TTM_PL_FLAG_WC
)
156 else if (placement
& TTM_PL_FLAG_UNCACHED
)
161 return ttm_tt_set_caching(ttm
, state
);
163 EXPORT_SYMBOL(ttm_tt_set_placement_caching
);
165 void ttm_tt_destroy(struct ttm_tt
*ttm
)
167 if (unlikely(ttm
== NULL
))
170 if (ttm
->state
== tt_bound
) {
174 if (likely(ttm
->pages
!= NULL
)) {
175 ttm
->bdev
->driver
->ttm_tt_unpopulate(ttm
);
176 ttm_tt_free_page_directory(ttm
);
179 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTENT_SWAP
) &&
181 fput(ttm
->swap_storage
);
183 ttm
->swap_storage
= NULL
;
184 ttm
->func
->destroy(ttm
);
187 int ttm_tt_init(struct ttm_tt
*ttm
, struct ttm_bo_device
*bdev
,
188 unsigned long size
, uint32_t page_flags
,
189 struct page
*dummy_read_page
)
192 ttm
->glob
= bdev
->glob
;
193 ttm
->num_pages
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
194 ttm
->caching_state
= tt_cached
;
195 ttm
->page_flags
= page_flags
;
196 ttm
->dummy_read_page
= dummy_read_page
;
197 ttm
->state
= tt_unpopulated
;
199 ttm_tt_alloc_page_directory(ttm
);
200 if (!ttm
->pages
|| !ttm
->dma_address
) {
202 printk(KERN_ERR TTM_PFX
"Failed allocating page table\n");
207 EXPORT_SYMBOL(ttm_tt_init
);
209 void ttm_tt_unbind(struct ttm_tt
*ttm
)
213 if (ttm
->state
== tt_bound
) {
214 ret
= ttm
->func
->unbind(ttm
);
216 ttm
->state
= tt_unbound
;
220 int ttm_tt_bind(struct ttm_tt
*ttm
, struct ttm_mem_reg
*bo_mem
)
227 if (ttm
->state
== tt_bound
)
230 ret
= ttm
->bdev
->driver
->ttm_tt_populate(ttm
);
234 ret
= ttm
->func
->bind(ttm
, bo_mem
);
235 if (unlikely(ret
!= 0))
238 ttm
->state
= tt_bound
;
242 EXPORT_SYMBOL(ttm_tt_bind
);
244 int ttm_tt_swapin(struct ttm_tt
*ttm
)
246 struct address_space
*swap_space
;
247 struct file
*swap_storage
;
248 struct page
*from_page
;
249 struct page
*to_page
;
255 swap_storage
= ttm
->swap_storage
;
256 BUG_ON(swap_storage
== NULL
);
258 swap_space
= swap_storage
->f_path
.dentry
->d_inode
->i_mapping
;
260 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
261 from_page
= shmem_read_mapping_page(swap_space
, i
);
262 if (IS_ERR(from_page
)) {
263 ret
= PTR_ERR(from_page
);
266 to_page
= ttm
->pages
[i
];
267 if (unlikely(to_page
== NULL
))
271 from_virtual
= kmap_atomic(from_page
, KM_USER0
);
272 to_virtual
= kmap_atomic(to_page
, KM_USER1
);
273 memcpy(to_virtual
, from_virtual
, PAGE_SIZE
);
274 kunmap_atomic(to_virtual
, KM_USER1
);
275 kunmap_atomic(from_virtual
, KM_USER0
);
277 page_cache_release(from_page
);
280 if (!(ttm
->page_flags
& TTM_PAGE_FLAG_PERSISTENT_SWAP
))
282 ttm
->swap_storage
= NULL
;
283 ttm
->page_flags
&= ~TTM_PAGE_FLAG_SWAPPED
;
290 int ttm_tt_swapout(struct ttm_tt
*ttm
, struct file
*persistent_swap_storage
)
292 struct address_space
*swap_space
;
293 struct file
*swap_storage
;
294 struct page
*from_page
;
295 struct page
*to_page
;
301 BUG_ON(ttm
->state
!= tt_unbound
&& ttm
->state
!= tt_unpopulated
);
302 BUG_ON(ttm
->caching_state
!= tt_cached
);
304 if (!persistent_swap_storage
) {
305 swap_storage
= shmem_file_setup("ttm swap",
306 ttm
->num_pages
<< PAGE_SHIFT
,
308 if (unlikely(IS_ERR(swap_storage
))) {
309 printk(KERN_ERR
"Failed allocating swap storage.\n");
310 return PTR_ERR(swap_storage
);
313 swap_storage
= persistent_swap_storage
;
315 swap_space
= swap_storage
->f_path
.dentry
->d_inode
->i_mapping
;
317 for (i
= 0; i
< ttm
->num_pages
; ++i
) {
318 from_page
= ttm
->pages
[i
];
319 if (unlikely(from_page
== NULL
))
321 to_page
= shmem_read_mapping_page(swap_space
, i
);
322 if (unlikely(IS_ERR(to_page
))) {
323 ret
= PTR_ERR(to_page
);
327 from_virtual
= kmap_atomic(from_page
, KM_USER0
);
328 to_virtual
= kmap_atomic(to_page
, KM_USER1
);
329 memcpy(to_virtual
, from_virtual
, PAGE_SIZE
);
330 kunmap_atomic(to_virtual
, KM_USER1
);
331 kunmap_atomic(from_virtual
, KM_USER0
);
333 set_page_dirty(to_page
);
334 mark_page_accessed(to_page
);
335 page_cache_release(to_page
);
338 ttm
->bdev
->driver
->ttm_tt_unpopulate(ttm
);
339 ttm
->swap_storage
= swap_storage
;
340 ttm
->page_flags
|= TTM_PAGE_FLAG_SWAPPED
;
341 if (persistent_swap_storage
)
342 ttm
->page_flags
|= TTM_PAGE_FLAG_PERSISTENT_SWAP
;
346 if (!persistent_swap_storage
)