Commit | Line | Data |
---|---|---|
8c8f1c95 AC |
1 | /************************************************************************** |
2 | * Copyright (c) 2007, Intel Corporation. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify it | |
5 | * under the terms and conditions of the GNU General Public License, | |
6 | * version 2, as published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope it will be useful, but WITHOUT | |
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
11 | * more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public License along with | |
14 | * this program; if not, write to the Free Software Foundation, Inc., | |
15 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
16 | * | |
17 | **************************************************************************/ | |
18 | #include <drm/drmP.h> | |
19 | #include "psb_drv.h" | |
20 | #include "psb_reg.h" | |
21 | ||
22 | /* | |
23 | * Code for the SGX MMU: | |
24 | */ | |
25 | ||
26 | /* | |
27 | * clflush on one processor only: | |
28 | * clflush should apparently flush the cache line on all processors in an | |
29 | * SMP system. | |
30 | */ | |
31 | ||
32 | /* | |
33 | * kmap atomic: | |
34 | * The usage of the slots must be completely encapsulated within a spinlock, and | |
35 | * no other functions that may be using the locks for other purposed may be | |
36 | * called from within the locked region. | |
37 | * Since the slots are per processor, this will guarantee that we are the only | |
38 | * user. | |
39 | */ | |
40 | ||
41 | /* | |
42 | * TODO: Inserting ptes from an interrupt handler: | |
43 | * This may be desirable for some SGX functionality where the GPU can fault in | |
44 | * needed pages. For that, we need to make an atomic insert_pages function, that | |
45 | * may fail. | |
46 | * If it fails, the caller need to insert the page using a workqueue function, | |
47 | * but on average it should be fast. | |
48 | */ | |
49 | ||
50 | struct psb_mmu_driver { | |
51 | /* protects driver- and pd structures. Always take in read mode | |
52 | * before taking the page table spinlock. | |
53 | */ | |
54 | struct rw_semaphore sem; | |
55 | ||
56 | /* protects page tables, directory tables and pt tables. | |
57 | * and pt structures. | |
58 | */ | |
59 | spinlock_t lock; | |
60 | ||
61 | atomic_t needs_tlbflush; | |
62 | ||
63 | uint8_t __iomem *register_map; | |
64 | struct psb_mmu_pd *default_pd; | |
65 | /*uint32_t bif_ctrl;*/ | |
66 | int has_clflush; | |
67 | int clflush_add; | |
68 | unsigned long clflush_mask; | |
69 | ||
70 | struct drm_psb_private *dev_priv; | |
71 | }; | |
72 | ||
73 | struct psb_mmu_pd; | |
74 | ||
75 | struct psb_mmu_pt { | |
76 | struct psb_mmu_pd *pd; | |
77 | uint32_t index; | |
78 | uint32_t count; | |
79 | struct page *p; | |
80 | uint32_t *v; | |
81 | }; | |
82 | ||
83 | struct psb_mmu_pd { | |
84 | struct psb_mmu_driver *driver; | |
85 | int hw_context; | |
86 | struct psb_mmu_pt **tables; | |
87 | struct page *p; | |
88 | struct page *dummy_pt; | |
89 | struct page *dummy_page; | |
90 | uint32_t pd_mask; | |
91 | uint32_t invalid_pde; | |
92 | uint32_t invalid_pte; | |
93 | }; | |
94 | ||
95 | static inline uint32_t psb_mmu_pt_index(uint32_t offset) | |
96 | { | |
97 | return (offset >> PSB_PTE_SHIFT) & 0x3FF; | |
98 | } | |
99 | ||
100 | static inline uint32_t psb_mmu_pd_index(uint32_t offset) | |
101 | { | |
102 | return offset >> PSB_PDE_SHIFT; | |
103 | } | |
104 | ||
105 | static inline void psb_clflush(void *addr) | |
106 | { | |
107 | __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory"); | |
108 | } | |
109 | ||
110 | static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, | |
111 | void *addr) | |
112 | { | |
113 | if (!driver->has_clflush) | |
114 | return; | |
115 | ||
116 | mb(); | |
117 | psb_clflush(addr); | |
118 | mb(); | |
119 | } | |
120 | ||
121 | static void psb_page_clflush(struct psb_mmu_driver *driver, struct page* page) | |
122 | { | |
123 | uint32_t clflush_add = driver->clflush_add >> PAGE_SHIFT; | |
124 | uint32_t clflush_count = PAGE_SIZE / clflush_add; | |
125 | int i; | |
126 | uint8_t *clf; | |
127 | ||
f0c5b592 | 128 | clf = kmap_atomic(page); |
8c8f1c95 AC |
129 | mb(); |
130 | for (i = 0; i < clflush_count; ++i) { | |
131 | psb_clflush(clf); | |
132 | clf += clflush_add; | |
133 | } | |
134 | mb(); | |
f0c5b592 | 135 | kunmap_atomic(clf); |
8c8f1c95 AC |
136 | } |
137 | ||
138 | static void psb_pages_clflush(struct psb_mmu_driver *driver, | |
139 | struct page *page[], unsigned long num_pages) | |
140 | { | |
141 | int i; | |
142 | ||
143 | if (!driver->has_clflush) | |
144 | return ; | |
145 | ||
146 | for (i = 0; i < num_pages; i++) | |
147 | psb_page_clflush(driver, *page++); | |
148 | } | |
149 | ||
150 | static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, | |
151 | int force) | |
152 | { | |
153 | atomic_set(&driver->needs_tlbflush, 0); | |
154 | } | |
155 | ||
156 | static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force) | |
157 | { | |
158 | down_write(&driver->sem); | |
159 | psb_mmu_flush_pd_locked(driver, force); | |
160 | up_write(&driver->sem); | |
161 | } | |
162 | ||
163 | void psb_mmu_flush(struct psb_mmu_driver *driver, int rc_prot) | |
164 | { | |
165 | if (rc_prot) | |
166 | down_write(&driver->sem); | |
167 | if (rc_prot) | |
168 | up_write(&driver->sem); | |
169 | } | |
170 | ||
171 | void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context) | |
172 | { | |
173 | /*ttm_tt_cache_flush(&pd->p, 1);*/ | |
174 | psb_pages_clflush(pd->driver, &pd->p, 1); | |
175 | down_write(&pd->driver->sem); | |
176 | wmb(); | |
177 | psb_mmu_flush_pd_locked(pd->driver, 1); | |
178 | pd->hw_context = hw_context; | |
179 | up_write(&pd->driver->sem); | |
180 | ||
181 | } | |
182 | ||
183 | static inline unsigned long psb_pd_addr_end(unsigned long addr, | |
184 | unsigned long end) | |
185 | { | |
186 | ||
187 | addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK; | |
188 | return (addr < end) ? addr : end; | |
189 | } | |
190 | ||
191 | static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type) | |
192 | { | |
193 | uint32_t mask = PSB_PTE_VALID; | |
194 | ||
195 | if (type & PSB_MMU_CACHED_MEMORY) | |
196 | mask |= PSB_PTE_CACHED; | |
197 | if (type & PSB_MMU_RO_MEMORY) | |
198 | mask |= PSB_PTE_RO; | |
199 | if (type & PSB_MMU_WO_MEMORY) | |
200 | mask |= PSB_PTE_WO; | |
201 | ||
202 | return (pfn << PAGE_SHIFT) | mask; | |
203 | } | |
204 | ||
205 | struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver, | |
206 | int trap_pagefaults, int invalid_type) | |
207 | { | |
208 | struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL); | |
209 | uint32_t *v; | |
210 | int i; | |
211 | ||
212 | if (!pd) | |
213 | return NULL; | |
214 | ||
215 | pd->p = alloc_page(GFP_DMA32); | |
216 | if (!pd->p) | |
217 | goto out_err1; | |
218 | pd->dummy_pt = alloc_page(GFP_DMA32); | |
219 | if (!pd->dummy_pt) | |
220 | goto out_err2; | |
221 | pd->dummy_page = alloc_page(GFP_DMA32); | |
222 | if (!pd->dummy_page) | |
223 | goto out_err3; | |
224 | ||
225 | if (!trap_pagefaults) { | |
226 | pd->invalid_pde = | |
227 | psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt), | |
228 | invalid_type); | |
229 | pd->invalid_pte = | |
230 | psb_mmu_mask_pte(page_to_pfn(pd->dummy_page), | |
231 | invalid_type); | |
232 | } else { | |
233 | pd->invalid_pde = 0; | |
234 | pd->invalid_pte = 0; | |
235 | } | |
236 | ||
237 | v = kmap(pd->dummy_pt); | |
238 | for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) | |
239 | v[i] = pd->invalid_pte; | |
240 | ||
241 | kunmap(pd->dummy_pt); | |
242 | ||
243 | v = kmap(pd->p); | |
244 | for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) | |
245 | v[i] = pd->invalid_pde; | |
246 | ||
247 | kunmap(pd->p); | |
248 | ||
249 | clear_page(kmap(pd->dummy_page)); | |
250 | kunmap(pd->dummy_page); | |
251 | ||
252 | pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024); | |
253 | if (!pd->tables) | |
254 | goto out_err4; | |
255 | ||
256 | pd->hw_context = -1; | |
257 | pd->pd_mask = PSB_PTE_VALID; | |
258 | pd->driver = driver; | |
259 | ||
260 | return pd; | |
261 | ||
262 | out_err4: | |
263 | __free_page(pd->dummy_page); | |
264 | out_err3: | |
265 | __free_page(pd->dummy_pt); | |
266 | out_err2: | |
267 | __free_page(pd->p); | |
268 | out_err1: | |
269 | kfree(pd); | |
270 | return NULL; | |
271 | } | |
272 | ||
3afad3c2 | 273 | static void psb_mmu_free_pt(struct psb_mmu_pt *pt) |
8c8f1c95 AC |
274 | { |
275 | __free_page(pt->p); | |
276 | kfree(pt); | |
277 | } | |
278 | ||
279 | void psb_mmu_free_pagedir(struct psb_mmu_pd *pd) | |
280 | { | |
281 | struct psb_mmu_driver *driver = pd->driver; | |
282 | struct psb_mmu_pt *pt; | |
283 | int i; | |
284 | ||
285 | down_write(&driver->sem); | |
286 | if (pd->hw_context != -1) | |
287 | psb_mmu_flush_pd_locked(driver, 1); | |
288 | ||
289 | /* Should take the spinlock here, but we don't need to do that | |
290 | since we have the semaphore in write mode. */ | |
291 | ||
292 | for (i = 0; i < 1024; ++i) { | |
293 | pt = pd->tables[i]; | |
294 | if (pt) | |
295 | psb_mmu_free_pt(pt); | |
296 | } | |
297 | ||
298 | vfree(pd->tables); | |
299 | __free_page(pd->dummy_page); | |
300 | __free_page(pd->dummy_pt); | |
301 | __free_page(pd->p); | |
302 | kfree(pd); | |
303 | up_write(&driver->sem); | |
304 | } | |
305 | ||
306 | static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd) | |
307 | { | |
308 | struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL); | |
309 | void *v; | |
310 | uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT; | |
311 | uint32_t clflush_count = PAGE_SIZE / clflush_add; | |
312 | spinlock_t *lock = &pd->driver->lock; | |
313 | uint8_t *clf; | |
314 | uint32_t *ptes; | |
315 | int i; | |
316 | ||
317 | if (!pt) | |
318 | return NULL; | |
319 | ||
320 | pt->p = alloc_page(GFP_DMA32); | |
321 | if (!pt->p) { | |
322 | kfree(pt); | |
323 | return NULL; | |
324 | } | |
325 | ||
326 | spin_lock(lock); | |
327 | ||
f0c5b592 | 328 | v = kmap_atomic(pt->p); |
8c8f1c95 AC |
329 | clf = (uint8_t *) v; |
330 | ptes = (uint32_t *) v; | |
331 | for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i) | |
332 | *ptes++ = pd->invalid_pte; | |
333 | ||
334 | ||
335 | if (pd->driver->has_clflush && pd->hw_context != -1) { | |
336 | mb(); | |
337 | for (i = 0; i < clflush_count; ++i) { | |
338 | psb_clflush(clf); | |
339 | clf += clflush_add; | |
340 | } | |
341 | mb(); | |
342 | } | |
343 | ||
f0c5b592 | 344 | kunmap_atomic(v); |
8c8f1c95 AC |
345 | spin_unlock(lock); |
346 | ||
347 | pt->count = 0; | |
348 | pt->pd = pd; | |
349 | pt->index = 0; | |
350 | ||
351 | return pt; | |
352 | } | |
353 | ||
3afad3c2 | 354 | static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd, |
8c8f1c95 AC |
355 | unsigned long addr) |
356 | { | |
357 | uint32_t index = psb_mmu_pd_index(addr); | |
358 | struct psb_mmu_pt *pt; | |
359 | uint32_t *v; | |
360 | spinlock_t *lock = &pd->driver->lock; | |
361 | ||
362 | spin_lock(lock); | |
363 | pt = pd->tables[index]; | |
364 | while (!pt) { | |
365 | spin_unlock(lock); | |
366 | pt = psb_mmu_alloc_pt(pd); | |
367 | if (!pt) | |
368 | return NULL; | |
369 | spin_lock(lock); | |
370 | ||
371 | if (pd->tables[index]) { | |
372 | spin_unlock(lock); | |
373 | psb_mmu_free_pt(pt); | |
374 | spin_lock(lock); | |
375 | pt = pd->tables[index]; | |
376 | continue; | |
377 | } | |
378 | ||
f0c5b592 | 379 | v = kmap_atomic(pd->p); |
8c8f1c95 AC |
380 | pd->tables[index] = pt; |
381 | v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask; | |
382 | pt->index = index; | |
f0c5b592 | 383 | kunmap_atomic((void *) v); |
8c8f1c95 AC |
384 | |
385 | if (pd->hw_context != -1) { | |
386 | psb_mmu_clflush(pd->driver, (void *) &v[index]); | |
387 | atomic_set(&pd->driver->needs_tlbflush, 1); | |
388 | } | |
389 | } | |
f0c5b592 | 390 | pt->v = kmap_atomic(pt->p); |
8c8f1c95 AC |
391 | return pt; |
392 | } | |
393 | ||
394 | static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd, | |
395 | unsigned long addr) | |
396 | { | |
397 | uint32_t index = psb_mmu_pd_index(addr); | |
398 | struct psb_mmu_pt *pt; | |
399 | spinlock_t *lock = &pd->driver->lock; | |
400 | ||
401 | spin_lock(lock); | |
402 | pt = pd->tables[index]; | |
403 | if (!pt) { | |
404 | spin_unlock(lock); | |
405 | return NULL; | |
406 | } | |
f0c5b592 | 407 | pt->v = kmap_atomic(pt->p); |
8c8f1c95 AC |
408 | return pt; |
409 | } | |
410 | ||
411 | static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt) | |
412 | { | |
413 | struct psb_mmu_pd *pd = pt->pd; | |
414 | uint32_t *v; | |
415 | ||
f0c5b592 | 416 | kunmap_atomic(pt->v); |
8c8f1c95 | 417 | if (pt->count == 0) { |
f0c5b592 | 418 | v = kmap_atomic(pd->p); |
8c8f1c95 AC |
419 | v[pt->index] = pd->invalid_pde; |
420 | pd->tables[pt->index] = NULL; | |
421 | ||
422 | if (pd->hw_context != -1) { | |
423 | psb_mmu_clflush(pd->driver, | |
424 | (void *) &v[pt->index]); | |
425 | atomic_set(&pd->driver->needs_tlbflush, 1); | |
426 | } | |
f0c5b592 | 427 | kunmap_atomic(pt->v); |
8c8f1c95 AC |
428 | spin_unlock(&pd->driver->lock); |
429 | psb_mmu_free_pt(pt); | |
430 | return; | |
431 | } | |
432 | spin_unlock(&pd->driver->lock); | |
433 | } | |
434 | ||
435 | static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, | |
436 | unsigned long addr, uint32_t pte) | |
437 | { | |
438 | pt->v[psb_mmu_pt_index(addr)] = pte; | |
439 | } | |
440 | ||
441 | static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt, | |
442 | unsigned long addr) | |
443 | { | |
444 | pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte; | |
445 | } | |
446 | ||
447 | ||
448 | void psb_mmu_mirror_gtt(struct psb_mmu_pd *pd, | |
449 | uint32_t mmu_offset, uint32_t gtt_start, | |
450 | uint32_t gtt_pages) | |
451 | { | |
452 | uint32_t *v; | |
453 | uint32_t start = psb_mmu_pd_index(mmu_offset); | |
454 | struct psb_mmu_driver *driver = pd->driver; | |
455 | int num_pages = gtt_pages; | |
456 | ||
457 | down_read(&driver->sem); | |
458 | spin_lock(&driver->lock); | |
459 | ||
f0c5b592 | 460 | v = kmap_atomic(pd->p); |
8c8f1c95 AC |
461 | v += start; |
462 | ||
463 | while (gtt_pages--) { | |
464 | *v++ = gtt_start | pd->pd_mask; | |
465 | gtt_start += PAGE_SIZE; | |
466 | } | |
467 | ||
468 | /*ttm_tt_cache_flush(&pd->p, num_pages);*/ | |
469 | psb_pages_clflush(pd->driver, &pd->p, num_pages); | |
f0c5b592 | 470 | kunmap_atomic(v); |
8c8f1c95 AC |
471 | spin_unlock(&driver->lock); |
472 | ||
473 | if (pd->hw_context != -1) | |
474 | atomic_set(&pd->driver->needs_tlbflush, 1); | |
475 | ||
476 | up_read(&pd->driver->sem); | |
477 | psb_mmu_flush_pd(pd->driver, 0); | |
478 | } | |
479 | ||
480 | struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver) | |
481 | { | |
482 | struct psb_mmu_pd *pd; | |
483 | ||
484 | /* down_read(&driver->sem); */ | |
485 | pd = driver->default_pd; | |
486 | /* up_read(&driver->sem); */ | |
487 | ||
488 | return pd; | |
489 | } | |
490 | ||
8c8f1c95 AC |
491 | void psb_mmu_driver_takedown(struct psb_mmu_driver *driver) |
492 | { | |
493 | psb_mmu_free_pagedir(driver->default_pd); | |
494 | kfree(driver); | |
495 | } | |
496 | ||
497 | struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers, | |
498 | int trap_pagefaults, | |
499 | int invalid_type, | |
500 | struct drm_psb_private *dev_priv) | |
501 | { | |
502 | struct psb_mmu_driver *driver; | |
503 | ||
504 | driver = kmalloc(sizeof(*driver), GFP_KERNEL); | |
505 | ||
506 | if (!driver) | |
507 | return NULL; | |
508 | driver->dev_priv = dev_priv; | |
509 | ||
510 | driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults, | |
511 | invalid_type); | |
512 | if (!driver->default_pd) | |
513 | goto out_err1; | |
514 | ||
515 | spin_lock_init(&driver->lock); | |
516 | init_rwsem(&driver->sem); | |
517 | down_write(&driver->sem); | |
518 | driver->register_map = registers; | |
519 | atomic_set(&driver->needs_tlbflush, 1); | |
520 | ||
521 | driver->has_clflush = 0; | |
522 | ||
523 | if (boot_cpu_has(X86_FEATURE_CLFLSH)) { | |
524 | uint32_t tfms, misc, cap0, cap4, clflush_size; | |
525 | ||
526 | /* | |
527 | * clflush size is determined at kernel setup for x86_64 | |
528 | * but not for i386. We have to do it here. | |
529 | */ | |
530 | ||
531 | cpuid(0x00000001, &tfms, &misc, &cap0, &cap4); | |
532 | clflush_size = ((misc >> 8) & 0xff) * 8; | |
533 | driver->has_clflush = 1; | |
534 | driver->clflush_add = | |
535 | PAGE_SIZE * clflush_size / sizeof(uint32_t); | |
536 | driver->clflush_mask = driver->clflush_add - 1; | |
537 | driver->clflush_mask = ~driver->clflush_mask; | |
538 | } | |
539 | ||
540 | up_write(&driver->sem); | |
541 | return driver; | |
542 | ||
543 | out_err1: | |
544 | kfree(driver); | |
545 | return NULL; | |
546 | } | |
547 | ||
548 | static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, | |
549 | unsigned long address, uint32_t num_pages, | |
550 | uint32_t desired_tile_stride, | |
551 | uint32_t hw_tile_stride) | |
552 | { | |
553 | struct psb_mmu_pt *pt; | |
554 | uint32_t rows = 1; | |
555 | uint32_t i; | |
556 | unsigned long addr; | |
557 | unsigned long end; | |
558 | unsigned long next; | |
559 | unsigned long add; | |
560 | unsigned long row_add; | |
561 | unsigned long clflush_add = pd->driver->clflush_add; | |
562 | unsigned long clflush_mask = pd->driver->clflush_mask; | |
563 | ||
564 | if (!pd->driver->has_clflush) { | |
565 | /*ttm_tt_cache_flush(&pd->p, num_pages);*/ | |
566 | psb_pages_clflush(pd->driver, &pd->p, num_pages); | |
567 | return; | |
568 | } | |
569 | ||
570 | if (hw_tile_stride) | |
571 | rows = num_pages / desired_tile_stride; | |
572 | else | |
573 | desired_tile_stride = num_pages; | |
574 | ||
575 | add = desired_tile_stride << PAGE_SHIFT; | |
576 | row_add = hw_tile_stride << PAGE_SHIFT; | |
577 | mb(); | |
578 | for (i = 0; i < rows; ++i) { | |
579 | ||
580 | addr = address; | |
581 | end = addr + add; | |
582 | ||
583 | do { | |
584 | next = psb_pd_addr_end(addr, end); | |
585 | pt = psb_mmu_pt_map_lock(pd, addr); | |
586 | if (!pt) | |
587 | continue; | |
588 | do { | |
589 | psb_clflush(&pt->v | |
590 | [psb_mmu_pt_index(addr)]); | |
591 | } while (addr += | |
592 | clflush_add, | |
593 | (addr & clflush_mask) < next); | |
594 | ||
595 | psb_mmu_pt_unmap_unlock(pt); | |
596 | } while (addr = next, next != end); | |
597 | address += row_add; | |
598 | } | |
599 | mb(); | |
600 | } | |
601 | ||
602 | void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd, | |
603 | unsigned long address, uint32_t num_pages) | |
604 | { | |
605 | struct psb_mmu_pt *pt; | |
606 | unsigned long addr; | |
607 | unsigned long end; | |
608 | unsigned long next; | |
609 | unsigned long f_address = address; | |
610 | ||
611 | down_read(&pd->driver->sem); | |
612 | ||
613 | addr = address; | |
614 | end = addr + (num_pages << PAGE_SHIFT); | |
615 | ||
616 | do { | |
617 | next = psb_pd_addr_end(addr, end); | |
618 | pt = psb_mmu_pt_alloc_map_lock(pd, addr); | |
619 | if (!pt) | |
620 | goto out; | |
621 | do { | |
622 | psb_mmu_invalidate_pte(pt, addr); | |
623 | --pt->count; | |
624 | } while (addr += PAGE_SIZE, addr < next); | |
625 | psb_mmu_pt_unmap_unlock(pt); | |
626 | ||
627 | } while (addr = next, next != end); | |
628 | ||
629 | out: | |
630 | if (pd->hw_context != -1) | |
631 | psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); | |
632 | ||
633 | up_read(&pd->driver->sem); | |
634 | ||
635 | if (pd->hw_context != -1) | |
636 | psb_mmu_flush(pd->driver, 0); | |
637 | ||
638 | return; | |
639 | } | |
640 | ||
641 | void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address, | |
642 | uint32_t num_pages, uint32_t desired_tile_stride, | |
643 | uint32_t hw_tile_stride) | |
644 | { | |
645 | struct psb_mmu_pt *pt; | |
646 | uint32_t rows = 1; | |
647 | uint32_t i; | |
648 | unsigned long addr; | |
649 | unsigned long end; | |
650 | unsigned long next; | |
651 | unsigned long add; | |
652 | unsigned long row_add; | |
653 | unsigned long f_address = address; | |
654 | ||
655 | if (hw_tile_stride) | |
656 | rows = num_pages / desired_tile_stride; | |
657 | else | |
658 | desired_tile_stride = num_pages; | |
659 | ||
660 | add = desired_tile_stride << PAGE_SHIFT; | |
661 | row_add = hw_tile_stride << PAGE_SHIFT; | |
662 | ||
663 | /* down_read(&pd->driver->sem); */ | |
664 | ||
665 | /* Make sure we only need to flush this processor's cache */ | |
666 | ||
667 | for (i = 0; i < rows; ++i) { | |
668 | ||
669 | addr = address; | |
670 | end = addr + add; | |
671 | ||
672 | do { | |
673 | next = psb_pd_addr_end(addr, end); | |
674 | pt = psb_mmu_pt_map_lock(pd, addr); | |
675 | if (!pt) | |
676 | continue; | |
677 | do { | |
678 | psb_mmu_invalidate_pte(pt, addr); | |
679 | --pt->count; | |
680 | ||
681 | } while (addr += PAGE_SIZE, addr < next); | |
682 | psb_mmu_pt_unmap_unlock(pt); | |
683 | ||
684 | } while (addr = next, next != end); | |
685 | address += row_add; | |
686 | } | |
687 | if (pd->hw_context != -1) | |
688 | psb_mmu_flush_ptes(pd, f_address, num_pages, | |
689 | desired_tile_stride, hw_tile_stride); | |
690 | ||
691 | /* up_read(&pd->driver->sem); */ | |
692 | ||
693 | if (pd->hw_context != -1) | |
694 | psb_mmu_flush(pd->driver, 0); | |
695 | } | |
696 | ||
697 | int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn, | |
698 | unsigned long address, uint32_t num_pages, | |
699 | int type) | |
700 | { | |
701 | struct psb_mmu_pt *pt; | |
702 | uint32_t pte; | |
703 | unsigned long addr; | |
704 | unsigned long end; | |
705 | unsigned long next; | |
706 | unsigned long f_address = address; | |
707 | int ret = 0; | |
708 | ||
709 | down_read(&pd->driver->sem); | |
710 | ||
711 | addr = address; | |
712 | end = addr + (num_pages << PAGE_SHIFT); | |
713 | ||
714 | do { | |
715 | next = psb_pd_addr_end(addr, end); | |
716 | pt = psb_mmu_pt_alloc_map_lock(pd, addr); | |
717 | if (!pt) { | |
718 | ret = -ENOMEM; | |
719 | goto out; | |
720 | } | |
721 | do { | |
722 | pte = psb_mmu_mask_pte(start_pfn++, type); | |
723 | psb_mmu_set_pte(pt, addr, pte); | |
724 | pt->count++; | |
725 | } while (addr += PAGE_SIZE, addr < next); | |
726 | psb_mmu_pt_unmap_unlock(pt); | |
727 | ||
728 | } while (addr = next, next != end); | |
729 | ||
730 | out: | |
731 | if (pd->hw_context != -1) | |
732 | psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1); | |
733 | ||
734 | up_read(&pd->driver->sem); | |
735 | ||
736 | if (pd->hw_context != -1) | |
737 | psb_mmu_flush(pd->driver, 1); | |
738 | ||
739 | return ret; | |
740 | } | |
741 | ||
742 | int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages, | |
743 | unsigned long address, uint32_t num_pages, | |
744 | uint32_t desired_tile_stride, | |
745 | uint32_t hw_tile_stride, int type) | |
746 | { | |
747 | struct psb_mmu_pt *pt; | |
748 | uint32_t rows = 1; | |
749 | uint32_t i; | |
750 | uint32_t pte; | |
751 | unsigned long addr; | |
752 | unsigned long end; | |
753 | unsigned long next; | |
754 | unsigned long add; | |
755 | unsigned long row_add; | |
756 | unsigned long f_address = address; | |
757 | int ret = 0; | |
758 | ||
759 | if (hw_tile_stride) { | |
760 | if (num_pages % desired_tile_stride != 0) | |
761 | return -EINVAL; | |
762 | rows = num_pages / desired_tile_stride; | |
763 | } else { | |
764 | desired_tile_stride = num_pages; | |
765 | } | |
766 | ||
767 | add = desired_tile_stride << PAGE_SHIFT; | |
768 | row_add = hw_tile_stride << PAGE_SHIFT; | |
769 | ||
770 | down_read(&pd->driver->sem); | |
771 | ||
772 | for (i = 0; i < rows; ++i) { | |
773 | ||
774 | addr = address; | |
775 | end = addr + add; | |
776 | ||
777 | do { | |
778 | next = psb_pd_addr_end(addr, end); | |
779 | pt = psb_mmu_pt_alloc_map_lock(pd, addr); | |
780 | if (!pt) { | |
781 | ret = -ENOMEM; | |
782 | goto out; | |
783 | } | |
784 | do { | |
785 | pte = | |
786 | psb_mmu_mask_pte(page_to_pfn(*pages++), | |
787 | type); | |
788 | psb_mmu_set_pte(pt, addr, pte); | |
789 | pt->count++; | |
790 | } while (addr += PAGE_SIZE, addr < next); | |
791 | psb_mmu_pt_unmap_unlock(pt); | |
792 | ||
793 | } while (addr = next, next != end); | |
794 | ||
795 | address += row_add; | |
796 | } | |
797 | out: | |
798 | if (pd->hw_context != -1) | |
799 | psb_mmu_flush_ptes(pd, f_address, num_pages, | |
800 | desired_tile_stride, hw_tile_stride); | |
801 | ||
802 | up_read(&pd->driver->sem); | |
803 | ||
804 | if (pd->hw_context != -1) | |
805 | psb_mmu_flush(pd->driver, 1); | |
806 | ||
807 | return ret; | |
808 | } | |
809 | ||
810 | int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual, | |
811 | unsigned long *pfn) | |
812 | { | |
813 | int ret; | |
814 | struct psb_mmu_pt *pt; | |
815 | uint32_t tmp; | |
816 | spinlock_t *lock = &pd->driver->lock; | |
817 | ||
818 | down_read(&pd->driver->sem); | |
819 | pt = psb_mmu_pt_map_lock(pd, virtual); | |
820 | if (!pt) { | |
821 | uint32_t *v; | |
822 | ||
823 | spin_lock(lock); | |
f0c5b592 | 824 | v = kmap_atomic(pd->p); |
8c8f1c95 | 825 | tmp = v[psb_mmu_pd_index(virtual)]; |
f0c5b592 | 826 | kunmap_atomic(v); |
8c8f1c95 AC |
827 | spin_unlock(lock); |
828 | ||
829 | if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) || | |
830 | !(pd->invalid_pte & PSB_PTE_VALID)) { | |
831 | ret = -EINVAL; | |
832 | goto out; | |
833 | } | |
834 | ret = 0; | |
835 | *pfn = pd->invalid_pte >> PAGE_SHIFT; | |
836 | goto out; | |
837 | } | |
838 | tmp = pt->v[psb_mmu_pt_index(virtual)]; | |
839 | if (!(tmp & PSB_PTE_VALID)) { | |
840 | ret = -EINVAL; | |
841 | } else { | |
842 | ret = 0; | |
843 | *pfn = tmp >> PAGE_SHIFT; | |
844 | } | |
845 | psb_mmu_pt_unmap_unlock(pt); | |
846 | out: | |
847 | up_read(&pd->driver->sem); | |
848 | return ret; | |
849 | } |