Commit | Line | Data |
---|---|---|
867e359b CM |
1 | /* |
2 | * Copyright 2010 Tilera Corporation. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation, version 2. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
11 | * NON INFRINGEMENT. See the GNU General Public License for | |
12 | * more details. | |
13 | */ | |
14 | ||
15 | #include <linux/highmem.h> | |
16 | #include <linux/module.h> | |
17 | #include <linux/pagemap.h> | |
18 | #include <asm/homecache.h> | |
19 | ||
20 | #define kmap_get_pte(vaddr) \ | |
21 | pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)),\ | |
22 | (vaddr)), (vaddr)) | |
23 | ||
24 | ||
25 | void *kmap(struct page *page) | |
26 | { | |
27 | void *kva; | |
28 | unsigned long flags; | |
29 | pte_t *ptep; | |
30 | ||
31 | might_sleep(); | |
32 | if (!PageHighMem(page)) | |
33 | return page_address(page); | |
34 | kva = kmap_high(page); | |
35 | ||
36 | /* | |
37 | * Rewrite the PTE under the lock. This ensures that the page | |
38 | * is not currently migrating. | |
39 | */ | |
40 | ptep = kmap_get_pte((unsigned long)kva); | |
41 | flags = homecache_kpte_lock(); | |
42 | set_pte_at(&init_mm, kva, ptep, mk_pte(page, page_to_kpgprot(page))); | |
43 | homecache_kpte_unlock(flags); | |
44 | ||
45 | return kva; | |
46 | } | |
47 | EXPORT_SYMBOL(kmap); | |
48 | ||
49 | void kunmap(struct page *page) | |
50 | { | |
51 | if (in_interrupt()) | |
52 | BUG(); | |
53 | if (!PageHighMem(page)) | |
54 | return; | |
55 | kunmap_high(page); | |
56 | } | |
57 | EXPORT_SYMBOL(kunmap); | |
58 | ||
867e359b CM |
59 | /* |
60 | * Describe a single atomic mapping of a page on a given cpu at a | |
61 | * given address, and allow it to be linked into a list. | |
62 | */ | |
63 | struct atomic_mapped_page { | |
64 | struct list_head list; | |
65 | struct page *page; | |
66 | int cpu; | |
67 | unsigned long va; | |
68 | }; | |
69 | ||
70 | static spinlock_t amp_lock = __SPIN_LOCK_UNLOCKED(&_lock); | |
71 | static struct list_head amp_list = LIST_HEAD_INIT(amp_list); | |
72 | ||
73 | /* | |
74 | * Combining this structure with a per-cpu declaration lets us give | |
75 | * each cpu an atomic_mapped_page structure per type. | |
76 | */ | |
77 | struct kmap_amps { | |
78 | struct atomic_mapped_page per_type[KM_TYPE_NR]; | |
79 | }; | |
0707ad30 | 80 | static DEFINE_PER_CPU(struct kmap_amps, amps); |
867e359b CM |
81 | |
82 | /* | |
83 | * Add a page and va, on this cpu, to the list of kmap_atomic pages, | |
84 | * and write the new pte to memory. Writing the new PTE under the | |
85 | * lock guarantees that it is either on the list before migration starts | |
86 | * (if we won the race), or set_pte() sets the migrating bit in the PTE | |
87 | * (if we lost the race). And doing it under the lock guarantees | |
88 | * that when kmap_atomic_fix_one_pte() comes along, it finds a valid | |
89 | * PTE in memory, iff the mapping is still on the amp_list. | |
90 | * | |
91 | * Finally, doing it under the lock lets us safely examine the page | |
92 | * to see if it is immutable or not, for the generic kmap_atomic() case. | |
93 | * If we examine it earlier we are exposed to a race where it looks | |
94 | * writable earlier, but becomes immutable before we write the PTE. | |
95 | */ | |
61d06c83 | 96 | static void kmap_atomic_register(struct page *page, int type, |
867e359b CM |
97 | unsigned long va, pte_t *ptep, pte_t pteval) |
98 | { | |
99 | unsigned long flags; | |
100 | struct atomic_mapped_page *amp; | |
101 | ||
102 | flags = homecache_kpte_lock(); | |
103 | spin_lock(&_lock); | |
104 | ||
105 | /* With interrupts disabled, now fill in the per-cpu info. */ | |
106 | amp = &__get_cpu_var(amps).per_type[type]; | |
107 | amp->page = page; | |
108 | amp->cpu = smp_processor_id(); | |
109 | amp->va = va; | |
110 | ||
111 | /* For generic kmap_atomic(), choose the PTE writability now. */ | |
112 | if (!pte_read(pteval)) | |
113 | pteval = mk_pte(page, page_to_kpgprot(page)); | |
114 | ||
115 | list_add(&->list, &_list); | |
116 | set_pte(ptep, pteval); | |
867e359b CM |
117 | |
118 | spin_unlock(&_lock); | |
119 | homecache_kpte_unlock(flags); | |
120 | } | |
121 | ||
122 | /* | |
123 | * Remove a page and va, on this cpu, from the list of kmap_atomic pages. | |
124 | * Linear-time search, but we count on the lists being short. | |
125 | * We don't need to adjust the PTE under the lock (as opposed to the | |
126 | * kmap_atomic_register() case), since we're just unconditionally | |
127 | * zeroing the PTE after it's off the list. | |
128 | */ | |
129 | static void kmap_atomic_unregister(struct page *page, unsigned long va) | |
130 | { | |
131 | unsigned long flags; | |
132 | struct atomic_mapped_page *amp; | |
133 | int cpu = smp_processor_id(); | |
134 | spin_lock_irqsave(&_lock, flags); | |
135 | list_for_each_entry(amp, &_list, list) { | |
136 | if (amp->page == page && amp->cpu == cpu && amp->va == va) | |
137 | break; | |
138 | } | |
139 | BUG_ON(&->list == &_list); | |
140 | list_del(&->list); | |
141 | spin_unlock_irqrestore(&_lock, flags); | |
142 | } | |
143 | ||
144 | /* Helper routine for kmap_atomic_fix_kpte(), below. */ | |
145 | static void kmap_atomic_fix_one_kpte(struct atomic_mapped_page *amp, | |
146 | int finished) | |
147 | { | |
148 | pte_t *ptep = kmap_get_pte(amp->va); | |
149 | if (!finished) { | |
150 | set_pte(ptep, pte_mkmigrate(*ptep)); | |
151 | flush_remote(0, 0, NULL, amp->va, PAGE_SIZE, PAGE_SIZE, | |
152 | cpumask_of(amp->cpu), NULL, 0); | |
153 | } else { | |
154 | /* | |
155 | * Rewrite a default kernel PTE for this page. | |
156 | * We rely on the fact that set_pte() writes the | |
157 | * present+migrating bits last. | |
158 | */ | |
159 | pte_t pte = mk_pte(amp->page, page_to_kpgprot(amp->page)); | |
160 | set_pte(ptep, pte); | |
161 | } | |
162 | } | |
163 | ||
164 | /* | |
165 | * This routine is a helper function for homecache_fix_kpte(); see | |
166 | * its comments for more information on the "finished" argument here. | |
167 | * | |
168 | * Note that we hold the lock while doing the remote flushes, which | |
169 | * will stall any unrelated cpus trying to do kmap_atomic operations. | |
170 | * We could just update the PTEs under the lock, and save away copies | |
171 | * of the structs (or just the va+cpu), then flush them after we | |
172 | * release the lock, but it seems easier just to do it all under the lock. | |
173 | */ | |
174 | void kmap_atomic_fix_kpte(struct page *page, int finished) | |
175 | { | |
176 | struct atomic_mapped_page *amp; | |
177 | unsigned long flags; | |
178 | spin_lock_irqsave(&_lock, flags); | |
179 | list_for_each_entry(amp, &_list, list) { | |
180 | if (amp->page == page) | |
181 | kmap_atomic_fix_one_kpte(amp, finished); | |
182 | } | |
183 | spin_unlock_irqrestore(&_lock, flags); | |
184 | } | |
185 | ||
186 | /* | |
187 | * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap | |
188 | * because the kmap code must perform a global TLB invalidation when | |
189 | * the kmap pool wraps. | |
190 | * | |
191 | * Note that they may be slower than on x86 (etc.) because unlike on | |
192 | * those platforms, we do have to take a global lock to map and unmap | |
193 | * pages on Tile (see above). | |
194 | * | |
195 | * When holding an atomic kmap is is not legal to sleep, so atomic | |
196 | * kmaps are appropriate for short, tight code paths only. | |
197 | */ | |
3e4d3af5 | 198 | void *kmap_atomic_prot(struct page *page, pgprot_t prot) |
867e359b | 199 | { |
867e359b | 200 | unsigned long vaddr; |
3e4d3af5 | 201 | int idx, type; |
867e359b CM |
202 | pte_t *pte; |
203 | ||
204 | /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ | |
205 | pagefault_disable(); | |
206 | ||
207 | /* Avoid icache flushes by disallowing atomic executable mappings. */ | |
208 | BUG_ON(pte_exec(prot)); | |
209 | ||
210 | if (!PageHighMem(page)) | |
211 | return page_address(page); | |
212 | ||
3e4d3af5 | 213 | type = kmap_atomic_idx_push(); |
867e359b CM |
214 | idx = type + KM_TYPE_NR*smp_processor_id(); |
215 | vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); | |
216 | pte = kmap_get_pte(vaddr); | |
217 | BUG_ON(!pte_none(*pte)); | |
218 | ||
219 | /* Register that this page is mapped atomically on this cpu. */ | |
220 | kmap_atomic_register(page, type, vaddr, pte, mk_pte(page, prot)); | |
221 | ||
222 | return (void *)vaddr; | |
223 | } | |
224 | EXPORT_SYMBOL(kmap_atomic_prot); | |
225 | ||
a24401bc | 226 | void *kmap_atomic(struct page *page) |
867e359b CM |
227 | { |
228 | /* PAGE_NONE is a magic value that tells us to check immutability. */ | |
38a6f426 | 229 | return kmap_atomic_prot(page, PAGE_NONE); |
867e359b | 230 | } |
a24401bc | 231 | EXPORT_SYMBOL(kmap_atomic); |
867e359b | 232 | |
3e4d3af5 | 233 | void __kunmap_atomic(void *kvaddr) |
867e359b CM |
234 | { |
235 | unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; | |
867e359b | 236 | |
3e4d3af5 PZ |
237 | if (vaddr >= __fix_to_virt(FIX_KMAP_END) && |
238 | vaddr <= __fix_to_virt(FIX_KMAP_BEGIN)) { | |
867e359b CM |
239 | pte_t *pte = kmap_get_pte(vaddr); |
240 | pte_t pteval = *pte; | |
3e4d3af5 PZ |
241 | int idx, type; |
242 | ||
20273941 | 243 | type = kmap_atomic_idx(); |
3e4d3af5 PZ |
244 | idx = type + KM_TYPE_NR*smp_processor_id(); |
245 | ||
246 | /* | |
247 | * Force other mappings to Oops if they try to access this pte | |
248 | * without first remapping it. Keeping stale mappings around | |
249 | * is a bad idea. | |
250 | */ | |
867e359b CM |
251 | BUG_ON(!pte_present(pteval) && !pte_migrating(pteval)); |
252 | kmap_atomic_unregister(pte_page(pteval), vaddr); | |
253 | kpte_clear_flush(pte, vaddr); | |
20273941 | 254 | kmap_atomic_idx_pop(); |
867e359b CM |
255 | } else { |
256 | /* Must be a lowmem page */ | |
257 | BUG_ON(vaddr < PAGE_OFFSET); | |
258 | BUG_ON(vaddr >= (unsigned long)high_memory); | |
259 | } | |
260 | ||
867e359b CM |
261 | pagefault_enable(); |
262 | } | |
3e4d3af5 | 263 | EXPORT_SYMBOL(__kunmap_atomic); |
867e359b CM |
264 | |
265 | /* | |
266 | * This API is supposed to allow us to map memory without a "struct page". | |
267 | * Currently we don't support this, though this may change in the future. | |
268 | */ | |
3e4d3af5 | 269 | void *kmap_atomic_pfn(unsigned long pfn) |
867e359b | 270 | { |
3e4d3af5 | 271 | return kmap_atomic(pfn_to_page(pfn)); |
867e359b | 272 | } |
3e4d3af5 | 273 | void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot) |
867e359b | 274 | { |
3e4d3af5 | 275 | return kmap_atomic_prot(pfn_to_page(pfn), prot); |
867e359b CM |
276 | } |
277 | ||
278 | struct page *kmap_atomic_to_page(void *ptr) | |
279 | { | |
280 | pte_t *pte; | |
281 | unsigned long vaddr = (unsigned long)ptr; | |
282 | ||
283 | if (vaddr < FIXADDR_START) | |
284 | return virt_to_page(ptr); | |
285 | ||
286 | pte = kmap_get_pte(vaddr); | |
287 | return pte_page(*pte); | |
288 | } |