Commit | Line | Data |
---|---|---|
b5eafe92 JF |
1 | /* |
2 | * Xen leaves the responsibility for maintaining p2m mappings to the | |
3 | * guests themselves, but it must also access and update the p2m array | |
4 | * during suspend/resume when all the pages are reallocated. | |
5 | * | |
6 | * The p2m table is logically a flat array, but we implement it as a | |
7 | * three-level tree to allow the address space to be sparse. | |
8 | * | |
9 | * Xen | |
10 | * | | |
11 | * p2m_top p2m_top_mfn | |
12 | * / \ / \ | |
13 | * p2m_mid p2m_mid p2m_mid_mfn p2m_mid_mfn | |
14 | * / \ / \ / / | |
15 | * p2m p2m p2m p2m p2m p2m p2m ... | |
16 | * | |
17 | * The p2m_mid_mfn pages are mapped by p2m_top_mfn_p. | |
18 | * | |
19 | * The p2m_top and p2m_top_mfn levels are limited to 1 page, so the | |
20 | * maximum representable pseudo-physical address space is: | |
21 | * P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE pages | |
22 | * | |
23 | * P2M_PER_PAGE depends on the architecture, as a mfn is always | |
24 | * unsigned long (8 bytes on 64-bit, 4 bytes on 32), leading to | |
a3118beb | 25 | * 512 and 1024 entries respectively. |
f4cec35b KRW |
26 | * |
27 | * In short, these structures contain the Machine Frame Number (MFN) of the PFN. | |
28 | * | |
29 | * However not all entries are filled with MFNs. Specifically for all other | |
30 | * leaf entries, or for the top root, or middle one, for which there is a void | |
31 | * entry, we assume it is "missing". So (for example) | |
32 | * pfn_to_mfn(0x90909090)=INVALID_P2M_ENTRY. | |
33 | * | |
34 | * We also have the possibility of setting 1-1 mappings on certain regions, so | |
35 | * that: | |
36 | * pfn_to_mfn(0xc0000)=0xc0000 | |
37 | * | |
38 | * The benefit of this is, that we can assume for non-RAM regions (think | |
3cb83e46 | 39 | * PCI BARs, or ACPI spaces), we can create mappings easily because we |
f4cec35b KRW |
40 | * get the PFN value to match the MFN. |
41 | * | |
42 | * For this to work efficiently we have one new page p2m_identity and | |
43 | * allocate (via reserved_brk) any other pages we need to cover the sides | |
44 | * (1GB or 4MB boundary violations). All entries in p2m_identity are set to | |
45 | * INVALID_P2M_ENTRY type (Xen toolstack only recognizes that and MFNs, | |
46 | * no other fancy value). | |
47 | * | |
48 | * On lookup we spot that the entry points to p2m_identity and return the | |
49 | * identity value instead of dereferencing and returning INVALID_P2M_ENTRY. | |
50 | * If the entry points to an allocated page, we just proceed as before and | |
51 | * return the PFN. If the PFN has IDENTITY_FRAME_BIT set we unmask that in | |
52 | * appropriate functions (pfn_to_mfn). | |
53 | * | |
54 | * The reason for having the IDENTITY_FRAME_BIT instead of just returning the | |
55 | * PFN is that we could find ourselves where pfn_to_mfn(pfn)==pfn for a | |
56 | * non-identity pfn. To protect ourselves against we elect to set (and get) the | |
57 | * IDENTITY_FRAME_BIT on all identity mapped PFNs. | |
58 | * | |
59 | * This simplistic diagram is used to explain the more subtle piece of code. | |
60 | * There is also a digram of the P2M at the end that can help. | |
61 | * Imagine your E820 looking as so: | |
62 | * | |
3cb83e46 | 63 | * 1GB 2GB 4GB |
f4cec35b KRW |
64 | * /-------------------+---------\/----\ /----------\ /---+-----\ |
65 | * | System RAM | Sys RAM ||ACPI| | reserved | | Sys RAM | | |
66 | * \-------------------+---------/\----/ \----------/ \---+-----/ | |
67 | * ^- 1029MB ^- 2001MB | |
68 | * | |
69 | * [1029MB = 263424 (0x40500), 2001MB = 512256 (0x7D100), | |
70 | * 2048MB = 524288 (0x80000)] | |
71 | * | |
72 | * And dom0_mem=max:3GB,1GB is passed in to the guest, meaning memory past 1GB | |
73 | * is actually not present (would have to kick the balloon driver to put it in). | |
74 | * | |
75 | * When we are told to set the PFNs for identity mapping (see patch: "xen/setup: | |
76 | * Set identity mapping for non-RAM E820 and E820 gaps.") we pass in the start | |
77 | * of the PFN and the end PFN (263424 and 512256 respectively). The first step | |
78 | * is to reserve_brk a top leaf page if the p2m[1] is missing. The top leaf page | |
79 | * covers 512^2 of page estate (1GB) and in case the start or end PFN is not | |
3cb83e46 DV |
80 | * aligned on 512^2*PAGE_SIZE (1GB) we reserve_brk new middle and leaf pages as |
81 | * required to split any existing p2m_mid_missing middle pages. | |
f4cec35b KRW |
82 | * |
83 | * With the E820 example above, 263424 is not 1GB aligned so we allocate a | |
84 | * reserve_brk page which will cover the PFNs estate from 0x40000 to 0x80000. | |
85 | * Each entry in the allocate page is "missing" (points to p2m_missing). | |
86 | * | |
87 | * Next stage is to determine if we need to do a more granular boundary check | |
88 | * on the 4MB (or 2MB depending on architecture) off the start and end pfn's. | |
89 | * We check if the start pfn and end pfn violate that boundary check, and if | |
3cb83e46 | 90 | * so reserve_brk a (p2m[x][y]) leaf page. This way we have a much finer |
f4cec35b KRW |
91 | * granularity of setting which PFNs are missing and which ones are identity. |
92 | * In our example 263424 and 512256 both fail the check so we reserve_brk two | |
93 | * pages. Populate them with INVALID_P2M_ENTRY (so they both have "missing" | |
94 | * values) and assign them to p2m[1][2] and p2m[1][488] respectively. | |
95 | * | |
96 | * At this point we would at minimum reserve_brk one page, but could be up to | |
97 | * three. Each call to set_phys_range_identity has at maximum a three page | |
98 | * cost. If we were to query the P2M at this stage, all those entries from | |
99 | * start PFN through end PFN (so 1029MB -> 2001MB) would return | |
100 | * INVALID_P2M_ENTRY ("missing"). | |
101 | * | |
102 | * The next step is to walk from the start pfn to the end pfn setting | |
103 | * the IDENTITY_FRAME_BIT on each PFN. This is done in set_phys_range_identity. | |
3cb83e46 DV |
104 | * If we find that the middle entry is pointing to p2m_missing we can swap it |
105 | * over to p2m_identity - this way covering 4MB (or 2MB) PFN space (and | |
106 | * similarly swapping p2m_mid_missing for p2m_mid_identity for larger regions). | |
107 | * At this point we do not need to worry about boundary aligment (so no need to | |
f4cec35b KRW |
108 | * reserve_brk a middle page, figure out which PFNs are "missing" and which |
109 | * ones are identity), as that has been done earlier. If we find that the | |
110 | * middle leaf is not occupied by p2m_identity or p2m_missing, we dereference | |
111 | * that page (which covers 512 PFNs) and set the appropriate PFN with | |
112 | * IDENTITY_FRAME_BIT. In our example 263424 and 512256 end up there, and we | |
113 | * set from p2m[1][2][256->511] and p2m[1][488][0->256] with | |
114 | * IDENTITY_FRAME_BIT set. | |
115 | * | |
116 | * All other regions that are void (or not filled) either point to p2m_missing | |
117 | * (considered missing) or have the default value of INVALID_P2M_ENTRY (also | |
118 | * considered missing). In our case, p2m[1][2][0->255] and p2m[1][488][257->511] | |
119 | * contain the INVALID_P2M_ENTRY value and are considered "missing." | |
120 | * | |
3cb83e46 DV |
121 | * Finally, the region beyond the end of of the E820 (4 GB in this example) |
122 | * is set to be identity (in case there are MMIO regions placed here). | |
123 | * | |
f4cec35b KRW |
124 | * This is what the p2m ends up looking (for the E820 above) with this |
125 | * fabulous drawing: | |
126 | * | |
127 | * p2m /--------------\ | |
128 | * /-----\ | &mfn_list[0],| /-----------------\ | |
129 | * | 0 |------>| &mfn_list[1],| /---------------\ | ~0, ~0, .. | | |
130 | * |-----| | ..., ~0, ~0 | | ~0, ~0, [x]---+----->| IDENTITY [@256] | | |
131 | * | 1 |---\ \--------------/ | [p2m_identity]+\ | IDENTITY [@257] | | |
132 | * |-----| \ | [p2m_identity]+\\ | .... | | |
133 | * | 2 |--\ \-------------------->| ... | \\ \----------------/ | |
134 | * |-----| \ \---------------/ \\ | |
3cb83e46 DV |
135 | * | 3 |-\ \ \\ p2m_identity [1] |
136 | * |-----| \ \-------------------->/---------------\ /-----------------\ | |
137 | * | .. |\ | | [p2m_identity]+-->| ~0, ~0, ~0, ... | | |
138 | * \-----/ | | | [p2m_identity]+-->| ..., ~0 | | |
139 | * | | | .... | \-----------------/ | |
140 | * | | +-[x], ~0, ~0.. +\ | |
141 | * | | \---------------/ \ | |
142 | * | | \-> /---------------\ | |
143 | * | V p2m_mid_missing p2m_missing | IDENTITY[@0] | | |
144 | * | /-----------------\ /------------\ | IDENTITY[@256]| | |
145 | * | | [p2m_missing] +---->| ~0, ~0, ...| | ~0, ~0, .... | | |
146 | * | | [p2m_missing] +---->| ..., ~0 | \---------------/ | |
147 | * | | ... | \------------/ | |
148 | * | \-----------------/ | |
149 | * | | |
150 | * | p2m_mid_identity | |
151 | * | /-----------------\ | |
152 | * \-->| [p2m_identity] +---->[1] | |
153 | * | [p2m_identity] +---->[1] | |
154 | * | ... | | |
155 | * \-----------------/ | |
f4cec35b KRW |
156 | * |
157 | * where ~0 is INVALID_P2M_ENTRY. IDENTITY is (PFN | IDENTITY_BIT) | |
b5eafe92 JF |
158 | */ |
159 | ||
160 | #include <linux/init.h> | |
161 | #include <linux/module.h> | |
448f2831 JF |
162 | #include <linux/list.h> |
163 | #include <linux/hash.h> | |
87f1d40a | 164 | #include <linux/sched.h> |
2222e71b | 165 | #include <linux/seq_file.h> |
2c185687 | 166 | #include <linux/bootmem.h> |
7108c9ce | 167 | #include <linux/slab.h> |
b5eafe92 JF |
168 | |
169 | #include <asm/cache.h> | |
170 | #include <asm/setup.h> | |
171 | ||
172 | #include <asm/xen/page.h> | |
173 | #include <asm/xen/hypercall.h> | |
174 | #include <asm/xen/hypervisor.h> | |
ee072640 | 175 | #include <xen/balloon.h> |
0930bba6 | 176 | #include <xen/grant_table.h> |
b5eafe92 | 177 | |
4fbb67e3 | 178 | #include "p2m.h" |
0930bba6 | 179 | #include "multicalls.h" |
b5eafe92 JF |
180 | #include "xen-ops.h" |
181 | ||
448f2831 JF |
182 | static void __init m2p_override_init(void); |
183 | ||
5b8e7d80 JG |
184 | unsigned long *xen_p2m_addr __read_mostly; |
185 | EXPORT_SYMBOL_GPL(xen_p2m_addr); | |
186 | unsigned long xen_p2m_size __read_mostly; | |
187 | EXPORT_SYMBOL_GPL(xen_p2m_size); | |
b5eafe92 | 188 | unsigned long xen_max_p2m_pfn __read_mostly; |
5b8e7d80 | 189 | EXPORT_SYMBOL_GPL(xen_max_p2m_pfn); |
b5eafe92 | 190 | |
2c185687 JG |
191 | static unsigned long *p2m_mid_missing_mfn; |
192 | static unsigned long *p2m_top_mfn; | |
193 | static unsigned long **p2m_top_mfn_p; | |
194 | ||
b5eafe92 JF |
195 | /* Placeholders for holes in the address space */ |
196 | static RESERVE_BRK_ARRAY(unsigned long, p2m_missing, P2M_PER_PAGE); | |
197 | static RESERVE_BRK_ARRAY(unsigned long *, p2m_mid_missing, P2M_MID_PER_PAGE); | |
b5eafe92 JF |
198 | |
199 | static RESERVE_BRK_ARRAY(unsigned long **, p2m_top, P2M_TOP_PER_PAGE); | |
b5eafe92 | 200 | |
f4cec35b | 201 | static RESERVE_BRK_ARRAY(unsigned long, p2m_identity, P2M_PER_PAGE); |
3cb83e46 | 202 | static RESERVE_BRK_ARRAY(unsigned long *, p2m_mid_identity, P2M_MID_PER_PAGE); |
f4cec35b | 203 | |
b5eafe92 | 204 | RESERVE_BRK(p2m_mid, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE))); |
b5eafe92 | 205 | |
7108c9ce JG |
206 | static int use_brk = 1; |
207 | ||
b5eafe92 JF |
208 | static inline unsigned p2m_top_index(unsigned long pfn) |
209 | { | |
210 | BUG_ON(pfn >= MAX_P2M_PFN); | |
211 | return pfn / (P2M_MID_PER_PAGE * P2M_PER_PAGE); | |
212 | } | |
213 | ||
214 | static inline unsigned p2m_mid_index(unsigned long pfn) | |
215 | { | |
216 | return (pfn / P2M_PER_PAGE) % P2M_MID_PER_PAGE; | |
217 | } | |
218 | ||
219 | static inline unsigned p2m_index(unsigned long pfn) | |
220 | { | |
221 | return pfn % P2M_PER_PAGE; | |
222 | } | |
223 | ||
224 | static void p2m_top_init(unsigned long ***top) | |
225 | { | |
226 | unsigned i; | |
227 | ||
228 | for (i = 0; i < P2M_TOP_PER_PAGE; i++) | |
229 | top[i] = p2m_mid_missing; | |
230 | } | |
231 | ||
232 | static void p2m_top_mfn_init(unsigned long *top) | |
233 | { | |
234 | unsigned i; | |
235 | ||
236 | for (i = 0; i < P2M_TOP_PER_PAGE; i++) | |
237 | top[i] = virt_to_mfn(p2m_mid_missing_mfn); | |
238 | } | |
239 | ||
240 | static void p2m_top_mfn_p_init(unsigned long **top) | |
241 | { | |
242 | unsigned i; | |
243 | ||
244 | for (i = 0; i < P2M_TOP_PER_PAGE; i++) | |
245 | top[i] = p2m_mid_missing_mfn; | |
246 | } | |
247 | ||
3cb83e46 | 248 | static void p2m_mid_init(unsigned long **mid, unsigned long *leaf) |
b5eafe92 JF |
249 | { |
250 | unsigned i; | |
251 | ||
252 | for (i = 0; i < P2M_MID_PER_PAGE; i++) | |
3cb83e46 | 253 | mid[i] = leaf; |
b5eafe92 JF |
254 | } |
255 | ||
3cb83e46 | 256 | static void p2m_mid_mfn_init(unsigned long *mid, unsigned long *leaf) |
b5eafe92 JF |
257 | { |
258 | unsigned i; | |
259 | ||
260 | for (i = 0; i < P2M_MID_PER_PAGE; i++) | |
3cb83e46 | 261 | mid[i] = virt_to_mfn(leaf); |
b5eafe92 JF |
262 | } |
263 | ||
264 | static void p2m_init(unsigned long *p2m) | |
265 | { | |
266 | unsigned i; | |
267 | ||
268 | for (i = 0; i < P2M_MID_PER_PAGE; i++) | |
269 | p2m[i] = INVALID_P2M_ENTRY; | |
270 | } | |
271 | ||
7108c9ce JG |
272 | static void * __ref alloc_p2m_page(void) |
273 | { | |
274 | if (unlikely(use_brk)) | |
275 | return extend_brk(PAGE_SIZE, PAGE_SIZE); | |
276 | ||
277 | if (unlikely(!slab_is_available())) | |
278 | return alloc_bootmem_align(PAGE_SIZE, PAGE_SIZE); | |
279 | ||
280 | return (void *)__get_free_page(GFP_KERNEL | __GFP_REPEAT); | |
281 | } | |
282 | ||
283 | /* Only to be called in case of a race for a page just allocated! */ | |
284 | static void free_p2m_page(void *p) | |
285 | { | |
286 | BUG_ON(!slab_is_available()); | |
287 | free_page((unsigned long)p); | |
288 | } | |
289 | ||
b5eafe92 JF |
290 | /* |
291 | * Build the parallel p2m_top_mfn and p2m_mid_mfn structures | |
292 | * | |
293 | * This is called both at boot time, and after resuming from suspend: | |
2c185687 | 294 | * - At boot time we're called rather early, and must use alloc_bootmem*() |
b5eafe92 JF |
295 | * to allocate memory. |
296 | * | |
297 | * - After resume we're called from within stop_machine, but the mfn | |
2c185687 | 298 | * tree should already be completely allocated. |
b5eafe92 | 299 | */ |
44b46c3e | 300 | void __ref xen_build_mfn_list_list(void) |
b5eafe92 JF |
301 | { |
302 | unsigned long pfn; | |
303 | ||
696fd7c5 KRW |
304 | if (xen_feature(XENFEAT_auto_translated_physmap)) |
305 | return; | |
306 | ||
b5eafe92 JF |
307 | /* Pre-initialize p2m_top_mfn to be completely missing */ |
308 | if (p2m_top_mfn == NULL) { | |
7108c9ce | 309 | p2m_mid_missing_mfn = alloc_p2m_page(); |
3cb83e46 | 310 | p2m_mid_mfn_init(p2m_mid_missing_mfn, p2m_missing); |
b5eafe92 | 311 | |
7108c9ce | 312 | p2m_top_mfn_p = alloc_p2m_page(); |
b5eafe92 JF |
313 | p2m_top_mfn_p_init(p2m_top_mfn_p); |
314 | ||
7108c9ce | 315 | p2m_top_mfn = alloc_p2m_page(); |
b5eafe92 JF |
316 | p2m_top_mfn_init(p2m_top_mfn); |
317 | } else { | |
318 | /* Reinitialise, mfn's all change after migration */ | |
3cb83e46 | 319 | p2m_mid_mfn_init(p2m_mid_missing_mfn, p2m_missing); |
b5eafe92 JF |
320 | } |
321 | ||
322 | for (pfn = 0; pfn < xen_max_p2m_pfn; pfn += P2M_PER_PAGE) { | |
323 | unsigned topidx = p2m_top_index(pfn); | |
324 | unsigned mididx = p2m_mid_index(pfn); | |
325 | unsigned long **mid; | |
326 | unsigned long *mid_mfn_p; | |
327 | ||
328 | mid = p2m_top[topidx]; | |
329 | mid_mfn_p = p2m_top_mfn_p[topidx]; | |
330 | ||
331 | /* Don't bother allocating any mfn mid levels if | |
332 | * they're just missing, just update the stored mfn, | |
333 | * since all could have changed over a migrate. | |
334 | */ | |
335 | if (mid == p2m_mid_missing) { | |
336 | BUG_ON(mididx); | |
337 | BUG_ON(mid_mfn_p != p2m_mid_missing_mfn); | |
338 | p2m_top_mfn[topidx] = virt_to_mfn(p2m_mid_missing_mfn); | |
339 | pfn += (P2M_MID_PER_PAGE - 1) * P2M_PER_PAGE; | |
340 | continue; | |
341 | } | |
342 | ||
343 | if (mid_mfn_p == p2m_mid_missing_mfn) { | |
344 | /* | |
345 | * XXX boot-time only! We should never find | |
346 | * missing parts of the mfn tree after | |
2c185687 | 347 | * runtime. |
b5eafe92 | 348 | */ |
7108c9ce | 349 | mid_mfn_p = alloc_p2m_page(); |
3cb83e46 | 350 | p2m_mid_mfn_init(mid_mfn_p, p2m_missing); |
b5eafe92 JF |
351 | |
352 | p2m_top_mfn_p[topidx] = mid_mfn_p; | |
353 | } | |
354 | ||
355 | p2m_top_mfn[topidx] = virt_to_mfn(mid_mfn_p); | |
356 | mid_mfn_p[mididx] = virt_to_mfn(mid[mididx]); | |
357 | } | |
358 | } | |
359 | ||
360 | void xen_setup_mfn_list_list(void) | |
361 | { | |
4dd322bc MR |
362 | if (xen_feature(XENFEAT_auto_translated_physmap)) |
363 | return; | |
364 | ||
b5eafe92 JF |
365 | BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info); |
366 | ||
367 | HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = | |
368 | virt_to_mfn(p2m_top_mfn); | |
369 | HYPERVISOR_shared_info->arch.max_pfn = xen_max_p2m_pfn; | |
370 | } | |
371 | ||
372 | /* Set up p2m_top to point to the domain-builder provided p2m pages */ | |
373 | void __init xen_build_dynamic_phys_to_machine(void) | |
374 | { | |
696fd7c5 KRW |
375 | unsigned long *mfn_list; |
376 | unsigned long max_pfn; | |
b5eafe92 JF |
377 | unsigned long pfn; |
378 | ||
696fd7c5 KRW |
379 | if (xen_feature(XENFEAT_auto_translated_physmap)) |
380 | return; | |
381 | ||
5b8e7d80 | 382 | xen_p2m_addr = (unsigned long *)xen_start_info->mfn_list; |
696fd7c5 KRW |
383 | mfn_list = (unsigned long *)xen_start_info->mfn_list; |
384 | max_pfn = min(MAX_DOMAIN_PAGES, xen_start_info->nr_pages); | |
b5eafe92 | 385 | xen_max_p2m_pfn = max_pfn; |
5b8e7d80 | 386 | xen_p2m_size = max_pfn; |
b5eafe92 | 387 | |
7108c9ce | 388 | p2m_missing = alloc_p2m_page(); |
b5eafe92 | 389 | p2m_init(p2m_missing); |
7108c9ce | 390 | p2m_identity = alloc_p2m_page(); |
3cb83e46 | 391 | p2m_init(p2m_identity); |
b5eafe92 | 392 | |
7108c9ce | 393 | p2m_mid_missing = alloc_p2m_page(); |
3cb83e46 | 394 | p2m_mid_init(p2m_mid_missing, p2m_missing); |
7108c9ce | 395 | p2m_mid_identity = alloc_p2m_page(); |
3cb83e46 | 396 | p2m_mid_init(p2m_mid_identity, p2m_identity); |
b5eafe92 | 397 | |
7108c9ce | 398 | p2m_top = alloc_p2m_page(); |
b5eafe92 JF |
399 | p2m_top_init(p2m_top); |
400 | ||
401 | /* | |
402 | * The domain builder gives us a pre-constructed p2m array in | |
403 | * mfn_list for all the pages initially given to us, so we just | |
404 | * need to graft that into our tree structure. | |
405 | */ | |
406 | for (pfn = 0; pfn < max_pfn; pfn += P2M_PER_PAGE) { | |
407 | unsigned topidx = p2m_top_index(pfn); | |
408 | unsigned mididx = p2m_mid_index(pfn); | |
409 | ||
410 | if (p2m_top[topidx] == p2m_mid_missing) { | |
7108c9ce | 411 | unsigned long **mid = alloc_p2m_page(); |
3cb83e46 | 412 | p2m_mid_init(mid, p2m_missing); |
b5eafe92 JF |
413 | |
414 | p2m_top[topidx] = mid; | |
415 | } | |
416 | ||
8e1b4cf2 SB |
417 | /* |
418 | * As long as the mfn_list has enough entries to completely | |
419 | * fill a p2m page, pointing into the array is ok. But if | |
420 | * not the entries beyond the last pfn will be undefined. | |
8e1b4cf2 SB |
421 | */ |
422 | if (unlikely(pfn + P2M_PER_PAGE > max_pfn)) { | |
423 | unsigned long p2midx; | |
cf04d120 SB |
424 | |
425 | p2midx = max_pfn % P2M_PER_PAGE; | |
426 | for ( ; p2midx < P2M_PER_PAGE; p2midx++) | |
427 | mfn_list[pfn + p2midx] = INVALID_P2M_ENTRY; | |
428 | } | |
429 | p2m_top[topidx][mididx] = &mfn_list[pfn]; | |
b5eafe92 JF |
430 | } |
431 | } | |
357a3cfb | 432 | #ifdef CONFIG_X86_64 |
357a3cfb KRW |
433 | unsigned long __init xen_revector_p2m_tree(void) |
434 | { | |
435 | unsigned long va_start; | |
436 | unsigned long va_end; | |
437 | unsigned long pfn; | |
3fc509fc | 438 | unsigned long pfn_free = 0; |
357a3cfb KRW |
439 | unsigned long *mfn_list = NULL; |
440 | unsigned long size; | |
441 | ||
7108c9ce | 442 | use_brk = 0; |
357a3cfb KRW |
443 | va_start = xen_start_info->mfn_list; |
444 | /*We copy in increments of P2M_PER_PAGE * sizeof(unsigned long), | |
445 | * so make sure it is rounded up to that */ | |
446 | size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long)); | |
447 | va_end = va_start + size; | |
448 | ||
449 | /* If we were revectored already, don't do it again. */ | |
450 | if (va_start <= __START_KERNEL_map && va_start >= __PAGE_OFFSET) | |
451 | return 0; | |
452 | ||
453 | mfn_list = alloc_bootmem_align(size, PAGE_SIZE); | |
454 | if (!mfn_list) { | |
455 | pr_warn("Could not allocate space for a new P2M tree!\n"); | |
456 | return xen_start_info->mfn_list; | |
457 | } | |
458 | /* Fill it out with INVALID_P2M_ENTRY value */ | |
459 | memset(mfn_list, 0xFF, size); | |
460 | ||
461 | for (pfn = 0; pfn < ALIGN(MAX_DOMAIN_PAGES, P2M_PER_PAGE); pfn += P2M_PER_PAGE) { | |
462 | unsigned topidx = p2m_top_index(pfn); | |
463 | unsigned mididx; | |
464 | unsigned long *mid_p; | |
465 | ||
466 | if (!p2m_top[topidx]) | |
467 | continue; | |
b5eafe92 | 468 | |
357a3cfb KRW |
469 | if (p2m_top[topidx] == p2m_mid_missing) |
470 | continue; | |
471 | ||
472 | mididx = p2m_mid_index(pfn); | |
473 | mid_p = p2m_top[topidx][mididx]; | |
474 | if (!mid_p) | |
475 | continue; | |
476 | if ((mid_p == p2m_missing) || (mid_p == p2m_identity)) | |
477 | continue; | |
b5eafe92 | 478 | |
357a3cfb KRW |
479 | if ((unsigned long)mid_p == INVALID_P2M_ENTRY) |
480 | continue; | |
481 | ||
482 | /* The old va. Rebase it on mfn_list */ | |
483 | if (mid_p >= (unsigned long *)va_start && mid_p <= (unsigned long *)va_end) { | |
484 | unsigned long *new; | |
485 | ||
3fc509fc KRW |
486 | if (pfn_free > (size / sizeof(unsigned long))) { |
487 | WARN(1, "Only allocated for %ld pages, but we want %ld!\n", | |
488 | size / sizeof(unsigned long), pfn_free); | |
489 | return 0; | |
490 | } | |
491 | new = &mfn_list[pfn_free]; | |
357a3cfb KRW |
492 | |
493 | copy_page(new, mid_p); | |
3fc509fc | 494 | p2m_top[topidx][mididx] = &mfn_list[pfn_free]; |
3fc509fc KRW |
495 | |
496 | pfn_free += P2M_PER_PAGE; | |
357a3cfb KRW |
497 | |
498 | } | |
499 | /* This should be the leafs allocated for identity from _brk. */ | |
500 | } | |
357a3cfb | 501 | |
5b8e7d80 JG |
502 | xen_p2m_size = xen_max_p2m_pfn; |
503 | xen_p2m_addr = mfn_list; | |
504 | ||
505 | xen_inv_extra_mem(); | |
506 | ||
97f4533a JG |
507 | m2p_override_init(); |
508 | return (unsigned long)mfn_list; | |
357a3cfb KRW |
509 | } |
510 | #else | |
511 | unsigned long __init xen_revector_p2m_tree(void) | |
512 | { | |
7108c9ce | 513 | use_brk = 0; |
5b8e7d80 JG |
514 | xen_p2m_size = xen_max_p2m_pfn; |
515 | xen_inv_extra_mem(); | |
97f4533a | 516 | m2p_override_init(); |
357a3cfb KRW |
517 | return 0; |
518 | } | |
519 | #endif | |
b5eafe92 JF |
520 | unsigned long get_phys_to_machine(unsigned long pfn) |
521 | { | |
522 | unsigned topidx, mididx, idx; | |
523 | ||
5b8e7d80 JG |
524 | if (unlikely(pfn >= xen_p2m_size)) { |
525 | if (pfn < xen_max_p2m_pfn) | |
526 | return xen_chk_extra_mem(pfn); | |
527 | ||
25b884a8 | 528 | return IDENTITY_FRAME(pfn); |
5b8e7d80 | 529 | } |
b5eafe92 JF |
530 | |
531 | topidx = p2m_top_index(pfn); | |
532 | mididx = p2m_mid_index(pfn); | |
533 | idx = p2m_index(pfn); | |
534 | ||
f4cec35b KRW |
535 | /* |
536 | * The INVALID_P2M_ENTRY is filled in both p2m_*identity | |
537 | * and in p2m_*missing, so returning the INVALID_P2M_ENTRY | |
538 | * would be wrong. | |
539 | */ | |
540 | if (p2m_top[topidx][mididx] == p2m_identity) | |
541 | return IDENTITY_FRAME(pfn); | |
542 | ||
b5eafe92 JF |
543 | return p2m_top[topidx][mididx][idx]; |
544 | } | |
545 | EXPORT_SYMBOL_GPL(get_phys_to_machine); | |
546 | ||
a3118beb | 547 | /* |
b5eafe92 JF |
548 | * Fully allocate the p2m structure for a given pfn. We need to check |
549 | * that both the top and mid levels are allocated, and make sure the | |
550 | * parallel mfn tree is kept in sync. We may race with other cpus, so | |
551 | * the new pages are installed with cmpxchg; if we lose the race then | |
552 | * simply free the page we allocated and use the one that's there. | |
553 | */ | |
554 | static bool alloc_p2m(unsigned long pfn) | |
555 | { | |
556 | unsigned topidx, mididx; | |
557 | unsigned long ***top_p, **mid; | |
558 | unsigned long *top_mfn_p, *mid_mfn; | |
3a0e94f8 | 559 | unsigned long *p2m_orig; |
b5eafe92 JF |
560 | |
561 | topidx = p2m_top_index(pfn); | |
562 | mididx = p2m_mid_index(pfn); | |
563 | ||
564 | top_p = &p2m_top[topidx]; | |
3a0e94f8 | 565 | mid = ACCESS_ONCE(*top_p); |
b5eafe92 JF |
566 | |
567 | if (mid == p2m_mid_missing) { | |
568 | /* Mid level is missing, allocate a new one */ | |
569 | mid = alloc_p2m_page(); | |
570 | if (!mid) | |
571 | return false; | |
572 | ||
3cb83e46 | 573 | p2m_mid_init(mid, p2m_missing); |
b5eafe92 JF |
574 | |
575 | if (cmpxchg(top_p, p2m_mid_missing, mid) != p2m_mid_missing) | |
576 | free_p2m_page(mid); | |
577 | } | |
578 | ||
579 | top_mfn_p = &p2m_top_mfn[topidx]; | |
3a0e94f8 | 580 | mid_mfn = ACCESS_ONCE(p2m_top_mfn_p[topidx]); |
b5eafe92 JF |
581 | |
582 | BUG_ON(virt_to_mfn(mid_mfn) != *top_mfn_p); | |
583 | ||
584 | if (mid_mfn == p2m_mid_missing_mfn) { | |
585 | /* Separately check the mid mfn level */ | |
586 | unsigned long missing_mfn; | |
587 | unsigned long mid_mfn_mfn; | |
239af7c7 | 588 | unsigned long old_mfn; |
b5eafe92 JF |
589 | |
590 | mid_mfn = alloc_p2m_page(); | |
591 | if (!mid_mfn) | |
592 | return false; | |
593 | ||
3cb83e46 | 594 | p2m_mid_mfn_init(mid_mfn, p2m_missing); |
b5eafe92 JF |
595 | |
596 | missing_mfn = virt_to_mfn(p2m_mid_missing_mfn); | |
597 | mid_mfn_mfn = virt_to_mfn(mid_mfn); | |
239af7c7 JG |
598 | old_mfn = cmpxchg(top_mfn_p, missing_mfn, mid_mfn_mfn); |
599 | if (old_mfn != missing_mfn) { | |
b5eafe92 | 600 | free_p2m_page(mid_mfn); |
239af7c7 JG |
601 | mid_mfn = mfn_to_virt(old_mfn); |
602 | } else { | |
b5eafe92 | 603 | p2m_top_mfn_p[topidx] = mid_mfn; |
239af7c7 | 604 | } |
b5eafe92 JF |
605 | } |
606 | ||
3a0e94f8 JG |
607 | p2m_orig = ACCESS_ONCE(p2m_top[topidx][mididx]); |
608 | if (p2m_orig == p2m_identity || p2m_orig == p2m_missing) { | |
b5eafe92 JF |
609 | /* p2m leaf page is missing */ |
610 | unsigned long *p2m; | |
611 | ||
612 | p2m = alloc_p2m_page(); | |
613 | if (!p2m) | |
614 | return false; | |
615 | ||
616 | p2m_init(p2m); | |
617 | ||
f4cec35b | 618 | if (cmpxchg(&mid[mididx], p2m_orig, p2m) != p2m_orig) |
b5eafe92 JF |
619 | free_p2m_page(p2m); |
620 | else | |
621 | mid_mfn[mididx] = virt_to_mfn(p2m); | |
622 | } | |
623 | ||
624 | return true; | |
625 | } | |
626 | ||
b83c6e55 | 627 | unsigned long __init set_phys_range_identity(unsigned long pfn_s, |
f4cec35b KRW |
628 | unsigned long pfn_e) |
629 | { | |
630 | unsigned long pfn; | |
631 | ||
5b8e7d80 | 632 | if (unlikely(pfn_s >= xen_p2m_size)) |
f4cec35b KRW |
633 | return 0; |
634 | ||
635 | if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) | |
636 | return pfn_e - pfn_s; | |
637 | ||
638 | if (pfn_s > pfn_e) | |
639 | return 0; | |
640 | ||
5b8e7d80 JG |
641 | if (pfn_e > xen_p2m_size) |
642 | pfn_e = xen_p2m_size; | |
f4cec35b | 643 | |
5b8e7d80 JG |
644 | for (pfn = pfn_s; pfn < pfn_e; pfn++) |
645 | xen_p2m_addr[pfn] = IDENTITY_FRAME(pfn); | |
f4cec35b KRW |
646 | |
647 | return pfn - pfn_s; | |
648 | } | |
649 | ||
b5eafe92 JF |
650 | /* Try to install p2m mapping; fail if intermediate bits missing */ |
651 | bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn) | |
652 | { | |
653 | unsigned topidx, mididx, idx; | |
654 | ||
2f558d40 SS |
655 | /* don't track P2M changes in autotranslate guests */ |
656 | if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) | |
6eaa412f | 657 | return true; |
2f558d40 | 658 | |
5b8e7d80 | 659 | if (unlikely(pfn >= xen_p2m_size)) { |
b5eafe92 JF |
660 | BUG_ON(mfn != INVALID_P2M_ENTRY); |
661 | return true; | |
662 | } | |
663 | ||
664 | topidx = p2m_top_index(pfn); | |
665 | mididx = p2m_mid_index(pfn); | |
666 | idx = p2m_index(pfn); | |
667 | ||
f4cec35b KRW |
668 | /* For sparse holes were the p2m leaf has real PFN along with |
669 | * PCI holes, stick in the PFN as the MFN value. | |
3cb83e46 DV |
670 | * |
671 | * set_phys_range_identity() will have allocated new middle | |
672 | * and leaf pages as required so an existing p2m_mid_missing | |
673 | * or p2m_missing mean that whole range will be identity so | |
674 | * these can be switched to p2m_mid_identity or p2m_identity. | |
f4cec35b KRW |
675 | */ |
676 | if (mfn != INVALID_P2M_ENTRY && (mfn & IDENTITY_FRAME_BIT)) { | |
3cb83e46 DV |
677 | if (p2m_top[topidx] == p2m_mid_identity) |
678 | return true; | |
679 | ||
680 | if (p2m_top[topidx] == p2m_mid_missing) { | |
681 | WARN_ON(cmpxchg(&p2m_top[topidx], p2m_mid_missing, | |
682 | p2m_mid_identity) != p2m_mid_missing); | |
683 | return true; | |
684 | } | |
685 | ||
f4cec35b KRW |
686 | if (p2m_top[topidx][mididx] == p2m_identity) |
687 | return true; | |
688 | ||
689 | /* Swap over from MISSING to IDENTITY if needed. */ | |
690 | if (p2m_top[topidx][mididx] == p2m_missing) { | |
c7617798 KRW |
691 | WARN_ON(cmpxchg(&p2m_top[topidx][mididx], p2m_missing, |
692 | p2m_identity) != p2m_missing); | |
f4cec35b KRW |
693 | return true; |
694 | } | |
695 | } | |
696 | ||
b5eafe92 JF |
697 | if (p2m_top[topidx][mididx] == p2m_missing) |
698 | return mfn == INVALID_P2M_ENTRY; | |
699 | ||
700 | p2m_top[topidx][mididx][idx] = mfn; | |
701 | ||
702 | return true; | |
703 | } | |
704 | ||
705 | bool set_phys_to_machine(unsigned long pfn, unsigned long mfn) | |
706 | { | |
b5eafe92 JF |
707 | if (unlikely(!__set_phys_to_machine(pfn, mfn))) { |
708 | if (!alloc_p2m(pfn)) | |
709 | return false; | |
710 | ||
711 | if (!__set_phys_to_machine(pfn, mfn)) | |
712 | return false; | |
713 | } | |
714 | ||
715 | return true; | |
716 | } | |
448f2831 JF |
717 | |
718 | #define M2P_OVERRIDE_HASH_SHIFT 10 | |
719 | #define M2P_OVERRIDE_HASH (1 << M2P_OVERRIDE_HASH_SHIFT) | |
720 | ||
97f4533a | 721 | static struct list_head *m2p_overrides; |
448f2831 JF |
722 | static DEFINE_SPINLOCK(m2p_override_lock); |
723 | ||
724 | static void __init m2p_override_init(void) | |
725 | { | |
726 | unsigned i; | |
727 | ||
97f4533a JG |
728 | m2p_overrides = alloc_bootmem_align( |
729 | sizeof(*m2p_overrides) * M2P_OVERRIDE_HASH, | |
730 | sizeof(unsigned long)); | |
448f2831 JF |
731 | |
732 | for (i = 0; i < M2P_OVERRIDE_HASH; i++) | |
733 | INIT_LIST_HEAD(&m2p_overrides[i]); | |
734 | } | |
735 | ||
736 | static unsigned long mfn_hash(unsigned long mfn) | |
737 | { | |
738 | return hash_long(mfn, M2P_OVERRIDE_HASH_SHIFT); | |
739 | } | |
740 | ||
741 | /* Add an MFN override for a particular page */ | |
820c4db2 JG |
742 | static int m2p_add_override(unsigned long mfn, struct page *page, |
743 | struct gnttab_map_grant_ref *kmap_op) | |
448f2831 JF |
744 | { |
745 | unsigned long flags; | |
87f1d40a | 746 | unsigned long pfn; |
6b08cfeb | 747 | unsigned long uninitialized_var(address); |
87f1d40a JF |
748 | unsigned level; |
749 | pte_t *ptep = NULL; | |
750 | ||
751 | pfn = page_to_pfn(page); | |
752 | if (!PageHighMem(page)) { | |
753 | address = (unsigned long)__va(pfn << PAGE_SHIFT); | |
754 | ptep = lookup_address(address, &level); | |
87f1d40a | 755 | if (WARN(ptep == NULL || level != PG_LEVEL_4K, |
6f58d89e | 756 | "m2p_add_override: pfn %lx not mapped", pfn)) |
87f1d40a JF |
757 | return -EINVAL; |
758 | } | |
b254244d | 759 | |
0930bba6 SS |
760 | if (kmap_op != NULL) { |
761 | if (!PageHighMem(page)) { | |
762 | struct multicall_space mcs = | |
763 | xen_mc_entry(sizeof(*kmap_op)); | |
764 | ||
765 | MULTI_grant_table_op(mcs.mc, | |
766 | GNTTABOP_map_grant_ref, kmap_op, 1); | |
767 | ||
768 | xen_mc_issue(PARAVIRT_LAZY_MMU); | |
769 | } | |
0930bba6 | 770 | } |
448f2831 JF |
771 | spin_lock_irqsave(&m2p_override_lock, flags); |
772 | list_add(&page->lru, &m2p_overrides[mfn_hash(mfn)]); | |
773 | spin_unlock_irqrestore(&m2p_override_lock, flags); | |
87f1d40a | 774 | |
b9e0d95c SS |
775 | /* p2m(m2p(mfn)) == mfn: the mfn is already present somewhere in |
776 | * this domain. Set the FOREIGN_FRAME_BIT in the p2m for the other | |
777 | * pfn so that the following mfn_to_pfn(mfn) calls will return the | |
778 | * pfn from the m2p_override (the backend pfn) instead. | |
779 | * We need to do this because the pages shared by the frontend | |
780 | * (xen-blkfront) can be already locked (lock_page, called by | |
781 | * do_read_cache_page); when the userspace backend tries to use them | |
782 | * with direct_IO, mfn_to_pfn returns the pfn of the frontend, so | |
783 | * do_blockdev_direct_IO is going to try to lock the same pages | |
784 | * again resulting in a deadlock. | |
785 | * As a side effect get_user_pages_fast might not be safe on the | |
786 | * frontend pages while they are being shared with the backend, | |
787 | * because mfn_to_pfn (that ends up being called by GUPF) will | |
788 | * return the backend pfn rather than the frontend pfn. */ | |
0160676b DV |
789 | pfn = mfn_to_pfn_no_overrides(mfn); |
790 | if (get_phys_to_machine(pfn) == mfn) | |
b9e0d95c SS |
791 | set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)); |
792 | ||
87f1d40a | 793 | return 0; |
448f2831 | 794 | } |
1429d46d | 795 | |
820c4db2 JG |
796 | int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, |
797 | struct gnttab_map_grant_ref *kmap_ops, | |
798 | struct page **pages, unsigned int count) | |
1429d46d ZK |
799 | { |
800 | int i, ret = 0; | |
801 | bool lazy = false; | |
820c4db2 | 802 | pte_t *pte; |
1429d46d ZK |
803 | |
804 | if (xen_feature(XENFEAT_auto_translated_physmap)) | |
805 | return 0; | |
806 | ||
807 | if (kmap_ops && | |
808 | !in_interrupt() && | |
809 | paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { | |
810 | arch_enter_lazy_mmu_mode(); | |
811 | lazy = true; | |
812 | } | |
813 | ||
814 | for (i = 0; i < count; i++) { | |
820c4db2 | 815 | unsigned long mfn, pfn; |
1429d46d | 816 | |
820c4db2 JG |
817 | /* Do not add to override if the map failed. */ |
818 | if (map_ops[i].status) | |
819 | continue; | |
820 | ||
821 | if (map_ops[i].flags & GNTMAP_contains_pte) { | |
822 | pte = (pte_t *)(mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) + | |
823 | (map_ops[i].host_addr & ~PAGE_MASK)); | |
824 | mfn = pte_mfn(*pte); | |
825 | } else { | |
826 | mfn = PFN_DOWN(map_ops[i].dev_bus_addr); | |
1429d46d | 827 | } |
820c4db2 | 828 | pfn = page_to_pfn(pages[i]); |
1429d46d | 829 | |
820c4db2 JG |
830 | WARN_ON(PagePrivate(pages[i])); |
831 | SetPagePrivate(pages[i]); | |
832 | set_page_private(pages[i], mfn); | |
833 | pages[i]->index = pfn_to_mfn(pfn); | |
1429d46d | 834 | |
820c4db2 JG |
835 | if (unlikely(!set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)))) { |
836 | ret = -ENOMEM; | |
1429d46d | 837 | goto out; |
820c4db2 JG |
838 | } |
839 | ||
840 | if (kmap_ops) { | |
841 | ret = m2p_add_override(mfn, pages[i], &kmap_ops[i]); | |
842 | if (ret) | |
843 | goto out; | |
844 | } | |
1429d46d ZK |
845 | } |
846 | ||
847 | out: | |
848 | if (lazy) | |
849 | arch_leave_lazy_mmu_mode(); | |
820c4db2 | 850 | |
1429d46d ZK |
851 | return ret; |
852 | } | |
820c4db2 | 853 | EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping); |
1429d46d | 854 | |
820c4db2 JG |
855 | static struct page *m2p_find_override(unsigned long mfn) |
856 | { | |
857 | unsigned long flags; | |
97f4533a | 858 | struct list_head *bucket; |
820c4db2 JG |
859 | struct page *p, *ret; |
860 | ||
97f4533a JG |
861 | if (unlikely(!m2p_overrides)) |
862 | return NULL; | |
863 | ||
820c4db2 | 864 | ret = NULL; |
97f4533a | 865 | bucket = &m2p_overrides[mfn_hash(mfn)]; |
820c4db2 JG |
866 | |
867 | spin_lock_irqsave(&m2p_override_lock, flags); | |
868 | ||
869 | list_for_each_entry(p, bucket, lru) { | |
870 | if (page_private(p) == mfn) { | |
871 | ret = p; | |
872 | break; | |
873 | } | |
874 | } | |
875 | ||
876 | spin_unlock_irqrestore(&m2p_override_lock, flags); | |
877 | ||
878 | return ret; | |
879 | } | |
880 | ||
881 | static int m2p_remove_override(struct page *page, | |
882 | struct gnttab_map_grant_ref *kmap_op, | |
883 | unsigned long mfn) | |
448f2831 JF |
884 | { |
885 | unsigned long flags; | |
9b705f0e | 886 | unsigned long pfn; |
6b08cfeb | 887 | unsigned long uninitialized_var(address); |
87f1d40a JF |
888 | unsigned level; |
889 | pte_t *ptep = NULL; | |
9b705f0e SS |
890 | |
891 | pfn = page_to_pfn(page); | |
87f1d40a JF |
892 | |
893 | if (!PageHighMem(page)) { | |
894 | address = (unsigned long)__va(pfn << PAGE_SHIFT); | |
895 | ptep = lookup_address(address, &level); | |
896 | ||
897 | if (WARN(ptep == NULL || level != PG_LEVEL_4K, | |
6f58d89e | 898 | "m2p_remove_override: pfn %lx not mapped", pfn)) |
87f1d40a JF |
899 | return -EINVAL; |
900 | } | |
9b705f0e | 901 | |
448f2831 JF |
902 | spin_lock_irqsave(&m2p_override_lock, flags); |
903 | list_del(&page->lru); | |
904 | spin_unlock_irqrestore(&m2p_override_lock, flags); | |
87f1d40a | 905 | |
2fc136ee | 906 | if (kmap_op != NULL) { |
0930bba6 SS |
907 | if (!PageHighMem(page)) { |
908 | struct multicall_space mcs; | |
ee072640 SS |
909 | struct gnttab_unmap_and_replace *unmap_op; |
910 | struct page *scratch_page = get_balloon_scratch_page(); | |
911 | unsigned long scratch_page_address = (unsigned long) | |
912 | __va(page_to_pfn(scratch_page) << PAGE_SHIFT); | |
0930bba6 SS |
913 | |
914 | /* | |
915 | * It might be that we queued all the m2p grant table | |
916 | * hypercalls in a multicall, then m2p_remove_override | |
917 | * get called before the multicall has actually been | |
918 | * issued. In this case handle is going to -1 because | |
919 | * it hasn't been modified yet. | |
920 | */ | |
2fc136ee | 921 | if (kmap_op->handle == -1) |
0930bba6 SS |
922 | xen_mc_flush(); |
923 | /* | |
2fc136ee | 924 | * Now if kmap_op->handle is negative it means that the |
0930bba6 SS |
925 | * hypercall actually returned an error. |
926 | */ | |
2fc136ee | 927 | if (kmap_op->handle == GNTST_general_error) { |
6f58d89e JG |
928 | pr_warn("m2p_remove_override: pfn %lx mfn %lx, failed to modify kernel mappings", |
929 | pfn, mfn); | |
d7f8f48d | 930 | put_balloon_scratch_page(); |
0930bba6 SS |
931 | return -1; |
932 | } | |
933 | ||
d7f8f48d BO |
934 | xen_mc_batch(); |
935 | ||
936 | mcs = __xen_mc_entry( | |
6f58d89e | 937 | sizeof(struct gnttab_unmap_and_replace)); |
0930bba6 | 938 | unmap_op = mcs.args; |
2fc136ee | 939 | unmap_op->host_addr = kmap_op->host_addr; |
ee072640 | 940 | unmap_op->new_addr = scratch_page_address; |
2fc136ee | 941 | unmap_op->handle = kmap_op->handle; |
0930bba6 SS |
942 | |
943 | MULTI_grant_table_op(mcs.mc, | |
6f58d89e | 944 | GNTTABOP_unmap_and_replace, unmap_op, 1); |
0930bba6 | 945 | |
ee072640 SS |
946 | mcs = __xen_mc_entry(0); |
947 | MULTI_update_va_mapping(mcs.mc, scratch_page_address, | |
d7f8f48d | 948 | pfn_pte(page_to_pfn(scratch_page), |
ee072640 | 949 | PAGE_KERNEL_RO), 0); |
d7f8f48d | 950 | |
ee072640 SS |
951 | xen_mc_issue(PARAVIRT_LAZY_MMU); |
952 | ||
2fc136ee | 953 | kmap_op->host_addr = 0; |
ee072640 | 954 | put_balloon_scratch_page(); |
0930bba6 | 955 | } |
2fc136ee | 956 | } |
87f1d40a | 957 | |
b9e0d95c SS |
958 | /* p2m(m2p(mfn)) == FOREIGN_FRAME(mfn): the mfn is already present |
959 | * somewhere in this domain, even before being added to the | |
960 | * m2p_override (see comment above in m2p_add_override). | |
961 | * If there are no other entries in the m2p_override corresponding | |
962 | * to this mfn, then remove the FOREIGN_FRAME_BIT from the p2m for | |
963 | * the original pfn (the one shared by the frontend): the backend | |
964 | * cannot do any IO on this page anymore because it has been | |
965 | * unshared. Removing the FOREIGN_FRAME_BIT from the p2m entry of | |
966 | * the original pfn causes mfn_to_pfn(mfn) to return the frontend | |
967 | * pfn again. */ | |
968 | mfn &= ~FOREIGN_FRAME_BIT; | |
0160676b DV |
969 | pfn = mfn_to_pfn_no_overrides(mfn); |
970 | if (get_phys_to_machine(pfn) == FOREIGN_FRAME(mfn) && | |
b9e0d95c SS |
971 | m2p_find_override(mfn) == NULL) |
972 | set_phys_to_machine(pfn, mfn); | |
973 | ||
87f1d40a | 974 | return 0; |
448f2831 JF |
975 | } |
976 | ||
820c4db2 JG |
977 | int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, |
978 | struct gnttab_map_grant_ref *kmap_ops, | |
979 | struct page **pages, unsigned int count) | |
448f2831 | 980 | { |
820c4db2 JG |
981 | int i, ret = 0; |
982 | bool lazy = false; | |
448f2831 | 983 | |
820c4db2 JG |
984 | if (xen_feature(XENFEAT_auto_translated_physmap)) |
985 | return 0; | |
448f2831 | 986 | |
820c4db2 JG |
987 | if (kmap_ops && |
988 | !in_interrupt() && | |
989 | paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { | |
990 | arch_enter_lazy_mmu_mode(); | |
991 | lazy = true; | |
992 | } | |
448f2831 | 993 | |
820c4db2 JG |
994 | for (i = 0; i < count; i++) { |
995 | unsigned long mfn = get_phys_to_machine(page_to_pfn(pages[i])); | |
996 | unsigned long pfn = page_to_pfn(pages[i]); | |
997 | ||
998 | if (mfn == INVALID_P2M_ENTRY || !(mfn & FOREIGN_FRAME_BIT)) { | |
999 | ret = -EINVAL; | |
1000 | goto out; | |
448f2831 | 1001 | } |
448f2831 | 1002 | |
820c4db2 JG |
1003 | set_page_private(pages[i], INVALID_P2M_ENTRY); |
1004 | WARN_ON(!PagePrivate(pages[i])); | |
1005 | ClearPagePrivate(pages[i]); | |
1006 | set_phys_to_machine(pfn, pages[i]->index); | |
448f2831 | 1007 | |
820c4db2 JG |
1008 | if (kmap_ops) |
1009 | ret = m2p_remove_override(pages[i], &kmap_ops[i], mfn); | |
1010 | if (ret) | |
1011 | goto out; | |
1012 | } | |
1013 | ||
1014 | out: | |
1015 | if (lazy) | |
1016 | arch_leave_lazy_mmu_mode(); | |
448f2831 JF |
1017 | return ret; |
1018 | } | |
820c4db2 | 1019 | EXPORT_SYMBOL_GPL(clear_foreign_p2m_mapping); |
448f2831 JF |
1020 | |
1021 | unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn) | |
1022 | { | |
1023 | struct page *p = m2p_find_override(mfn); | |
1024 | unsigned long ret = pfn; | |
1025 | ||
1026 | if (p) | |
1027 | ret = page_to_pfn(p); | |
1028 | ||
1029 | return ret; | |
1030 | } | |
e1b478e4 | 1031 | EXPORT_SYMBOL_GPL(m2p_find_override_pfn); |
2222e71b KRW |
1032 | |
1033 | #ifdef CONFIG_XEN_DEBUG_FS | |
a867db10 KRW |
1034 | #include <linux/debugfs.h> |
1035 | #include "debugfs.h" | |
1036 | static int p2m_dump_show(struct seq_file *m, void *v) | |
2222e71b KRW |
1037 | { |
1038 | static const char * const level_name[] = { "top", "middle", | |
8404877e | 1039 | "entry", "abnormal", "error"}; |
2222e71b KRW |
1040 | #define TYPE_IDENTITY 0 |
1041 | #define TYPE_MISSING 1 | |
1042 | #define TYPE_PFN 2 | |
1043 | #define TYPE_UNKNOWN 3 | |
a491dbef KRW |
1044 | static const char * const type_name[] = { |
1045 | [TYPE_IDENTITY] = "identity", | |
1046 | [TYPE_MISSING] = "missing", | |
1047 | [TYPE_PFN] = "pfn", | |
1048 | [TYPE_UNKNOWN] = "abnormal"}; | |
2222e71b KRW |
1049 | unsigned long pfn, prev_pfn_type = 0, prev_pfn_level = 0; |
1050 | unsigned int uninitialized_var(prev_level); | |
1051 | unsigned int uninitialized_var(prev_type); | |
1052 | ||
1053 | if (!p2m_top) | |
1054 | return 0; | |
1055 | ||
1056 | for (pfn = 0; pfn < MAX_DOMAIN_PAGES; pfn++) { | |
1057 | unsigned topidx = p2m_top_index(pfn); | |
1058 | unsigned mididx = p2m_mid_index(pfn); | |
1059 | unsigned idx = p2m_index(pfn); | |
1060 | unsigned lvl, type; | |
1061 | ||
1062 | lvl = 4; | |
1063 | type = TYPE_UNKNOWN; | |
1064 | if (p2m_top[topidx] == p2m_mid_missing) { | |
1065 | lvl = 0; type = TYPE_MISSING; | |
1066 | } else if (p2m_top[topidx] == NULL) { | |
1067 | lvl = 0; type = TYPE_UNKNOWN; | |
1068 | } else if (p2m_top[topidx][mididx] == NULL) { | |
1069 | lvl = 1; type = TYPE_UNKNOWN; | |
1070 | } else if (p2m_top[topidx][mididx] == p2m_identity) { | |
1071 | lvl = 1; type = TYPE_IDENTITY; | |
1072 | } else if (p2m_top[topidx][mididx] == p2m_missing) { | |
1073 | lvl = 1; type = TYPE_MISSING; | |
1074 | } else if (p2m_top[topidx][mididx][idx] == 0) { | |
1075 | lvl = 2; type = TYPE_UNKNOWN; | |
1076 | } else if (p2m_top[topidx][mididx][idx] == IDENTITY_FRAME(pfn)) { | |
1077 | lvl = 2; type = TYPE_IDENTITY; | |
1078 | } else if (p2m_top[topidx][mididx][idx] == INVALID_P2M_ENTRY) { | |
1079 | lvl = 2; type = TYPE_MISSING; | |
1080 | } else if (p2m_top[topidx][mididx][idx] == pfn) { | |
1081 | lvl = 2; type = TYPE_PFN; | |
1082 | } else if (p2m_top[topidx][mididx][idx] != pfn) { | |
1083 | lvl = 2; type = TYPE_PFN; | |
1084 | } | |
1085 | if (pfn == 0) { | |
1086 | prev_level = lvl; | |
1087 | prev_type = type; | |
1088 | } | |
1089 | if (pfn == MAX_DOMAIN_PAGES-1) { | |
1090 | lvl = 3; | |
1091 | type = TYPE_UNKNOWN; | |
1092 | } | |
1093 | if (prev_type != type) { | |
1094 | seq_printf(m, " [0x%lx->0x%lx] %s\n", | |
1095 | prev_pfn_type, pfn, type_name[prev_type]); | |
1096 | prev_pfn_type = pfn; | |
1097 | prev_type = type; | |
1098 | } | |
1099 | if (prev_level != lvl) { | |
1100 | seq_printf(m, " [0x%lx->0x%lx] level %s\n", | |
1101 | prev_pfn_level, pfn, level_name[prev_level]); | |
1102 | prev_pfn_level = pfn; | |
1103 | prev_level = lvl; | |
1104 | } | |
1105 | } | |
1106 | return 0; | |
1107 | #undef TYPE_IDENTITY | |
1108 | #undef TYPE_MISSING | |
1109 | #undef TYPE_PFN | |
1110 | #undef TYPE_UNKNOWN | |
1111 | } | |
a867db10 KRW |
1112 | |
1113 | static int p2m_dump_open(struct inode *inode, struct file *filp) | |
1114 | { | |
1115 | return single_open(filp, p2m_dump_show, NULL); | |
1116 | } | |
1117 | ||
1118 | static const struct file_operations p2m_dump_fops = { | |
1119 | .open = p2m_dump_open, | |
1120 | .read = seq_read, | |
1121 | .llseek = seq_lseek, | |
1122 | .release = single_release, | |
1123 | }; | |
1124 | ||
1125 | static struct dentry *d_mmu_debug; | |
1126 | ||
1127 | static int __init xen_p2m_debugfs(void) | |
1128 | { | |
1129 | struct dentry *d_xen = xen_init_debugfs(); | |
1130 | ||
1131 | if (d_xen == NULL) | |
1132 | return -ENOMEM; | |
1133 | ||
1134 | d_mmu_debug = debugfs_create_dir("mmu", d_xen); | |
1135 | ||
1136 | debugfs_create_file("p2m", 0600, d_mmu_debug, NULL, &p2m_dump_fops); | |
1137 | return 0; | |
1138 | } | |
1139 | fs_initcall(xen_p2m_debugfs); | |
1140 | #endif /* CONFIG_XEN_DEBUG_FS */ |