Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * pSeries_lpar.c | |
3 | * Copyright (C) 2001 Todd Inglett, IBM Corporation | |
4 | * | |
5 | * pSeries LPAR support. | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License as published by | |
9 | * the Free Software Foundation; either version 2 of the License, or | |
10 | * (at your option) any later version. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public License | |
18 | * along with this program; if not, write to the Free Software | |
19 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
20 | */ | |
21 | ||
f7ebf352 ME |
22 | /* Enables debugging of low-level hash table routines - careful! */ |
23 | #undef DEBUG | |
1da177e4 | 24 | |
1da177e4 LT |
25 | #include <linux/kernel.h> |
26 | #include <linux/dma-mapping.h> | |
463ce0e1 | 27 | #include <linux/console.h> |
1da177e4 LT |
28 | #include <asm/processor.h> |
29 | #include <asm/mmu.h> | |
30 | #include <asm/page.h> | |
31 | #include <asm/pgtable.h> | |
32 | #include <asm/machdep.h> | |
33 | #include <asm/abs_addr.h> | |
34 | #include <asm/mmu_context.h> | |
1da177e4 LT |
35 | #include <asm/iommu.h> |
36 | #include <asm/tlbflush.h> | |
37 | #include <asm/tlb.h> | |
38 | #include <asm/prom.h> | |
1da177e4 | 39 | #include <asm/cputable.h> |
dcad47fc | 40 | #include <asm/udbg.h> |
2249ca9d | 41 | #include <asm/smp.h> |
c8cd093a | 42 | #include <asm/trace.h> |
a1218720 ME |
43 | |
44 | #include "plpar_wrappers.h" | |
21cf9133 | 45 | #include "pseries.h" |
1da177e4 | 46 | |
1da177e4 | 47 | |
b9377ffc | 48 | /* in hvCall.S */ |
1da177e4 | 49 | EXPORT_SYMBOL(plpar_hcall); |
b9377ffc | 50 | EXPORT_SYMBOL(plpar_hcall9); |
1da177e4 | 51 | EXPORT_SYMBOL(plpar_hcall_norets); |
b9377ffc | 52 | |
1da177e4 LT |
53 | extern void pSeries_find_serial_port(void); |
54 | ||
55 | ||
541b2755 | 56 | static int vtermno; /* virtual terminal# for udbg */ |
1da177e4 LT |
57 | |
58 | #define __ALIGNED__ __attribute__((__aligned__(sizeof(long)))) | |
51d3082f | 59 | static void udbg_hvsi_putc(char c) |
1da177e4 LT |
60 | { |
61 | /* packet's seqno isn't used anyways */ | |
62 | uint8_t packet[] __ALIGNED__ = { 0xff, 5, 0, 0, c }; | |
63 | int rc; | |
64 | ||
65 | if (c == '\n') | |
66 | udbg_hvsi_putc('\r'); | |
67 | ||
68 | do { | |
69 | rc = plpar_put_term_char(vtermno, sizeof(packet), packet); | |
706c8c93 | 70 | } while (rc == H_BUSY); |
1da177e4 LT |
71 | } |
72 | ||
73 | static long hvsi_udbg_buf_len; | |
74 | static uint8_t hvsi_udbg_buf[256]; | |
75 | ||
76 | static int udbg_hvsi_getc_poll(void) | |
77 | { | |
78 | unsigned char ch; | |
79 | int rc, i; | |
80 | ||
81 | if (hvsi_udbg_buf_len == 0) { | |
82 | rc = plpar_get_term_char(vtermno, &hvsi_udbg_buf_len, hvsi_udbg_buf); | |
706c8c93 | 83 | if (rc != H_SUCCESS || hvsi_udbg_buf[0] != 0xff) { |
1da177e4 LT |
84 | /* bad read or non-data packet */ |
85 | hvsi_udbg_buf_len = 0; | |
86 | } else { | |
87 | /* remove the packet header */ | |
88 | for (i = 4; i < hvsi_udbg_buf_len; i++) | |
89 | hvsi_udbg_buf[i-4] = hvsi_udbg_buf[i]; | |
90 | hvsi_udbg_buf_len -= 4; | |
91 | } | |
92 | } | |
93 | ||
94 | if (hvsi_udbg_buf_len <= 0 || hvsi_udbg_buf_len > 256) { | |
95 | /* no data ready */ | |
96 | hvsi_udbg_buf_len = 0; | |
97 | return -1; | |
98 | } | |
99 | ||
100 | ch = hvsi_udbg_buf[0]; | |
101 | /* shift remaining data down */ | |
102 | for (i = 1; i < hvsi_udbg_buf_len; i++) { | |
103 | hvsi_udbg_buf[i-1] = hvsi_udbg_buf[i]; | |
104 | } | |
105 | hvsi_udbg_buf_len--; | |
106 | ||
107 | return ch; | |
108 | } | |
109 | ||
bb6b9b28 | 110 | static int udbg_hvsi_getc(void) |
1da177e4 LT |
111 | { |
112 | int ch; | |
113 | for (;;) { | |
114 | ch = udbg_hvsi_getc_poll(); | |
115 | if (ch == -1) { | |
116 | /* This shouldn't be needed...but... */ | |
117 | volatile unsigned long delay; | |
118 | for (delay=0; delay < 2000000; delay++) | |
119 | ; | |
120 | } else { | |
121 | return ch; | |
122 | } | |
123 | } | |
124 | } | |
125 | ||
51d3082f | 126 | static void udbg_putcLP(char c) |
1da177e4 LT |
127 | { |
128 | char buf[16]; | |
129 | unsigned long rc; | |
130 | ||
131 | if (c == '\n') | |
132 | udbg_putcLP('\r'); | |
133 | ||
134 | buf[0] = c; | |
135 | do { | |
136 | rc = plpar_put_term_char(vtermno, 1, buf); | |
706c8c93 | 137 | } while(rc == H_BUSY); |
1da177e4 LT |
138 | } |
139 | ||
140 | /* Buffered chars getc */ | |
141 | static long inbuflen; | |
142 | static long inbuf[2]; /* must be 2 longs */ | |
143 | ||
144 | static int udbg_getc_pollLP(void) | |
145 | { | |
146 | /* The interface is tricky because it may return up to 16 chars. | |
147 | * We save them statically for future calls to udbg_getc(). | |
148 | */ | |
149 | char ch, *buf = (char *)inbuf; | |
150 | int i; | |
151 | long rc; | |
152 | if (inbuflen == 0) { | |
153 | /* get some more chars. */ | |
154 | inbuflen = 0; | |
155 | rc = plpar_get_term_char(vtermno, &inbuflen, buf); | |
706c8c93 | 156 | if (rc != H_SUCCESS) |
1da177e4 LT |
157 | inbuflen = 0; /* otherwise inbuflen is garbage */ |
158 | } | |
159 | if (inbuflen <= 0 || inbuflen > 16) { | |
160 | /* Catch error case as well as other oddities (corruption) */ | |
161 | inbuflen = 0; | |
162 | return -1; | |
163 | } | |
164 | ch = buf[0]; | |
165 | for (i = 1; i < inbuflen; i++) /* shuffle them down. */ | |
166 | buf[i-1] = buf[i]; | |
167 | inbuflen--; | |
168 | return ch; | |
169 | } | |
170 | ||
bb6b9b28 | 171 | static int udbg_getcLP(void) |
1da177e4 LT |
172 | { |
173 | int ch; | |
174 | for (;;) { | |
175 | ch = udbg_getc_pollLP(); | |
176 | if (ch == -1) { | |
177 | /* This shouldn't be needed...but... */ | |
178 | volatile unsigned long delay; | |
179 | for (delay=0; delay < 2000000; delay++) | |
180 | ; | |
181 | } else { | |
182 | return ch; | |
183 | } | |
184 | } | |
185 | } | |
186 | ||
187 | /* call this from early_init() for a working debug console on | |
188 | * vterm capable LPAR machines | |
189 | */ | |
463ce0e1 | 190 | void __init udbg_init_debug_lpar(void) |
1da177e4 LT |
191 | { |
192 | vtermno = 0; | |
c8f1c8be MM |
193 | udbg_putc = udbg_putcLP; |
194 | udbg_getc = udbg_getcLP; | |
195 | udbg_getc_poll = udbg_getc_pollLP; | |
cb1e2ab4 ME |
196 | |
197 | register_early_udbg_console(); | |
1da177e4 LT |
198 | } |
199 | ||
200 | /* returns 0 if couldn't find or use /chosen/stdout as console */ | |
463ce0e1 | 201 | void __init find_udbg_vterm(void) |
1da177e4 LT |
202 | { |
203 | struct device_node *stdout_node; | |
954a46e2 JK |
204 | const u32 *termno; |
205 | const char *name; | |
1da177e4 LT |
206 | |
207 | /* find the boot console from /chosen/stdout */ | |
208 | if (!of_chosen) | |
463ce0e1 | 209 | return; |
e2eb6392 | 210 | name = of_get_property(of_chosen, "linux,stdout-path", NULL); |
1da177e4 | 211 | if (name == NULL) |
463ce0e1 | 212 | return; |
1da177e4 LT |
213 | stdout_node = of_find_node_by_path(name); |
214 | if (!stdout_node) | |
463ce0e1 | 215 | return; |
e2eb6392 | 216 | name = of_get_property(stdout_node, "name", NULL); |
1da177e4 LT |
217 | if (!name) { |
218 | printk(KERN_WARNING "stdout node missing 'name' property!\n"); | |
219 | goto out; | |
220 | } | |
221 | ||
463ce0e1 BH |
222 | /* Check if it's a virtual terminal */ |
223 | if (strncmp(name, "vty", 3) != 0) | |
224 | goto out; | |
e2eb6392 | 225 | termno = of_get_property(stdout_node, "reg", NULL); |
463ce0e1 BH |
226 | if (termno == NULL) |
227 | goto out; | |
228 | vtermno = termno[0]; | |
229 | ||
55b61fec | 230 | if (of_device_is_compatible(stdout_node, "hvterm1")) { |
463ce0e1 BH |
231 | udbg_putc = udbg_putcLP; |
232 | udbg_getc = udbg_getcLP; | |
233 | udbg_getc_poll = udbg_getc_pollLP; | |
5faae2e5 | 234 | add_preferred_console("hvc", termno[0] & 0xff, NULL); |
55b61fec | 235 | } else if (of_device_is_compatible(stdout_node, "hvterm-protocol")) { |
463ce0e1 BH |
236 | vtermno = termno[0]; |
237 | udbg_putc = udbg_hvsi_putc; | |
238 | udbg_getc = udbg_hvsi_getc; | |
239 | udbg_getc_poll = udbg_hvsi_getc_poll; | |
5faae2e5 | 240 | add_preferred_console("hvsi", termno[0] & 0xff, NULL); |
1da177e4 | 241 | } |
1da177e4 LT |
242 | out: |
243 | of_node_put(stdout_node); | |
1da177e4 LT |
244 | } |
245 | ||
246 | void vpa_init(int cpu) | |
247 | { | |
248 | int hwcpu = get_hard_smp_processor_id(cpu); | |
2f6093c8 | 249 | unsigned long addr; |
1da177e4 | 250 | long ret; |
cf9efce0 PM |
251 | struct paca_struct *pp; |
252 | struct dtl_entry *dtl; | |
233ccd0d OJ |
253 | |
254 | if (cpu_has_feature(CPU_FTR_ALTIVEC)) | |
8154c5d2 | 255 | lppaca_of(cpu).vmxregs_in_use = 1; |
233ccd0d | 256 | |
8154c5d2 | 257 | addr = __pa(&lppaca_of(cpu)); |
2f6093c8 | 258 | ret = register_vpa(hwcpu, addr); |
1da177e4 | 259 | |
2f6093c8 | 260 | if (ret) { |
1da177e4 LT |
261 | printk(KERN_ERR "WARNING: vpa_init: VPA registration for " |
262 | "cpu %d (hw %d) of area %lx returns %ld\n", | |
2f6093c8 MN |
263 | cpu, hwcpu, addr, ret); |
264 | return; | |
265 | } | |
266 | /* | |
267 | * PAPR says this feature is SLB-Buffer but firmware never | |
268 | * reports that. All SPLPAR support SLB shadow buffer. | |
269 | */ | |
270 | addr = __pa(&slb_shadow[cpu]); | |
271 | if (firmware_has_feature(FW_FEATURE_SPLPAR)) { | |
272 | ret = register_slb_shadow(hwcpu, addr); | |
273 | if (ret) | |
274 | printk(KERN_ERR | |
275 | "WARNING: vpa_init: SLB shadow buffer " | |
276 | "registration for cpu %d (hw %d) of area %lx " | |
277 | "returns %ld\n", cpu, hwcpu, addr, ret); | |
278 | } | |
cf9efce0 PM |
279 | |
280 | /* | |
281 | * Register dispatch trace log, if one has been allocated. | |
282 | */ | |
283 | pp = &paca[cpu]; | |
284 | dtl = pp->dispatch_log; | |
285 | if (dtl) { | |
286 | pp->dtl_ridx = 0; | |
287 | pp->dtl_curr = dtl; | |
288 | lppaca_of(cpu).dtl_idx = 0; | |
289 | ||
290 | /* hypervisor reads buffer length from this field */ | |
291 | dtl->enqueue_to_dispatch_time = DISPATCH_LOG_BYTES; | |
292 | ret = register_dtl(hwcpu, __pa(dtl)); | |
293 | if (ret) | |
294 | pr_warn("DTL registration failed for cpu %d (%ld)\n", | |
295 | cpu, ret); | |
296 | lppaca_of(cpu).dtl_enable_mask = 2; | |
297 | } | |
1da177e4 LT |
298 | } |
299 | ||
035223fb | 300 | static long pSeries_lpar_hpte_insert(unsigned long hpte_group, |
3c726f8d BH |
301 | unsigned long va, unsigned long pa, |
302 | unsigned long rflags, unsigned long vflags, | |
1189be65 | 303 | int psize, int ssize) |
1da177e4 | 304 | { |
1da177e4 LT |
305 | unsigned long lpar_rc; |
306 | unsigned long flags; | |
307 | unsigned long slot; | |
96e28449 | 308 | unsigned long hpte_v, hpte_r; |
1da177e4 | 309 | |
3c726f8d | 310 | if (!(vflags & HPTE_V_BOLTED)) |
551a232c | 311 | pr_devel("hpte_insert(group=%lx, va=%016lx, pa=%016lx, " |
f7ebf352 ME |
312 | "rflags=%lx, vflags=%lx, psize=%d)\n", |
313 | hpte_group, va, pa, rflags, vflags, psize); | |
3c726f8d | 314 | |
1189be65 | 315 | hpte_v = hpte_encode_v(va, psize, ssize) | vflags | HPTE_V_VALID; |
3c726f8d BH |
316 | hpte_r = hpte_encode_r(pa, psize) | rflags; |
317 | ||
318 | if (!(vflags & HPTE_V_BOLTED)) | |
551a232c | 319 | pr_devel(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r); |
3c726f8d | 320 | |
1da177e4 LT |
321 | /* Now fill in the actual HPTE */ |
322 | /* Set CEC cookie to 0 */ | |
323 | /* Zero page = 0 */ | |
324 | /* I-cache Invalidate = 0 */ | |
325 | /* I-cache synchronize = 0 */ | |
326 | /* Exact = 0 */ | |
327 | flags = 0; | |
328 | ||
3c726f8d | 329 | /* Make pHyp happy */ |
443dcac4 | 330 | if ((rflags & _PAGE_NO_CACHE) & !(rflags & _PAGE_WRITETHRU)) |
96e28449 | 331 | hpte_r &= ~_PAGE_COHERENT; |
1da177e4 | 332 | |
b9377ffc | 333 | lpar_rc = plpar_pte_enter(flags, hpte_group, hpte_v, hpte_r, &slot); |
706c8c93 | 334 | if (unlikely(lpar_rc == H_PTEG_FULL)) { |
3c726f8d | 335 | if (!(vflags & HPTE_V_BOLTED)) |
551a232c | 336 | pr_devel(" full\n"); |
1da177e4 | 337 | return -1; |
3c726f8d | 338 | } |
1da177e4 LT |
339 | |
340 | /* | |
341 | * Since we try and ioremap PHBs we don't own, the pte insert | |
342 | * will fail. However we must catch the failure in hash_page | |
343 | * or we will loop forever, so return -2 in this case. | |
344 | */ | |
706c8c93 | 345 | if (unlikely(lpar_rc != H_SUCCESS)) { |
3c726f8d | 346 | if (!(vflags & HPTE_V_BOLTED)) |
551a232c | 347 | pr_devel(" lpar err %lu\n", lpar_rc); |
1da177e4 | 348 | return -2; |
3c726f8d BH |
349 | } |
350 | if (!(vflags & HPTE_V_BOLTED)) | |
551a232c | 351 | pr_devel(" -> slot: %lu\n", slot & 7); |
1da177e4 LT |
352 | |
353 | /* Because of iSeries, we have to pass down the secondary | |
354 | * bucket bit here as well | |
355 | */ | |
96e28449 | 356 | return (slot & 7) | (!!(vflags & HPTE_V_SECONDARY) << 3); |
1da177e4 LT |
357 | } |
358 | ||
359 | static DEFINE_SPINLOCK(pSeries_lpar_tlbie_lock); | |
360 | ||
361 | static long pSeries_lpar_hpte_remove(unsigned long hpte_group) | |
362 | { | |
363 | unsigned long slot_offset; | |
364 | unsigned long lpar_rc; | |
365 | int i; | |
366 | unsigned long dummy1, dummy2; | |
367 | ||
368 | /* pick a random slot to start at */ | |
369 | slot_offset = mftb() & 0x7; | |
370 | ||
371 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
372 | ||
373 | /* don't remove a bolted entry */ | |
374 | lpar_rc = plpar_pte_remove(H_ANDCOND, hpte_group + slot_offset, | |
375 | (0x1UL << 4), &dummy1, &dummy2); | |
706c8c93 | 376 | if (lpar_rc == H_SUCCESS) |
1da177e4 | 377 | return i; |
706c8c93 | 378 | BUG_ON(lpar_rc != H_NOT_FOUND); |
1da177e4 LT |
379 | |
380 | slot_offset++; | |
381 | slot_offset &= 0x7; | |
382 | } | |
383 | ||
384 | return -1; | |
385 | } | |
386 | ||
387 | static void pSeries_lpar_hptab_clear(void) | |
388 | { | |
389 | unsigned long size_bytes = 1UL << ppc64_pft_size; | |
390 | unsigned long hpte_count = size_bytes >> 4; | |
d504bed6 MN |
391 | struct { |
392 | unsigned long pteh; | |
393 | unsigned long ptel; | |
394 | } ptes[4]; | |
b7abc5c5 | 395 | long lpar_rc; |
d504bed6 MN |
396 | int i, j; |
397 | ||
398 | /* Read in batches of 4, | |
399 | * invalidate only valid entries not in the VRMA | |
400 | * hpte_count will be a multiple of 4 | |
401 | */ | |
402 | for (i = 0; i < hpte_count; i += 4) { | |
403 | lpar_rc = plpar_pte_read_4_raw(0, i, (void *)ptes); | |
404 | if (lpar_rc != H_SUCCESS) | |
405 | continue; | |
406 | for (j = 0; j < 4; j++){ | |
407 | if ((ptes[j].pteh & HPTE_V_VRMA_MASK) == | |
408 | HPTE_V_VRMA_MASK) | |
409 | continue; | |
410 | if (ptes[j].pteh & HPTE_V_VALID) | |
411 | plpar_pte_remove_raw(0, i + j, 0, | |
412 | &(ptes[j].pteh), &(ptes[j].ptel)); | |
b7abc5c5 SS |
413 | } |
414 | } | |
1da177e4 LT |
415 | } |
416 | ||
1189be65 PM |
417 | /* |
418 | * This computes the AVPN and B fields of the first dword of a HPTE, | |
419 | * for use when we want to match an existing PTE. The bottom 7 bits | |
420 | * of the returned value are zero. | |
421 | */ | |
422 | static inline unsigned long hpte_encode_avpn(unsigned long va, int psize, | |
423 | int ssize) | |
424 | { | |
425 | unsigned long v; | |
426 | ||
427 | v = (va >> 23) & ~(mmu_psize_defs[psize].avpnm); | |
428 | v <<= HPTE_V_AVPN_SHIFT; | |
429 | v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT; | |
430 | return v; | |
431 | } | |
432 | ||
1da177e4 LT |
433 | /* |
434 | * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and | |
435 | * the low 3 bits of flags happen to line up. So no transform is needed. | |
436 | * We can probably optimize here and assume the high bits of newpp are | |
437 | * already zero. For now I am paranoid. | |
438 | */ | |
3c726f8d BH |
439 | static long pSeries_lpar_hpte_updatepp(unsigned long slot, |
440 | unsigned long newpp, | |
441 | unsigned long va, | |
1189be65 | 442 | int psize, int ssize, int local) |
1da177e4 LT |
443 | { |
444 | unsigned long lpar_rc; | |
445 | unsigned long flags = (newpp & 7) | H_AVPN; | |
3c726f8d | 446 | unsigned long want_v; |
1da177e4 | 447 | |
1189be65 | 448 | want_v = hpte_encode_avpn(va, psize, ssize); |
1da177e4 | 449 | |
551a232c | 450 | pr_devel(" update: avpnv=%016lx, hash=%016lx, f=%lx, psize: %d ...", |
f7ebf352 | 451 | want_v, slot, flags, psize); |
1da177e4 | 452 | |
1189be65 | 453 | lpar_rc = plpar_pte_protect(flags, slot, want_v); |
3c726f8d | 454 | |
706c8c93 | 455 | if (lpar_rc == H_NOT_FOUND) { |
551a232c | 456 | pr_devel("not found !\n"); |
1da177e4 | 457 | return -1; |
3c726f8d BH |
458 | } |
459 | ||
551a232c | 460 | pr_devel("ok\n"); |
1da177e4 | 461 | |
706c8c93 | 462 | BUG_ON(lpar_rc != H_SUCCESS); |
1da177e4 LT |
463 | |
464 | return 0; | |
465 | } | |
466 | ||
467 | static unsigned long pSeries_lpar_hpte_getword0(unsigned long slot) | |
468 | { | |
469 | unsigned long dword0; | |
470 | unsigned long lpar_rc; | |
471 | unsigned long dummy_word1; | |
472 | unsigned long flags; | |
473 | ||
474 | /* Read 1 pte at a time */ | |
475 | /* Do not need RPN to logical page translation */ | |
476 | /* No cross CEC PFT access */ | |
477 | flags = 0; | |
478 | ||
479 | lpar_rc = plpar_pte_read(flags, slot, &dword0, &dummy_word1); | |
480 | ||
706c8c93 | 481 | BUG_ON(lpar_rc != H_SUCCESS); |
1da177e4 LT |
482 | |
483 | return dword0; | |
484 | } | |
485 | ||
1189be65 | 486 | static long pSeries_lpar_hpte_find(unsigned long va, int psize, int ssize) |
1da177e4 LT |
487 | { |
488 | unsigned long hash; | |
1189be65 | 489 | unsigned long i; |
1da177e4 | 490 | long slot; |
3c726f8d | 491 | unsigned long want_v, hpte_v; |
1da177e4 | 492 | |
1189be65 PM |
493 | hash = hpt_hash(va, mmu_psize_defs[psize].shift, ssize); |
494 | want_v = hpte_encode_avpn(va, psize, ssize); | |
495 | ||
496 | /* Bolted entries are always in the primary group */ | |
497 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
498 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
499 | hpte_v = pSeries_lpar_hpte_getword0(slot); | |
500 | ||
501 | if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) | |
502 | /* HPTE matches */ | |
503 | return slot; | |
504 | ++slot; | |
1da177e4 LT |
505 | } |
506 | ||
507 | return -1; | |
508 | } | |
509 | ||
510 | static void pSeries_lpar_hpte_updateboltedpp(unsigned long newpp, | |
3c726f8d | 511 | unsigned long ea, |
1189be65 | 512 | int psize, int ssize) |
1da177e4 | 513 | { |
3c726f8d | 514 | unsigned long lpar_rc, slot, vsid, va, flags; |
1da177e4 | 515 | |
1189be65 PM |
516 | vsid = get_kernel_vsid(ea, ssize); |
517 | va = hpt_va(ea, vsid, ssize); | |
1da177e4 | 518 | |
1189be65 | 519 | slot = pSeries_lpar_hpte_find(va, psize, ssize); |
1da177e4 LT |
520 | BUG_ON(slot == -1); |
521 | ||
522 | flags = newpp & 7; | |
523 | lpar_rc = plpar_pte_protect(flags, slot, 0); | |
524 | ||
706c8c93 | 525 | BUG_ON(lpar_rc != H_SUCCESS); |
1da177e4 LT |
526 | } |
527 | ||
528 | static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long va, | |
1189be65 | 529 | int psize, int ssize, int local) |
1da177e4 | 530 | { |
3c726f8d | 531 | unsigned long want_v; |
1da177e4 LT |
532 | unsigned long lpar_rc; |
533 | unsigned long dummy1, dummy2; | |
534 | ||
551a232c | 535 | pr_devel(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n", |
f7ebf352 | 536 | slot, va, psize, local); |
1da177e4 | 537 | |
1189be65 PM |
538 | want_v = hpte_encode_avpn(va, psize, ssize); |
539 | lpar_rc = plpar_pte_remove(H_AVPN, slot, want_v, &dummy1, &dummy2); | |
706c8c93 | 540 | if (lpar_rc == H_NOT_FOUND) |
1da177e4 LT |
541 | return; |
542 | ||
706c8c93 | 543 | BUG_ON(lpar_rc != H_SUCCESS); |
1da177e4 LT |
544 | } |
545 | ||
f8c8803b BP |
546 | static void pSeries_lpar_hpte_removebolted(unsigned long ea, |
547 | int psize, int ssize) | |
548 | { | |
549 | unsigned long slot, vsid, va; | |
550 | ||
551 | vsid = get_kernel_vsid(ea, ssize); | |
552 | va = hpt_va(ea, vsid, ssize); | |
553 | ||
554 | slot = pSeries_lpar_hpte_find(va, psize, ssize); | |
555 | BUG_ON(slot == -1); | |
556 | ||
557 | pSeries_lpar_hpte_invalidate(slot, va, psize, ssize, 0); | |
558 | } | |
559 | ||
f03e64f2 PM |
560 | /* Flag bits for H_BULK_REMOVE */ |
561 | #define HBR_REQUEST 0x4000000000000000UL | |
562 | #define HBR_RESPONSE 0x8000000000000000UL | |
563 | #define HBR_END 0xc000000000000000UL | |
564 | #define HBR_AVPN 0x0200000000000000UL | |
565 | #define HBR_ANDCOND 0x0100000000000000UL | |
566 | ||
1da177e4 LT |
567 | /* |
568 | * Take a spinlock around flushes to avoid bouncing the hypervisor tlbie | |
569 | * lock. | |
570 | */ | |
035223fb | 571 | static void pSeries_lpar_flush_hash_range(unsigned long number, int local) |
1da177e4 | 572 | { |
f03e64f2 | 573 | unsigned long i, pix, rc; |
12e86f92 | 574 | unsigned long flags = 0; |
1da177e4 LT |
575 | struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); |
576 | int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); | |
f03e64f2 PM |
577 | unsigned long param[9]; |
578 | unsigned long va; | |
579 | unsigned long hash, index, shift, hidx, slot; | |
580 | real_pte_t pte; | |
1189be65 | 581 | int psize, ssize; |
1da177e4 LT |
582 | |
583 | if (lock_tlbie) | |
584 | spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags); | |
585 | ||
f03e64f2 | 586 | psize = batch->psize; |
1189be65 | 587 | ssize = batch->ssize; |
f03e64f2 PM |
588 | pix = 0; |
589 | for (i = 0; i < number; i++) { | |
590 | va = batch->vaddr[i]; | |
591 | pte = batch->pte[i]; | |
592 | pte_iterate_hashed_subpages(pte, psize, va, index, shift) { | |
1189be65 | 593 | hash = hpt_hash(va, shift, ssize); |
f03e64f2 PM |
594 | hidx = __rpte_to_hidx(pte, index); |
595 | if (hidx & _PTEIDX_SECONDARY) | |
596 | hash = ~hash; | |
597 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
598 | slot += hidx & _PTEIDX_GROUP_IX; | |
12e86f92 PM |
599 | if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) { |
600 | pSeries_lpar_hpte_invalidate(slot, va, psize, | |
1189be65 | 601 | ssize, local); |
12e86f92 PM |
602 | } else { |
603 | param[pix] = HBR_REQUEST | HBR_AVPN | slot; | |
1189be65 PM |
604 | param[pix+1] = hpte_encode_avpn(va, psize, |
605 | ssize); | |
12e86f92 PM |
606 | pix += 2; |
607 | if (pix == 8) { | |
608 | rc = plpar_hcall9(H_BULK_REMOVE, param, | |
f03e64f2 PM |
609 | param[0], param[1], param[2], |
610 | param[3], param[4], param[5], | |
611 | param[6], param[7]); | |
12e86f92 PM |
612 | BUG_ON(rc != H_SUCCESS); |
613 | pix = 0; | |
614 | } | |
f03e64f2 PM |
615 | } |
616 | } pte_iterate_hashed_end(); | |
617 | } | |
618 | if (pix) { | |
619 | param[pix] = HBR_END; | |
620 | rc = plpar_hcall9(H_BULK_REMOVE, param, param[0], param[1], | |
621 | param[2], param[3], param[4], param[5], | |
622 | param[6], param[7]); | |
623 | BUG_ON(rc != H_SUCCESS); | |
624 | } | |
1da177e4 LT |
625 | |
626 | if (lock_tlbie) | |
627 | spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags); | |
628 | } | |
629 | ||
4e89a2d8 WS |
630 | static int __init disable_bulk_remove(char *str) |
631 | { | |
632 | if (strcmp(str, "off") == 0 && | |
633 | firmware_has_feature(FW_FEATURE_BULK_REMOVE)) { | |
634 | printk(KERN_INFO "Disabling BULK_REMOVE firmware feature"); | |
635 | powerpc_firmware_features &= ~FW_FEATURE_BULK_REMOVE; | |
636 | } | |
637 | return 1; | |
638 | } | |
639 | ||
640 | __setup("bulk_remove=", disable_bulk_remove); | |
641 | ||
7d0daae4 | 642 | void __init hpte_init_lpar(void) |
1da177e4 LT |
643 | { |
644 | ppc_md.hpte_invalidate = pSeries_lpar_hpte_invalidate; | |
645 | ppc_md.hpte_updatepp = pSeries_lpar_hpte_updatepp; | |
646 | ppc_md.hpte_updateboltedpp = pSeries_lpar_hpte_updateboltedpp; | |
647 | ppc_md.hpte_insert = pSeries_lpar_hpte_insert; | |
648 | ppc_md.hpte_remove = pSeries_lpar_hpte_remove; | |
f8c8803b | 649 | ppc_md.hpte_removebolted = pSeries_lpar_hpte_removebolted; |
1da177e4 LT |
650 | ppc_md.flush_hash_range = pSeries_lpar_flush_hash_range; |
651 | ppc_md.hpte_clear_all = pSeries_lpar_hptab_clear; | |
1da177e4 | 652 | } |
14f966e7 RJ |
653 | |
654 | #ifdef CONFIG_PPC_SMLPAR | |
655 | #define CMO_FREE_HINT_DEFAULT 1 | |
656 | static int cmo_free_hint_flag = CMO_FREE_HINT_DEFAULT; | |
657 | ||
658 | static int __init cmo_free_hint(char *str) | |
659 | { | |
660 | char *parm; | |
661 | parm = strstrip(str); | |
662 | ||
663 | if (strcasecmp(parm, "no") == 0 || strcasecmp(parm, "off") == 0) { | |
664 | printk(KERN_INFO "cmo_free_hint: CMO free page hinting is not active.\n"); | |
665 | cmo_free_hint_flag = 0; | |
666 | return 1; | |
667 | } | |
668 | ||
669 | cmo_free_hint_flag = 1; | |
670 | printk(KERN_INFO "cmo_free_hint: CMO free page hinting is active.\n"); | |
671 | ||
672 | if (strcasecmp(parm, "yes") == 0 || strcasecmp(parm, "on") == 0) | |
673 | return 1; | |
674 | ||
675 | return 0; | |
676 | } | |
677 | ||
678 | __setup("cmo_free_hint=", cmo_free_hint); | |
679 | ||
680 | static void pSeries_set_page_state(struct page *page, int order, | |
681 | unsigned long state) | |
682 | { | |
683 | int i, j; | |
684 | unsigned long cmo_page_sz, addr; | |
685 | ||
686 | cmo_page_sz = cmo_get_page_size(); | |
687 | addr = __pa((unsigned long)page_address(page)); | |
688 | ||
689 | for (i = 0; i < (1 << order); i++, addr += PAGE_SIZE) { | |
690 | for (j = 0; j < PAGE_SIZE; j += cmo_page_sz) | |
691 | plpar_hcall_norets(H_PAGE_INIT, state, addr + j, 0); | |
692 | } | |
693 | } | |
694 | ||
695 | void arch_free_page(struct page *page, int order) | |
696 | { | |
697 | if (!cmo_free_hint_flag || !firmware_has_feature(FW_FEATURE_CMO)) | |
698 | return; | |
699 | ||
700 | pSeries_set_page_state(page, order, H_PAGE_SET_UNUSED); | |
701 | } | |
702 | EXPORT_SYMBOL(arch_free_page); | |
703 | ||
704 | #endif | |
c8cd093a AB |
705 | |
706 | #ifdef CONFIG_TRACEPOINTS | |
707 | /* | |
708 | * We optimise our hcall path by placing hcall_tracepoint_refcount | |
709 | * directly in the TOC so we can check if the hcall tracepoints are | |
710 | * enabled via a single load. | |
711 | */ | |
712 | ||
713 | /* NB: reg/unreg are called while guarded with the tracepoints_mutex */ | |
714 | extern long hcall_tracepoint_refcount; | |
715 | ||
57cdfdf8 AB |
716 | /* |
717 | * Since the tracing code might execute hcalls we need to guard against | |
718 | * recursion. One example of this are spinlocks calling H_YIELD on | |
719 | * shared processor partitions. | |
720 | */ | |
721 | static DEFINE_PER_CPU(unsigned int, hcall_trace_depth); | |
722 | ||
c8cd093a AB |
723 | void hcall_tracepoint_regfunc(void) |
724 | { | |
725 | hcall_tracepoint_refcount++; | |
726 | } | |
727 | ||
728 | void hcall_tracepoint_unregfunc(void) | |
729 | { | |
730 | hcall_tracepoint_refcount--; | |
731 | } | |
732 | ||
6f26353c | 733 | void __trace_hcall_entry(unsigned long opcode, unsigned long *args) |
c8cd093a | 734 | { |
57cdfdf8 AB |
735 | unsigned long flags; |
736 | unsigned int *depth; | |
737 | ||
738 | local_irq_save(flags); | |
739 | ||
740 | depth = &__get_cpu_var(hcall_trace_depth); | |
741 | ||
742 | if (*depth) | |
743 | goto out; | |
744 | ||
745 | (*depth)++; | |
6f26353c | 746 | trace_hcall_entry(opcode, args); |
57cdfdf8 AB |
747 | (*depth)--; |
748 | ||
749 | out: | |
750 | local_irq_restore(flags); | |
c8cd093a AB |
751 | } |
752 | ||
6f26353c AB |
753 | void __trace_hcall_exit(long opcode, unsigned long retval, |
754 | unsigned long *retbuf) | |
c8cd093a | 755 | { |
57cdfdf8 AB |
756 | unsigned long flags; |
757 | unsigned int *depth; | |
758 | ||
759 | local_irq_save(flags); | |
760 | ||
761 | depth = &__get_cpu_var(hcall_trace_depth); | |
762 | ||
763 | if (*depth) | |
764 | goto out; | |
765 | ||
766 | (*depth)++; | |
6f26353c | 767 | trace_hcall_exit(opcode, retval, retbuf); |
57cdfdf8 AB |
768 | (*depth)--; |
769 | ||
770 | out: | |
771 | local_irq_restore(flags); | |
c8cd093a AB |
772 | } |
773 | #endif |