Commit | Line | Data |
---|---|---|
948cf67c | 1 | /* |
7230c564 | 2 | * This file contains the power_save function for Power7 CPUs. |
948cf67c BH |
3 | * |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation; either version | |
7 | * 2 of the License, or (at your option) any later version. | |
8 | */ | |
9 | ||
10 | #include <linux/threads.h> | |
11 | #include <asm/processor.h> | |
12 | #include <asm/page.h> | |
13 | #include <asm/cputable.h> | |
14 | #include <asm/thread_info.h> | |
15 | #include <asm/ppc_asm.h> | |
16 | #include <asm/asm-offsets.h> | |
17 | #include <asm/ppc-opcode.h> | |
7230c564 | 18 | #include <asm/hw_irq.h> |
f0888f70 | 19 | #include <asm/kvm_book3s_asm.h> |
97eb001f | 20 | #include <asm/opal.h> |
7cba160a | 21 | #include <asm/cpuidle.h> |
77b54e9f | 22 | #include <asm/mmu-hash64.h> |
948cf67c BH |
23 | |
24 | #undef DEBUG | |
25 | ||
77b54e9f SP |
26 | /* |
27 | * Use unused space in the interrupt stack to save and restore | |
28 | * registers for winkle support. | |
29 | */ | |
30 | #define _SDR1 GPR3 | |
31 | #define _RPR GPR4 | |
32 | #define _SPURR GPR5 | |
33 | #define _PURR GPR6 | |
34 | #define _TSCR GPR7 | |
35 | #define _DSCR GPR8 | |
36 | #define _AMOR GPR9 | |
37 | #define _WORT GPR10 | |
38 | #define _WORC GPR11 | |
39 | ||
aca79d2b | 40 | /* Idle state entry routines */ |
948cf67c | 41 | |
aca79d2b VS |
42 | #define IDLE_STATE_ENTER_SEQ(IDLE_INST) \ |
43 | /* Magic NAP/SLEEP/WINKLE mode enter sequence */ \ | |
44 | std r0,0(r1); \ | |
45 | ptesync; \ | |
46 | ld r0,0(r1); \ | |
47 | 1: cmp cr0,r0,r0; \ | |
48 | bne 1b; \ | |
49 | IDLE_INST; \ | |
50 | b . | |
948cf67c | 51 | |
aca79d2b VS |
52 | .text |
53 | ||
54 | /* | |
55 | * Pass requested state in r3: | |
7cba160a | 56 | * r3 - PNV_THREAD_NAP/SLEEP/WINKLE |
8d6f7c5a ME |
57 | * |
58 | * To check IRQ_HAPPENED in r4 | |
59 | * 0 - don't check | |
60 | * 1 - check | |
aca79d2b VS |
61 | */ |
62 | _GLOBAL(power7_powersave_common) | |
63 | /* Use r3 to pass state nap/sleep/winkle */ | |
948cf67c BH |
64 | /* NAP is a state loss, we create a regs frame on the |
65 | * stack, fill it up with the state we care about and | |
66 | * stick a pointer to it in PACAR1. We really only | |
67 | * need to save PC, some CR bits and the NV GPRs, | |
68 | * but for now an interrupt frame will do. | |
69 | */ | |
70 | mflr r0 | |
71 | std r0,16(r1) | |
72 | stdu r1,-INT_FRAME_SIZE(r1) | |
73 | std r0,_LINK(r1) | |
74 | std r0,_NIP(r1) | |
75 | ||
76 | #ifndef CONFIG_SMP | |
77 | /* Make sure FPU, VSX etc... are flushed as we may lose | |
78 | * state when going to nap mode | |
79 | */ | |
b1576fec | 80 | bl discard_lazy_cpu_state |
948cf67c BH |
81 | #endif /* CONFIG_SMP */ |
82 | ||
83 | /* Hard disable interrupts */ | |
84 | mfmsr r9 | |
85 | rldicl r9,r9,48,1 | |
86 | rotldi r9,r9,16 | |
87 | mtmsrd r9,1 /* hard-disable interrupts */ | |
7230c564 BH |
88 | |
89 | /* Check if something happened while soft-disabled */ | |
90 | lbz r0,PACAIRQHAPPENED(r13) | |
d6a4f709 | 91 | andi. r0,r0,~PACA_IRQ_HARD_DIS@l |
7230c564 | 92 | beq 1f |
8d6f7c5a ME |
93 | cmpwi cr0,r4,0 |
94 | beq 1f | |
7230c564 BH |
95 | addi r1,r1,INT_FRAME_SIZE |
96 | ld r0,16(r1) | |
97 | mtlr r0 | |
98 | blr | |
99 | ||
100 | 1: /* We mark irqs hard disabled as this is the state we'll | |
101 | * be in when returning and we need to tell arch_local_irq_restore() | |
102 | * about it | |
103 | */ | |
104 | li r0,PACA_IRQ_HARD_DIS | |
105 | stb r0,PACAIRQHAPPENED(r13) | |
106 | ||
107 | /* We haven't lost state ... yet */ | |
948cf67c | 108 | li r0,0 |
2fde6d20 | 109 | stb r0,PACA_NAPSTATELOST(r13) |
948cf67c BH |
110 | |
111 | /* Continue saving state */ | |
112 | SAVE_GPR(2, r1) | |
113 | SAVE_NVGPRS(r1) | |
aca79d2b VS |
114 | mfcr r4 |
115 | std r4,_CCR(r1) | |
948cf67c BH |
116 | std r9,_MSR(r1) |
117 | std r1,PACAR1(r13) | |
118 | ||
8117ac6a PM |
119 | /* |
120 | * Go to real mode to do the nap, as required by the architecture. | |
121 | * Also, we need to be in real mode before setting hwthread_state, | |
122 | * because as soon as we do that, another thread can switch | |
123 | * the MMU context to the guest. | |
124 | */ | |
125 | LOAD_REG_IMMEDIATE(r5, MSR_IDLE) | |
126 | li r6, MSR_RI | |
127 | andc r6, r9, r6 | |
128 | LOAD_REG_ADDR(r7, power7_enter_nap_mode) | |
129 | mtmsrd r6, 1 /* clear RI before setting SRR0/1 */ | |
130 | mtspr SPRN_SRR0, r7 | |
131 | mtspr SPRN_SRR1, r5 | |
132 | rfid | |
133 | ||
134 | .globl power7_enter_nap_mode | |
135 | power7_enter_nap_mode: | |
9975f5e3 | 136 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
f0888f70 PM |
137 | /* Tell KVM we're napping */ |
138 | li r4,KVM_HWTHREAD_IN_NAP | |
139 | stb r4,HSTATE_HWTHREAD_STATE(r13) | |
140 | #endif | |
7cba160a | 141 | stb r3,PACA_THREAD_IDLE_STATE(r13) |
77b54e9f SP |
142 | cmpwi cr3,r3,PNV_THREAD_SLEEP |
143 | bge cr3,2f | |
aca79d2b VS |
144 | IDLE_STATE_ENTER_SEQ(PPC_NAP) |
145 | /* No return */ | |
7cba160a SP |
146 | 2: |
147 | /* Sleep or winkle */ | |
148 | lbz r7,PACA_THREAD_MASK(r13) | |
149 | ld r14,PACA_CORE_IDLE_STATE_PTR(r13) | |
150 | lwarx_loop1: | |
151 | lwarx r15,0,r14 | |
152 | andc r15,r15,r7 /* Clear thread bit */ | |
153 | ||
154 | andi. r15,r15,PNV_CORE_IDLE_THREAD_BITS | |
155 | ||
156 | /* | |
157 | * If cr0 = 0, then current thread is the last thread of the core entering | |
158 | * sleep. Last thread needs to execute the hardware bug workaround code if | |
159 | * required by the platform. | |
160 | * Make the workaround call unconditionally here. The below branch call is | |
161 | * patched out when the idle states are discovered if the platform does not | |
162 | * require it. | |
163 | */ | |
164 | .global pnv_fastsleep_workaround_at_entry | |
165 | pnv_fastsleep_workaround_at_entry: | |
166 | beq fastsleep_workaround_at_entry | |
167 | ||
168 | stwcx. r15,0,r14 | |
169 | bne- lwarx_loop1 | |
170 | isync | |
171 | ||
77b54e9f SP |
172 | common_enter: /* common code for all the threads entering sleep or winkle */ |
173 | bgt cr3,enter_winkle | |
7cba160a SP |
174 | IDLE_STATE_ENTER_SEQ(PPC_SLEEP) |
175 | ||
176 | fastsleep_workaround_at_entry: | |
177 | ori r15,r15,PNV_CORE_IDLE_LOCK_BIT | |
178 | stwcx. r15,0,r14 | |
179 | bne- lwarx_loop1 | |
180 | isync | |
181 | ||
182 | /* Fast sleep workaround */ | |
183 | li r3,1 | |
184 | li r4,1 | |
185 | li r0,OPAL_CONFIG_CPU_IDLE_STATE | |
186 | bl opal_call_realmode | |
187 | ||
188 | /* Clear Lock bit */ | |
189 | li r0,0 | |
190 | lwsync | |
191 | stw r0,0(r14) | |
192 | b common_enter | |
193 | ||
77b54e9f SP |
194 | enter_winkle: |
195 | /* | |
196 | * Note all register i.e per-core, per-subcore or per-thread is saved | |
197 | * here since any thread in the core might wake up first | |
198 | */ | |
199 | mfspr r3,SPRN_SDR1 | |
200 | std r3,_SDR1(r1) | |
201 | mfspr r3,SPRN_RPR | |
202 | std r3,_RPR(r1) | |
203 | mfspr r3,SPRN_SPURR | |
204 | std r3,_SPURR(r1) | |
205 | mfspr r3,SPRN_PURR | |
206 | std r3,_PURR(r1) | |
207 | mfspr r3,SPRN_TSCR | |
208 | std r3,_TSCR(r1) | |
209 | mfspr r3,SPRN_DSCR | |
210 | std r3,_DSCR(r1) | |
211 | mfspr r3,SPRN_AMOR | |
212 | std r3,_AMOR(r1) | |
213 | mfspr r3,SPRN_WORT | |
214 | std r3,_WORT(r1) | |
215 | mfspr r3,SPRN_WORC | |
216 | std r3,_WORC(r1) | |
217 | IDLE_STATE_ENTER_SEQ(PPC_WINKLE) | |
f0888f70 | 218 | |
aca79d2b VS |
219 | _GLOBAL(power7_idle) |
220 | /* Now check if user or arch enabled NAP mode */ | |
221 | LOAD_REG_ADDRBASE(r3,powersave_nap) | |
222 | lwz r4,ADDROFF(powersave_nap)(r3) | |
223 | cmpwi 0,r4,0 | |
224 | beqlr | |
8d6f7c5a | 225 | li r3, 1 |
aca79d2b VS |
226 | /* fall through */ |
227 | ||
228 | _GLOBAL(power7_nap) | |
8d6f7c5a | 229 | mr r4,r3 |
7cba160a | 230 | li r3,PNV_THREAD_NAP |
aca79d2b VS |
231 | b power7_powersave_common |
232 | /* No return */ | |
233 | ||
234 | _GLOBAL(power7_sleep) | |
7cba160a | 235 | li r3,PNV_THREAD_SLEEP |
c733cf83 | 236 | li r4,1 |
aca79d2b VS |
237 | b power7_powersave_common |
238 | /* No return */ | |
948cf67c | 239 | |
77b54e9f SP |
240 | _GLOBAL(power7_winkle) |
241 | li r3,3 | |
242 | li r4,1 | |
243 | b power7_powersave_common | |
244 | /* No return */ | |
245 | ||
bbdb760d MS |
246 | #define CHECK_HMI_INTERRUPT \ |
247 | mfspr r0,SPRN_SRR1; \ | |
248 | BEGIN_FTR_SECTION_NESTED(66); \ | |
249 | rlwinm r0,r0,45-31,0xf; /* extract wake reason field (P8) */ \ | |
250 | FTR_SECTION_ELSE_NESTED(66); \ | |
251 | rlwinm r0,r0,45-31,0xe; /* P7 wake reason field is 3 bits */ \ | |
252 | ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66); \ | |
253 | cmpwi r0,0xa; /* Hypervisor maintenance ? */ \ | |
254 | bne 20f; \ | |
255 | /* Invoke opal call to handle hmi */ \ | |
256 | ld r2,PACATOC(r13); \ | |
257 | ld r1,PACAR1(r13); \ | |
258 | std r3,ORIG_GPR3(r1); /* Save original r3 */ \ | |
7cba160a | 259 | li r0,OPAL_HANDLE_HMI; /* Pass opal token argument*/ \ |
db97efff | 260 | bl opal_call_realmode; \ |
bbdb760d MS |
261 | ld r3,ORIG_GPR3(r1); /* Restore original r3 */ \ |
262 | 20: nop; | |
263 | ||
264 | ||
97eb001f VS |
265 | _GLOBAL(power7_wakeup_tb_loss) |
266 | ld r2,PACATOC(r13); | |
267 | ld r1,PACAR1(r13) | |
7cba160a SP |
268 | /* |
269 | * Before entering any idle state, the NVGPRs are saved in the stack | |
270 | * and they are restored before switching to the process context. Hence | |
271 | * until they are restored, they are free to be used. | |
272 | * | |
273 | * Save SRR1 in a NVGPR as it might be clobbered in opal_call_realmode | |
274 | * (called in CHECK_HMI_INTERRUPT). SRR1 is required to determine the | |
275 | * wakeup reason if we branch to kvm_start_guest. | |
276 | */ | |
97eb001f | 277 | |
7cba160a | 278 | mfspr r16,SPRN_SRR1 |
bbdb760d MS |
279 | BEGIN_FTR_SECTION |
280 | CHECK_HMI_INTERRUPT | |
281 | END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) | |
7cba160a SP |
282 | |
283 | lbz r7,PACA_THREAD_MASK(r13) | |
284 | ld r14,PACA_CORE_IDLE_STATE_PTR(r13) | |
285 | lwarx_loop2: | |
286 | lwarx r15,0,r14 | |
287 | andi. r9,r15,PNV_CORE_IDLE_LOCK_BIT | |
288 | /* | |
289 | * Lock bit is set in one of the 2 cases- | |
290 | * a. In the sleep/winkle enter path, the last thread is executing | |
291 | * fastsleep workaround code. | |
292 | * b. In the wake up path, another thread is executing fastsleep | |
293 | * workaround undo code or resyncing timebase or restoring context | |
294 | * In either case loop until the lock bit is cleared. | |
295 | */ | |
296 | bne core_idle_lock_held | |
297 | ||
298 | cmpwi cr2,r15,0 | |
77b54e9f SP |
299 | lbz r4,PACA_SUBCORE_SIBLING_MASK(r13) |
300 | and r4,r4,r15 | |
301 | cmpwi cr1,r4,0 /* Check if first in subcore */ | |
302 | ||
303 | /* | |
304 | * At this stage | |
305 | * cr1 - 0b0100 if first thread to wakeup in subcore | |
306 | * cr2 - 0b0100 if first thread to wakeup in core | |
307 | * cr3- 0b0010 if waking up from sleep or winkle | |
308 | * cr4 - 0b0100 if waking up from winkle | |
309 | */ | |
310 | ||
7cba160a SP |
311 | or r15,r15,r7 /* Set thread bit */ |
312 | ||
77b54e9f | 313 | beq cr1,first_thread_in_subcore |
7cba160a | 314 | |
77b54e9f | 315 | /* Not first thread in subcore to wake up */ |
7cba160a SP |
316 | stwcx. r15,0,r14 |
317 | bne- lwarx_loop2 | |
318 | isync | |
319 | b common_exit | |
320 | ||
321 | core_idle_lock_held: | |
322 | HMT_LOW | |
323 | core_idle_lock_loop: | |
324 | lwz r15,0(14) | |
325 | andi. r9,r15,PNV_CORE_IDLE_LOCK_BIT | |
326 | bne core_idle_lock_loop | |
327 | HMT_MEDIUM | |
328 | b lwarx_loop2 | |
329 | ||
77b54e9f SP |
330 | first_thread_in_subcore: |
331 | /* First thread in subcore to wakeup */ | |
7cba160a SP |
332 | ori r15,r15,PNV_CORE_IDLE_LOCK_BIT |
333 | stwcx. r15,0,r14 | |
334 | bne- lwarx_loop2 | |
335 | isync | |
336 | ||
77b54e9f SP |
337 | /* |
338 | * If waking up from sleep, subcore state is not lost. Hence | |
339 | * skip subcore state restore | |
340 | */ | |
341 | bne cr4,subcore_state_restored | |
342 | ||
343 | /* Restore per-subcore state */ | |
344 | ld r4,_SDR1(r1) | |
345 | mtspr SPRN_SDR1,r4 | |
346 | ld r4,_RPR(r1) | |
347 | mtspr SPRN_RPR,r4 | |
348 | ld r4,_AMOR(r1) | |
349 | mtspr SPRN_AMOR,r4 | |
350 | ||
351 | subcore_state_restored: | |
352 | /* | |
353 | * Check if the thread is also the first thread in the core. If not, | |
354 | * skip to clear_lock. | |
355 | */ | |
356 | bne cr2,clear_lock | |
357 | ||
358 | first_thread_in_core: | |
359 | ||
7cba160a SP |
360 | /* |
361 | * First thread in the core waking up from fastsleep. It needs to | |
362 | * call the fastsleep workaround code if the platform requires it. | |
363 | * Call it unconditionally here. The below branch instruction will | |
364 | * be patched out when the idle states are discovered if platform | |
365 | * does not require workaround. | |
366 | */ | |
367 | .global pnv_fastsleep_workaround_at_exit | |
368 | pnv_fastsleep_workaround_at_exit: | |
369 | b fastsleep_workaround_at_exit | |
370 | ||
371 | timebase_resync: | |
372 | /* Do timebase resync if we are waking up from sleep. Use cr3 value | |
373 | * set in exceptions-64s.S */ | |
374 | ble cr3,clear_lock | |
97eb001f | 375 | /* Time base re-sync */ |
7cba160a | 376 | li r0,OPAL_RESYNC_TIMEBASE |
db97efff | 377 | bl opal_call_realmode; |
97eb001f VS |
378 | /* TODO: Check r3 for failure */ |
379 | ||
77b54e9f SP |
380 | /* |
381 | * If waking up from sleep, per core state is not lost, skip to | |
382 | * clear_lock. | |
383 | */ | |
384 | bne cr4,clear_lock | |
385 | ||
386 | /* Restore per core state */ | |
387 | ld r4,_TSCR(r1) | |
388 | mtspr SPRN_TSCR,r4 | |
389 | ld r4,_WORC(r1) | |
390 | mtspr SPRN_WORC,r4 | |
391 | ||
7cba160a SP |
392 | clear_lock: |
393 | andi. r15,r15,PNV_CORE_IDLE_THREAD_BITS | |
394 | lwsync | |
395 | stw r15,0(r14) | |
396 | ||
397 | common_exit: | |
77b54e9f SP |
398 | /* |
399 | * Common to all threads. | |
400 | * | |
401 | * If waking up from sleep, hypervisor state is not lost. Hence | |
402 | * skip hypervisor state restore. | |
403 | */ | |
404 | bne cr4,hypervisor_state_restored | |
405 | ||
406 | /* Waking up from winkle */ | |
407 | ||
408 | /* Restore per thread state */ | |
409 | bl __restore_cpu_power8 | |
410 | ||
411 | /* Restore SLB from PACA */ | |
412 | ld r8,PACA_SLBSHADOWPTR(r13) | |
413 | ||
414 | .rept SLB_NUM_BOLTED | |
415 | li r3, SLBSHADOW_SAVEAREA | |
416 | LDX_BE r5, r8, r3 | |
417 | addi r3, r3, 8 | |
418 | LDX_BE r6, r8, r3 | |
419 | andis. r7,r5,SLB_ESID_V@h | |
420 | beq 1f | |
421 | slbmte r6,r5 | |
422 | 1: addi r8,r8,16 | |
423 | .endr | |
424 | ||
425 | ld r4,_SPURR(r1) | |
426 | mtspr SPRN_SPURR,r4 | |
427 | ld r4,_PURR(r1) | |
428 | mtspr SPRN_PURR,r4 | |
429 | ld r4,_DSCR(r1) | |
430 | mtspr SPRN_DSCR,r4 | |
431 | ld r4,_WORT(r1) | |
432 | mtspr SPRN_WORT,r4 | |
433 | ||
434 | hypervisor_state_restored: | |
435 | ||
7cba160a SP |
436 | li r5,PNV_THREAD_RUNNING |
437 | stb r5,PACA_THREAD_IDLE_STATE(r13) | |
438 | ||
439 | mtspr SPRN_SRR1,r16 | |
440 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE | |
441 | li r0,KVM_HWTHREAD_IN_KERNEL | |
442 | stb r0,HSTATE_HWTHREAD_STATE(r13) | |
443 | /* Order setting hwthread_state vs. testing hwthread_req */ | |
444 | sync | |
445 | lbz r0,HSTATE_HWTHREAD_REQ(r13) | |
446 | cmpwi r0,0 | |
447 | beq 6f | |
448 | b kvm_start_guest | |
449 | 6: | |
450 | #endif | |
451 | ||
97eb001f VS |
452 | REST_NVGPRS(r1) |
453 | REST_GPR(2, r1) | |
454 | ld r3,_CCR(r1) | |
455 | ld r4,_MSR(r1) | |
456 | ld r5,_NIP(r1) | |
457 | addi r1,r1,INT_FRAME_SIZE | |
458 | mtcr r3 | |
459 | mfspr r3,SPRN_SRR1 /* Return SRR1 */ | |
460 | mtspr SPRN_SRR1,r4 | |
461 | mtspr SPRN_SRR0,r5 | |
462 | rfid | |
463 | ||
7cba160a SP |
464 | fastsleep_workaround_at_exit: |
465 | li r3,1 | |
466 | li r4,0 | |
467 | li r0,OPAL_CONFIG_CPU_IDLE_STATE | |
468 | bl opal_call_realmode | |
469 | b timebase_resync | |
470 | ||
56548fc0 PM |
471 | /* |
472 | * R3 here contains the value that will be returned to the caller | |
473 | * of power7_nap. | |
474 | */ | |
948cf67c | 475 | _GLOBAL(power7_wakeup_loss) |
948cf67c | 476 | ld r1,PACAR1(r13) |
bbdb760d MS |
477 | BEGIN_FTR_SECTION |
478 | CHECK_HMI_INTERRUPT | |
479 | END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) | |
948cf67c BH |
480 | REST_NVGPRS(r1) |
481 | REST_GPR(2, r1) | |
56548fc0 | 482 | ld r6,_CCR(r1) |
948cf67c BH |
483 | ld r4,_MSR(r1) |
484 | ld r5,_NIP(r1) | |
485 | addi r1,r1,INT_FRAME_SIZE | |
56548fc0 | 486 | mtcr r6 |
948cf67c BH |
487 | mtspr SPRN_SRR1,r4 |
488 | mtspr SPRN_SRR0,r5 | |
489 | rfid | |
490 | ||
56548fc0 PM |
491 | /* |
492 | * R3 here contains the value that will be returned to the caller | |
493 | * of power7_nap. | |
494 | */ | |
948cf67c | 495 | _GLOBAL(power7_wakeup_noloss) |
2fde6d20 PM |
496 | lbz r0,PACA_NAPSTATELOST(r13) |
497 | cmpwi r0,0 | |
b1576fec | 498 | bne power7_wakeup_loss |
bbdb760d MS |
499 | BEGIN_FTR_SECTION |
500 | CHECK_HMI_INTERRUPT | |
501 | END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) | |
948cf67c BH |
502 | ld r1,PACAR1(r13) |
503 | ld r4,_MSR(r1) | |
504 | ld r5,_NIP(r1) | |
505 | addi r1,r1,INT_FRAME_SIZE | |
506 | mtspr SPRN_SRR1,r4 | |
507 | mtspr SPRN_SRR0,r5 | |
508 | rfid |