Commit | Line | Data |
---|---|---|
b4072df4 PM |
1 | /* |
2 | * This program is free software; you can redistribute it and/or modify | |
3 | * it under the terms of the GNU General Public License, version 2, as | |
4 | * published by the Free Software Foundation. | |
5 | * | |
6 | * Copyright 2012 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> | |
7 | */ | |
8 | ||
9 | #include <linux/types.h> | |
10 | #include <linux/string.h> | |
11 | #include <linux/kvm.h> | |
12 | #include <linux/kvm_host.h> | |
13 | #include <linux/kernel.h> | |
14 | #include <asm/opal.h> | |
36df96f8 | 15 | #include <asm/mce.h> |
fd7bacbc MS |
16 | #include <asm/machdep.h> |
17 | #include <asm/cputhreads.h> | |
18 | #include <asm/hmi.h> | |
b4072df4 PM |
19 | |
20 | /* SRR1 bits for machine check on POWER7 */ | |
21 | #define SRR1_MC_LDSTERR (1ul << (63-42)) | |
22 | #define SRR1_MC_IFETCH_SH (63-45) | |
23 | #define SRR1_MC_IFETCH_MASK 0x7 | |
24 | #define SRR1_MC_IFETCH_SLBPAR 2 /* SLB parity error */ | |
25 | #define SRR1_MC_IFETCH_SLBMULTI 3 /* SLB multi-hit */ | |
26 | #define SRR1_MC_IFETCH_SLBPARMULTI 4 /* SLB parity + multi-hit */ | |
27 | #define SRR1_MC_IFETCH_TLBMULTI 5 /* I-TLB multi-hit */ | |
28 | ||
29 | /* DSISR bits for machine check on POWER7 */ | |
30 | #define DSISR_MC_DERAT_MULTI 0x800 /* D-ERAT multi-hit */ | |
31 | #define DSISR_MC_TLB_MULTI 0x400 /* D-TLB multi-hit */ | |
32 | #define DSISR_MC_SLB_PARITY 0x100 /* SLB parity error */ | |
33 | #define DSISR_MC_SLB_MULTI 0x080 /* SLB multi-hit */ | |
34 | #define DSISR_MC_SLB_PARMULTI 0x040 /* SLB parity + multi-hit */ | |
35 | ||
36 | /* POWER7 SLB flush and reload */ | |
37 | static void reload_slb(struct kvm_vcpu *vcpu) | |
38 | { | |
39 | struct slb_shadow *slb; | |
40 | unsigned long i, n; | |
41 | ||
42 | /* First clear out SLB */ | |
43 | asm volatile("slbmte %0,%0; slbia" : : "r" (0)); | |
44 | ||
45 | /* Do they have an SLB shadow buffer registered? */ | |
46 | slb = vcpu->arch.slb_shadow.pinned_addr; | |
47 | if (!slb) | |
48 | return; | |
49 | ||
50 | /* Sanity check */ | |
02407552 | 51 | n = min_t(u32, be32_to_cpu(slb->persistent), SLB_MIN_SIZE); |
b4072df4 PM |
52 | if ((void *) &slb->save_area[n] > vcpu->arch.slb_shadow.pinned_end) |
53 | return; | |
54 | ||
55 | /* Load up the SLB from that */ | |
56 | for (i = 0; i < n; ++i) { | |
02407552 AG |
57 | unsigned long rb = be64_to_cpu(slb->save_area[i].esid); |
58 | unsigned long rs = be64_to_cpu(slb->save_area[i].vsid); | |
b4072df4 PM |
59 | |
60 | rb = (rb & ~0xFFFul) | i; /* insert entry number */ | |
61 | asm volatile("slbmte %0,%1" : : "r" (rs), "r" (rb)); | |
62 | } | |
63 | } | |
64 | ||
b4072df4 PM |
65 | /* |
66 | * On POWER7, see if we can handle a machine check that occurred inside | |
67 | * the guest in real mode, without switching to the host partition. | |
68 | * | |
69 | * Returns: 0 => exit guest, 1 => deliver machine check to guest | |
70 | */ | |
71 | static long kvmppc_realmode_mc_power7(struct kvm_vcpu *vcpu) | |
72 | { | |
73 | unsigned long srr1 = vcpu->arch.shregs.msr; | |
36df96f8 | 74 | struct machine_check_event mce_evt; |
b4072df4 PM |
75 | long handled = 1; |
76 | ||
77 | if (srr1 & SRR1_MC_LDSTERR) { | |
78 | /* error on load/store */ | |
79 | unsigned long dsisr = vcpu->arch.shregs.dsisr; | |
80 | ||
81 | if (dsisr & (DSISR_MC_SLB_PARMULTI | DSISR_MC_SLB_MULTI | | |
82 | DSISR_MC_SLB_PARITY | DSISR_MC_DERAT_MULTI)) { | |
83 | /* flush and reload SLB; flushes D-ERAT too */ | |
84 | reload_slb(vcpu); | |
85 | dsisr &= ~(DSISR_MC_SLB_PARMULTI | DSISR_MC_SLB_MULTI | | |
86 | DSISR_MC_SLB_PARITY | DSISR_MC_DERAT_MULTI); | |
87 | } | |
88 | if (dsisr & DSISR_MC_TLB_MULTI) { | |
04407050 | 89 | if (cur_cpu_spec && cur_cpu_spec->flush_tlb) |
45706bb5 | 90 | cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_LPID); |
b4072df4 PM |
91 | dsisr &= ~DSISR_MC_TLB_MULTI; |
92 | } | |
93 | /* Any other errors we don't understand? */ | |
94 | if (dsisr & 0xffffffffUL) | |
95 | handled = 0; | |
96 | } | |
97 | ||
98 | switch ((srr1 >> SRR1_MC_IFETCH_SH) & SRR1_MC_IFETCH_MASK) { | |
99 | case 0: | |
100 | break; | |
101 | case SRR1_MC_IFETCH_SLBPAR: | |
102 | case SRR1_MC_IFETCH_SLBMULTI: | |
103 | case SRR1_MC_IFETCH_SLBPARMULTI: | |
104 | reload_slb(vcpu); | |
105 | break; | |
106 | case SRR1_MC_IFETCH_TLBMULTI: | |
04407050 | 107 | if (cur_cpu_spec && cur_cpu_spec->flush_tlb) |
45706bb5 | 108 | cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_LPID); |
b4072df4 PM |
109 | break; |
110 | default: | |
111 | handled = 0; | |
112 | } | |
113 | ||
114 | /* | |
36df96f8 MS |
115 | * See if we have already handled the condition in the linux host. |
116 | * We assume that if the condition is recovered then linux host | |
b4072df4 PM |
117 | * will have generated an error log event that we will pick |
118 | * up and log later. | |
74845bc2 MS |
119 | * Don't release mce event now. We will queue up the event so that |
120 | * we can log the MCE event info on host console. | |
b4072df4 | 121 | */ |
36df96f8 MS |
122 | if (!get_mce_event(&mce_evt, MCE_EVENT_DONTRELEASE)) |
123 | goto out; | |
124 | ||
125 | if (mce_evt.version == MCE_V1 && | |
126 | (mce_evt.severity == MCE_SEV_NO_ERROR || | |
127 | mce_evt.disposition == MCE_DISPOSITION_RECOVERED)) | |
b4072df4 PM |
128 | handled = 1; |
129 | ||
36df96f8 MS |
130 | out: |
131 | /* | |
74845bc2 MS |
132 | * We are now going enter guest either through machine check |
133 | * interrupt (for unhandled errors) or will continue from | |
134 | * current HSRR0 (for handled errors) in guest. Hence | |
135 | * queue up the event so that we can log it from host console later. | |
36df96f8 | 136 | */ |
74845bc2 | 137 | machine_check_queue_event(); |
b4072df4 PM |
138 | |
139 | return handled; | |
140 | } | |
141 | ||
142 | long kvmppc_realmode_machine_check(struct kvm_vcpu *vcpu) | |
143 | { | |
c17b98cf | 144 | return kvmppc_realmode_mc_power7(vcpu); |
b4072df4 | 145 | } |
fd7bacbc MS |
146 | |
147 | /* Check if dynamic split is in force and return subcore size accordingly. */ | |
148 | static inline int kvmppc_cur_subcore_size(void) | |
149 | { | |
150 | if (local_paca->kvm_hstate.kvm_split_mode) | |
151 | return local_paca->kvm_hstate.kvm_split_mode->subcore_size; | |
152 | ||
153 | return threads_per_subcore; | |
154 | } | |
155 | ||
156 | void kvmppc_subcore_enter_guest(void) | |
157 | { | |
158 | int thread_id, subcore_id; | |
159 | ||
160 | thread_id = cpu_thread_in_core(local_paca->paca_index); | |
161 | subcore_id = thread_id / kvmppc_cur_subcore_size(); | |
162 | ||
163 | local_paca->sibling_subcore_state->in_guest[subcore_id] = 1; | |
164 | } | |
165 | ||
166 | void kvmppc_subcore_exit_guest(void) | |
167 | { | |
168 | int thread_id, subcore_id; | |
169 | ||
170 | thread_id = cpu_thread_in_core(local_paca->paca_index); | |
171 | subcore_id = thread_id / kvmppc_cur_subcore_size(); | |
172 | ||
173 | local_paca->sibling_subcore_state->in_guest[subcore_id] = 0; | |
174 | } | |
175 | ||
176 | static bool kvmppc_tb_resync_required(void) | |
177 | { | |
178 | if (test_and_set_bit(CORE_TB_RESYNC_REQ_BIT, | |
179 | &local_paca->sibling_subcore_state->flags)) | |
180 | return false; | |
181 | ||
182 | return true; | |
183 | } | |
184 | ||
185 | static void kvmppc_tb_resync_done(void) | |
186 | { | |
187 | clear_bit(CORE_TB_RESYNC_REQ_BIT, | |
188 | &local_paca->sibling_subcore_state->flags); | |
189 | } | |
190 | ||
191 | /* | |
192 | * kvmppc_realmode_hmi_handler() is called only by primary thread during | |
193 | * guest exit path. | |
194 | * | |
195 | * There are multiple reasons why HMI could occur, one of them is | |
196 | * Timebase (TB) error. If this HMI is due to TB error, then TB would | |
197 | * have been in stopped state. The opal hmi handler Will fix it and | |
198 | * restore the TB value with host timebase value. For HMI caused due | |
199 | * to non-TB errors, opal hmi handler will not touch/restore TB register | |
200 | * and hence there won't be any change in TB value. | |
201 | * | |
202 | * Since we are not sure about the cause of this HMI, we can't be sure | |
203 | * about the content of TB register whether it holds guest or host timebase | |
204 | * value. Hence the idea is to resync the TB on every HMI, so that we | |
205 | * know about the exact state of the TB value. Resync TB call will | |
206 | * restore TB to host timebase. | |
207 | * | |
208 | * Things to consider: | |
209 | * - On TB error, HMI interrupt is reported on all the threads of the core | |
210 | * that has encountered TB error irrespective of split-core mode. | |
211 | * - The very first thread on the core that get chance to fix TB error | |
212 | * would rsync the TB with local chipTOD value. | |
213 | * - The resync TB is a core level action i.e. it will sync all the TBs | |
214 | * in that core independent of split-core mode. This means if we trigger | |
215 | * TB sync from a thread from one subcore, it would affect TB values of | |
216 | * sibling subcores of the same core. | |
217 | * | |
218 | * All threads need to co-ordinate before making opal hmi handler. | |
219 | * All threads will use sibling_subcore_state->in_guest[] (shared by all | |
220 | * threads in the core) in paca which holds information about whether | |
221 | * sibling subcores are in Guest mode or host mode. The in_guest[] array | |
222 | * is of size MAX_SUBCORE_PER_CORE=4, indexed using subcore id to set/unset | |
223 | * subcore status. Only primary threads from each subcore is responsible | |
224 | * to set/unset its designated array element while entering/exiting the | |
225 | * guset. | |
226 | * | |
227 | * After invoking opal hmi handler call, one of the thread (of entire core) | |
228 | * will need to resync the TB. Bit 63 from subcore state bitmap flags | |
229 | * (sibling_subcore_state->flags) will be used to co-ordinate between | |
230 | * primary threads to decide who takes up the responsibility. | |
231 | * | |
232 | * This is what we do: | |
233 | * - Primary thread from each subcore tries to set resync required bit[63] | |
234 | * of paca->sibling_subcore_state->flags. | |
235 | * - The first primary thread that is able to set the flag takes the | |
236 | * responsibility of TB resync. (Let us call it as thread leader) | |
237 | * - All other threads which are in host will call | |
238 | * wait_for_subcore_guest_exit() and wait for in_guest[0-3] from | |
239 | * paca->sibling_subcore_state to get cleared. | |
240 | * - All the primary thread will clear its subcore status from subcore | |
241 | * state in_guest[] array respectively. | |
242 | * - Once all primary threads clear in_guest[0-3], all of them will invoke | |
243 | * opal hmi handler. | |
244 | * - Now all threads will wait for TB resync to complete by invoking | |
245 | * wait_for_tb_resync() except the thread leader. | |
246 | * - Thread leader will do a TB resync by invoking opal_resync_timebase() | |
247 | * call and the it will clear the resync required bit. | |
248 | * - All other threads will now come out of resync wait loop and proceed | |
249 | * with individual execution. | |
250 | * - On return of this function, primary thread will signal all | |
251 | * secondary threads to proceed. | |
252 | * - All secondary threads will eventually call opal hmi handler on | |
253 | * their exit path. | |
254 | */ | |
255 | ||
256 | long kvmppc_realmode_hmi_handler(void) | |
257 | { | |
258 | int ptid = local_paca->kvm_hstate.ptid; | |
259 | bool resync_req; | |
260 | ||
261 | /* This is only called on primary thread. */ | |
262 | BUG_ON(ptid != 0); | |
263 | __this_cpu_inc(irq_stat.hmi_exceptions); | |
264 | ||
265 | /* | |
266 | * By now primary thread has already completed guest->host | |
267 | * partition switch but haven't signaled secondaries yet. | |
268 | * All the secondary threads on this subcore is waiting | |
269 | * for primary thread to signal them to go ahead. | |
270 | * | |
271 | * For threads from subcore which isn't in guest, they all will | |
272 | * wait until all other subcores on this core exit the guest. | |
273 | * | |
274 | * Now set the resync required bit. If you are the first to | |
275 | * set this bit then kvmppc_tb_resync_required() function will | |
276 | * return true. For rest all other subcores | |
277 | * kvmppc_tb_resync_required() will return false. | |
278 | * | |
279 | * If resync_req == true, then this thread is responsible to | |
280 | * initiate TB resync after hmi handler has completed. | |
281 | * All other threads on this core will wait until this thread | |
282 | * clears the resync required bit flag. | |
283 | */ | |
284 | resync_req = kvmppc_tb_resync_required(); | |
285 | ||
286 | /* Reset the subcore status to indicate it has exited guest */ | |
287 | kvmppc_subcore_exit_guest(); | |
288 | ||
289 | /* | |
290 | * Wait for other subcores on this core to exit the guest. | |
291 | * All the primary threads and threads from subcore that are | |
292 | * not in guest will wait here until all subcores are out | |
293 | * of guest context. | |
294 | */ | |
295 | wait_for_subcore_guest_exit(); | |
296 | ||
297 | /* | |
298 | * At this point we are sure that primary threads from each | |
299 | * subcore on this core have completed guest->host partition | |
300 | * switch. Now it is safe to call HMI handler. | |
301 | */ | |
302 | if (ppc_md.hmi_exception_early) | |
303 | ppc_md.hmi_exception_early(NULL); | |
304 | ||
305 | /* | |
306 | * Check if this thread is responsible to resync TB. | |
307 | * All other threads will wait until this thread completes the | |
308 | * TB resync. | |
309 | */ | |
310 | if (resync_req) { | |
311 | opal_resync_timebase(); | |
312 | /* Reset TB resync req bit */ | |
313 | kvmppc_tb_resync_done(); | |
314 | } else { | |
315 | wait_for_tb_resync(); | |
316 | } | |
317 | return 0; | |
318 | } |