Commit | Line | Data |
---|---|---|
9d670280 JX |
1 | /* |
2 | * ICSWX and ACOP Management | |
3 | * | |
4 | * Copyright (C) 2011 Anton Blanchard, IBM Corp. <anton@samba.org> | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * as published by the Free Software Foundation; either version | |
9 | * 2 of the License, or (at your option) any later version. | |
10 | * | |
11 | */ | |
12 | ||
13 | #include <linux/sched.h> | |
14 | #include <linux/kernel.h> | |
15 | #include <linux/errno.h> | |
16 | #include <linux/types.h> | |
17 | #include <linux/mm.h> | |
18 | #include <linux/spinlock.h> | |
19 | #include <linux/module.h> | |
c3dcf53a JX |
20 | #include <linux/uaccess.h> |
21 | ||
9d670280 JX |
22 | #include "icswx.h" |
23 | ||
24 | /* | |
25 | * The processor and its L2 cache cause the icswx instruction to | |
26 | * generate a COP_REQ transaction on PowerBus. The transaction has no | |
27 | * address, and the processor does not perform an MMU access to | |
28 | * authenticate the transaction. The command portion of the PowerBus | |
29 | * COP_REQ transaction includes the LPAR_ID (LPID) and the coprocessor | |
30 | * Process ID (PID), which the coprocessor compares to the authorized | |
31 | * LPID and PID held in the coprocessor, to determine if the process | |
32 | * is authorized to generate the transaction. The data of the COP_REQ | |
33 | * transaction is 128-byte or less in size and is placed in cacheable | |
34 | * memory on a 128-byte cache line boundary. | |
35 | * | |
36 | * The task to use a coprocessor should use use_cop() to mark the use | |
37 | * of the Coprocessor Type (CT) and context switching. On a server | |
38 | * class processor, the PID register is used only for coprocessor | |
39 | * management + * and so a coprocessor PID is allocated before | |
40 | * executing icswx + * instruction. Drop_cop() is used to free the | |
41 | * coprocessor PID. | |
42 | * | |
43 | * Example: | |
44 | * Host Fabric Interface (HFI) is a PowerPC network coprocessor. | |
45 | * Each HFI have multiple windows. Each HFI window serves as a | |
46 | * network device sending to and receiving from HFI network. | |
47 | * HFI immediate send function uses icswx instruction. The immediate | |
48 | * send function allows small (single cache-line) packets be sent | |
49 | * without using the regular HFI send FIFO and doorbell, which are | |
50 | * much slower than immediate send. | |
51 | * | |
52 | * For each task intending to use HFI immediate send, the HFI driver | |
53 | * calls use_cop() to obtain a coprocessor PID for the task. | |
54 | * The HFI driver then allocate a free HFI window and save the | |
55 | * coprocessor PID to the HFI window to allow the task to use the | |
56 | * HFI window. | |
57 | * | |
58 | * The HFI driver repeatedly creates immediate send packets and | |
59 | * issues icswx instruction to send data through the HFI window. | |
60 | * The HFI compares the coprocessor PID in the CPU PID register | |
61 | * to the PID held in the HFI window to determine if the transaction | |
62 | * is allowed. | |
63 | * | |
64 | * When the task to release the HFI window, the HFI driver calls | |
65 | * drop_cop() to release the coprocessor PID. | |
66 | */ | |
67 | ||
68 | void switch_cop(struct mm_struct *next) | |
69 | { | |
aac3d0c8 | 70 | #ifdef CONFIG_PPC_ICSWX_PID |
9d670280 JX |
71 | mtspr(SPRN_PID, next->context.cop_pid); |
72 | #endif | |
73 | mtspr(SPRN_ACOP, next->context.acop); | |
74 | } | |
75 | ||
76 | /** | |
77 | * Start using a coprocessor. | |
78 | * @acop: mask of coprocessor to be used. | |
79 | * @mm: The mm the coprocessor to associate with. Most likely current mm. | |
80 | * | |
81 | * Return a positive PID if successful. Negative errno otherwise. | |
82 | * The returned PID will be fed to the coprocessor to determine if an | |
83 | * icswx transaction is authenticated. | |
84 | */ | |
85 | int use_cop(unsigned long acop, struct mm_struct *mm) | |
86 | { | |
87 | int ret; | |
88 | ||
89 | if (!cpu_has_feature(CPU_FTR_ICSWX)) | |
90 | return -ENODEV; | |
91 | ||
92 | if (!mm || !acop) | |
93 | return -EINVAL; | |
94 | ||
95 | /* The page_table_lock ensures mm_users won't change under us */ | |
96 | spin_lock(&mm->page_table_lock); | |
97 | spin_lock(mm->context.cop_lockp); | |
98 | ||
99 | ret = get_cop_pid(mm); | |
100 | if (ret < 0) | |
101 | goto out; | |
102 | ||
103 | /* update acop */ | |
104 | mm->context.acop |= acop; | |
105 | ||
106 | sync_cop(mm); | |
107 | ||
108 | /* | |
109 | * If this is a threaded process then there might be other threads | |
110 | * running. We need to send an IPI to force them to pick up any | |
111 | * change in PID and ACOP. | |
112 | */ | |
113 | if (atomic_read(&mm->mm_users) > 1) | |
114 | smp_call_function(sync_cop, mm, 1); | |
115 | ||
116 | out: | |
117 | spin_unlock(mm->context.cop_lockp); | |
118 | spin_unlock(&mm->page_table_lock); | |
119 | ||
120 | return ret; | |
121 | } | |
122 | EXPORT_SYMBOL_GPL(use_cop); | |
123 | ||
124 | /** | |
125 | * Stop using a coprocessor. | |
126 | * @acop: mask of coprocessor to be stopped. | |
127 | * @mm: The mm the coprocessor associated with. | |
128 | */ | |
129 | void drop_cop(unsigned long acop, struct mm_struct *mm) | |
130 | { | |
131 | int free_pid; | |
132 | ||
133 | if (!cpu_has_feature(CPU_FTR_ICSWX)) | |
134 | return; | |
135 | ||
136 | if (WARN_ON_ONCE(!mm)) | |
137 | return; | |
138 | ||
139 | /* The page_table_lock ensures mm_users won't change under us */ | |
140 | spin_lock(&mm->page_table_lock); | |
141 | spin_lock(mm->context.cop_lockp); | |
142 | ||
143 | mm->context.acop &= ~acop; | |
144 | ||
145 | free_pid = disable_cop_pid(mm); | |
146 | sync_cop(mm); | |
147 | ||
148 | /* | |
149 | * If this is a threaded process then there might be other threads | |
150 | * running. We need to send an IPI to force them to pick up any | |
151 | * change in PID and ACOP. | |
152 | */ | |
153 | if (atomic_read(&mm->mm_users) > 1) | |
154 | smp_call_function(sync_cop, mm, 1); | |
155 | ||
156 | if (free_pid != COP_PID_NONE) | |
157 | free_cop_pid(free_pid); | |
158 | ||
159 | spin_unlock(mm->context.cop_lockp); | |
160 | spin_unlock(&mm->page_table_lock); | |
161 | } | |
162 | EXPORT_SYMBOL_GPL(drop_cop); | |
c3dcf53a JX |
163 | |
164 | static int acop_use_cop(int ct) | |
165 | { | |
de801de1 | 166 | /* There is no alternate policy, yet */ |
c3dcf53a JX |
167 | return -1; |
168 | } | |
169 | ||
170 | /* | |
171 | * Get the instruction word at the NIP | |
172 | */ | |
173 | static u32 acop_get_inst(struct pt_regs *regs) | |
174 | { | |
175 | u32 inst; | |
176 | u32 __user *p; | |
177 | ||
178 | p = (u32 __user *)regs->nip; | |
179 | if (!access_ok(VERIFY_READ, p, sizeof(*p))) | |
180 | return 0; | |
181 | ||
182 | if (__get_user(inst, p)) | |
183 | return 0; | |
184 | ||
185 | return inst; | |
186 | } | |
187 | ||
188 | /** | |
189 | * @regs: regsiters at time of interrupt | |
190 | * @address: storage address | |
191 | * @error_code: Fault code, usually the DSISR or ESR depending on | |
192 | * processor type | |
193 | * | |
194 | * Return 0 if we are able to resolve the data storage fault that | |
195 | * results from a CT miss in the ACOP register. | |
196 | */ | |
197 | int acop_handle_fault(struct pt_regs *regs, unsigned long address, | |
198 | unsigned long error_code) | |
199 | { | |
200 | int ct; | |
201 | u32 inst = 0; | |
202 | ||
203 | if (!cpu_has_feature(CPU_FTR_ICSWX)) { | |
204 | pr_info("No coprocessors available"); | |
205 | _exception(SIGILL, regs, ILL_ILLOPN, address); | |
206 | } | |
207 | ||
208 | if (!user_mode(regs)) { | |
209 | /* this could happen if the HV denies the | |
210 | * kernel access, for now we just die */ | |
211 | die("ICSWX from kernel failed", regs, SIGSEGV); | |
212 | } | |
213 | ||
214 | /* Some implementations leave us a hint for the CT */ | |
215 | ct = ICSWX_GET_CT_HINT(error_code); | |
216 | if (ct < 0) { | |
217 | /* we have to peek at the instruction word to figure out CT */ | |
218 | u32 ccw; | |
219 | u32 rs; | |
220 | ||
221 | inst = acop_get_inst(regs); | |
222 | if (inst == 0) | |
223 | return -1; | |
224 | ||
225 | rs = (inst >> (31 - 10)) & 0x1f; | |
226 | ccw = regs->gpr[rs]; | |
227 | ct = (ccw >> 16) & 0x3f; | |
228 | } | |
229 | ||
de801de1 JX |
230 | /* |
231 | * We could be here because another thread has enabled acop | |
232 | * but the ACOP register has yet to be updated. | |
233 | * | |
234 | * This should have been taken care of by the IPI to sync all | |
235 | * the threads (see smp_call_function(sync_cop, mm, 1)), but | |
236 | * that could take forever if there are a significant amount | |
237 | * of threads. | |
238 | * | |
239 | * Given the number of threads on some of these systems, | |
240 | * perhaps this is the best way to sync ACOP rather than whack | |
241 | * every thread with an IPI. | |
242 | */ | |
243 | if ((acop_copro_type_bit(ct) & current->active_mm->context.acop) != 0) { | |
244 | sync_cop(current->active_mm); | |
245 | return 0; | |
246 | } | |
247 | ||
248 | /* check for alternate policy */ | |
c3dcf53a JX |
249 | if (!acop_use_cop(ct)) |
250 | return 0; | |
251 | ||
252 | /* at this point the CT is unknown to the system */ | |
de801de1 | 253 | pr_warn("%s[%d]: Coprocessor %d is unavailable\n", |
c3dcf53a JX |
254 | current->comm, current->pid, ct); |
255 | ||
256 | /* get inst if we don't already have it */ | |
257 | if (inst == 0) { | |
258 | inst = acop_get_inst(regs); | |
259 | if (inst == 0) | |
260 | return -1; | |
261 | } | |
262 | ||
263 | /* Check if the instruction is the "record form" */ | |
264 | if (inst & 1) { | |
265 | /* | |
266 | * the instruction is "record" form so we can reject | |
267 | * using CR0 | |
268 | */ | |
269 | regs->ccr &= ~(0xful << 28); | |
270 | regs->ccr |= ICSWX_RC_NOT_FOUND << 28; | |
271 | ||
272 | /* Move on to the next instruction */ | |
273 | regs->nip += 4; | |
274 | } else { | |
275 | /* | |
276 | * There is no architected mechanism to report a bad | |
277 | * CT so we could either SIGILL or report nothing. | |
278 | * Since the non-record version should only bu used | |
279 | * for "hints" or "don't care" we should probably do | |
280 | * nothing. However, I could see how some people | |
281 | * might want an SIGILL so it here if you want it. | |
282 | */ | |
283 | #ifdef CONFIG_PPC_ICSWX_USE_SIGILL | |
284 | _exception(SIGILL, regs, ILL_ILLOPN, address); | |
285 | #else | |
286 | regs->nip += 4; | |
287 | #endif | |
288 | } | |
289 | ||
290 | return 0; | |
291 | } | |
292 | EXPORT_SYMBOL_GPL(acop_handle_fault); |