Commit | Line | Data |
---|---|---|
94bd2708 DN |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
a374c57b | 6 | * Copyright (c) 2008-2009 Silicon Graphics, Inc. All Rights Reserved. |
94bd2708 DN |
7 | */ |
8 | ||
9 | /* | |
10 | * Cross Partition Communication (XPC) sn2-based functions. | |
11 | * | |
12 | * Architecture specific implementation of common functions. | |
13 | * | |
14 | */ | |
15 | ||
e17d416b | 16 | #include <linux/delay.h> |
5a0e3ad6 | 17 | #include <linux/slab.h> |
94bd2708 | 18 | #include <asm/uncached.h> |
261f3b49 | 19 | #include <asm/sn/mspec.h> |
94bd2708 DN |
20 | #include <asm/sn/sn_sal.h> |
21 | #include "xpc.h" | |
22 | ||
ee6665e3 DN |
23 | /* |
24 | * Define the number of u64s required to represent all the C-brick nasids | |
25 | * as a bitmap. The cross-partition kernel modules deal only with | |
26 | * C-brick nasids, thus the need for bitmaps which don't account for | |
27 | * odd-numbered (non C-brick) nasids. | |
28 | */ | |
29 | #define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2) | |
30 | #define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8) | |
31 | #define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64) | |
32 | ||
33 | /* | |
34 | * Memory for XPC's amo variables is allocated by the MSPEC driver. These | |
35 | * pages are located in the lowest granule. The lowest granule uses 4k pages | |
36 | * for cached references and an alternate TLB handler to never provide a | |
37 | * cacheable mapping for the entire region. This will prevent speculative | |
38 | * reading of cached copies of our lines from being issued which will cause | |
39 | * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64 | |
40 | * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of | |
41 | * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify | |
42 | * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote | |
43 | * partitions (i.e., XPCs) consider themselves currently engaged with the | |
44 | * local XPC and 1 amo variable to request partition deactivation. | |
45 | */ | |
46 | #define XPC_NOTIFY_IRQ_AMOS_SN2 0 | |
47 | #define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \ | |
48 | XP_MAX_NPARTITIONS_SN2) | |
49 | #define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \ | |
50 | XP_NASID_MASK_WORDS_SN2) | |
51 | #define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1) | |
52 | ||
53 | /* | |
54 | * Buffer used to store a local copy of portions of a remote partition's | |
55 | * reserved page (either its header and part_nasids mask, or its vars). | |
56 | */ | |
ee6665e3 | 57 | static void *xpc_remote_copy_buffer_base_sn2; |
5b8669df | 58 | static char *xpc_remote_copy_buffer_sn2; |
ee6665e3 | 59 | |
8e85c23e DN |
60 | static struct xpc_vars_sn2 *xpc_vars_sn2; |
61 | static struct xpc_vars_part_sn2 *xpc_vars_part_sn2; | |
94bd2708 | 62 | |
5b8669df | 63 | static int |
a7665b0a | 64 | xpc_setup_partitions_sn2(void) |
5b8669df DN |
65 | { |
66 | /* nothing needs to be done */ | |
67 | return 0; | |
68 | } | |
69 | ||
6f2584f4 | 70 | static void |
a7665b0a | 71 | xpc_teardown_partitions_sn2(void) |
6f2584f4 JS |
72 | { |
73 | /* nothing needs to be done */ | |
74 | } | |
75 | ||
6e41017a | 76 | /* SH_IPI_ACCESS shub register value on startup */ |
8e85c23e DN |
77 | static u64 xpc_sh1_IPI_access_sn2; |
78 | static u64 xpc_sh2_IPI_access0_sn2; | |
79 | static u64 xpc_sh2_IPI_access1_sn2; | |
80 | static u64 xpc_sh2_IPI_access2_sn2; | |
81 | static u64 xpc_sh2_IPI_access3_sn2; | |
6e41017a DN |
82 | |
83 | /* | |
84 | * Change protections to allow IPI operations. | |
85 | */ | |
86 | static void | |
87 | xpc_allow_IPI_ops_sn2(void) | |
88 | { | |
89 | int node; | |
90 | int nasid; | |
91 | ||
ea57f80c | 92 | /* !!! The following should get moved into SAL. */ |
6e41017a | 93 | if (is_shub2()) { |
8e85c23e | 94 | xpc_sh2_IPI_access0_sn2 = |
6e41017a | 95 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0)); |
8e85c23e | 96 | xpc_sh2_IPI_access1_sn2 = |
6e41017a | 97 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1)); |
8e85c23e | 98 | xpc_sh2_IPI_access2_sn2 = |
6e41017a | 99 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2)); |
8e85c23e | 100 | xpc_sh2_IPI_access3_sn2 = |
6e41017a DN |
101 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3)); |
102 | ||
103 | for_each_online_node(node) { | |
104 | nasid = cnodeid_to_nasid(node); | |
105 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), | |
106 | -1UL); | |
107 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), | |
108 | -1UL); | |
109 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), | |
110 | -1UL); | |
111 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), | |
112 | -1UL); | |
113 | } | |
114 | } else { | |
8e85c23e | 115 | xpc_sh1_IPI_access_sn2 = |
6e41017a DN |
116 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS)); |
117 | ||
118 | for_each_online_node(node) { | |
119 | nasid = cnodeid_to_nasid(node); | |
120 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), | |
121 | -1UL); | |
122 | } | |
123 | } | |
124 | } | |
125 | ||
126 | /* | |
127 | * Restrict protections to disallow IPI operations. | |
128 | */ | |
129 | static void | |
130 | xpc_disallow_IPI_ops_sn2(void) | |
131 | { | |
132 | int node; | |
133 | int nasid; | |
134 | ||
ea57f80c | 135 | /* !!! The following should get moved into SAL. */ |
6e41017a DN |
136 | if (is_shub2()) { |
137 | for_each_online_node(node) { | |
138 | nasid = cnodeid_to_nasid(node); | |
139 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), | |
8e85c23e | 140 | xpc_sh2_IPI_access0_sn2); |
6e41017a | 141 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), |
8e85c23e | 142 | xpc_sh2_IPI_access1_sn2); |
6e41017a | 143 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), |
8e85c23e | 144 | xpc_sh2_IPI_access2_sn2); |
6e41017a | 145 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), |
8e85c23e | 146 | xpc_sh2_IPI_access3_sn2); |
6e41017a DN |
147 | } |
148 | } else { | |
149 | for_each_online_node(node) { | |
150 | nasid = cnodeid_to_nasid(node); | |
151 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), | |
8e85c23e | 152 | xpc_sh1_IPI_access_sn2); |
6e41017a DN |
153 | } |
154 | } | |
155 | } | |
156 | ||
33ba3c77 | 157 | /* |
7fb5e59d DN |
158 | * The following set of functions are used for the sending and receiving of |
159 | * IRQs (also known as IPIs). There are two flavors of IRQs, one that is | |
160 | * associated with partition activity (SGI_XPC_ACTIVATE) and the other that | |
161 | * is associated with channel activity (SGI_XPC_NOTIFY). | |
33ba3c77 DN |
162 | */ |
163 | ||
164 | static u64 | |
c39838ce | 165 | xpc_receive_IRQ_amo_sn2(struct amo *amo) |
33ba3c77 DN |
166 | { |
167 | return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR); | |
168 | } | |
169 | ||
170 | static enum xp_retval | |
c39838ce DN |
171 | xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid, |
172 | int vector) | |
33ba3c77 DN |
173 | { |
174 | int ret = 0; | |
175 | unsigned long irq_flags; | |
176 | ||
177 | local_irq_save(irq_flags); | |
178 | ||
179 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag); | |
180 | sn_send_IPI_phys(nasid, phys_cpuid, vector, 0); | |
181 | ||
182 | /* | |
183 | * We must always use the nofault function regardless of whether we | |
184 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
185 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 186 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
187 | */ |
188 | ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable), | |
189 | xp_nofault_PIOR_target)); | |
190 | ||
191 | local_irq_restore(irq_flags); | |
192 | ||
261f3b49 | 193 | return (ret == 0) ? xpSuccess : xpPioReadError; |
33ba3c77 DN |
194 | } |
195 | ||
c39838ce | 196 | static struct amo * |
7fb5e59d | 197 | xpc_init_IRQ_amo_sn2(int index) |
33ba3c77 | 198 | { |
8e85c23e | 199 | struct amo *amo = xpc_vars_sn2->amos_page + index; |
33ba3c77 | 200 | |
c39838ce | 201 | (void)xpc_receive_IRQ_amo_sn2(amo); /* clear amo variable */ |
33ba3c77 DN |
202 | return amo; |
203 | } | |
204 | ||
205 | /* | |
7fb5e59d | 206 | * Functions associated with SGI_XPC_ACTIVATE IRQ. |
33ba3c77 DN |
207 | */ |
208 | ||
6e41017a DN |
209 | /* |
210 | * Notify the heartbeat check thread that an activate IRQ has been received. | |
211 | */ | |
212 | static irqreturn_t | |
213 | xpc_handle_activate_IRQ_sn2(int irq, void *dev_id) | |
214 | { | |
5b8669df DN |
215 | unsigned long irq_flags; |
216 | ||
217 | spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
218 | xpc_activate_IRQ_rcvd++; | |
219 | spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
220 | ||
6e41017a DN |
221 | wake_up_interruptible(&xpc_activate_IRQ_wq); |
222 | return IRQ_HANDLED; | |
223 | } | |
224 | ||
33ba3c77 | 225 | /* |
c39838ce | 226 | * Flag the appropriate amo variable and send an IRQ to the specified node. |
33ba3c77 DN |
227 | */ |
228 | static void | |
a812dcc3 DN |
229 | xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid, |
230 | int to_nasid, int to_phys_cpuid) | |
33ba3c77 | 231 | { |
c39838ce | 232 | struct amo *amos = (struct amo *)__va(amos_page_pa + |
ee6665e3 | 233 | (XPC_ACTIVATE_IRQ_AMOS_SN2 * |
c39838ce | 234 | sizeof(struct amo))); |
33ba3c77 | 235 | |
04de7418 DN |
236 | (void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)], |
237 | BIT_MASK(from_nasid / 2), to_nasid, | |
33ba3c77 DN |
238 | to_phys_cpuid, SGI_XPC_ACTIVATE); |
239 | } | |
240 | ||
241 | static void | |
7fb5e59d | 242 | xpc_send_local_activate_IRQ_sn2(int from_nasid) |
33ba3c77 | 243 | { |
5b8669df | 244 | unsigned long irq_flags; |
8e85c23e | 245 | struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa + |
ee6665e3 | 246 | (XPC_ACTIVATE_IRQ_AMOS_SN2 * |
c39838ce | 247 | sizeof(struct amo))); |
33ba3c77 DN |
248 | |
249 | /* fake the sending and receipt of an activate IRQ from remote nasid */ | |
04de7418 DN |
250 | FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable), |
251 | FETCHOP_OR, BIT_MASK(from_nasid / 2)); | |
252 | ||
5b8669df DN |
253 | spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); |
254 | xpc_activate_IRQ_rcvd++; | |
255 | spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
256 | ||
6e41017a | 257 | wake_up_interruptible(&xpc_activate_IRQ_wq); |
33ba3c77 DN |
258 | } |
259 | ||
a47d5dac | 260 | /* |
7fb5e59d | 261 | * Functions associated with SGI_XPC_NOTIFY IRQ. |
a47d5dac | 262 | */ |
33ba3c77 | 263 | |
a47d5dac | 264 | /* |
7fb5e59d | 265 | * Check to see if any chctl flags were sent from the specified partition. |
a47d5dac | 266 | */ |
33ba3c77 | 267 | static void |
7fb5e59d | 268 | xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part) |
33ba3c77 | 269 | { |
7fb5e59d | 270 | union xpc_channel_ctl_flags chctl; |
a47d5dac | 271 | unsigned long irq_flags; |
33ba3c77 | 272 | |
7fb5e59d DN |
273 | chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2. |
274 | local_chctl_amo_va); | |
275 | if (chctl.all_flags == 0) | |
a47d5dac DN |
276 | return; |
277 | ||
7fb5e59d DN |
278 | spin_lock_irqsave(&part->chctl_lock, irq_flags); |
279 | part->chctl.all_flags |= chctl.all_flags; | |
280 | spin_unlock_irqrestore(&part->chctl_lock, irq_flags); | |
a47d5dac | 281 | |
7fb5e59d | 282 | dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags=" |
f509e34a | 283 | "0x%llx\n", XPC_PARTID(part), chctl.all_flags); |
a47d5dac DN |
284 | |
285 | xpc_wakeup_channel_mgr(part); | |
33ba3c77 DN |
286 | } |
287 | ||
a47d5dac DN |
288 | /* |
289 | * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified | |
290 | * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more | |
c39838ce | 291 | * than one partition, we use an amo structure per partition to indicate |
7fb5e59d | 292 | * whether a partition has sent an IRQ or not. If it has, then wake up the |
a47d5dac DN |
293 | * associated kthread to handle it. |
294 | * | |
7fb5e59d | 295 | * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC |
a47d5dac DN |
296 | * running on other partitions. |
297 | * | |
298 | * Noteworthy Arguments: | |
299 | * | |
300 | * irq - Interrupt ReQuest number. NOT USED. | |
301 | * | |
7fb5e59d | 302 | * dev_id - partid of IRQ's potential sender. |
a47d5dac DN |
303 | */ |
304 | static irqreturn_t | |
305 | xpc_handle_notify_IRQ_sn2(int irq, void *dev_id) | |
33ba3c77 | 306 | { |
a47d5dac DN |
307 | short partid = (short)(u64)dev_id; |
308 | struct xpc_partition *part = &xpc_partitions[partid]; | |
309 | ||
261f3b49 | 310 | DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2); |
a47d5dac DN |
311 | |
312 | if (xpc_part_ref(part)) { | |
7fb5e59d | 313 | xpc_check_for_sent_chctl_flags_sn2(part); |
a47d5dac DN |
314 | |
315 | xpc_part_deref(part); | |
316 | } | |
317 | return IRQ_HANDLED; | |
33ba3c77 DN |
318 | } |
319 | ||
320 | /* | |
7fb5e59d DN |
321 | * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor |
322 | * because the write to their associated amo variable completed after the IRQ | |
a47d5dac | 323 | * was received. |
33ba3c77 | 324 | */ |
a47d5dac | 325 | static void |
7fb5e59d | 326 | xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition *part) |
a47d5dac DN |
327 | { |
328 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; | |
329 | ||
330 | if (xpc_part_ref(part)) { | |
7fb5e59d | 331 | xpc_check_for_sent_chctl_flags_sn2(part); |
a47d5dac DN |
332 | |
333 | part_sn2->dropped_notify_IRQ_timer.expires = jiffies + | |
7fb5e59d | 334 | XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; |
a47d5dac DN |
335 | add_timer(&part_sn2->dropped_notify_IRQ_timer); |
336 | xpc_part_deref(part); | |
337 | } | |
338 | } | |
33ba3c77 DN |
339 | |
340 | /* | |
7fb5e59d | 341 | * Send a notify IRQ to the remote partition that is associated with the |
33ba3c77 DN |
342 | * specified channel. |
343 | */ | |
344 | static void | |
7fb5e59d DN |
345 | xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, |
346 | char *chctl_flag_string, unsigned long *irq_flags) | |
33ba3c77 DN |
347 | { |
348 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
a47d5dac | 349 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
7fb5e59d | 350 | union xpc_channel_ctl_flags chctl = { 0 }; |
33ba3c77 DN |
351 | enum xp_retval ret; |
352 | ||
83469b55 | 353 | if (likely(part->act_state != XPC_P_AS_DEACTIVATING)) { |
7fb5e59d DN |
354 | chctl.flags[ch->number] = chctl_flag; |
355 | ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va, | |
356 | chctl.all_flags, | |
357 | part_sn2->notify_IRQ_nasid, | |
358 | part_sn2->notify_IRQ_phys_cpuid, | |
33ba3c77 DN |
359 | SGI_XPC_NOTIFY); |
360 | dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n", | |
7fb5e59d | 361 | chctl_flag_string, ch->partid, ch->number, ret); |
33ba3c77 DN |
362 | if (unlikely(ret != xpSuccess)) { |
363 | if (irq_flags != NULL) | |
364 | spin_unlock_irqrestore(&ch->lock, *irq_flags); | |
365 | XPC_DEACTIVATE_PARTITION(part, ret); | |
366 | if (irq_flags != NULL) | |
367 | spin_lock_irqsave(&ch->lock, *irq_flags); | |
368 | } | |
369 | } | |
370 | } | |
371 | ||
7fb5e59d DN |
372 | #define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \ |
373 | xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f) | |
33ba3c77 DN |
374 | |
375 | /* | |
376 | * Make it look like the remote partition, which is associated with the | |
7fb5e59d DN |
377 | * specified channel, sent us a notify IRQ. This faked IRQ will be handled |
378 | * by xpc_check_for_dropped_notify_IRQ_sn2(). | |
33ba3c77 DN |
379 | */ |
380 | static void | |
7fb5e59d DN |
381 | xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, |
382 | char *chctl_flag_string) | |
33ba3c77 DN |
383 | { |
384 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
7fb5e59d | 385 | union xpc_channel_ctl_flags chctl = { 0 }; |
33ba3c77 | 386 | |
7fb5e59d DN |
387 | chctl.flags[ch->number] = chctl_flag; |
388 | FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va-> | |
389 | variable), FETCHOP_OR, chctl.all_flags); | |
33ba3c77 | 390 | dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n", |
7fb5e59d | 391 | chctl_flag_string, ch->partid, ch->number); |
33ba3c77 DN |
392 | } |
393 | ||
7fb5e59d DN |
394 | #define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \ |
395 | xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f) | |
33ba3c77 DN |
396 | |
397 | static void | |
7fb5e59d DN |
398 | xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch, |
399 | unsigned long *irq_flags) | |
33ba3c77 | 400 | { |
5b8669df | 401 | struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; |
33ba3c77 DN |
402 | |
403 | args->reason = ch->reason; | |
7fb5e59d | 404 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags); |
33ba3c77 DN |
405 | } |
406 | ||
407 | static void | |
7fb5e59d | 408 | xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 409 | { |
7fb5e59d | 410 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags); |
33ba3c77 DN |
411 | } |
412 | ||
413 | static void | |
7fb5e59d | 414 | xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 415 | { |
5b8669df | 416 | struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; |
33ba3c77 | 417 | |
bd3e64c1 | 418 | args->entry_size = ch->entry_size; |
33ba3c77 | 419 | args->local_nentries = ch->local_nentries; |
7fb5e59d | 420 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags); |
33ba3c77 DN |
421 | } |
422 | ||
423 | static void | |
7fb5e59d | 424 | xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 425 | { |
5b8669df | 426 | struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; |
33ba3c77 DN |
427 | |
428 | args->remote_nentries = ch->remote_nentries; | |
429 | args->local_nentries = ch->local_nentries; | |
5b8669df | 430 | args->local_msgqueue_pa = xp_pa(ch->sn.sn2.local_msgqueue); |
7fb5e59d | 431 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags); |
33ba3c77 DN |
432 | } |
433 | ||
efdd06ed RH |
434 | static void |
435 | xpc_send_chctl_opencomplete_sn2(struct xpc_channel *ch, | |
436 | unsigned long *irq_flags) | |
437 | { | |
438 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENCOMPLETE, irq_flags); | |
439 | } | |
440 | ||
33ba3c77 | 441 | static void |
7fb5e59d | 442 | xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch) |
33ba3c77 | 443 | { |
7fb5e59d | 444 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL); |
33ba3c77 DN |
445 | } |
446 | ||
447 | static void | |
7fb5e59d | 448 | xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch) |
33ba3c77 | 449 | { |
7fb5e59d | 450 | XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST); |
33ba3c77 DN |
451 | } |
452 | ||
6f2584f4 | 453 | static enum xp_retval |
5b8669df DN |
454 | xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch, |
455 | unsigned long msgqueue_pa) | |
456 | { | |
457 | ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa; | |
6f2584f4 | 458 | return xpSuccess; |
5b8669df DN |
459 | } |
460 | ||
33ba3c77 DN |
461 | /* |
462 | * This next set of functions are used to keep track of when a partition is | |
463 | * potentially engaged in accessing memory belonging to another partition. | |
464 | */ | |
465 | ||
466 | static void | |
a47d5dac | 467 | xpc_indicate_partition_engaged_sn2(struct xpc_partition *part) |
33ba3c77 DN |
468 | { |
469 | unsigned long irq_flags; | |
c39838ce | 470 | struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + |
ee6665e3 | 471 | (XPC_ENGAGED_PARTITIONS_AMO_SN2 * |
c39838ce | 472 | sizeof(struct amo))); |
33ba3c77 DN |
473 | |
474 | local_irq_save(irq_flags); | |
475 | ||
c39838ce | 476 | /* set bit corresponding to our partid in remote partition's amo */ |
33ba3c77 | 477 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, |
04de7418 DN |
478 | BIT(sn_partition_id)); |
479 | ||
33ba3c77 DN |
480 | /* |
481 | * We must always use the nofault function regardless of whether we | |
482 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
483 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 484 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
485 | */ |
486 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
487 | variable), | |
488 | xp_nofault_PIOR_target)); | |
489 | ||
490 | local_irq_restore(irq_flags); | |
491 | } | |
492 | ||
493 | static void | |
a47d5dac | 494 | xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part) |
33ba3c77 | 495 | { |
a47d5dac | 496 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
33ba3c77 | 497 | unsigned long irq_flags; |
c39838ce | 498 | struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + |
ee6665e3 | 499 | (XPC_ENGAGED_PARTITIONS_AMO_SN2 * |
c39838ce | 500 | sizeof(struct amo))); |
33ba3c77 DN |
501 | |
502 | local_irq_save(irq_flags); | |
503 | ||
c39838ce | 504 | /* clear bit corresponding to our partid in remote partition's amo */ |
33ba3c77 | 505 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
04de7418 DN |
506 | ~BIT(sn_partition_id)); |
507 | ||
33ba3c77 DN |
508 | /* |
509 | * We must always use the nofault function regardless of whether we | |
510 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
511 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 512 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
513 | */ |
514 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
515 | variable), | |
516 | xp_nofault_PIOR_target)); | |
517 | ||
518 | local_irq_restore(irq_flags); | |
33ba3c77 | 519 | |
33ba3c77 | 520 | /* |
a47d5dac | 521 | * Send activate IRQ to get other side to see that we've cleared our |
c39838ce | 522 | * bit in their engaged partitions amo. |
33ba3c77 | 523 | */ |
7fb5e59d | 524 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
525 | cnodeid_to_nasid(0), |
526 | part_sn2->activate_IRQ_nasid, | |
527 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 DN |
528 | } |
529 | ||
5b8669df DN |
530 | static void |
531 | xpc_assume_partition_disengaged_sn2(short partid) | |
532 | { | |
533 | struct amo *amo = xpc_vars_sn2->amos_page + | |
534 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
535 | ||
536 | /* clear bit(s) based on partid mask in our partition's amo */ | |
537 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, | |
538 | ~BIT(partid)); | |
539 | } | |
540 | ||
a47d5dac DN |
541 | static int |
542 | xpc_partition_engaged_sn2(short partid) | |
33ba3c77 | 543 | { |
8e85c23e DN |
544 | struct amo *amo = xpc_vars_sn2->amos_page + |
545 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 546 | |
c39838ce | 547 | /* our partition's amo variable ANDed with partid mask */ |
33ba3c77 | 548 | return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & |
04de7418 | 549 | BIT(partid)) != 0; |
33ba3c77 DN |
550 | } |
551 | ||
a47d5dac DN |
552 | static int |
553 | xpc_any_partition_engaged_sn2(void) | |
33ba3c77 | 554 | { |
8e85c23e DN |
555 | struct amo *amo = xpc_vars_sn2->amos_page + |
556 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 557 | |
c39838ce | 558 | /* our partition's amo variable */ |
a47d5dac | 559 | return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0; |
33ba3c77 DN |
560 | } |
561 | ||
6e41017a DN |
562 | /* original protection values for each node */ |
563 | static u64 xpc_prot_vec_sn2[MAX_NUMNODES]; | |
564 | ||
565 | /* | |
c39838ce | 566 | * Change protections to allow amo operations on non-Shub 1.1 systems. |
6e41017a DN |
567 | */ |
568 | static enum xp_retval | |
c39838ce | 569 | xpc_allow_amo_ops_sn2(struct amo *amos_page) |
6e41017a | 570 | { |
6c1c325d | 571 | enum xp_retval ret = xpSuccess; |
6e41017a DN |
572 | |
573 | /* | |
574 | * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST | |
575 | * collides with memory operations. On those systems we call | |
c39838ce | 576 | * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead. |
6e41017a | 577 | */ |
6c1c325d DN |
578 | if (!enable_shub_wars_1_1()) |
579 | ret = xp_expand_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE); | |
580 | ||
581 | return ret; | |
6e41017a DN |
582 | } |
583 | ||
584 | /* | |
c39838ce | 585 | * Change protections to allow amo operations on Shub 1.1 systems. |
6e41017a DN |
586 | */ |
587 | static void | |
c39838ce | 588 | xpc_allow_amo_ops_shub_wars_1_1_sn2(void) |
6e41017a DN |
589 | { |
590 | int node; | |
591 | int nasid; | |
592 | ||
593 | if (!enable_shub_wars_1_1()) | |
594 | return; | |
595 | ||
596 | for_each_online_node(node) { | |
597 | nasid = cnodeid_to_nasid(node); | |
598 | /* save current protection values */ | |
599 | xpc_prot_vec_sn2[node] = | |
600 | (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid, | |
601 | SH1_MD_DQLP_MMR_DIR_PRIVEC0)); | |
602 | /* open up everything */ | |
603 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, | |
604 | SH1_MD_DQLP_MMR_DIR_PRIVEC0), | |
605 | -1UL); | |
606 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, | |
607 | SH1_MD_DQRP_MMR_DIR_PRIVEC0), | |
608 | -1UL); | |
609 | } | |
610 | } | |
611 | ||
261f3b49 | 612 | static enum xp_retval |
a812dcc3 | 613 | xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa, |
261f3b49 DN |
614 | size_t *len) |
615 | { | |
616 | s64 status; | |
617 | enum xp_retval ret; | |
618 | ||
f509e34a RD |
619 | status = sn_partition_reserved_page_pa((u64)buf, cookie, |
620 | (u64 *)rp_pa, (u64 *)len); | |
261f3b49 DN |
621 | if (status == SALRET_OK) |
622 | ret = xpSuccess; | |
623 | else if (status == SALRET_MORE_PASSES) | |
624 | ret = xpNeedMoreInfo; | |
625 | else | |
626 | ret = xpSalError; | |
627 | ||
628 | return ret; | |
629 | } | |
630 | ||
631 | ||
5b8669df | 632 | static int |
a7665b0a | 633 | xpc_setup_rsvd_page_sn2(struct xpc_rsvd_page *rp) |
94bd2708 | 634 | { |
c39838ce | 635 | struct amo *amos_page; |
94bd2708 DN |
636 | int i; |
637 | int ret; | |
638 | ||
8e85c23e | 639 | xpc_vars_sn2 = XPC_RP_VARS(rp); |
94bd2708 | 640 | |
a374c57b | 641 | rp->sn.sn2.vars_pa = xp_pa(xpc_vars_sn2); |
94bd2708 | 642 | |
e17d416b | 643 | /* vars_part array follows immediately after vars */ |
8e85c23e DN |
644 | xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) + |
645 | XPC_RP_VARS_SIZE); | |
e17d416b | 646 | |
94bd2708 | 647 | /* |
8e85c23e DN |
648 | * Before clearing xpc_vars_sn2, see if a page of amos had been |
649 | * previously allocated. If not we'll need to allocate one and set | |
650 | * permissions so that cross-partition amos are allowed. | |
94bd2708 | 651 | * |
c39838ce | 652 | * The allocated amo page needs MCA reporting to remain disabled after |
94bd2708 | 653 | * XPC has unloaded. To make this work, we keep a copy of the pointer |
8e85c23e | 654 | * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure, |
94bd2708 | 655 | * which is pointed to by the reserved page, and re-use that saved copy |
c39838ce | 656 | * on subsequent loads of XPC. This amo page is never freed, and its |
94bd2708 DN |
657 | * memory protections are never restricted. |
658 | */ | |
8e85c23e | 659 | amos_page = xpc_vars_sn2->amos_page; |
94bd2708 | 660 | if (amos_page == NULL) { |
c39838ce | 661 | amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1)); |
94bd2708 | 662 | if (amos_page == NULL) { |
c39838ce | 663 | dev_err(xpc_part, "can't allocate page of amos\n"); |
5b8669df | 664 | return -ENOMEM; |
94bd2708 DN |
665 | } |
666 | ||
667 | /* | |
c39838ce DN |
668 | * Open up amo-R/W to cpu. This is done on Shub 1.1 systems |
669 | * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called. | |
94bd2708 | 670 | */ |
c39838ce | 671 | ret = xpc_allow_amo_ops_sn2(amos_page); |
6e41017a | 672 | if (ret != xpSuccess) { |
c39838ce | 673 | dev_err(xpc_part, "can't allow amo operations\n"); |
6e41017a DN |
674 | uncached_free_page(__IA64_UNCACHED_OFFSET | |
675 | TO_PHYS((u64)amos_page), 1); | |
5b8669df | 676 | return -EPERM; |
94bd2708 DN |
677 | } |
678 | } | |
679 | ||
8e85c23e DN |
680 | /* clear xpc_vars_sn2 */ |
681 | memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2)); | |
94bd2708 | 682 | |
8e85c23e DN |
683 | xpc_vars_sn2->version = XPC_V_VERSION; |
684 | xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0); | |
685 | xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0); | |
a812dcc3 | 686 | xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2); |
8e85c23e DN |
687 | xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page); |
688 | xpc_vars_sn2->amos_page = amos_page; /* save for next load of XPC */ | |
94bd2708 | 689 | |
8e85c23e DN |
690 | /* clear xpc_vars_part_sn2 */ |
691 | memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) * | |
261f3b49 | 692 | XP_MAX_NPARTITIONS_SN2); |
94bd2708 | 693 | |
c39838ce | 694 | /* initialize the activate IRQ related amo variables */ |
04de7418 | 695 | for (i = 0; i < xpc_nasid_mask_nlongs; i++) |
ee6665e3 | 696 | (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i); |
94bd2708 | 697 | |
c39838ce | 698 | /* initialize the engaged remote partitions related amo variables */ |
ee6665e3 DN |
699 | (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2); |
700 | (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2); | |
33ba3c77 | 701 | |
5b8669df | 702 | return 0; |
33ba3c77 DN |
703 | } |
704 | ||
a374c57b RH |
705 | static int |
706 | xpc_hb_allowed_sn2(short partid, void *heartbeating_to_mask) | |
707 | { | |
708 | return test_bit(partid, heartbeating_to_mask); | |
709 | } | |
710 | ||
711 | static void | |
712 | xpc_allow_hb_sn2(short partid) | |
713 | { | |
714 | DBUG_ON(xpc_vars_sn2 == NULL); | |
715 | set_bit(partid, xpc_vars_sn2->heartbeating_to_mask); | |
716 | } | |
717 | ||
718 | static void | |
719 | xpc_disallow_hb_sn2(short partid) | |
720 | { | |
721 | DBUG_ON(xpc_vars_sn2 == NULL); | |
722 | clear_bit(partid, xpc_vars_sn2->heartbeating_to_mask); | |
723 | } | |
724 | ||
725 | static void | |
726 | xpc_disallow_all_hbs_sn2(void) | |
727 | { | |
728 | DBUG_ON(xpc_vars_sn2 == NULL); | |
729 | bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, xp_max_npartitions); | |
730 | } | |
731 | ||
33ba3c77 DN |
732 | static void |
733 | xpc_increment_heartbeat_sn2(void) | |
734 | { | |
8e85c23e | 735 | xpc_vars_sn2->heartbeat++; |
33ba3c77 DN |
736 | } |
737 | ||
738 | static void | |
739 | xpc_offline_heartbeat_sn2(void) | |
740 | { | |
741 | xpc_increment_heartbeat_sn2(); | |
8e85c23e | 742 | xpc_vars_sn2->heartbeat_offline = 1; |
33ba3c77 DN |
743 | } |
744 | ||
745 | static void | |
746 | xpc_online_heartbeat_sn2(void) | |
747 | { | |
748 | xpc_increment_heartbeat_sn2(); | |
8e85c23e | 749 | xpc_vars_sn2->heartbeat_offline = 0; |
33ba3c77 DN |
750 | } |
751 | ||
752 | static void | |
753 | xpc_heartbeat_init_sn2(void) | |
754 | { | |
8e85c23e | 755 | DBUG_ON(xpc_vars_sn2 == NULL); |
33ba3c77 | 756 | |
8e85c23e | 757 | bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2); |
33ba3c77 DN |
758 | xpc_online_heartbeat_sn2(); |
759 | } | |
760 | ||
761 | static void | |
762 | xpc_heartbeat_exit_sn2(void) | |
763 | { | |
764 | xpc_offline_heartbeat_sn2(); | |
765 | } | |
766 | ||
61deb86e DN |
767 | static enum xp_retval |
768 | xpc_get_remote_heartbeat_sn2(struct xpc_partition *part) | |
33ba3c77 DN |
769 | { |
770 | struct xpc_vars_sn2 *remote_vars; | |
33ba3c77 DN |
771 | enum xp_retval ret; |
772 | ||
ee6665e3 | 773 | remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; |
33ba3c77 | 774 | |
61deb86e DN |
775 | /* pull the remote vars structure that contains the heartbeat */ |
776 | ret = xp_remote_memcpy(xp_pa(remote_vars), | |
777 | part->sn.sn2.remote_vars_pa, | |
778 | XPC_RP_VARS_SIZE); | |
779 | if (ret != xpSuccess) | |
780 | return ret; | |
33ba3c77 | 781 | |
f509e34a RD |
782 | dev_dbg(xpc_part, "partid=%d, heartbeat=%lld, last_heartbeat=%lld, " |
783 | "heartbeat_offline=%lld, HB_mask[0]=0x%lx\n", XPC_PARTID(part), | |
61deb86e DN |
784 | remote_vars->heartbeat, part->last_heartbeat, |
785 | remote_vars->heartbeat_offline, | |
786 | remote_vars->heartbeating_to_mask[0]); | |
787 | ||
788 | if ((remote_vars->heartbeat == part->last_heartbeat && | |
a374c57b RH |
789 | !remote_vars->heartbeat_offline) || |
790 | !xpc_hb_allowed_sn2(sn_partition_id, | |
791 | remote_vars->heartbeating_to_mask)) { | |
61deb86e DN |
792 | ret = xpNoHeartbeat; |
793 | } else { | |
33ba3c77 DN |
794 | part->last_heartbeat = remote_vars->heartbeat; |
795 | } | |
61deb86e DN |
796 | |
797 | return ret; | |
33ba3c77 DN |
798 | } |
799 | ||
800 | /* | |
801 | * Get a copy of the remote partition's XPC variables from the reserved page. | |
802 | * | |
803 | * remote_vars points to a buffer that is cacheline aligned for BTE copies and | |
804 | * assumed to be of size XPC_RP_VARS_SIZE. | |
805 | */ | |
806 | static enum xp_retval | |
a812dcc3 DN |
807 | xpc_get_remote_vars_sn2(unsigned long remote_vars_pa, |
808 | struct xpc_vars_sn2 *remote_vars) | |
33ba3c77 DN |
809 | { |
810 | enum xp_retval ret; | |
811 | ||
812 | if (remote_vars_pa == 0) | |
813 | return xpVarsNotSet; | |
814 | ||
815 | /* pull over the cross partition variables */ | |
a812dcc3 | 816 | ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa, |
33ba3c77 DN |
817 | XPC_RP_VARS_SIZE); |
818 | if (ret != xpSuccess) | |
819 | return ret; | |
820 | ||
821 | if (XPC_VERSION_MAJOR(remote_vars->version) != | |
822 | XPC_VERSION_MAJOR(XPC_V_VERSION)) { | |
823 | return xpBadVersion; | |
824 | } | |
94bd2708 DN |
825 | |
826 | return xpSuccess; | |
827 | } | |
828 | ||
33ba3c77 | 829 | static void |
a47d5dac | 830 | xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp, |
a812dcc3 | 831 | unsigned long remote_rp_pa, int nasid) |
33ba3c77 | 832 | { |
7fb5e59d | 833 | xpc_send_local_activate_IRQ_sn2(nasid); |
a47d5dac DN |
834 | } |
835 | ||
836 | static void | |
837 | xpc_request_partition_reactivation_sn2(struct xpc_partition *part) | |
838 | { | |
7fb5e59d | 839 | xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid); |
a47d5dac DN |
840 | } |
841 | ||
842 | static void | |
843 | xpc_request_partition_deactivation_sn2(struct xpc_partition *part) | |
844 | { | |
845 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; | |
846 | unsigned long irq_flags; | |
c39838ce | 847 | struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + |
ee6665e3 | 848 | (XPC_DEACTIVATE_REQUEST_AMO_SN2 * |
c39838ce | 849 | sizeof(struct amo))); |
a47d5dac DN |
850 | |
851 | local_irq_save(irq_flags); | |
852 | ||
c39838ce | 853 | /* set bit corresponding to our partid in remote partition's amo */ |
a47d5dac | 854 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, |
04de7418 DN |
855 | BIT(sn_partition_id)); |
856 | ||
a47d5dac DN |
857 | /* |
858 | * We must always use the nofault function regardless of whether we | |
859 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
860 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 861 | * keep sending IRQs and amos to it until the heartbeat times out. |
a47d5dac DN |
862 | */ |
863 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
864 | variable), | |
865 | xp_nofault_PIOR_target)); | |
866 | ||
867 | local_irq_restore(irq_flags); | |
868 | ||
869 | /* | |
870 | * Send activate IRQ to get other side to see that we've set our | |
c39838ce | 871 | * bit in their deactivate request amo. |
a47d5dac | 872 | */ |
7fb5e59d | 873 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
874 | cnodeid_to_nasid(0), |
875 | part_sn2->activate_IRQ_nasid, | |
876 | part_sn2->activate_IRQ_phys_cpuid); | |
877 | } | |
878 | ||
879 | static void | |
880 | xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part) | |
881 | { | |
882 | unsigned long irq_flags; | |
c39838ce | 883 | struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + |
ee6665e3 | 884 | (XPC_DEACTIVATE_REQUEST_AMO_SN2 * |
c39838ce | 885 | sizeof(struct amo))); |
a47d5dac DN |
886 | |
887 | local_irq_save(irq_flags); | |
888 | ||
c39838ce | 889 | /* clear bit corresponding to our partid in remote partition's amo */ |
a47d5dac | 890 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
04de7418 DN |
891 | ~BIT(sn_partition_id)); |
892 | ||
a47d5dac DN |
893 | /* |
894 | * We must always use the nofault function regardless of whether we | |
895 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
896 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 897 | * keep sending IRQs and amos to it until the heartbeat times out. |
a47d5dac DN |
898 | */ |
899 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
900 | variable), | |
901 | xp_nofault_PIOR_target)); | |
902 | ||
903 | local_irq_restore(irq_flags); | |
904 | } | |
905 | ||
906 | static int | |
907 | xpc_partition_deactivation_requested_sn2(short partid) | |
908 | { | |
8e85c23e DN |
909 | struct amo *amo = xpc_vars_sn2->amos_page + |
910 | XPC_DEACTIVATE_REQUEST_AMO_SN2; | |
a47d5dac | 911 | |
c39838ce | 912 | /* our partition's amo variable ANDed with partid mask */ |
a47d5dac | 913 | return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & |
04de7418 | 914 | BIT(partid)) != 0; |
33ba3c77 DN |
915 | } |
916 | ||
917 | /* | |
918 | * Update the remote partition's info. | |
919 | */ | |
920 | static void | |
921 | xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version, | |
81fe7883 | 922 | unsigned long *remote_rp_ts_jiffies, |
a812dcc3 DN |
923 | unsigned long remote_rp_pa, |
924 | unsigned long remote_vars_pa, | |
33ba3c77 DN |
925 | struct xpc_vars_sn2 *remote_vars) |
926 | { | |
a47d5dac DN |
927 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
928 | ||
33ba3c77 DN |
929 | part->remote_rp_version = remote_rp_version; |
930 | dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n", | |
931 | part->remote_rp_version); | |
932 | ||
81fe7883 DN |
933 | part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies; |
934 | dev_dbg(xpc_part, " remote_rp_ts_jiffies = 0x%016lx\n", | |
935 | part->remote_rp_ts_jiffies); | |
33ba3c77 DN |
936 | |
937 | part->remote_rp_pa = remote_rp_pa; | |
938 | dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa); | |
939 | ||
a47d5dac | 940 | part_sn2->remote_vars_pa = remote_vars_pa; |
33ba3c77 | 941 | dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n", |
a47d5dac | 942 | part_sn2->remote_vars_pa); |
33ba3c77 | 943 | |
158bc69e | 944 | part->last_heartbeat = remote_vars->heartbeat - 1; |
f509e34a | 945 | dev_dbg(xpc_part, " last_heartbeat = 0x%016llx\n", |
33ba3c77 DN |
946 | part->last_heartbeat); |
947 | ||
a47d5dac | 948 | part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa; |
33ba3c77 | 949 | dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n", |
a47d5dac | 950 | part_sn2->remote_vars_part_pa); |
33ba3c77 | 951 | |
a47d5dac DN |
952 | part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid; |
953 | dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n", | |
954 | part_sn2->activate_IRQ_nasid); | |
33ba3c77 | 955 | |
a47d5dac DN |
956 | part_sn2->activate_IRQ_phys_cpuid = |
957 | remote_vars->activate_IRQ_phys_cpuid; | |
958 | dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n", | |
959 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 | 960 | |
a47d5dac | 961 | part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa; |
33ba3c77 | 962 | dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n", |
a47d5dac | 963 | part_sn2->remote_amos_page_pa); |
33ba3c77 | 964 | |
a47d5dac | 965 | part_sn2->remote_vars_version = remote_vars->version; |
33ba3c77 | 966 | dev_dbg(xpc_part, " remote_vars_version = 0x%x\n", |
a47d5dac | 967 | part_sn2->remote_vars_version); |
33ba3c77 DN |
968 | } |
969 | ||
970 | /* | |
7fb5e59d DN |
971 | * Prior code has determined the nasid which generated a activate IRQ. |
972 | * Inspect that nasid to determine if its partition needs to be activated | |
973 | * or deactivated. | |
33ba3c77 | 974 | * |
7fb5e59d | 975 | * A partition is considered "awaiting activation" if our partition |
33ba3c77 DN |
976 | * flags indicate it is not active and it has a heartbeat. A |
977 | * partition is considered "awaiting deactivation" if our partition | |
978 | * flags indicate it is active but it has no heartbeat or it is not | |
979 | * sending its heartbeat to us. | |
980 | * | |
981 | * To determine the heartbeat, the remote nasid must have a properly | |
982 | * initialized reserved page. | |
983 | */ | |
984 | static void | |
6e41017a | 985 | xpc_identify_activate_IRQ_req_sn2(int nasid) |
33ba3c77 DN |
986 | { |
987 | struct xpc_rsvd_page *remote_rp; | |
988 | struct xpc_vars_sn2 *remote_vars; | |
a812dcc3 DN |
989 | unsigned long remote_rp_pa; |
990 | unsigned long remote_vars_pa; | |
33ba3c77 DN |
991 | int remote_rp_version; |
992 | int reactivate = 0; | |
81fe7883 | 993 | unsigned long remote_rp_ts_jiffies = 0; |
33ba3c77 DN |
994 | short partid; |
995 | struct xpc_partition *part; | |
a47d5dac | 996 | struct xpc_partition_sn2 *part_sn2; |
33ba3c77 DN |
997 | enum xp_retval ret; |
998 | ||
999 | /* pull over the reserved page structure */ | |
1000 | ||
ee6665e3 | 1001 | remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
1002 | |
1003 | ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa); | |
1004 | if (ret != xpSuccess) { | |
1005 | dev_warn(xpc_part, "unable to get reserved page from nasid %d, " | |
1006 | "which sent interrupt, reason=%d\n", nasid, ret); | |
1007 | return; | |
1008 | } | |
1009 | ||
a374c57b | 1010 | remote_vars_pa = remote_rp->sn.sn2.vars_pa; |
33ba3c77 | 1011 | remote_rp_version = remote_rp->version; |
81fe7883 | 1012 | remote_rp_ts_jiffies = remote_rp->ts_jiffies; |
33ba3c77 DN |
1013 | |
1014 | partid = remote_rp->SAL_partid; | |
1015 | part = &xpc_partitions[partid]; | |
a47d5dac | 1016 | part_sn2 = &part->sn.sn2; |
33ba3c77 DN |
1017 | |
1018 | /* pull over the cross partition variables */ | |
1019 | ||
ee6665e3 | 1020 | remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
1021 | |
1022 | ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars); | |
1023 | if (ret != xpSuccess) { | |
33ba3c77 DN |
1024 | dev_warn(xpc_part, "unable to get XPC variables from nasid %d, " |
1025 | "which sent interrupt, reason=%d\n", nasid, ret); | |
1026 | ||
1027 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1028 | return; | |
1029 | } | |
1030 | ||
6e41017a | 1031 | part->activate_IRQ_rcvd++; |
33ba3c77 DN |
1032 | |
1033 | dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = " | |
f509e34a RD |
1034 | "%lld:0x%lx\n", (int)nasid, (int)partid, |
1035 | part->activate_IRQ_rcvd, | |
33ba3c77 DN |
1036 | remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]); |
1037 | ||
1038 | if (xpc_partition_disengaged(part) && | |
83469b55 | 1039 | part->act_state == XPC_P_AS_INACTIVE) { |
33ba3c77 DN |
1040 | |
1041 | xpc_update_partition_info_sn2(part, remote_rp_version, | |
81fe7883 DN |
1042 | &remote_rp_ts_jiffies, |
1043 | remote_rp_pa, remote_vars_pa, | |
1044 | remote_vars); | |
33ba3c77 | 1045 | |
a47d5dac DN |
1046 | if (xpc_partition_deactivation_requested_sn2(partid)) { |
1047 | /* | |
1048 | * Other side is waiting on us to deactivate even though | |
1049 | * we already have. | |
1050 | */ | |
1051 | return; | |
33ba3c77 DN |
1052 | } |
1053 | ||
1054 | xpc_activate_partition(part); | |
1055 | return; | |
1056 | } | |
1057 | ||
1058 | DBUG_ON(part->remote_rp_version == 0); | |
a47d5dac | 1059 | DBUG_ON(part_sn2->remote_vars_version == 0); |
33ba3c77 | 1060 | |
81fe7883 | 1061 | if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) { |
33ba3c77 | 1062 | |
a47d5dac | 1063 | /* the other side rebooted */ |
33ba3c77 | 1064 | |
a47d5dac DN |
1065 | DBUG_ON(xpc_partition_engaged_sn2(partid)); |
1066 | DBUG_ON(xpc_partition_deactivation_requested_sn2(partid)); | |
33ba3c77 DN |
1067 | |
1068 | xpc_update_partition_info_sn2(part, remote_rp_version, | |
81fe7883 DN |
1069 | &remote_rp_ts_jiffies, |
1070 | remote_rp_pa, remote_vars_pa, | |
1071 | remote_vars); | |
33ba3c77 | 1072 | reactivate = 1; |
33ba3c77 DN |
1073 | } |
1074 | ||
a47d5dac | 1075 | if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) { |
33ba3c77 DN |
1076 | /* still waiting on other side to disengage from us */ |
1077 | return; | |
1078 | } | |
1079 | ||
a47d5dac | 1080 | if (reactivate) |
33ba3c77 | 1081 | XPC_DEACTIVATE_PARTITION(part, xpReactivating); |
a47d5dac | 1082 | else if (xpc_partition_deactivation_requested_sn2(partid)) |
33ba3c77 | 1083 | XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown); |
33ba3c77 DN |
1084 | } |
1085 | ||
1086 | /* | |
c39838ce | 1087 | * Loop through the activation amo variables and process any bits |
33ba3c77 DN |
1088 | * which are set. Each bit indicates a nasid sending a partition |
1089 | * activation or deactivation request. | |
1090 | * | |
1091 | * Return #of IRQs detected. | |
1092 | */ | |
1093 | int | |
6e41017a | 1094 | xpc_identify_activate_IRQ_sender_sn2(void) |
33ba3c77 | 1095 | { |
04de7418 DN |
1096 | int l; |
1097 | int b; | |
1098 | unsigned long nasid_mask_long; | |
33ba3c77 DN |
1099 | u64 nasid; /* remote nasid */ |
1100 | int n_IRQs_detected = 0; | |
c39838ce | 1101 | struct amo *act_amos; |
33ba3c77 | 1102 | |
8e85c23e | 1103 | act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2; |
33ba3c77 | 1104 | |
04de7418 DN |
1105 | /* scan through activate amo variables looking for non-zero entries */ |
1106 | for (l = 0; l < xpc_nasid_mask_nlongs; l++) { | |
33ba3c77 DN |
1107 | |
1108 | if (xpc_exiting) | |
1109 | break; | |
1110 | ||
04de7418 DN |
1111 | nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]); |
1112 | ||
1113 | b = find_first_bit(&nasid_mask_long, BITS_PER_LONG); | |
1114 | if (b >= BITS_PER_LONG) { | |
1115 | /* no IRQs from nasids in this amo variable */ | |
33ba3c77 DN |
1116 | continue; |
1117 | } | |
1118 | ||
04de7418 DN |
1119 | dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l, |
1120 | nasid_mask_long); | |
33ba3c77 DN |
1121 | |
1122 | /* | |
1123 | * If this nasid has been added to the machine since | |
1124 | * our partition was reset, this will retain the | |
1125 | * remote nasid in our reserved pages machine mask. | |
1126 | * This is used in the event of module reload. | |
1127 | */ | |
04de7418 | 1128 | xpc_mach_nasids[l] |= nasid_mask_long; |
33ba3c77 DN |
1129 | |
1130 | /* locate the nasid(s) which sent interrupts */ | |
1131 | ||
04de7418 DN |
1132 | do { |
1133 | n_IRQs_detected++; | |
1134 | nasid = (l * BITS_PER_LONG + b) * 2; | |
f509e34a | 1135 | dev_dbg(xpc_part, "interrupt from nasid %lld\n", nasid); |
04de7418 DN |
1136 | xpc_identify_activate_IRQ_req_sn2(nasid); |
1137 | ||
1138 | b = find_next_bit(&nasid_mask_long, BITS_PER_LONG, | |
1139 | b + 1); | |
1140 | } while (b < BITS_PER_LONG); | |
33ba3c77 DN |
1141 | } |
1142 | return n_IRQs_detected; | |
1143 | } | |
1144 | ||
1145 | static void | |
5b8669df | 1146 | xpc_process_activate_IRQ_rcvd_sn2(void) |
33ba3c77 | 1147 | { |
5b8669df DN |
1148 | unsigned long irq_flags; |
1149 | int n_IRQs_expected; | |
33ba3c77 DN |
1150 | int n_IRQs_detected; |
1151 | ||
5b8669df DN |
1152 | spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); |
1153 | n_IRQs_expected = xpc_activate_IRQ_rcvd; | |
1154 | xpc_activate_IRQ_rcvd = 0; | |
1155 | spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
1156 | ||
6e41017a | 1157 | n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2(); |
33ba3c77 | 1158 | if (n_IRQs_detected < n_IRQs_expected) { |
c39838ce | 1159 | /* retry once to help avoid missing amo */ |
6e41017a | 1160 | (void)xpc_identify_activate_IRQ_sender_sn2(); |
33ba3c77 DN |
1161 | } |
1162 | } | |
1163 | ||
185c3a1b | 1164 | /* |
5b8669df | 1165 | * Setup the channel structures that are sn2 specific. |
e17d416b DN |
1166 | */ |
1167 | static enum xp_retval | |
a7665b0a | 1168 | xpc_setup_ch_structures_sn2(struct xpc_partition *part) |
e17d416b | 1169 | { |
a47d5dac | 1170 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
5b8669df | 1171 | struct xpc_channel_sn2 *ch_sn2; |
e17d416b DN |
1172 | enum xp_retval retval; |
1173 | int ret; | |
1174 | int cpuid; | |
1175 | int ch_number; | |
e17d416b DN |
1176 | struct timer_list *timer; |
1177 | short partid = XPC_PARTID(part); | |
1178 | ||
e17d416b DN |
1179 | /* allocate all the required GET/PUT values */ |
1180 | ||
185c3a1b | 1181 | part_sn2->local_GPs = |
5b8669df DN |
1182 | xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL, |
1183 | &part_sn2->local_GPs_base); | |
a47d5dac | 1184 | if (part_sn2->local_GPs == NULL) { |
e17d416b DN |
1185 | dev_err(xpc_chan, "can't get memory for local get/put " |
1186 | "values\n"); | |
5b8669df | 1187 | return xpNoMemory; |
e17d416b DN |
1188 | } |
1189 | ||
185c3a1b | 1190 | part_sn2->remote_GPs = |
5b8669df DN |
1191 | xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL, |
1192 | &part_sn2->remote_GPs_base); | |
a47d5dac | 1193 | if (part_sn2->remote_GPs == NULL) { |
e17d416b DN |
1194 | dev_err(xpc_chan, "can't get memory for remote get/put " |
1195 | "values\n"); | |
1196 | retval = xpNoMemory; | |
5b8669df | 1197 | goto out_1; |
e17d416b DN |
1198 | } |
1199 | ||
a47d5dac | 1200 | part_sn2->remote_GPs_pa = 0; |
e17d416b DN |
1201 | |
1202 | /* allocate all the required open and close args */ | |
1203 | ||
5b8669df DN |
1204 | part_sn2->local_openclose_args = |
1205 | xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE, | |
1206 | GFP_KERNEL, &part_sn2-> | |
1207 | local_openclose_args_base); | |
1208 | if (part_sn2->local_openclose_args == NULL) { | |
e17d416b DN |
1209 | dev_err(xpc_chan, "can't get memory for local connect args\n"); |
1210 | retval = xpNoMemory; | |
5b8669df | 1211 | goto out_2; |
e17d416b DN |
1212 | } |
1213 | ||
a47d5dac | 1214 | part_sn2->remote_openclose_args_pa = 0; |
e17d416b | 1215 | |
7fb5e59d | 1216 | part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid); |
e17d416b | 1217 | |
7fb5e59d DN |
1218 | part_sn2->notify_IRQ_nasid = 0; |
1219 | part_sn2->notify_IRQ_phys_cpuid = 0; | |
1220 | part_sn2->remote_chctl_amo_va = NULL; | |
e17d416b | 1221 | |
7fb5e59d | 1222 | sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid); |
a47d5dac | 1223 | ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2, |
7fb5e59d | 1224 | IRQF_SHARED, part_sn2->notify_IRQ_owner, |
a47d5dac | 1225 | (void *)(u64)partid); |
e17d416b DN |
1226 | if (ret != 0) { |
1227 | dev_err(xpc_chan, "can't register NOTIFY IRQ handler, " | |
1228 | "errno=%d\n", -ret); | |
1229 | retval = xpLackOfResources; | |
5b8669df | 1230 | goto out_3; |
e17d416b DN |
1231 | } |
1232 | ||
7fb5e59d | 1233 | /* Setup a timer to check for dropped notify IRQs */ |
a47d5dac | 1234 | timer = &part_sn2->dropped_notify_IRQ_timer; |
e17d416b | 1235 | init_timer(timer); |
a47d5dac | 1236 | timer->function = |
7fb5e59d | 1237 | (void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2; |
e17d416b | 1238 | timer->data = (unsigned long)part; |
7fb5e59d | 1239 | timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; |
e17d416b DN |
1240 | add_timer(timer); |
1241 | ||
e17d416b | 1242 | for (ch_number = 0; ch_number < part->nchannels; ch_number++) { |
5b8669df | 1243 | ch_sn2 = &part->channels[ch_number].sn.sn2; |
e17d416b | 1244 | |
5b8669df DN |
1245 | ch_sn2->local_GP = &part_sn2->local_GPs[ch_number]; |
1246 | ch_sn2->local_openclose_args = | |
1247 | &part_sn2->local_openclose_args[ch_number]; | |
e17d416b | 1248 | |
5b8669df | 1249 | mutex_init(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1250 | } |
1251 | ||
e17d416b DN |
1252 | /* |
1253 | * Setup the per partition specific variables required by the | |
1254 | * remote partition to establish channel connections with us. | |
1255 | * | |
1256 | * The setting of the magic # indicates that these per partition | |
1257 | * specific variables are ready to be used. | |
1258 | */ | |
a812dcc3 | 1259 | xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs); |
8e85c23e | 1260 | xpc_vars_part_sn2[partid].openclose_args_pa = |
5b8669df | 1261 | xp_pa(part_sn2->local_openclose_args); |
8e85c23e | 1262 | xpc_vars_part_sn2[partid].chctl_amo_pa = |
a812dcc3 | 1263 | xp_pa(part_sn2->local_chctl_amo_va); |
e17d416b | 1264 | cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */ |
8e85c23e DN |
1265 | xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid); |
1266 | xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid = | |
1267 | cpu_physical_id(cpuid); | |
1268 | xpc_vars_part_sn2[partid].nchannels = part->nchannels; | |
5b8669df | 1269 | xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1_SN2; |
e17d416b DN |
1270 | |
1271 | return xpSuccess; | |
1272 | ||
5b8669df | 1273 | /* setup of ch structures failed */ |
e17d416b | 1274 | out_3: |
5b8669df DN |
1275 | kfree(part_sn2->local_openclose_args_base); |
1276 | part_sn2->local_openclose_args = NULL; | |
1277 | out_2: | |
a47d5dac DN |
1278 | kfree(part_sn2->remote_GPs_base); |
1279 | part_sn2->remote_GPs = NULL; | |
5b8669df | 1280 | out_1: |
a47d5dac DN |
1281 | kfree(part_sn2->local_GPs_base); |
1282 | part_sn2->local_GPs = NULL; | |
e17d416b DN |
1283 | return retval; |
1284 | } | |
1285 | ||
1286 | /* | |
5b8669df | 1287 | * Teardown the channel structures that are sn2 specific. |
e17d416b DN |
1288 | */ |
1289 | static void | |
a7665b0a | 1290 | xpc_teardown_ch_structures_sn2(struct xpc_partition *part) |
e17d416b | 1291 | { |
a47d5dac | 1292 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1293 | short partid = XPC_PARTID(part); |
1294 | ||
1295 | /* | |
5b8669df DN |
1296 | * Indicate that the variables specific to the remote partition are no |
1297 | * longer available for its use. | |
e17d416b | 1298 | */ |
8e85c23e | 1299 | xpc_vars_part_sn2[partid].magic = 0; |
e17d416b | 1300 | |
e17d416b | 1301 | /* in case we've still got outstanding timers registered... */ |
a47d5dac | 1302 | del_timer_sync(&part_sn2->dropped_notify_IRQ_timer); |
5b8669df | 1303 | free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid); |
e17d416b | 1304 | |
5b8669df DN |
1305 | kfree(part_sn2->local_openclose_args_base); |
1306 | part_sn2->local_openclose_args = NULL; | |
a47d5dac DN |
1307 | kfree(part_sn2->remote_GPs_base); |
1308 | part_sn2->remote_GPs = NULL; | |
1309 | kfree(part_sn2->local_GPs_base); | |
1310 | part_sn2->local_GPs = NULL; | |
7fb5e59d | 1311 | part_sn2->local_chctl_amo_va = NULL; |
e17d416b DN |
1312 | } |
1313 | ||
1314 | /* | |
1315 | * Create a wrapper that hides the underlying mechanism for pulling a cacheline | |
1316 | * (or multiple cachelines) from a remote partition. | |
1317 | * | |
a812dcc3 | 1318 | * src_pa must be a cacheline aligned physical address on the remote partition. |
e17d416b DN |
1319 | * dst must be a cacheline aligned virtual address on this partition. |
1320 | * cnt must be cacheline sized | |
1321 | */ | |
ea57f80c | 1322 | /* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */ |
e17d416b DN |
1323 | static enum xp_retval |
1324 | xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst, | |
a812dcc3 | 1325 | const unsigned long src_pa, size_t cnt) |
e17d416b DN |
1326 | { |
1327 | enum xp_retval ret; | |
1328 | ||
a812dcc3 DN |
1329 | DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa)); |
1330 | DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst)); | |
e17d416b DN |
1331 | DBUG_ON(cnt != L1_CACHE_ALIGN(cnt)); |
1332 | ||
83469b55 | 1333 | if (part->act_state == XPC_P_AS_DEACTIVATING) |
e17d416b DN |
1334 | return part->reason; |
1335 | ||
a812dcc3 | 1336 | ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt); |
e17d416b DN |
1337 | if (ret != xpSuccess) { |
1338 | dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed," | |
1339 | " ret=%d\n", XPC_PARTID(part), ret); | |
1340 | } | |
1341 | return ret; | |
1342 | } | |
1343 | ||
1344 | /* | |
1345 | * Pull the remote per partition specific variables from the specified | |
1346 | * partition. | |
1347 | */ | |
1348 | static enum xp_retval | |
1349 | xpc_pull_remote_vars_part_sn2(struct xpc_partition *part) | |
1350 | { | |
a47d5dac | 1351 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1352 | u8 buffer[L1_CACHE_BYTES * 2]; |
1353 | struct xpc_vars_part_sn2 *pulled_entry_cacheline = | |
1354 | (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer); | |
1355 | struct xpc_vars_part_sn2 *pulled_entry; | |
a812dcc3 DN |
1356 | unsigned long remote_entry_cacheline_pa; |
1357 | unsigned long remote_entry_pa; | |
e17d416b DN |
1358 | short partid = XPC_PARTID(part); |
1359 | enum xp_retval ret; | |
1360 | ||
1361 | /* pull the cacheline that contains the variables we're interested in */ | |
1362 | ||
a47d5dac DN |
1363 | DBUG_ON(part_sn2->remote_vars_part_pa != |
1364 | L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa)); | |
e17d416b DN |
1365 | DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2); |
1366 | ||
a47d5dac | 1367 | remote_entry_pa = part_sn2->remote_vars_part_pa + |
e17d416b DN |
1368 | sn_partition_id * sizeof(struct xpc_vars_part_sn2); |
1369 | ||
1370 | remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1)); | |
1371 | ||
1372 | pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline | |
1373 | + (remote_entry_pa & | |
1374 | (L1_CACHE_BYTES - 1))); | |
1375 | ||
1376 | ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline, | |
a812dcc3 | 1377 | remote_entry_cacheline_pa, |
e17d416b DN |
1378 | L1_CACHE_BYTES); |
1379 | if (ret != xpSuccess) { | |
1380 | dev_dbg(xpc_chan, "failed to pull XPC vars_part from " | |
1381 | "partition %d, ret=%d\n", partid, ret); | |
1382 | return ret; | |
1383 | } | |
1384 | ||
1385 | /* see if they've been set up yet */ | |
1386 | ||
5b8669df DN |
1387 | if (pulled_entry->magic != XPC_VP_MAGIC1_SN2 && |
1388 | pulled_entry->magic != XPC_VP_MAGIC2_SN2) { | |
e17d416b DN |
1389 | |
1390 | if (pulled_entry->magic != 0) { | |
1391 | dev_dbg(xpc_chan, "partition %d's XPC vars_part for " | |
f509e34a | 1392 | "partition %d has bad magic value (=0x%llx)\n", |
e17d416b DN |
1393 | partid, sn_partition_id, pulled_entry->magic); |
1394 | return xpBadMagic; | |
1395 | } | |
1396 | ||
1397 | /* they've not been initialized yet */ | |
1398 | return xpRetry; | |
1399 | } | |
1400 | ||
5b8669df | 1401 | if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1_SN2) { |
e17d416b DN |
1402 | |
1403 | /* validate the variables */ | |
1404 | ||
1405 | if (pulled_entry->GPs_pa == 0 || | |
1406 | pulled_entry->openclose_args_pa == 0 || | |
7fb5e59d | 1407 | pulled_entry->chctl_amo_pa == 0) { |
e17d416b DN |
1408 | |
1409 | dev_err(xpc_chan, "partition %d's XPC vars_part for " | |
1410 | "partition %d are not valid\n", partid, | |
1411 | sn_partition_id); | |
1412 | return xpInvalidAddress; | |
1413 | } | |
1414 | ||
1415 | /* the variables we imported look to be valid */ | |
1416 | ||
a47d5dac DN |
1417 | part_sn2->remote_GPs_pa = pulled_entry->GPs_pa; |
1418 | part_sn2->remote_openclose_args_pa = | |
e17d416b | 1419 | pulled_entry->openclose_args_pa; |
7fb5e59d | 1420 | part_sn2->remote_chctl_amo_va = |
c39838ce | 1421 | (struct amo *)__va(pulled_entry->chctl_amo_pa); |
7fb5e59d DN |
1422 | part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid; |
1423 | part_sn2->notify_IRQ_phys_cpuid = | |
1424 | pulled_entry->notify_IRQ_phys_cpuid; | |
e17d416b DN |
1425 | |
1426 | if (part->nchannels > pulled_entry->nchannels) | |
1427 | part->nchannels = pulled_entry->nchannels; | |
1428 | ||
1429 | /* let the other side know that we've pulled their variables */ | |
1430 | ||
5b8669df | 1431 | xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2_SN2; |
e17d416b DN |
1432 | } |
1433 | ||
5b8669df | 1434 | if (pulled_entry->magic == XPC_VP_MAGIC1_SN2) |
e17d416b DN |
1435 | return xpRetry; |
1436 | ||
1437 | return xpSuccess; | |
1438 | } | |
1439 | ||
1440 | /* | |
1441 | * Establish first contact with the remote partititon. This involves pulling | |
1442 | * the XPC per partition variables from the remote partition and waiting for | |
1443 | * the remote partition to pull ours. | |
1444 | */ | |
1445 | static enum xp_retval | |
1446 | xpc_make_first_contact_sn2(struct xpc_partition *part) | |
1447 | { | |
a47d5dac | 1448 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1449 | enum xp_retval ret; |
1450 | ||
33ba3c77 | 1451 | /* |
c39838ce | 1452 | * Register the remote partition's amos with SAL so it can handle |
33ba3c77 DN |
1453 | * and cleanup errors within that address range should the remote |
1454 | * partition go down. We don't unregister this range because it is | |
1455 | * difficult to tell when outstanding writes to the remote partition | |
1456 | * are finished and thus when it is safe to unregister. This should | |
1457 | * not result in wasted space in the SAL xp_addr_region table because | |
1458 | * we should get the same page for remote_amos_page_pa after module | |
1459 | * reloads and system reboots. | |
1460 | */ | |
a47d5dac | 1461 | if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa, |
33ba3c77 DN |
1462 | PAGE_SIZE, 1) < 0) { |
1463 | dev_warn(xpc_part, "xpc_activating(%d) failed to register " | |
1464 | "xp_addr region\n", XPC_PARTID(part)); | |
1465 | ||
1466 | ret = xpPhysAddrRegFailed; | |
1467 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1468 | return ret; | |
1469 | } | |
1470 | ||
a47d5dac DN |
1471 | /* |
1472 | * Send activate IRQ to get other side to activate if they've not | |
1473 | * already begun to do so. | |
1474 | */ | |
7fb5e59d | 1475 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
1476 | cnodeid_to_nasid(0), |
1477 | part_sn2->activate_IRQ_nasid, | |
1478 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 | 1479 | |
e17d416b DN |
1480 | while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) { |
1481 | if (ret != xpRetry) { | |
1482 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1483 | return ret; | |
1484 | } | |
1485 | ||
1486 | dev_dbg(xpc_part, "waiting to make first contact with " | |
1487 | "partition %d\n", XPC_PARTID(part)); | |
1488 | ||
1489 | /* wait a 1/4 of a second or so */ | |
1490 | (void)msleep_interruptible(250); | |
1491 | ||
83469b55 | 1492 | if (part->act_state == XPC_P_AS_DEACTIVATING) |
e17d416b DN |
1493 | return part->reason; |
1494 | } | |
1495 | ||
1496 | return xpSuccess; | |
1497 | } | |
1498 | ||
1499 | /* | |
7fb5e59d | 1500 | * Get the chctl flags and pull the openclose args and/or remote GPs as needed. |
e17d416b DN |
1501 | */ |
1502 | static u64 | |
7fb5e59d | 1503 | xpc_get_chctl_all_flags_sn2(struct xpc_partition *part) |
e17d416b | 1504 | { |
a47d5dac | 1505 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b | 1506 | unsigned long irq_flags; |
7fb5e59d | 1507 | union xpc_channel_ctl_flags chctl; |
e17d416b DN |
1508 | enum xp_retval ret; |
1509 | ||
1510 | /* | |
7fb5e59d | 1511 | * See if there are any chctl flags to be handled. |
e17d416b DN |
1512 | */ |
1513 | ||
7fb5e59d DN |
1514 | spin_lock_irqsave(&part->chctl_lock, irq_flags); |
1515 | chctl = part->chctl; | |
1516 | if (chctl.all_flags != 0) | |
1517 | part->chctl.all_flags = 0; | |
e17d416b | 1518 | |
7fb5e59d | 1519 | spin_unlock_irqrestore(&part->chctl_lock, irq_flags); |
e17d416b | 1520 | |
7fb5e59d | 1521 | if (xpc_any_openclose_chctl_flags_set(&chctl)) { |
a47d5dac DN |
1522 | ret = xpc_pull_remote_cachelines_sn2(part, part-> |
1523 | remote_openclose_args, | |
a812dcc3 | 1524 | part_sn2-> |
e17d416b DN |
1525 | remote_openclose_args_pa, |
1526 | XPC_OPENCLOSE_ARGS_SIZE); | |
1527 | if (ret != xpSuccess) { | |
1528 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1529 | ||
1530 | dev_dbg(xpc_chan, "failed to pull openclose args from " | |
1531 | "partition %d, ret=%d\n", XPC_PARTID(part), | |
1532 | ret); | |
1533 | ||
7fb5e59d DN |
1534 | /* don't bother processing chctl flags anymore */ |
1535 | chctl.all_flags = 0; | |
e17d416b DN |
1536 | } |
1537 | } | |
1538 | ||
7fb5e59d | 1539 | if (xpc_any_msg_chctl_flags_set(&chctl)) { |
a47d5dac | 1540 | ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs, |
a812dcc3 | 1541 | part_sn2->remote_GPs_pa, |
e17d416b DN |
1542 | XPC_GP_SIZE); |
1543 | if (ret != xpSuccess) { | |
1544 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1545 | ||
1546 | dev_dbg(xpc_chan, "failed to pull GPs from partition " | |
1547 | "%d, ret=%d\n", XPC_PARTID(part), ret); | |
1548 | ||
7fb5e59d DN |
1549 | /* don't bother processing chctl flags anymore */ |
1550 | chctl.all_flags = 0; | |
e17d416b DN |
1551 | } |
1552 | } | |
1553 | ||
7fb5e59d | 1554 | return chctl.all_flags; |
e17d416b DN |
1555 | } |
1556 | ||
185c3a1b DN |
1557 | /* |
1558 | * Allocate the local message queue and the notify queue. | |
1559 | */ | |
1560 | static enum xp_retval | |
1561 | xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch) | |
1562 | { | |
5b8669df | 1563 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
185c3a1b DN |
1564 | unsigned long irq_flags; |
1565 | int nentries; | |
1566 | size_t nbytes; | |
1567 | ||
1568 | for (nentries = ch->local_nentries; nentries > 0; nentries--) { | |
1569 | ||
bd3e64c1 | 1570 | nbytes = nentries * ch->entry_size; |
5b8669df DN |
1571 | ch_sn2->local_msgqueue = |
1572 | xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, | |
1573 | &ch_sn2->local_msgqueue_base); | |
1574 | if (ch_sn2->local_msgqueue == NULL) | |
185c3a1b DN |
1575 | continue; |
1576 | ||
bd3e64c1 | 1577 | nbytes = nentries * sizeof(struct xpc_notify_sn2); |
5b8669df DN |
1578 | ch_sn2->notify_queue = kzalloc(nbytes, GFP_KERNEL); |
1579 | if (ch_sn2->notify_queue == NULL) { | |
1580 | kfree(ch_sn2->local_msgqueue_base); | |
1581 | ch_sn2->local_msgqueue = NULL; | |
185c3a1b DN |
1582 | continue; |
1583 | } | |
1584 | ||
1585 | spin_lock_irqsave(&ch->lock, irq_flags); | |
1586 | if (nentries < ch->local_nentries) { | |
1587 | dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, " | |
1588 | "partid=%d, channel=%d\n", nentries, | |
1589 | ch->local_nentries, ch->partid, ch->number); | |
1590 | ||
1591 | ch->local_nentries = nentries; | |
1592 | } | |
1593 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
1594 | return xpSuccess; | |
1595 | } | |
1596 | ||
1597 | dev_dbg(xpc_chan, "can't get memory for local message queue and notify " | |
1598 | "queue, partid=%d, channel=%d\n", ch->partid, ch->number); | |
1599 | return xpNoMemory; | |
1600 | } | |
1601 | ||
1602 | /* | |
1603 | * Allocate the cached remote message queue. | |
1604 | */ | |
1605 | static enum xp_retval | |
1606 | xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch) | |
1607 | { | |
5b8669df | 1608 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
185c3a1b DN |
1609 | unsigned long irq_flags; |
1610 | int nentries; | |
1611 | size_t nbytes; | |
1612 | ||
1613 | DBUG_ON(ch->remote_nentries <= 0); | |
1614 | ||
1615 | for (nentries = ch->remote_nentries; nentries > 0; nentries--) { | |
1616 | ||
bd3e64c1 | 1617 | nbytes = nentries * ch->entry_size; |
5b8669df DN |
1618 | ch_sn2->remote_msgqueue = |
1619 | xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, &ch_sn2-> | |
1620 | remote_msgqueue_base); | |
1621 | if (ch_sn2->remote_msgqueue == NULL) | |
185c3a1b DN |
1622 | continue; |
1623 | ||
1624 | spin_lock_irqsave(&ch->lock, irq_flags); | |
1625 | if (nentries < ch->remote_nentries) { | |
1626 | dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, " | |
1627 | "partid=%d, channel=%d\n", nentries, | |
1628 | ch->remote_nentries, ch->partid, ch->number); | |
1629 | ||
1630 | ch->remote_nentries = nentries; | |
1631 | } | |
1632 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
1633 | return xpSuccess; | |
1634 | } | |
1635 | ||
1636 | dev_dbg(xpc_chan, "can't get memory for cached remote message queue, " | |
1637 | "partid=%d, channel=%d\n", ch->partid, ch->number); | |
1638 | return xpNoMemory; | |
1639 | } | |
1640 | ||
1641 | /* | |
1642 | * Allocate message queues and other stuff associated with a channel. | |
1643 | * | |
1644 | * Note: Assumes all of the channel sizes are filled in. | |
1645 | */ | |
1646 | static enum xp_retval | |
5b8669df | 1647 | xpc_setup_msg_structures_sn2(struct xpc_channel *ch) |
185c3a1b | 1648 | { |
5b8669df | 1649 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
185c3a1b DN |
1650 | enum xp_retval ret; |
1651 | ||
1652 | DBUG_ON(ch->flags & XPC_C_SETUP); | |
1653 | ||
1654 | ret = xpc_allocate_local_msgqueue_sn2(ch); | |
1655 | if (ret == xpSuccess) { | |
1656 | ||
1657 | ret = xpc_allocate_remote_msgqueue_sn2(ch); | |
1658 | if (ret != xpSuccess) { | |
5b8669df DN |
1659 | kfree(ch_sn2->local_msgqueue_base); |
1660 | ch_sn2->local_msgqueue = NULL; | |
1661 | kfree(ch_sn2->notify_queue); | |
1662 | ch_sn2->notify_queue = NULL; | |
185c3a1b DN |
1663 | } |
1664 | } | |
1665 | return ret; | |
1666 | } | |
1667 | ||
1668 | /* | |
1669 | * Free up message queues and other stuff that were allocated for the specified | |
1670 | * channel. | |
185c3a1b DN |
1671 | */ |
1672 | static void | |
5b8669df | 1673 | xpc_teardown_msg_structures_sn2(struct xpc_channel *ch) |
185c3a1b DN |
1674 | { |
1675 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
1676 | ||
1677 | DBUG_ON(!spin_is_locked(&ch->lock)); | |
185c3a1b | 1678 | |
5b8669df | 1679 | ch_sn2->remote_msgqueue_pa = 0; |
185c3a1b DN |
1680 | |
1681 | ch_sn2->local_GP->get = 0; | |
1682 | ch_sn2->local_GP->put = 0; | |
1683 | ch_sn2->remote_GP.get = 0; | |
1684 | ch_sn2->remote_GP.put = 0; | |
1685 | ch_sn2->w_local_GP.get = 0; | |
1686 | ch_sn2->w_local_GP.put = 0; | |
1687 | ch_sn2->w_remote_GP.get = 0; | |
1688 | ch_sn2->w_remote_GP.put = 0; | |
1689 | ch_sn2->next_msg_to_pull = 0; | |
1690 | ||
1691 | if (ch->flags & XPC_C_SETUP) { | |
1692 | dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n", | |
1693 | ch->flags, ch->partid, ch->number); | |
1694 | ||
5b8669df DN |
1695 | kfree(ch_sn2->local_msgqueue_base); |
1696 | ch_sn2->local_msgqueue = NULL; | |
1697 | kfree(ch_sn2->remote_msgqueue_base); | |
1698 | ch_sn2->remote_msgqueue = NULL; | |
1699 | kfree(ch_sn2->notify_queue); | |
1700 | ch_sn2->notify_queue = NULL; | |
185c3a1b DN |
1701 | } |
1702 | } | |
1703 | ||
a47d5dac DN |
1704 | /* |
1705 | * Notify those who wanted to be notified upon delivery of their message. | |
1706 | */ | |
1707 | static void | |
1708 | xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put) | |
1709 | { | |
bd3e64c1 | 1710 | struct xpc_notify_sn2 *notify; |
a47d5dac DN |
1711 | u8 notify_type; |
1712 | s64 get = ch->sn.sn2.w_remote_GP.get - 1; | |
1713 | ||
1714 | while (++get < put && atomic_read(&ch->n_to_notify) > 0) { | |
1715 | ||
5b8669df | 1716 | notify = &ch->sn.sn2.notify_queue[get % ch->local_nentries]; |
a47d5dac DN |
1717 | |
1718 | /* | |
1719 | * See if the notify entry indicates it was associated with | |
1720 | * a message who's sender wants to be notified. It is possible | |
1721 | * that it is, but someone else is doing or has done the | |
1722 | * notification. | |
1723 | */ | |
1724 | notify_type = notify->type; | |
1725 | if (notify_type == 0 || | |
1726 | cmpxchg(¬ify->type, notify_type, 0) != notify_type) { | |
1727 | continue; | |
1728 | } | |
1729 | ||
1730 | DBUG_ON(notify_type != XPC_N_CALL); | |
1731 | ||
1732 | atomic_dec(&ch->n_to_notify); | |
1733 | ||
1734 | if (notify->func != NULL) { | |
bd3e64c1 | 1735 | dev_dbg(xpc_chan, "notify->func() called, notify=0x%p " |
f509e34a | 1736 | "msg_number=%lld partid=%d channel=%d\n", |
a47d5dac DN |
1737 | (void *)notify, get, ch->partid, ch->number); |
1738 | ||
1739 | notify->func(reason, ch->partid, ch->number, | |
1740 | notify->key); | |
1741 | ||
bd3e64c1 | 1742 | dev_dbg(xpc_chan, "notify->func() returned, notify=0x%p" |
f509e34a | 1743 | " msg_number=%lld partid=%d channel=%d\n", |
bd3e64c1 | 1744 | (void *)notify, get, ch->partid, ch->number); |
a47d5dac DN |
1745 | } |
1746 | } | |
1747 | } | |
1748 | ||
1749 | static void | |
1750 | xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch) | |
1751 | { | |
1752 | xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put); | |
1753 | } | |
1754 | ||
1755 | /* | |
1756 | * Clear some of the msg flags in the local message queue. | |
1757 | */ | |
1758 | static inline void | |
1759 | xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch) | |
1760 | { | |
1761 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
bd3e64c1 | 1762 | struct xpc_msg_sn2 *msg; |
a47d5dac DN |
1763 | s64 get; |
1764 | ||
1765 | get = ch_sn2->w_remote_GP.get; | |
1766 | do { | |
bd3e64c1 DN |
1767 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue + |
1768 | (get % ch->local_nentries) * | |
1769 | ch->entry_size); | |
252523ef | 1770 | DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); |
a47d5dac DN |
1771 | msg->flags = 0; |
1772 | } while (++get < ch_sn2->remote_GP.get); | |
1773 | } | |
1774 | ||
1775 | /* | |
1776 | * Clear some of the msg flags in the remote message queue. | |
1777 | */ | |
1778 | static inline void | |
1779 | xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch) | |
1780 | { | |
1781 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
bd3e64c1 | 1782 | struct xpc_msg_sn2 *msg; |
6e873ec7 | 1783 | s64 put, remote_nentries = ch->remote_nentries; |
a47d5dac | 1784 | |
252523ef | 1785 | /* flags are zeroed when the buffer is allocated */ |
6e873ec7 | 1786 | if (ch_sn2->remote_GP.put < remote_nentries) |
252523ef RH |
1787 | return; |
1788 | ||
6e873ec7 | 1789 | put = max(ch_sn2->w_remote_GP.put, remote_nentries); |
a47d5dac | 1790 | do { |
bd3e64c1 | 1791 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + |
6e873ec7 | 1792 | (put % remote_nentries) * |
bd3e64c1 | 1793 | ch->entry_size); |
252523ef RH |
1794 | DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); |
1795 | DBUG_ON(!(msg->flags & XPC_M_SN2_DONE)); | |
6e873ec7 | 1796 | DBUG_ON(msg->number != put - remote_nentries); |
a47d5dac DN |
1797 | msg->flags = 0; |
1798 | } while (++put < ch_sn2->remote_GP.put); | |
1799 | } | |
1800 | ||
bd3e64c1 DN |
1801 | static int |
1802 | xpc_n_of_deliverable_payloads_sn2(struct xpc_channel *ch) | |
1803 | { | |
1804 | return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get; | |
1805 | } | |
1806 | ||
a47d5dac | 1807 | static void |
7fb5e59d | 1808 | xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number) |
a47d5dac DN |
1809 | { |
1810 | struct xpc_channel *ch = &part->channels[ch_number]; | |
1811 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
bd3e64c1 | 1812 | int npayloads_sent; |
a47d5dac DN |
1813 | |
1814 | ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number]; | |
1815 | ||
1816 | /* See what, if anything, has changed for each connected channel */ | |
1817 | ||
1818 | xpc_msgqueue_ref(ch); | |
1819 | ||
1820 | if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get && | |
1821 | ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) { | |
1822 | /* nothing changed since GPs were last pulled */ | |
1823 | xpc_msgqueue_deref(ch); | |
1824 | return; | |
1825 | } | |
1826 | ||
1827 | if (!(ch->flags & XPC_C_CONNECTED)) { | |
1828 | xpc_msgqueue_deref(ch); | |
1829 | return; | |
1830 | } | |
1831 | ||
1832 | /* | |
1833 | * First check to see if messages recently sent by us have been | |
1834 | * received by the other side. (The remote GET value will have | |
1835 | * changed since we last looked at it.) | |
1836 | */ | |
1837 | ||
1838 | if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) { | |
1839 | ||
1840 | /* | |
1841 | * We need to notify any senders that want to be notified | |
1842 | * that their sent messages have been received by their | |
1843 | * intended recipients. We need to do this before updating | |
1844 | * w_remote_GP.get so that we don't allocate the same message | |
1845 | * queue entries prematurely (see xpc_allocate_msg()). | |
1846 | */ | |
1847 | if (atomic_read(&ch->n_to_notify) > 0) { | |
1848 | /* | |
1849 | * Notify senders that messages sent have been | |
1850 | * received and delivered by the other side. | |
1851 | */ | |
1852 | xpc_notify_senders_sn2(ch, xpMsgDelivered, | |
1853 | ch_sn2->remote_GP.get); | |
1854 | } | |
1855 | ||
1856 | /* | |
1857 | * Clear msg->flags in previously sent messages, so that | |
1858 | * they're ready for xpc_allocate_msg(). | |
1859 | */ | |
1860 | xpc_clear_local_msgqueue_flags_sn2(ch); | |
1861 | ||
1862 | ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get; | |
1863 | ||
f509e34a | 1864 | dev_dbg(xpc_chan, "w_remote_GP.get changed to %lld, partid=%d, " |
a47d5dac DN |
1865 | "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid, |
1866 | ch->number); | |
1867 | ||
1868 | /* | |
1869 | * If anyone was waiting for message queue entries to become | |
1870 | * available, wake them up. | |
1871 | */ | |
1872 | if (atomic_read(&ch->n_on_msg_allocate_wq) > 0) | |
1873 | wake_up(&ch->msg_allocate_wq); | |
1874 | } | |
1875 | ||
1876 | /* | |
1877 | * Now check for newly sent messages by the other side. (The remote | |
1878 | * PUT value will have changed since we last looked at it.) | |
1879 | */ | |
1880 | ||
1881 | if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) { | |
1882 | /* | |
1883 | * Clear msg->flags in previously received messages, so that | |
bd3e64c1 | 1884 | * they're ready for xpc_get_deliverable_payload_sn2(). |
a47d5dac DN |
1885 | */ |
1886 | xpc_clear_remote_msgqueue_flags_sn2(ch); | |
1887 | ||
69b3bb65 | 1888 | smp_wmb(); /* ensure flags have been cleared before bte_copy */ |
a47d5dac DN |
1889 | ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put; |
1890 | ||
f509e34a | 1891 | dev_dbg(xpc_chan, "w_remote_GP.put changed to %lld, partid=%d, " |
a47d5dac DN |
1892 | "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid, |
1893 | ch->number); | |
1894 | ||
bd3e64c1 DN |
1895 | npayloads_sent = xpc_n_of_deliverable_payloads_sn2(ch); |
1896 | if (npayloads_sent > 0) { | |
a47d5dac DN |
1897 | dev_dbg(xpc_chan, "msgs waiting to be copied and " |
1898 | "delivered=%d, partid=%d, channel=%d\n", | |
bd3e64c1 | 1899 | npayloads_sent, ch->partid, ch->number); |
a47d5dac DN |
1900 | |
1901 | if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) | |
bd3e64c1 | 1902 | xpc_activate_kthreads(ch, npayloads_sent); |
a47d5dac DN |
1903 | } |
1904 | } | |
1905 | ||
1906 | xpc_msgqueue_deref(ch); | |
1907 | } | |
1908 | ||
bd3e64c1 | 1909 | static struct xpc_msg_sn2 * |
e17d416b DN |
1910 | xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get) |
1911 | { | |
1912 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
a47d5dac | 1913 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
a812dcc3 | 1914 | unsigned long remote_msg_pa; |
bd3e64c1 | 1915 | struct xpc_msg_sn2 *msg; |
a812dcc3 DN |
1916 | u32 msg_index; |
1917 | u32 nmsgs; | |
e17d416b DN |
1918 | u64 msg_offset; |
1919 | enum xp_retval ret; | |
1920 | ||
a47d5dac | 1921 | if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) { |
e17d416b DN |
1922 | /* we were interrupted by a signal */ |
1923 | return NULL; | |
1924 | } | |
1925 | ||
a47d5dac | 1926 | while (get >= ch_sn2->next_msg_to_pull) { |
e17d416b DN |
1927 | |
1928 | /* pull as many messages as are ready and able to be pulled */ | |
1929 | ||
a47d5dac | 1930 | msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries; |
e17d416b | 1931 | |
a47d5dac DN |
1932 | DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put); |
1933 | nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull; | |
e17d416b DN |
1934 | if (msg_index + nmsgs > ch->remote_nentries) { |
1935 | /* ignore the ones that wrap the msg queue for now */ | |
1936 | nmsgs = ch->remote_nentries - msg_index; | |
1937 | } | |
1938 | ||
bd3e64c1 DN |
1939 | msg_offset = msg_index * ch->entry_size; |
1940 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + | |
5b8669df DN |
1941 | msg_offset); |
1942 | remote_msg_pa = ch_sn2->remote_msgqueue_pa + msg_offset; | |
e17d416b | 1943 | |
a812dcc3 | 1944 | ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa, |
bd3e64c1 | 1945 | nmsgs * ch->entry_size); |
e17d416b DN |
1946 | if (ret != xpSuccess) { |
1947 | ||
1948 | dev_dbg(xpc_chan, "failed to pull %d msgs starting with" | |
f509e34a | 1949 | " msg %lld from partition %d, channel=%d, " |
a47d5dac | 1950 | "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull, |
e17d416b DN |
1951 | ch->partid, ch->number, ret); |
1952 | ||
1953 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1954 | ||
a47d5dac | 1955 | mutex_unlock(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1956 | return NULL; |
1957 | } | |
1958 | ||
a47d5dac | 1959 | ch_sn2->next_msg_to_pull += nmsgs; |
e17d416b DN |
1960 | } |
1961 | ||
a47d5dac | 1962 | mutex_unlock(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1963 | |
1964 | /* return the message we were looking for */ | |
bd3e64c1 DN |
1965 | msg_offset = (get % ch->remote_nentries) * ch->entry_size; |
1966 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + msg_offset); | |
e17d416b DN |
1967 | |
1968 | return msg; | |
1969 | } | |
1970 | ||
1971 | /* | |
bd3e64c1 | 1972 | * Get the next deliverable message's payload. |
e17d416b | 1973 | */ |
bd3e64c1 DN |
1974 | static void * |
1975 | xpc_get_deliverable_payload_sn2(struct xpc_channel *ch) | |
e17d416b | 1976 | { |
a47d5dac | 1977 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 DN |
1978 | struct xpc_msg_sn2 *msg; |
1979 | void *payload = NULL; | |
e17d416b DN |
1980 | s64 get; |
1981 | ||
1982 | do { | |
1983 | if (ch->flags & XPC_C_DISCONNECTING) | |
1984 | break; | |
1985 | ||
a47d5dac | 1986 | get = ch_sn2->w_local_GP.get; |
69b3bb65 | 1987 | smp_rmb(); /* guarantee that .get loads before .put */ |
a47d5dac | 1988 | if (get == ch_sn2->w_remote_GP.put) |
e17d416b DN |
1989 | break; |
1990 | ||
1991 | /* There are messages waiting to be pulled and delivered. | |
1992 | * We need to try to secure one for ourselves. We'll do this | |
1993 | * by trying to increment w_local_GP.get and hope that no one | |
1994 | * else beats us to it. If they do, we'll we'll simply have | |
1995 | * to try again for the next one. | |
1996 | */ | |
1997 | ||
a47d5dac | 1998 | if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) { |
e17d416b DN |
1999 | /* we got the entry referenced by get */ |
2000 | ||
f509e34a | 2001 | dev_dbg(xpc_chan, "w_local_GP.get changed to %lld, " |
e17d416b DN |
2002 | "partid=%d, channel=%d\n", get + 1, |
2003 | ch->partid, ch->number); | |
2004 | ||
2005 | /* pull the message from the remote partition */ | |
2006 | ||
2007 | msg = xpc_pull_remote_msg_sn2(ch, get); | |
2008 | ||
17e21616 RH |
2009 | if (msg != NULL) { |
2010 | DBUG_ON(msg->number != get); | |
2011 | DBUG_ON(msg->flags & XPC_M_SN2_DONE); | |
2012 | DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); | |
e17d416b | 2013 | |
17e21616 RH |
2014 | payload = &msg->payload; |
2015 | } | |
e17d416b DN |
2016 | break; |
2017 | } | |
2018 | ||
2019 | } while (1); | |
2020 | ||
bd3e64c1 | 2021 | return payload; |
e17d416b DN |
2022 | } |
2023 | ||
33ba3c77 DN |
2024 | /* |
2025 | * Now we actually send the messages that are ready to be sent by advancing | |
7fb5e59d DN |
2026 | * the local message queue's Put value and then send a chctl msgrequest to the |
2027 | * recipient partition. | |
33ba3c77 DN |
2028 | */ |
2029 | static void | |
2030 | xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put) | |
2031 | { | |
a47d5dac | 2032 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 | 2033 | struct xpc_msg_sn2 *msg; |
33ba3c77 | 2034 | s64 put = initial_put + 1; |
7fb5e59d | 2035 | int send_msgrequest = 0; |
33ba3c77 DN |
2036 | |
2037 | while (1) { | |
2038 | ||
2039 | while (1) { | |
a47d5dac | 2040 | if (put == ch_sn2->w_local_GP.put) |
33ba3c77 DN |
2041 | break; |
2042 | ||
bd3e64c1 DN |
2043 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2-> |
2044 | local_msgqueue + (put % | |
2045 | ch->local_nentries) * | |
2046 | ch->entry_size); | |
33ba3c77 | 2047 | |
bd3e64c1 | 2048 | if (!(msg->flags & XPC_M_SN2_READY)) |
33ba3c77 DN |
2049 | break; |
2050 | ||
2051 | put++; | |
2052 | } | |
2053 | ||
2054 | if (put == initial_put) { | |
2055 | /* nothing's changed */ | |
2056 | break; | |
2057 | } | |
2058 | ||
a47d5dac | 2059 | if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) != |
33ba3c77 DN |
2060 | initial_put) { |
2061 | /* someone else beat us to it */ | |
a47d5dac | 2062 | DBUG_ON(ch_sn2->local_GP->put < initial_put); |
33ba3c77 DN |
2063 | break; |
2064 | } | |
2065 | ||
2066 | /* we just set the new value of local_GP->put */ | |
2067 | ||
f509e34a | 2068 | dev_dbg(xpc_chan, "local_GP->put changed to %lld, partid=%d, " |
33ba3c77 DN |
2069 | "channel=%d\n", put, ch->partid, ch->number); |
2070 | ||
7fb5e59d | 2071 | send_msgrequest = 1; |
33ba3c77 DN |
2072 | |
2073 | /* | |
2074 | * We need to ensure that the message referenced by | |
bd3e64c1 | 2075 | * local_GP->put is not XPC_M_SN2_READY or that local_GP->put |
33ba3c77 DN |
2076 | * equals w_local_GP.put, so we'll go have a look. |
2077 | */ | |
2078 | initial_put = put; | |
2079 | } | |
2080 | ||
7fb5e59d DN |
2081 | if (send_msgrequest) |
2082 | xpc_send_chctl_msgrequest_sn2(ch); | |
33ba3c77 DN |
2083 | } |
2084 | ||
2085 | /* | |
2086 | * Allocate an entry for a message from the message queue associated with the | |
2087 | * specified channel. | |
2088 | */ | |
2089 | static enum xp_retval | |
2090 | xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags, | |
bd3e64c1 | 2091 | struct xpc_msg_sn2 **address_of_msg) |
33ba3c77 | 2092 | { |
a47d5dac | 2093 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 | 2094 | struct xpc_msg_sn2 *msg; |
33ba3c77 DN |
2095 | enum xp_retval ret; |
2096 | s64 put; | |
2097 | ||
33ba3c77 DN |
2098 | /* |
2099 | * Get the next available message entry from the local message queue. | |
2100 | * If none are available, we'll make sure that we grab the latest | |
2101 | * GP values. | |
2102 | */ | |
2103 | ret = xpTimeout; | |
2104 | ||
2105 | while (1) { | |
2106 | ||
a47d5dac | 2107 | put = ch_sn2->w_local_GP.put; |
69b3bb65 | 2108 | smp_rmb(); /* guarantee that .put loads before .get */ |
a47d5dac | 2109 | if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) { |
33ba3c77 DN |
2110 | |
2111 | /* There are available message entries. We need to try | |
2112 | * to secure one for ourselves. We'll do this by trying | |
2113 | * to increment w_local_GP.put as long as someone else | |
2114 | * doesn't beat us to it. If they do, we'll have to | |
2115 | * try again. | |
2116 | */ | |
a47d5dac DN |
2117 | if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) == |
2118 | put) { | |
33ba3c77 DN |
2119 | /* we got the entry referenced by put */ |
2120 | break; | |
2121 | } | |
2122 | continue; /* try again */ | |
2123 | } | |
2124 | ||
2125 | /* | |
2126 | * There aren't any available msg entries at this time. | |
2127 | * | |
2128 | * In waiting for a message entry to become available, | |
7fb5e59d DN |
2129 | * we set a timeout in case the other side is not sending |
2130 | * completion interrupts. This lets us fake a notify IRQ | |
2131 | * that will cause the notify IRQ handler to fetch the latest | |
2132 | * GP values as if an interrupt was sent by the other side. | |
33ba3c77 DN |
2133 | */ |
2134 | if (ret == xpTimeout) | |
7fb5e59d | 2135 | xpc_send_chctl_local_msgrequest_sn2(ch); |
33ba3c77 | 2136 | |
97bf1aa1 | 2137 | if (flags & XPC_NOWAIT) |
33ba3c77 | 2138 | return xpNoWait; |
33ba3c77 DN |
2139 | |
2140 | ret = xpc_allocate_msg_wait(ch); | |
97bf1aa1 | 2141 | if (ret != xpInterrupted && ret != xpTimeout) |
33ba3c77 | 2142 | return ret; |
33ba3c77 DN |
2143 | } |
2144 | ||
2145 | /* get the message's address and initialize it */ | |
bd3e64c1 DN |
2146 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue + |
2147 | (put % ch->local_nentries) * | |
2148 | ch->entry_size); | |
33ba3c77 DN |
2149 | |
2150 | DBUG_ON(msg->flags != 0); | |
2151 | msg->number = put; | |
2152 | ||
f509e34a RD |
2153 | dev_dbg(xpc_chan, "w_local_GP.put changed to %lld; msg=0x%p, " |
2154 | "msg_number=%lld, partid=%d, channel=%d\n", put + 1, | |
33ba3c77 DN |
2155 | (void *)msg, msg->number, ch->partid, ch->number); |
2156 | ||
2157 | *address_of_msg = msg; | |
33ba3c77 DN |
2158 | return xpSuccess; |
2159 | } | |
2160 | ||
2161 | /* | |
2162 | * Common code that does the actual sending of the message by advancing the | |
7fb5e59d DN |
2163 | * local message queue's Put value and sends a chctl msgrequest to the |
2164 | * partition the message is being sent to. | |
33ba3c77 DN |
2165 | */ |
2166 | static enum xp_retval | |
bd3e64c1 DN |
2167 | xpc_send_payload_sn2(struct xpc_channel *ch, u32 flags, void *payload, |
2168 | u16 payload_size, u8 notify_type, xpc_notify_func func, | |
2169 | void *key) | |
33ba3c77 DN |
2170 | { |
2171 | enum xp_retval ret = xpSuccess; | |
5b8669df | 2172 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 DN |
2173 | struct xpc_msg_sn2 *msg = msg; |
2174 | struct xpc_notify_sn2 *notify = notify; | |
97bf1aa1 DN |
2175 | s64 msg_number; |
2176 | s64 put; | |
33ba3c77 DN |
2177 | |
2178 | DBUG_ON(notify_type == XPC_N_CALL && func == NULL); | |
97bf1aa1 | 2179 | |
bd3e64c1 | 2180 | if (XPC_MSG_SIZE(payload_size) > ch->entry_size) |
97bf1aa1 DN |
2181 | return xpPayloadTooBig; |
2182 | ||
2183 | xpc_msgqueue_ref(ch); | |
33ba3c77 DN |
2184 | |
2185 | if (ch->flags & XPC_C_DISCONNECTING) { | |
97bf1aa1 DN |
2186 | ret = ch->reason; |
2187 | goto out_1; | |
2188 | } | |
2189 | if (!(ch->flags & XPC_C_CONNECTED)) { | |
2190 | ret = xpNotConnected; | |
2191 | goto out_1; | |
33ba3c77 DN |
2192 | } |
2193 | ||
97bf1aa1 DN |
2194 | ret = xpc_allocate_msg_sn2(ch, flags, &msg); |
2195 | if (ret != xpSuccess) | |
2196 | goto out_1; | |
2197 | ||
2198 | msg_number = msg->number; | |
2199 | ||
33ba3c77 DN |
2200 | if (notify_type != 0) { |
2201 | /* | |
2202 | * Tell the remote side to send an ACK interrupt when the | |
2203 | * message has been delivered. | |
2204 | */ | |
bd3e64c1 | 2205 | msg->flags |= XPC_M_SN2_INTERRUPT; |
33ba3c77 DN |
2206 | |
2207 | atomic_inc(&ch->n_to_notify); | |
2208 | ||
5b8669df | 2209 | notify = &ch_sn2->notify_queue[msg_number % ch->local_nentries]; |
33ba3c77 DN |
2210 | notify->func = func; |
2211 | notify->key = key; | |
2212 | notify->type = notify_type; | |
2213 | ||
ea57f80c | 2214 | /* ??? Is a mb() needed here? */ |
33ba3c77 DN |
2215 | |
2216 | if (ch->flags & XPC_C_DISCONNECTING) { | |
2217 | /* | |
2218 | * An error occurred between our last error check and | |
2219 | * this one. We will try to clear the type field from | |
2220 | * the notify entry. If we succeed then | |
2221 | * xpc_disconnect_channel() didn't already process | |
2222 | * the notify entry. | |
2223 | */ | |
2224 | if (cmpxchg(¬ify->type, notify_type, 0) == | |
2225 | notify_type) { | |
2226 | atomic_dec(&ch->n_to_notify); | |
2227 | ret = ch->reason; | |
2228 | } | |
97bf1aa1 | 2229 | goto out_1; |
33ba3c77 DN |
2230 | } |
2231 | } | |
2232 | ||
97bf1aa1 DN |
2233 | memcpy(&msg->payload, payload, payload_size); |
2234 | ||
bd3e64c1 | 2235 | msg->flags |= XPC_M_SN2_READY; |
33ba3c77 DN |
2236 | |
2237 | /* | |
2238 | * The preceding store of msg->flags must occur before the following | |
a47d5dac | 2239 | * load of local_GP->put. |
33ba3c77 | 2240 | */ |
69b3bb65 | 2241 | smp_mb(); |
33ba3c77 DN |
2242 | |
2243 | /* see if the message is next in line to be sent, if so send it */ | |
2244 | ||
5b8669df | 2245 | put = ch_sn2->local_GP->put; |
33ba3c77 DN |
2246 | if (put == msg_number) |
2247 | xpc_send_msgs_sn2(ch, put); | |
2248 | ||
97bf1aa1 | 2249 | out_1: |
33ba3c77 DN |
2250 | xpc_msgqueue_deref(ch); |
2251 | return ret; | |
2252 | } | |
2253 | ||
2254 | /* | |
2255 | * Now we actually acknowledge the messages that have been delivered and ack'd | |
2256 | * by advancing the cached remote message queue's Get value and if requested | |
7fb5e59d | 2257 | * send a chctl msgrequest to the message sender's partition. |
bd3e64c1 DN |
2258 | * |
2259 | * If a message has XPC_M_SN2_INTERRUPT set, send an interrupt to the partition | |
2260 | * that sent the message. | |
33ba3c77 DN |
2261 | */ |
2262 | static void | |
2263 | xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags) | |
2264 | { | |
a47d5dac | 2265 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 | 2266 | struct xpc_msg_sn2 *msg; |
33ba3c77 | 2267 | s64 get = initial_get + 1; |
7fb5e59d | 2268 | int send_msgrequest = 0; |
33ba3c77 DN |
2269 | |
2270 | while (1) { | |
2271 | ||
2272 | while (1) { | |
a47d5dac | 2273 | if (get == ch_sn2->w_local_GP.get) |
33ba3c77 DN |
2274 | break; |
2275 | ||
bd3e64c1 DN |
2276 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2-> |
2277 | remote_msgqueue + (get % | |
2278 | ch->remote_nentries) * | |
2279 | ch->entry_size); | |
33ba3c77 | 2280 | |
bd3e64c1 | 2281 | if (!(msg->flags & XPC_M_SN2_DONE)) |
33ba3c77 DN |
2282 | break; |
2283 | ||
2284 | msg_flags |= msg->flags; | |
2285 | get++; | |
2286 | } | |
2287 | ||
2288 | if (get == initial_get) { | |
2289 | /* nothing's changed */ | |
2290 | break; | |
2291 | } | |
2292 | ||
a47d5dac | 2293 | if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) != |
33ba3c77 DN |
2294 | initial_get) { |
2295 | /* someone else beat us to it */ | |
a47d5dac | 2296 | DBUG_ON(ch_sn2->local_GP->get <= initial_get); |
33ba3c77 DN |
2297 | break; |
2298 | } | |
2299 | ||
2300 | /* we just set the new value of local_GP->get */ | |
2301 | ||
f509e34a | 2302 | dev_dbg(xpc_chan, "local_GP->get changed to %lld, partid=%d, " |
33ba3c77 DN |
2303 | "channel=%d\n", get, ch->partid, ch->number); |
2304 | ||
bd3e64c1 | 2305 | send_msgrequest = (msg_flags & XPC_M_SN2_INTERRUPT); |
33ba3c77 DN |
2306 | |
2307 | /* | |
2308 | * We need to ensure that the message referenced by | |
bd3e64c1 | 2309 | * local_GP->get is not XPC_M_SN2_DONE or that local_GP->get |
33ba3c77 DN |
2310 | * equals w_local_GP.get, so we'll go have a look. |
2311 | */ | |
2312 | initial_get = get; | |
2313 | } | |
2314 | ||
7fb5e59d DN |
2315 | if (send_msgrequest) |
2316 | xpc_send_chctl_msgrequest_sn2(ch); | |
33ba3c77 DN |
2317 | } |
2318 | ||
2319 | static void | |
bd3e64c1 | 2320 | xpc_received_payload_sn2(struct xpc_channel *ch, void *payload) |
33ba3c77 | 2321 | { |
bd3e64c1 DN |
2322 | struct xpc_msg_sn2 *msg; |
2323 | s64 msg_number; | |
33ba3c77 | 2324 | s64 get; |
bd3e64c1 DN |
2325 | |
2326 | msg = container_of(payload, struct xpc_msg_sn2, payload); | |
2327 | msg_number = msg->number; | |
33ba3c77 | 2328 | |
f509e34a | 2329 | dev_dbg(xpc_chan, "msg=0x%p, msg_number=%lld, partid=%d, channel=%d\n", |
33ba3c77 DN |
2330 | (void *)msg, msg_number, ch->partid, ch->number); |
2331 | ||
252523ef | 2332 | DBUG_ON((((u64)msg - (u64)ch->sn.sn2.remote_msgqueue) / ch->entry_size) != |
33ba3c77 | 2333 | msg_number % ch->remote_nentries); |
252523ef | 2334 | DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); |
bd3e64c1 | 2335 | DBUG_ON(msg->flags & XPC_M_SN2_DONE); |
33ba3c77 | 2336 | |
bd3e64c1 | 2337 | msg->flags |= XPC_M_SN2_DONE; |
33ba3c77 DN |
2338 | |
2339 | /* | |
2340 | * The preceding store of msg->flags must occur before the following | |
a47d5dac | 2341 | * load of local_GP->get. |
33ba3c77 | 2342 | */ |
69b3bb65 | 2343 | smp_mb(); |
33ba3c77 DN |
2344 | |
2345 | /* | |
2346 | * See if this message is next in line to be acknowledged as having | |
2347 | * been delivered. | |
2348 | */ | |
a47d5dac | 2349 | get = ch->sn.sn2.local_GP->get; |
33ba3c77 DN |
2350 | if (get == msg_number) |
2351 | xpc_acknowledge_msgs_sn2(ch, get, msg->flags); | |
2352 | } | |
2353 | ||
a7665b0a RH |
2354 | static struct xpc_arch_operations xpc_arch_ops_sn2 = { |
2355 | .setup_partitions = xpc_setup_partitions_sn2, | |
2356 | .teardown_partitions = xpc_teardown_partitions_sn2, | |
2357 | .process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2, | |
2358 | .get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2, | |
2359 | .setup_rsvd_page = xpc_setup_rsvd_page_sn2, | |
2360 | ||
2361 | .allow_hb = xpc_allow_hb_sn2, | |
2362 | .disallow_hb = xpc_disallow_hb_sn2, | |
2363 | .disallow_all_hbs = xpc_disallow_all_hbs_sn2, | |
2364 | .increment_heartbeat = xpc_increment_heartbeat_sn2, | |
2365 | .offline_heartbeat = xpc_offline_heartbeat_sn2, | |
2366 | .online_heartbeat = xpc_online_heartbeat_sn2, | |
2367 | .heartbeat_init = xpc_heartbeat_init_sn2, | |
2368 | .heartbeat_exit = xpc_heartbeat_exit_sn2, | |
2369 | .get_remote_heartbeat = xpc_get_remote_heartbeat_sn2, | |
2370 | ||
2371 | .request_partition_activation = | |
2372 | xpc_request_partition_activation_sn2, | |
2373 | .request_partition_reactivation = | |
2374 | xpc_request_partition_reactivation_sn2, | |
2375 | .request_partition_deactivation = | |
2376 | xpc_request_partition_deactivation_sn2, | |
2377 | .cancel_partition_deactivation_request = | |
2378 | xpc_cancel_partition_deactivation_request_sn2, | |
2379 | ||
2380 | .setup_ch_structures = xpc_setup_ch_structures_sn2, | |
2381 | .teardown_ch_structures = xpc_teardown_ch_structures_sn2, | |
2382 | ||
2383 | .make_first_contact = xpc_make_first_contact_sn2, | |
2384 | ||
2385 | .get_chctl_all_flags = xpc_get_chctl_all_flags_sn2, | |
2386 | .send_chctl_closerequest = xpc_send_chctl_closerequest_sn2, | |
2387 | .send_chctl_closereply = xpc_send_chctl_closereply_sn2, | |
2388 | .send_chctl_openrequest = xpc_send_chctl_openrequest_sn2, | |
2389 | .send_chctl_openreply = xpc_send_chctl_openreply_sn2, | |
2390 | .send_chctl_opencomplete = xpc_send_chctl_opencomplete_sn2, | |
2391 | .process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2, | |
2392 | ||
2393 | .save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_sn2, | |
2394 | ||
2395 | .setup_msg_structures = xpc_setup_msg_structures_sn2, | |
2396 | .teardown_msg_structures = xpc_teardown_msg_structures_sn2, | |
2397 | ||
2398 | .indicate_partition_engaged = xpc_indicate_partition_engaged_sn2, | |
2399 | .indicate_partition_disengaged = xpc_indicate_partition_disengaged_sn2, | |
2400 | .partition_engaged = xpc_partition_engaged_sn2, | |
2401 | .any_partition_engaged = xpc_any_partition_engaged_sn2, | |
2402 | .assume_partition_disengaged = xpc_assume_partition_disengaged_sn2, | |
2403 | ||
2404 | .n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_sn2, | |
2405 | .send_payload = xpc_send_payload_sn2, | |
2406 | .get_deliverable_payload = xpc_get_deliverable_payload_sn2, | |
2407 | .received_payload = xpc_received_payload_sn2, | |
2408 | .notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2, | |
2409 | }; | |
2410 | ||
6e41017a | 2411 | int |
94bd2708 DN |
2412 | xpc_init_sn2(void) |
2413 | { | |
6e41017a | 2414 | int ret; |
ee6665e3 | 2415 | size_t buf_size; |
6e41017a | 2416 | |
a7665b0a | 2417 | xpc_arch_ops = xpc_arch_ops_sn2; |
bd3e64c1 DN |
2418 | |
2419 | if (offsetof(struct xpc_msg_sn2, payload) > XPC_MSG_HDR_MAX_SIZE) { | |
2420 | dev_err(xpc_part, "header portion of struct xpc_msg_sn2 is " | |
2421 | "larger than %d\n", XPC_MSG_HDR_MAX_SIZE); | |
2422 | return -E2BIG; | |
2423 | } | |
6e41017a | 2424 | |
ee6665e3 DN |
2425 | buf_size = max(XPC_RP_VARS_SIZE, |
2426 | XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2); | |
2427 | xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size, | |
2428 | GFP_KERNEL, | |
2429 | &xpc_remote_copy_buffer_base_sn2); | |
2430 | if (xpc_remote_copy_buffer_sn2 == NULL) { | |
2431 | dev_err(xpc_part, "can't get memory for remote copy buffer\n"); | |
2432 | return -ENOMEM; | |
2433 | } | |
2434 | ||
c39838ce | 2435 | /* open up protections for IPI and [potentially] amo operations */ |
6e41017a | 2436 | xpc_allow_IPI_ops_sn2(); |
c39838ce | 2437 | xpc_allow_amo_ops_shub_wars_1_1_sn2(); |
6e41017a DN |
2438 | |
2439 | /* | |
2440 | * This is safe to do before the xpc_hb_checker thread has started | |
2441 | * because the handler releases a wait queue. If an interrupt is | |
2442 | * received before the thread is waiting, it will not go to sleep, | |
2443 | * but rather immediately process the interrupt. | |
2444 | */ | |
2445 | ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0, | |
2446 | "xpc hb", NULL); | |
2447 | if (ret != 0) { | |
2448 | dev_err(xpc_part, "can't register ACTIVATE IRQ handler, " | |
2449 | "errno=%d\n", -ret); | |
2450 | xpc_disallow_IPI_ops_sn2(); | |
ee6665e3 | 2451 | kfree(xpc_remote_copy_buffer_base_sn2); |
6e41017a DN |
2452 | } |
2453 | return ret; | |
94bd2708 DN |
2454 | } |
2455 | ||
2456 | void | |
2457 | xpc_exit_sn2(void) | |
2458 | { | |
6e41017a DN |
2459 | free_irq(SGI_XPC_ACTIVATE, NULL); |
2460 | xpc_disallow_IPI_ops_sn2(); | |
ee6665e3 | 2461 | kfree(xpc_remote_copy_buffer_base_sn2); |
94bd2708 | 2462 | } |