Commit | Line | Data |
---|---|---|
951f22d5 | 1 | /* |
951f22d5 MS |
2 | * Out of line spinlock code. |
3 | * | |
a53c8fab | 4 | * Copyright IBM Corp. 2004, 2006 |
951f22d5 MS |
5 | * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) |
6 | */ | |
7 | ||
8 | #include <linux/types.h> | |
9 | #include <linux/module.h> | |
10 | #include <linux/spinlock.h> | |
11 | #include <linux/init.h> | |
8b646bd7 | 12 | #include <linux/smp.h> |
951f22d5 MS |
13 | #include <asm/io.h> |
14 | ||
2c72a44e MS |
15 | int spin_retry = -1; |
16 | ||
17 | static int __init spin_retry_init(void) | |
18 | { | |
19 | if (spin_retry < 0) | |
20 | spin_retry = MACHINE_HAS_CAD ? 10 : 1000; | |
21 | return 0; | |
22 | } | |
23 | early_initcall(spin_retry_init); | |
951f22d5 MS |
24 | |
25 | /** | |
26 | * spin_retry= parameter | |
27 | */ | |
28 | static int __init spin_retry_setup(char *str) | |
29 | { | |
30 | spin_retry = simple_strtoul(str, &str, 0); | |
31 | return 1; | |
32 | } | |
33 | __setup("spin_retry=", spin_retry_setup); | |
34 | ||
2c72a44e MS |
35 | static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old) |
36 | { | |
37 | asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock)); | |
38 | } | |
39 | ||
419123f9 MS |
40 | static inline int cpu_is_preempted(int cpu) |
41 | { | |
42 | if (test_cpu_flag_of(CIF_ENABLED_WAIT, cpu)) | |
43 | return 0; | |
44 | if (smp_vcpu_scheduled(cpu)) | |
45 | return 0; | |
46 | return 1; | |
47 | } | |
48 | ||
0199c4e6 | 49 | void arch_spin_lock_wait(arch_spinlock_t *lp) |
951f22d5 | 50 | { |
6c8cd5bb | 51 | unsigned int cpu = SPINLOCK_LOCKVAL; |
59b69787 | 52 | unsigned int owner; |
db1c4515 | 53 | int count, first_diag; |
951f22d5 | 54 | |
db1c4515 | 55 | first_diag = 1; |
951f22d5 | 56 | while (1) { |
470ada6b MS |
57 | owner = ACCESS_ONCE(lp->lock); |
58 | /* Try to get the lock if it is free. */ | |
59 | if (!owner) { | |
60 | if (_raw_compare_and_swap(&lp->lock, 0, cpu)) | |
61 | return; | |
62 | continue; | |
951f22d5 | 63 | } |
db1c4515 | 64 | /* First iteration: check if the lock owner is running. */ |
419123f9 | 65 | if (first_diag && cpu_is_preempted(~owner)) { |
470ada6b | 66 | smp_yield_cpu(~owner); |
db1c4515 | 67 | first_diag = 0; |
470ada6b MS |
68 | continue; |
69 | } | |
70 | /* Loop for a while on the lock value. */ | |
71 | count = spin_retry; | |
72 | do { | |
2c72a44e MS |
73 | if (MACHINE_HAS_CAD) |
74 | _raw_compare_and_delay(&lp->lock, owner); | |
470ada6b MS |
75 | owner = ACCESS_ONCE(lp->lock); |
76 | } while (owner && count-- > 0); | |
77 | if (!owner) | |
78 | continue; | |
79 | /* | |
80 | * For multiple layers of hypervisors, e.g. z/VM + LPAR | |
db1c4515 MS |
81 | * yield the CPU unconditionally. For LPAR rely on the |
82 | * sense running status. | |
470ada6b | 83 | */ |
419123f9 | 84 | if (!MACHINE_IS_LPAR || cpu_is_preempted(~owner)) { |
8b646bd7 | 85 | smp_yield_cpu(~owner); |
db1c4515 MS |
86 | first_diag = 0; |
87 | } | |
951f22d5 MS |
88 | } |
89 | } | |
0199c4e6 | 90 | EXPORT_SYMBOL(arch_spin_lock_wait); |
951f22d5 | 91 | |
0199c4e6 | 92 | void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) |
894cdde2 | 93 | { |
6c8cd5bb | 94 | unsigned int cpu = SPINLOCK_LOCKVAL; |
59b69787 | 95 | unsigned int owner; |
db1c4515 | 96 | int count, first_diag; |
894cdde2 HH |
97 | |
98 | local_irq_restore(flags); | |
db1c4515 | 99 | first_diag = 1; |
894cdde2 | 100 | while (1) { |
470ada6b MS |
101 | owner = ACCESS_ONCE(lp->lock); |
102 | /* Try to get the lock if it is free. */ | |
103 | if (!owner) { | |
104 | local_irq_disable(); | |
105 | if (_raw_compare_and_swap(&lp->lock, 0, cpu)) | |
106 | return; | |
107 | local_irq_restore(flags); | |
108 | } | |
109 | /* Check if the lock owner is running. */ | |
419123f9 | 110 | if (first_diag && cpu_is_preempted(~owner)) { |
470ada6b | 111 | smp_yield_cpu(~owner); |
db1c4515 | 112 | first_diag = 0; |
470ada6b | 113 | continue; |
894cdde2 | 114 | } |
470ada6b MS |
115 | /* Loop for a while on the lock value. */ |
116 | count = spin_retry; | |
117 | do { | |
2c72a44e MS |
118 | if (MACHINE_HAS_CAD) |
119 | _raw_compare_and_delay(&lp->lock, owner); | |
470ada6b MS |
120 | owner = ACCESS_ONCE(lp->lock); |
121 | } while (owner && count-- > 0); | |
122 | if (!owner) | |
123 | continue; | |
124 | /* | |
125 | * For multiple layers of hypervisors, e.g. z/VM + LPAR | |
db1c4515 MS |
126 | * yield the CPU unconditionally. For LPAR rely on the |
127 | * sense running status. | |
470ada6b | 128 | */ |
419123f9 | 129 | if (!MACHINE_IS_LPAR || cpu_is_preempted(~owner)) { |
8b646bd7 | 130 | smp_yield_cpu(~owner); |
db1c4515 MS |
131 | first_diag = 0; |
132 | } | |
894cdde2 HH |
133 | } |
134 | } | |
0199c4e6 | 135 | EXPORT_SYMBOL(arch_spin_lock_wait_flags); |
894cdde2 | 136 | |
0199c4e6 | 137 | int arch_spin_trylock_retry(arch_spinlock_t *lp) |
951f22d5 | 138 | { |
2c72a44e MS |
139 | unsigned int cpu = SPINLOCK_LOCKVAL; |
140 | unsigned int owner; | |
3c1fcfe2 | 141 | int count; |
951f22d5 | 142 | |
2c72a44e MS |
143 | for (count = spin_retry; count > 0; count--) { |
144 | owner = ACCESS_ONCE(lp->lock); | |
145 | /* Try to get the lock if it is free. */ | |
146 | if (!owner) { | |
147 | if (_raw_compare_and_swap(&lp->lock, 0, cpu)) | |
148 | return 1; | |
149 | } else if (MACHINE_HAS_CAD) | |
150 | _raw_compare_and_delay(&lp->lock, owner); | |
151 | } | |
951f22d5 MS |
152 | return 0; |
153 | } | |
0199c4e6 | 154 | EXPORT_SYMBOL(arch_spin_trylock_retry); |
951f22d5 | 155 | |
fb3a6bbc | 156 | void _raw_read_lock_wait(arch_rwlock_t *rw) |
951f22d5 | 157 | { |
d59b93da | 158 | unsigned int owner, old; |
951f22d5 MS |
159 | int count = spin_retry; |
160 | ||
bbae71bf MS |
161 | #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES |
162 | __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD); | |
163 | #endif | |
d59b93da | 164 | owner = 0; |
951f22d5 MS |
165 | while (1) { |
166 | if (count-- <= 0) { | |
419123f9 | 167 | if (owner && cpu_is_preempted(~owner)) |
d59b93da | 168 | smp_yield_cpu(~owner); |
951f22d5 MS |
169 | count = spin_retry; |
170 | } | |
bae8f567 | 171 | old = ACCESS_ONCE(rw->lock); |
d59b93da | 172 | owner = ACCESS_ONCE(rw->owner); |
2c72a44e MS |
173 | if ((int) old < 0) { |
174 | if (MACHINE_HAS_CAD) | |
175 | _raw_compare_and_delay(&rw->lock, old); | |
96567161 | 176 | continue; |
2c72a44e | 177 | } |
5b3f683e | 178 | if (_raw_compare_and_swap(&rw->lock, old, old + 1)) |
951f22d5 MS |
179 | return; |
180 | } | |
181 | } | |
182 | EXPORT_SYMBOL(_raw_read_lock_wait); | |
183 | ||
fb3a6bbc | 184 | int _raw_read_trylock_retry(arch_rwlock_t *rw) |
951f22d5 MS |
185 | { |
186 | unsigned int old; | |
187 | int count = spin_retry; | |
188 | ||
189 | while (count-- > 0) { | |
bae8f567 | 190 | old = ACCESS_ONCE(rw->lock); |
2c72a44e MS |
191 | if ((int) old < 0) { |
192 | if (MACHINE_HAS_CAD) | |
193 | _raw_compare_and_delay(&rw->lock, old); | |
96567161 | 194 | continue; |
2c72a44e | 195 | } |
5b3f683e | 196 | if (_raw_compare_and_swap(&rw->lock, old, old + 1)) |
951f22d5 MS |
197 | return 1; |
198 | } | |
199 | return 0; | |
200 | } | |
201 | EXPORT_SYMBOL(_raw_read_trylock_retry); | |
202 | ||
bbae71bf MS |
203 | #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES |
204 | ||
205 | void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev) | |
206 | { | |
207 | unsigned int owner, old; | |
208 | int count = spin_retry; | |
209 | ||
210 | owner = 0; | |
211 | while (1) { | |
212 | if (count-- <= 0) { | |
419123f9 | 213 | if (owner && cpu_is_preempted(~owner)) |
bbae71bf MS |
214 | smp_yield_cpu(~owner); |
215 | count = spin_retry; | |
216 | } | |
217 | old = ACCESS_ONCE(rw->lock); | |
218 | owner = ACCESS_ONCE(rw->owner); | |
e0af21c5 | 219 | smp_mb(); |
bbae71bf MS |
220 | if ((int) old >= 0) { |
221 | prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR); | |
222 | old = prev; | |
223 | } | |
224 | if ((old & 0x7fffffff) == 0 && (int) prev >= 0) | |
225 | break; | |
2c72a44e MS |
226 | if (MACHINE_HAS_CAD) |
227 | _raw_compare_and_delay(&rw->lock, old); | |
bbae71bf MS |
228 | } |
229 | } | |
230 | EXPORT_SYMBOL(_raw_write_lock_wait); | |
231 | ||
232 | #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ | |
233 | ||
fb3a6bbc | 234 | void _raw_write_lock_wait(arch_rwlock_t *rw) |
951f22d5 | 235 | { |
94232a43 | 236 | unsigned int owner, old, prev; |
951f22d5 MS |
237 | int count = spin_retry; |
238 | ||
94232a43 | 239 | prev = 0x80000000; |
d59b93da | 240 | owner = 0; |
951f22d5 MS |
241 | while (1) { |
242 | if (count-- <= 0) { | |
419123f9 | 243 | if (owner && cpu_is_preempted(~owner)) |
d59b93da | 244 | smp_yield_cpu(~owner); |
951f22d5 MS |
245 | count = spin_retry; |
246 | } | |
bae8f567 | 247 | old = ACCESS_ONCE(rw->lock); |
d59b93da | 248 | owner = ACCESS_ONCE(rw->owner); |
94232a43 MS |
249 | if ((int) old >= 0 && |
250 | _raw_compare_and_swap(&rw->lock, old, old | 0x80000000)) | |
251 | prev = old; | |
252 | else | |
e0af21c5 | 253 | smp_mb(); |
94232a43 MS |
254 | if ((old & 0x7fffffff) == 0 && (int) prev >= 0) |
255 | break; | |
2c72a44e MS |
256 | if (MACHINE_HAS_CAD) |
257 | _raw_compare_and_delay(&rw->lock, old); | |
951f22d5 MS |
258 | } |
259 | } | |
260 | EXPORT_SYMBOL(_raw_write_lock_wait); | |
261 | ||
bbae71bf MS |
262 | #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ |
263 | ||
fb3a6bbc | 264 | int _raw_write_trylock_retry(arch_rwlock_t *rw) |
951f22d5 | 265 | { |
bae8f567 | 266 | unsigned int old; |
951f22d5 MS |
267 | int count = spin_retry; |
268 | ||
269 | while (count-- > 0) { | |
bae8f567 | 270 | old = ACCESS_ONCE(rw->lock); |
2c72a44e MS |
271 | if (old) { |
272 | if (MACHINE_HAS_CAD) | |
273 | _raw_compare_and_delay(&rw->lock, old); | |
96567161 | 274 | continue; |
2c72a44e | 275 | } |
5b3f683e | 276 | if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000)) |
951f22d5 MS |
277 | return 1; |
278 | } | |
279 | return 0; | |
280 | } | |
281 | EXPORT_SYMBOL(_raw_write_trylock_retry); | |
d59b93da MS |
282 | |
283 | void arch_lock_relax(unsigned int cpu) | |
284 | { | |
285 | if (!cpu) | |
286 | return; | |
419123f9 | 287 | if (MACHINE_IS_LPAR && !cpu_is_preempted(~cpu)) |
d59b93da MS |
288 | return; |
289 | smp_yield_cpu(~cpu); | |
290 | } | |
291 | EXPORT_SYMBOL(arch_lock_relax); |