Commit | Line | Data |
---|---|---|
951f22d5 | 1 | /* |
951f22d5 MS |
2 | * Out of line spinlock code. |
3 | * | |
a53c8fab | 4 | * Copyright IBM Corp. 2004, 2006 |
951f22d5 MS |
5 | * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) |
6 | */ | |
7 | ||
8 | #include <linux/types.h> | |
9 | #include <linux/module.h> | |
10 | #include <linux/spinlock.h> | |
11 | #include <linux/init.h> | |
8b646bd7 | 12 | #include <linux/smp.h> |
951f22d5 MS |
13 | #include <asm/io.h> |
14 | ||
2c72a44e MS |
15 | int spin_retry = -1; |
16 | ||
17 | static int __init spin_retry_init(void) | |
18 | { | |
19 | if (spin_retry < 0) | |
20 | spin_retry = MACHINE_HAS_CAD ? 10 : 1000; | |
21 | return 0; | |
22 | } | |
23 | early_initcall(spin_retry_init); | |
951f22d5 MS |
24 | |
25 | /** | |
26 | * spin_retry= parameter | |
27 | */ | |
28 | static int __init spin_retry_setup(char *str) | |
29 | { | |
30 | spin_retry = simple_strtoul(str, &str, 0); | |
31 | return 1; | |
32 | } | |
33 | __setup("spin_retry=", spin_retry_setup); | |
34 | ||
2c72a44e MS |
35 | static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old) |
36 | { | |
37 | asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock)); | |
38 | } | |
39 | ||
0199c4e6 | 40 | void arch_spin_lock_wait(arch_spinlock_t *lp) |
951f22d5 | 41 | { |
6c8cd5bb | 42 | unsigned int cpu = SPINLOCK_LOCKVAL; |
59b69787 | 43 | unsigned int owner; |
2e4006b3 | 44 | int count; |
951f22d5 MS |
45 | |
46 | while (1) { | |
470ada6b MS |
47 | owner = ACCESS_ONCE(lp->lock); |
48 | /* Try to get the lock if it is free. */ | |
49 | if (!owner) { | |
50 | if (_raw_compare_and_swap(&lp->lock, 0, cpu)) | |
51 | return; | |
52 | continue; | |
951f22d5 | 53 | } |
470ada6b MS |
54 | /* Check if the lock owner is running. */ |
55 | if (!smp_vcpu_scheduled(~owner)) { | |
56 | smp_yield_cpu(~owner); | |
57 | continue; | |
58 | } | |
59 | /* Loop for a while on the lock value. */ | |
60 | count = spin_retry; | |
61 | do { | |
2c72a44e MS |
62 | if (MACHINE_HAS_CAD) |
63 | _raw_compare_and_delay(&lp->lock, owner); | |
470ada6b MS |
64 | owner = ACCESS_ONCE(lp->lock); |
65 | } while (owner && count-- > 0); | |
66 | if (!owner) | |
67 | continue; | |
68 | /* | |
69 | * For multiple layers of hypervisors, e.g. z/VM + LPAR | |
70 | * yield the CPU if the lock is still unavailable. | |
71 | */ | |
72 | if (!MACHINE_IS_LPAR) | |
8b646bd7 | 73 | smp_yield_cpu(~owner); |
951f22d5 MS |
74 | } |
75 | } | |
0199c4e6 | 76 | EXPORT_SYMBOL(arch_spin_lock_wait); |
951f22d5 | 77 | |
0199c4e6 | 78 | void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) |
894cdde2 | 79 | { |
6c8cd5bb | 80 | unsigned int cpu = SPINLOCK_LOCKVAL; |
59b69787 | 81 | unsigned int owner; |
2e4006b3 | 82 | int count; |
894cdde2 HH |
83 | |
84 | local_irq_restore(flags); | |
85 | while (1) { | |
470ada6b MS |
86 | owner = ACCESS_ONCE(lp->lock); |
87 | /* Try to get the lock if it is free. */ | |
88 | if (!owner) { | |
89 | local_irq_disable(); | |
90 | if (_raw_compare_and_swap(&lp->lock, 0, cpu)) | |
91 | return; | |
92 | local_irq_restore(flags); | |
93 | } | |
94 | /* Check if the lock owner is running. */ | |
95 | if (!smp_vcpu_scheduled(~owner)) { | |
96 | smp_yield_cpu(~owner); | |
97 | continue; | |
894cdde2 | 98 | } |
470ada6b MS |
99 | /* Loop for a while on the lock value. */ |
100 | count = spin_retry; | |
101 | do { | |
2c72a44e MS |
102 | if (MACHINE_HAS_CAD) |
103 | _raw_compare_and_delay(&lp->lock, owner); | |
470ada6b MS |
104 | owner = ACCESS_ONCE(lp->lock); |
105 | } while (owner && count-- > 0); | |
106 | if (!owner) | |
107 | continue; | |
108 | /* | |
109 | * For multiple layers of hypervisors, e.g. z/VM + LPAR | |
110 | * yield the CPU if the lock is still unavailable. | |
111 | */ | |
112 | if (!MACHINE_IS_LPAR) | |
8b646bd7 | 113 | smp_yield_cpu(~owner); |
894cdde2 HH |
114 | } |
115 | } | |
0199c4e6 | 116 | EXPORT_SYMBOL(arch_spin_lock_wait_flags); |
894cdde2 | 117 | |
0199c4e6 | 118 | int arch_spin_trylock_retry(arch_spinlock_t *lp) |
951f22d5 | 119 | { |
2c72a44e MS |
120 | unsigned int cpu = SPINLOCK_LOCKVAL; |
121 | unsigned int owner; | |
3c1fcfe2 | 122 | int count; |
951f22d5 | 123 | |
2c72a44e MS |
124 | for (count = spin_retry; count > 0; count--) { |
125 | owner = ACCESS_ONCE(lp->lock); | |
126 | /* Try to get the lock if it is free. */ | |
127 | if (!owner) { | |
128 | if (_raw_compare_and_swap(&lp->lock, 0, cpu)) | |
129 | return 1; | |
130 | } else if (MACHINE_HAS_CAD) | |
131 | _raw_compare_and_delay(&lp->lock, owner); | |
132 | } | |
951f22d5 MS |
133 | return 0; |
134 | } | |
0199c4e6 | 135 | EXPORT_SYMBOL(arch_spin_trylock_retry); |
951f22d5 | 136 | |
fb3a6bbc | 137 | void _raw_read_lock_wait(arch_rwlock_t *rw) |
951f22d5 | 138 | { |
d59b93da | 139 | unsigned int owner, old; |
951f22d5 MS |
140 | int count = spin_retry; |
141 | ||
bbae71bf MS |
142 | #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES |
143 | __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD); | |
144 | #endif | |
d59b93da | 145 | owner = 0; |
951f22d5 MS |
146 | while (1) { |
147 | if (count-- <= 0) { | |
d59b93da MS |
148 | if (owner && !smp_vcpu_scheduled(~owner)) |
149 | smp_yield_cpu(~owner); | |
951f22d5 MS |
150 | count = spin_retry; |
151 | } | |
bae8f567 | 152 | old = ACCESS_ONCE(rw->lock); |
d59b93da | 153 | owner = ACCESS_ONCE(rw->owner); |
2c72a44e MS |
154 | if ((int) old < 0) { |
155 | if (MACHINE_HAS_CAD) | |
156 | _raw_compare_and_delay(&rw->lock, old); | |
96567161 | 157 | continue; |
2c72a44e | 158 | } |
5b3f683e | 159 | if (_raw_compare_and_swap(&rw->lock, old, old + 1)) |
951f22d5 MS |
160 | return; |
161 | } | |
162 | } | |
163 | EXPORT_SYMBOL(_raw_read_lock_wait); | |
164 | ||
fb3a6bbc | 165 | int _raw_read_trylock_retry(arch_rwlock_t *rw) |
951f22d5 MS |
166 | { |
167 | unsigned int old; | |
168 | int count = spin_retry; | |
169 | ||
170 | while (count-- > 0) { | |
bae8f567 | 171 | old = ACCESS_ONCE(rw->lock); |
2c72a44e MS |
172 | if ((int) old < 0) { |
173 | if (MACHINE_HAS_CAD) | |
174 | _raw_compare_and_delay(&rw->lock, old); | |
96567161 | 175 | continue; |
2c72a44e | 176 | } |
5b3f683e | 177 | if (_raw_compare_and_swap(&rw->lock, old, old + 1)) |
951f22d5 MS |
178 | return 1; |
179 | } | |
180 | return 0; | |
181 | } | |
182 | EXPORT_SYMBOL(_raw_read_trylock_retry); | |
183 | ||
bbae71bf MS |
184 | #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES |
185 | ||
186 | void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev) | |
187 | { | |
188 | unsigned int owner, old; | |
189 | int count = spin_retry; | |
190 | ||
191 | owner = 0; | |
192 | while (1) { | |
193 | if (count-- <= 0) { | |
194 | if (owner && !smp_vcpu_scheduled(~owner)) | |
195 | smp_yield_cpu(~owner); | |
196 | count = spin_retry; | |
197 | } | |
198 | old = ACCESS_ONCE(rw->lock); | |
199 | owner = ACCESS_ONCE(rw->owner); | |
e0af21c5 | 200 | smp_mb(); |
bbae71bf MS |
201 | if ((int) old >= 0) { |
202 | prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR); | |
203 | old = prev; | |
204 | } | |
205 | if ((old & 0x7fffffff) == 0 && (int) prev >= 0) | |
206 | break; | |
2c72a44e MS |
207 | if (MACHINE_HAS_CAD) |
208 | _raw_compare_and_delay(&rw->lock, old); | |
bbae71bf MS |
209 | } |
210 | } | |
211 | EXPORT_SYMBOL(_raw_write_lock_wait); | |
212 | ||
213 | #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ | |
214 | ||
fb3a6bbc | 215 | void _raw_write_lock_wait(arch_rwlock_t *rw) |
951f22d5 | 216 | { |
94232a43 | 217 | unsigned int owner, old, prev; |
951f22d5 MS |
218 | int count = spin_retry; |
219 | ||
94232a43 | 220 | prev = 0x80000000; |
d59b93da | 221 | owner = 0; |
951f22d5 MS |
222 | while (1) { |
223 | if (count-- <= 0) { | |
d59b93da MS |
224 | if (owner && !smp_vcpu_scheduled(~owner)) |
225 | smp_yield_cpu(~owner); | |
951f22d5 MS |
226 | count = spin_retry; |
227 | } | |
bae8f567 | 228 | old = ACCESS_ONCE(rw->lock); |
d59b93da | 229 | owner = ACCESS_ONCE(rw->owner); |
94232a43 MS |
230 | if ((int) old >= 0 && |
231 | _raw_compare_and_swap(&rw->lock, old, old | 0x80000000)) | |
232 | prev = old; | |
233 | else | |
e0af21c5 | 234 | smp_mb(); |
94232a43 MS |
235 | if ((old & 0x7fffffff) == 0 && (int) prev >= 0) |
236 | break; | |
2c72a44e MS |
237 | if (MACHINE_HAS_CAD) |
238 | _raw_compare_and_delay(&rw->lock, old); | |
951f22d5 MS |
239 | } |
240 | } | |
241 | EXPORT_SYMBOL(_raw_write_lock_wait); | |
242 | ||
bbae71bf MS |
243 | #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ |
244 | ||
fb3a6bbc | 245 | int _raw_write_trylock_retry(arch_rwlock_t *rw) |
951f22d5 | 246 | { |
bae8f567 | 247 | unsigned int old; |
951f22d5 MS |
248 | int count = spin_retry; |
249 | ||
250 | while (count-- > 0) { | |
bae8f567 | 251 | old = ACCESS_ONCE(rw->lock); |
2c72a44e MS |
252 | if (old) { |
253 | if (MACHINE_HAS_CAD) | |
254 | _raw_compare_and_delay(&rw->lock, old); | |
96567161 | 255 | continue; |
2c72a44e | 256 | } |
5b3f683e | 257 | if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000)) |
951f22d5 MS |
258 | return 1; |
259 | } | |
260 | return 0; | |
261 | } | |
262 | EXPORT_SYMBOL(_raw_write_trylock_retry); | |
d59b93da MS |
263 | |
264 | void arch_lock_relax(unsigned int cpu) | |
265 | { | |
266 | if (!cpu) | |
267 | return; | |
268 | if (MACHINE_IS_LPAR && smp_vcpu_scheduled(~cpu)) | |
269 | return; | |
270 | smp_yield_cpu(~cpu); | |
271 | } | |
272 | EXPORT_SYMBOL(arch_lock_relax); |