Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Copyright (2004) Linus Torvalds | |
3 | * | |
4 | * Author: Zwane Mwaikambo <zwane@fsmlabs.com> | |
5 | * | |
fb1c8f93 IM |
6 | * Copyright (2004, 2005) Ingo Molnar |
7 | * | |
8 | * This file contains the spinlock/rwlock implementations for the | |
9 | * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them) | |
0cb91a22 AK |
10 | * |
11 | * Note that some architectures have special knowledge about the | |
12 | * stack frames of these functions in their profile_pc. If you | |
13 | * change anything significant here that could change the stack | |
14 | * frame contact the architecture maintainers. | |
1da177e4 LT |
15 | */ |
16 | ||
1da177e4 LT |
17 | #include <linux/linkage.h> |
18 | #include <linux/preempt.h> | |
19 | #include <linux/spinlock.h> | |
20 | #include <linux/interrupt.h> | |
8a25d5de | 21 | #include <linux/debug_locks.h> |
9984de1a | 22 | #include <linux/export.h> |
1da177e4 | 23 | |
8e13c7b7 TG |
24 | /* |
25 | * If lockdep is enabled then we use the non-preemption spin-ops | |
26 | * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are | |
27 | * not re-enabled during lock-acquire (which the preempt-spin-ops do): | |
28 | */ | |
29 | #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) | |
30 | /* | |
31 | * The __lock_function inlines are taken from | |
32 | * include/linux/spinlock_api_smp.h | |
33 | */ | |
34 | #else | |
c2f21ce2 TG |
35 | #define raw_read_can_lock(l) read_can_lock(l) |
36 | #define raw_write_can_lock(l) write_can_lock(l) | |
c14c338c WD |
37 | |
38 | /* | |
39 | * Some architectures can relax in favour of the CPU owning the lock. | |
40 | */ | |
41 | #ifndef arch_read_relax | |
42 | # define arch_read_relax(l) cpu_relax() | |
43 | #endif | |
44 | #ifndef arch_write_relax | |
45 | # define arch_write_relax(l) cpu_relax() | |
46 | #endif | |
47 | #ifndef arch_spin_relax | |
48 | # define arch_spin_relax(l) cpu_relax() | |
49 | #endif | |
50 | ||
8e13c7b7 TG |
51 | /* |
52 | * We build the __lock_function inlines here. They are too large for | |
53 | * inlining all over the place, but here is only one user per function | |
54 | * which embedds them into the calling _lock_function below. | |
55 | * | |
56 | * This could be a long-held lock. We both prepare to spin for a long | |
57 | * time (making _this_ CPU preemptable if possible), and we also signal | |
58 | * towards that other CPU that it should break the lock ASAP. | |
59 | */ | |
60 | #define BUILD_LOCK_OPS(op, locktype) \ | |
9c1721aa | 61 | void __lockfunc __raw_##op##_lock(locktype##_t *lock) \ |
8e13c7b7 TG |
62 | { \ |
63 | for (;;) { \ | |
64 | preempt_disable(); \ | |
9828ea9d | 65 | if (likely(do_raw_##op##_trylock(lock))) \ |
8e13c7b7 TG |
66 | break; \ |
67 | preempt_enable(); \ | |
68 | \ | |
69 | if (!(lock)->break_lock) \ | |
70 | (lock)->break_lock = 1; \ | |
c2f21ce2 | 71 | while (!raw_##op##_can_lock(lock) && (lock)->break_lock)\ |
0199c4e6 | 72 | arch_##op##_relax(&lock->raw_lock); \ |
8e13c7b7 TG |
73 | } \ |
74 | (lock)->break_lock = 0; \ | |
75 | } \ | |
76 | \ | |
9c1721aa | 77 | unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \ |
8e13c7b7 TG |
78 | { \ |
79 | unsigned long flags; \ | |
80 | \ | |
81 | for (;;) { \ | |
82 | preempt_disable(); \ | |
83 | local_irq_save(flags); \ | |
9828ea9d | 84 | if (likely(do_raw_##op##_trylock(lock))) \ |
8e13c7b7 TG |
85 | break; \ |
86 | local_irq_restore(flags); \ | |
87 | preempt_enable(); \ | |
88 | \ | |
89 | if (!(lock)->break_lock) \ | |
90 | (lock)->break_lock = 1; \ | |
c2f21ce2 | 91 | while (!raw_##op##_can_lock(lock) && (lock)->break_lock)\ |
0199c4e6 | 92 | arch_##op##_relax(&lock->raw_lock); \ |
8e13c7b7 TG |
93 | } \ |
94 | (lock)->break_lock = 0; \ | |
95 | return flags; \ | |
96 | } \ | |
97 | \ | |
9c1721aa | 98 | void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock) \ |
8e13c7b7 | 99 | { \ |
9c1721aa | 100 | _raw_##op##_lock_irqsave(lock); \ |
8e13c7b7 TG |
101 | } \ |
102 | \ | |
9c1721aa | 103 | void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ |
8e13c7b7 TG |
104 | { \ |
105 | unsigned long flags; \ | |
106 | \ | |
107 | /* */ \ | |
108 | /* Careful: we must exclude softirqs too, hence the */ \ | |
109 | /* irq-disabling. We use the generic preemption-aware */ \ | |
110 | /* function: */ \ | |
111 | /**/ \ | |
9c1721aa | 112 | flags = _raw_##op##_lock_irqsave(lock); \ |
8e13c7b7 TG |
113 | local_bh_disable(); \ |
114 | local_irq_restore(flags); \ | |
115 | } \ | |
116 | ||
117 | /* | |
118 | * Build preemption-friendly versions of the following | |
119 | * lock-spinning functions: | |
120 | * | |
121 | * __[spin|read|write]_lock() | |
122 | * __[spin|read|write]_lock_irq() | |
123 | * __[spin|read|write]_lock_irqsave() | |
124 | * __[spin|read|write]_lock_bh() | |
125 | */ | |
c2f21ce2 | 126 | BUILD_LOCK_OPS(spin, raw_spinlock); |
8e13c7b7 TG |
127 | BUILD_LOCK_OPS(read, rwlock); |
128 | BUILD_LOCK_OPS(write, rwlock); | |
129 | ||
130 | #endif | |
131 | ||
6beb0009 | 132 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK |
9c1721aa | 133 | int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) |
1da177e4 | 134 | { |
9c1721aa | 135 | return __raw_spin_trylock(lock); |
1da177e4 | 136 | } |
9c1721aa | 137 | EXPORT_SYMBOL(_raw_spin_trylock); |
892a7c67 | 138 | #endif |
1da177e4 | 139 | |
b7b40ade | 140 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH |
9c1721aa | 141 | int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) |
1da177e4 | 142 | { |
9c1721aa | 143 | return __raw_spin_trylock_bh(lock); |
1da177e4 | 144 | } |
9c1721aa | 145 | EXPORT_SYMBOL(_raw_spin_trylock_bh); |
892a7c67 | 146 | #endif |
1da177e4 | 147 | |
b7b40ade | 148 | #ifndef CONFIG_INLINE_SPIN_LOCK |
9c1721aa | 149 | void __lockfunc _raw_spin_lock(raw_spinlock_t *lock) |
1da177e4 | 150 | { |
9c1721aa | 151 | __raw_spin_lock(lock); |
1da177e4 | 152 | } |
9c1721aa | 153 | EXPORT_SYMBOL(_raw_spin_lock); |
892a7c67 | 154 | #endif |
1da177e4 | 155 | |
6beb0009 | 156 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE |
9c1721aa | 157 | unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) |
1da177e4 | 158 | { |
9c1721aa | 159 | return __raw_spin_lock_irqsave(lock); |
1da177e4 | 160 | } |
9c1721aa | 161 | EXPORT_SYMBOL(_raw_spin_lock_irqsave); |
892a7c67 | 162 | #endif |
1da177e4 | 163 | |
6beb0009 | 164 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ |
9c1721aa | 165 | void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) |
1da177e4 | 166 | { |
9c1721aa | 167 | __raw_spin_lock_irq(lock); |
1da177e4 | 168 | } |
9c1721aa | 169 | EXPORT_SYMBOL(_raw_spin_lock_irq); |
892a7c67 | 170 | #endif |
1da177e4 | 171 | |
6beb0009 | 172 | #ifndef CONFIG_INLINE_SPIN_LOCK_BH |
9c1721aa | 173 | void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) |
1da177e4 | 174 | { |
9c1721aa | 175 | __raw_spin_lock_bh(lock); |
1da177e4 | 176 | } |
9c1721aa | 177 | EXPORT_SYMBOL(_raw_spin_lock_bh); |
892a7c67 | 178 | #endif |
1da177e4 | 179 | |
e335e3eb | 180 | #ifdef CONFIG_UNINLINE_SPIN_UNLOCK |
9c1721aa | 181 | void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) |
1da177e4 | 182 | { |
9c1721aa | 183 | __raw_spin_unlock(lock); |
1da177e4 | 184 | } |
9c1721aa | 185 | EXPORT_SYMBOL(_raw_spin_unlock); |
892a7c67 | 186 | #endif |
1da177e4 | 187 | |
b7b40ade | 188 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE |
9c1721aa | 189 | void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) |
1da177e4 | 190 | { |
9c1721aa | 191 | __raw_spin_unlock_irqrestore(lock, flags); |
1da177e4 | 192 | } |
9c1721aa | 193 | EXPORT_SYMBOL(_raw_spin_unlock_irqrestore); |
892a7c67 | 194 | #endif |
1da177e4 | 195 | |
b7b40ade | 196 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ |
9c1721aa | 197 | void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) |
1da177e4 | 198 | { |
9c1721aa | 199 | __raw_spin_unlock_irq(lock); |
1da177e4 | 200 | } |
9c1721aa | 201 | EXPORT_SYMBOL(_raw_spin_unlock_irq); |
892a7c67 | 202 | #endif |
1da177e4 | 203 | |
b7b40ade | 204 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH |
9c1721aa | 205 | void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) |
1da177e4 | 206 | { |
9c1721aa | 207 | __raw_spin_unlock_bh(lock); |
1da177e4 | 208 | } |
9c1721aa | 209 | EXPORT_SYMBOL(_raw_spin_unlock_bh); |
892a7c67 | 210 | #endif |
1da177e4 | 211 | |
b7b40ade | 212 | #ifndef CONFIG_INLINE_READ_TRYLOCK |
9c1721aa | 213 | int __lockfunc _raw_read_trylock(rwlock_t *lock) |
1da177e4 | 214 | { |
9c1721aa | 215 | return __raw_read_trylock(lock); |
1da177e4 | 216 | } |
9c1721aa | 217 | EXPORT_SYMBOL(_raw_read_trylock); |
892a7c67 | 218 | #endif |
1da177e4 | 219 | |
b7b40ade | 220 | #ifndef CONFIG_INLINE_READ_LOCK |
9c1721aa | 221 | void __lockfunc _raw_read_lock(rwlock_t *lock) |
1da177e4 | 222 | { |
9c1721aa | 223 | __raw_read_lock(lock); |
1da177e4 | 224 | } |
9c1721aa | 225 | EXPORT_SYMBOL(_raw_read_lock); |
892a7c67 | 226 | #endif |
1da177e4 | 227 | |
b7b40ade | 228 | #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE |
9c1721aa | 229 | unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) |
1da177e4 | 230 | { |
9c1721aa | 231 | return __raw_read_lock_irqsave(lock); |
1da177e4 | 232 | } |
9c1721aa | 233 | EXPORT_SYMBOL(_raw_read_lock_irqsave); |
892a7c67 | 234 | #endif |
1da177e4 | 235 | |
b7b40ade | 236 | #ifndef CONFIG_INLINE_READ_LOCK_IRQ |
9c1721aa | 237 | void __lockfunc _raw_read_lock_irq(rwlock_t *lock) |
1da177e4 | 238 | { |
9c1721aa | 239 | __raw_read_lock_irq(lock); |
1da177e4 | 240 | } |
9c1721aa | 241 | EXPORT_SYMBOL(_raw_read_lock_irq); |
892a7c67 | 242 | #endif |
1da177e4 | 243 | |
b7b40ade | 244 | #ifndef CONFIG_INLINE_READ_LOCK_BH |
9c1721aa | 245 | void __lockfunc _raw_read_lock_bh(rwlock_t *lock) |
1da177e4 | 246 | { |
9c1721aa | 247 | __raw_read_lock_bh(lock); |
1da177e4 | 248 | } |
9c1721aa | 249 | EXPORT_SYMBOL(_raw_read_lock_bh); |
892a7c67 | 250 | #endif |
1da177e4 | 251 | |
6beb0009 | 252 | #ifndef CONFIG_INLINE_READ_UNLOCK |
9c1721aa | 253 | void __lockfunc _raw_read_unlock(rwlock_t *lock) |
1da177e4 | 254 | { |
9c1721aa | 255 | __raw_read_unlock(lock); |
1da177e4 | 256 | } |
9c1721aa | 257 | EXPORT_SYMBOL(_raw_read_unlock); |
892a7c67 | 258 | #endif |
1da177e4 | 259 | |
6beb0009 | 260 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE |
9c1721aa | 261 | void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) |
1da177e4 | 262 | { |
9c1721aa | 263 | __raw_read_unlock_irqrestore(lock, flags); |
1da177e4 | 264 | } |
9c1721aa | 265 | EXPORT_SYMBOL(_raw_read_unlock_irqrestore); |
892a7c67 | 266 | #endif |
1da177e4 | 267 | |
6beb0009 | 268 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ |
9c1721aa | 269 | void __lockfunc _raw_read_unlock_irq(rwlock_t *lock) |
1da177e4 | 270 | { |
9c1721aa | 271 | __raw_read_unlock_irq(lock); |
1da177e4 | 272 | } |
9c1721aa | 273 | EXPORT_SYMBOL(_raw_read_unlock_irq); |
892a7c67 | 274 | #endif |
1da177e4 | 275 | |
6beb0009 | 276 | #ifndef CONFIG_INLINE_READ_UNLOCK_BH |
9c1721aa | 277 | void __lockfunc _raw_read_unlock_bh(rwlock_t *lock) |
1da177e4 | 278 | { |
9c1721aa | 279 | __raw_read_unlock_bh(lock); |
1da177e4 | 280 | } |
9c1721aa | 281 | EXPORT_SYMBOL(_raw_read_unlock_bh); |
892a7c67 | 282 | #endif |
1da177e4 | 283 | |
b7b40ade | 284 | #ifndef CONFIG_INLINE_WRITE_TRYLOCK |
9c1721aa | 285 | int __lockfunc _raw_write_trylock(rwlock_t *lock) |
b7b40ade | 286 | { |
9c1721aa | 287 | return __raw_write_trylock(lock); |
b7b40ade | 288 | } |
9c1721aa | 289 | EXPORT_SYMBOL(_raw_write_trylock); |
b7b40ade TG |
290 | #endif |
291 | ||
292 | #ifndef CONFIG_INLINE_WRITE_LOCK | |
9c1721aa | 293 | void __lockfunc _raw_write_lock(rwlock_t *lock) |
b7b40ade | 294 | { |
9c1721aa | 295 | __raw_write_lock(lock); |
b7b40ade | 296 | } |
9c1721aa | 297 | EXPORT_SYMBOL(_raw_write_lock); |
b7b40ade TG |
298 | #endif |
299 | ||
300 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE | |
9c1721aa | 301 | unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) |
b7b40ade | 302 | { |
9c1721aa | 303 | return __raw_write_lock_irqsave(lock); |
b7b40ade | 304 | } |
9c1721aa | 305 | EXPORT_SYMBOL(_raw_write_lock_irqsave); |
b7b40ade TG |
306 | #endif |
307 | ||
308 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ | |
9c1721aa | 309 | void __lockfunc _raw_write_lock_irq(rwlock_t *lock) |
b7b40ade | 310 | { |
9c1721aa | 311 | __raw_write_lock_irq(lock); |
b7b40ade | 312 | } |
9c1721aa | 313 | EXPORT_SYMBOL(_raw_write_lock_irq); |
b7b40ade TG |
314 | #endif |
315 | ||
316 | #ifndef CONFIG_INLINE_WRITE_LOCK_BH | |
9c1721aa | 317 | void __lockfunc _raw_write_lock_bh(rwlock_t *lock) |
b7b40ade | 318 | { |
9c1721aa | 319 | __raw_write_lock_bh(lock); |
b7b40ade | 320 | } |
9c1721aa | 321 | EXPORT_SYMBOL(_raw_write_lock_bh); |
b7b40ade TG |
322 | #endif |
323 | ||
324 | #ifndef CONFIG_INLINE_WRITE_UNLOCK | |
9c1721aa | 325 | void __lockfunc _raw_write_unlock(rwlock_t *lock) |
b7b40ade | 326 | { |
9c1721aa | 327 | __raw_write_unlock(lock); |
b7b40ade | 328 | } |
9c1721aa | 329 | EXPORT_SYMBOL(_raw_write_unlock); |
b7b40ade TG |
330 | #endif |
331 | ||
6beb0009 | 332 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE |
9c1721aa | 333 | void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) |
1da177e4 | 334 | { |
9c1721aa | 335 | __raw_write_unlock_irqrestore(lock, flags); |
1da177e4 | 336 | } |
9c1721aa | 337 | EXPORT_SYMBOL(_raw_write_unlock_irqrestore); |
892a7c67 | 338 | #endif |
1da177e4 | 339 | |
6beb0009 | 340 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ |
9c1721aa | 341 | void __lockfunc _raw_write_unlock_irq(rwlock_t *lock) |
1da177e4 | 342 | { |
9c1721aa | 343 | __raw_write_unlock_irq(lock); |
1da177e4 | 344 | } |
9c1721aa | 345 | EXPORT_SYMBOL(_raw_write_unlock_irq); |
892a7c67 | 346 | #endif |
1da177e4 | 347 | |
6beb0009 | 348 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH |
9c1721aa | 349 | void __lockfunc _raw_write_unlock_bh(rwlock_t *lock) |
1da177e4 | 350 | { |
9c1721aa | 351 | __raw_write_unlock_bh(lock); |
1da177e4 | 352 | } |
9c1721aa | 353 | EXPORT_SYMBOL(_raw_write_unlock_bh); |
892a7c67 | 354 | #endif |
1da177e4 | 355 | |
b7b40ade TG |
356 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
357 | ||
9c1721aa | 358 | void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass) |
1da177e4 | 359 | { |
b7b40ade TG |
360 | preempt_disable(); |
361 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | |
9828ea9d | 362 | LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); |
1da177e4 | 363 | } |
9c1721aa | 364 | EXPORT_SYMBOL(_raw_spin_lock_nested); |
b7b40ade | 365 | |
113948d8 TG |
366 | void __lockfunc _raw_spin_lock_bh_nested(raw_spinlock_t *lock, int subclass) |
367 | { | |
368 | __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); | |
369 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | |
370 | LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); | |
371 | } | |
372 | EXPORT_SYMBOL(_raw_spin_lock_bh_nested); | |
373 | ||
9c1721aa | 374 | unsigned long __lockfunc _raw_spin_lock_irqsave_nested(raw_spinlock_t *lock, |
b7b40ade TG |
375 | int subclass) |
376 | { | |
377 | unsigned long flags; | |
378 | ||
379 | local_irq_save(flags); | |
380 | preempt_disable(); | |
381 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | |
9828ea9d TG |
382 | LOCK_CONTENDED_FLAGS(lock, do_raw_spin_trylock, do_raw_spin_lock, |
383 | do_raw_spin_lock_flags, &flags); | |
b7b40ade TG |
384 | return flags; |
385 | } | |
9c1721aa | 386 | EXPORT_SYMBOL(_raw_spin_lock_irqsave_nested); |
b7b40ade | 387 | |
9c1721aa | 388 | void __lockfunc _raw_spin_lock_nest_lock(raw_spinlock_t *lock, |
b7b40ade TG |
389 | struct lockdep_map *nest_lock) |
390 | { | |
391 | preempt_disable(); | |
392 | spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); | |
9828ea9d | 393 | LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); |
b7b40ade | 394 | } |
9c1721aa | 395 | EXPORT_SYMBOL(_raw_spin_lock_nest_lock); |
b7b40ade | 396 | |
892a7c67 | 397 | #endif |
1da177e4 | 398 | |
0764d23c | 399 | notrace int in_lock_functions(unsigned long addr) |
1da177e4 LT |
400 | { |
401 | /* Linker adds these: start and end of __lockfunc functions */ | |
402 | extern char __lock_text_start[], __lock_text_end[]; | |
403 | ||
404 | return addr >= (unsigned long)__lock_text_start | |
405 | && addr < (unsigned long)__lock_text_end; | |
406 | } | |
407 | EXPORT_SYMBOL(in_lock_functions); |