Commit | Line | Data |
---|---|---|
620a6fd1 | 1 | /* |
f30c2269 | 2 | * include/asm-generic/mutex-xchg.h |
620a6fd1 IM |
3 | * |
4 | * Generic implementation of the mutex fastpath, based on xchg(). | |
5 | * | |
e358c1a2 | 6 | * NOTE: An xchg based implementation might be less optimal than an atomic |
620a6fd1 IM |
7 | * decrement/increment based implementation. If your architecture |
8 | * has a reasonable atomic dec/inc then you should probably use | |
9 | * asm-generic/mutex-dec.h instead, or you could open-code an | |
10 | * optimized version in asm/mutex.h. | |
11 | */ | |
12 | #ifndef _ASM_GENERIC_MUTEX_XCHG_H | |
13 | #define _ASM_GENERIC_MUTEX_XCHG_H | |
14 | ||
15 | /** | |
16 | * __mutex_fastpath_lock - try to take the lock by moving the count | |
17 | * from 1 to a 0 value | |
18 | * @count: pointer of type atomic_t | |
19 | * @fail_fn: function to call if the original value was not 1 | |
20 | * | |
21 | * Change the count from 1 to a value lower than 1, and call <fail_fn> if it | |
22 | * wasn't 1 originally. This function MUST leave the value lower than 1 | |
23 | * even when the "1" assertion wasn't true. | |
24 | */ | |
e358c1a2 | 25 | static inline void |
144b2a91 | 26 | __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *)) |
e358c1a2 NP |
27 | { |
28 | if (unlikely(atomic_xchg(count, 0) != 1)) | |
0bce9c46 WD |
29 | /* |
30 | * We failed to acquire the lock, so mark it contended | |
31 | * to ensure that any waiting tasks are woken up by the | |
32 | * unlock slow path. | |
33 | */ | |
81a43ada | 34 | if (likely(atomic_xchg_acquire(count, -1) != 1)) |
0bce9c46 | 35 | fail_fn(count); |
e358c1a2 | 36 | } |
620a6fd1 IM |
37 | |
38 | /** | |
39 | * __mutex_fastpath_lock_retval - try to take the lock by moving the count | |
40 | * from 1 to a 0 value | |
41 | * @count: pointer of type atomic_t | |
620a6fd1 | 42 | * |
a41b56ef ML |
43 | * Change the count from 1 to a value lower than 1. This function returns 0 |
44 | * if the fastpath succeeds, or -1 otherwise. | |
620a6fd1 IM |
45 | */ |
46 | static inline int | |
a41b56ef | 47 | __mutex_fastpath_lock_retval(atomic_t *count) |
620a6fd1 | 48 | { |
81a43ada | 49 | if (unlikely(atomic_xchg_acquire(count, 0) != 1)) |
0bce9c46 | 50 | if (likely(atomic_xchg(count, -1) != 1)) |
a41b56ef | 51 | return -1; |
a8ddac7e | 52 | return 0; |
620a6fd1 IM |
53 | } |
54 | ||
55 | /** | |
56 | * __mutex_fastpath_unlock - try to promote the mutex from 0 to 1 | |
57 | * @count: pointer of type atomic_t | |
58 | * @fail_fn: function to call if the original value was not 0 | |
59 | * | |
60 | * try to promote the mutex from 0 to 1. if it wasn't 0, call <function> | |
61 | * In the failure case, this function is allowed to either set the value to | |
62 | * 1, or to set it to a value lower than one. | |
63 | * If the implementation sets it to a value of lower than one, the | |
64 | * __mutex_slowpath_needs_to_unlock() macro needs to return 1, it needs | |
65 | * to return 0 otherwise. | |
66 | */ | |
e358c1a2 | 67 | static inline void |
144b2a91 | 68 | __mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *)) |
e358c1a2 | 69 | { |
81a43ada | 70 | if (unlikely(atomic_xchg_release(count, 1) != 0)) |
e358c1a2 NP |
71 | fail_fn(count); |
72 | } | |
620a6fd1 IM |
73 | |
74 | #define __mutex_slowpath_needs_to_unlock() 0 | |
75 | ||
76 | /** | |
77 | * __mutex_fastpath_trylock - try to acquire the mutex, without waiting | |
78 | * | |
79 | * @count: pointer of type atomic_t | |
80 | * @fail_fn: spinlock based trylock implementation | |
81 | * | |
82 | * Change the count from 1 to a value lower than 1, and return 0 (failure) | |
83 | * if it wasn't 1 originally, or return 1 (success) otherwise. This function | |
84 | * MUST leave the value lower than 1 even when the "1" assertion wasn't true. | |
85 | * Additionally, if the value was < 0 originally, this function must not leave | |
86 | * it to 0 on failure. | |
87 | * | |
88 | * If the architecture has no effective trylock variant, it should call the | |
89 | * <fail_fn> spinlock-based trylock variant unconditionally. | |
90 | */ | |
91 | static inline int | |
92 | __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) | |
93 | { | |
81a43ada | 94 | int prev = atomic_xchg_acquire(count, 0); |
620a6fd1 IM |
95 | |
96 | if (unlikely(prev < 0)) { | |
97 | /* | |
98 | * The lock was marked contended so we must restore that | |
99 | * state. If while doing so we get back a prev value of 1 | |
100 | * then we just own it. | |
101 | * | |
102 | * [ In the rare case of the mutex going to 1, to 0, to -1 | |
103 | * and then back to 0 in this few-instructions window, | |
104 | * this has the potential to trigger the slowpath for the | |
105 | * owner's unlock path needlessly, but that's not a problem | |
106 | * in practice. ] | |
107 | */ | |
81a43ada | 108 | prev = atomic_xchg_acquire(count, prev); |
620a6fd1 IM |
109 | if (prev < 0) |
110 | prev = 0; | |
111 | } | |
620a6fd1 IM |
112 | |
113 | return prev; | |
114 | } | |
115 | ||
116 | #endif |