Commit | Line | Data |
---|---|---|
9390ef0c | 1 | #include <linux/atomic.h> |
a1fd3e24 ON |
2 | #include <linux/rwsem.h> |
3 | #include <linux/percpu.h> | |
4 | #include <linux/wait.h> | |
8ebe3473 | 5 | #include <linux/lockdep.h> |
a1fd3e24 ON |
6 | #include <linux/percpu-rwsem.h> |
7 | #include <linux/rcupdate.h> | |
8 | #include <linux/sched.h> | |
9 | #include <linux/errno.h> | |
10 | ||
8ebe3473 ON |
11 | int __percpu_init_rwsem(struct percpu_rw_semaphore *brw, |
12 | const char *name, struct lock_class_key *rwsem_key) | |
a1fd3e24 ON |
13 | { |
14 | brw->fast_read_ctr = alloc_percpu(int); | |
15 | if (unlikely(!brw->fast_read_ctr)) | |
16 | return -ENOMEM; | |
17 | ||
8ebe3473 ON |
18 | /* ->rw_sem represents the whole percpu_rw_semaphore for lockdep */ |
19 | __init_rwsem(&brw->rw_sem, name, rwsem_key); | |
9390ef0c | 20 | atomic_set(&brw->write_ctr, 0); |
a1fd3e24 ON |
21 | atomic_set(&brw->slow_read_ctr, 0); |
22 | init_waitqueue_head(&brw->write_waitq); | |
23 | return 0; | |
24 | } | |
302707fd | 25 | EXPORT_SYMBOL_GPL(__percpu_init_rwsem); |
a1fd3e24 ON |
26 | |
27 | void percpu_free_rwsem(struct percpu_rw_semaphore *brw) | |
28 | { | |
95b19f68 ON |
29 | /* |
30 | * XXX: temporary kludge. The error path in alloc_super() | |
31 | * assumes that percpu_free_rwsem() is safe after kzalloc(). | |
32 | */ | |
33 | if (!brw->fast_read_ctr) | |
34 | return; | |
35 | ||
a1fd3e24 ON |
36 | free_percpu(brw->fast_read_ctr); |
37 | brw->fast_read_ctr = NULL; /* catch use after free bugs */ | |
38 | } | |
39 | ||
40 | /* | |
41 | * This is the fast-path for down_read/up_read, it only needs to ensure | |
9390ef0c | 42 | * there is no pending writer (atomic_read(write_ctr) == 0) and inc/dec the |
a1fd3e24 ON |
43 | * fast per-cpu counter. The writer uses synchronize_sched_expedited() to |
44 | * serialize with the preempt-disabled section below. | |
45 | * | |
46 | * The nontrivial part is that we should guarantee acquire/release semantics | |
47 | * in case when | |
48 | * | |
49 | * R_W: down_write() comes after up_read(), the writer should see all | |
50 | * changes done by the reader | |
51 | * or | |
52 | * W_R: down_read() comes after up_write(), the reader should see all | |
53 | * changes done by the writer | |
54 | * | |
55 | * If this helper fails the callers rely on the normal rw_semaphore and | |
56 | * atomic_dec_and_test(), so in this case we have the necessary barriers. | |
57 | * | |
9390ef0c | 58 | * But if it succeeds we do not have any barriers, atomic_read(write_ctr) or |
a1fd3e24 ON |
59 | * __this_cpu_add() below can be reordered with any LOAD/STORE done by the |
60 | * reader inside the critical section. See the comments in down_write and | |
61 | * up_write below. | |
62 | */ | |
63 | static bool update_fast_ctr(struct percpu_rw_semaphore *brw, unsigned int val) | |
64 | { | |
65 | bool success = false; | |
66 | ||
67 | preempt_disable(); | |
9390ef0c | 68 | if (likely(!atomic_read(&brw->write_ctr))) { |
a1fd3e24 ON |
69 | __this_cpu_add(*brw->fast_read_ctr, val); |
70 | success = true; | |
71 | } | |
72 | preempt_enable(); | |
73 | ||
74 | return success; | |
75 | } | |
76 | ||
77 | /* | |
78 | * Like the normal down_read() this is not recursive, the writer can | |
79 | * come after the first percpu_down_read() and create the deadlock. | |
8ebe3473 ON |
80 | * |
81 | * Note: returns with lock_is_held(brw->rw_sem) == T for lockdep, | |
82 | * percpu_up_read() does rwsem_release(). This pairs with the usage | |
83 | * of ->rw_sem in percpu_down/up_write(). | |
a1fd3e24 ON |
84 | */ |
85 | void percpu_down_read(struct percpu_rw_semaphore *brw) | |
86 | { | |
8ebe3473 ON |
87 | might_sleep(); |
88 | if (likely(update_fast_ctr(brw, +1))) { | |
89 | rwsem_acquire_read(&brw->rw_sem.dep_map, 0, 0, _RET_IP_); | |
a1fd3e24 | 90 | return; |
8ebe3473 | 91 | } |
a1fd3e24 ON |
92 | |
93 | down_read(&brw->rw_sem); | |
94 | atomic_inc(&brw->slow_read_ctr); | |
8ebe3473 ON |
95 | /* avoid up_read()->rwsem_release() */ |
96 | __up_read(&brw->rw_sem); | |
a1fd3e24 | 97 | } |
302707fd | 98 | EXPORT_SYMBOL_GPL(percpu_down_read); |
a1fd3e24 | 99 | |
9287f692 ON |
100 | int percpu_down_read_trylock(struct percpu_rw_semaphore *brw) |
101 | { | |
102 | if (unlikely(!update_fast_ctr(brw, +1))) { | |
103 | if (!__down_read_trylock(&brw->rw_sem)) | |
104 | return 0; | |
105 | atomic_inc(&brw->slow_read_ctr); | |
106 | __up_read(&brw->rw_sem); | |
107 | } | |
108 | ||
109 | rwsem_acquire_read(&brw->rw_sem.dep_map, 0, 1, _RET_IP_); | |
110 | return 1; | |
111 | } | |
112 | ||
a1fd3e24 ON |
113 | void percpu_up_read(struct percpu_rw_semaphore *brw) |
114 | { | |
8ebe3473 ON |
115 | rwsem_release(&brw->rw_sem.dep_map, 1, _RET_IP_); |
116 | ||
a1fd3e24 ON |
117 | if (likely(update_fast_ctr(brw, -1))) |
118 | return; | |
119 | ||
120 | /* false-positive is possible but harmless */ | |
121 | if (atomic_dec_and_test(&brw->slow_read_ctr)) | |
122 | wake_up_all(&brw->write_waitq); | |
123 | } | |
302707fd | 124 | EXPORT_SYMBOL_GPL(percpu_up_read); |
a1fd3e24 ON |
125 | |
126 | static int clear_fast_ctr(struct percpu_rw_semaphore *brw) | |
127 | { | |
128 | unsigned int sum = 0; | |
129 | int cpu; | |
130 | ||
131 | for_each_possible_cpu(cpu) { | |
132 | sum += per_cpu(*brw->fast_read_ctr, cpu); | |
133 | per_cpu(*brw->fast_read_ctr, cpu) = 0; | |
134 | } | |
135 | ||
136 | return sum; | |
137 | } | |
138 | ||
139 | /* | |
9390ef0c ON |
140 | * A writer increments ->write_ctr to force the readers to switch to the |
141 | * slow mode, note the atomic_read() check in update_fast_ctr(). | |
a1fd3e24 ON |
142 | * |
143 | * After that the readers can only inc/dec the slow ->slow_read_ctr counter, | |
144 | * ->fast_read_ctr is stable. Once the writer moves its sum into the slow | |
145 | * counter it represents the number of active readers. | |
146 | * | |
147 | * Finally the writer takes ->rw_sem for writing and blocks the new readers, | |
148 | * then waits until the slow counter becomes zero. | |
149 | */ | |
150 | void percpu_down_write(struct percpu_rw_semaphore *brw) | |
151 | { | |
9390ef0c ON |
152 | /* tell update_fast_ctr() there is a pending writer */ |
153 | atomic_inc(&brw->write_ctr); | |
a1fd3e24 | 154 | /* |
9390ef0c | 155 | * 1. Ensures that write_ctr != 0 is visible to any down_read/up_read |
a1fd3e24 ON |
156 | * so that update_fast_ctr() can't succeed. |
157 | * | |
158 | * 2. Ensures we see the result of every previous this_cpu_add() in | |
159 | * update_fast_ctr(). | |
160 | * | |
161 | * 3. Ensures that if any reader has exited its critical section via | |
162 | * fast-path, it executes a full memory barrier before we return. | |
163 | * See R_W case in the comment above update_fast_ctr(). | |
164 | */ | |
165 | synchronize_sched_expedited(); | |
166 | ||
9390ef0c ON |
167 | /* exclude other writers, and block the new readers completely */ |
168 | down_write(&brw->rw_sem); | |
169 | ||
a1fd3e24 ON |
170 | /* nobody can use fast_read_ctr, move its sum into slow_read_ctr */ |
171 | atomic_add(clear_fast_ctr(brw), &brw->slow_read_ctr); | |
172 | ||
a1fd3e24 ON |
173 | /* wait for all readers to complete their percpu_up_read() */ |
174 | wait_event(brw->write_waitq, !atomic_read(&brw->slow_read_ctr)); | |
175 | } | |
302707fd | 176 | EXPORT_SYMBOL_GPL(percpu_down_write); |
a1fd3e24 ON |
177 | |
178 | void percpu_up_write(struct percpu_rw_semaphore *brw) | |
179 | { | |
9390ef0c | 180 | /* release the lock, but the readers can't use the fast-path */ |
a1fd3e24 | 181 | up_write(&brw->rw_sem); |
a1fd3e24 ON |
182 | /* |
183 | * Insert the barrier before the next fast-path in down_read, | |
184 | * see W_R case in the comment above update_fast_ctr(). | |
185 | */ | |
186 | synchronize_sched_expedited(); | |
9390ef0c ON |
187 | /* the last writer unblocks update_fast_ctr() */ |
188 | atomic_dec(&brw->write_ctr); | |
a1fd3e24 | 189 | } |
302707fd | 190 | EXPORT_SYMBOL_GPL(percpu_up_write); |