Commit | Line | Data |
---|---|---|
867e359b CM |
1 | /* |
2 | * Copyright 2010 Tilera Corporation. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation, version 2. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
11 | * NON INFRINGEMENT. See the GNU General Public License for | |
12 | * more details. | |
13 | */ | |
14 | ||
15 | #include <linux/spinlock.h> | |
16 | #include <linux/module.h> | |
17 | #include <asm/processor.h> | |
3c5ead52 | 18 | #include <arch/spr_def.h> |
867e359b CM |
19 | |
20 | #include "spinlock_common.h" | |
21 | ||
22 | void arch_spin_lock(arch_spinlock_t *lock) | |
23 | { | |
24 | int my_ticket; | |
25 | int iterations = 0; | |
26 | int delta; | |
27 | ||
28 | while ((my_ticket = __insn_tns((void *)&lock->next_ticket)) & 1) | |
29 | delay_backoff(iterations++); | |
30 | ||
31 | /* Increment the next ticket number, implicitly releasing tns lock. */ | |
32 | lock->next_ticket = my_ticket + TICKET_QUANTUM; | |
33 | ||
34 | /* Wait until it's our turn. */ | |
35 | while ((delta = my_ticket - lock->current_ticket) != 0) | |
36 | relax((128 / CYCLES_PER_RELAX_LOOP) * delta); | |
37 | } | |
38 | EXPORT_SYMBOL(arch_spin_lock); | |
39 | ||
40 | int arch_spin_trylock(arch_spinlock_t *lock) | |
41 | { | |
42 | /* | |
43 | * Grab a ticket; no need to retry if it's busy, we'll just | |
44 | * treat that the same as "locked", since someone else | |
45 | * will lock it momentarily anyway. | |
46 | */ | |
47 | int my_ticket = __insn_tns((void *)&lock->next_ticket); | |
48 | ||
49 | if (my_ticket == lock->current_ticket) { | |
50 | /* Not currently locked, so lock it by keeping this ticket. */ | |
51 | lock->next_ticket = my_ticket + TICKET_QUANTUM; | |
52 | /* Success! */ | |
53 | return 1; | |
54 | } | |
55 | ||
56 | if (!(my_ticket & 1)) { | |
57 | /* Release next_ticket. */ | |
58 | lock->next_ticket = my_ticket; | |
59 | } | |
60 | ||
61 | return 0; | |
62 | } | |
63 | EXPORT_SYMBOL(arch_spin_trylock); | |
64 | ||
65 | void arch_spin_unlock_wait(arch_spinlock_t *lock) | |
66 | { | |
67 | u32 iterations = 0; | |
14c3dec2 CM |
68 | int curr = READ_ONCE(lock->current_ticket); |
69 | int next = READ_ONCE(lock->next_ticket); | |
70 | ||
71 | /* Return immediately if unlocked. */ | |
72 | if (next == curr) | |
73 | return; | |
74 | ||
75 | /* Wait until the current locker has released the lock. */ | |
76 | do { | |
867e359b | 77 | delay_backoff(iterations++); |
14c3dec2 | 78 | } while (READ_ONCE(lock->current_ticket) == curr); |
867e359b CM |
79 | } |
80 | EXPORT_SYMBOL(arch_spin_unlock_wait); | |
81 | ||
82 | /* | |
83 | * The low byte is always reserved to be the marker for a "tns" operation | |
84 | * since the low bit is set to "1" by a tns. The next seven bits are | |
85 | * zeroes. The next byte holds the "next" writer value, i.e. the ticket | |
86 | * available for the next task that wants to write. The third byte holds | |
87 | * the current writer value, i.e. the writer who holds the current ticket. | |
88 | * If current == next == 0, there are no interested writers. | |
89 | */ | |
90 | #define WR_NEXT_SHIFT _WR_NEXT_SHIFT | |
91 | #define WR_CURR_SHIFT _WR_CURR_SHIFT | |
92 | #define WR_WIDTH _WR_WIDTH | |
93 | #define WR_MASK ((1 << WR_WIDTH) - 1) | |
94 | ||
95 | /* | |
96 | * The last eight bits hold the active reader count. This has to be | |
97 | * zero before a writer can start to write. | |
98 | */ | |
99 | #define RD_COUNT_SHIFT _RD_COUNT_SHIFT | |
100 | #define RD_COUNT_WIDTH _RD_COUNT_WIDTH | |
101 | #define RD_COUNT_MASK ((1 << RD_COUNT_WIDTH) - 1) | |
102 | ||
103 | ||
3c5ead52 CM |
104 | /* |
105 | * We can get the read lock if everything but the reader bits (which | |
106 | * are in the high part of the word) is zero, i.e. no active or | |
107 | * waiting writers, no tns. | |
108 | * | |
109 | * We guard the tns/store-back with an interrupt critical section to | |
110 | * preserve the semantic that the same read lock can be acquired in an | |
111 | * interrupt context. | |
112 | */ | |
4833d7f0 | 113 | int arch_read_trylock(arch_rwlock_t *rwlock) |
867e359b | 114 | { |
3c5ead52 CM |
115 | u32 val; |
116 | __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 1); | |
117 | val = __insn_tns((int *)&rwlock->lock); | |
118 | if (likely((val << _RD_COUNT_WIDTH) == 0)) { | |
119 | val += 1 << RD_COUNT_SHIFT; | |
120 | rwlock->lock = val; | |
121 | __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); | |
122 | BUG_ON(val == 0); /* we don't expect wraparound */ | |
123 | return 1; | |
867e359b | 124 | } |
3c5ead52 CM |
125 | if ((val & 1) == 0) |
126 | rwlock->lock = val; | |
127 | __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); | |
128 | return 0; | |
867e359b | 129 | } |
3c5ead52 | 130 | EXPORT_SYMBOL(arch_read_trylock); |
867e359b CM |
131 | |
132 | /* | |
3c5ead52 | 133 | * Spin doing arch_read_trylock() until we acquire the lock. |
867e359b CM |
134 | * ISSUE: This approach can permanently starve readers. A reader who sees |
135 | * a writer could instead take a ticket lock (just like a writer would), | |
136 | * and atomically enter read mode (with 1 reader) when it gets the ticket. | |
3c5ead52 | 137 | * This way both readers and writers would always make forward progress |
867e359b CM |
138 | * in a finite time. |
139 | */ | |
3c5ead52 | 140 | void arch_read_lock(arch_rwlock_t *rwlock) |
867e359b CM |
141 | { |
142 | u32 iterations = 0; | |
3c5ead52 | 143 | while (unlikely(!arch_read_trylock(rwlock))) |
867e359b | 144 | delay_backoff(iterations++); |
3c5ead52 CM |
145 | } |
146 | EXPORT_SYMBOL(arch_read_lock); | |
147 | ||
148 | void arch_read_unlock(arch_rwlock_t *rwlock) | |
149 | { | |
150 | u32 val, iterations = 0; | |
151 | ||
152 | mb(); /* guarantee anything modified under the lock is visible */ | |
153 | for (;;) { | |
154 | __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 1); | |
867e359b | 155 | val = __insn_tns((int *)&rwlock->lock); |
cf8c1daf | 156 | if (likely((val & 1) == 0)) { |
3c5ead52 CM |
157 | rwlock->lock = val - (1 << _RD_COUNT_SHIFT); |
158 | __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); | |
159 | break; | |
160 | } | |
161 | __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); | |
162 | delay_backoff(iterations++); | |
163 | } | |
867e359b | 164 | } |
3c5ead52 | 165 | EXPORT_SYMBOL(arch_read_unlock); |
867e359b | 166 | |
3c5ead52 CM |
167 | /* |
168 | * We don't need an interrupt critical section here (unlike for | |
169 | * arch_read_lock) since we should never use a bare write lock where | |
170 | * it could be interrupted by code that could try to re-acquire it. | |
171 | */ | |
172 | void arch_write_lock(arch_rwlock_t *rwlock) | |
867e359b CM |
173 | { |
174 | /* | |
175 | * The trailing underscore on this variable (and curr_ below) | |
176 | * reminds us that the high bits are garbage; we mask them out | |
177 | * when we compare them. | |
178 | */ | |
179 | u32 my_ticket_; | |
24f3f6b5 | 180 | u32 iterations = 0; |
3c5ead52 CM |
181 | u32 val = __insn_tns((int *)&rwlock->lock); |
182 | ||
183 | if (likely(val == 0)) { | |
184 | rwlock->lock = 1 << _WR_NEXT_SHIFT; | |
185 | return; | |
186 | } | |
867e359b | 187 | |
24f3f6b5 CM |
188 | /* |
189 | * Wait until there are no readers, then bump up the next | |
190 | * field and capture the ticket value. | |
191 | */ | |
192 | for (;;) { | |
193 | if (!(val & 1)) { | |
194 | if ((val >> RD_COUNT_SHIFT) == 0) | |
195 | break; | |
196 | rwlock->lock = val; | |
197 | } | |
198 | delay_backoff(iterations++); | |
199 | val = __insn_tns((int *)&rwlock->lock); | |
200 | } | |
867e359b | 201 | |
24f3f6b5 CM |
202 | /* Take out the next ticket and extract my ticket value. */ |
203 | rwlock->lock = __insn_addb(val, 1 << WR_NEXT_SHIFT); | |
867e359b CM |
204 | my_ticket_ = val >> WR_NEXT_SHIFT; |
205 | ||
24f3f6b5 | 206 | /* Wait until the "current" field matches our ticket. */ |
867e359b CM |
207 | for (;;) { |
208 | u32 curr_ = val >> WR_CURR_SHIFT; | |
24f3f6b5 | 209 | u32 delta = ((my_ticket_ - curr_) & WR_MASK); |
867e359b CM |
210 | if (likely(delta == 0)) |
211 | break; | |
212 | ||
213 | /* Delay based on how many lock-holders are still out there. */ | |
214 | relax((256 / CYCLES_PER_RELAX_LOOP) * delta); | |
215 | ||
216 | /* | |
217 | * Get a non-tns value to check; we don't need to tns | |
218 | * it ourselves. Since we're not tns'ing, we retry | |
219 | * more rapidly to get a valid value. | |
220 | */ | |
221 | while ((val = rwlock->lock) & 1) | |
222 | relax(4); | |
223 | } | |
224 | } | |
3c5ead52 | 225 | EXPORT_SYMBOL(arch_write_lock); |
867e359b | 226 | |
3c5ead52 | 227 | int arch_write_trylock(arch_rwlock_t *rwlock) |
867e359b | 228 | { |
3c5ead52 | 229 | u32 val = __insn_tns((int *)&rwlock->lock); |
867e359b | 230 | |
3c5ead52 CM |
231 | /* |
232 | * If a tns is in progress, or there's a waiting or active locker, | |
233 | * or active readers, we can't take the lock, so give up. | |
234 | */ | |
235 | if (unlikely(val != 0)) { | |
236 | if (!(val & 1)) | |
237 | rwlock->lock = val; | |
238 | return 0; | |
239 | } | |
867e359b | 240 | |
3c5ead52 CM |
241 | /* Set the "next" field to mark it locked. */ |
242 | rwlock->lock = 1 << _WR_NEXT_SHIFT; | |
243 | return 1; | |
867e359b | 244 | } |
3c5ead52 | 245 | EXPORT_SYMBOL(arch_write_trylock); |
867e359b | 246 | |
3c5ead52 | 247 | void arch_write_unlock(arch_rwlock_t *rwlock) |
867e359b | 248 | { |
3c5ead52 CM |
249 | u32 val, eq, mask; |
250 | ||
251 | mb(); /* guarantee anything modified under the lock is visible */ | |
252 | val = __insn_tns((int *)&rwlock->lock); | |
253 | if (likely(val == (1 << _WR_NEXT_SHIFT))) { | |
254 | rwlock->lock = 0; | |
255 | return; | |
256 | } | |
257 | while (unlikely(val & 1)) { | |
258 | /* Limited backoff since we are the highest-priority task. */ | |
259 | relax(4); | |
260 | val = __insn_tns((int *)&rwlock->lock); | |
261 | } | |
262 | mask = 1 << WR_CURR_SHIFT; | |
263 | val = __insn_addb(val, mask); | |
264 | eq = __insn_seqb(val, val << (WR_CURR_SHIFT - WR_NEXT_SHIFT)); | |
265 | val = __insn_mz(eq & mask, val); | |
266 | rwlock->lock = val; | |
867e359b | 267 | } |
3c5ead52 | 268 | EXPORT_SYMBOL(arch_write_unlock); |