Commit | Line | Data |
---|---|---|
867e359b CM |
1 | /* |
2 | * Copyright 2010 Tilera Corporation. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation, version 2. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
11 | * NON INFRINGEMENT. See the GNU General Public License for | |
12 | * more details. | |
13 | */ | |
14 | ||
15 | #include <linux/spinlock.h> | |
16 | #include <linux/module.h> | |
17 | #include <asm/processor.h> | |
3c5ead52 | 18 | #include <arch/spr_def.h> |
867e359b CM |
19 | |
20 | #include "spinlock_common.h" | |
21 | ||
22 | void arch_spin_lock(arch_spinlock_t *lock) | |
23 | { | |
24 | int my_ticket; | |
25 | int iterations = 0; | |
26 | int delta; | |
27 | ||
28 | while ((my_ticket = __insn_tns((void *)&lock->next_ticket)) & 1) | |
29 | delay_backoff(iterations++); | |
30 | ||
31 | /* Increment the next ticket number, implicitly releasing tns lock. */ | |
32 | lock->next_ticket = my_ticket + TICKET_QUANTUM; | |
33 | ||
34 | /* Wait until it's our turn. */ | |
35 | while ((delta = my_ticket - lock->current_ticket) != 0) | |
36 | relax((128 / CYCLES_PER_RELAX_LOOP) * delta); | |
37 | } | |
38 | EXPORT_SYMBOL(arch_spin_lock); | |
39 | ||
40 | int arch_spin_trylock(arch_spinlock_t *lock) | |
41 | { | |
42 | /* | |
43 | * Grab a ticket; no need to retry if it's busy, we'll just | |
44 | * treat that the same as "locked", since someone else | |
45 | * will lock it momentarily anyway. | |
46 | */ | |
47 | int my_ticket = __insn_tns((void *)&lock->next_ticket); | |
48 | ||
49 | if (my_ticket == lock->current_ticket) { | |
50 | /* Not currently locked, so lock it by keeping this ticket. */ | |
51 | lock->next_ticket = my_ticket + TICKET_QUANTUM; | |
52 | /* Success! */ | |
53 | return 1; | |
54 | } | |
55 | ||
56 | if (!(my_ticket & 1)) { | |
57 | /* Release next_ticket. */ | |
58 | lock->next_ticket = my_ticket; | |
59 | } | |
60 | ||
61 | return 0; | |
62 | } | |
63 | EXPORT_SYMBOL(arch_spin_trylock); | |
64 | ||
65 | void arch_spin_unlock_wait(arch_spinlock_t *lock) | |
66 | { | |
67 | u32 iterations = 0; | |
68 | while (arch_spin_is_locked(lock)) | |
69 | delay_backoff(iterations++); | |
70 | } | |
71 | EXPORT_SYMBOL(arch_spin_unlock_wait); | |
72 | ||
73 | /* | |
74 | * The low byte is always reserved to be the marker for a "tns" operation | |
75 | * since the low bit is set to "1" by a tns. The next seven bits are | |
76 | * zeroes. The next byte holds the "next" writer value, i.e. the ticket | |
77 | * available for the next task that wants to write. The third byte holds | |
78 | * the current writer value, i.e. the writer who holds the current ticket. | |
79 | * If current == next == 0, there are no interested writers. | |
80 | */ | |
81 | #define WR_NEXT_SHIFT _WR_NEXT_SHIFT | |
82 | #define WR_CURR_SHIFT _WR_CURR_SHIFT | |
83 | #define WR_WIDTH _WR_WIDTH | |
84 | #define WR_MASK ((1 << WR_WIDTH) - 1) | |
85 | ||
86 | /* | |
87 | * The last eight bits hold the active reader count. This has to be | |
88 | * zero before a writer can start to write. | |
89 | */ | |
90 | #define RD_COUNT_SHIFT _RD_COUNT_SHIFT | |
91 | #define RD_COUNT_WIDTH _RD_COUNT_WIDTH | |
92 | #define RD_COUNT_MASK ((1 << RD_COUNT_WIDTH) - 1) | |
93 | ||
94 | ||
3c5ead52 CM |
95 | /* |
96 | * We can get the read lock if everything but the reader bits (which | |
97 | * are in the high part of the word) is zero, i.e. no active or | |
98 | * waiting writers, no tns. | |
99 | * | |
100 | * We guard the tns/store-back with an interrupt critical section to | |
101 | * preserve the semantic that the same read lock can be acquired in an | |
102 | * interrupt context. | |
103 | */ | |
4833d7f0 | 104 | int arch_read_trylock(arch_rwlock_t *rwlock) |
867e359b | 105 | { |
3c5ead52 CM |
106 | u32 val; |
107 | __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 1); | |
108 | val = __insn_tns((int *)&rwlock->lock); | |
109 | if (likely((val << _RD_COUNT_WIDTH) == 0)) { | |
110 | val += 1 << RD_COUNT_SHIFT; | |
111 | rwlock->lock = val; | |
112 | __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); | |
113 | BUG_ON(val == 0); /* we don't expect wraparound */ | |
114 | return 1; | |
867e359b | 115 | } |
3c5ead52 CM |
116 | if ((val & 1) == 0) |
117 | rwlock->lock = val; | |
118 | __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); | |
119 | return 0; | |
867e359b | 120 | } |
3c5ead52 | 121 | EXPORT_SYMBOL(arch_read_trylock); |
867e359b CM |
122 | |
123 | /* | |
3c5ead52 | 124 | * Spin doing arch_read_trylock() until we acquire the lock. |
867e359b CM |
125 | * ISSUE: This approach can permanently starve readers. A reader who sees |
126 | * a writer could instead take a ticket lock (just like a writer would), | |
127 | * and atomically enter read mode (with 1 reader) when it gets the ticket. | |
3c5ead52 | 128 | * This way both readers and writers would always make forward progress |
867e359b CM |
129 | * in a finite time. |
130 | */ | |
3c5ead52 | 131 | void arch_read_lock(arch_rwlock_t *rwlock) |
867e359b CM |
132 | { |
133 | u32 iterations = 0; | |
3c5ead52 | 134 | while (unlikely(!arch_read_trylock(rwlock))) |
867e359b | 135 | delay_backoff(iterations++); |
3c5ead52 CM |
136 | } |
137 | EXPORT_SYMBOL(arch_read_lock); | |
138 | ||
139 | void arch_read_unlock(arch_rwlock_t *rwlock) | |
140 | { | |
141 | u32 val, iterations = 0; | |
142 | ||
143 | mb(); /* guarantee anything modified under the lock is visible */ | |
144 | for (;;) { | |
145 | __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 1); | |
867e359b | 146 | val = __insn_tns((int *)&rwlock->lock); |
cf8c1daf | 147 | if (likely((val & 1) == 0)) { |
3c5ead52 CM |
148 | rwlock->lock = val - (1 << _RD_COUNT_SHIFT); |
149 | __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); | |
150 | break; | |
151 | } | |
152 | __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); | |
153 | delay_backoff(iterations++); | |
154 | } | |
867e359b | 155 | } |
3c5ead52 | 156 | EXPORT_SYMBOL(arch_read_unlock); |
867e359b | 157 | |
3c5ead52 CM |
158 | /* |
159 | * We don't need an interrupt critical section here (unlike for | |
160 | * arch_read_lock) since we should never use a bare write lock where | |
161 | * it could be interrupted by code that could try to re-acquire it. | |
162 | */ | |
163 | void arch_write_lock(arch_rwlock_t *rwlock) | |
867e359b CM |
164 | { |
165 | /* | |
166 | * The trailing underscore on this variable (and curr_ below) | |
167 | * reminds us that the high bits are garbage; we mask them out | |
168 | * when we compare them. | |
169 | */ | |
170 | u32 my_ticket_; | |
24f3f6b5 | 171 | u32 iterations = 0; |
3c5ead52 CM |
172 | u32 val = __insn_tns((int *)&rwlock->lock); |
173 | ||
174 | if (likely(val == 0)) { | |
175 | rwlock->lock = 1 << _WR_NEXT_SHIFT; | |
176 | return; | |
177 | } | |
867e359b | 178 | |
24f3f6b5 CM |
179 | /* |
180 | * Wait until there are no readers, then bump up the next | |
181 | * field and capture the ticket value. | |
182 | */ | |
183 | for (;;) { | |
184 | if (!(val & 1)) { | |
185 | if ((val >> RD_COUNT_SHIFT) == 0) | |
186 | break; | |
187 | rwlock->lock = val; | |
188 | } | |
189 | delay_backoff(iterations++); | |
190 | val = __insn_tns((int *)&rwlock->lock); | |
191 | } | |
867e359b | 192 | |
24f3f6b5 CM |
193 | /* Take out the next ticket and extract my ticket value. */ |
194 | rwlock->lock = __insn_addb(val, 1 << WR_NEXT_SHIFT); | |
867e359b CM |
195 | my_ticket_ = val >> WR_NEXT_SHIFT; |
196 | ||
24f3f6b5 | 197 | /* Wait until the "current" field matches our ticket. */ |
867e359b CM |
198 | for (;;) { |
199 | u32 curr_ = val >> WR_CURR_SHIFT; | |
24f3f6b5 | 200 | u32 delta = ((my_ticket_ - curr_) & WR_MASK); |
867e359b CM |
201 | if (likely(delta == 0)) |
202 | break; | |
203 | ||
204 | /* Delay based on how many lock-holders are still out there. */ | |
205 | relax((256 / CYCLES_PER_RELAX_LOOP) * delta); | |
206 | ||
207 | /* | |
208 | * Get a non-tns value to check; we don't need to tns | |
209 | * it ourselves. Since we're not tns'ing, we retry | |
210 | * more rapidly to get a valid value. | |
211 | */ | |
212 | while ((val = rwlock->lock) & 1) | |
213 | relax(4); | |
214 | } | |
215 | } | |
3c5ead52 | 216 | EXPORT_SYMBOL(arch_write_lock); |
867e359b | 217 | |
3c5ead52 | 218 | int arch_write_trylock(arch_rwlock_t *rwlock) |
867e359b | 219 | { |
3c5ead52 | 220 | u32 val = __insn_tns((int *)&rwlock->lock); |
867e359b | 221 | |
3c5ead52 CM |
222 | /* |
223 | * If a tns is in progress, or there's a waiting or active locker, | |
224 | * or active readers, we can't take the lock, so give up. | |
225 | */ | |
226 | if (unlikely(val != 0)) { | |
227 | if (!(val & 1)) | |
228 | rwlock->lock = val; | |
229 | return 0; | |
230 | } | |
867e359b | 231 | |
3c5ead52 CM |
232 | /* Set the "next" field to mark it locked. */ |
233 | rwlock->lock = 1 << _WR_NEXT_SHIFT; | |
234 | return 1; | |
867e359b | 235 | } |
3c5ead52 | 236 | EXPORT_SYMBOL(arch_write_trylock); |
867e359b | 237 | |
3c5ead52 | 238 | void arch_write_unlock(arch_rwlock_t *rwlock) |
867e359b | 239 | { |
3c5ead52 CM |
240 | u32 val, eq, mask; |
241 | ||
242 | mb(); /* guarantee anything modified under the lock is visible */ | |
243 | val = __insn_tns((int *)&rwlock->lock); | |
244 | if (likely(val == (1 << _WR_NEXT_SHIFT))) { | |
245 | rwlock->lock = 0; | |
246 | return; | |
247 | } | |
248 | while (unlikely(val & 1)) { | |
249 | /* Limited backoff since we are the highest-priority task. */ | |
250 | relax(4); | |
251 | val = __insn_tns((int *)&rwlock->lock); | |
252 | } | |
253 | mask = 1 << WR_CURR_SHIFT; | |
254 | val = __insn_addb(val, mask); | |
255 | eq = __insn_seqb(val, val << (WR_CURR_SHIFT - WR_NEXT_SHIFT)); | |
256 | val = __insn_mz(eq & mask, val); | |
257 | rwlock->lock = val; | |
867e359b | 258 | } |
3c5ead52 | 259 | EXPORT_SYMBOL(arch_write_unlock); |