Commit | Line | Data |
---|---|---|
c0385b24 WD |
1 | /* |
2 | * Based on arch/arm/include/asm/atomic.h | |
3 | * | |
4 | * Copyright (C) 1996 Russell King. | |
5 | * Copyright (C) 2002 Deep Blue Solutions Ltd. | |
6 | * Copyright (C) 2012 ARM Ltd. | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify | |
9 | * it under the terms of the GNU General Public License version 2 as | |
10 | * published by the Free Software Foundation. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public License | |
18 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
19 | */ | |
20 | ||
21 | #ifndef __ASM_ATOMIC_LSE_H | |
22 | #define __ASM_ATOMIC_LSE_H | |
23 | ||
24 | #ifndef __ARM64_IN_ATOMIC_IMPL | |
25 | #error "please don't include this file directly" | |
26 | #endif | |
27 | ||
c09d6a04 WD |
28 | #define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op) |
29 | ||
30 | static inline void atomic_andnot(int i, atomic_t *v) | |
31 | { | |
32 | register int w0 asm ("w0") = i; | |
33 | register atomic_t *x1 asm ("x1") = v; | |
34 | ||
35 | asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(andnot), | |
36 | " stclr %w[i], %[v]\n") | |
37 | : [i] "+r" (w0), [v] "+Q" (v->counter) | |
38 | : "r" (x1) | |
39 | : "x30"); | |
40 | } | |
41 | ||
42 | static inline void atomic_or(int i, atomic_t *v) | |
43 | { | |
44 | register int w0 asm ("w0") = i; | |
45 | register atomic_t *x1 asm ("x1") = v; | |
46 | ||
47 | asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(or), | |
48 | " stset %w[i], %[v]\n") | |
49 | : [i] "+r" (w0), [v] "+Q" (v->counter) | |
50 | : "r" (x1) | |
51 | : "x30"); | |
52 | } | |
53 | ||
54 | static inline void atomic_xor(int i, atomic_t *v) | |
55 | { | |
56 | register int w0 asm ("w0") = i; | |
57 | register atomic_t *x1 asm ("x1") = v; | |
58 | ||
59 | asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(xor), | |
60 | " steor %w[i], %[v]\n") | |
61 | : [i] "+r" (w0), [v] "+Q" (v->counter) | |
62 | : "r" (x1) | |
63 | : "x30"); | |
64 | } | |
65 | ||
66 | static inline void atomic_add(int i, atomic_t *v) | |
67 | { | |
68 | register int w0 asm ("w0") = i; | |
69 | register atomic_t *x1 asm ("x1") = v; | |
70 | ||
71 | asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(add), | |
72 | " stadd %w[i], %[v]\n") | |
73 | : [i] "+r" (w0), [v] "+Q" (v->counter) | |
74 | : "r" (x1) | |
75 | : "x30"); | |
76 | } | |
77 | ||
305d454a WD |
78 | #define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \ |
79 | static inline int atomic_add_return##name(int i, atomic_t *v) \ | |
80 | { \ | |
81 | register int w0 asm ("w0") = i; \ | |
82 | register atomic_t *x1 asm ("x1") = v; \ | |
83 | \ | |
84 | asm volatile(ARM64_LSE_ATOMIC_INSN( \ | |
85 | /* LL/SC */ \ | |
86 | " nop\n" \ | |
87 | __LL_SC_ATOMIC(add_return##name), \ | |
88 | /* LSE atomics */ \ | |
89 | " ldadd" #mb " %w[i], w30, %[v]\n" \ | |
90 | " add %w[i], %w[i], w30") \ | |
91 | : [i] "+r" (w0), [v] "+Q" (v->counter) \ | |
92 | : "r" (x1) \ | |
93 | : "x30" , ##cl); \ | |
94 | \ | |
95 | return w0; \ | |
96 | } | |
c09d6a04 | 97 | |
305d454a WD |
98 | ATOMIC_OP_ADD_RETURN(_relaxed, ) |
99 | ATOMIC_OP_ADD_RETURN(_acquire, a, "memory") | |
100 | ATOMIC_OP_ADD_RETURN(_release, l, "memory") | |
101 | ATOMIC_OP_ADD_RETURN( , al, "memory") | |
c09d6a04 | 102 | |
305d454a | 103 | #undef ATOMIC_OP_ADD_RETURN |
c09d6a04 WD |
104 | |
105 | static inline void atomic_and(int i, atomic_t *v) | |
106 | { | |
107 | register int w0 asm ("w0") = i; | |
108 | register atomic_t *x1 asm ("x1") = v; | |
109 | ||
110 | asm volatile(ARM64_LSE_ATOMIC_INSN( | |
111 | /* LL/SC */ | |
112 | " nop\n" | |
113 | __LL_SC_ATOMIC(and), | |
114 | /* LSE atomics */ | |
115 | " mvn %w[i], %w[i]\n" | |
116 | " stclr %w[i], %[v]") | |
117 | : [i] "+r" (w0), [v] "+Q" (v->counter) | |
118 | : "r" (x1) | |
119 | : "x30"); | |
120 | } | |
121 | ||
122 | static inline void atomic_sub(int i, atomic_t *v) | |
123 | { | |
124 | register int w0 asm ("w0") = i; | |
125 | register atomic_t *x1 asm ("x1") = v; | |
126 | ||
127 | asm volatile(ARM64_LSE_ATOMIC_INSN( | |
128 | /* LL/SC */ | |
129 | " nop\n" | |
130 | __LL_SC_ATOMIC(sub), | |
131 | /* LSE atomics */ | |
132 | " neg %w[i], %w[i]\n" | |
133 | " stadd %w[i], %[v]") | |
134 | : [i] "+r" (w0), [v] "+Q" (v->counter) | |
135 | : "r" (x1) | |
136 | : "x30"); | |
137 | } | |
138 | ||
305d454a WD |
139 | #define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \ |
140 | static inline int atomic_sub_return##name(int i, atomic_t *v) \ | |
141 | { \ | |
142 | register int w0 asm ("w0") = i; \ | |
143 | register atomic_t *x1 asm ("x1") = v; \ | |
144 | \ | |
145 | asm volatile(ARM64_LSE_ATOMIC_INSN( \ | |
146 | /* LL/SC */ \ | |
147 | " nop\n" \ | |
148 | __LL_SC_ATOMIC(sub_return##name) \ | |
149 | " nop", \ | |
150 | /* LSE atomics */ \ | |
151 | " neg %w[i], %w[i]\n" \ | |
152 | " ldadd" #mb " %w[i], w30, %[v]\n" \ | |
153 | " add %w[i], %w[i], w30") \ | |
154 | : [i] "+r" (w0), [v] "+Q" (v->counter) \ | |
155 | : "r" (x1) \ | |
156 | : "x30" , ##cl); \ | |
157 | \ | |
158 | return w0; \ | |
c09d6a04 | 159 | } |
c0385b24 | 160 | |
305d454a WD |
161 | ATOMIC_OP_SUB_RETURN(_relaxed, ) |
162 | ATOMIC_OP_SUB_RETURN(_acquire, a, "memory") | |
163 | ATOMIC_OP_SUB_RETURN(_release, l, "memory") | |
164 | ATOMIC_OP_SUB_RETURN( , al, "memory") | |
165 | ||
166 | #undef ATOMIC_OP_SUB_RETURN | |
c09d6a04 | 167 | #undef __LL_SC_ATOMIC |
c0385b24 | 168 | |
c09d6a04 WD |
169 | #define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op) |
170 | ||
171 | static inline void atomic64_andnot(long i, atomic64_t *v) | |
172 | { | |
173 | register long x0 asm ("x0") = i; | |
174 | register atomic64_t *x1 asm ("x1") = v; | |
175 | ||
176 | asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(andnot), | |
177 | " stclr %[i], %[v]\n") | |
178 | : [i] "+r" (x0), [v] "+Q" (v->counter) | |
179 | : "r" (x1) | |
180 | : "x30"); | |
181 | } | |
182 | ||
183 | static inline void atomic64_or(long i, atomic64_t *v) | |
184 | { | |
185 | register long x0 asm ("x0") = i; | |
186 | register atomic64_t *x1 asm ("x1") = v; | |
187 | ||
188 | asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(or), | |
189 | " stset %[i], %[v]\n") | |
190 | : [i] "+r" (x0), [v] "+Q" (v->counter) | |
191 | : "r" (x1) | |
192 | : "x30"); | |
193 | } | |
194 | ||
195 | static inline void atomic64_xor(long i, atomic64_t *v) | |
196 | { | |
197 | register long x0 asm ("x0") = i; | |
198 | register atomic64_t *x1 asm ("x1") = v; | |
199 | ||
200 | asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(xor), | |
201 | " steor %[i], %[v]\n") | |
202 | : [i] "+r" (x0), [v] "+Q" (v->counter) | |
203 | : "r" (x1) | |
204 | : "x30"); | |
205 | } | |
206 | ||
207 | static inline void atomic64_add(long i, atomic64_t *v) | |
208 | { | |
209 | register long x0 asm ("x0") = i; | |
210 | register atomic64_t *x1 asm ("x1") = v; | |
211 | ||
212 | asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(add), | |
213 | " stadd %[i], %[v]\n") | |
214 | : [i] "+r" (x0), [v] "+Q" (v->counter) | |
215 | : "r" (x1) | |
216 | : "x30"); | |
217 | } | |
218 | ||
305d454a WD |
219 | #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \ |
220 | static inline long atomic64_add_return##name(long i, atomic64_t *v) \ | |
221 | { \ | |
222 | register long x0 asm ("x0") = i; \ | |
223 | register atomic64_t *x1 asm ("x1") = v; \ | |
224 | \ | |
225 | asm volatile(ARM64_LSE_ATOMIC_INSN( \ | |
226 | /* LL/SC */ \ | |
227 | " nop\n" \ | |
228 | __LL_SC_ATOMIC64(add_return##name), \ | |
229 | /* LSE atomics */ \ | |
230 | " ldadd" #mb " %[i], x30, %[v]\n" \ | |
231 | " add %[i], %[i], x30") \ | |
232 | : [i] "+r" (x0), [v] "+Q" (v->counter) \ | |
233 | : "r" (x1) \ | |
234 | : "x30" , ##cl); \ | |
235 | \ | |
236 | return x0; \ | |
237 | } | |
c09d6a04 | 238 | |
305d454a WD |
239 | ATOMIC64_OP_ADD_RETURN(_relaxed, ) |
240 | ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory") | |
241 | ATOMIC64_OP_ADD_RETURN(_release, l, "memory") | |
242 | ATOMIC64_OP_ADD_RETURN( , al, "memory") | |
c09d6a04 | 243 | |
305d454a | 244 | #undef ATOMIC64_OP_ADD_RETURN |
c09d6a04 WD |
245 | |
246 | static inline void atomic64_and(long i, atomic64_t *v) | |
247 | { | |
248 | register long x0 asm ("x0") = i; | |
249 | register atomic64_t *x1 asm ("x1") = v; | |
250 | ||
251 | asm volatile(ARM64_LSE_ATOMIC_INSN( | |
252 | /* LL/SC */ | |
253 | " nop\n" | |
254 | __LL_SC_ATOMIC64(and), | |
255 | /* LSE atomics */ | |
256 | " mvn %[i], %[i]\n" | |
257 | " stclr %[i], %[v]") | |
258 | : [i] "+r" (x0), [v] "+Q" (v->counter) | |
259 | : "r" (x1) | |
260 | : "x30"); | |
261 | } | |
262 | ||
263 | static inline void atomic64_sub(long i, atomic64_t *v) | |
264 | { | |
265 | register long x0 asm ("x0") = i; | |
266 | register atomic64_t *x1 asm ("x1") = v; | |
267 | ||
268 | asm volatile(ARM64_LSE_ATOMIC_INSN( | |
269 | /* LL/SC */ | |
270 | " nop\n" | |
271 | __LL_SC_ATOMIC64(sub), | |
272 | /* LSE atomics */ | |
273 | " neg %[i], %[i]\n" | |
274 | " stadd %[i], %[v]") | |
275 | : [i] "+r" (x0), [v] "+Q" (v->counter) | |
276 | : "r" (x1) | |
277 | : "x30"); | |
278 | } | |
279 | ||
305d454a WD |
280 | #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \ |
281 | static inline long atomic64_sub_return##name(long i, atomic64_t *v) \ | |
282 | { \ | |
283 | register long x0 asm ("x0") = i; \ | |
284 | register atomic64_t *x1 asm ("x1") = v; \ | |
285 | \ | |
286 | asm volatile(ARM64_LSE_ATOMIC_INSN( \ | |
287 | /* LL/SC */ \ | |
288 | " nop\n" \ | |
289 | __LL_SC_ATOMIC64(sub_return##name) \ | |
290 | " nop", \ | |
291 | /* LSE atomics */ \ | |
292 | " neg %[i], %[i]\n" \ | |
293 | " ldadd" #mb " %[i], x30, %[v]\n" \ | |
294 | " add %[i], %[i], x30") \ | |
295 | : [i] "+r" (x0), [v] "+Q" (v->counter) \ | |
296 | : "r" (x1) \ | |
297 | : "x30" , ##cl); \ | |
298 | \ | |
299 | return x0; \ | |
300 | } | |
c09d6a04 | 301 | |
305d454a WD |
302 | ATOMIC64_OP_SUB_RETURN(_relaxed, ) |
303 | ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory") | |
304 | ATOMIC64_OP_SUB_RETURN(_release, l, "memory") | |
305 | ATOMIC64_OP_SUB_RETURN( , al, "memory") | |
c09d6a04 | 306 | |
305d454a | 307 | #undef ATOMIC64_OP_SUB_RETURN |
c0385b24 WD |
308 | |
309 | static inline long atomic64_dec_if_positive(atomic64_t *v) | |
310 | { | |
c09d6a04 | 311 | register long x0 asm ("x0") = (long)v; |
c0385b24 | 312 | |
c09d6a04 WD |
313 | asm volatile(ARM64_LSE_ATOMIC_INSN( |
314 | /* LL/SC */ | |
315 | " nop\n" | |
316 | __LL_SC_ATOMIC64(dec_if_positive) | |
317 | " nop\n" | |
318 | " nop\n" | |
319 | " nop\n" | |
320 | " nop\n" | |
321 | " nop", | |
322 | /* LSE atomics */ | |
323 | "1: ldr x30, %[v]\n" | |
324 | " subs %[ret], x30, #1\n" | |
db26217e | 325 | " b.lt 2f\n" |
c09d6a04 WD |
326 | " casal x30, %[ret], %[v]\n" |
327 | " sub x30, x30, #1\n" | |
328 | " sub x30, x30, %[ret]\n" | |
329 | " cbnz x30, 1b\n" | |
330 | "2:") | |
331 | : [ret] "+&r" (x0), [v] "+Q" (v->counter) | |
c0385b24 WD |
332 | : |
333 | : "x30", "cc", "memory"); | |
334 | ||
335 | return x0; | |
336 | } | |
337 | ||
c09d6a04 WD |
338 | #undef __LL_SC_ATOMIC64 |
339 | ||
c342f782 WD |
340 | #define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op) |
341 | ||
342 | #define __CMPXCHG_CASE(w, sz, name, mb, cl...) \ | |
343 | static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \ | |
344 | unsigned long old, \ | |
345 | unsigned long new) \ | |
346 | { \ | |
347 | register unsigned long x0 asm ("x0") = (unsigned long)ptr; \ | |
348 | register unsigned long x1 asm ("x1") = old; \ | |
349 | register unsigned long x2 asm ("x2") = new; \ | |
350 | \ | |
351 | asm volatile(ARM64_LSE_ATOMIC_INSN( \ | |
352 | /* LL/SC */ \ | |
484c96db WD |
353 | " nop\n" \ |
354 | __LL_SC_CMPXCHG(name) \ | |
355 | " nop", \ | |
c342f782 WD |
356 | /* LSE atomics */ \ |
357 | " mov " #w "30, %" #w "[old]\n" \ | |
358 | " cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \ | |
359 | " mov %" #w "[ret], " #w "30") \ | |
360 | : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \ | |
361 | : [old] "r" (x1), [new] "r" (x2) \ | |
362 | : "x30" , ##cl); \ | |
363 | \ | |
364 | return x0; \ | |
365 | } | |
366 | ||
305d454a WD |
367 | __CMPXCHG_CASE(w, b, 1, ) |
368 | __CMPXCHG_CASE(w, h, 2, ) | |
369 | __CMPXCHG_CASE(w, , 4, ) | |
370 | __CMPXCHG_CASE(x, , 8, ) | |
371 | __CMPXCHG_CASE(w, b, acq_1, a, "memory") | |
372 | __CMPXCHG_CASE(w, h, acq_2, a, "memory") | |
373 | __CMPXCHG_CASE(w, , acq_4, a, "memory") | |
374 | __CMPXCHG_CASE(x, , acq_8, a, "memory") | |
375 | __CMPXCHG_CASE(w, b, rel_1, l, "memory") | |
376 | __CMPXCHG_CASE(w, h, rel_2, l, "memory") | |
377 | __CMPXCHG_CASE(w, , rel_4, l, "memory") | |
378 | __CMPXCHG_CASE(x, , rel_8, l, "memory") | |
379 | __CMPXCHG_CASE(w, b, mb_1, al, "memory") | |
380 | __CMPXCHG_CASE(w, h, mb_2, al, "memory") | |
381 | __CMPXCHG_CASE(w, , mb_4, al, "memory") | |
382 | __CMPXCHG_CASE(x, , mb_8, al, "memory") | |
c342f782 WD |
383 | |
384 | #undef __LL_SC_CMPXCHG | |
385 | #undef __CMPXCHG_CASE | |
386 | ||
e9a4b795 WD |
387 | #define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op) |
388 | ||
389 | #define __CMPXCHG_DBL(name, mb, cl...) \ | |
390 | static inline int __cmpxchg_double##name(unsigned long old1, \ | |
391 | unsigned long old2, \ | |
392 | unsigned long new1, \ | |
393 | unsigned long new2, \ | |
394 | volatile void *ptr) \ | |
395 | { \ | |
396 | unsigned long oldval1 = old1; \ | |
397 | unsigned long oldval2 = old2; \ | |
398 | register unsigned long x0 asm ("x0") = old1; \ | |
399 | register unsigned long x1 asm ("x1") = old2; \ | |
400 | register unsigned long x2 asm ("x2") = new1; \ | |
401 | register unsigned long x3 asm ("x3") = new2; \ | |
402 | register unsigned long x4 asm ("x4") = (unsigned long)ptr; \ | |
403 | \ | |
404 | asm volatile(ARM64_LSE_ATOMIC_INSN( \ | |
405 | /* LL/SC */ \ | |
406 | " nop\n" \ | |
407 | " nop\n" \ | |
408 | " nop\n" \ | |
409 | __LL_SC_CMPXCHG_DBL(name), \ | |
410 | /* LSE atomics */ \ | |
411 | " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\ | |
412 | " eor %[old1], %[old1], %[oldval1]\n" \ | |
413 | " eor %[old2], %[old2], %[oldval2]\n" \ | |
414 | " orr %[old1], %[old1], %[old2]") \ | |
415 | : [old1] "+r" (x0), [old2] "+r" (x1), \ | |
416 | [v] "+Q" (*(unsigned long *)ptr) \ | |
417 | : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \ | |
418 | [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \ | |
419 | : "x30" , ##cl); \ | |
420 | \ | |
421 | return x0; \ | |
422 | } | |
423 | ||
424 | __CMPXCHG_DBL( , ) | |
425 | __CMPXCHG_DBL(_mb, al, "memory") | |
426 | ||
427 | #undef __LL_SC_CMPXCHG_DBL | |
428 | #undef __CMPXCHG_DBL | |
429 | ||
c0385b24 | 430 | #endif /* __ASM_ATOMIC_LSE_H */ |