arch/arm64/include/asm/atomic_ll_sc.h

   1 /*
   2  * Based on arch/arm/include/asm/atomic.h
   3  *
   4  * Copyright (C) 1996 Russell King.
   5  * Copyright (C) 2002 Deep Blue Solutions Ltd.
   6  * Copyright (C) 2012 ARM Ltd.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #ifndef __ASM_ATOMIC_LL_SC_H
  22 #define __ASM_ATOMIC_LL_SC_H
  23
  24 #ifndef __ARM64_IN_ATOMIC_IMPL
  25 #error "please don't include this file directly"
  26 #endif
  27
  28 /*
  29  * AArch64 UP and SMP safe atomic ops.  We use load exclusive and
  30  * store exclusive to ensure that these are atomic.  We may loop
  31  * to ensure that the update happens.
  32  *
  33  * NOTE: these functions do *not* follow the PCS and must explicitly
  34  * save any clobbered registers other than x0 (regardless of return
  35  * value).  This is achieved through -fcall-saved-* compiler flags for
  36  * this file, which unfortunately don't work on a per-function basis
  37  * (the optimize attribute silently ignores these options).
  38  */
  39
  40 #define ATOMIC_OP(op, asm_op)                                           \
  41 __LL_SC_INLINE void                                                     \
  42 __LL_SC_PREFIX(atomic_##op(int i, atomic_t *v))                         \
  43 {                                                                       \
  44         unsigned long tmp;                                              \
  45         int result;                                                     \
  46                                                                         \
  47         asm volatile("// atomic_" #op "\n"                              \
  48 "1:     ldxr    %w0, %2\n"                                              \
  49 "       " #asm_op "     %w0, %w0, %w3\n"                                \
  50 "       stxr    %w1, %w0, %2\n"                                         \
  51 "       cbnz    %w1, 1b"                                                \
  52         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
  53         : "Ir" (i));                                                    \
  54 }                                                                       \
  55 __LL_SC_EXPORT(atomic_##op);
  56
  57 #define ATOMIC_OP_RETURN(op, asm_op)                                    \
  58 __LL_SC_INLINE int                                                      \
  59 __LL_SC_PREFIX(atomic_##op##_return(int i, atomic_t *v))                \
  60 {                                                                       \
  61         unsigned long tmp;                                              \
  62         int result;                                                     \
  63                                                                         \
  64         asm volatile("// atomic_" #op "_return\n"                       \
  65 "1:     ldxr    %w0, %2\n"                                              \
  66 "       " #asm_op "     %w0, %w0, %w3\n"                                \
  67 "       stlxr   %w1, %w0, %2\n"                                         \
  68 "       cbnz    %w1, 1b"                                                \
  69         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
  70         : "Ir" (i)                                                      \
  71         : "memory");                                                    \
  72                                                                         \
  73         smp_mb();                                                       \
  74         return result;                                                  \
  75 }                                                                       \
  76 __LL_SC_EXPORT(atomic_##op##_return);
  77
  78 #define ATOMIC_OPS(op, asm_op)                                          \
  79         ATOMIC_OP(op, asm_op)                                           \
  80         ATOMIC_OP_RETURN(op, asm_op)
  81
  82 ATOMIC_OPS(add, add)
  83 ATOMIC_OPS(sub, sub)
  84
  85 ATOMIC_OP(and, and)
  86 ATOMIC_OP(andnot, bic)
  87 ATOMIC_OP(or, orr)
  88 ATOMIC_OP(xor, eor)
  89
  90 #undef ATOMIC_OPS
  91 #undef ATOMIC_OP_RETURN
  92 #undef ATOMIC_OP
  93
  94 __LL_SC_INLINE int
  95 __LL_SC_PREFIX(atomic_cmpxchg(atomic_t *ptr, int old, int new))
  96 {
  97         unsigned long tmp;
  98         int oldval;
  99
 100         asm volatile("// atomic_cmpxchg\n"
 101 "1:     ldxr    %w1, %2\n"
 102 "       eor     %w0, %w1, %w3\n"
 103 "       cbnz    %w0, 2f\n"
 104 "       stlxr   %w0, %w4, %2\n"
 105 "       cbnz    %w0, 1b\n"
 106 "       dmb     ish\n"
 107 "2:"
 108         : "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
 109         : "Lr" (old), "r" (new)
 110         : "memory");
 111
 112         return oldval;
 113 }
 114 __LL_SC_EXPORT(atomic_cmpxchg);
 115
 116 #define ATOMIC64_OP(op, asm_op)                                         \
 117 __LL_SC_INLINE void                                                     \
 118 __LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v))                    \
 119 {                                                                       \
 120         long result;                                                    \
 121         unsigned long tmp;                                              \
 122                                                                         \
 123         asm volatile("// atomic64_" #op "\n"                            \
 124 "1:     ldxr    %0, %2\n"                                               \
 125 "       " #asm_op "     %0, %0, %3\n"                                   \
 126 "       stxr    %w1, %0, %2\n"                                          \
 127 "       cbnz    %w1, 1b"                                                \
 128         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
 129         : "Ir" (i));                                                    \
 130 }                                                                       \
 131 __LL_SC_EXPORT(atomic64_##op);
 132
 133 #define ATOMIC64_OP_RETURN(op, asm_op)                                  \
 134 __LL_SC_INLINE long                                                     \
 135 __LL_SC_PREFIX(atomic64_##op##_return(long i, atomic64_t *v))           \
 136 {                                                                       \
 137         long result;                                                    \
 138         unsigned long tmp;                                              \
 139                                                                         \
 140         asm volatile("// atomic64_" #op "_return\n"                     \
 141 "1:     ldxr    %0, %2\n"                                               \
 142 "       " #asm_op "     %0, %0, %3\n"                                   \
 143 "       stlxr   %w1, %0, %2\n"                                          \
 144 "       cbnz    %w1, 1b"                                                \
 145         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
 146         : "Ir" (i)                                                      \
 147         : "memory");                                                    \
 148                                                                         \
 149         smp_mb();                                                       \
 150         return result;                                                  \
 151 }                                                                       \
 152 __LL_SC_EXPORT(atomic64_##op##_return);
 153
 154 #define ATOMIC64_OPS(op, asm_op)                                        \
 155         ATOMIC64_OP(op, asm_op)                                         \
 156         ATOMIC64_OP_RETURN(op, asm_op)
 157
 158 ATOMIC64_OPS(add, add)
 159 ATOMIC64_OPS(sub, sub)
 160
 161 ATOMIC64_OP(and, and)
 162 ATOMIC64_OP(andnot, bic)
 163 ATOMIC64_OP(or, orr)
 164 ATOMIC64_OP(xor, eor)
 165
 166 #undef ATOMIC64_OPS
 167 #undef ATOMIC64_OP_RETURN
 168 #undef ATOMIC64_OP
 169
 170 __LL_SC_INLINE long
 171 __LL_SC_PREFIX(atomic64_cmpxchg(atomic64_t *ptr, long old, long new))
 172 {
 173         long oldval;
 174         unsigned long res;
 175
 176         asm volatile("// atomic64_cmpxchg\n"
 177 "1:     ldxr    %1, %2\n"
 178 "       eor     %0, %1, %3\n"
 179 "       cbnz    %w0, 2f\n"
 180 "       stlxr   %w0, %4, %2\n"
 181 "       cbnz    %w0, 1b\n"
 182 "       dmb     ish\n"
 183 "2:"
 184         : "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter)
 185         : "Lr" (old), "r" (new)
 186         : "memory");
 187
 188         return oldval;
 189 }
 190 __LL_SC_EXPORT(atomic64_cmpxchg);
 191
 192 __LL_SC_INLINE long
 193 __LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
 194 {
 195         long result;
 196         unsigned long tmp;
 197
 198         asm volatile("// atomic64_dec_if_positive\n"
 199 "1:     ldxr    %0, %2\n"
 200 "       subs    %0, %0, #1\n"
 201 "       b.mi    2f\n"
 202 "       stlxr   %w1, %0, %2\n"
 203 "       cbnz    %w1, 1b\n"
 204 "       dmb     ish\n"
 205 "2:"
 206         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
 207         :
 208         : "cc", "memory");
 209
 210         return result;
 211 }
 212 __LL_SC_EXPORT(atomic64_dec_if_positive);
 213
 214 #define __CMPXCHG_CASE(w, sz, name, mb, rel, cl)                        \
 215 __LL_SC_INLINE unsigned long                                            \
 216 __LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr,                \
 217                                      unsigned long old,                 \
 218                                      unsigned long new))                \
 219 {                                                                       \
 220         unsigned long tmp, oldval;                                      \
 221                                                                         \
 222         asm volatile(                                                   \
 223         "1:     ldxr" #sz "\t%" #w "[oldval], %[v]\n"                   \
 224         "       eor     %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n"  \
 225         "       cbnz    %" #w "[tmp], 2f\n"                             \
 226         "       st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n"     \
 227         "       cbnz    %w[tmp], 1b\n"                                  \
 228         "       " #mb "\n"                                              \
 229         "       mov     %" #w "[oldval], %" #w "[old]\n"                \
 230         "2:"                                                            \
 231         : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval),                   \
 232           [v] "+Q" (*(unsigned long *)ptr)                              \
 233         : [old] "Lr" (old), [new] "r" (new)                             \
 234         : cl);                                                          \
 235                                                                         \
 236         return oldval;                                                  \
 237 }                                                                       \
 238 __LL_SC_EXPORT(__cmpxchg_case_##name);
 239
 240 __CMPXCHG_CASE(w, b,    1,        ,  ,         )
 241 __CMPXCHG_CASE(w, h,    2,        ,  ,         )
 242 __CMPXCHG_CASE(w,  ,    4,        ,  ,         )
 243 __CMPXCHG_CASE( ,  ,    8,        ,  ,         )
 244 __CMPXCHG_CASE(w, b, mb_1, dmb ish, l, "memory")
 245 __CMPXCHG_CASE(w, h, mb_2, dmb ish, l, "memory")
 246 __CMPXCHG_CASE(w,  , mb_4, dmb ish, l, "memory")
 247 __CMPXCHG_CASE( ,  , mb_8, dmb ish, l, "memory")
 248
 249 #undef __CMPXCHG_CASE
 250
 251 #define __CMPXCHG_DBL(name, mb, rel, cl)                                \
 252 __LL_SC_INLINE int                                                      \
 253 __LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1,               \
 254                                       unsigned long old2,               \
 255                                       unsigned long new1,               \
 256                                       unsigned long new2,               \
 257                                       volatile void *ptr))              \
 258 {                                                                       \
 259         unsigned long tmp, ret;                                         \
 260                                                                         \
 261         asm volatile("// __cmpxchg_double" #name "\n"                   \
 262         "1:     ldxp    %0, %1, %2\n"                                   \
 263         "       eor     %0, %0, %3\n"                                   \
 264         "       eor     %1, %1, %4\n"                                   \
 265         "       orr     %1, %0, %1\n"                                   \
 266         "       cbnz    %1, 2f\n"                                       \
 267         "       st" #rel "xp    %w0, %5, %6, %2\n"                      \
 268         "       cbnz    %w0, 1b\n"                                      \
 269         "       " #mb "\n"                                              \
 270         "2:"                                                            \
 271         : "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr)        \
 272         : "r" (old1), "r" (old2), "r" (new1), "r" (new2)                \
 273         : cl);                                                          \
 274                                                                         \
 275         return ret;                                                     \
 276 }                                                                       \
 277 __LL_SC_EXPORT(__cmpxchg_double##name);
 278
 279 __CMPXCHG_DBL(   ,        ,  ,         )
 280 __CMPXCHG_DBL(_mb, dmb ish, l, "memory")
 281
 282 #undef __CMPXCHG_DBL
 283
 284 #endif  /* __ASM_ATOMIC_LL_SC_H */