arch/mips/include/asm/cmpxchg.h

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
   7  */
   8 #ifndef __ASM_CMPXCHG_H
   9 #define __ASM_CMPXCHG_H
  10
  11 #include <linux/irqflags.h>
  12 #include <asm/war.h>
  13
  14 static inline unsigned long __xchg_u32(volatile int * m, unsigned int val)
  15 {
  16         __u32 retval;
  17
  18         smp_mb__before_llsc();
  19
  20         if (kernel_uses_llsc && R10000_LLSC_WAR) {
  21                 unsigned long dummy;
  22
  23                 __asm__ __volatile__(
  24                 "       .set    mips3                                   \n"
  25                 "1:     ll      %0, %3                  # xchg_u32      \n"
  26                 "       .set    mips0                                   \n"
  27                 "       move    %2, %z4                                 \n"
  28                 "       .set    mips3                                   \n"
  29                 "       sc      %2, %1                                  \n"
  30                 "       beqzl   %2, 1b                                  \n"
  31                 "       .set    mips0                                   \n"
  32                 : "=&r" (retval), "=m" (*m), "=&r" (dummy)
  33                 : "R" (*m), "Jr" (val)
  34                 : "memory");
  35         } else if (kernel_uses_llsc) {
  36                 unsigned long dummy;
  37
  38                 do {
  39                         __asm__ __volatile__(
  40                         "       .set    mips3                           \n"
  41                         "       ll      %0, %3          # xchg_u32      \n"
  42                         "       .set    mips0                           \n"
  43                         "       move    %2, %z4                         \n"
  44                         "       .set    mips3                           \n"
  45                         "       sc      %2, %1                          \n"
  46                         "       .set    mips0                           \n"
  47                         : "=&r" (retval), "=m" (*m), "=&r" (dummy)
  48                         : "R" (*m), "Jr" (val)
  49                         : "memory");
  50                 } while (unlikely(!dummy));
  51         } else {
  52                 unsigned long flags;
  53
  54                 raw_local_irq_save(flags);
  55                 retval = *m;
  56                 *m = val;
  57                 raw_local_irq_restore(flags);   /* implies memory barrier  */
  58         }
  59
  60         smp_llsc_mb();
  61
  62         return retval;
  63 }
  64
  65 #ifdef CONFIG_64BIT
  66 static inline __u64 __xchg_u64(volatile __u64 * m, __u64 val)
  67 {
  68         __u64 retval;
  69
  70         smp_mb__before_llsc();
  71
  72         if (kernel_uses_llsc && R10000_LLSC_WAR) {
  73                 unsigned long dummy;
  74
  75                 __asm__ __volatile__(
  76                 "       .set    mips3                                   \n"
  77                 "1:     lld     %0, %3                  # xchg_u64      \n"
  78                 "       move    %2, %z4                                 \n"
  79                 "       scd     %2, %1                                  \n"
  80                 "       beqzl   %2, 1b                                  \n"
  81                 "       .set    mips0                                   \n"
  82                 : "=&r" (retval), "=m" (*m), "=&r" (dummy)
  83                 : "R" (*m), "Jr" (val)
  84                 : "memory");
  85         } else if (kernel_uses_llsc) {
  86                 unsigned long dummy;
  87
  88                 do {
  89                         __asm__ __volatile__(
  90                         "       .set    mips3                           \n"
  91                         "       lld     %0, %3          # xchg_u64      \n"
  92                         "       move    %2, %z4                         \n"
  93                         "       scd     %2, %1                          \n"
  94                         "       .set    mips0                           \n"
  95                         : "=&r" (retval), "=m" (*m), "=&r" (dummy)
  96                         : "R" (*m), "Jr" (val)
  97                         : "memory");
  98                 } while (unlikely(!dummy));
  99         } else {
 100                 unsigned long flags;
 101
 102                 raw_local_irq_save(flags);
 103                 retval = *m;
 104                 *m = val;
 105                 raw_local_irq_restore(flags);   /* implies memory barrier  */
 106         }
 107
 108         smp_llsc_mb();
 109
 110         return retval;
 111 }
 112 #else
 113 extern __u64 __xchg_u64_unsupported_on_32bit_kernels(volatile __u64 * m, __u64 val);
 114 #define __xchg_u64 __xchg_u64_unsupported_on_32bit_kernels
 115 #endif
 116
 117 static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size)
 118 {
 119         switch (size) {
 120         case 4:
 121                 return __xchg_u32(ptr, x);
 122         case 8:
 123                 return __xchg_u64(ptr, x);
 124         }
 125
 126         return x;
 127 }
 128
 129 #define xchg(ptr, x)                                                    \
 130 ({                                                                      \
 131         BUILD_BUG_ON(sizeof(*(ptr)) & ~0xc);                            \
 132                                                                         \
 133         ((__typeof__(*(ptr)))                                           \
 134                 __xchg((unsigned long)(x), (ptr), sizeof(*(ptr))));     \
 135 })
 136
 137 #define __HAVE_ARCH_CMPXCHG 1
 138
 139 #define __cmpxchg_asm(ld, st, m, old, new)                              \
 140 ({                                                                      \
 141         __typeof(*(m)) __ret;                                           \
 142                                                                         \
 143         if (kernel_uses_llsc && R10000_LLSC_WAR) {                      \
 144                 __asm__ __volatile__(                                   \
 145                 "       .set    push                            \n"     \
 146                 "       .set    noat                            \n"     \
 147                 "       .set    mips3                           \n"     \
 148                 "1:     " ld "  %0, %2          # __cmpxchg_asm \n"     \
 149                 "       bne     %0, %z3, 2f                     \n"     \
 150                 "       .set    mips0                           \n"     \
 151                 "       move    $1, %z4                         \n"     \
 152                 "       .set    mips3                           \n"     \
 153                 "       " st "  $1, %1                          \n"     \
 154                 "       beqzl   $1, 1b                          \n"     \
 155                 "2:                                             \n"     \
 156                 "       .set    pop                             \n"     \
 157                 : "=&r" (__ret), "=R" (*m)                              \
 158                 : "R" (*m), "Jr" (old), "Jr" (new)                      \
 159                 : "memory");                                            \
 160         } else if (kernel_uses_llsc) {                                  \
 161                 __asm__ __volatile__(                                   \
 162                 "       .set    push                            \n"     \
 163                 "       .set    noat                            \n"     \
 164                 "       .set    mips3                           \n"     \
 165                 "1:     " ld "  %0, %2          # __cmpxchg_asm \n"     \
 166                 "       bne     %0, %z3, 2f                     \n"     \
 167                 "       .set    mips0                           \n"     \
 168                 "       move    $1, %z4                         \n"     \
 169                 "       .set    mips3                           \n"     \
 170                 "       " st "  $1, %1                          \n"     \
 171                 "       beqz    $1, 1b                          \n"     \
 172                 "       .set    pop                             \n"     \
 173                 "2:                                             \n"     \
 174                 : "=&r" (__ret), "=R" (*m)                              \
 175                 : "R" (*m), "Jr" (old), "Jr" (new)                      \
 176                 : "memory");                                            \
 177         } else {                                                        \
 178                 unsigned long __flags;                                  \
 179                                                                         \
 180                 raw_local_irq_save(__flags);                            \
 181                 __ret = *m;                                             \
 182                 if (__ret == old)                                       \
 183                         *m = new;                                       \
 184                 raw_local_irq_restore(__flags);                         \
 185         }                                                               \
 186                                                                         \
 187         __ret;                                                          \
 188 })
 189
 190 /*
 191  * This function doesn't exist, so you'll get a linker error
 192  * if something tries to do an invalid cmpxchg().
 193  */
 194 extern void __cmpxchg_called_with_bad_pointer(void);
 195
 196 #define __cmpxchg(ptr, old, new, pre_barrier, post_barrier)             \
 197 ({                                                                      \
 198         __typeof__(ptr) __ptr = (ptr);                                  \
 199         __typeof__(*(ptr)) __old = (old);                               \
 200         __typeof__(*(ptr)) __new = (new);                               \
 201         __typeof__(*(ptr)) __res = 0;                                   \
 202                                                                         \
 203         pre_barrier;                                                    \
 204                                                                         \
 205         switch (sizeof(*(__ptr))) {                                     \
 206         case 4:                                                         \
 207                 __res = __cmpxchg_asm("ll", "sc", __ptr, __old, __new); \
 208                 break;                                                  \
 209         case 8:                                                         \
 210                 if (sizeof(long) == 8) {                                \
 211                         __res = __cmpxchg_asm("lld", "scd", __ptr,      \
 212                                            __old, __new);               \
 213                         break;                                          \
 214                 }                                                       \
 215         default:                                                        \
 216                 __cmpxchg_called_with_bad_pointer();                    \
 217                 break;                                                  \
 218         }                                                               \
 219                                                                         \
 220         post_barrier;                                                   \
 221                                                                         \
 222         __res;                                                          \
 223 })
 224
 225 #define cmpxchg(ptr, old, new)          __cmpxchg(ptr, old, new, smp_mb__before_llsc(), smp_llsc_mb())
 226 #define cmpxchg_local(ptr, old, new)    __cmpxchg(ptr, old, new, , )
 227
 228 #define cmpxchg64(ptr, o, n)                                            \
 229   ({                                                                    \
 230         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 231         cmpxchg((ptr), (o), (n));                                       \
 232   })
 233
 234 #ifdef CONFIG_64BIT
 235 #define cmpxchg64_local(ptr, o, n)                                      \
 236   ({                                                                    \
 237         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 238         cmpxchg_local((ptr), (o), (n));                                 \
 239   })
 240 #else
 241 #include <asm-generic/cmpxchg-local.h>
 242 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
 243 #endif
 244
 245 #endif /* __ASM_CMPXCHG_H */