arch/arc/include/asm/bitops.h

   1 /*
   2  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #ifndef _ASM_BITOPS_H
  10 #define _ASM_BITOPS_H
  11
  12 #ifndef _LINUX_BITOPS_H
  13 #error only <linux/bitops.h> can be included directly
  14 #endif
  15
  16 #ifndef __ASSEMBLY__
  17
  18 #include <linux/types.h>
  19 #include <linux/compiler.h>
  20 #include <asm/barrier.h>
  21
  22 /*
  23  * Hardware assisted read-modify-write using ARC700 LLOCK/SCOND insns.
  24  * The Kconfig glue ensures that in SMP, this is only set if the container
  25  * SoC/platform has cross-core coherent LLOCK/SCOND
  26  */
  27 #if defined(CONFIG_ARC_HAS_LLSC)
  28
  29 static inline void set_bit(unsigned long nr, volatile unsigned long *m)
  30 {
  31         unsigned int temp;
  32
  33         m += nr >> 5;
  34
  35         /*
  36          * ARC ISA micro-optimization:
  37          *
  38          * Instructions dealing with bitpos only consider lower 5 bits (0-31)
  39          * e.g (x << 33) is handled like (x << 1) by ASL instruction
  40          *  (mem pointer still needs adjustment to point to next word)
  41          *
  42          * Hence the masking to clamp @nr arg can be elided in general.
  43          *
  44          * However if @nr is a constant (above assumed it in a register),
  45          * and greater than 31, gcc can optimize away (x << 33) to 0,
  46          * as overflow, given the 32-bit ISA. Thus masking needs to be done
  47          * for constant @nr, but no code is generated due to const prop.
  48          */
  49         if (__builtin_constant_p(nr))
  50                 nr &= 0x1f;
  51
  52         __asm__ __volatile__(
  53         "1:     llock   %0, [%1]        \n"
  54         "       bset    %0, %0, %2      \n"
  55         "       scond   %0, [%1]        \n"
  56         "       bnz     1b      \n"
  57         : "=&r"(temp)
  58         : "r"(m), "ir"(nr)
  59         : "cc");
  60 }
  61
  62 static inline void clear_bit(unsigned long nr, volatile unsigned long *m)
  63 {
  64         unsigned int temp;
  65
  66         m += nr >> 5;
  67
  68         if (__builtin_constant_p(nr))
  69                 nr &= 0x1f;
  70
  71         __asm__ __volatile__(
  72         "1:     llock   %0, [%1]        \n"
  73         "       bclr    %0, %0, %2      \n"
  74         "       scond   %0, [%1]        \n"
  75         "       bnz     1b      \n"
  76         : "=&r"(temp)
  77         : "r"(m), "ir"(nr)
  78         : "cc");
  79 }
  80
  81 static inline void change_bit(unsigned long nr, volatile unsigned long *m)
  82 {
  83         unsigned int temp;
  84
  85         m += nr >> 5;
  86
  87         if (__builtin_constant_p(nr))
  88                 nr &= 0x1f;
  89
  90         __asm__ __volatile__(
  91         "1:     llock   %0, [%1]        \n"
  92         "       bxor    %0, %0, %2      \n"
  93         "       scond   %0, [%1]        \n"
  94         "       bnz     1b              \n"
  95         : "=&r"(temp)
  96         : "r"(m), "ir"(nr)
  97         : "cc");
  98 }
  99
 100 /*
 101  * Semantically:
 102  *    Test the bit
 103  *    if clear
 104  *        set it and return 0 (old value)
 105  *    else
 106  *        return 1 (old value).
 107  *
 108  * Since ARC lacks a equivalent h/w primitive, the bit is set unconditionally
 109  * and the old value of bit is returned
 110  */
 111 static inline int test_and_set_bit(unsigned long nr, volatile unsigned long *m)
 112 {
 113         unsigned long old, temp;
 114
 115         m += nr >> 5;
 116
 117         if (__builtin_constant_p(nr))
 118                 nr &= 0x1f;
 119
 120         __asm__ __volatile__(
 121         "1:     llock   %0, [%2]        \n"
 122         "       bset    %1, %0, %3      \n"
 123         "       scond   %1, [%2]        \n"
 124         "       bnz     1b              \n"
 125         : "=&r"(old), "=&r"(temp)
 126         : "r"(m), "ir"(nr)
 127         : "cc");
 128
 129         return (old & (1 << nr)) != 0;
 130 }
 131
 132 static inline int
 133 test_and_clear_bit(unsigned long nr, volatile unsigned long *m)
 134 {
 135         unsigned int old, temp;
 136
 137         m += nr >> 5;
 138
 139         if (__builtin_constant_p(nr))
 140                 nr &= 0x1f;
 141
 142         __asm__ __volatile__(
 143         "1:     llock   %0, [%2]        \n"
 144         "       bclr    %1, %0, %3      \n"
 145         "       scond   %1, [%2]        \n"
 146         "       bnz     1b              \n"
 147         : "=&r"(old), "=&r"(temp)
 148         : "r"(m), "ir"(nr)
 149         : "cc");
 150
 151         return (old & (1 << nr)) != 0;
 152 }
 153
 154 static inline int
 155 test_and_change_bit(unsigned long nr, volatile unsigned long *m)
 156 {
 157         unsigned int old, temp;
 158
 159         m += nr >> 5;
 160
 161         if (__builtin_constant_p(nr))
 162                 nr &= 0x1f;
 163
 164         __asm__ __volatile__(
 165         "1:     llock   %0, [%2]        \n"
 166         "       bxor    %1, %0, %3      \n"
 167         "       scond   %1, [%2]        \n"
 168         "       bnz     1b              \n"
 169         : "=&r"(old), "=&r"(temp)
 170         : "r"(m), "ir"(nr)
 171         : "cc");
 172
 173         return (old & (1 << nr)) != 0;
 174 }
 175
 176 #else   /* !CONFIG_ARC_HAS_LLSC */
 177
 178 #include <asm/smp.h>
 179
 180 /*
 181  * Non hardware assisted Atomic-R-M-W
 182  * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
 183  *
 184  * There's "significant" micro-optimization in writing our own variants of
 185  * bitops (over generic variants)
 186  *
 187  * (1) The generic APIs have "signed" @nr while we have it "unsigned"
 188  *     This avoids extra code to be generated for pointer arithmatic, since
 189  *     is "not sure" that index is NOT -ve
 190  * (2) Utilize the fact that ARCompact bit fidding insn (BSET/BCLR/ASL) etc
 191  *     only consider bottom 5 bits of @nr, so NO need to mask them off.
 192  *     (GCC Quirk: however for constant @nr we still need to do the masking
 193  *             at compile time)
 194  */
 195
 196 static inline void set_bit(unsigned long nr, volatile unsigned long *m)
 197 {
 198         unsigned long temp, flags;
 199         m += nr >> 5;
 200
 201         if (__builtin_constant_p(nr))
 202                 nr &= 0x1f;
 203
 204         bitops_lock(flags);
 205
 206         temp = *m;
 207         *m = temp | (1UL << nr);
 208
 209         bitops_unlock(flags);
 210 }
 211
 212 static inline void clear_bit(unsigned long nr, volatile unsigned long *m)
 213 {
 214         unsigned long temp, flags;
 215         m += nr >> 5;
 216
 217         if (__builtin_constant_p(nr))
 218                 nr &= 0x1f;
 219
 220         bitops_lock(flags);
 221
 222         temp = *m;
 223         *m = temp & ~(1UL << nr);
 224
 225         bitops_unlock(flags);
 226 }
 227
 228 static inline void change_bit(unsigned long nr, volatile unsigned long *m)
 229 {
 230         unsigned long temp, flags;
 231         m += nr >> 5;
 232
 233         if (__builtin_constant_p(nr))
 234                 nr &= 0x1f;
 235
 236         bitops_lock(flags);
 237
 238         temp = *m;
 239         *m = temp ^ (1UL << nr);
 240
 241         bitops_unlock(flags);
 242 }
 243
 244 static inline int test_and_set_bit(unsigned long nr, volatile unsigned long *m)
 245 {
 246         unsigned long old, flags;
 247         m += nr >> 5;
 248
 249         if (__builtin_constant_p(nr))
 250                 nr &= 0x1f;
 251
 252         bitops_lock(flags);
 253
 254         old = *m;
 255         *m = old | (1 << nr);
 256
 257         bitops_unlock(flags);
 258
 259         return (old & (1 << nr)) != 0;
 260 }
 261
 262 static inline int
 263 test_and_clear_bit(unsigned long nr, volatile unsigned long *m)
 264 {
 265         unsigned long old, flags;
 266         m += nr >> 5;
 267
 268         if (__builtin_constant_p(nr))
 269                 nr &= 0x1f;
 270
 271         bitops_lock(flags);
 272
 273         old = *m;
 274         *m = old & ~(1 << nr);
 275
 276         bitops_unlock(flags);
 277
 278         return (old & (1 << nr)) != 0;
 279 }
 280
 281 static inline int
 282 test_and_change_bit(unsigned long nr, volatile unsigned long *m)
 283 {
 284         unsigned long old, flags;
 285         m += nr >> 5;
 286
 287         if (__builtin_constant_p(nr))
 288                 nr &= 0x1f;
 289
 290         bitops_lock(flags);
 291
 292         old = *m;
 293         *m = old ^ (1 << nr);
 294
 295         bitops_unlock(flags);
 296
 297         return (old & (1 << nr)) != 0;
 298 }
 299
 300 #endif /* CONFIG_ARC_HAS_LLSC */
 301
 302 /***************************************
 303  * Non atomic variants
 304  **************************************/
 305
 306 static inline void __set_bit(unsigned long nr, volatile unsigned long *m)
 307 {
 308         unsigned long temp;
 309         m += nr >> 5;
 310
 311         if (__builtin_constant_p(nr))
 312                 nr &= 0x1f;
 313
 314         temp = *m;
 315         *m = temp | (1UL << nr);
 316 }
 317
 318 static inline void __clear_bit(unsigned long nr, volatile unsigned long *m)
 319 {
 320         unsigned long temp;
 321         m += nr >> 5;
 322
 323         if (__builtin_constant_p(nr))
 324                 nr &= 0x1f;
 325
 326         temp = *m;
 327         *m = temp & ~(1UL << nr);
 328 }
 329
 330 static inline void __change_bit(unsigned long nr, volatile unsigned long *m)
 331 {
 332         unsigned long temp;
 333         m += nr >> 5;
 334
 335         if (__builtin_constant_p(nr))
 336                 nr &= 0x1f;
 337
 338         temp = *m;
 339         *m = temp ^ (1UL << nr);
 340 }
 341
 342 static inline int
 343 __test_and_set_bit(unsigned long nr, volatile unsigned long *m)
 344 {
 345         unsigned long old;
 346         m += nr >> 5;
 347
 348         if (__builtin_constant_p(nr))
 349                 nr &= 0x1f;
 350
 351         old = *m;
 352         *m = old | (1 << nr);
 353
 354         return (old & (1 << nr)) != 0;
 355 }
 356
 357 static inline int
 358 __test_and_clear_bit(unsigned long nr, volatile unsigned long *m)
 359 {
 360         unsigned long old;
 361         m += nr >> 5;
 362
 363         if (__builtin_constant_p(nr))
 364                 nr &= 0x1f;
 365
 366         old = *m;
 367         *m = old & ~(1 << nr);
 368
 369         return (old & (1 << nr)) != 0;
 370 }
 371
 372 static inline int
 373 __test_and_change_bit(unsigned long nr, volatile unsigned long *m)
 374 {
 375         unsigned long old;
 376         m += nr >> 5;
 377
 378         if (__builtin_constant_p(nr))
 379                 nr &= 0x1f;
 380
 381         old = *m;
 382         *m = old ^ (1 << nr);
 383
 384         return (old & (1 << nr)) != 0;
 385 }
 386
 387 /*
 388  * This routine doesn't need to be atomic.
 389  */
 390 static inline int
 391 test_bit(unsigned int nr, const volatile unsigned long *addr)
 392 {
 393         unsigned long mask;
 394
 395         addr += nr >> 5;
 396
 397         if (__builtin_constant_p(nr))
 398                 nr &= 0x1f;
 399
 400         mask = 1 << nr;
 401
 402         return ((mask & *addr) != 0);
 403 }
 404
 405 #ifdef CONFIG_ISA_ARCOMPACT
 406
 407 /*
 408  * Count the number of zeros, starting from MSB
 409  * Helper for fls( ) friends
 410  * This is a pure count, so (1-32) or (0-31) doesn't apply
 411  * It could be 0 to 32, based on num of 0's in there
 412  * clz(0x8000_0000) = 0, clz(0xFFFF_FFFF)=0, clz(0) = 32, clz(1) = 31
 413  */
 414 static inline __attribute__ ((const)) int clz(unsigned int x)
 415 {
 416         unsigned int res;
 417
 418         __asm__ __volatile__(
 419         "       norm.f  %0, %1          \n"
 420         "       mov.n   %0, 0           \n"
 421         "       add.p   %0, %0, 1       \n"
 422         : "=r"(res)
 423         : "r"(x)
 424         : "cc");
 425
 426         return res;
 427 }
 428
 429 static inline int constant_fls(int x)
 430 {
 431         int r = 32;
 432
 433         if (!x)
 434                 return 0;
 435         if (!(x & 0xffff0000u)) {
 436                 x <<= 16;
 437                 r -= 16;
 438         }
 439         if (!(x & 0xff000000u)) {
 440                 x <<= 8;
 441                 r -= 8;
 442         }
 443         if (!(x & 0xf0000000u)) {
 444                 x <<= 4;
 445                 r -= 4;
 446         }
 447         if (!(x & 0xc0000000u)) {
 448                 x <<= 2;
 449                 r -= 2;
 450         }
 451         if (!(x & 0x80000000u)) {
 452                 x <<= 1;
 453                 r -= 1;
 454         }
 455         return r;
 456 }
 457
 458 /*
 459  * fls = Find Last Set in word
 460  * @result: [1-32]
 461  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
 462  */
 463 static inline __attribute__ ((const)) int fls(unsigned long x)
 464 {
 465         if (__builtin_constant_p(x))
 466                return constant_fls(x);
 467
 468         return 32 - clz(x);
 469 }
 470
 471 /*
 472  * __fls: Similar to fls, but zero based (0-31)
 473  */
 474 static inline __attribute__ ((const)) int __fls(unsigned long x)
 475 {
 476         if (!x)
 477                 return 0;
 478         else
 479                 return fls(x) - 1;
 480 }
 481
 482 /*
 483  * ffs = Find First Set in word (LSB to MSB)
 484  * @result: [1-32], 0 if all 0's
 485  */
 486 #define ffs(x)  ({ unsigned long __t = (x); fls(__t & -__t); })
 487
 488 /*
 489  * __ffs: Similar to ffs, but zero based (0-31)
 490  */
 491 static inline __attribute__ ((const)) int __ffs(unsigned long word)
 492 {
 493         if (!word)
 494                 return word;
 495
 496         return ffs(word) - 1;
 497 }
 498
 499 #else   /* CONFIG_ISA_ARCV2 */
 500
 501 /*
 502  * fls = Find Last Set in word
 503  * @result: [1-32]
 504  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
 505  */
 506 static inline __attribute__ ((const)) int fls(unsigned long x)
 507 {
 508         int n;
 509
 510         asm volatile(
 511         "       fls.f   %0, %1          \n"  /* 0:31; 0(Z) if src 0 */
 512         "       add.nz  %0, %0, 1       \n"  /* 0:31 -> 1:32 */
 513         : "=r"(n)       /* Early clobber not needed */
 514         : "r"(x)
 515         : "cc");
 516
 517         return n;
 518 }
 519
 520 /*
 521  * __fls: Similar to fls, but zero based (0-31). Also 0 if no bit set
 522  */
 523 static inline __attribute__ ((const)) int __fls(unsigned long x)
 524 {
 525         /* FLS insn has exactly same semantics as the API */
 526         return  __builtin_arc_fls(x);
 527 }
 528
 529 /*
 530  * ffs = Find First Set in word (LSB to MSB)
 531  * @result: [1-32], 0 if all 0's
 532  */
 533 static inline __attribute__ ((const)) int ffs(unsigned long x)
 534 {
 535         int n;
 536
 537         asm volatile(
 538         "       ffs.f   %0, %1          \n"  /* 0:31; 31(Z) if src 0 */
 539         "       add.nz  %0, %0, 1       \n"  /* 0:31 -> 1:32 */
 540         "       mov.z   %0, 0           \n"  /* 31(Z)-> 0 */
 541         : "=r"(n)       /* Early clobber not needed */
 542         : "r"(x)
 543         : "cc");
 544
 545         return n;
 546 }
 547
 548 /*
 549  * __ffs: Similar to ffs, but zero based (0-31)
 550  */
 551 static inline __attribute__ ((const)) int __ffs(unsigned long x)
 552 {
 553         int n;
 554
 555         asm volatile(
 556         "       ffs.f   %0, %1          \n"  /* 0:31; 31(Z) if src 0 */
 557         "       mov.z   %0, 0           \n"  /* 31(Z)-> 0 */
 558         : "=r"(n)
 559         : "r"(x)
 560         : "cc");
 561
 562         return n;
 563
 564 }
 565
 566 #endif  /* CONFIG_ISA_ARCOMPACT */
 567
 568 /*
 569  * ffz = Find First Zero in word.
 570  * @return:[0-31], 32 if all 1's
 571  */
 572 #define ffz(x)  __ffs(~(x))
 573
 574 #include <asm-generic/bitops/hweight.h>
 575 #include <asm-generic/bitops/fls64.h>
 576 #include <asm-generic/bitops/sched.h>
 577 #include <asm-generic/bitops/lock.h>
 578
 579 #include <asm-generic/bitops/find.h>
 580 #include <asm-generic/bitops/le.h>
 581 #include <asm-generic/bitops/ext2-atomic-setbit.h>
 582
 583 #endif /* !__ASSEMBLY__ */
 584
 585 #endif