Commit | Line | Data |
---|---|---|
a00736e9 SR |
1 | #ifndef _SPARC64_BACKOFF_H |
2 | #define _SPARC64_BACKOFF_H | |
3 | ||
187818cd DM |
4 | /* The macros in this file implement an exponential backoff facility |
5 | * for atomic operations. | |
6 | * | |
7 | * When multiple threads compete on an atomic operation, it is | |
8 | * possible for one thread to be continually denied a successful | |
9 | * completion of the compare-and-swap instruction. Heavily | |
10 | * threaded cpu implementations like Niagara can compound this | |
11 | * problem even further. | |
12 | * | |
13 | * When an atomic operation fails and needs to be retried, we spin a | |
14 | * certain number of times. At each subsequent failure of the same | |
15 | * operation we double the spin count, realizing an exponential | |
16 | * backoff. | |
17 | * | |
18 | * When we spin, we try to use an operation that will cause the | |
19 | * current cpu strand to block, and therefore make the core fully | |
20 | * available to any other other runnable strands. There are two | |
21 | * options, based upon cpu capabilities. | |
22 | * | |
23 | * On all cpus prior to SPARC-T4 we do three dummy reads of the | |
24 | * condition code register. Each read blocks the strand for something | |
25 | * between 40 and 50 cpu cycles. | |
26 | * | |
27 | * For SPARC-T4 and later we have a special "pause" instruction | |
28 | * available. This is implemented using writes to register %asr27. | |
29 | * The cpu will block the number of cycles written into the register, | |
30 | * unless a disrupting trap happens first. SPARC-T4 specifically | |
31 | * implements pause with a granularity of 8 cycles. Each strand has | |
32 | * an internal pause counter which decrements every 8 cycles. So the | |
33 | * chip shifts the %asr27 value down by 3 bits, and writes the result | |
34 | * into the pause counter. If a value smaller than 8 is written, the | |
35 | * chip blocks for 1 cycle. | |
36 | * | |
37 | * To achieve the same amount of backoff as the three %ccr reads give | |
38 | * on earlier chips, we shift the backoff value up by 7 bits. (Three | |
39 | * %ccr reads block for about 128 cycles, 1 << 7 == 128) We write the | |
40 | * whole amount we want to block into the pause register, rather than | |
41 | * loop writing 128 each time. | |
42 | */ | |
43 | ||
a00736e9 SR |
44 | #define BACKOFF_LIMIT (4 * 1024) |
45 | ||
46 | #ifdef CONFIG_SMP | |
47 | ||
48 | #define BACKOFF_SETUP(reg) \ | |
49 | mov 1, reg | |
50 | ||
0f58189d DM |
51 | #define BACKOFF_LABEL(spin_label, continue_label) \ |
52 | spin_label | |
53 | ||
e9b9eb59 DM |
54 | #define BACKOFF_SPIN(reg, tmp, label) \ |
55 | mov reg, tmp; \ | |
56 | 88: rd %ccr, %g0; \ | |
57 | rd %ccr, %g0; \ | |
58 | rd %ccr, %g0; \ | |
187818cd | 59 | .section .pause_3insn_patch,"ax";\ |
e9b9eb59 DM |
60 | .word 88b; \ |
61 | sllx tmp, 7, tmp; \ | |
62 | wr tmp, 0, %asr27; \ | |
63 | clr tmp; \ | |
64 | .previous; \ | |
65 | brnz,pt tmp, 88b; \ | |
66 | sub tmp, 1, tmp; \ | |
67 | set BACKOFF_LIMIT, tmp; \ | |
68 | cmp reg, tmp; \ | |
69 | bg,pn %xcc, label; \ | |
70 | nop; \ | |
71 | ba,pt %xcc, label; \ | |
72 | sllx reg, 1, reg; | |
a00736e9 SR |
73 | |
74 | #else | |
75 | ||
76 | #define BACKOFF_SETUP(reg) | |
0f58189d DM |
77 | |
78 | #define BACKOFF_LABEL(spin_label, continue_label) \ | |
79 | continue_label | |
80 | ||
81 | #define BACKOFF_SPIN(reg, tmp, label) | |
a00736e9 SR |
82 | |
83 | #endif | |
84 | ||
85 | #endif /* _SPARC64_BACKOFF_H */ |