1 // SPDX-License-Identifier: LGPL-2.1-only
15 #include <rseq/rseq.h>
21 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
23 struct percpu_lock_entry
{
25 } __attribute__((aligned(128)));
28 struct percpu_lock_entry c
[CPU_SETSIZE
];
31 struct test_data_entry
{
33 } __attribute__((aligned(128)));
35 struct spinlock_test_data
{
36 struct percpu_lock lock
;
37 struct test_data_entry c
[CPU_SETSIZE
];
41 struct percpu_list_node
{
43 struct percpu_list_node
*next
;
46 struct percpu_list_entry
{
47 struct percpu_list_node
*head
;
48 } __attribute__((aligned(128)));
51 struct percpu_list_entry c
[CPU_SETSIZE
];
54 /* A simple percpu spinlock. Returns the cpu lock was acquired on. */
55 static int rseq_this_cpu_lock(struct percpu_lock
*lock
)
62 cpu
= rseq_cpu_start();
63 ret
= rseq_cmpeqv_storev(&lock
->c
[cpu
].v
,
65 if (rseq_likely(!ret
))
67 /* Retry if comparison fails or rseq aborts. */
70 * Acquire semantic when taking lock after control dependency.
71 * Matches rseq_smp_store_release().
73 rseq_smp_acquire__after_ctrl_dep();
77 static void rseq_percpu_unlock(struct percpu_lock
*lock
, int cpu
)
79 assert(lock
->c
[cpu
].v
== 1);
81 * Release lock, with release semantic. Matches
82 * rseq_smp_acquire__after_ctrl_dep().
84 rseq_smp_store_release(&lock
->c
[cpu
].v
, 0);
87 static void *test_percpu_spinlock_thread(void *arg
)
89 struct spinlock_test_data
*data
= (struct spinlock_test_data
*) arg
;
92 if (rseq_register_current_thread()) {
93 fprintf(stderr
, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
94 errno
, strerror(errno
));
97 for (i
= 0; i
< data
->reps
; i
++) {
98 cpu
= rseq_this_cpu_lock(&data
->lock
);
100 rseq_percpu_unlock(&data
->lock
, cpu
);
102 if (rseq_unregister_current_thread()) {
103 fprintf(stderr
, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
104 errno
, strerror(errno
));
112 * A simple test which implements a sharded counter using a per-cpu
113 * lock. Obviously real applications might prefer to simply use a
114 * per-cpu increment; however, this is reasonable for a test and the
115 * lock can be extended to synchronize more complicated operations.
117 static void test_percpu_spinlock(void)
119 const int num_threads
= 200;
121 uint64_t sum
, expected_sum
;
122 pthread_t test_threads
[num_threads
];
123 struct spinlock_test_data data
;
127 memset(&data
, 0, sizeof(data
));
130 for (i
= 0; i
< num_threads
; i
++)
131 pthread_create(&test_threads
[i
], NULL
,
132 test_percpu_spinlock_thread
, &data
);
134 for (i
= 0; i
< num_threads
; i
++)
135 pthread_join(test_threads
[i
], NULL
);
138 for (i
= 0; i
< CPU_SETSIZE
; i
++)
139 sum
+= data
.c
[i
].count
;
141 expected_sum
= (uint64_t)data
.reps
* num_threads
;
143 ok(sum
== expected_sum
, "spinlock - sum (%" PRIu64
" == %" PRIu64
")", sum
, expected_sum
);
146 static void this_cpu_list_push(struct percpu_list
*list
,
147 struct percpu_list_node
*node
,
153 intptr_t *targetptr
, newval
, expect
;
156 cpu
= rseq_cpu_start();
157 /* Load list->c[cpu].head with single-copy atomicity. */
158 expect
= (intptr_t)RSEQ_READ_ONCE(list
->c
[cpu
].head
);
159 newval
= (intptr_t)node
;
160 targetptr
= (intptr_t *)&list
->c
[cpu
].head
;
161 node
->next
= (struct percpu_list_node
*)expect
;
162 ret
= rseq_cmpeqv_storev(targetptr
, expect
, newval
, cpu
);
163 if (rseq_likely(!ret
))
165 /* Retry if comparison fails or rseq aborts. */
172 * Unlike a traditional lock-less linked list; the availability of a
173 * rseq primitive allows us to implement pop without concerns over
176 static struct percpu_list_node
*this_cpu_list_pop(struct percpu_list
*list
,
180 struct percpu_list_node
*head
;
181 intptr_t *targetptr
, expectnot
, *load
;
182 int offset
, ret
, cpu
;
184 cpu
= rseq_cpu_start();
185 targetptr
= (intptr_t *)&list
->c
[cpu
].head
;
186 expectnot
= (intptr_t)NULL
;
187 offset
= offsetof(struct percpu_list_node
, next
);
188 load
= (intptr_t *)&head
;
189 ret
= rseq_cmpnev_storeoffp_load(targetptr
, expectnot
,
191 if (rseq_likely(!ret
)) {
198 /* Retry if rseq aborts. */
203 * __percpu_list_pop is not safe against concurrent accesses. Should
204 * only be used on lists that are not concurrently modified.
206 static struct percpu_list_node
*__percpu_list_pop(struct percpu_list
*list
, int cpu
)
208 struct percpu_list_node
*node
;
210 node
= list
->c
[cpu
].head
;
213 list
->c
[cpu
].head
= node
->next
;
217 static void *test_percpu_list_thread(void *arg
)
220 struct percpu_list
*list
= (struct percpu_list
*)arg
;
222 if (rseq_register_current_thread()) {
223 fprintf(stderr
, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
224 errno
, strerror(errno
));
228 for (i
= 0; i
< 100000; i
++) {
229 struct percpu_list_node
*node
;
231 node
= this_cpu_list_pop(list
, NULL
);
232 sched_yield(); /* encourage shuffling */
234 this_cpu_list_push(list
, node
, NULL
);
237 if (rseq_unregister_current_thread()) {
238 fprintf(stderr
, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
239 errno
, strerror(errno
));
246 /* Simultaneous modification to a per-cpu linked list from many threads. */
247 static void test_percpu_list(void)
250 uint64_t sum
= 0, expected_sum
= 0;
251 struct percpu_list list
;
252 pthread_t test_threads
[200];
253 cpu_set_t allowed_cpus
;
257 memset(&list
, 0, sizeof(list
));
259 /* Generate list entries for every usable cpu. */
260 sched_getaffinity(0, sizeof(allowed_cpus
), &allowed_cpus
);
261 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
262 if (!CPU_ISSET(i
, &allowed_cpus
))
264 for (j
= 1; j
<= 100; j
++) {
265 struct percpu_list_node
*node
;
269 node
= (struct percpu_list_node
*) malloc(sizeof(*node
));
272 node
->next
= list
.c
[i
].head
;
273 list
.c
[i
].head
= node
;
277 for (i
= 0; i
< 200; i
++)
278 pthread_create(&test_threads
[i
], NULL
,
279 test_percpu_list_thread
, &list
);
281 for (i
= 0; i
< 200; i
++)
282 pthread_join(test_threads
[i
], NULL
);
284 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
285 struct percpu_list_node
*node
;
287 if (!CPU_ISSET(i
, &allowed_cpus
))
290 while ((node
= __percpu_list_pop(&list
, i
))) {
297 * All entries should now be accounted for (unless some external
298 * actor is interfering with our allowed affinity while this
301 ok(sum
== expected_sum
, "percpu_list - sum (%" PRIu64
" == %" PRIu64
")", sum
, expected_sum
);
306 plan_tests(NR_TESTS
);
308 if (!rseq_available()) {
309 skip(NR_TESTS
, "The rseq syscall is unavailable");
313 if (rseq_register_current_thread()) {
314 fail("rseq_register_current_thread(...) failed(%d): %s\n",
315 errno
, strerror(errno
));
318 pass("Registered current thread with rseq");
321 test_percpu_spinlock();
324 if (rseq_unregister_current_thread()) {
325 fail("rseq_unregister_current_thread(...) failed(%d): %s\n",
326 errno
, strerror(errno
));
329 pass("Unregistered current thread with rseq");