1 // SPDX-License-Identifier: LGPL-2.1-only
6 #include <linux/membarrier.h>
16 #include <sys/types.h>
22 static int loop_cnt
[NR_INJECT
+ 1];
24 static int loop_cnt_1
asm("asm_loop_cnt_1") __attribute__((used
));
25 static int loop_cnt_2
asm("asm_loop_cnt_2") __attribute__((used
));
26 static int loop_cnt_3
asm("asm_loop_cnt_3") __attribute__((used
));
27 static int loop_cnt_4
asm("asm_loop_cnt_4") __attribute__((used
));
28 static int loop_cnt_5
asm("asm_loop_cnt_5") __attribute__((used
));
29 static int loop_cnt_6
asm("asm_loop_cnt_6") __attribute__((used
));
31 static int opt_modulo
, verbose
;
33 static int opt_yield
, opt_signal
, opt_sleep
,
34 opt_disable_rseq
, opt_threads
= 200,
35 opt_disable_mod
= 0, opt_test
= 's', opt_mb
= 0;
37 #ifndef RSEQ_SKIP_FASTPATH
38 static long long opt_reps
= 5000;
40 static long long opt_reps
= 100;
43 static __thread
__attribute__((tls_model("initial-exec")))
44 unsigned int signals_delivered
;
48 static inline pid_t
rseq_gettid(void)
50 return syscall(__NR_gettid
);
53 static __thread
__attribute__((tls_model("initial-exec"), unused
))
54 int yield_mod_cnt
, nr_abort
;
56 #define printf_verbose(fmt, ...) \
59 printf(fmt, ## __VA_ARGS__); \
64 #define INJECT_ASM_REG "eax"
66 #define RSEQ_INJECT_CLOBBER \
69 #define RSEQ_INJECT_ASM(n) \
70 "mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \
71 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
74 "dec %%" INJECT_ASM_REG "\n\t" \
78 #elif defined(__x86_64__)
80 #define INJECT_ASM_REG_P "rax"
81 #define INJECT_ASM_REG "eax"
83 #define RSEQ_INJECT_CLOBBER \
87 #define RSEQ_INJECT_ASM(n) \
88 "lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG_P "\n\t" \
89 "mov (%%" INJECT_ASM_REG_P "), %%" INJECT_ASM_REG "\n\t" \
90 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
93 "dec %%" INJECT_ASM_REG "\n\t" \
97 #elif defined(__s390__)
99 #define RSEQ_INJECT_INPUT \
100 , [loop_cnt_1]"m"(loop_cnt[1]) \
101 , [loop_cnt_2]"m"(loop_cnt[2]) \
102 , [loop_cnt_3]"m"(loop_cnt[3]) \
103 , [loop_cnt_4]"m"(loop_cnt[4]) \
104 , [loop_cnt_5]"m"(loop_cnt[5]) \
105 , [loop_cnt_6]"m"(loop_cnt[6])
107 #define INJECT_ASM_REG "r12"
109 #define RSEQ_INJECT_CLOBBER \
112 #define RSEQ_INJECT_ASM(n) \
113 "l %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
114 "ltr %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG "\n\t" \
117 "ahi %%" INJECT_ASM_REG ", -1\n\t" \
121 #elif defined(__ARMEL__)
123 #define RSEQ_INJECT_INPUT \
124 , [loop_cnt_1]"m"(loop_cnt[1]) \
125 , [loop_cnt_2]"m"(loop_cnt[2]) \
126 , [loop_cnt_3]"m"(loop_cnt[3]) \
127 , [loop_cnt_4]"m"(loop_cnt[4]) \
128 , [loop_cnt_5]"m"(loop_cnt[5]) \
129 , [loop_cnt_6]"m"(loop_cnt[6])
131 #define INJECT_ASM_REG "r4"
133 #define RSEQ_INJECT_CLOBBER \
136 #define RSEQ_INJECT_ASM(n) \
137 "ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
138 "cmp " INJECT_ASM_REG ", #0\n\t" \
141 "subs " INJECT_ASM_REG ", #1\n\t" \
145 #elif defined(__AARCH64EL__)
147 #define RSEQ_INJECT_INPUT \
148 , [loop_cnt_1] "Qo" (loop_cnt[1]) \
149 , [loop_cnt_2] "Qo" (loop_cnt[2]) \
150 , [loop_cnt_3] "Qo" (loop_cnt[3]) \
151 , [loop_cnt_4] "Qo" (loop_cnt[4]) \
152 , [loop_cnt_5] "Qo" (loop_cnt[5]) \
153 , [loop_cnt_6] "Qo" (loop_cnt[6])
155 #define INJECT_ASM_REG RSEQ_ASM_TMP_REG32
157 #define RSEQ_INJECT_ASM(n) \
158 " ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n" \
159 " cbz " INJECT_ASM_REG ", 333f\n" \
161 " sub " INJECT_ASM_REG ", " INJECT_ASM_REG ", #1\n" \
162 " cbnz " INJECT_ASM_REG ", 222b\n" \
165 #elif defined(__PPC__)
167 #define RSEQ_INJECT_INPUT \
168 , [loop_cnt_1]"m"(loop_cnt[1]) \
169 , [loop_cnt_2]"m"(loop_cnt[2]) \
170 , [loop_cnt_3]"m"(loop_cnt[3]) \
171 , [loop_cnt_4]"m"(loop_cnt[4]) \
172 , [loop_cnt_5]"m"(loop_cnt[5]) \
173 , [loop_cnt_6]"m"(loop_cnt[6])
175 #define INJECT_ASM_REG "r18"
177 #define RSEQ_INJECT_CLOBBER \
180 #define RSEQ_INJECT_ASM(n) \
181 "lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
182 "cmpwi %%" INJECT_ASM_REG ", 0\n\t" \
185 "subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \
189 #elif defined(__mips__)
191 #define RSEQ_INJECT_INPUT \
192 , [loop_cnt_1]"m"(loop_cnt[1]) \
193 , [loop_cnt_2]"m"(loop_cnt[2]) \
194 , [loop_cnt_3]"m"(loop_cnt[3]) \
195 , [loop_cnt_4]"m"(loop_cnt[4]) \
196 , [loop_cnt_5]"m"(loop_cnt[5]) \
197 , [loop_cnt_6]"m"(loop_cnt[6])
199 #define INJECT_ASM_REG "$5"
201 #define RSEQ_INJECT_CLOBBER \
204 #define RSEQ_INJECT_ASM(n) \
205 "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
206 "beqz " INJECT_ASM_REG ", 333f\n\t" \
208 "addiu " INJECT_ASM_REG ", -1\n\t" \
209 "bnez " INJECT_ASM_REG ", 222b\n\t" \
213 #error unsupported target
216 #define RSEQ_INJECT_FAILED \
219 #define RSEQ_INJECT_C(n) \
221 int loc_i, loc_nr_loops = loop_cnt[n]; \
223 for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \
226 if (loc_nr_loops == -1 && opt_modulo) { \
227 if (yield_mod_cnt == opt_modulo - 1) { \
229 poll(NULL, 0, opt_sleep); \
243 #define printf_verbose(fmt, ...)
245 #endif /* BENCHMARK */
247 #include <rseq/rseq.h>
249 struct percpu_lock_entry
{
251 } __attribute__((aligned(128)));
254 struct percpu_lock_entry c
[CPU_SETSIZE
];
257 struct test_data_entry
{
259 } __attribute__((aligned(128)));
261 struct spinlock_test_data
{
262 struct percpu_lock lock
;
263 struct test_data_entry c
[CPU_SETSIZE
];
266 struct spinlock_thread_test_data
{
267 struct spinlock_test_data
*data
;
272 struct inc_test_data
{
273 struct test_data_entry c
[CPU_SETSIZE
];
276 struct inc_thread_test_data
{
277 struct inc_test_data
*data
;
282 struct percpu_list_node
{
284 struct percpu_list_node
*next
;
287 struct percpu_list_entry
{
288 struct percpu_list_node
*head
;
289 } __attribute__((aligned(128)));
292 struct percpu_list_entry c
[CPU_SETSIZE
];
295 #define BUFFER_ITEM_PER_CPU 100
297 struct percpu_buffer_node
{
301 struct percpu_buffer_entry
{
304 struct percpu_buffer_node
**array
;
305 } __attribute__((aligned(128)));
307 struct percpu_buffer
{
308 struct percpu_buffer_entry c
[CPU_SETSIZE
];
311 #define MEMCPY_BUFFER_ITEM_PER_CPU 100
313 struct percpu_memcpy_buffer_node
{
318 struct percpu_memcpy_buffer_entry
{
321 struct percpu_memcpy_buffer_node
*array
;
322 } __attribute__((aligned(128)));
324 struct percpu_memcpy_buffer
{
325 struct percpu_memcpy_buffer_entry c
[CPU_SETSIZE
];
328 /* A simple percpu spinlock. Grabs lock on current cpu. */
329 static int rseq_this_cpu_lock(struct percpu_lock
*lock
)
336 cpu
= rseq_cpu_start();
337 ret
= rseq_cmpeqv_storev(&lock
->c
[cpu
].v
,
339 if (rseq_likely(!ret
))
341 /* Retry if comparison fails or rseq aborts. */
344 * Acquire semantic when taking lock after control dependency.
345 * Matches rseq_smp_store_release().
347 rseq_smp_acquire__after_ctrl_dep();
351 static void rseq_percpu_unlock(struct percpu_lock
*lock
, int cpu
)
353 assert(lock
->c
[cpu
].v
== 1);
355 * Release lock, with release semantic. Matches
356 * rseq_smp_acquire__after_ctrl_dep().
358 rseq_smp_store_release(&lock
->c
[cpu
].v
, 0);
361 static void *test_percpu_spinlock_thread(void *arg
)
363 struct spinlock_thread_test_data
*thread_data
= (struct spinlock_thread_test_data
*) arg
;
364 struct spinlock_test_data
*data
= thread_data
->data
;
367 if (!opt_disable_rseq
&& thread_data
->reg
&&
368 rseq_register_current_thread())
370 reps
= thread_data
->reps
;
371 for (i
= 0; i
< reps
; i
++) {
372 int cpu
= rseq_this_cpu_lock(&data
->lock
);
373 data
->c
[cpu
].count
++;
374 rseq_percpu_unlock(&data
->lock
, cpu
);
376 if (i
!= 0 && !(i
% (reps
/ 10)))
377 printf_verbose("tid %d: count %lld\n",
378 (int) rseq_gettid(), i
);
381 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
382 (int) rseq_gettid(), nr_abort
, signals_delivered
);
383 if (!opt_disable_rseq
&& thread_data
->reg
&&
384 rseq_unregister_current_thread())
390 * A simple test which implements a sharded counter using a per-cpu
391 * lock. Obviously real applications might prefer to simply use a
392 * per-cpu increment; however, this is reasonable for a test and the
393 * lock can be extended to synchronize more complicated operations.
395 static void test_percpu_spinlock(void)
397 const int num_threads
= opt_threads
;
400 pthread_t test_threads
[num_threads
];
401 struct spinlock_test_data data
;
402 struct spinlock_thread_test_data thread_data
[num_threads
];
404 memset(&data
, 0, sizeof(data
));
405 for (i
= 0; i
< num_threads
; i
++) {
406 thread_data
[i
].reps
= opt_reps
;
407 if (opt_disable_mod
<= 0 || (i
% opt_disable_mod
))
408 thread_data
[i
].reg
= 1;
410 thread_data
[i
].reg
= 0;
411 thread_data
[i
].data
= &data
;
412 ret
= pthread_create(&test_threads
[i
], NULL
,
413 test_percpu_spinlock_thread
,
417 perror("pthread_create");
422 for (i
= 0; i
< num_threads
; i
++) {
423 ret
= pthread_join(test_threads
[i
], NULL
);
426 perror("pthread_join");
432 for (i
= 0; i
< CPU_SETSIZE
; i
++)
433 sum
+= data
.c
[i
].count
;
435 assert(sum
== (uint64_t)opt_reps
* num_threads
);
438 static void *test_percpu_inc_thread(void *arg
)
440 struct inc_thread_test_data
*thread_data
= (struct inc_thread_test_data
*) arg
;
441 struct inc_test_data
*data
= thread_data
->data
;
444 if (!opt_disable_rseq
&& thread_data
->reg
&&
445 rseq_register_current_thread())
447 reps
= thread_data
->reps
;
448 for (i
= 0; i
< reps
; i
++) {
454 cpu
= rseq_cpu_start();
455 ret
= rseq_addv(&data
->c
[cpu
].count
, 1, cpu
);
456 } while (rseq_unlikely(ret
));
458 if (i
!= 0 && !(i
% (reps
/ 10)))
459 printf_verbose("tid %d: count %lld\n",
460 (int) rseq_gettid(), i
);
463 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
464 (int) rseq_gettid(), nr_abort
, signals_delivered
);
465 if (!opt_disable_rseq
&& thread_data
->reg
&&
466 rseq_unregister_current_thread())
471 static void test_percpu_inc(void)
473 const int num_threads
= opt_threads
;
476 pthread_t test_threads
[num_threads
];
477 struct inc_test_data data
;
478 struct inc_thread_test_data thread_data
[num_threads
];
480 memset(&data
, 0, sizeof(data
));
481 for (i
= 0; i
< num_threads
; i
++) {
482 thread_data
[i
].reps
= opt_reps
;
483 if (opt_disable_mod
<= 0 || (i
% opt_disable_mod
))
484 thread_data
[i
].reg
= 1;
486 thread_data
[i
].reg
= 0;
487 thread_data
[i
].data
= &data
;
488 ret
= pthread_create(&test_threads
[i
], NULL
,
489 test_percpu_inc_thread
,
493 perror("pthread_create");
498 for (i
= 0; i
< num_threads
; i
++) {
499 ret
= pthread_join(test_threads
[i
], NULL
);
502 perror("pthread_join");
508 for (i
= 0; i
< CPU_SETSIZE
; i
++)
509 sum
+= data
.c
[i
].count
;
511 assert(sum
== (uint64_t)opt_reps
* num_threads
);
514 static void this_cpu_list_push(struct percpu_list
*list
,
515 struct percpu_list_node
*node
,
521 intptr_t *targetptr
, newval
, expect
;
524 cpu
= rseq_cpu_start();
525 /* Load list->c[cpu].head with single-copy atomicity. */
526 expect
= (intptr_t)RSEQ_READ_ONCE(list
->c
[cpu
].head
);
527 newval
= (intptr_t)node
;
528 targetptr
= (intptr_t *)&list
->c
[cpu
].head
;
529 node
->next
= (struct percpu_list_node
*)expect
;
530 ret
= rseq_cmpeqv_storev(targetptr
, expect
, newval
, cpu
);
531 if (rseq_likely(!ret
))
533 /* Retry if comparison fails or rseq aborts. */
540 * Unlike a traditional lock-less linked list; the availability of a
541 * rseq primitive allows us to implement pop without concerns over
544 static struct percpu_list_node
*this_cpu_list_pop(struct percpu_list
*list
,
547 struct percpu_list_node
*node
= NULL
;
551 struct percpu_list_node
*head
;
552 intptr_t *targetptr
, expectnot
, *load
;
556 cpu
= rseq_cpu_start();
557 targetptr
= (intptr_t *)&list
->c
[cpu
].head
;
558 expectnot
= (intptr_t)NULL
;
559 offset
= offsetof(struct percpu_list_node
, next
);
560 load
= (intptr_t *)&head
;
561 ret
= rseq_cmpnev_storeoffp_load(targetptr
, expectnot
,
563 if (rseq_likely(!ret
)) {
569 /* Retry if rseq aborts. */
577 * __percpu_list_pop is not safe against concurrent accesses. Should
578 * only be used on lists that are not concurrently modified.
580 static struct percpu_list_node
*__percpu_list_pop(struct percpu_list
*list
, int cpu
)
582 struct percpu_list_node
*node
;
584 node
= list
->c
[cpu
].head
;
587 list
->c
[cpu
].head
= node
->next
;
591 static void *test_percpu_list_thread(void *arg
)
594 struct percpu_list
*list
= (struct percpu_list
*)arg
;
596 if (!opt_disable_rseq
&& rseq_register_current_thread())
600 for (i
= 0; i
< reps
; i
++) {
601 struct percpu_list_node
*node
;
603 node
= this_cpu_list_pop(list
, NULL
);
605 sched_yield(); /* encourage shuffling */
607 this_cpu_list_push(list
, node
, NULL
);
610 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
611 (int) rseq_gettid(), nr_abort
, signals_delivered
);
612 if (!opt_disable_rseq
&& rseq_unregister_current_thread())
618 /* Simultaneous modification to a per-cpu linked list from many threads. */
619 static void test_percpu_list(void)
621 const int num_threads
= opt_threads
;
623 uint64_t sum
= 0, expected_sum
= 0;
624 struct percpu_list list
;
625 pthread_t test_threads
[num_threads
];
626 cpu_set_t allowed_cpus
;
628 memset(&list
, 0, sizeof(list
));
630 /* Generate list entries for every usable cpu. */
631 sched_getaffinity(0, sizeof(allowed_cpus
), &allowed_cpus
);
632 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
633 if (!CPU_ISSET(i
, &allowed_cpus
))
635 for (j
= 1; j
<= 100; j
++) {
636 struct percpu_list_node
*node
;
640 node
= (struct percpu_list_node
*) malloc(sizeof(*node
));
643 node
->next
= list
.c
[i
].head
;
644 list
.c
[i
].head
= node
;
648 for (i
= 0; i
< num_threads
; i
++) {
649 ret
= pthread_create(&test_threads
[i
], NULL
,
650 test_percpu_list_thread
, &list
);
653 perror("pthread_create");
658 for (i
= 0; i
< num_threads
; i
++) {
659 ret
= pthread_join(test_threads
[i
], NULL
);
662 perror("pthread_join");
667 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
668 struct percpu_list_node
*node
;
670 if (!CPU_ISSET(i
, &allowed_cpus
))
673 while ((node
= __percpu_list_pop(&list
, i
))) {
680 * All entries should now be accounted for (unless some external
681 * actor is interfering with our allowed affinity while this
684 assert(sum
== expected_sum
);
687 static bool this_cpu_buffer_push(struct percpu_buffer
*buffer
,
688 struct percpu_buffer_node
*node
,
695 intptr_t *targetptr_spec
, newval_spec
;
696 intptr_t *targetptr_final
, newval_final
;
700 cpu
= rseq_cpu_start();
701 offset
= RSEQ_READ_ONCE(buffer
->c
[cpu
].offset
);
702 if (offset
== buffer
->c
[cpu
].buflen
)
704 newval_spec
= (intptr_t)node
;
705 targetptr_spec
= (intptr_t *)&buffer
->c
[cpu
].array
[offset
];
706 newval_final
= offset
+ 1;
707 targetptr_final
= &buffer
->c
[cpu
].offset
;
709 ret
= rseq_cmpeqv_trystorev_storev_release(
710 targetptr_final
, offset
, targetptr_spec
,
711 newval_spec
, newval_final
, cpu
);
713 ret
= rseq_cmpeqv_trystorev_storev(targetptr_final
,
714 offset
, targetptr_spec
, newval_spec
,
716 if (rseq_likely(!ret
)) {
720 /* Retry if comparison fails or rseq aborts. */
727 static struct percpu_buffer_node
*this_cpu_buffer_pop(struct percpu_buffer
*buffer
,
730 struct percpu_buffer_node
*head
;
734 intptr_t *targetptr
, newval
;
738 cpu
= rseq_cpu_start();
739 /* Load offset with single-copy atomicity. */
740 offset
= RSEQ_READ_ONCE(buffer
->c
[cpu
].offset
);
745 head
= RSEQ_READ_ONCE(buffer
->c
[cpu
].array
[offset
- 1]);
747 targetptr
= (intptr_t *)&buffer
->c
[cpu
].offset
;
748 ret
= rseq_cmpeqv_cmpeqv_storev(targetptr
, offset
,
749 (intptr_t *)&buffer
->c
[cpu
].array
[offset
- 1],
750 (intptr_t)head
, newval
, cpu
);
751 if (rseq_likely(!ret
))
753 /* Retry if comparison fails or rseq aborts. */
761 * __percpu_buffer_pop is not safe against concurrent accesses. Should
762 * only be used on buffers that are not concurrently modified.
764 static struct percpu_buffer_node
*__percpu_buffer_pop(struct percpu_buffer
*buffer
,
767 struct percpu_buffer_node
*head
;
770 offset
= buffer
->c
[cpu
].offset
;
773 head
= buffer
->c
[cpu
].array
[offset
- 1];
774 buffer
->c
[cpu
].offset
= offset
- 1;
778 static void *test_percpu_buffer_thread(void *arg
)
781 struct percpu_buffer
*buffer
= (struct percpu_buffer
*)arg
;
783 if (!opt_disable_rseq
&& rseq_register_current_thread())
787 for (i
= 0; i
< reps
; i
++) {
788 struct percpu_buffer_node
*node
;
790 node
= this_cpu_buffer_pop(buffer
, NULL
);
792 sched_yield(); /* encourage shuffling */
794 if (!this_cpu_buffer_push(buffer
, node
, NULL
)) {
795 /* Should increase buffer size. */
801 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
802 (int) rseq_gettid(), nr_abort
, signals_delivered
);
803 if (!opt_disable_rseq
&& rseq_unregister_current_thread())
809 /* Simultaneous modification to a per-cpu buffer from many threads. */
810 static void test_percpu_buffer(void)
812 const int num_threads
= opt_threads
;
814 uint64_t sum
= 0, expected_sum
= 0;
815 struct percpu_buffer buffer
;
816 pthread_t test_threads
[num_threads
];
817 cpu_set_t allowed_cpus
;
819 memset(&buffer
, 0, sizeof(buffer
));
821 /* Generate list entries for every usable cpu. */
822 sched_getaffinity(0, sizeof(allowed_cpus
), &allowed_cpus
);
823 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
824 if (!CPU_ISSET(i
, &allowed_cpus
))
826 /* Worse-case is every item in same CPU. */
828 (struct percpu_buffer_node
**)
829 malloc(sizeof(*buffer
.c
[i
].array
) * CPU_SETSIZE
*
830 BUFFER_ITEM_PER_CPU
);
831 assert(buffer
.c
[i
].array
);
832 buffer
.c
[i
].buflen
= CPU_SETSIZE
* BUFFER_ITEM_PER_CPU
;
833 for (j
= 1; j
<= BUFFER_ITEM_PER_CPU
; j
++) {
834 struct percpu_buffer_node
*node
;
839 * We could theoretically put the word-sized
840 * "data" directly in the buffer. However, we
841 * want to model objects that would not fit
842 * within a single word, so allocate an object
845 node
= (struct percpu_buffer_node
*) malloc(sizeof(*node
));
848 buffer
.c
[i
].array
[j
- 1] = node
;
849 buffer
.c
[i
].offset
++;
853 for (i
= 0; i
< num_threads
; i
++) {
854 ret
= pthread_create(&test_threads
[i
], NULL
,
855 test_percpu_buffer_thread
, &buffer
);
858 perror("pthread_create");
863 for (i
= 0; i
< num_threads
; i
++) {
864 ret
= pthread_join(test_threads
[i
], NULL
);
867 perror("pthread_join");
872 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
873 struct percpu_buffer_node
*node
;
875 if (!CPU_ISSET(i
, &allowed_cpus
))
878 while ((node
= __percpu_buffer_pop(&buffer
, i
))) {
882 free(buffer
.c
[i
].array
);
886 * All entries should now be accounted for (unless some external
887 * actor is interfering with our allowed affinity while this
890 assert(sum
== expected_sum
);
893 static bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer
*buffer
,
894 struct percpu_memcpy_buffer_node item
,
901 intptr_t *targetptr_final
, newval_final
, offset
;
902 char *destptr
, *srcptr
;
906 cpu
= rseq_cpu_start();
907 /* Load offset with single-copy atomicity. */
908 offset
= RSEQ_READ_ONCE(buffer
->c
[cpu
].offset
);
909 if (offset
== buffer
->c
[cpu
].buflen
)
911 destptr
= (char *)&buffer
->c
[cpu
].array
[offset
];
912 srcptr
= (char *)&item
;
913 /* copylen must be <= 4kB. */
914 copylen
= sizeof(item
);
915 newval_final
= offset
+ 1;
916 targetptr_final
= &buffer
->c
[cpu
].offset
;
918 ret
= rseq_cmpeqv_trymemcpy_storev_release(
919 targetptr_final
, offset
,
920 destptr
, srcptr
, copylen
,
923 ret
= rseq_cmpeqv_trymemcpy_storev(targetptr_final
,
924 offset
, destptr
, srcptr
, copylen
,
926 if (rseq_likely(!ret
)) {
930 /* Retry if comparison fails or rseq aborts. */
937 static bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer
*buffer
,
938 struct percpu_memcpy_buffer_node
*item
,
945 intptr_t *targetptr_final
, newval_final
, offset
;
946 char *destptr
, *srcptr
;
950 cpu
= rseq_cpu_start();
951 /* Load offset with single-copy atomicity. */
952 offset
= RSEQ_READ_ONCE(buffer
->c
[cpu
].offset
);
955 destptr
= (char *)item
;
956 srcptr
= (char *)&buffer
->c
[cpu
].array
[offset
- 1];
957 /* copylen must be <= 4kB. */
958 copylen
= sizeof(*item
);
959 newval_final
= offset
- 1;
960 targetptr_final
= &buffer
->c
[cpu
].offset
;
961 ret
= rseq_cmpeqv_trymemcpy_storev(targetptr_final
,
962 offset
, destptr
, srcptr
, copylen
,
964 if (rseq_likely(!ret
)) {
968 /* Retry if comparison fails or rseq aborts. */
976 * __percpu_memcpy_buffer_pop is not safe against concurrent accesses. Should
977 * only be used on buffers that are not concurrently modified.
979 static bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer
*buffer
,
980 struct percpu_memcpy_buffer_node
*item
,
985 offset
= buffer
->c
[cpu
].offset
;
988 memcpy(item
, &buffer
->c
[cpu
].array
[offset
- 1], sizeof(*item
));
989 buffer
->c
[cpu
].offset
= offset
- 1;
993 static void *test_percpu_memcpy_buffer_thread(void *arg
)
996 struct percpu_memcpy_buffer
*buffer
= (struct percpu_memcpy_buffer
*)arg
;
998 if (!opt_disable_rseq
&& rseq_register_current_thread())
1002 for (i
= 0; i
< reps
; i
++) {
1003 struct percpu_memcpy_buffer_node item
;
1006 result
= this_cpu_memcpy_buffer_pop(buffer
, &item
, NULL
);
1008 sched_yield(); /* encourage shuffling */
1010 if (!this_cpu_memcpy_buffer_push(buffer
, item
, NULL
)) {
1011 /* Should increase buffer size. */
1017 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
1018 (int) rseq_gettid(), nr_abort
, signals_delivered
);
1019 if (!opt_disable_rseq
&& rseq_unregister_current_thread())
1025 /* Simultaneous modification to a per-cpu buffer from many threads. */
1026 static void test_percpu_memcpy_buffer(void)
1028 const int num_threads
= opt_threads
;
1030 uint64_t sum
= 0, expected_sum
= 0;
1031 struct percpu_memcpy_buffer buffer
;
1032 pthread_t test_threads
[num_threads
];
1033 cpu_set_t allowed_cpus
;
1035 memset(&buffer
, 0, sizeof(buffer
));
1037 /* Generate list entries for every usable cpu. */
1038 sched_getaffinity(0, sizeof(allowed_cpus
), &allowed_cpus
);
1039 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
1040 if (!CPU_ISSET(i
, &allowed_cpus
))
1042 /* Worse-case is every item in same CPU. */
1044 (struct percpu_memcpy_buffer_node
*)
1045 malloc(sizeof(*buffer
.c
[i
].array
) * CPU_SETSIZE
*
1046 MEMCPY_BUFFER_ITEM_PER_CPU
);
1047 assert(buffer
.c
[i
].array
);
1048 buffer
.c
[i
].buflen
= CPU_SETSIZE
* MEMCPY_BUFFER_ITEM_PER_CPU
;
1049 for (j
= 1; j
<= MEMCPY_BUFFER_ITEM_PER_CPU
; j
++) {
1050 expected_sum
+= 2 * j
+ 1;
1053 * We could theoretically put the word-sized
1054 * "data" directly in the buffer. However, we
1055 * want to model objects that would not fit
1056 * within a single word, so allocate an object
1059 buffer
.c
[i
].array
[j
- 1].data1
= j
;
1060 buffer
.c
[i
].array
[j
- 1].data2
= j
+ 1;
1061 buffer
.c
[i
].offset
++;
1065 for (i
= 0; i
< num_threads
; i
++) {
1066 ret
= pthread_create(&test_threads
[i
], NULL
,
1067 test_percpu_memcpy_buffer_thread
,
1071 perror("pthread_create");
1076 for (i
= 0; i
< num_threads
; i
++) {
1077 ret
= pthread_join(test_threads
[i
], NULL
);
1080 perror("pthread_join");
1085 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
1086 struct percpu_memcpy_buffer_node item
;
1088 if (!CPU_ISSET(i
, &allowed_cpus
))
1091 while (__percpu_memcpy_buffer_pop(&buffer
, &item
, i
)) {
1095 free(buffer
.c
[i
].array
);
1099 * All entries should now be accounted for (unless some external
1100 * actor is interfering with our allowed affinity while this
1103 assert(sum
== expected_sum
);
1107 static void test_signal_interrupt_handler(__attribute__ ((unused
)) int signo
)
1109 signals_delivered
++;
1112 static int set_signal_handler(void)
1115 struct sigaction sa
;
1118 ret
= sigemptyset(&sigset
);
1120 perror("sigemptyset");
1124 sa
.sa_handler
= test_signal_interrupt_handler
;
1125 sa
.sa_mask
= sigset
;
1127 ret
= sigaction(SIGUSR1
, &sa
, NULL
);
1129 perror("sigaction");
1133 printf_verbose("Signal handler set for SIGUSR1\n");
1138 /* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */
1139 #ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
1140 struct test_membarrier_thread_args
{
1142 intptr_t percpu_list_ptr
;
1145 /* Worker threads modify data in their "active" percpu lists. */
1147 void *test_membarrier_worker_thread(void *arg
)
1149 struct test_membarrier_thread_args
*args
=
1150 (struct test_membarrier_thread_args
*)arg
;
1151 const int iters
= opt_reps
;
1154 if (rseq_register_current_thread()) {
1155 fprintf(stderr
, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
1156 errno
, strerror(errno
));
1160 /* Wait for initialization. */
1161 while (!rseq_smp_load_acquire(&args
->percpu_list_ptr
)) { }
1163 for (i
= 0; i
< iters
; ++i
) {
1167 int cpu
= rseq_cpu_start();
1169 ret
= rseq_offset_deref_addv(&args
->percpu_list_ptr
,
1170 sizeof(struct percpu_list_entry
) * cpu
, 1, cpu
);
1171 } while (rseq_unlikely(ret
));
1174 if (rseq_unregister_current_thread()) {
1175 fprintf(stderr
, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
1176 errno
, strerror(errno
));
1183 void test_membarrier_init_percpu_list(struct percpu_list
*list
)
1187 memset(list
, 0, sizeof(*list
));
1188 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
1189 struct percpu_list_node
*node
;
1191 node
= (struct percpu_list_node
*) malloc(sizeof(*node
));
1195 list
->c
[i
].head
= node
;
1200 void test_membarrier_free_percpu_list(struct percpu_list
*list
)
1204 for (i
= 0; i
< CPU_SETSIZE
; i
++)
1205 free(list
->c
[i
].head
);
1209 int sys_membarrier(int cmd
, int flags
, int cpu_id
)
1211 return syscall(__NR_membarrier
, cmd
, flags
, cpu_id
);
1215 * The manager thread swaps per-cpu lists that worker threads see,
1216 * and validates that there are no unexpected modifications.
1219 void *test_membarrier_manager_thread(void *arg
)
1221 struct test_membarrier_thread_args
*args
=
1222 (struct test_membarrier_thread_args
*)arg
;
1223 struct percpu_list list_a
, list_b
;
1224 intptr_t expect_a
= 0, expect_b
= 0;
1225 int cpu_a
= 0, cpu_b
= 0;
1227 if (rseq_register_current_thread()) {
1228 fprintf(stderr
, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
1229 errno
, strerror(errno
));
1234 test_membarrier_init_percpu_list(&list_a
);
1235 test_membarrier_init_percpu_list(&list_b
);
1237 /* Initialize lists before publishing them. */
1240 RSEQ_WRITE_ONCE(args
->percpu_list_ptr
, (intptr_t)&list_a
);
1242 while (!RSEQ_READ_ONCE(args
->stop
)) {
1243 /* list_a is "active". */
1244 cpu_a
= rand() % CPU_SETSIZE
;
1246 * As list_b is "inactive", we should never see changes
1249 if (expect_b
!= RSEQ_READ_ONCE(list_b
.c
[cpu_b
].head
->data
)) {
1250 fprintf(stderr
, "Membarrier test failed\n");
1254 /* Make list_b "active". */
1255 RSEQ_WRITE_ONCE(args
->percpu_list_ptr
, (intptr_t)&list_b
);
1256 if (sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ
,
1257 MEMBARRIER_CMD_FLAG_CPU
, cpu_a
) &&
1258 errno
!= ENXIO
/* missing CPU */) {
1259 perror("sys_membarrier");
1263 * Cpu A should now only modify list_b, so the values
1264 * in list_a should be stable.
1266 expect_a
= RSEQ_READ_ONCE(list_a
.c
[cpu_a
].head
->data
);
1268 cpu_b
= rand() % CPU_SETSIZE
;
1270 * As list_a is "inactive", we should never see changes
1273 if (expect_a
!= RSEQ_READ_ONCE(list_a
.c
[cpu_a
].head
->data
)) {
1274 fprintf(stderr
, "Membarrier test failed\n");
1278 /* Make list_a "active". */
1279 RSEQ_WRITE_ONCE(args
->percpu_list_ptr
, (intptr_t)&list_a
);
1280 if (sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ
,
1281 MEMBARRIER_CMD_FLAG_CPU
, cpu_b
) &&
1282 errno
!= ENXIO
/* missing CPU */) {
1283 perror("sys_membarrier");
1286 /* Remember a value from list_b. */
1287 expect_b
= RSEQ_READ_ONCE(list_b
.c
[cpu_b
].head
->data
);
1290 test_membarrier_free_percpu_list(&list_a
);
1291 test_membarrier_free_percpu_list(&list_b
);
1293 if (rseq_unregister_current_thread()) {
1294 fprintf(stderr
, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
1295 errno
, strerror(errno
));
1302 void test_membarrier(void)
1304 const int num_threads
= opt_threads
;
1305 struct test_membarrier_thread_args thread_args
;
1306 pthread_t worker_threads
[num_threads
];
1307 pthread_t manager_thread
;
1310 if (sys_membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ
, 0, 0)) {
1311 perror("sys_membarrier");
1315 thread_args
.stop
= 0;
1316 thread_args
.percpu_list_ptr
= 0;
1317 ret
= pthread_create(&manager_thread
, NULL
,
1318 test_membarrier_manager_thread
, &thread_args
);
1321 perror("pthread_create");
1325 for (i
= 0; i
< num_threads
; i
++) {
1326 ret
= pthread_create(&worker_threads
[i
], NULL
,
1327 test_membarrier_worker_thread
, &thread_args
);
1330 perror("pthread_create");
1336 for (i
= 0; i
< num_threads
; i
++) {
1337 ret
= pthread_join(worker_threads
[i
], NULL
);
1340 perror("pthread_join");
1345 RSEQ_WRITE_ONCE(thread_args
.stop
, 1);
1346 ret
= pthread_join(manager_thread
, NULL
);
1349 perror("pthread_join");
1353 #else /* RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV */
1355 void test_membarrier(void)
1357 fprintf(stderr
, "rseq_offset_deref_addv is not implemented on this architecture. "
1358 "Skipping membarrier test.\n");
1362 static void show_usage(char **argv
)
1364 printf("Usage : %s <OPTIONS>\n",
1366 printf("OPTIONS:\n");
1367 printf(" [-1 loops] Number of loops for delay injection 1\n");
1368 printf(" [-2 loops] Number of loops for delay injection 2\n");
1369 printf(" [-3 loops] Number of loops for delay injection 3\n");
1370 printf(" [-4 loops] Number of loops for delay injection 4\n");
1371 printf(" [-5 loops] Number of loops for delay injection 5\n");
1372 printf(" [-6 loops] Number of loops for delay injection 6\n");
1373 printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n");
1374 printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n");
1375 printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n");
1376 printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n");
1377 printf(" [-y] Yield\n");
1378 printf(" [-k] Kill thread with signal\n");
1379 printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n");
1380 printf(" [-t N] Number of threads (default 200)\n");
1381 printf(" [-r N] Number of repetitions per thread (default 5000)\n");
1382 printf(" [-d] Disable rseq system call (no initialization)\n");
1383 printf(" [-D M] Disable rseq for each M threads\n");
1384 printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\n");
1385 printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n");
1386 printf(" [-c] Check if the rseq syscall is available.\n");
1387 printf(" [-v] Verbose output.\n");
1388 printf(" [-h] Show this help.\n");
1392 int main(int argc
, char **argv
)
1396 for (i
= 1; i
< argc
; i
++) {
1397 if (argv
[i
][0] != '-')
1399 switch (argv
[i
][1]) {
1413 loop_cnt
[argv
[i
][1] - '0'] = atol(argv
[i
+ 1]);
1421 opt_modulo
= atol(argv
[i
+ 1]);
1422 if (opt_modulo
< 0) {
1433 opt_sleep
= atol(argv
[i
+ 1]);
1434 if (opt_sleep
< 0) {
1447 opt_disable_rseq
= 1;
1454 opt_disable_mod
= atol(argv
[i
+ 1]);
1455 if (opt_disable_mod
< 0) {
1466 opt_threads
= atol(argv
[i
+ 1]);
1467 if (opt_threads
< 0) {
1478 opt_reps
= atoll(argv
[i
+ 1]);
1493 opt_test
= *argv
[i
+ 1];
1515 if (rseq_available()) {
1516 printf_verbose("The rseq syscall is available.\n");
1519 printf_verbose("The rseq syscall is unavailable.\n");
1528 loop_cnt_1
= loop_cnt
[1];
1529 loop_cnt_2
= loop_cnt
[2];
1530 loop_cnt_3
= loop_cnt
[3];
1531 loop_cnt_4
= loop_cnt
[4];
1532 loop_cnt_5
= loop_cnt
[5];
1533 loop_cnt_6
= loop_cnt
[6];
1535 if (set_signal_handler())
1538 if (!opt_disable_rseq
&& rseq_register_current_thread())
1542 printf_verbose("spinlock\n");
1543 test_percpu_spinlock();
1546 printf_verbose("linked list\n");
1550 printf_verbose("buffer\n");
1551 test_percpu_buffer();
1554 printf_verbose("memcpy buffer\n");
1555 test_percpu_memcpy_buffer();
1558 printf_verbose("counter increment\n");
1562 printf_verbose("membarrier\n");
1566 if (!opt_disable_rseq
&& rseq_unregister_current_thread())