1 // SPDX-License-Identifier: MIT
2 // SPDX-FileCopyrightText: 2020-2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
7 #include <linux/version.h>
8 #include <linux/membarrier.h>
18 #include <sys/types.h>
23 #include <rseq/mempool.h>
25 #if LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0)
27 MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ
= (1 << 7),
28 MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ
= (1 << 8),
32 MEMBARRIER_CMD_FLAG_CPU
= (1 << 0),
37 static int loop_cnt
[NR_INJECT
+ 1];
39 static int loop_cnt_1
asm("asm_loop_cnt_1") __attribute__((used
));
40 static int loop_cnt_2
asm("asm_loop_cnt_2") __attribute__((used
));
41 static int loop_cnt_3
asm("asm_loop_cnt_3") __attribute__((used
));
42 static int loop_cnt_4
asm("asm_loop_cnt_4") __attribute__((used
));
43 static int loop_cnt_5
asm("asm_loop_cnt_5") __attribute__((used
));
44 static int loop_cnt_6
asm("asm_loop_cnt_6") __attribute__((used
));
46 static int opt_modulo
, verbose
;
48 static int opt_yield
, opt_signal
, opt_sleep
,
49 opt_disable_rseq
, opt_threads
= 200,
50 opt_disable_mod
= 0, opt_test
= 's';
52 static long long opt_reps
= 5000;
54 static __thread
__attribute__((tls_model("initial-exec")))
55 unsigned int signals_delivered
;
57 static inline pid_t
rseq_gettid(void)
59 return syscall(__NR_gettid
);
64 static __thread
__attribute__((tls_model("initial-exec"), unused
))
65 int yield_mod_cnt
, nr_abort
;
67 #define printf_verbose(fmt, ...) \
70 printf(fmt, ## __VA_ARGS__); \
75 #define INJECT_ASM_REG "eax"
77 #define RSEQ_INJECT_CLOBBER \
81 * Use ip-relative addressing to get the loop counter.
83 #define __RSEQ_INJECT_ASM(n, ref_ip, ref_label) \
84 "movl " __rseq_str(ref_ip) ", %%" INJECT_ASM_REG "\n\t" \
85 "leal ( asm_loop_cnt_" #n " - " __rseq_str(ref_label) "b)(%%" INJECT_ASM_REG "), %%" INJECT_ASM_REG "\n\t" \
86 "movl (%%" INJECT_ASM_REG "), %%" INJECT_ASM_REG "\n\t" \
87 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
90 "dec %%" INJECT_ASM_REG "\n\t" \
94 #define RSEQ_INJECT_ASM(n) \
95 __RSEQ_INJECT_ASM(n, %[ref_ip], RSEQ_ASM_REF_LABEL)
97 #elif defined(__x86_64__)
99 #define INJECT_ASM_REG_P "rax"
100 #define INJECT_ASM_REG "eax"
102 #define RSEQ_INJECT_CLOBBER \
106 #define RSEQ_INJECT_ASM(n) \
107 "lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG_P "\n\t" \
108 "mov (%%" INJECT_ASM_REG_P "), %%" INJECT_ASM_REG "\n\t" \
109 "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
112 "dec %%" INJECT_ASM_REG "\n\t" \
116 #elif defined(__s390__)
118 #define RSEQ_INJECT_INPUT \
119 , [loop_cnt_1]"m"(loop_cnt[1]) \
120 , [loop_cnt_2]"m"(loop_cnt[2]) \
121 , [loop_cnt_3]"m"(loop_cnt[3]) \
122 , [loop_cnt_4]"m"(loop_cnt[4]) \
123 , [loop_cnt_5]"m"(loop_cnt[5]) \
124 , [loop_cnt_6]"m"(loop_cnt[6])
126 #define INJECT_ASM_REG "r12"
128 #define RSEQ_INJECT_CLOBBER \
131 #define RSEQ_INJECT_ASM(n) \
132 "l %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
133 "ltr %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG "\n\t" \
136 "ahi %%" INJECT_ASM_REG ", -1\n\t" \
140 #elif defined(__ARMEL__)
142 #define RSEQ_INJECT_INPUT \
143 , [loop_cnt_1]"m"(loop_cnt[1]) \
144 , [loop_cnt_2]"m"(loop_cnt[2]) \
145 , [loop_cnt_3]"m"(loop_cnt[3]) \
146 , [loop_cnt_4]"m"(loop_cnt[4]) \
147 , [loop_cnt_5]"m"(loop_cnt[5]) \
148 , [loop_cnt_6]"m"(loop_cnt[6])
150 #define INJECT_ASM_REG "r4"
152 #define RSEQ_INJECT_CLOBBER \
155 #define RSEQ_INJECT_ASM(n) \
156 "ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
157 "cmp " INJECT_ASM_REG ", #0\n\t" \
160 "subs " INJECT_ASM_REG ", #1\n\t" \
164 #elif defined(__AARCH64EL__)
166 #define RSEQ_INJECT_INPUT \
167 , [loop_cnt_1] "Qo" (loop_cnt[1]) \
168 , [loop_cnt_2] "Qo" (loop_cnt[2]) \
169 , [loop_cnt_3] "Qo" (loop_cnt[3]) \
170 , [loop_cnt_4] "Qo" (loop_cnt[4]) \
171 , [loop_cnt_5] "Qo" (loop_cnt[5]) \
172 , [loop_cnt_6] "Qo" (loop_cnt[6])
174 #define INJECT_ASM_REG RSEQ_ASM_TMP_REG32
176 #define RSEQ_INJECT_ASM(n) \
177 " ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n" \
178 " cbz " INJECT_ASM_REG ", 333f\n" \
180 " sub " INJECT_ASM_REG ", " INJECT_ASM_REG ", #1\n" \
181 " cbnz " INJECT_ASM_REG ", 222b\n" \
184 #elif defined(__PPC__)
186 #define RSEQ_INJECT_INPUT \
187 , [loop_cnt_1]"m"(loop_cnt[1]) \
188 , [loop_cnt_2]"m"(loop_cnt[2]) \
189 , [loop_cnt_3]"m"(loop_cnt[3]) \
190 , [loop_cnt_4]"m"(loop_cnt[4]) \
191 , [loop_cnt_5]"m"(loop_cnt[5]) \
192 , [loop_cnt_6]"m"(loop_cnt[6])
194 #define INJECT_ASM_REG "r18"
196 #define RSEQ_INJECT_CLOBBER \
199 #define RSEQ_INJECT_ASM(n) \
200 "lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
201 "cmpwi %%" INJECT_ASM_REG ", 0\n\t" \
204 "subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \
208 #elif defined(__mips__)
210 #define RSEQ_INJECT_INPUT \
211 , [loop_cnt_1]"m"(loop_cnt[1]) \
212 , [loop_cnt_2]"m"(loop_cnt[2]) \
213 , [loop_cnt_3]"m"(loop_cnt[3]) \
214 , [loop_cnt_4]"m"(loop_cnt[4]) \
215 , [loop_cnt_5]"m"(loop_cnt[5]) \
216 , [loop_cnt_6]"m"(loop_cnt[6])
218 #define INJECT_ASM_REG "$5"
220 #define RSEQ_INJECT_CLOBBER \
223 #define RSEQ_INJECT_ASM(n) \
224 "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
225 "beqz " INJECT_ASM_REG ", 333f\n\t" \
227 "addiu " INJECT_ASM_REG ", -1\n\t" \
228 "bnez " INJECT_ASM_REG ", 222b\n\t" \
231 #elif defined(__riscv)
233 #define RSEQ_INJECT_INPUT \
234 , [loop_cnt_1]"m"(loop_cnt[1]) \
235 , [loop_cnt_2]"m"(loop_cnt[2]) \
236 , [loop_cnt_3]"m"(loop_cnt[3]) \
237 , [loop_cnt_4]"m"(loop_cnt[4]) \
238 , [loop_cnt_5]"m"(loop_cnt[5]) \
239 , [loop_cnt_6]"m"(loop_cnt[6])
241 #define INJECT_ASM_REG "t1"
243 #define RSEQ_INJECT_CLOBBER \
246 #define RSEQ_INJECT_ASM(n) \
247 "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
248 "beqz " INJECT_ASM_REG ", 333f\n\t" \
250 "addi " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t" \
251 "bnez " INJECT_ASM_REG ", 222b\n\t" \
255 #error unsupported target
258 #define RSEQ_INJECT_FAILED \
261 #define RSEQ_INJECT_C(n) \
263 int loc_i, loc_nr_loops = loop_cnt[n]; \
265 for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \
268 if (loc_nr_loops == -1 && opt_modulo) { \
269 if (yield_mod_cnt == opt_modulo - 1) { \
271 poll(NULL, 0, opt_sleep); \
285 #define printf_verbose(fmt, ...)
287 #endif /* BENCHMARK */
289 #include <rseq/rseq.h>
291 static enum rseq_mo opt_mo
= RSEQ_MO_RELAXED
;
293 static int sys_membarrier(int cmd
, int flags
, int cpu_id
)
295 return syscall(__NR_membarrier
, cmd
, flags
, cpu_id
);
298 #ifdef rseq_arch_has_load_cbne_load_add_load_add_store
299 #define TEST_MEMBARRIER
302 #ifdef BUILDOPT_RSEQ_PERCPU_MM_CID
303 # define RSEQ_PERCPU RSEQ_PERCPU_MM_CID
305 int get_current_cpu_id(void)
307 return rseq_current_mm_cid();
310 bool rseq_validate_cpu_id(void)
312 return rseq_mm_cid_available();
315 bool rseq_use_cpu_index(void)
317 return false; /* Use mm_cid */
319 # ifdef TEST_MEMBARRIER
321 * Membarrier does not currently support targeting a mm_cid, so
322 * issue the barrier on all cpus.
325 int rseq_membarrier_expedited(__attribute__ ((unused
)) int cpu
)
327 return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ
,
330 # endif /* TEST_MEMBARRIER */
332 # define RSEQ_PERCPU RSEQ_PERCPU_CPU_ID
334 int get_current_cpu_id(void)
336 return rseq_cpu_start();
339 bool rseq_validate_cpu_id(void)
341 return rseq_current_cpu_raw() >= 0;
344 bool rseq_use_cpu_index(void)
346 return true; /* Use cpu_id as index. */
348 # ifdef TEST_MEMBARRIER
350 int rseq_membarrier_expedited(int cpu
)
352 return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ
,
353 MEMBARRIER_CMD_FLAG_CPU
, cpu
);
355 # endif /* TEST_MEMBARRIER */
362 struct spinlock_test_data
{
363 struct percpu_lock lock
;
367 struct spinlock_thread_test_data
{
368 struct spinlock_test_data __rseq_percpu
*data
;
373 struct inc_test_data
{
377 struct inc_thread_test_data
{
378 struct inc_test_data __rseq_percpu
*data
;
383 struct percpu_list_node
{
385 struct percpu_list_node
*next
;
389 struct percpu_list_node
*head
;
392 #define BUFFER_ITEM_PER_CPU 100
394 struct percpu_buffer_node
{
398 struct percpu_buffer
{
401 struct percpu_buffer_node
**array
;
404 #define MEMCPY_BUFFER_ITEM_PER_CPU 100
406 struct percpu_memcpy_buffer_node
{
411 struct percpu_memcpy_buffer
{
414 struct percpu_memcpy_buffer_node
*array
;
417 /* A simple percpu spinlock. Grabs lock on current cpu. */
418 static int rseq_this_cpu_lock(struct percpu_lock __rseq_percpu
*lock
)
425 cpu
= get_current_cpu_id();
427 fprintf(stderr
, "pid: %d: tid: %d, cpu: %d: cid: %d\n",
428 getpid(), (int) rseq_gettid(), rseq_current_cpu_raw(), cpu
);
431 ret
= rseq_load_cbne_store__ptr(RSEQ_MO_RELAXED
, RSEQ_PERCPU
,
432 &rseq_percpu_ptr(lock
, cpu
)->v
,
434 if (rseq_likely(!ret
))
436 /* Retry if comparison fails or rseq aborts. */
439 * Acquire semantic when taking lock after control dependency.
440 * Matches rseq_smp_store_release().
442 rseq_smp_acquire__after_ctrl_dep();
446 static void rseq_percpu_unlock(struct percpu_lock __rseq_percpu
*lock
, int cpu
)
448 assert(rseq_percpu_ptr(lock
, cpu
)->v
== 1);
450 * Release lock, with release semantic. Matches
451 * rseq_smp_acquire__after_ctrl_dep().
453 rseq_smp_store_release(&rseq_percpu_ptr(lock
, cpu
)->v
, 0);
456 static void *test_percpu_spinlock_thread(void *arg
)
458 struct spinlock_thread_test_data
*thread_data
= (struct spinlock_thread_test_data
*) arg
;
459 struct spinlock_test_data __rseq_percpu
*data
= thread_data
->data
;
462 if (!opt_disable_rseq
&& thread_data
->reg
&&
463 rseq_register_current_thread())
465 reps
= thread_data
->reps
;
466 for (i
= 0; i
< reps
; i
++) {
467 int cpu
= rseq_this_cpu_lock(&data
->lock
);
468 rseq_percpu_ptr(data
, cpu
)->count
++;
469 rseq_percpu_unlock(&data
->lock
, cpu
);
471 if (i
!= 0 && !(i
% (reps
/ 10)))
472 printf_verbose("tid %d: count %lld\n",
473 (int) rseq_gettid(), i
);
476 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
477 (int) rseq_gettid(), nr_abort
, signals_delivered
);
478 if (!opt_disable_rseq
&& thread_data
->reg
&&
479 rseq_unregister_current_thread())
485 * A simple test which implements a sharded counter using a per-cpu
486 * lock. Obviously real applications might prefer to simply use a
487 * per-cpu increment; however, this is reasonable for a test and the
488 * lock can be extended to synchronize more complicated operations.
490 static void test_percpu_spinlock(void)
492 const int num_threads
= opt_threads
;
495 pthread_t test_threads
[num_threads
];
496 struct spinlock_test_data __rseq_percpu
*data
;
497 struct spinlock_thread_test_data thread_data
[num_threads
];
498 struct rseq_percpu_pool
*mempool
;
500 mempool
= rseq_percpu_pool_create("spinlock_test_data",
501 sizeof(struct spinlock_test_data
),
502 0, CPU_SETSIZE
, NULL
);
504 perror("rseq_percpu_pool_create");
507 data
= (struct spinlock_test_data __rseq_percpu
*)rseq_percpu_zmalloc(mempool
);
509 perror("rseq_percpu_zmalloc");
513 for (i
= 0; i
< num_threads
; i
++) {
514 thread_data
[i
].reps
= opt_reps
;
515 if (opt_disable_mod
<= 0 || (i
% opt_disable_mod
))
516 thread_data
[i
].reg
= 1;
518 thread_data
[i
].reg
= 0;
519 thread_data
[i
].data
= data
;
520 ret
= pthread_create(&test_threads
[i
], NULL
,
521 test_percpu_spinlock_thread
,
525 perror("pthread_create");
530 for (i
= 0; i
< num_threads
; i
++) {
531 ret
= pthread_join(test_threads
[i
], NULL
);
534 perror("pthread_join");
540 for (i
= 0; i
< CPU_SETSIZE
; i
++)
541 sum
+= rseq_percpu_ptr(data
, i
)->count
;
543 assert(sum
== (uint64_t)opt_reps
* num_threads
);
544 rseq_percpu_free(data
);
545 ret
= rseq_percpu_pool_destroy(mempool
);
547 perror("rseq_percpu_pool_destroy");
552 static void *test_percpu_inc_thread(void *arg
)
554 struct inc_thread_test_data
*thread_data
= (struct inc_thread_test_data
*) arg
;
555 struct inc_test_data __rseq_percpu
*data
= thread_data
->data
;
558 if (!opt_disable_rseq
&& thread_data
->reg
&&
559 rseq_register_current_thread())
561 reps
= thread_data
->reps
;
562 for (i
= 0; i
< reps
; i
++) {
568 cpu
= get_current_cpu_id();
569 ret
= rseq_load_add_store__ptr(RSEQ_MO_RELAXED
, RSEQ_PERCPU
,
570 &rseq_percpu_ptr(data
, cpu
)->count
, 1, cpu
);
571 } while (rseq_unlikely(ret
));
573 if (i
!= 0 && !(i
% (reps
/ 10)))
574 printf_verbose("tid %d: count %lld\n",
575 (int) rseq_gettid(), i
);
578 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
579 (int) rseq_gettid(), nr_abort
, signals_delivered
);
580 if (!opt_disable_rseq
&& thread_data
->reg
&&
581 rseq_unregister_current_thread())
586 static void test_percpu_inc(void)
588 const int num_threads
= opt_threads
;
591 pthread_t test_threads
[num_threads
];
592 struct inc_test_data __rseq_percpu
*data
;
593 struct inc_thread_test_data thread_data
[num_threads
];
594 struct rseq_percpu_pool
*mempool
;
596 mempool
= rseq_percpu_pool_create("inc_test_data",
597 sizeof(struct inc_test_data
),
598 0, CPU_SETSIZE
, NULL
);
600 perror("rseq_percpu_pool_create");
603 data
= (struct inc_test_data __rseq_percpu
*)rseq_percpu_zmalloc(mempool
);
605 perror("rseq_percpu_zmalloc");
609 for (i
= 0; i
< num_threads
; i
++) {
610 thread_data
[i
].reps
= opt_reps
;
611 if (opt_disable_mod
<= 0 || (i
% opt_disable_mod
))
612 thread_data
[i
].reg
= 1;
614 thread_data
[i
].reg
= 0;
615 thread_data
[i
].data
= data
;
616 ret
= pthread_create(&test_threads
[i
], NULL
,
617 test_percpu_inc_thread
,
621 perror("pthread_create");
626 for (i
= 0; i
< num_threads
; i
++) {
627 ret
= pthread_join(test_threads
[i
], NULL
);
630 perror("pthread_join");
636 for (i
= 0; i
< CPU_SETSIZE
; i
++)
637 sum
+= rseq_percpu_ptr(data
, i
)->count
;
639 assert(sum
== (uint64_t)opt_reps
* num_threads
);
640 rseq_percpu_free(data
);
641 ret
= rseq_percpu_pool_destroy(mempool
);
643 perror("rseq_percpu_pool_destroy");
648 static void this_cpu_list_push(struct percpu_list __rseq_percpu
*list
,
649 struct percpu_list_node
*node
,
655 intptr_t *targetptr
, newval
, expect
;
656 struct percpu_list
*cpulist
;
659 cpu
= get_current_cpu_id();
660 cpulist
= rseq_percpu_ptr(list
, cpu
);
661 /* Load list->c[cpu].head with single-copy atomicity. */
662 expect
= (intptr_t)RSEQ_READ_ONCE(cpulist
->head
);
663 newval
= (intptr_t)node
;
664 targetptr
= (intptr_t *)&cpulist
->head
;
665 node
->next
= (struct percpu_list_node
*)expect
;
666 ret
= rseq_load_cbne_store__ptr(RSEQ_MO_RELAXED
, RSEQ_PERCPU
,
667 targetptr
, expect
, newval
, cpu
);
668 if (rseq_likely(!ret
))
670 /* Retry if comparison fails or rseq aborts. */
677 * Unlike a traditional lock-less linked list; the availability of a
678 * rseq primitive allows us to implement pop without concerns over
681 static struct percpu_list_node
*this_cpu_list_pop(struct percpu_list __rseq_percpu
*list
,
684 struct percpu_list_node
*node
= NULL
;
688 struct percpu_list_node
*head
;
689 intptr_t *targetptr
, expectnot
, *load
;
690 struct percpu_list
*cpulist
;
694 cpu
= get_current_cpu_id();
695 cpulist
= rseq_percpu_ptr(list
, cpu
);
696 targetptr
= (intptr_t *)&cpulist
->head
;
697 expectnot
= (intptr_t)NULL
;
698 offset
= offsetof(struct percpu_list_node
, next
);
699 load
= (intptr_t *)&head
;
700 ret
= rseq_load_cbeq_store_add_load_store__ptr(RSEQ_MO_RELAXED
, RSEQ_PERCPU
,
701 targetptr
, expectnot
,
703 if (rseq_likely(!ret
)) {
709 /* Retry if rseq aborts. */
717 * __percpu_list_pop is not safe against concurrent accesses. Should
718 * only be used on lists that are not concurrently modified.
720 static struct percpu_list_node
*__percpu_list_pop(struct percpu_list __rseq_percpu
*list
, int cpu
)
722 struct percpu_list
*cpulist
= rseq_percpu_ptr(list
, cpu
);
723 struct percpu_list_node
*node
;
725 node
= cpulist
->head
;
728 cpulist
->head
= node
->next
;
732 static void *test_percpu_list_thread(void *arg
)
735 struct percpu_list __rseq_percpu
*list
= (struct percpu_list __rseq_percpu
*)arg
;
737 if (!opt_disable_rseq
&& rseq_register_current_thread())
741 for (i
= 0; i
< reps
; i
++) {
742 struct percpu_list_node
*node
;
744 node
= this_cpu_list_pop(list
, NULL
);
746 sched_yield(); /* encourage shuffling */
748 this_cpu_list_push(list
, node
, NULL
);
751 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
752 (int) rseq_gettid(), nr_abort
, signals_delivered
);
753 if (!opt_disable_rseq
&& rseq_unregister_current_thread())
759 /* Simultaneous modification to a per-cpu linked list from many threads. */
760 static void test_percpu_list(void)
762 const int num_threads
= opt_threads
;
764 uint64_t sum
= 0, expected_sum
= 0;
765 struct percpu_list __rseq_percpu
*list
;
766 pthread_t test_threads
[num_threads
];
767 cpu_set_t allowed_cpus
;
768 struct rseq_percpu_pool
*mempool
;
770 mempool
= rseq_percpu_pool_create("percpu_list", sizeof(struct percpu_list
),
771 0, CPU_SETSIZE
, NULL
);
773 perror("rseq_percpu_pool_create");
776 list
= (struct percpu_list __rseq_percpu
*)rseq_percpu_zmalloc(mempool
);
778 perror("rseq_percpu_zmalloc");
782 /* Generate list entries for every usable cpu. */
783 sched_getaffinity(0, sizeof(allowed_cpus
), &allowed_cpus
);
784 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
785 if (rseq_use_cpu_index() && !CPU_ISSET(i
, &allowed_cpus
))
787 for (j
= 1; j
<= 100; j
++) {
788 struct percpu_list
*cpulist
= rseq_percpu_ptr(list
, i
);
789 struct percpu_list_node
*node
;
793 node
= (struct percpu_list_node
*) malloc(sizeof(*node
));
796 node
->next
= cpulist
->head
;
797 cpulist
->head
= node
;
801 for (i
= 0; i
< num_threads
; i
++) {
802 ret
= pthread_create(&test_threads
[i
], NULL
,
803 test_percpu_list_thread
, list
);
806 perror("pthread_create");
811 for (i
= 0; i
< num_threads
; i
++) {
812 ret
= pthread_join(test_threads
[i
], NULL
);
815 perror("pthread_join");
820 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
821 struct percpu_list_node
*node
;
823 if (rseq_use_cpu_index() && !CPU_ISSET(i
, &allowed_cpus
))
826 while ((node
= __percpu_list_pop(list
, i
))) {
833 * All entries should now be accounted for (unless some external
834 * actor is interfering with our allowed affinity while this
837 assert(sum
== expected_sum
);
838 rseq_percpu_free(list
);
839 ret
= rseq_percpu_pool_destroy(mempool
);
841 perror("rseq_percpu_pool_destroy");
846 static bool this_cpu_buffer_push(struct percpu_buffer __rseq_percpu
*buffer
,
847 struct percpu_buffer_node
*node
,
854 struct percpu_buffer
*cpubuffer
;
855 intptr_t *targetptr_spec
, newval_spec
;
856 intptr_t *targetptr_final
, newval_final
;
860 cpu
= get_current_cpu_id();
861 cpubuffer
= rseq_percpu_ptr(buffer
, cpu
);
862 offset
= RSEQ_READ_ONCE(cpubuffer
->offset
);
863 if (offset
== cpubuffer
->buflen
)
865 newval_spec
= (intptr_t)node
;
866 targetptr_spec
= (intptr_t *)&cpubuffer
->array
[offset
];
867 newval_final
= offset
+ 1;
868 targetptr_final
= &cpubuffer
->offset
;
869 ret
= rseq_load_cbne_store_store__ptr(opt_mo
, RSEQ_PERCPU
,
870 targetptr_final
, offset
, targetptr_spec
,
871 newval_spec
, newval_final
, cpu
);
872 if (rseq_likely(!ret
)) {
876 /* Retry if comparison fails or rseq aborts. */
883 static struct percpu_buffer_node
*this_cpu_buffer_pop(struct percpu_buffer __rseq_percpu
*buffer
,
886 struct percpu_buffer_node
*head
;
890 struct percpu_buffer
*cpubuffer
;
891 intptr_t *targetptr
, newval
;
895 cpu
= get_current_cpu_id();
896 cpubuffer
= rseq_percpu_ptr(buffer
, cpu
);
897 /* Load offset with single-copy atomicity. */
898 offset
= RSEQ_READ_ONCE(cpubuffer
->offset
);
903 head
= RSEQ_READ_ONCE(cpubuffer
->array
[offset
- 1]);
905 targetptr
= (intptr_t *)&cpubuffer
->offset
;
906 ret
= rseq_load_cbne_load_cbne_store__ptr(RSEQ_MO_RELAXED
, RSEQ_PERCPU
,
908 (intptr_t *)&cpubuffer
->array
[offset
- 1],
909 (intptr_t)head
, newval
, cpu
);
910 if (rseq_likely(!ret
))
912 /* Retry if comparison fails or rseq aborts. */
920 * __percpu_buffer_pop is not safe against concurrent accesses. Should
921 * only be used on buffers that are not concurrently modified.
923 static struct percpu_buffer_node
*__percpu_buffer_pop(struct percpu_buffer __rseq_percpu
*buffer
,
926 struct percpu_buffer
*cpubuffer
;
927 struct percpu_buffer_node
*head
;
930 cpubuffer
= rseq_percpu_ptr(buffer
, cpu
);
931 offset
= cpubuffer
->offset
;
934 head
= cpubuffer
->array
[offset
- 1];
935 cpubuffer
->offset
= offset
- 1;
939 static void *test_percpu_buffer_thread(void *arg
)
942 struct percpu_buffer __rseq_percpu
*buffer
= (struct percpu_buffer __rseq_percpu
*)arg
;
944 if (!opt_disable_rseq
&& rseq_register_current_thread())
948 for (i
= 0; i
< reps
; i
++) {
949 struct percpu_buffer_node
*node
;
951 node
= this_cpu_buffer_pop(buffer
, NULL
);
953 sched_yield(); /* encourage shuffling */
955 if (!this_cpu_buffer_push(buffer
, node
, NULL
)) {
956 /* Should increase buffer size. */
962 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
963 (int) rseq_gettid(), nr_abort
, signals_delivered
);
964 if (!opt_disable_rseq
&& rseq_unregister_current_thread())
970 /* Simultaneous modification to a per-cpu buffer from many threads. */
971 static void test_percpu_buffer(void)
973 const int num_threads
= opt_threads
;
975 uint64_t sum
= 0, expected_sum
= 0;
976 struct percpu_buffer __rseq_percpu
*buffer
;
977 pthread_t test_threads
[num_threads
];
978 cpu_set_t allowed_cpus
;
979 struct rseq_percpu_pool
*mempool
;
981 mempool
= rseq_percpu_pool_create("percpu_buffer", sizeof(struct percpu_buffer
),
982 0, CPU_SETSIZE
, NULL
);
984 perror("rseq_percpu_pool_create");
987 buffer
= (struct percpu_buffer __rseq_percpu
*)rseq_percpu_zmalloc(mempool
);
989 perror("rseq_percpu_zmalloc");
993 /* Generate list entries for every usable cpu. */
994 sched_getaffinity(0, sizeof(allowed_cpus
), &allowed_cpus
);
995 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
996 struct percpu_buffer
*cpubuffer
;
998 if (rseq_use_cpu_index() && !CPU_ISSET(i
, &allowed_cpus
))
1000 cpubuffer
= rseq_percpu_ptr(buffer
, i
);
1001 /* Worse-case is every item in same CPU. */
1003 (struct percpu_buffer_node
**)
1004 malloc(sizeof(*cpubuffer
->array
) * CPU_SETSIZE
*
1005 BUFFER_ITEM_PER_CPU
);
1006 assert(cpubuffer
->array
);
1007 cpubuffer
->buflen
= CPU_SETSIZE
* BUFFER_ITEM_PER_CPU
;
1008 for (j
= 1; j
<= BUFFER_ITEM_PER_CPU
; j
++) {
1009 struct percpu_buffer_node
*node
;
1014 * We could theoretically put the word-sized
1015 * "data" directly in the buffer. However, we
1016 * want to model objects that would not fit
1017 * within a single word, so allocate an object
1020 node
= (struct percpu_buffer_node
*) malloc(sizeof(*node
));
1023 cpubuffer
->array
[j
- 1] = node
;
1024 cpubuffer
->offset
++;
1028 for (i
= 0; i
< num_threads
; i
++) {
1029 ret
= pthread_create(&test_threads
[i
], NULL
,
1030 test_percpu_buffer_thread
, buffer
);
1033 perror("pthread_create");
1038 for (i
= 0; i
< num_threads
; i
++) {
1039 ret
= pthread_join(test_threads
[i
], NULL
);
1042 perror("pthread_join");
1047 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
1048 struct percpu_buffer
*cpubuffer
;
1049 struct percpu_buffer_node
*node
;
1051 if (rseq_use_cpu_index() && !CPU_ISSET(i
, &allowed_cpus
))
1054 cpubuffer
= rseq_percpu_ptr(buffer
, i
);
1055 while ((node
= __percpu_buffer_pop(buffer
, i
))) {
1059 free(cpubuffer
->array
);
1063 * All entries should now be accounted for (unless some external
1064 * actor is interfering with our allowed affinity while this
1067 assert(sum
== expected_sum
);
1068 rseq_percpu_free(buffer
);
1069 ret
= rseq_percpu_pool_destroy(mempool
);
1071 perror("rseq_percpu_pool_destroy");
1076 static bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer __rseq_percpu
*buffer
,
1077 struct percpu_memcpy_buffer_node item
,
1080 bool result
= false;
1084 struct percpu_memcpy_buffer
*cpubuffer
;
1085 intptr_t *targetptr_final
, newval_final
, offset
;
1086 char *destptr
, *srcptr
;
1090 cpu
= get_current_cpu_id();
1091 cpubuffer
= rseq_percpu_ptr(buffer
, cpu
);
1092 /* Load offset with single-copy atomicity. */
1093 offset
= RSEQ_READ_ONCE(cpubuffer
->offset
);
1094 if (offset
== cpubuffer
->buflen
)
1096 destptr
= (char *)&cpubuffer
->array
[offset
];
1097 srcptr
= (char *)&item
;
1098 /* copylen must be <= 4kB. */
1099 copylen
= sizeof(item
);
1100 newval_final
= offset
+ 1;
1101 targetptr_final
= &cpubuffer
->offset
;
1102 ret
= rseq_load_cbne_memcpy_store__ptr(
1103 opt_mo
, RSEQ_PERCPU
,
1104 targetptr_final
, offset
,
1105 destptr
, srcptr
, copylen
,
1107 if (rseq_likely(!ret
)) {
1111 /* Retry if comparison fails or rseq aborts. */
1118 static bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer __rseq_percpu
*buffer
,
1119 struct percpu_memcpy_buffer_node
*item
,
1122 bool result
= false;
1126 struct percpu_memcpy_buffer
*cpubuffer
;
1127 intptr_t *targetptr_final
, newval_final
, offset
;
1128 char *destptr
, *srcptr
;
1132 cpu
= get_current_cpu_id();
1133 cpubuffer
= rseq_percpu_ptr(buffer
, cpu
);
1134 /* Load offset with single-copy atomicity. */
1135 offset
= RSEQ_READ_ONCE(cpubuffer
->offset
);
1138 destptr
= (char *)item
;
1139 srcptr
= (char *)&cpubuffer
->array
[offset
- 1];
1140 /* copylen must be <= 4kB. */
1141 copylen
= sizeof(*item
);
1142 newval_final
= offset
- 1;
1143 targetptr_final
= &cpubuffer
->offset
;
1144 ret
= rseq_load_cbne_memcpy_store__ptr(RSEQ_MO_RELAXED
, RSEQ_PERCPU
,
1145 targetptr_final
, offset
, destptr
, srcptr
, copylen
,
1147 if (rseq_likely(!ret
)) {
1151 /* Retry if comparison fails or rseq aborts. */
1159 * __percpu_memcpy_buffer_pop is not safe against concurrent accesses. Should
1160 * only be used on buffers that are not concurrently modified.
1162 static bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer __rseq_percpu
*buffer
,
1163 struct percpu_memcpy_buffer_node
*item
,
1166 struct percpu_memcpy_buffer
*cpubuffer
;
1169 cpubuffer
= rseq_percpu_ptr(buffer
, cpu
);
1170 offset
= cpubuffer
->offset
;
1173 memcpy(item
, &cpubuffer
->array
[offset
- 1], sizeof(*item
));
1174 cpubuffer
->offset
= offset
- 1;
1178 static void *test_percpu_memcpy_buffer_thread(void *arg
)
1181 struct percpu_memcpy_buffer __rseq_percpu
*buffer
= (struct percpu_memcpy_buffer __rseq_percpu
*)arg
;
1183 if (!opt_disable_rseq
&& rseq_register_current_thread())
1187 for (i
= 0; i
< reps
; i
++) {
1188 struct percpu_memcpy_buffer_node item
;
1191 result
= this_cpu_memcpy_buffer_pop(buffer
, &item
, NULL
);
1193 sched_yield(); /* encourage shuffling */
1195 if (!this_cpu_memcpy_buffer_push(buffer
, item
, NULL
)) {
1196 /* Should increase buffer size. */
1202 printf_verbose("tid %d: number of rseq abort: %d, signals delivered: %u\n",
1203 (int) rseq_gettid(), nr_abort
, signals_delivered
);
1204 if (!opt_disable_rseq
&& rseq_unregister_current_thread())
1210 /* Simultaneous modification to a per-cpu buffer from many threads. */
1211 static void test_percpu_memcpy_buffer(void)
1213 const int num_threads
= opt_threads
;
1215 uint64_t sum
= 0, expected_sum
= 0;
1216 struct percpu_memcpy_buffer
*buffer
;
1217 pthread_t test_threads
[num_threads
];
1218 cpu_set_t allowed_cpus
;
1219 struct rseq_percpu_pool
*mempool
;
1221 mempool
= rseq_percpu_pool_create("percpu_memcpy_buffer",
1222 sizeof(struct percpu_memcpy_buffer
),
1223 0, CPU_SETSIZE
, NULL
);
1225 perror("rseq_percpu_pool_create");
1228 buffer
= (struct percpu_memcpy_buffer __rseq_percpu
*)rseq_percpu_zmalloc(mempool
);
1230 perror("rseq_percpu_zmalloc");
1234 /* Generate list entries for every usable cpu. */
1235 sched_getaffinity(0, sizeof(allowed_cpus
), &allowed_cpus
);
1236 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
1237 struct percpu_memcpy_buffer
*cpubuffer
;
1239 if (rseq_use_cpu_index() && !CPU_ISSET(i
, &allowed_cpus
))
1241 cpubuffer
= rseq_percpu_ptr(buffer
, i
);
1242 /* Worse-case is every item in same CPU. */
1244 (struct percpu_memcpy_buffer_node
*)
1245 malloc(sizeof(*cpubuffer
->array
) * CPU_SETSIZE
*
1246 MEMCPY_BUFFER_ITEM_PER_CPU
);
1247 assert(cpubuffer
->array
);
1248 cpubuffer
->buflen
= CPU_SETSIZE
* MEMCPY_BUFFER_ITEM_PER_CPU
;
1249 for (j
= 1; j
<= MEMCPY_BUFFER_ITEM_PER_CPU
; j
++) {
1250 expected_sum
+= 2 * j
+ 1;
1253 * We could theoretically put the word-sized
1254 * "data" directly in the buffer. However, we
1255 * want to model objects that would not fit
1256 * within a single word, so allocate an object
1259 cpubuffer
->array
[j
- 1].data1
= j
;
1260 cpubuffer
->array
[j
- 1].data2
= j
+ 1;
1261 cpubuffer
->offset
++;
1265 for (i
= 0; i
< num_threads
; i
++) {
1266 ret
= pthread_create(&test_threads
[i
], NULL
,
1267 test_percpu_memcpy_buffer_thread
,
1271 perror("pthread_create");
1276 for (i
= 0; i
< num_threads
; i
++) {
1277 ret
= pthread_join(test_threads
[i
], NULL
);
1280 perror("pthread_join");
1285 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
1286 struct percpu_memcpy_buffer_node item
;
1287 struct percpu_memcpy_buffer
*cpubuffer
;
1289 if (rseq_use_cpu_index() && !CPU_ISSET(i
, &allowed_cpus
))
1292 cpubuffer
= rseq_percpu_ptr(buffer
, i
);
1293 while (__percpu_memcpy_buffer_pop(buffer
, &item
, i
)) {
1297 free(cpubuffer
->array
);
1301 * All entries should now be accounted for (unless some external
1302 * actor is interfering with our allowed affinity while this
1305 assert(sum
== expected_sum
);
1306 rseq_percpu_free(buffer
);
1307 ret
= rseq_percpu_pool_destroy(mempool
);
1309 perror("rseq_percpu_pool_destroy");
1314 static void test_signal_interrupt_handler(__attribute__ ((unused
)) int signo
)
1316 signals_delivered
++;
1319 static int set_signal_handler(void)
1322 struct sigaction sa
;
1325 ret
= sigemptyset(&sigset
);
1327 perror("sigemptyset");
1331 sa
.sa_handler
= test_signal_interrupt_handler
;
1332 sa
.sa_mask
= sigset
;
1334 ret
= sigaction(SIGUSR1
, &sa
, NULL
);
1336 perror("sigaction");
1340 printf_verbose("Signal handler set for SIGUSR1\n");
1346 bool membarrier_private_expedited_rseq_available(void)
1348 int status
= sys_membarrier(MEMBARRIER_CMD_QUERY
, 0, 0);
1351 perror("membarrier");
1354 if (!(status
& MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ
))
1359 /* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */
1360 #ifdef TEST_MEMBARRIER
1361 struct test_membarrier_thread_args
{
1362 struct rseq_percpu_pool
*mempool
;
1363 struct percpu_list __rseq_percpu
*percpu_list_ptr
;
1367 /* Worker threads modify data in their "active" percpu lists. */
1369 void *test_membarrier_worker_thread(void *arg
)
1371 struct test_membarrier_thread_args
*args
=
1372 (struct test_membarrier_thread_args
*)arg
;
1373 const long long iters
= opt_reps
;
1376 if (rseq_register_current_thread()) {
1377 fprintf(stderr
, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
1378 errno
, strerror(errno
));
1382 /* Wait for initialization. */
1383 while (!rseq_smp_load_acquire(&args
->percpu_list_ptr
)) { }
1385 for (i
= 0; i
< iters
; ++i
) {
1389 int cpu
= get_current_cpu_id();
1390 struct percpu_list __rseq_percpu
*list
= RSEQ_READ_ONCE(args
->percpu_list_ptr
);
1391 struct percpu_list
*cpulist
= rseq_percpu_ptr(list
, cpu
);
1393 ret
= rseq_load_cbne_load_add_load_add_store__ptr(RSEQ_MO_RELAXED
, RSEQ_PERCPU
,
1394 (intptr_t *) &args
->percpu_list_ptr
,
1395 (intptr_t) list
, (intptr_t *) &cpulist
->head
, 0, 1, cpu
);
1396 } while (rseq_unlikely(ret
));
1399 if (rseq_unregister_current_thread()) {
1400 fprintf(stderr
, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
1401 errno
, strerror(errno
));
1408 struct percpu_list __rseq_percpu
*test_membarrier_alloc_percpu_list(struct rseq_percpu_pool
*mempool
)
1410 struct percpu_list __rseq_percpu
*list
;
1413 list
= (struct percpu_list __rseq_percpu
*)rseq_percpu_zmalloc(mempool
);
1415 perror("rseq_percpu_zmalloc");
1418 for (i
= 0; i
< CPU_SETSIZE
; i
++) {
1419 struct percpu_list
*cpulist
= rseq_percpu_ptr(list
, i
);
1420 struct percpu_list_node
*node
;
1422 node
= (struct percpu_list_node
*) malloc(sizeof(*node
));
1426 cpulist
->head
= node
;
1432 void test_membarrier_free_percpu_list(struct percpu_list __rseq_percpu
*list
)
1436 for (i
= 0; i
< CPU_SETSIZE
; i
++)
1437 free(rseq_percpu_ptr(list
, i
)->head
);
1438 rseq_percpu_free(list
);
1442 long long test_membarrier_count_percpu_list(struct percpu_list __rseq_percpu
*list
)
1444 long long total_count
= 0;
1447 for (i
= 0; i
< CPU_SETSIZE
; i
++)
1448 total_count
+= rseq_percpu_ptr(list
, i
)->head
->data
;
1453 * The manager thread swaps per-cpu lists that worker threads see,
1454 * and validates that there are no unexpected modifications.
1457 void *test_membarrier_manager_thread(void *arg
)
1459 struct test_membarrier_thread_args
*args
=
1460 (struct test_membarrier_thread_args
*)arg
;
1461 struct percpu_list __rseq_percpu
*list_a
, __rseq_percpu
*list_b
;
1462 intptr_t expect_a
= 0, expect_b
= 0;
1463 int cpu_a
= 0, cpu_b
= 0;
1464 struct rseq_percpu_pool
*mempool
;
1466 long long total_count
= 0;
1468 mempool
= rseq_percpu_pool_create("percpu_list", sizeof(struct percpu_list
),
1469 0, CPU_SETSIZE
, NULL
);
1471 perror("rseq_percpu_pool_create");
1474 args
->mempool
= mempool
;
1476 if (rseq_register_current_thread()) {
1477 fprintf(stderr
, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
1478 errno
, strerror(errno
));
1483 list_a
= test_membarrier_alloc_percpu_list(mempool
);
1485 list_b
= test_membarrier_alloc_percpu_list(mempool
);
1488 /* Initialize lists before publishing them. */
1491 RSEQ_WRITE_ONCE(args
->percpu_list_ptr
, list_a
);
1493 while (!RSEQ_READ_ONCE(args
->stop
)) {
1494 /* list_a is "active". */
1495 cpu_a
= rand() % CPU_SETSIZE
;
1497 * As list_b is "inactive", we should never see changes
1500 if (expect_b
!= RSEQ_READ_ONCE(rseq_percpu_ptr(list_b
, cpu_b
)->head
->data
)) {
1501 fprintf(stderr
, "Membarrier test failed\n");
1505 /* Make list_b "active". */
1506 RSEQ_WRITE_ONCE(args
->percpu_list_ptr
, list_b
);
1507 if (rseq_membarrier_expedited(cpu_a
) &&
1508 errno
!= ENXIO
/* missing CPU */) {
1509 perror("sys_membarrier");
1513 * Cpu A should now only modify list_b, so the values
1514 * in list_a should be stable.
1516 expect_a
= RSEQ_READ_ONCE(rseq_percpu_ptr(list_a
, cpu_a
)->head
->data
);
1518 cpu_b
= rand() % CPU_SETSIZE
;
1520 * As list_a is "inactive", we should never see changes
1523 if (expect_a
!= RSEQ_READ_ONCE(rseq_percpu_ptr(list_a
, cpu_a
)->head
->data
)) {
1524 fprintf(stderr
, "Membarrier test failed\n");
1528 /* Make list_a "active". */
1529 RSEQ_WRITE_ONCE(args
->percpu_list_ptr
, list_a
);
1530 if (rseq_membarrier_expedited(cpu_b
) &&
1531 errno
!= ENXIO
/* missing CPU */) {
1532 perror("sys_membarrier");
1535 /* Remember a value from list_b. */
1536 expect_b
= RSEQ_READ_ONCE(rseq_percpu_ptr(list_b
, cpu_b
)->head
->data
);
1539 total_count
+= test_membarrier_count_percpu_list(list_a
);
1540 total_count
+= test_membarrier_count_percpu_list(list_b
);
1542 /* Validate that we observe the right number of increments. */
1543 if (total_count
!= opt_threads
* opt_reps
) {
1544 fprintf(stderr
, "Error: Observed %lld increments, expected %lld\n",
1545 total_count
, opt_threads
* opt_reps
);
1548 test_membarrier_free_percpu_list(list_a
);
1549 test_membarrier_free_percpu_list(list_b
);
1551 if (rseq_unregister_current_thread()) {
1552 fprintf(stderr
, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
1553 errno
, strerror(errno
));
1556 ret
= rseq_percpu_pool_destroy(mempool
);
1558 perror("rseq_percpu_pool_destroy");
1566 void test_membarrier(void)
1568 const int num_threads
= opt_threads
;
1569 struct test_membarrier_thread_args thread_args
;
1570 pthread_t worker_threads
[num_threads
];
1571 pthread_t manager_thread
;
1574 if (!membarrier_private_expedited_rseq_available()) {
1575 fprintf(stderr
, "Membarrier private expedited rseq not available. "
1576 "Skipping membarrier test.\n");
1579 if (sys_membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ
, 0, 0)) {
1580 perror("sys_membarrier");
1584 thread_args
.percpu_list_ptr
= NULL
;
1585 thread_args
.stop
= 0;
1586 ret
= pthread_create(&manager_thread
, NULL
,
1587 test_membarrier_manager_thread
, &thread_args
);
1590 perror("pthread_create");
1594 for (i
= 0; i
< num_threads
; i
++) {
1595 ret
= pthread_create(&worker_threads
[i
], NULL
,
1596 test_membarrier_worker_thread
, &thread_args
);
1599 perror("pthread_create");
1605 for (i
= 0; i
< num_threads
; i
++) {
1606 ret
= pthread_join(worker_threads
[i
], NULL
);
1609 perror("pthread_join");
1614 RSEQ_WRITE_ONCE(thread_args
.stop
, 1);
1615 ret
= pthread_join(manager_thread
, NULL
);
1618 perror("pthread_join");
1622 #else /* TEST_MEMBARRIER */
1624 void test_membarrier(void)
1626 if (!membarrier_private_expedited_rseq_available()) {
1627 fprintf(stderr
, "Membarrier private expedited rseq not available. "
1628 "Skipping membarrier test.\n");
1631 fprintf(stderr
, "rseq_load_cbne_load_add_load_add_store__ptr is not implemented on this architecture. "
1632 "Skipping membarrier test.\n");
1636 static void show_usage(char **argv
)
1638 printf("Usage : %s <OPTIONS>\n",
1640 printf("OPTIONS:\n");
1641 printf(" [-1 loops] Number of loops for delay injection 1\n");
1642 printf(" [-2 loops] Number of loops for delay injection 2\n");
1643 printf(" [-3 loops] Number of loops for delay injection 3\n");
1644 printf(" [-4 loops] Number of loops for delay injection 4\n");
1645 printf(" [-5 loops] Number of loops for delay injection 5\n");
1646 printf(" [-6 loops] Number of loops for delay injection 6\n");
1647 printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n");
1648 printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n");
1649 printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n");
1650 printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n");
1651 printf(" [-y] Yield\n");
1652 printf(" [-k] Kill thread with signal\n");
1653 printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n");
1654 printf(" [-t N] Number of threads (default 200)\n");
1655 printf(" [-r N] Number of repetitions per thread (default 5000)\n");
1656 printf(" [-d] Disable rseq system call (no initialization)\n");
1657 printf(" [-D M] Disable rseq for each M threads\n");
1658 printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\n");
1659 printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n");
1660 printf(" [-c] Check if the rseq syscall is available.\n");
1661 printf(" [-v] Verbose output.\n");
1662 printf(" [-h] Show this help.\n");
1666 int main(int argc
, char **argv
)
1670 for (i
= 1; i
< argc
; i
++) {
1671 if (argv
[i
][0] != '-')
1673 switch (argv
[i
][1]) {
1687 loop_cnt
[argv
[i
][1] - '0'] = atol(argv
[i
+ 1]);
1695 opt_modulo
= atol(argv
[i
+ 1]);
1696 if (opt_modulo
< 0) {
1707 opt_sleep
= atol(argv
[i
+ 1]);
1708 if (opt_sleep
< 0) {
1721 opt_disable_rseq
= 1;
1728 opt_disable_mod
= atol(argv
[i
+ 1]);
1729 if (opt_disable_mod
< 0) {
1740 opt_threads
= atol(argv
[i
+ 1]);
1741 if (opt_threads
< 0) {
1752 opt_reps
= atoll(argv
[i
+ 1]);
1767 opt_test
= *argv
[i
+ 1];
1786 opt_mo
= RSEQ_MO_RELEASE
;
1789 if (rseq_available(RSEQ_AVAILABLE_QUERY_KERNEL
)) {
1790 printf_verbose("The rseq syscall is available.\n");
1793 printf_verbose("The rseq syscall is unavailable.\n");
1802 loop_cnt_1
= loop_cnt
[1];
1803 loop_cnt_2
= loop_cnt
[2];
1804 loop_cnt_3
= loop_cnt
[3];
1805 loop_cnt_4
= loop_cnt
[4];
1806 loop_cnt_5
= loop_cnt
[5];
1807 loop_cnt_6
= loop_cnt
[6];
1809 if (set_signal_handler())
1812 if (!opt_disable_rseq
&& rseq_register_current_thread())
1814 if (!opt_disable_rseq
&& !rseq_validate_cpu_id()) {
1815 printf_verbose("The rseq cpu id getter is unavailable\n");
1820 printf_verbose("spinlock\n");
1821 test_percpu_spinlock();
1824 printf_verbose("linked list\n");
1828 printf_verbose("buffer\n");
1829 test_percpu_buffer();
1832 printf_verbose("memcpy buffer\n");
1833 test_percpu_memcpy_buffer();
1836 printf_verbose("counter increment\n");
1840 printf_verbose("membarrier\n");
1844 if (!opt_disable_rseq
&& rseq_unregister_current_thread())