From 369688a5b0ebe06be3b04a27ed77e0958e4af77e Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Tue, 25 Apr 2023 17:55:05 -0400 Subject: [PATCH] Implement parametrized mm_cid test Adapt to the rseq.h API changes introduced by commits "selftests/rseq: : Template memory ordering and percpu access mode". Build a new param_test_mm_cid, param_test_mm_cid_benchmark, and param_test_mm_cid_compare_twice executables to test the new "mm_cid" rseq field. Signed-off-by: Mathieu Desnoyers Change-Id: I36405e733876abd7d4f04f07e256e7029c2ea1d2 --- .gitignore | 6 ++ tests/Makefile.am | 32 ++++++++- tests/param_test.c | 149 ++++++++++++++++++++++++++------------- tests/run_param_test.tap | 8 ++- 4 files changed, 144 insertions(+), 51 deletions(-) diff --git a/.gitignore b/.gitignore index bfd59ec..6863b6e 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,12 @@ dkms.conf /tests/param_test_benchmark_cxx /tests/param_test_compare_twice /tests/param_test_compare_twice_cxx +/tests/param_test_mm_cid +/tests/param_test_mm_cid_cxx +/tests/param_test_mm_cid_benchmark +/tests/param_test_mm_cid_benchmark_cxx +/tests/param_test_mm_cid_compare_twice +/tests/param_test_mm_cid_compare_twice_cxx #automake /include/config.h diff --git a/tests/Makefile.am b/tests/Makefile.am index afc3d9e..b2292ad 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -21,10 +21,16 @@ noinst_PROGRAMS = \ basic_test_cxx.tap \ param_test \ param_test_cxx \ + param_test_mm_cid \ + param_test_mm_cid_cxx \ param_test_benchmark \ param_test_benchmark_cxx \ + param_test_mm_cid_benchmark \ + param_test_mm_cid_benchmark_cxx \ param_test_compare_twice \ - param_test_compare_twice_cxx + param_test_compare_twice_cxx \ + param_test_mm_cid_compare_twice \ + param_test_mm_cid_compare_twice_cxx dist_noinst_SCRIPTS = \ run_param_test.tap \ @@ -56,6 +62,14 @@ param_test_LDADD = $(top_builddir)/src/librseq.la $(DL_LIBS) param_test_cxx_SOURCES = param_test_cxx.cpp param_test_cxx_LDADD = $(top_builddir)/src/librseq.la $(DL_LIBS) +param_test_mm_cid_SOURCES = param_test.c +param_test_mm_cid_CPPFLAGS = $(AM_CPPFLAGS) -DBUILDOPT_RSEQ_PERCPU_MM_CID +param_test_mm_cid_LDADD = $(top_builddir)/src/librseq.la $(DL_LIBS) + +param_test_mm_cid_cxx_SOURCES = param_test_cxx.cpp +param_test_mm_cid_cxx_CPPFLAGS = $(AM_CPPFLAGS) -DBUILDOPT_RSEQ_PERCPU_MM_CID +param_test_mm_cid_cxx_LDADD = $(top_builddir)/src/librseq.la $(DL_LIBS) + param_test_benchmark_SOURCES = param_test.c param_test_benchmark_CPPFLAGS = $(AM_CPPFLAGS) -DBENCHMARK param_test_benchmark_LDADD = $(top_builddir)/src/librseq.la $(DL_LIBS) @@ -64,6 +78,14 @@ param_test_benchmark_cxx_SOURCES = param_test_cxx.cpp param_test_benchmark_cxx_CPPFLAGS = $(AM_CPPFLAGS) -DBENCHMARK param_test_benchmark_cxx_LDADD = $(top_builddir)/src/librseq.la $(DL_LIBS) +param_test_mm_cid_benchmark_SOURCES = param_test.c +param_test_mm_cid_benchmark_CPPFLAGS = $(AM_CPPFLAGS) -DBENCHMARK -DBUILDOPT_RSEQ_PERCPU_MM_CID +param_test_mm_cid_benchmark_LDADD = $(top_builddir)/src/librseq.la $(DL_LIBS) + +param_test_mm_cid_benchmark_cxx_SOURCES = param_test_cxx.cpp +param_test_mm_cid_benchmark_cxx_CPPFLAGS = $(AM_CPPFLAGS) -DBENCHMARK -DBUILDOPT_RSEQ_PERCPU_MM_CID +param_test_mm_cid_benchmark_cxx_LDADD = $(top_builddir)/src/librseq.la $(DL_LIBS) + param_test_compare_twice_SOURCES = param_test.c param_test_compare_twice_CPPFLAGS = $(AM_CPPFLAGS) -DRSEQ_COMPARE_TWICE param_test_compare_twice_LDADD = $(top_builddir)/src/librseq.la $(DL_LIBS) @@ -72,6 +94,14 @@ param_test_compare_twice_cxx_SOURCES = param_test_cxx.cpp param_test_compare_twice_cxx_CPPFLAGS = $(AM_CPPFLAGS) -DRSEQ_COMPARE_TWICE param_test_compare_twice_cxx_LDADD = $(top_builddir)/src/librseq.la $(DL_LIBS) +param_test_mm_cid_compare_twice_SOURCES = param_test.c +param_test_mm_cid_compare_twice_CPPFLAGS = $(AM_CPPFLAGS) -DRSEQ_COMPARE_TWICE -DBUILDOPT_RSEQ_PERCPU_MM_CID +param_test_mm_cid_compare_twice_LDADD = $(top_builddir)/src/librseq.la $(DL_LIBS) + +param_test_mm_cid_compare_twice_cxx_SOURCES = param_test_cxx.cpp +param_test_mm_cid_compare_twice_cxx_CPPFLAGS = $(AM_CPPFLAGS) -DRSEQ_COMPARE_TWICE -DBUILDOPT_RSEQ_PERCPU_MM_CID +param_test_mm_cid_compare_twice_cxx_LDADD = $(top_builddir)/src/librseq.la $(DL_LIBS) + TESTS = \ basic_percpu_ops_test.tap \ basic_percpu_ops_test_cxx.tap \ diff --git a/tests/param_test.c b/tests/param_test.c index c8aacba..a77ac58 100644 --- a/tests/param_test.c +++ b/tests/param_test.c @@ -19,6 +19,7 @@ #include #include #include +#include #if LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0) enum { @@ -45,7 +46,7 @@ static int opt_modulo, verbose; static int opt_yield, opt_signal, opt_sleep, opt_disable_rseq, opt_threads = 200, - opt_disable_mod = 0, opt_test = 's', opt_mb = 0; + opt_disable_mod = 0, opt_test = 's'; static long long opt_reps = 5000; @@ -278,6 +279,63 @@ int yield_mod_cnt, nr_abort; #include +static enum rseq_mo opt_mo = RSEQ_MO_RELAXED; + +static int sys_membarrier(int cmd, int flags, int cpu_id) +{ + return syscall(__NR_membarrier, cmd, flags, cpu_id); +} + +#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV +#define TEST_MEMBARRIER +#endif + +#ifdef BUILDOPT_RSEQ_PERCPU_MM_CID +# define RSEQ_PERCPU RSEQ_PERCPU_MM_CID +static +int get_current_cpu_id(void) +{ + return rseq_current_mm_cid(); +} +static +bool rseq_validate_cpu_id(void) +{ + return rseq_mm_cid_available(); +} +# ifdef TEST_MEMBARRIER +/* + * Membarrier does not currently support targeting a mm_cid, so + * issue the barrier on all cpus. + */ +static +int rseq_membarrier_expedited(__attribute__ ((unused)) int cpu) +{ + return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, + 0, 0); +} +# endif /* TEST_MEMBARRIER */ +#else +# define RSEQ_PERCPU RSEQ_PERCPU_CPU_ID +static +int get_current_cpu_id(void) +{ + return rseq_cpu_start(); +} +static +bool rseq_validate_cpu_id(void) +{ + return rseq_current_cpu_raw() >= 0; +} +# ifdef TEST_MEMBARRIER +static +int rseq_membarrier_expedited(int cpu) +{ + return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, + MEMBARRIER_CMD_FLAG_CPU, cpu); +} +# endif /* TEST_MEMBARRIER */ +#endif + struct percpu_lock_entry { intptr_t v; } __attribute__((aligned(128))); @@ -365,8 +423,9 @@ static int rseq_this_cpu_lock(struct percpu_lock *lock) for (;;) { int ret; - cpu = rseq_cpu_start(); - ret = rseq_cmpeqv_storev(&lock->c[cpu].v, + cpu = get_current_cpu_id(); + ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + &lock->c[cpu].v, 0, 1, cpu); if (rseq_likely(!ret)) break; @@ -483,8 +542,9 @@ static void *test_percpu_inc_thread(void *arg) do { int cpu; - cpu = rseq_cpu_start(); - ret = rseq_addv(&data->c[cpu].count, 1, cpu); + cpu = get_current_cpu_id(); + ret = rseq_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU, + &data->c[cpu].count, 1, cpu); } while (rseq_unlikely(ret)); #ifndef BENCHMARK if (i != 0 && !(i % (reps / 10))) @@ -553,13 +613,14 @@ static void this_cpu_list_push(struct percpu_list *list, intptr_t *targetptr, newval, expect; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load list->c[cpu].head with single-copy atomicity. */ expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); newval = (intptr_t)node; targetptr = (intptr_t *)&list->c[cpu].head; node->next = (struct percpu_list_node *)expect; - ret = rseq_cmpeqv_storev(targetptr, expect, newval, cpu); + ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, expect, newval, cpu); if (rseq_likely(!ret)) break; /* Retry if comparison fails or rseq aborts. */ @@ -585,13 +646,14 @@ static struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, long offset; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); targetptr = (intptr_t *)&list->c[cpu].head; expectnot = (intptr_t)NULL; offset = offsetof(struct percpu_list_node, next); load = (intptr_t *)&head; - ret = rseq_cmpnev_storeoffp_load(targetptr, expectnot, - offset, load, cpu); + ret = rseq_cmpnev_storeoffp_load(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, expectnot, + offset, load, cpu); if (rseq_likely(!ret)) { node = head; break; @@ -729,7 +791,7 @@ static bool this_cpu_buffer_push(struct percpu_buffer *buffer, intptr_t offset; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == buffer->c[cpu].buflen) break; @@ -737,14 +799,9 @@ static bool this_cpu_buffer_push(struct percpu_buffer *buffer, targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset]; newval_final = offset + 1; targetptr_final = &buffer->c[cpu].offset; - if (opt_mb) - ret = rseq_cmpeqv_trystorev_storev_release( - targetptr_final, offset, targetptr_spec, - newval_spec, newval_final, cpu); - else - ret = rseq_cmpeqv_trystorev_storev(targetptr_final, - offset, targetptr_spec, newval_spec, - newval_final, cpu); + ret = rseq_cmpeqv_trystorev_storev(opt_mo, RSEQ_PERCPU, + targetptr_final, offset, targetptr_spec, + newval_spec, newval_final, cpu); if (rseq_likely(!ret)) { result = true; break; @@ -767,7 +824,7 @@ static struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buff intptr_t offset; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load offset with single-copy atomicity. */ offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == 0) { @@ -777,7 +834,8 @@ static struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buff head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]); newval = offset - 1; targetptr = (intptr_t *)&buffer->c[cpu].offset; - ret = rseq_cmpeqv_cmpeqv_storev(targetptr, offset, + ret = rseq_cmpeqv_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, offset, (intptr_t *)&buffer->c[cpu].array[offset - 1], (intptr_t)head, newval, cpu); if (rseq_likely(!ret)) @@ -935,7 +993,7 @@ static bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer, size_t copylen; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load offset with single-copy atomicity. */ offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == buffer->c[cpu].buflen) @@ -946,15 +1004,11 @@ static bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer, copylen = sizeof(item); newval_final = offset + 1; targetptr_final = &buffer->c[cpu].offset; - if (opt_mb) - ret = rseq_cmpeqv_trymemcpy_storev_release( - targetptr_final, offset, - destptr, srcptr, copylen, - newval_final, cpu); - else - ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final, - offset, destptr, srcptr, copylen, - newval_final, cpu); + ret = rseq_cmpeqv_trymemcpy_storev( + opt_mo, RSEQ_PERCPU, + targetptr_final, offset, + destptr, srcptr, copylen, + newval_final, cpu); if (rseq_likely(!ret)) { result = true; break; @@ -979,7 +1033,7 @@ static bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, size_t copylen; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load offset with single-copy atomicity. */ offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == 0) @@ -990,8 +1044,8 @@ static bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, copylen = sizeof(*item); newval_final = offset - 1; targetptr_final = &buffer->c[cpu].offset; - ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final, - offset, destptr, srcptr, copylen, + ret = rseq_cmpeqv_trymemcpy_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr_final, offset, destptr, srcptr, copylen, newval_final, cpu); if (rseq_likely(!ret)) { result = true; @@ -1167,12 +1221,6 @@ static int set_signal_handler(void) return ret; } -static -int sys_membarrier(int cmd, int flags, int cpu_id) -{ - return syscall(__NR_membarrier, cmd, flags, cpu_id); -} - static bool membarrier_private_expedited_rseq_available(void) { @@ -1188,7 +1236,7 @@ bool membarrier_private_expedited_rseq_available(void) } /* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */ -#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV +#ifdef TEST_MEMBARRIER struct test_membarrier_thread_args { int stop; intptr_t percpu_list_ptr; @@ -1216,9 +1264,10 @@ void *test_membarrier_worker_thread(void *arg) int ret; do { - int cpu = rseq_cpu_start(); + int cpu = get_current_cpu_id(); - ret = rseq_offset_deref_addv(&args->percpu_list_ptr, + ret = rseq_offset_deref_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU, + &args->percpu_list_ptr, sizeof(struct percpu_list_entry) * cpu, 1, cpu); } while (rseq_unlikely(ret)); } @@ -1299,8 +1348,7 @@ void *test_membarrier_manager_thread(void *arg) /* Make list_b "active". */ RSEQ_WRITE_ONCE(args->percpu_list_ptr, (intptr_t)&list_b); - if (sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, - MEMBARRIER_CMD_FLAG_CPU, cpu_a) && + if (rseq_membarrier_expedited(cpu_a) && errno != ENXIO /* missing CPU */) { perror("sys_membarrier"); abort(); @@ -1323,8 +1371,7 @@ void *test_membarrier_manager_thread(void *arg) /* Make list_a "active". */ RSEQ_WRITE_ONCE(args->percpu_list_ptr, (intptr_t)&list_a); - if (sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, - MEMBARRIER_CMD_FLAG_CPU, cpu_b) && + if (rseq_membarrier_expedited(cpu_b) && errno != ENXIO /* missing CPU */) { perror("sys_membarrier"); abort(); @@ -1401,7 +1448,7 @@ void test_membarrier(void) abort(); } } -#else /* RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV */ +#else /* TEST_MEMBARRIER */ static void test_membarrier(void) { @@ -1565,7 +1612,7 @@ int main(int argc, char **argv) verbose = 1; break; case 'M': - opt_mb = 1; + opt_mo = RSEQ_MO_RELEASE; break; case 'c': if (rseq_available(RSEQ_AVAILABLE_QUERY_KERNEL)) { @@ -1593,6 +1640,10 @@ int main(int argc, char **argv) if (!opt_disable_rseq && rseq_register_current_thread()) goto error; + if (!opt_disable_rseq && !rseq_validate_cpu_id()) { + fprintf(stderr, "Error: cpu id getter unavailable\n"); + goto error; + } switch (opt_test) { case 's': printf_verbose("spinlock\n"); diff --git a/tests/run_param_test.tap b/tests/run_param_test.tap index cd1e6e2..a8c5716 100755 --- a/tests/run_param_test.tap +++ b/tests/run_param_test.tap @@ -32,6 +32,12 @@ function do_test() "$RSEQ_TESTS_BUILDDIR"/param_test_compare_twice "${args[@]}" -r ${REPS} -t ${NR_THREADS} "${EXTRA_ARGS[@]}" ok $? "Running compare-twice test ${test_name}" + + "$RSEQ_TESTS_BUILDDIR"/param_test_mm_cid "${args[@]}" -r ${REPS} -t ${NR_THREADS} "${EXTRA_ARGS[@]}" + ok $? "Running mm_cid test ${test_name}" + + "$RSEQ_TESTS_BUILDDIR"/param_test_mm_cid_compare_twice "${args[@]}" -r ${REPS} -t ${NR_THREADS} "${EXTRA_ARGS[@]}" + ok $? "Running compare-twice mm_cid test ${test_name}" } function do_tests() @@ -77,7 +83,7 @@ function do_tests_inject() if [[ $? == 2 ]]; then plan_skip_all "The rseq syscall is unavailable" else - plan_tests $(( 2 * 8 * 37 )) + plan_tests $(( 4 * 8 * 37 )) fi diag "Default parameters" -- 2.34.1