#include <errno.h>
#include <stddef.h>
#include <stdbool.h>
-#include <rseq/percpu-alloc.h>
-
-#define PERCPU_POOL_LEN (1024*1024) /* 1MB */
+#include <rseq/mempool.h>
#if LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0)
enum {
#define RSEQ_INJECT_CLOBBER \
, INJECT_ASM_REG
-#define RSEQ_INJECT_ASM(n) \
- "mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \
+/*
+ * Use ip-relative addressing to get the loop counter.
+ */
+#define __RSEQ_INJECT_ASM(n, ref_ip, ref_label) \
+ "movl " __rseq_str(ref_ip) ", %%" INJECT_ASM_REG "\n\t" \
+ "leal ( asm_loop_cnt_" #n " - " __rseq_str(ref_label) "b)(%%" INJECT_ASM_REG "), %%" INJECT_ASM_REG "\n\t" \
+ "movl (%%" INJECT_ASM_REG "), %%" INJECT_ASM_REG "\n\t" \
"test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
"jz 333f\n\t" \
"222:\n\t" \
"jnz 222b\n\t" \
"333:\n\t"
+#define RSEQ_INJECT_ASM(n) \
+ __RSEQ_INJECT_ASM(n, %[ref_ip], RSEQ_ASM_REF_LABEL)
+
#elif defined(__x86_64__)
#define INJECT_ASM_REG_P "rax"
return syscall(__NR_membarrier, cmd, flags, cpu_id);
}
-#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
+#ifdef rseq_arch_has_load_cbne_load_add_load_add_store
#define TEST_MEMBARRIER
#endif
struct percpu_list_node *next;
};
-struct percpu_list_entry {
- struct percpu_list_node *head;
-} __attribute__((aligned(128)));
-
struct percpu_list {
- struct percpu_list_entry c[CPU_SETSIZE];
+ struct percpu_list_node *head;
};
#define BUFFER_ITEM_PER_CPU 100
intptr_t data;
};
-struct percpu_buffer_entry {
+struct percpu_buffer {
intptr_t offset;
intptr_t buflen;
struct percpu_buffer_node **array;
-} __attribute__((aligned(128)));
-
-struct percpu_buffer {
- struct percpu_buffer_entry c[CPU_SETSIZE];
};
#define MEMCPY_BUFFER_ITEM_PER_CPU 100
uint64_t data2;
};
-struct percpu_memcpy_buffer_entry {
+struct percpu_memcpy_buffer {
intptr_t offset;
intptr_t buflen;
struct percpu_memcpy_buffer_node *array;
-} __attribute__((aligned(128)));
-
-struct percpu_memcpy_buffer {
- struct percpu_memcpy_buffer_entry c[CPU_SETSIZE];
};
/* A simple percpu spinlock. Grabs lock on current cpu. */
pthread_t test_threads[num_threads];
struct spinlock_test_data __rseq_percpu *data;
struct spinlock_thread_test_data thread_data[num_threads];
- struct rseq_percpu_pool *mempool;
+ struct rseq_mempool *mempool;
- mempool = rseq_percpu_pool_create(sizeof(struct spinlock_test_data),
- PERCPU_POOL_LEN, CPU_SETSIZE, PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0, 0);
+ mempool = rseq_mempool_create("spinlock_test_data",
+ sizeof(struct spinlock_test_data),
+ 0, CPU_SETSIZE, NULL);
if (!mempool) {
- perror("rseq_percpu_pool_create");
+ perror("rseq_mempool_create");
abort();
}
data = (struct spinlock_test_data __rseq_percpu *)rseq_percpu_zmalloc(mempool);
assert(sum == (uint64_t)opt_reps * num_threads);
rseq_percpu_free(data);
- ret = rseq_percpu_pool_destroy(mempool);
+ ret = rseq_mempool_destroy(mempool);
if (ret) {
- perror("rseq_percpu_pool_destroy");
+ perror("rseq_mempool_destroy");
abort();
}
}
pthread_t test_threads[num_threads];
struct inc_test_data __rseq_percpu *data;
struct inc_thread_test_data thread_data[num_threads];
- struct rseq_percpu_pool *mempool;
+ struct rseq_mempool *mempool;
- mempool = rseq_percpu_pool_create(sizeof(struct inc_test_data),
- PERCPU_POOL_LEN, CPU_SETSIZE, PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0, 0);
+ mempool = rseq_mempool_create("inc_test_data",
+ sizeof(struct inc_test_data),
+ 0, CPU_SETSIZE, NULL);
if (!mempool) {
- perror("rseq_percpu_pool_create");
+ perror("rseq_mempool_create");
abort();
}
data = (struct inc_test_data __rseq_percpu *)rseq_percpu_zmalloc(mempool);
assert(sum == (uint64_t)opt_reps * num_threads);
rseq_percpu_free(data);
- ret = rseq_percpu_pool_destroy(mempool);
+ ret = rseq_mempool_destroy(mempool);
if (ret) {
- perror("rseq_percpu_pool_destroy");
+ perror("rseq_mempool_destroy");
abort();
}
}
-static void this_cpu_list_push(struct percpu_list *list,
+static void this_cpu_list_push(struct percpu_list __rseq_percpu *list,
struct percpu_list_node *node,
int *_cpu)
{
for (;;) {
intptr_t *targetptr, newval, expect;
+ struct percpu_list *cpulist;
int ret;
cpu = get_current_cpu_id();
+ cpulist = rseq_percpu_ptr(list, cpu);
/* Load list->c[cpu].head with single-copy atomicity. */
- expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head);
+ expect = (intptr_t)RSEQ_READ_ONCE(cpulist->head);
newval = (intptr_t)node;
- targetptr = (intptr_t *)&list->c[cpu].head;
+ targetptr = (intptr_t *)&cpulist->head;
node->next = (struct percpu_list_node *)expect;
ret = rseq_load_cbne_store__ptr(RSEQ_MO_RELAXED, RSEQ_PERCPU,
targetptr, expect, newval, cpu);
* rseq primitive allows us to implement pop without concerns over
* ABA-type races.
*/
-static struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list,
+static struct percpu_list_node *this_cpu_list_pop(struct percpu_list __rseq_percpu *list,
int *_cpu)
{
struct percpu_list_node *node = NULL;
for (;;) {
struct percpu_list_node *head;
intptr_t *targetptr, expectnot, *load;
+ struct percpu_list *cpulist;
long offset;
int ret;
cpu = get_current_cpu_id();
- targetptr = (intptr_t *)&list->c[cpu].head;
+ cpulist = rseq_percpu_ptr(list, cpu);
+ targetptr = (intptr_t *)&cpulist->head;
expectnot = (intptr_t)NULL;
offset = offsetof(struct percpu_list_node, next);
load = (intptr_t *)&head;
* __percpu_list_pop is not safe against concurrent accesses. Should
* only be used on lists that are not concurrently modified.
*/
-static struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu)
+static struct percpu_list_node *__percpu_list_pop(struct percpu_list __rseq_percpu *list, int cpu)
{
+ struct percpu_list *cpulist = rseq_percpu_ptr(list, cpu);
struct percpu_list_node *node;
- node = list->c[cpu].head;
+ node = cpulist->head;
if (!node)
return NULL;
- list->c[cpu].head = node->next;
+ cpulist->head = node->next;
return node;
}
static void *test_percpu_list_thread(void *arg)
{
long long i, reps;
- struct percpu_list *list = (struct percpu_list *)arg;
+ struct percpu_list __rseq_percpu *list = (struct percpu_list __rseq_percpu *)arg;
if (!opt_disable_rseq && rseq_register_current_thread())
abort();
const int num_threads = opt_threads;
int i, j, ret;
uint64_t sum = 0, expected_sum = 0;
- struct percpu_list list;
+ struct percpu_list __rseq_percpu *list;
pthread_t test_threads[num_threads];
cpu_set_t allowed_cpus;
+ struct rseq_mempool *mempool;
- memset(&list, 0, sizeof(list));
+ mempool = rseq_mempool_create("percpu_list", sizeof(struct percpu_list),
+ 0, CPU_SETSIZE, NULL);
+ if (!mempool) {
+ perror("rseq_mempool_create");
+ abort();
+ }
+ list = (struct percpu_list __rseq_percpu *)rseq_percpu_zmalloc(mempool);
+ if (!list) {
+ perror("rseq_percpu_zmalloc");
+ abort();
+ }
/* Generate list entries for every usable cpu. */
sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus))
continue;
for (j = 1; j <= 100; j++) {
+ struct percpu_list *cpulist = rseq_percpu_ptr(list, i);
struct percpu_list_node *node;
expected_sum += j;
node = (struct percpu_list_node *) malloc(sizeof(*node));
assert(node);
node->data = j;
- node->next = list.c[i].head;
- list.c[i].head = node;
+ node->next = cpulist->head;
+ cpulist->head = node;
}
}
for (i = 0; i < num_threads; i++) {
ret = pthread_create(&test_threads[i], NULL,
- test_percpu_list_thread, &list);
+ test_percpu_list_thread, list);
if (ret) {
errno = ret;
perror("pthread_create");
if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus))
continue;
- while ((node = __percpu_list_pop(&list, i))) {
+ while ((node = __percpu_list_pop(list, i))) {
sum += node->data;
free(node);
}
* test is running).
*/
assert(sum == expected_sum);
+ rseq_percpu_free(list);
+ ret = rseq_mempool_destroy(mempool);
+ if (ret) {
+ perror("rseq_mempool_destroy");
+ abort();
+ }
}
-static bool this_cpu_buffer_push(struct percpu_buffer *buffer,
+static bool this_cpu_buffer_push(struct percpu_buffer __rseq_percpu *buffer,
struct percpu_buffer_node *node,
int *_cpu)
{
int cpu;
for (;;) {
+ struct percpu_buffer *cpubuffer;
intptr_t *targetptr_spec, newval_spec;
intptr_t *targetptr_final, newval_final;
intptr_t offset;
int ret;
cpu = get_current_cpu_id();
- offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
- if (offset == buffer->c[cpu].buflen)
+ cpubuffer = rseq_percpu_ptr(buffer, cpu);
+ offset = RSEQ_READ_ONCE(cpubuffer->offset);
+ if (offset == cpubuffer->buflen)
break;
newval_spec = (intptr_t)node;
- targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset];
+ targetptr_spec = (intptr_t *)&cpubuffer->array[offset];
newval_final = offset + 1;
- targetptr_final = &buffer->c[cpu].offset;
+ targetptr_final = &cpubuffer->offset;
ret = rseq_load_cbne_store_store__ptr(opt_mo, RSEQ_PERCPU,
targetptr_final, offset, targetptr_spec,
newval_spec, newval_final, cpu);
return result;
}
-static struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer,
+static struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer __rseq_percpu *buffer,
int *_cpu)
{
struct percpu_buffer_node *head;
int cpu;
for (;;) {
+ struct percpu_buffer *cpubuffer;
intptr_t *targetptr, newval;
intptr_t offset;
int ret;
cpu = get_current_cpu_id();
+ cpubuffer = rseq_percpu_ptr(buffer, cpu);
/* Load offset with single-copy atomicity. */
- offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
+ offset = RSEQ_READ_ONCE(cpubuffer->offset);
if (offset == 0) {
head = NULL;
break;
}
- head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]);
+ head = RSEQ_READ_ONCE(cpubuffer->array[offset - 1]);
newval = offset - 1;
- targetptr = (intptr_t *)&buffer->c[cpu].offset;
+ targetptr = (intptr_t *)&cpubuffer->offset;
ret = rseq_load_cbne_load_cbne_store__ptr(RSEQ_MO_RELAXED, RSEQ_PERCPU,
targetptr, offset,
- (intptr_t *)&buffer->c[cpu].array[offset - 1],
+ (intptr_t *)&cpubuffer->array[offset - 1],
(intptr_t)head, newval, cpu);
if (rseq_likely(!ret))
break;
* __percpu_buffer_pop is not safe against concurrent accesses. Should
* only be used on buffers that are not concurrently modified.
*/
-static struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer *buffer,
+static struct percpu_buffer_node *__percpu_buffer_pop(struct percpu_buffer __rseq_percpu *buffer,
int cpu)
{
+ struct percpu_buffer *cpubuffer;
struct percpu_buffer_node *head;
intptr_t offset;
- offset = buffer->c[cpu].offset;
+ cpubuffer = rseq_percpu_ptr(buffer, cpu);
+ offset = cpubuffer->offset;
if (offset == 0)
return NULL;
- head = buffer->c[cpu].array[offset - 1];
- buffer->c[cpu].offset = offset - 1;
+ head = cpubuffer->array[offset - 1];
+ cpubuffer->offset = offset - 1;
return head;
}
static void *test_percpu_buffer_thread(void *arg)
{
long long i, reps;
- struct percpu_buffer *buffer = (struct percpu_buffer *)arg;
+ struct percpu_buffer __rseq_percpu *buffer = (struct percpu_buffer __rseq_percpu *)arg;
if (!opt_disable_rseq && rseq_register_current_thread())
abort();
const int num_threads = opt_threads;
int i, j, ret;
uint64_t sum = 0, expected_sum = 0;
- struct percpu_buffer buffer;
+ struct percpu_buffer __rseq_percpu *buffer;
pthread_t test_threads[num_threads];
cpu_set_t allowed_cpus;
+ struct rseq_mempool *mempool;
- memset(&buffer, 0, sizeof(buffer));
+ mempool = rseq_mempool_create("percpu_buffer", sizeof(struct percpu_buffer),
+ 0, CPU_SETSIZE, NULL);
+ if (!mempool) {
+ perror("rseq_mempool_create");
+ abort();
+ }
+ buffer = (struct percpu_buffer __rseq_percpu *)rseq_percpu_zmalloc(mempool);
+ if (!buffer) {
+ perror("rseq_percpu_zmalloc");
+ abort();
+ }
/* Generate list entries for every usable cpu. */
sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
for (i = 0; i < CPU_SETSIZE; i++) {
+ struct percpu_buffer *cpubuffer;
+
if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus))
continue;
+ cpubuffer = rseq_percpu_ptr(buffer, i);
/* Worse-case is every item in same CPU. */
- buffer.c[i].array =
+ cpubuffer->array =
(struct percpu_buffer_node **)
- malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
+ malloc(sizeof(*cpubuffer->array) * CPU_SETSIZE *
BUFFER_ITEM_PER_CPU);
- assert(buffer.c[i].array);
- buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU;
+ assert(cpubuffer->array);
+ cpubuffer->buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU;
for (j = 1; j <= BUFFER_ITEM_PER_CPU; j++) {
struct percpu_buffer_node *node;
node = (struct percpu_buffer_node *) malloc(sizeof(*node));
assert(node);
node->data = j;
- buffer.c[i].array[j - 1] = node;
- buffer.c[i].offset++;
+ cpubuffer->array[j - 1] = node;
+ cpubuffer->offset++;
}
}
for (i = 0; i < num_threads; i++) {
ret = pthread_create(&test_threads[i], NULL,
- test_percpu_buffer_thread, &buffer);
+ test_percpu_buffer_thread, buffer);
if (ret) {
errno = ret;
perror("pthread_create");
}
for (i = 0; i < CPU_SETSIZE; i++) {
+ struct percpu_buffer *cpubuffer;
struct percpu_buffer_node *node;
if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus))
continue;
- while ((node = __percpu_buffer_pop(&buffer, i))) {
+ cpubuffer = rseq_percpu_ptr(buffer, i);
+ while ((node = __percpu_buffer_pop(buffer, i))) {
sum += node->data;
free(node);
}
- free(buffer.c[i].array);
+ free(cpubuffer->array);
}
/*
* test is running).
*/
assert(sum == expected_sum);
+ rseq_percpu_free(buffer);
+ ret = rseq_mempool_destroy(mempool);
+ if (ret) {
+ perror("rseq_mempool_destroy");
+ abort();
+ }
}
-static bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer,
+static bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer __rseq_percpu *buffer,
struct percpu_memcpy_buffer_node item,
int *_cpu)
{
int cpu;
for (;;) {
+ struct percpu_memcpy_buffer *cpubuffer;
intptr_t *targetptr_final, newval_final, offset;
char *destptr, *srcptr;
size_t copylen;
int ret;
cpu = get_current_cpu_id();
+ cpubuffer = rseq_percpu_ptr(buffer, cpu);
/* Load offset with single-copy atomicity. */
- offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
- if (offset == buffer->c[cpu].buflen)
+ offset = RSEQ_READ_ONCE(cpubuffer->offset);
+ if (offset == cpubuffer->buflen)
break;
- destptr = (char *)&buffer->c[cpu].array[offset];
+ destptr = (char *)&cpubuffer->array[offset];
srcptr = (char *)&item;
/* copylen must be <= 4kB. */
copylen = sizeof(item);
newval_final = offset + 1;
- targetptr_final = &buffer->c[cpu].offset;
+ targetptr_final = &cpubuffer->offset;
ret = rseq_load_cbne_memcpy_store__ptr(
opt_mo, RSEQ_PERCPU,
targetptr_final, offset,
return result;
}
-static bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
+static bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer __rseq_percpu *buffer,
struct percpu_memcpy_buffer_node *item,
int *_cpu)
{
int cpu;
for (;;) {
+ struct percpu_memcpy_buffer *cpubuffer;
intptr_t *targetptr_final, newval_final, offset;
char *destptr, *srcptr;
size_t copylen;
int ret;
cpu = get_current_cpu_id();
+ cpubuffer = rseq_percpu_ptr(buffer, cpu);
/* Load offset with single-copy atomicity. */
- offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
+ offset = RSEQ_READ_ONCE(cpubuffer->offset);
if (offset == 0)
break;
destptr = (char *)item;
- srcptr = (char *)&buffer->c[cpu].array[offset - 1];
+ srcptr = (char *)&cpubuffer->array[offset - 1];
/* copylen must be <= 4kB. */
copylen = sizeof(*item);
newval_final = offset - 1;
- targetptr_final = &buffer->c[cpu].offset;
+ targetptr_final = &cpubuffer->offset;
ret = rseq_load_cbne_memcpy_store__ptr(RSEQ_MO_RELAXED, RSEQ_PERCPU,
targetptr_final, offset, destptr, srcptr, copylen,
newval_final, cpu);
* __percpu_memcpy_buffer_pop is not safe against concurrent accesses. Should
* only be used on buffers that are not concurrently modified.
*/
-static bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer,
+static bool __percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer __rseq_percpu *buffer,
struct percpu_memcpy_buffer_node *item,
int cpu)
{
+ struct percpu_memcpy_buffer *cpubuffer;
intptr_t offset;
- offset = buffer->c[cpu].offset;
+ cpubuffer = rseq_percpu_ptr(buffer, cpu);
+ offset = cpubuffer->offset;
if (offset == 0)
return false;
- memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item));
- buffer->c[cpu].offset = offset - 1;
+ memcpy(item, &cpubuffer->array[offset - 1], sizeof(*item));
+ cpubuffer->offset = offset - 1;
return true;
}
static void *test_percpu_memcpy_buffer_thread(void *arg)
{
long long i, reps;
- struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg;
+ struct percpu_memcpy_buffer __rseq_percpu *buffer = (struct percpu_memcpy_buffer __rseq_percpu *)arg;
if (!opt_disable_rseq && rseq_register_current_thread())
abort();
const int num_threads = opt_threads;
int i, j, ret;
uint64_t sum = 0, expected_sum = 0;
- struct percpu_memcpy_buffer buffer;
+ struct percpu_memcpy_buffer *buffer;
pthread_t test_threads[num_threads];
cpu_set_t allowed_cpus;
+ struct rseq_mempool *mempool;
- memset(&buffer, 0, sizeof(buffer));
+ mempool = rseq_mempool_create("percpu_memcpy_buffer",
+ sizeof(struct percpu_memcpy_buffer),
+ 0, CPU_SETSIZE, NULL);
+ if (!mempool) {
+ perror("rseq_mempool_create");
+ abort();
+ }
+ buffer = (struct percpu_memcpy_buffer __rseq_percpu *)rseq_percpu_zmalloc(mempool);
+ if (!buffer) {
+ perror("rseq_percpu_zmalloc");
+ abort();
+ }
/* Generate list entries for every usable cpu. */
sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus);
for (i = 0; i < CPU_SETSIZE; i++) {
+ struct percpu_memcpy_buffer *cpubuffer;
+
if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus))
continue;
+ cpubuffer = rseq_percpu_ptr(buffer, i);
/* Worse-case is every item in same CPU. */
- buffer.c[i].array =
+ cpubuffer->array =
(struct percpu_memcpy_buffer_node *)
- malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE *
+ malloc(sizeof(*cpubuffer->array) * CPU_SETSIZE *
MEMCPY_BUFFER_ITEM_PER_CPU);
- assert(buffer.c[i].array);
- buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU;
+ assert(cpubuffer->array);
+ cpubuffer->buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU;
for (j = 1; j <= MEMCPY_BUFFER_ITEM_PER_CPU; j++) {
expected_sum += 2 * j + 1;
* within a single word, so allocate an object
* for each node.
*/
- buffer.c[i].array[j - 1].data1 = j;
- buffer.c[i].array[j - 1].data2 = j + 1;
- buffer.c[i].offset++;
+ cpubuffer->array[j - 1].data1 = j;
+ cpubuffer->array[j - 1].data2 = j + 1;
+ cpubuffer->offset++;
}
}
for (i = 0; i < num_threads; i++) {
ret = pthread_create(&test_threads[i], NULL,
test_percpu_memcpy_buffer_thread,
- &buffer);
+ buffer);
if (ret) {
errno = ret;
perror("pthread_create");
for (i = 0; i < CPU_SETSIZE; i++) {
struct percpu_memcpy_buffer_node item;
+ struct percpu_memcpy_buffer *cpubuffer;
if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus))
continue;
- while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) {
+ cpubuffer = rseq_percpu_ptr(buffer, i);
+ while (__percpu_memcpy_buffer_pop(buffer, &item, i)) {
sum += item.data1;
sum += item.data2;
}
- free(buffer.c[i].array);
+ free(cpubuffer->array);
}
/*
* test is running).
*/
assert(sum == expected_sum);
+ rseq_percpu_free(buffer);
+ ret = rseq_mempool_destroy(mempool);
+ if (ret) {
+ perror("rseq_mempool_destroy");
+ abort();
+ }
}
-
static void test_signal_interrupt_handler(__attribute__ ((unused)) int signo)
{
signals_delivered++;
/* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */
#ifdef TEST_MEMBARRIER
struct test_membarrier_thread_args {
+ struct rseq_mempool *mempool;
+ struct percpu_list __rseq_percpu *percpu_list_ptr;
int stop;
- intptr_t percpu_list_ptr;
};
/* Worker threads modify data in their "active" percpu lists. */
{
struct test_membarrier_thread_args *args =
(struct test_membarrier_thread_args *)arg;
- const int iters = opt_reps;
- int i;
+ const long long iters = opt_reps;
+ long long i;
if (rseq_register_current_thread()) {
fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
do {
int cpu = get_current_cpu_id();
+ struct percpu_list __rseq_percpu *list = RSEQ_READ_ONCE(args->percpu_list_ptr);
+ struct percpu_list *cpulist = rseq_percpu_ptr(list, cpu);
- ret = rseq_load_add_load_add_store__ptr(RSEQ_MO_RELAXED, RSEQ_PERCPU,
- &args->percpu_list_ptr,
- sizeof(struct percpu_list_entry) * cpu, 1, cpu);
+ ret = rseq_load_cbne_load_add_load_add_store__ptr(RSEQ_MO_RELAXED, RSEQ_PERCPU,
+ (intptr_t *) &args->percpu_list_ptr,
+ (intptr_t) list, (intptr_t *) &cpulist->head, 0, 1, cpu);
} while (rseq_unlikely(ret));
}
}
static
-void test_membarrier_init_percpu_list(struct percpu_list *list)
+struct percpu_list __rseq_percpu *test_membarrier_alloc_percpu_list(struct rseq_mempool *mempool)
{
+ struct percpu_list __rseq_percpu *list;
int i;
- memset(list, 0, sizeof(*list));
+ list = (struct percpu_list __rseq_percpu *)rseq_percpu_zmalloc(mempool);
+ if (!list) {
+ perror("rseq_percpu_zmalloc");
+ return NULL;
+ }
for (i = 0; i < CPU_SETSIZE; i++) {
+ struct percpu_list *cpulist = rseq_percpu_ptr(list, i);
struct percpu_list_node *node;
node = (struct percpu_list_node *) malloc(sizeof(*node));
assert(node);
node->data = 0;
node->next = NULL;
- list->c[i].head = node;
+ cpulist->head = node;
}
+ return list;
}
static
-void test_membarrier_free_percpu_list(struct percpu_list *list)
+void test_membarrier_free_percpu_list(struct percpu_list __rseq_percpu *list)
{
int i;
for (i = 0; i < CPU_SETSIZE; i++)
- free(list->c[i].head);
+ free(rseq_percpu_ptr(list, i)->head);
+ rseq_percpu_free(list);
+}
+
+static
+long long test_membarrier_count_percpu_list(struct percpu_list __rseq_percpu *list)
+{
+ long long total_count = 0;
+ int i;
+
+ for (i = 0; i < CPU_SETSIZE; i++)
+ total_count += rseq_percpu_ptr(list, i)->head->data;
+ return total_count;
}
/*
{
struct test_membarrier_thread_args *args =
(struct test_membarrier_thread_args *)arg;
- struct percpu_list list_a, list_b;
+ struct percpu_list __rseq_percpu *list_a, __rseq_percpu *list_b;
intptr_t expect_a = 0, expect_b = 0;
int cpu_a = 0, cpu_b = 0;
+ struct rseq_mempool *mempool;
+ int ret;
+ long long total_count = 0;
+
+ mempool = rseq_mempool_create("percpu_list", sizeof(struct percpu_list),
+ 0, CPU_SETSIZE, NULL);
+ if (!mempool) {
+ perror("rseq_mempool_create");
+ abort();
+ }
+ args->mempool = mempool;
if (rseq_register_current_thread()) {
fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
}
/* Init lists. */
- test_membarrier_init_percpu_list(&list_a);
- test_membarrier_init_percpu_list(&list_b);
+ list_a = test_membarrier_alloc_percpu_list(mempool);
+ assert(list_a);
+ list_b = test_membarrier_alloc_percpu_list(mempool);
+ assert(list_b);
/* Initialize lists before publishing them. */
rseq_smp_wmb();
- RSEQ_WRITE_ONCE(args->percpu_list_ptr, (intptr_t)&list_a);
+ RSEQ_WRITE_ONCE(args->percpu_list_ptr, list_a);
while (!RSEQ_READ_ONCE(args->stop)) {
/* list_a is "active". */
* As list_b is "inactive", we should never see changes
* to list_b.
*/
- if (expect_b != RSEQ_READ_ONCE(list_b.c[cpu_b].head->data)) {
+ if (expect_b != RSEQ_READ_ONCE(rseq_percpu_ptr(list_b, cpu_b)->head->data)) {
fprintf(stderr, "Membarrier test failed\n");
abort();
}
/* Make list_b "active". */
- RSEQ_WRITE_ONCE(args->percpu_list_ptr, (intptr_t)&list_b);
+ RSEQ_WRITE_ONCE(args->percpu_list_ptr, list_b);
if (rseq_membarrier_expedited(cpu_a) &&
errno != ENXIO /* missing CPU */) {
perror("sys_membarrier");
* Cpu A should now only modify list_b, so the values
* in list_a should be stable.
*/
- expect_a = RSEQ_READ_ONCE(list_a.c[cpu_a].head->data);
+ expect_a = RSEQ_READ_ONCE(rseq_percpu_ptr(list_a, cpu_a)->head->data);
cpu_b = rand() % CPU_SETSIZE;
/*
* As list_a is "inactive", we should never see changes
* to list_a.
*/
- if (expect_a != RSEQ_READ_ONCE(list_a.c[cpu_a].head->data)) {
+ if (expect_a != RSEQ_READ_ONCE(rseq_percpu_ptr(list_a, cpu_a)->head->data)) {
fprintf(stderr, "Membarrier test failed\n");
abort();
}
/* Make list_a "active". */
- RSEQ_WRITE_ONCE(args->percpu_list_ptr, (intptr_t)&list_a);
+ RSEQ_WRITE_ONCE(args->percpu_list_ptr, list_a);
if (rseq_membarrier_expedited(cpu_b) &&
errno != ENXIO /* missing CPU */) {
perror("sys_membarrier");
abort();
}
/* Remember a value from list_b. */
- expect_b = RSEQ_READ_ONCE(list_b.c[cpu_b].head->data);
+ expect_b = RSEQ_READ_ONCE(rseq_percpu_ptr(list_b, cpu_b)->head->data);
}
- test_membarrier_free_percpu_list(&list_a);
- test_membarrier_free_percpu_list(&list_b);
+ total_count += test_membarrier_count_percpu_list(list_a);
+ total_count += test_membarrier_count_percpu_list(list_b);
+
+ /* Validate that we observe the right number of increments. */
+ if (total_count != opt_threads * opt_reps) {
+ fprintf(stderr, "Error: Observed %lld increments, expected %lld\n",
+ total_count, opt_threads * opt_reps);
+ abort();
+ }
+ test_membarrier_free_percpu_list(list_a);
+ test_membarrier_free_percpu_list(list_b);
if (rseq_unregister_current_thread()) {
fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
errno, strerror(errno));
abort();
}
+ ret = rseq_mempool_destroy(mempool);
+ if (ret) {
+ perror("rseq_mempool_destroy");
+ abort();
+ }
+
return NULL;
}
abort();
}
+ thread_args.percpu_list_ptr = NULL;
thread_args.stop = 0;
- thread_args.percpu_list_ptr = 0;
ret = pthread_create(&manager_thread, NULL,
test_membarrier_manager_thread, &thread_args);
if (ret) {
"Skipping membarrier test.\n");
return;
}
- fprintf(stderr, "rseq_load_add_load_add_store__ptr is not implemented on this architecture. "
+ fprintf(stderr, "rseq_load_cbne_load_add_load_add_store__ptr is not implemented on this architecture. "
"Skipping membarrier test.\n");
}
#endif