X-Git-Url: http://drtracing.org/?a=blobdiff_plain;f=src%2Frcu.c;h=9652725fdc1f037e9bf2eada107fdf954c489b90;hb=873bbf16c6bcfe2c11fca7e76dd7284c5afbee99;hp=3c6dafbb35b4f6f6d1b51dd56648111909a57381;hpb=bddcdc92ce71eac0bcb0e2e404b9fdd261679f32;p=libside.git

diff --git a/src/rcu.c b/src/rcu.c
index 3c6dafb..9652725 100644
--- a/src/rcu.c
+++ b/src/rcu.c
@@ -30,6 +30,88 @@ membarrier(int cmd, unsigned int flags, int cpu_id)
 	return syscall(__NR_membarrier, cmd, flags, cpu_id);
 }
 
+/*
+ * Wait/wakeup scheme with single waiter/many wakers.
+ */
+static
+void wait_gp_prepare(struct side_rcu_gp_state *gp_state)
+{
+	__atomic_store_n(&gp_state->futex, -1, __ATOMIC_RELAXED);
+	/*
+	 * This memory barrier (H) pairs with memory barrier (F). It
+	 * orders store to futex before load of RCU reader's counter
+	 * state, thus ensuring that load of RCU reader's counters does
+	 * not leak outside of futex state=-1.
+	 */
+	if (side_rcu_rseq_membarrier_available) {
+		if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0, 0)) {
+			perror("membarrier");
+			abort();
+		}
+	} else {
+		__atomic_thread_fence(__ATOMIC_SEQ_CST);
+	}
+}
+
+static
+void wait_gp_end(struct side_rcu_gp_state *gp_state)
+{
+	/*
+	 * This memory barrier (G) pairs with memory barrier (F). It
+	 * orders load of RCU reader's counter state before storing the
+	 * futex value, thus ensuring that load of RCU reader's counters
+	 * does not leak outside of futex state=-1.
+	 */
+	if (side_rcu_rseq_membarrier_available) {
+		if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0, 0)) {
+			perror("membarrier");
+			abort();
+		}
+	} else {
+		__atomic_thread_fence(__ATOMIC_SEQ_CST);
+	}
+	__atomic_store_n(&gp_state->futex, 0, __ATOMIC_RELAXED);
+}
+
+static
+void wait_gp(struct side_rcu_gp_state *gp_state)
+{
+	/*
+	 * This memory barrier (G) pairs with memory barrier (F). It
+	 * orders load of RCU reader's counter state before loading the
+	 * futex value.
+	 */
+	if (side_rcu_rseq_membarrier_available) {
+		if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0, 0)) {
+			perror("membarrier");
+			abort();
+		}
+	} else {
+		__atomic_thread_fence(__ATOMIC_SEQ_CST);
+	}
+	while (__atomic_load_n(&gp_state->futex, __ATOMIC_RELAXED) == -1) {
+		if (!futex(&gp_state->futex, FUTEX_WAIT, -1, NULL, NULL, 0)) {
+			/*
+			 * May be awakened by either spurious wake up or
+			 * because the state is now as expected.
+			 */
+			continue;
+		}
+		switch (errno) {
+		case EWOULDBLOCK:
+			/* Value already changed. */
+			return;
+		case EINTR:
+			/* Retry if interrupted by signal. */
+			break;	/* Get out of switch. */
+		default:
+			/* Unexpected error. */
+			abort();
+		}
+	}
+	return;
+}
+
 /* active_readers is an input/output parameter. */
 static
 void check_active_readers(struct side_rcu_gp_state *gp_state, bool *active_readers)
@@ -109,11 +191,13 @@ void wait_for_prev_period_readers(struct side_rcu_gp_state *gp_state, bool *acti
 	 * previous period.
 	 */
 	for (;;) {
+		wait_gp_prepare(gp_state);
 		check_active_readers(gp_state, active_readers);
-		if (!active_readers[prev_period])
+		if (!active_readers[prev_period]) {
+			wait_gp_end(gp_state);
 			break;
-		/* Retry after 10ms. */
-		poll(NULL, 0, 10);
+		}
+		wait_gp(gp_state);
 	}
 }
 
@@ -207,7 +291,8 @@ void side_rcu_gp_init(struct side_rcu_gp_state *rcu_gp)
 	if (!rcu_gp->nr_cpus)
 		abort();
 	pthread_mutex_init(&rcu_gp->gp_lock, NULL);
-	rcu_gp->percpu_state = calloc(rcu_gp->nr_cpus, sizeof(struct side_rcu_cpu_gp_state));
+	rcu_gp->percpu_state = (struct side_rcu_cpu_gp_state *)
+		calloc(rcu_gp->nr_cpus, sizeof(struct side_rcu_cpu_gp_state));
 	if (!rcu_gp->percpu_state)
 		abort();
 	if (!membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0, 0))