X-Git-Url: http://drtracing.org/?a=blobdiff_plain;f=src%2Frcu.h;h=4db3500566abc002042d9f63a6b617d4c24d76de;hb=873bbf16c6bcfe2c11fca7e76dd7284c5afbee99;hp=f2221478c9ec484094c1bad7839cb39b7116100d;hpb=cd50bc06e32848c725369e1f07b6e3a910e7c846;p=libside.git

diff --git a/src/rcu.h b/src/rcu.h
index f222147..4db3500 100644
--- a/src/rcu.h
+++ b/src/rcu.h
@@ -70,33 +70,34 @@ void side_rcu_wake_up_gp(struct side_rcu_gp_state *gp_state)
 static inline
 void side_rcu_read_begin(struct side_rcu_gp_state *gp_state, struct side_rcu_read_state *read_state)
 {
-	unsigned int period = __atomic_load_n(&gp_state->period, __ATOMIC_RELAXED);
 	struct side_rcu_percpu_count *begin_cpu_count;
 	struct side_rcu_cpu_gp_state *cpu_gp_state;
+	unsigned int period;
 	int cpu;
 
-	if (side_likely(side_rcu_rseq_membarrier_available)) {
-		cpu = rseq_cpu_start();
-		cpu_gp_state = &gp_state->percpu_state[cpu];
-		read_state->percpu_count = begin_cpu_count = &cpu_gp_state->count[period];
-		read_state->cpu = cpu;
-		if (side_likely(!rseq_addv((intptr_t *)&begin_cpu_count->rseq_begin, 1, cpu))) {
-			/*
-			 * This compiler barrier (A) is paired with membarrier() at (C),
-			 * (D), (E). It effectively upgrades this compiler barrier to a
-			 * SEQ_CST fence with respect to the paired barriers.
-			 *
-			 * This barrier (A) ensures that the contents of the read-side
-			 * critical section does not leak before the "begin" counter
-			 * increment. It pairs with memory barriers (D) and (E).
-			 *
-			 * This barrier (A) also ensures that the "begin" increment is
-			 * before the "end" increment. It pairs with memory barrier (C).
-			 * It is redundant with barrier (B) for that purpose.
-			 */
-			rseq_barrier();
-			return;
-		}
+	cpu = rseq_cpu_start();
+	period = __atomic_load_n(&gp_state->period, __ATOMIC_RELAXED);
+	cpu_gp_state = &gp_state->percpu_state[cpu];
+	read_state->percpu_count = begin_cpu_count = &cpu_gp_state->count[period];
+	read_state->cpu = cpu;
+	if (side_likely(side_rcu_rseq_membarrier_available &&
+			!rseq_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU_CPU_ID,
+				   (intptr_t *)&begin_cpu_count->rseq_begin, 1, cpu))) {
+		/*
+		 * This compiler barrier (A) is paired with membarrier() at (C),
+		 * (D), (E). It effectively upgrades this compiler barrier to a
+		 * SEQ_CST fence with respect to the paired barriers.
+		 *
+		 * This barrier (A) ensures that the contents of the read-side
+		 * critical section does not leak before the "begin" counter
+		 * increment. It pairs with memory barriers (D) and (E).
+		 *
+		 * This barrier (A) also ensures that the "begin" increment is
+		 * before the "end" increment. It pairs with memory barrier (C).
+		 * It is redundant with barrier (B) for that purpose.
+		 */
+		rseq_barrier();
+		return;
 	}
 	/* Fallback to atomic increment and SEQ_CST. */
 	cpu = sched_getcpu();
@@ -114,39 +115,38 @@ void side_rcu_read_end(struct side_rcu_gp_state *gp_state, struct side_rcu_read_
 	struct side_rcu_percpu_count *begin_cpu_count = read_state->percpu_count;
 	int cpu = read_state->cpu;
 
-	if (side_likely(side_rcu_rseq_membarrier_available)) {
+	/*
+	 * This compiler barrier (B) is paired with membarrier() at (C),
+	 * (D), (E). It effectively upgrades this compiler barrier to a
+	 * SEQ_CST fence with respect to the paired barriers.
+	 *
+	 * This barrier (B) ensures that the contents of the read-side
+	 * critical section does not leak after the "end" counter
+	 * increment. It pairs with memory barriers (D) and (E).
+	 *
+	 * This barrier (B) also ensures that the "begin" increment is
+	 * before the "end" increment. It pairs with memory barrier (C).
+	 * It is redundant with barrier (A) for that purpose.
+	 */
+	rseq_barrier();
+	if (side_likely(side_rcu_rseq_membarrier_available &&
+			!rseq_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU_CPU_ID,
+				   (intptr_t *)&begin_cpu_count->rseq_end, 1, cpu))) {
 		/*
-		 * This compiler barrier (B) is paired with membarrier() at (C),
-		 * (D), (E). It effectively upgrades this compiler barrier to a
-		 * SEQ_CST fence with respect to the paired barriers.
-		 *
-		 * This barrier (B) ensures that the contents of the read-side
-		 * critical section does not leak after the "end" counter
-		 * increment. It pairs with memory barriers (D) and (E).
-		 *
-		 * This barrier (B) also ensures that the "begin" increment is
-		 * before the "end" increment. It pairs with memory barrier (C).
-		 * It is redundant with barrier (A) for that purpose.
+		 * This barrier (F) is paired with membarrier()
+		 * at (G). It orders increment of the begin/end
+		 * counters before load/store to the futex.
 		 */
 		rseq_barrier();
-		if (side_likely(!rseq_addv((intptr_t *)&begin_cpu_count->rseq_end, 1, cpu))) {
-			/*
-			 * This barrier (F) is paired with membarrier()
-			 * at (G). It orders increment of the begin/end
-			 * counters before load/store to the futex.
-			 */
-			rseq_barrier();
-			goto end;
-		}
+		goto end;
 	}
 	/* Fallback to atomic increment and SEQ_CST. */
 	(void) __atomic_add_fetch(&begin_cpu_count->end, 1, __ATOMIC_SEQ_CST);
 	/*
-	 * This barrier (F) is paired with SEQ_CST barrier or
-	 * membarrier() at (G). It orders increment of the begin/end
-	 * counters before load/store to the futex.
+	 * This barrier (F) implied by SEQ_CST is paired with SEQ_CST
+	 * barrier or membarrier() at (G). It orders increment of the
+	 * begin/end counters before load/store to the futex.
 	 */
-	__atomic_thread_fence(__ATOMIC_SEQ_CST);
 end:
 	side_rcu_wake_up_gp(gp_state);
 }
@@ -158,7 +158,7 @@ end:
 		(_____side_v); \
 	})
 
-#define side_rcu_assign_pointer(p, v)	__atomic_store_n(&(p), v, __ATOMIC_RELEASE); \
+#define side_rcu_assign_pointer(p, v)	__atomic_store_n(&(p), v, __ATOMIC_RELEASE);
 
 void side_rcu_wait_grace_period(struct side_rcu_gp_state *gp_state) __attribute__((visibility("hidden")));
 void side_rcu_gp_init(struct side_rcu_gp_state *rcu_gp) __attribute__((visibility("hidden")));