X-Git-Url: http://drtracing.org/?a=blobdiff_plain;f=src%2Frcu.h;h=4db3500566abc002042d9f63a6b617d4c24d76de;hb=f8188f94f6e17919ca75063e4d43faca0fd03af8;hp=37eaaec199a301aabe628fb65359adedfe47e10f;hpb=c3a2bccf45edaa6b5ffc8366bd8d3274a9a7ac4d;p=libside.git

diff --git a/src/rcu.h b/src/rcu.h
index 37eaaec..4db3500 100644
--- a/src/rcu.h
+++ b/src/rcu.h
@@ -25,11 +25,11 @@ struct side_rcu_percpu_count {
 	uintptr_t rseq_begin;
 	uintptr_t end;
 	uintptr_t rseq_end;
-}  __attribute__((__aligned__(SIDE_CACHE_LINE_SIZE)));
+};
 
 struct side_rcu_cpu_gp_state {
 	struct side_rcu_percpu_count count[2];
-};
+} __attribute__((__aligned__(SIDE_CACHE_LINE_SIZE)));
 
 struct side_rcu_gp_state {
 	struct side_rcu_cpu_gp_state *percpu_state;
@@ -39,6 +39,11 @@ struct side_rcu_gp_state {
 	pthread_mutex_t gp_lock;
 };
 
+struct side_rcu_read_state {
+	struct side_rcu_percpu_count *percpu_count;
+	int cpu;
+};
+
 extern unsigned int side_rcu_rseq_membarrier_available __attribute__((visibility("hidden")));
 
 static inline
@@ -63,87 +68,85 @@ void side_rcu_wake_up_gp(struct side_rcu_gp_state *gp_state)
 }
 
 static inline
-unsigned int side_rcu_read_begin(struct side_rcu_gp_state *gp_state)
+void side_rcu_read_begin(struct side_rcu_gp_state *gp_state, struct side_rcu_read_state *read_state)
 {
-	unsigned int period = __atomic_load_n(&gp_state->period, __ATOMIC_RELAXED);
+	struct side_rcu_percpu_count *begin_cpu_count;
 	struct side_rcu_cpu_gp_state *cpu_gp_state;
+	unsigned int period;
 	int cpu;
 
-	if (side_likely(side_rcu_rseq_membarrier_available)) {
-		cpu = rseq_cpu_start();
-		cpu_gp_state = &gp_state->percpu_state[cpu];
-		if (side_likely(!rseq_addv((intptr_t *)&cpu_gp_state->count[period].rseq_begin, 1, cpu))) {
-			/*
-			 * This compiler barrier (A) is paired with membarrier() at (C),
-			 * (D), (E). It effectively upgrades this compiler barrier to a
-			 * SEQ_CST fence with respect to the paired barriers.
-			 *
-			 * This barrier (A) ensures that the contents of the read-side
-			 * critical section does not leak before the "begin" counter
-			 * increment. It pairs with memory barriers (D) and (E).
-			 *
-			 * This barrier (A) also ensures that the "begin" increment is
-			 * before the "end" increment. It pairs with memory barrier (C).
-			 * It is redundant with barrier (B) for that purpose.
-			 */
-			rseq_barrier();
-			return period;
-		}
-	}
-	/* Fallback to atomic increment and SEQ_CST. */
-	cpu = sched_getcpu();
-	if (side_unlikely(cpu < 0))
-		cpu = 0;
+	cpu = rseq_cpu_start();
+	period = __atomic_load_n(&gp_state->period, __ATOMIC_RELAXED);
 	cpu_gp_state = &gp_state->percpu_state[cpu];
-	(void) __atomic_add_fetch(&cpu_gp_state->count[period].begin, 1, __ATOMIC_SEQ_CST);
-	return period;
-}
-
-static inline
-void side_rcu_read_end(struct side_rcu_gp_state *gp_state, unsigned int period)
-{
-	struct side_rcu_cpu_gp_state *cpu_gp_state;
-	int cpu;
-
-	if (side_likely(side_rcu_rseq_membarrier_available)) {
+	read_state->percpu_count = begin_cpu_count = &cpu_gp_state->count[period];
+	read_state->cpu = cpu;
+	if (side_likely(side_rcu_rseq_membarrier_available &&
+			!rseq_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU_CPU_ID,
+				   (intptr_t *)&begin_cpu_count->rseq_begin, 1, cpu))) {
 		/*
-		 * This compiler barrier (B) is paired with membarrier() at (C),
+		 * This compiler barrier (A) is paired with membarrier() at (C),
 		 * (D), (E). It effectively upgrades this compiler barrier to a
 		 * SEQ_CST fence with respect to the paired barriers.
 		 *
-		 * This barrier (B) ensures that the contents of the read-side
-		 * critical section does not leak after the "end" counter
+		 * This barrier (A) ensures that the contents of the read-side
+		 * critical section does not leak before the "begin" counter
 		 * increment. It pairs with memory barriers (D) and (E).
 		 *
-		 * This barrier (B) also ensures that the "begin" increment is
+		 * This barrier (A) also ensures that the "begin" increment is
 		 * before the "end" increment. It pairs with memory barrier (C).
-		 * It is redundant with barrier (A) for that purpose.
+		 * It is redundant with barrier (B) for that purpose.
 		 */
 		rseq_barrier();
-		cpu = rseq_cpu_start();
-		cpu_gp_state = &gp_state->percpu_state[cpu];
-		if (side_likely(!rseq_addv((intptr_t *)&cpu_gp_state->count[period].rseq_end, 1, cpu))) {
-			/*
-			 * This barrier (F) is paired with membarrier()
-			 * at (G). It orders increment of the begin/end
-			 * counters before load/store to the futex.
-			 */
-			rseq_barrier();
-			goto end;
-		}
+		return;
 	}
 	/* Fallback to atomic increment and SEQ_CST. */
 	cpu = sched_getcpu();
 	if (side_unlikely(cpu < 0))
 		cpu = 0;
+	read_state->cpu = cpu;
 	cpu_gp_state = &gp_state->percpu_state[cpu];
-	(void) __atomic_add_fetch(&cpu_gp_state->count[period].end, 1, __ATOMIC_SEQ_CST);
+	read_state->percpu_count = begin_cpu_count = &cpu_gp_state->count[period];
+	(void) __atomic_add_fetch(&begin_cpu_count->begin, 1, __ATOMIC_SEQ_CST);
+}
+
+static inline
+void side_rcu_read_end(struct side_rcu_gp_state *gp_state, struct side_rcu_read_state *read_state)
+{
+	struct side_rcu_percpu_count *begin_cpu_count = read_state->percpu_count;
+	int cpu = read_state->cpu;
+
+	/*
+	 * This compiler barrier (B) is paired with membarrier() at (C),
+	 * (D), (E). It effectively upgrades this compiler barrier to a
+	 * SEQ_CST fence with respect to the paired barriers.
+	 *
+	 * This barrier (B) ensures that the contents of the read-side
+	 * critical section does not leak after the "end" counter
+	 * increment. It pairs with memory barriers (D) and (E).
+	 *
+	 * This barrier (B) also ensures that the "begin" increment is
+	 * before the "end" increment. It pairs with memory barrier (C).
+	 * It is redundant with barrier (A) for that purpose.
+	 */
+	rseq_barrier();
+	if (side_likely(side_rcu_rseq_membarrier_available &&
+			!rseq_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU_CPU_ID,
+				   (intptr_t *)&begin_cpu_count->rseq_end, 1, cpu))) {
+		/*
+		 * This barrier (F) is paired with membarrier()
+		 * at (G). It orders increment of the begin/end
+		 * counters before load/store to the futex.
+		 */
+		rseq_barrier();
+		goto end;
+	}
+	/* Fallback to atomic increment and SEQ_CST. */
+	(void) __atomic_add_fetch(&begin_cpu_count->end, 1, __ATOMIC_SEQ_CST);
 	/*
-	 * This barrier (F) is paired with SEQ_CST barrier or
-	 * membarrier() at (G). It orders increment of the begin/end
-	 * counters before load/store to the futex.
+	 * This barrier (F) implied by SEQ_CST is paired with SEQ_CST
+	 * barrier or membarrier() at (G). It orders increment of the
+	 * begin/end counters before load/store to the futex.
 	 */
-	__atomic_thread_fence(__ATOMIC_SEQ_CST);
 end:
 	side_rcu_wake_up_gp(gp_state);
 }
@@ -155,7 +158,7 @@ end:
 		(_____side_v); \
 	})
 
-#define side_rcu_assign_pointer(p, v)	__atomic_store_n(&(p), v, __ATOMIC_RELEASE); \
+#define side_rcu_assign_pointer(p, v)	__atomic_store_n(&(p), v, __ATOMIC_RELEASE);
 
 void side_rcu_wait_grace_period(struct side_rcu_gp_state *gp_state) __attribute__((visibility("hidden")));
 void side_rcu_gp_init(struct side_rcu_gp_state *rcu_gp) __attribute__((visibility("hidden")));