1 // SPDX-License-Identifier: MIT
3 * Copyright 2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
14 #include <sys/syscall.h>
15 #include <linux/membarrier.h>
21 membarrier(int cmd
, unsigned int flags
, int cpu_id
)
23 return syscall(__NR_membarrier
, cmd
, flags
, cpu_id
);
26 /* active_readers is an input/output parameter. */
28 void check_active_readers(struct side_rcu_gp_state
*gp_state
, bool *active_readers
)
30 uintptr_t sum
[2] = { 0, 0 }; /* begin - end */
33 for (i
= 0; i
< gp_state
->nr_cpus
; i
++) {
34 struct side_rcu_cpu_gp_state
*cpu_state
= &gp_state
->percpu_state
[i
];
36 if (active_readers
[0]) {
37 sum
[0] -= __atomic_load_n(&cpu_state
->count
[0].end
, __ATOMIC_RELAXED
);
38 sum
[0] -= __atomic_load_n(&cpu_state
->count
[0].rseq_end
, __ATOMIC_RELAXED
);
40 if (active_readers
[1]) {
41 sum
[1] -= __atomic_load_n(&cpu_state
->count
[1].end
, __ATOMIC_RELAXED
);
42 sum
[1] -= __atomic_load_n(&cpu_state
->count
[1].rseq_end
, __ATOMIC_RELAXED
);
47 * This memory barrier (C) pairs with either of memory barriers
48 * (A) or (B) (one is sufficient).
50 * Read end counts before begin counts. Reading "end" before
51 * "begin" counts ensures we never see an "end" without having
52 * seen its associated "begin", because "begin" is always
53 * incremented before "end", as guaranteed by memory barriers
56 if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED
, 0, 0))
59 for (i
= 0; i
< gp_state
->nr_cpus
; i
++) {
60 struct side_rcu_cpu_gp_state
*cpu_state
= &gp_state
->percpu_state
[i
];
62 if (active_readers
[0]) {
63 sum
[0] += __atomic_load_n(&cpu_state
->count
[0].begin
, __ATOMIC_RELAXED
);
64 sum
[0] += __atomic_load_n(&cpu_state
->count
[0].rseq_begin
, __ATOMIC_RELAXED
);
66 if (active_readers
[1]) {
67 sum
[1] += __atomic_load_n(&cpu_state
->count
[1].begin
, __ATOMIC_RELAXED
);
68 sum
[1] += __atomic_load_n(&cpu_state
->count
[1].rseq_begin
, __ATOMIC_RELAXED
);
71 if (active_readers
[0])
72 active_readers
[0] = sum
[0];
73 if (active_readers
[1])
74 active_readers
[1] = sum
[1];
78 * Wait for previous period to have no active readers.
80 * active_readers is an input/output parameter.
83 void wait_for_prev_period_readers(struct side_rcu_gp_state
*gp_state
, bool *active_readers
)
85 unsigned int prev_period
= gp_state
->period
^ 1;
88 * If a prior active readers scan already observed that no
89 * readers are present for the previous period, there is no need
92 if (!active_readers
[prev_period
])
95 * Wait for the sum of CPU begin/end counts to match for the
99 check_active_readers(gp_state
, active_readers
);
100 if (!active_readers
[prev_period
])
102 /* Retry after 10ms. */
108 * The grace period completes when it observes that there are no active
109 * readers within each of the periods.
111 * The active_readers state is initially true for each period, until the
112 * grace period observes that no readers are present for each given
113 * period, at which point the active_readers state becomes false.
115 void side_rcu_wait_grace_period(struct side_rcu_gp_state
*gp_state
)
117 bool active_readers
[2] = { true, true };
120 * This memory barrier (D) pairs with memory barriers (A) and
121 * (B) on the read-side.
123 * It orders prior loads and stores before the "end"/"begin"
124 * reader state loads. In other words, it orders prior loads and
125 * stores before observation of active readers quiescence,
126 * effectively ensuring that read-side critical sections which
127 * exist after the grace period completes are ordered after
128 * loads and stores performed before the grace period.
130 if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED
, 0, 0))
134 * First scan through all cpus, for both period. If no readers
135 * are accounted for, we have observed quiescence and can
136 * complete the grace period immediately.
138 check_active_readers(gp_state
, active_readers
);
139 if (!active_readers
[0] && !active_readers
[1])
142 pthread_mutex_lock(&gp_state
->gp_lock
);
144 wait_for_prev_period_readers(gp_state
, active_readers
);
146 * If the reader scan detected that there are no readers in the
147 * current period as well, we can complete the grace period
150 if (!active_readers
[gp_state
->period
])
153 /* Flip period: 0 -> 1, 1 -> 0. */
154 (void) __atomic_xor_fetch(&gp_state
->period
, 1, __ATOMIC_RELAXED
);
156 wait_for_prev_period_readers(gp_state
, active_readers
);
158 pthread_mutex_unlock(&gp_state
->gp_lock
);
161 * This memory barrier (E) pairs with memory barriers (A) and
162 * (B) on the read-side.
164 * It orders the "end"/"begin" reader state loads before
165 * following loads and stores. In other words, it orders
166 * observation of active readers quiescence before following
167 * loads and stores, effectively ensuring that read-side
168 * critical sections which existed prior to the grace period
169 * are ordered before loads and stores performed after the grace
172 if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED
, 0, 0))
176 void side_rcu_gp_init(struct side_rcu_gp_state
*rcu_gp
)
178 memset(rcu_gp
, 0, sizeof(*rcu_gp
));
179 rcu_gp
->nr_cpus
= get_possible_cpus_array_len();
180 if (!rcu_gp
->nr_cpus
)
182 pthread_mutex_init(&rcu_gp
->gp_lock
, NULL
);
183 rcu_gp
->percpu_state
= calloc(rcu_gp
->nr_cpus
, sizeof(struct side_rcu_cpu_gp_state
));
184 if (!rcu_gp
->percpu_state
)
186 if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED
, 0, 0))
190 void side_rcu_gp_exit(struct side_rcu_gp_state
*rcu_gp
)
192 rseq_prepare_unload();
193 pthread_mutex_destroy(&rcu_gp
->gp_lock
);
194 free(rcu_gp
->percpu_state
);