src/rcu.h

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright 2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   4  */
   5
   6 #ifndef _SIDE_RCU_H
   7 #define _SIDE_RCU_H
   8
   9 #include <sched.h>
  10 #include <stdint.h>
  11 #include <pthread.h>
  12 #include <stdbool.h>
  13 #include <poll.h>
  14 #include <side/trace.h>
  15 #include <rseq/rseq.h>
  16
  17 #define SIDE_CACHE_LINE_SIZE            256
  18
  19 struct side_rcu_percpu_count {
  20         uintptr_t begin;
  21         uintptr_t rseq_begin;
  22         uintptr_t end;
  23         uintptr_t rseq_end;
  24 }  __attribute__((__aligned__(SIDE_CACHE_LINE_SIZE)));
  25
  26 struct side_rcu_cpu_gp_state {
  27         struct side_rcu_percpu_count count[2];
  28 };
  29
  30 struct side_rcu_gp_state {
  31         struct side_rcu_cpu_gp_state *percpu_state;
  32         int nr_cpus;
  33         unsigned int period;
  34         pthread_mutex_t gp_lock;
  35 };
  36
  37 extern unsigned int side_rcu_rseq_membarrier_available __attribute__((visibility("hidden")));
  38
  39 //TODO: implement wait/wakeup for grace period using sys_futex
  40 static inline
  41 unsigned int side_rcu_read_begin(struct side_rcu_gp_state *gp_state)
  42 {
  43         unsigned int period = __atomic_load_n(&gp_state->period, __ATOMIC_RELAXED);
  44         struct side_rcu_cpu_gp_state *cpu_gp_state;
  45         int cpu;
  46
  47         if (side_likely(side_rcu_rseq_membarrier_available)) {
  48                 cpu = rseq_cpu_start();
  49                 cpu_gp_state = &gp_state->percpu_state[cpu];
  50                 if (side_likely(!rseq_addv((intptr_t *)&cpu_gp_state->count[period].rseq_begin, 1, cpu))) {
  51                         /*
  52                          * This compiler barrier (A) is paired with membarrier() at (C),
  53                          * (D), (E). It effectively upgrades this compiler barrier to a
  54                          * SEQ_CST fence with respect to the paired barriers.
  55                          *
  56                          * This barrier (A) ensures that the contents of the read-side
  57                          * critical section does not leak before the "begin" counter
  58                          * increment. It pairs with memory barriers (D) and (E).
  59                          *
  60                          * This barrier (A) also ensures that the "begin" increment is
  61                          * before the "end" increment. It pairs with memory barrier (C).
  62                          * It is redundant with barrier (B) for that purpose.
  63                          */
  64                         rseq_barrier();
  65                         return period;
  66                 }
  67         }
  68         /* Fallback to atomic increment and SEQ_CST. */
  69         cpu = sched_getcpu();
  70         if (side_unlikely(cpu < 0))
  71                 cpu = 0;
  72         cpu_gp_state = &gp_state->percpu_state[cpu];
  73         (void) __atomic_add_fetch(&cpu_gp_state->count[period].begin, 1, __ATOMIC_SEQ_CST);
  74         return period;
  75 }
  76
  77 static inline
  78 void side_rcu_read_end(struct side_rcu_gp_state *gp_state, unsigned int period)
  79 {
  80         struct side_rcu_cpu_gp_state *cpu_gp_state;
  81         int cpu;
  82
  83         if (side_likely(side_rcu_rseq_membarrier_available)) {
  84                 /*
  85                  * This compiler barrier (B) is paired with membarrier() at (C),
  86                  * (D), (E). It effectively upgrades this compiler barrier to a
  87                  * SEQ_CST fence with respect to the paired barriers.
  88                  *
  89                  * This barrier (B) ensures that the contents of the read-side
  90                  * critical section does not leak after the "end" counter
  91                  * increment. It pairs with memory barriers (D) and (E).
  92                  *
  93                  * This barrier (B) also ensures that the "begin" increment is
  94                  * before the "end" increment. It pairs with memory barrier (C).
  95                  * It is redundant with barrier (A) for that purpose.
  96                  */
  97                 rseq_barrier();
  98                 cpu = rseq_cpu_start();
  99                 cpu_gp_state = &gp_state->percpu_state[cpu];
 100                 if (side_likely(!rseq_addv((intptr_t *)&cpu_gp_state->count[period].rseq_end, 1, cpu)))
 101                         return;
 102         }
 103         /* Fallback to atomic increment and SEQ_CST. */
 104         cpu = sched_getcpu();
 105         if (side_unlikely(cpu < 0))
 106                 cpu = 0;
 107         cpu_gp_state = &gp_state->percpu_state[cpu];
 108         (void) __atomic_add_fetch(&cpu_gp_state->count[period].end, 1, __ATOMIC_SEQ_CST);
 109
 110
 111 }
 112
 113 #define side_rcu_dereference(p) \
 114         __extension__ \
 115         ({ \
 116                 __typeof__(p) _____side_v = __atomic_load_n(&(p), __ATOMIC_CONSUME); \
 117                 (_____side_v); \
 118         })
 119
 120 #define side_rcu_assign_pointer(p, v)   __atomic_store_n(&(p), v, __ATOMIC_RELEASE); \
 121
 122 void side_rcu_wait_grace_period(struct side_rcu_gp_state *gp_state) __attribute__((visibility("hidden")));
 123 void side_rcu_gp_init(struct side_rcu_gp_state *rcu_gp) __attribute__((visibility("hidden")));
 124 void side_rcu_gp_exit(struct side_rcu_gp_state *rcu_gp) __attribute__((visibility("hidden")));
 125
 126 #endif /* _SIDE_RCU_H */