Introduce rb_replace_node_rcu()

Implement an RCU-safe variant of rb_replace_node() and rearrange
rb_replace_node() to do things in the same order.

Signed-off-by: David Howells <dhowells@redhat.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>

diff --git a/include/linux/rbtree.h b/include/linux/rbtree.h
index b6900099ea81c9445dfaec79ca07573726034787..e585018498d594f7c12abe6d3935f321009b3cd8 100644 (file)
--- a/include/linux/rbtree.h
+++ b/include/linux/rbtree.h
@@ -76,6 +76,8 @@ extern struct rb_node *rb_next_postorder(const struct rb_node *);
 /* Fast replacement of a single node without remove/rebalance/add/rebalance */
 extern void rb_replace_node(struct rb_node *victim, struct rb_node *new,
                            struct rb_root *root);
+extern void rb_replace_node_rcu(struct rb_node *victim, struct rb_node *new,
+                               struct rb_root *root);
 
 static inline void rb_link_node(struct rb_node *node, struct rb_node *parent,
                                struct rb_node **rb_link)

diff --git a/include/linux/rbtree_augmented.h b/include/linux/rbtree_augmented.h
index 14d7b831b63a8377dce8161f3bf0be55bf385258..d076183e49bec0b0c843f8b067fbd3d25cc4bce0 100644 (file)
--- a/include/linux/rbtree_augmented.h
+++ b/include/linux/rbtree_augmented.h
@@ -130,6 +130,19 @@ __rb_change_child(struct rb_node *old, struct rb_node *new,
                WRITE_ONCE(root->rb_node, new);
 }
 
+static inline void
+__rb_change_child_rcu(struct rb_node *old, struct rb_node *new,
+                     struct rb_node *parent, struct rb_root *root)
+{
+       if (parent) {
+               if (parent->rb_left == old)
+                       rcu_assign_pointer(parent->rb_left, new);
+               else
+                       rcu_assign_pointer(parent->rb_right, new);
+       } else
+               rcu_assign_pointer(root->rb_node, new);
+}
+
 extern void __rb_erase_color(struct rb_node *parent, struct rb_root *root,
        void (*augment_rotate)(struct rb_node *old, struct rb_node *new));
 

diff --git a/lib/rbtree.c b/lib/rbtree.c
index 1356454e36de9f1c083b2c84f297ba4878498c0a..eb8a19fee11003adbb5ca62fd6eefa8033d6dbcd 100644 (file)
--- a/lib/rbtree.c
+++ b/lib/rbtree.c
@@ -539,17 +539,39 @@ void rb_replace_node(struct rb_node *victim, struct rb_node *new,
 {
        struct rb_node *parent = rb_parent(victim);
 
+       /* Copy the pointers/colour from the victim to the replacement */
+       *new = *victim;
+
        /* Set the surrounding nodes to point to the replacement */
-       __rb_change_child(victim, new, parent, root);
        if (victim->rb_left)
                rb_set_parent(victim->rb_left, new);
        if (victim->rb_right)
                rb_set_parent(victim->rb_right, new);
+       __rb_change_child(victim, new, parent, root);
+}
+EXPORT_SYMBOL(rb_replace_node);
+
+void rb_replace_node_rcu(struct rb_node *victim, struct rb_node *new,
+                        struct rb_root *root)
+{
+       struct rb_node *parent = rb_parent(victim);
 
        /* Copy the pointers/colour from the victim to the replacement */
        *new = *victim;
+
+       /* Set the surrounding nodes to point to the replacement */
+       if (victim->rb_left)
+               rb_set_parent(victim->rb_left, new);
+       if (victim->rb_right)
+               rb_set_parent(victim->rb_right, new);
+
+       /* Set the parent's pointer to the new node last after an RCU barrier
+        * so that the pointers onwards are seen to be set correctly when doing
+        * an RCU walk over the tree.
+        */
+       __rb_change_child_rcu(victim, new, parent, root);
 }
-EXPORT_SYMBOL(rb_replace_node);
+EXPORT_SYMBOL(rb_replace_node_rcu);
 
 static struct rb_node *rb_left_deepest_node(const struct rb_node *node)
 {