rculist: Use WRITE_ONCE() when deleting from reader-visible list

The various RCU list-deletion macros (list_del_rcu(),
hlist_del_init_rcu(), hlist_del_rcu(), hlist_bl_del_init_rcu(),
hlist_bl_del_rcu(), hlist_nulls_del_init_rcu(), and hlist_nulls_del_rcu())
do plain stores into the ->next pointer of the preceding list elemment.
Unfortunately, the compiler is within its rights to (for example) use
byte-at-a-time writes to update the pointer, which would fatally confuse
concurrent readers.  This patch therefore adds the needed WRITE_ONCE()
macros.

KernelThreadSanitizer (KTSAN) reported the __hlist_del() issue, which
is a problem when __hlist_del() is invoked by hlist_del_rcu().

Reported-by: Dmitry Vyukov <dvyukov@google.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>

diff --git a/include/linux/list.h b/include/linux/list.h
index 3e3e64a6100241b328a82f39f77ab403583637c6..993395a2e55c5483c890b40e216193d95c85bd64 100644 (file)
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -87,7 +87,7 @@ static inline void list_add_tail(struct list_head *new, struct list_head *head)
 static inline void __list_del(struct list_head * prev, struct list_head * next)
 {
        next->prev = prev;
-       prev->next = next;
+       WRITE_ONCE(prev->next, next);
 }
 
 /**
@@ -615,7 +615,8 @@ static inline void __hlist_del(struct hlist_node *n)
 {
        struct hlist_node *next = n->next;
        struct hlist_node **pprev = n->pprev;
-       *pprev = next;
+
+       WRITE_ONCE(*pprev, next);
        if (next)
                next->pprev = pprev;
 }

diff --git a/include/linux/list_bl.h b/include/linux/list_bl.h
index 2eb88556c5c5b0ee0cb0fc664e9d13dc852fea6e..8132214e8efd2930ff752fcc4c37da7d23b9643f 100644 (file)
--- a/include/linux/list_bl.h
+++ b/include/linux/list_bl.h
@@ -93,9 +93,10 @@ static inline void __hlist_bl_del(struct hlist_bl_node *n)
        LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK);
 
        /* pprev may be `first`, so be careful not to lose the lock bit */
-       *pprev = (struct hlist_bl_node *)
+       WRITE_ONCE(*pprev,
+                  (struct hlist_bl_node *)
                        ((unsigned long)next |
-                        ((unsigned long)*pprev & LIST_BL_LOCKMASK));
+                        ((unsigned long)*pprev & LIST_BL_LOCKMASK)));
        if (next)
                next->pprev = pprev;
 }

diff --git a/include/linux/list_nulls.h b/include/linux/list_nulls.h
index f266661d2666596d808bbe8756c91239f5b8e6dc..444d2b1313bda37647b1660e4582202a7e3b66e4 100644 (file)
--- a/include/linux/list_nulls.h
+++ b/include/linux/list_nulls.h
@@ -76,7 +76,8 @@ static inline void __hlist_nulls_del(struct hlist_nulls_node *n)
 {
        struct hlist_nulls_node *next = n->next;
        struct hlist_nulls_node **pprev = n->pprev;
-       *pprev = next;
+
+       WRITE_ONCE(*pprev, next);
        if (!is_a_nulls(next))
                next->pprev = pprev;
 }