rcu: Associate quiescent-state reports with grace period

[deliverable/linux.git] / kernel / rcu / tree.c

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index b3684b2846770321420de82292e38fb51c0195e1..8fcc64ed858c1e7548d3b3ddc759ac5f1a9f3c91 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2132,25 +2132,32 @@ static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
  * Similar to rcu_report_qs_rdp(), for which it is a helper function.
  * Allows quiescent states for a group of CPUs to be reported at one go
  * to the specified rcu_node structure, though all the CPUs in the group
- * must be represented by the same rcu_node structure (which need not be
- * a leaf rcu_node structure, though it often will be).  That structure's
- * lock must be held upon entry, and it is released before return.
+ * must be represented by the same rcu_node structure (which need not be a
+ * leaf rcu_node structure, though it often will be).  The gps parameter
+ * is the grace-period snapshot, which means that the quiescent states
+ * are valid only if rnp->gpnum is equal to gps.  That structure's lock
+ * must be held upon entry, and it is released before return.
  */
 static void
 rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
-                 struct rcu_node *rnp, unsigned long flags)
+                 struct rcu_node *rnp, unsigned long gps, unsigned long flags)
        __releases(rnp->lock)
 {
+       unsigned long oldmask = 0;
        struct rcu_node *rnp_c;
 
        /* Walk up the rcu_node hierarchy. */
        for (;;) {
-               if (!(rnp->qsmask & mask)) {
+               if (!(rnp->qsmask & mask) || rnp->gpnum != gps) {
 
-                       /* Our bit has already been cleared, so done. */
+                       /*
+                        * Our bit has already been cleared, or the
+                        * relevant grace period is already over, so done.
+                        */
                        raw_spin_unlock_irqrestore(&rnp->lock, flags);
                        return;
                }
+               WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
                rnp->qsmask &= ~mask;
                trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum,
                                                 mask, rnp->qsmask, rnp->level,
@@ -2174,7 +2181,7 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
                rnp = rnp->parent;
                raw_spin_lock_irqsave(&rnp->lock, flags);
                smp_mb__after_unlock_lock();
-               WARN_ON_ONCE(rnp_c->qsmask);
+               oldmask = rnp_c->qsmask;
        }
 
        /*
@@ -2196,6 +2203,7 @@ static void rcu_report_unblock_qs_rnp(struct rcu_state *rsp,
                                      struct rcu_node *rnp, unsigned long flags)
        __releases(rnp->lock)
 {
+       unsigned long gps;
        unsigned long mask;
        struct rcu_node *rnp_p;
 
@@ -2215,12 +2223,13 @@ static void rcu_report_unblock_qs_rnp(struct rcu_state *rsp,
                return;
        }
 
-       /* Report up the rest of the hierarchy. */
+       /* Report up the rest of the hierarchy, tracking current ->gpnum. */
+       gps = rnp->gpnum;
        mask = rnp->grpmask;
        raw_spin_unlock(&rnp->lock);    /* irqs remain disabled. */
        raw_spin_lock(&rnp_p->lock);    /* irqs already disabled. */
        smp_mb__after_unlock_lock();
-       rcu_report_qs_rnp(mask, rsp, rnp_p, flags);
+       rcu_report_qs_rnp(mask, rsp, rnp_p, gps, flags);
 }
 
 /*
@@ -2271,7 +2280,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
                 */
                needwake = rcu_accelerate_cbs(rsp, rnp, rdp);
 
-               rcu_report_qs_rnp(mask, rsp, rnp, flags); /* rlses rnp->lock */
+               rcu_report_qs_rnp(mask, rsp, rnp, rnp->gpnum, flags);
+               /* ^^^ Released rnp->lock */
                if (needwake)
                        rcu_gp_kthread_wake(rsp);
        }
@@ -2747,8 +2757,8 @@ static void force_qs_rnp(struct rcu_state *rsp,
                        }
                }
                if (mask != 0) {
-                       /* Idle/offline CPUs, report. */
-                       rcu_report_qs_rnp(mask, rsp, rnp, flags);
+                       /* Idle/offline CPUs, report (releases rnp->lock. */
+                       rcu_report_qs_rnp(mask, rsp, rnp, rnp->gpnum, flags);
                } else {
                        /* Nothing to do here, so just drop the lock. */
                        raw_spin_unlock_irqrestore(&rnp->lock, flags);