rcu: Add rcu_normal kernel parameter to suppress expediting

Although expedited grace periods can be quite useful, and although their
OS jitter has been greatly reduced, they can still pose problems for
extreme real-time workloads.  This commit therefore adds a rcu_normal
kernel boot parameter (which can also be manipulated via sysfs)
to suppress expedited grace periods, that is, to treat requests for
expedited grace periods as if they were requests for normal grace periods.
If both rcu_expedited and rcu_normal are specified, rcu_normal wins.
This means that if you are relying on expedited grace periods to speed up
boot, you will want to specify rcu_expedited on the kernel command line,
and then specify rcu_normal via sysfs once boot completes.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt
index 742f69d18fc8989ae28d9c0662d6bf334109dddd..7673943d30850c2e70f4ebd27596df8a2c8388be 100644 (file)
--- a/Documentation/kernel-parameters.txt
+++ b/Documentation/kernel-parameters.txt
@@ -3296,6 +3296,12 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
        rcutorture.verbose= [KNL]
                        Enable additional printk() statements.
 
+       rcupdate.rcu_cpu_stall_suppress= [KNL]
+                       Suppress RCU CPU stall warning messages.
+
+       rcupdate.rcu_cpu_stall_timeout= [KNL]
+                       Set timeout for RCU CPU stall warning messages.
+
        rcupdate.rcu_expedited= [KNL]
                        Use expedited grace-period primitives, for
                        example, synchronize_rcu_expedited() instead
@@ -3303,11 +3309,14 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
                        but can increase CPU utilization, degrade
                        real-time latency, and degrade energy efficiency.
 
-       rcupdate.rcu_cpu_stall_suppress= [KNL]
-                       Suppress RCU CPU stall warning messages.
-
-       rcupdate.rcu_cpu_stall_timeout= [KNL]
-                       Set timeout for RCU CPU stall warning messages.
+       rcupdate.rcu_normal= [KNL]
+                       Use only normal grace-period primitives,
+                       for example, synchronize_rcu() instead of
+                       synchronize_rcu_expedited().  This improves
+                       real-time latency, CPU utilization, and energy
+                       efficiency, but can expose users to increased
+                       grace-period latency.  This parameter overrides
+                       rcupdate.rcu_expedited.
 
        rcupdate.rcu_task_stall_timeout= [KNL]
                        Set timeout in jiffies for RCU task stall warning

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index a0189ba67fde721a824cc7c8db4b1f015197059e..98d9f30c02d43ab41154b110795d8bfb8449fd3b 100644 (file)
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -49,9 +49,14 @@
 #include <asm/barrier.h>
 
 extern int rcu_expedited; /* for sysctl */
+extern int rcu_normal;    /* also for sysctl */
 
 #ifdef CONFIG_TINY_RCU
 /* Tiny RCU doesn't expedite, as its purpose in life is instead to be tiny. */
+static inline bool rcu_gp_is_normal(void)  /* Internal RCU use. */
+{
+       return true;
+}
 static inline bool rcu_gp_is_expedited(void)  /* Internal RCU use. */
 {
        return false;
@@ -65,6 +70,7 @@ static inline void rcu_unexpedite_gp(void)
 {
 }
 #else /* #ifdef CONFIG_TINY_RCU */
+bool rcu_gp_is_normal(void);     /* Internal RCU use. */
 bool rcu_gp_is_expedited(void);  /* Internal RCU use. */
 void rcu_expedite_gp(void);
 void rcu_unexpedite_gp(void);

diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c
index e83b264640615c47c31cce539f31014dc11b0776..b4e2fa52d8bc4feb4feaace94da14aae0b1ed23a 100644 (file)
--- a/kernel/ksysfs.c
+++ b/kernel/ksysfs.c
@@ -20,7 +20,7 @@
 #include <linux/capability.h>
 #include <linux/compiler.h>
 
-#include <linux/rcupdate.h>    /* rcu_expedited */
+#include <linux/rcupdate.h>    /* rcu_expedited and rcu_normal */
 
 #define KERNEL_ATTR_RO(_name) \
 static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
@@ -148,7 +148,7 @@ int rcu_expedited;
 static ssize_t rcu_expedited_show(struct kobject *kobj,
                                  struct kobj_attribute *attr, char *buf)
 {
-       return sprintf(buf, "%d\n", rcu_expedited);
+       return sprintf(buf, "%d\n", READ_ONCE(rcu_expedited));
 }
 static ssize_t rcu_expedited_store(struct kobject *kobj,
                                   struct kobj_attribute *attr,
@@ -161,6 +161,23 @@ static ssize_t rcu_expedited_store(struct kobject *kobj,
 }
 KERNEL_ATTR_RW(rcu_expedited);
 
+int rcu_normal;
+static ssize_t rcu_normal_show(struct kobject *kobj,
+                              struct kobj_attribute *attr, char *buf)
+{
+       return sprintf(buf, "%d\n", READ_ONCE(rcu_normal));
+}
+static ssize_t rcu_normal_store(struct kobject *kobj,
+                               struct kobj_attribute *attr,
+                               const char *buf, size_t count)
+{
+       if (kstrtoint(buf, 0, &rcu_normal))
+               return -EINVAL;
+
+       return count;
+}
+KERNEL_ATTR_RW(rcu_normal);
+
 /*
  * Make /sys/kernel/notes give the raw contents of our kernel .notes section.
  */
@@ -203,6 +220,7 @@ static struct attribute * kernel_attrs[] = {
        &vmcoreinfo_attr.attr,
 #endif
        &rcu_expedited_attr.attr,
+       &rcu_normal_attr.attr,
        NULL
 };
 

diff --git a/kernel/rcu/srcu.c b/kernel/rcu/srcu.c
index a63a1ea5a41bf450f6b162674149a5928fca3530..9b9cdd549caa848111247b35b19109da7af9099d 100644 (file)
--- a/kernel/rcu/srcu.c
+++ b/kernel/rcu/srcu.c
@@ -489,7 +489,7 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
  */
 void synchronize_srcu(struct srcu_struct *sp)
 {
-       __synchronize_srcu(sp, rcu_gp_is_expedited()
+       __synchronize_srcu(sp, (rcu_gp_is_expedited() && !rcu_gp_is_normal())
                           ? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
                           : SYNCHRONIZE_SRCU_TRYCOUNT);
 }

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 6a652d1f3d7f88e3ea9aae5fdadd0d6a5a2b4522..489992997c06eb543a7fb1d97236661bea009530 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3841,6 +3841,12 @@ void synchronize_sched_expedited(void)
        if (rcu_blocking_is_gp())
                return;
 
+       /* If expedited grace periods are prohibited, fall back to normal. */
+       if (rcu_gp_is_normal()) {
+               wait_rcu_gp(call_rcu_sched);
+               return;
+       }
+
        /* Take a snapshot of the sequence number.  */
        s = rcu_exp_gp_seq_snap(rsp);
 

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 57ba873d2f180066ef2d9b8bbd5d333297679c4d..d45df378155166a08c4b44e492ff9f98651b4b74 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -746,6 +746,12 @@ void synchronize_rcu_expedited(void)
        struct rcu_state *rsp = rcu_state_p;
        unsigned long s;
 
+       /* If expedited grace periods are prohibited, fall back to normal. */
+       if (rcu_gp_is_normal()) {
+               wait_rcu_gp(call_rcu);
+               return;
+       }
+
        s = rcu_exp_gp_seq_snap(rsp);
 
        rnp_unlock = exp_funnel_lock(rsp, s);

diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index 5f748c5a40f0756b2d6c97fa615cde551649db4a..8fccda3a794d9069d78dee99a74857af4b1ede95 100644 (file)
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -61,6 +61,7 @@ MODULE_ALIAS("rcupdate");
 #define MODULE_PARAM_PREFIX "rcupdate."
 
 module_param(rcu_expedited, int, 0);
+module_param(rcu_normal, int, 0);
 
 #if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_PREEMPT_COUNT)
 /**
@@ -113,6 +114,17 @@ EXPORT_SYMBOL(rcu_read_lock_sched_held);
 
 #ifndef CONFIG_TINY_RCU
 
+/*
+ * Should expedited grace-period primitives always fall back to their
+ * non-expedited counterparts?  Intended for use within RCU.  Note
+ * that if the user specifies both rcu_expedited and rcu_normal, then
+ * rcu_normal wins.
+ */
+bool rcu_gp_is_normal(void)
+{
+       return READ_ONCE(rcu_normal);
+}
+
 static atomic_t rcu_expedited_nesting =
        ATOMIC_INIT(IS_ENABLED(CONFIG_RCU_EXPEDITE_BOOT) ? 1 : 0);