Merge git://www.linux-watchdog.org/linux-watchdog

[deliverable/linux.git] / kernel / jump_label.c

diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 2e7cc1e4b4b53f0b522f9072652466bb00a39d2e..f7dd15d537f9b1135eb01036bf2fa3a5594ed9b6 100644 (file)
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -165,16 +165,22 @@ static inline bool static_key_type(struct static_key *key)
 
 static inline struct static_key *jump_entry_key(struct jump_entry *entry)
 {
-       return (struct static_key *)((unsigned long)entry->key);
+       return (struct static_key *)((unsigned long)entry->key & ~1UL);
+}
+
+static bool jump_entry_branch(struct jump_entry *entry)
+{
+       return (unsigned long)entry->key & 1UL;
 }
 
 static enum jump_label_type jump_label_type(struct jump_entry *entry)
 {
        struct static_key *key = jump_entry_key(entry);
        bool enabled = static_key_enabled(key);
-       bool type = static_key_type(key);
+       bool branch = jump_entry_branch(entry);
 
-       return enabled ^ type;
+       /* See the comment in linux/jump_label.h */
+       return enabled ^ branch;
 }
 
 static void __jump_label_update(struct static_key *key,
@@ -205,7 +211,10 @@ void __init jump_label_init(void)
        for (iter = iter_start; iter < iter_stop; iter++) {
                struct static_key *iterk;
 
-               arch_jump_label_transform_static(iter, jump_label_type(iter));
+               /* rewrite NOPs */
+               if (jump_label_type(iter) == JUMP_LABEL_NOP)
+                       arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
+
                iterk = jump_entry_key(iter);
                if (iterk == key)
                        continue;
@@ -225,6 +234,16 @@ void __init jump_label_init(void)
 
 #ifdef CONFIG_MODULES
 
+static enum jump_label_type jump_label_init_type(struct jump_entry *entry)
+{
+       struct static_key *key = jump_entry_key(entry);
+       bool type = static_key_type(key);
+       bool branch = jump_entry_branch(entry);
+
+       /* See the comment in linux/jump_label.h */
+       return type ^ branch;
+}
+
 struct static_key_mod {
        struct static_key_mod *next;
        struct jump_entry *entries;
@@ -276,8 +295,11 @@ void jump_label_apply_nops(struct module *mod)
        if (iter_start == iter_stop)
                return;
 
-       for (iter = iter_start; iter < iter_stop; iter++)
-               arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
+       for (iter = iter_start; iter < iter_stop; iter++) {
+               /* Only write NOPs for arch_branch_static(). */
+               if (jump_label_init_type(iter) == JUMP_LABEL_NOP)
+                       arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
+       }
 }
 
 static int jump_label_add_module(struct module *mod)
@@ -318,7 +340,8 @@ static int jump_label_add_module(struct module *mod)
                jlm->next = key->next;
                key->next = jlm;
 
-               if (jump_label_type(iter) == JUMP_LABEL_JMP)
+               /* Only update if we've changed from our initial state */
+               if (jump_label_type(iter) != jump_label_init_type(iter))
                        __jump_label_update(key, iter, iter_stop);
        }
 
@@ -459,4 +482,41 @@ static void jump_label_update(struct static_key *key)
                __jump_label_update(key, entry, stop);
 }
 
-#endif
+#ifdef CONFIG_STATIC_KEYS_SELFTEST
+static DEFINE_STATIC_KEY_TRUE(sk_true);
+static DEFINE_STATIC_KEY_FALSE(sk_false);
+
+static __init int jump_label_test(void)
+{
+       int i;
+
+       for (i = 0; i < 2; i++) {
+               WARN_ON(static_key_enabled(&sk_true.key) != true);
+               WARN_ON(static_key_enabled(&sk_false.key) != false);
+
+               WARN_ON(!static_branch_likely(&sk_true));
+               WARN_ON(!static_branch_unlikely(&sk_true));
+               WARN_ON(static_branch_likely(&sk_false));
+               WARN_ON(static_branch_unlikely(&sk_false));
+
+               static_branch_disable(&sk_true);
+               static_branch_enable(&sk_false);
+
+               WARN_ON(static_key_enabled(&sk_true.key) == true);
+               WARN_ON(static_key_enabled(&sk_false.key) == false);
+
+               WARN_ON(static_branch_likely(&sk_true));
+               WARN_ON(static_branch_unlikely(&sk_true));
+               WARN_ON(!static_branch_likely(&sk_false));
+               WARN_ON(!static_branch_unlikely(&sk_false));
+
+               static_branch_enable(&sk_true);
+               static_branch_disable(&sk_false);
+       }
+
+       return 0;
+}
+late_initcall(jump_label_test);
+#endif /* STATIC_KEYS_SELFTEST */
+
+#endif /* HAVE_JUMP_LABEL */