2 * core.c - Kernel Live Patching Core
4 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
5 * Copyright (C) 2014 SUSE
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, see <http://www.gnu.org/licenses/>.
21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
23 #include <linux/module.h>
24 #include <linux/kernel.h>
25 #include <linux/mutex.h>
26 #include <linux/slab.h>
27 #include <linux/ftrace.h>
28 #include <linux/list.h>
29 #include <linux/kallsyms.h>
30 #include <linux/livepatch.h>
33 * struct klp_ops - structure for tracking registered ftrace ops structs
35 * A single ftrace_ops is shared between all enabled replacement functions
36 * (klp_func structs) which have the same old_addr. This allows the switch
37 * between function versions to happen instantaneously by updating the klp_ops
38 * struct's func_stack list. The winner is the klp_func at the top of the
39 * func_stack (front of the list).
41 * @node: node for the global klp_ops list
42 * @func_stack: list head for the stack of klp_func's (active func is on top)
43 * @fops: registered ftrace ops struct
46 struct list_head node
;
47 struct list_head func_stack
;
48 struct ftrace_ops fops
;
52 * The klp_mutex protects the global lists and state transitions of any
53 * structure reachable from them. References to any structure must be obtained
54 * under mutex protection (except in klp_ftrace_handler(), which uses RCU to
55 * ensure it gets consistent data).
57 static DEFINE_MUTEX(klp_mutex
);
59 static LIST_HEAD(klp_patches
);
60 static LIST_HEAD(klp_ops
);
62 static struct kobject
*klp_root_kobj
;
64 static struct klp_ops
*klp_find_ops(unsigned long old_addr
)
67 struct klp_func
*func
;
69 list_for_each_entry(ops
, &klp_ops
, node
) {
70 func
= list_first_entry(&ops
->func_stack
, struct klp_func
,
72 if (func
->old_addr
== old_addr
)
79 static bool klp_is_module(struct klp_object
*obj
)
84 static bool klp_is_object_loaded(struct klp_object
*obj
)
86 return !obj
->name
|| obj
->mod
;
89 /* sets obj->mod if object is not vmlinux and module is found */
90 static void klp_find_object_module(struct klp_object
*obj
)
94 if (!klp_is_module(obj
))
97 mutex_lock(&module_mutex
);
99 * We do not want to block removal of patched modules and therefore
100 * we do not take a reference here. The patches are removed by
101 * a going module handler instead.
103 mod
= find_module(obj
->name
);
105 * Do not mess work of the module coming and going notifiers.
106 * Note that the patch might still be needed before the going handler
107 * is called. Module functions can be called even in the GOING state
108 * until mod->exit() finishes. This is especially important for
109 * patches that modify semantic of the functions.
111 if (mod
&& mod
->klp_alive
)
114 mutex_unlock(&module_mutex
);
117 /* klp_mutex must be held by caller */
118 static bool klp_is_patch_registered(struct klp_patch
*patch
)
120 struct klp_patch
*mypatch
;
122 list_for_each_entry(mypatch
, &klp_patches
, list
)
123 if (mypatch
== patch
)
129 static bool klp_initialized(void)
131 return klp_root_kobj
;
134 struct klp_find_arg
{
139 * If count == 0, the symbol was not found. If count == 1, a unique
140 * match was found and addr is set. If count > 1, there is
141 * unresolvable ambiguity among "count" number of symbols with the same
142 * name in the same object.
147 static int klp_find_callback(void *data
, const char *name
,
148 struct module
*mod
, unsigned long addr
)
150 struct klp_find_arg
*args
= data
;
152 if ((mod
&& !args
->objname
) || (!mod
&& args
->objname
))
155 if (strcmp(args
->name
, name
))
158 if (args
->objname
&& strcmp(args
->objname
, mod
->name
))
162 * args->addr might be overwritten if another match is found
163 * but klp_find_object_symbol() handles this and only returns the
164 * addr if count == 1.
172 static int klp_find_object_symbol(const char *objname
, const char *name
,
175 struct klp_find_arg args
= {
182 kallsyms_on_each_symbol(klp_find_callback
, &args
);
185 pr_err("symbol '%s' not found in symbol table\n", name
);
186 else if (args
.count
> 1)
187 pr_err("unresolvable ambiguity (%lu matches) on symbol '%s' in object '%s'\n",
188 args
.count
, name
, objname
);
198 struct klp_verify_args
{
200 const unsigned long addr
;
203 static int klp_verify_callback(void *data
, const char *name
,
204 struct module
*mod
, unsigned long addr
)
206 struct klp_verify_args
*args
= data
;
209 !strcmp(args
->name
, name
) &&
216 static int klp_verify_vmlinux_symbol(const char *name
, unsigned long addr
)
218 struct klp_verify_args args
= {
223 if (kallsyms_on_each_symbol(klp_verify_callback
, &args
))
226 pr_err("symbol '%s' not found at specified address 0x%016lx, kernel mismatch?\n",
231 static int klp_find_verify_func_addr(struct klp_object
*obj
,
232 struct klp_func
*func
)
236 #if defined(CONFIG_RANDOMIZE_BASE)
237 /* KASLR is enabled, disregard old_addr from user */
241 if (!func
->old_addr
|| klp_is_module(obj
))
242 ret
= klp_find_object_symbol(obj
->name
, func
->old_name
,
245 ret
= klp_verify_vmlinux_symbol(func
->old_name
,
252 * external symbols are located outside the parent object (where the parent
253 * object is either vmlinux or the kmod being patched).
255 static int klp_find_external_symbol(struct module
*pmod
, const char *name
,
258 const struct kernel_symbol
*sym
;
260 /* first, check if it's an exported symbol */
262 sym
= find_symbol(name
, NULL
, NULL
, true, true);
270 /* otherwise check if it's in another .o within the patch module */
271 return klp_find_object_symbol(pmod
->name
, name
, addr
);
274 static int klp_write_object_relocations(struct module
*pmod
,
275 struct klp_object
*obj
)
278 struct klp_reloc
*reloc
;
280 if (WARN_ON(!klp_is_object_loaded(obj
)))
283 if (WARN_ON(!obj
->relocs
))
286 for (reloc
= obj
->relocs
; reloc
->name
; reloc
++) {
287 if (!klp_is_module(obj
)) {
288 ret
= klp_verify_vmlinux_symbol(reloc
->name
,
293 /* module, reloc->val needs to be discovered */
295 ret
= klp_find_external_symbol(pmod
,
299 ret
= klp_find_object_symbol(obj
->mod
->name
,
305 ret
= klp_write_module_reloc(pmod
, reloc
->type
, reloc
->loc
,
306 reloc
->val
+ reloc
->addend
);
308 pr_err("relocation failed for symbol '%s' at 0x%016lx (%d)\n",
309 reloc
->name
, reloc
->val
, ret
);
317 static void notrace
klp_ftrace_handler(unsigned long ip
,
318 unsigned long parent_ip
,
319 struct ftrace_ops
*fops
,
320 struct pt_regs
*regs
)
323 struct klp_func
*func
;
325 ops
= container_of(fops
, struct klp_ops
, fops
);
328 func
= list_first_or_null_rcu(&ops
->func_stack
, struct klp_func
,
330 if (WARN_ON_ONCE(!func
))
333 klp_arch_set_pc(regs
, (unsigned long)func
->new_func
);
338 static int klp_disable_func(struct klp_func
*func
)
343 if (WARN_ON(func
->state
!= KLP_ENABLED
))
346 if (WARN_ON(!func
->old_addr
))
349 ops
= klp_find_ops(func
->old_addr
);
353 if (list_is_singular(&ops
->func_stack
)) {
354 ret
= unregister_ftrace_function(&ops
->fops
);
356 pr_err("failed to unregister ftrace handler for function '%s' (%d)\n",
357 func
->old_name
, ret
);
361 ret
= ftrace_set_filter_ip(&ops
->fops
, func
->old_addr
, 1, 0);
363 pr_warn("function unregister succeeded but failed to clear the filter\n");
365 list_del_rcu(&func
->stack_node
);
366 list_del(&ops
->node
);
369 list_del_rcu(&func
->stack_node
);
372 func
->state
= KLP_DISABLED
;
377 static int klp_enable_func(struct klp_func
*func
)
382 if (WARN_ON(!func
->old_addr
))
385 if (WARN_ON(func
->state
!= KLP_DISABLED
))
388 ops
= klp_find_ops(func
->old_addr
);
390 ops
= kzalloc(sizeof(*ops
), GFP_KERNEL
);
394 ops
->fops
.func
= klp_ftrace_handler
;
395 ops
->fops
.flags
= FTRACE_OPS_FL_SAVE_REGS
|
396 FTRACE_OPS_FL_DYNAMIC
|
397 FTRACE_OPS_FL_IPMODIFY
;
399 list_add(&ops
->node
, &klp_ops
);
401 INIT_LIST_HEAD(&ops
->func_stack
);
402 list_add_rcu(&func
->stack_node
, &ops
->func_stack
);
404 ret
= ftrace_set_filter_ip(&ops
->fops
, func
->old_addr
, 0, 0);
406 pr_err("failed to set ftrace filter for function '%s' (%d)\n",
407 func
->old_name
, ret
);
411 ret
= register_ftrace_function(&ops
->fops
);
413 pr_err("failed to register ftrace handler for function '%s' (%d)\n",
414 func
->old_name
, ret
);
415 ftrace_set_filter_ip(&ops
->fops
, func
->old_addr
, 1, 0);
421 list_add_rcu(&func
->stack_node
, &ops
->func_stack
);
424 func
->state
= KLP_ENABLED
;
429 list_del_rcu(&func
->stack_node
);
430 list_del(&ops
->node
);
435 static int klp_disable_object(struct klp_object
*obj
)
437 struct klp_func
*func
;
440 for (func
= obj
->funcs
; func
->old_name
; func
++) {
441 if (func
->state
!= KLP_ENABLED
)
444 ret
= klp_disable_func(func
);
449 obj
->state
= KLP_DISABLED
;
454 static int klp_enable_object(struct klp_object
*obj
)
456 struct klp_func
*func
;
459 if (WARN_ON(obj
->state
!= KLP_DISABLED
))
462 if (WARN_ON(!klp_is_object_loaded(obj
)))
465 for (func
= obj
->funcs
; func
->old_name
; func
++) {
466 ret
= klp_enable_func(func
);
470 obj
->state
= KLP_ENABLED
;
475 WARN_ON(klp_disable_object(obj
));
479 static int __klp_disable_patch(struct klp_patch
*patch
)
481 struct klp_object
*obj
;
484 /* enforce stacking: only the last enabled patch can be disabled */
485 if (!list_is_last(&patch
->list
, &klp_patches
) &&
486 list_next_entry(patch
, list
)->state
== KLP_ENABLED
)
489 pr_notice("disabling patch '%s'\n", patch
->mod
->name
);
491 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
492 if (obj
->state
!= KLP_ENABLED
)
495 ret
= klp_disable_object(obj
);
500 patch
->state
= KLP_DISABLED
;
506 * klp_disable_patch() - disables a registered patch
507 * @patch: The registered, enabled patch to be disabled
509 * Unregisters the patched functions from ftrace.
511 * Return: 0 on success, otherwise error
513 int klp_disable_patch(struct klp_patch
*patch
)
517 mutex_lock(&klp_mutex
);
519 if (!klp_is_patch_registered(patch
)) {
524 if (patch
->state
== KLP_DISABLED
) {
529 ret
= __klp_disable_patch(patch
);
532 mutex_unlock(&klp_mutex
);
535 EXPORT_SYMBOL_GPL(klp_disable_patch
);
537 static int __klp_enable_patch(struct klp_patch
*patch
)
539 struct klp_object
*obj
;
542 if (WARN_ON(patch
->state
!= KLP_DISABLED
))
545 /* enforce stacking: only the first disabled patch can be enabled */
546 if (patch
->list
.prev
!= &klp_patches
&&
547 list_prev_entry(patch
, list
)->state
== KLP_DISABLED
)
550 pr_notice_once("tainting kernel with TAINT_LIVEPATCH\n");
551 add_taint(TAINT_LIVEPATCH
, LOCKDEP_STILL_OK
);
553 pr_notice("enabling patch '%s'\n", patch
->mod
->name
);
555 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
556 klp_find_object_module(obj
);
558 if (!klp_is_object_loaded(obj
))
561 ret
= klp_enable_object(obj
);
566 patch
->state
= KLP_ENABLED
;
571 WARN_ON(__klp_disable_patch(patch
));
576 * klp_enable_patch() - enables a registered patch
577 * @patch: The registered, disabled patch to be enabled
579 * Performs the needed symbol lookups and code relocations,
580 * then registers the patched functions with ftrace.
582 * Return: 0 on success, otherwise error
584 int klp_enable_patch(struct klp_patch
*patch
)
588 mutex_lock(&klp_mutex
);
590 if (!klp_is_patch_registered(patch
)) {
595 ret
= __klp_enable_patch(patch
);
598 mutex_unlock(&klp_mutex
);
601 EXPORT_SYMBOL_GPL(klp_enable_patch
);
606 * /sys/kernel/livepatch
607 * /sys/kernel/livepatch/<patch>
608 * /sys/kernel/livepatch/<patch>/enabled
609 * /sys/kernel/livepatch/<patch>/<object>
610 * /sys/kernel/livepatch/<patch>/<object>/<func>
613 static ssize_t
enabled_store(struct kobject
*kobj
, struct kobj_attribute
*attr
,
614 const char *buf
, size_t count
)
616 struct klp_patch
*patch
;
620 ret
= kstrtoul(buf
, 10, &val
);
624 if (val
!= KLP_DISABLED
&& val
!= KLP_ENABLED
)
627 patch
= container_of(kobj
, struct klp_patch
, kobj
);
629 mutex_lock(&klp_mutex
);
631 if (val
== patch
->state
) {
632 /* already in requested state */
637 if (val
== KLP_ENABLED
) {
638 ret
= __klp_enable_patch(patch
);
642 ret
= __klp_disable_patch(patch
);
647 mutex_unlock(&klp_mutex
);
652 mutex_unlock(&klp_mutex
);
656 static ssize_t
enabled_show(struct kobject
*kobj
,
657 struct kobj_attribute
*attr
, char *buf
)
659 struct klp_patch
*patch
;
661 patch
= container_of(kobj
, struct klp_patch
, kobj
);
662 return snprintf(buf
, PAGE_SIZE
-1, "%d\n", patch
->state
);
665 static struct kobj_attribute enabled_kobj_attr
= __ATTR_RW(enabled
);
666 static struct attribute
*klp_patch_attrs
[] = {
667 &enabled_kobj_attr
.attr
,
671 static void klp_kobj_release_patch(struct kobject
*kobj
)
674 * Once we have a consistency model we'll need to module_put() the
675 * patch module here. See klp_register_patch() for more details.
679 static struct kobj_type klp_ktype_patch
= {
680 .release
= klp_kobj_release_patch
,
681 .sysfs_ops
= &kobj_sysfs_ops
,
682 .default_attrs
= klp_patch_attrs
,
685 static void klp_kobj_release_func(struct kobject
*kobj
)
689 static struct kobj_type klp_ktype_func
= {
690 .release
= klp_kobj_release_func
,
691 .sysfs_ops
= &kobj_sysfs_ops
,
695 * Free all functions' kobjects in the array up to some limit. When limit is
696 * NULL, all kobjects are freed.
698 static void klp_free_funcs_limited(struct klp_object
*obj
,
699 struct klp_func
*limit
)
701 struct klp_func
*func
;
703 for (func
= obj
->funcs
; func
->old_name
&& func
!= limit
; func
++)
704 kobject_put(&func
->kobj
);
707 /* Clean up when a patched object is unloaded */
708 static void klp_free_object_loaded(struct klp_object
*obj
)
710 struct klp_func
*func
;
714 for (func
= obj
->funcs
; func
->old_name
; func
++)
719 * Free all objects' kobjects in the array up to some limit. When limit is
720 * NULL, all kobjects are freed.
722 static void klp_free_objects_limited(struct klp_patch
*patch
,
723 struct klp_object
*limit
)
725 struct klp_object
*obj
;
727 for (obj
= patch
->objs
; obj
->funcs
&& obj
!= limit
; obj
++) {
728 klp_free_funcs_limited(obj
, NULL
);
729 kobject_put(obj
->kobj
);
733 static void klp_free_patch(struct klp_patch
*patch
)
735 klp_free_objects_limited(patch
, NULL
);
736 if (!list_empty(&patch
->list
))
737 list_del(&patch
->list
);
738 kobject_put(&patch
->kobj
);
741 static int klp_init_func(struct klp_object
*obj
, struct klp_func
*func
)
743 INIT_LIST_HEAD(&func
->stack_node
);
744 func
->state
= KLP_DISABLED
;
746 return kobject_init_and_add(&func
->kobj
, &klp_ktype_func
,
747 obj
->kobj
, "%s", func
->old_name
);
750 /* parts of the initialization that is done only when the object is loaded */
751 static int klp_init_object_loaded(struct klp_patch
*patch
,
752 struct klp_object
*obj
)
754 struct klp_func
*func
;
758 ret
= klp_write_object_relocations(patch
->mod
, obj
);
763 for (func
= obj
->funcs
; func
->old_name
; func
++) {
764 ret
= klp_find_verify_func_addr(obj
, func
);
772 static int klp_init_object(struct klp_patch
*patch
, struct klp_object
*obj
)
774 struct klp_func
*func
;
781 obj
->state
= KLP_DISABLED
;
784 klp_find_object_module(obj
);
786 name
= klp_is_module(obj
) ? obj
->name
: "vmlinux";
787 obj
->kobj
= kobject_create_and_add(name
, &patch
->kobj
);
791 for (func
= obj
->funcs
; func
->old_name
; func
++) {
792 ret
= klp_init_func(obj
, func
);
797 if (klp_is_object_loaded(obj
)) {
798 ret
= klp_init_object_loaded(patch
, obj
);
806 klp_free_funcs_limited(obj
, func
);
807 kobject_put(obj
->kobj
);
811 static int klp_init_patch(struct klp_patch
*patch
)
813 struct klp_object
*obj
;
819 mutex_lock(&klp_mutex
);
821 patch
->state
= KLP_DISABLED
;
823 ret
= kobject_init_and_add(&patch
->kobj
, &klp_ktype_patch
,
824 klp_root_kobj
, "%s", patch
->mod
->name
);
828 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
829 ret
= klp_init_object(patch
, obj
);
834 list_add_tail(&patch
->list
, &klp_patches
);
836 mutex_unlock(&klp_mutex
);
841 klp_free_objects_limited(patch
, obj
);
842 kobject_put(&patch
->kobj
);
844 mutex_unlock(&klp_mutex
);
849 * klp_unregister_patch() - unregisters a patch
850 * @patch: Disabled patch to be unregistered
852 * Frees the data structures and removes the sysfs interface.
854 * Return: 0 on success, otherwise error
856 int klp_unregister_patch(struct klp_patch
*patch
)
860 mutex_lock(&klp_mutex
);
862 if (!klp_is_patch_registered(patch
)) {
867 if (patch
->state
== KLP_ENABLED
) {
872 klp_free_patch(patch
);
875 mutex_unlock(&klp_mutex
);
878 EXPORT_SYMBOL_GPL(klp_unregister_patch
);
881 * klp_register_patch() - registers a patch
882 * @patch: Patch to be registered
884 * Initializes the data structure associated with the patch and
885 * creates the sysfs interface.
887 * Return: 0 on success, otherwise error
889 int klp_register_patch(struct klp_patch
*patch
)
893 if (!klp_initialized())
896 if (!patch
|| !patch
->mod
)
900 * A reference is taken on the patch module to prevent it from being
901 * unloaded. Right now, we don't allow patch modules to unload since
902 * there is currently no method to determine if a thread is still
903 * running in the patched code contained in the patch module once
904 * the ftrace registration is successful.
906 if (!try_module_get(patch
->mod
))
909 ret
= klp_init_patch(patch
);
911 module_put(patch
->mod
);
915 EXPORT_SYMBOL_GPL(klp_register_patch
);
917 static void klp_module_notify_coming(struct klp_patch
*patch
,
918 struct klp_object
*obj
)
920 struct module
*pmod
= patch
->mod
;
921 struct module
*mod
= obj
->mod
;
924 ret
= klp_init_object_loaded(patch
, obj
);
928 if (patch
->state
== KLP_DISABLED
)
931 pr_notice("applying patch '%s' to loading module '%s'\n",
932 pmod
->name
, mod
->name
);
934 ret
= klp_enable_object(obj
);
939 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
940 pmod
->name
, mod
->name
, ret
);
943 static void klp_module_notify_going(struct klp_patch
*patch
,
944 struct klp_object
*obj
)
946 struct module
*pmod
= patch
->mod
;
947 struct module
*mod
= obj
->mod
;
950 if (patch
->state
== KLP_DISABLED
)
953 pr_notice("reverting patch '%s' on unloading module '%s'\n",
954 pmod
->name
, mod
->name
);
956 ret
= klp_disable_object(obj
);
958 pr_warn("failed to revert patch '%s' on module '%s' (%d)\n",
959 pmod
->name
, mod
->name
, ret
);
962 klp_free_object_loaded(obj
);
965 static int klp_module_notify(struct notifier_block
*nb
, unsigned long action
,
968 struct module
*mod
= data
;
969 struct klp_patch
*patch
;
970 struct klp_object
*obj
;
972 if (action
!= MODULE_STATE_COMING
&& action
!= MODULE_STATE_GOING
)
975 mutex_lock(&klp_mutex
);
978 * Each module has to know that the notifier has been called.
979 * We never know what module will get patched by a new patch.
981 if (action
== MODULE_STATE_COMING
)
982 mod
->klp_alive
= true;
983 else /* MODULE_STATE_GOING */
984 mod
->klp_alive
= false;
986 list_for_each_entry(patch
, &klp_patches
, list
) {
987 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
988 if (!klp_is_module(obj
) || strcmp(obj
->name
, mod
->name
))
991 if (action
== MODULE_STATE_COMING
) {
993 klp_module_notify_coming(patch
, obj
);
994 } else /* MODULE_STATE_GOING */
995 klp_module_notify_going(patch
, obj
);
1001 mutex_unlock(&klp_mutex
);
1006 static struct notifier_block klp_module_nb
= {
1007 .notifier_call
= klp_module_notify
,
1008 .priority
= INT_MIN
+1, /* called late but before ftrace notifier */
1011 static int klp_init(void)
1015 ret
= klp_check_compiler_support();
1017 pr_info("Your compiler is too old; turning off.\n");
1021 ret
= register_module_notifier(&klp_module_nb
);
1025 klp_root_kobj
= kobject_create_and_add("livepatch", kernel_kobj
);
1026 if (!klp_root_kobj
) {
1034 unregister_module_notifier(&klp_module_nb
);
1038 module_init(klp_init
);