Merge branch 'kbuild' of git://git.kernel.org/pub/scm/linux/kernel/git/mmarek/kbuild

[deliverable/linux.git] / kernel / bpf / syscall.c

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 637397059f763564b535cfda2e4eaa9bf1d34fad..2a2efe1bc76c7a2ef2e0a4af3ebec7f988a4b4a6 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -18,6 +18,8 @@
 #include <linux/filter.h>
 #include <linux/version.h>
 
+DEFINE_PER_CPU(int, bpf_prog_active);
+
 int sysctl_unprivileged_bpf_disabled __read_mostly;
 
 static LIST_HEAD(bpf_map_types);
@@ -46,6 +48,19 @@ void bpf_register_map_type(struct bpf_map_type_list *tl)
        list_add(&tl->list_node, &bpf_map_types);
 }
 
+int bpf_map_precharge_memlock(u32 pages)
+{
+       struct user_struct *user = get_current_user();
+       unsigned long memlock_limit, cur;
+
+       memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+       cur = atomic_long_read(&user->locked_vm);
+       free_uid(user);
+       if (cur + pages > memlock_limit)
+               return -EPERM;
+       return 0;
+}
+
 static int bpf_map_charge_memlock(struct bpf_map *map)
 {
        struct user_struct *user = get_current_user();
@@ -151,7 +166,7 @@ int bpf_map_new_fd(struct bpf_map *map)
                   offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
                   sizeof(attr->CMD##_LAST_FIELD)) != NULL
 
-#define BPF_MAP_CREATE_LAST_FIELD max_entries
+#define BPF_MAP_CREATE_LAST_FIELD map_flags
 /* called via syscall */
 static int map_create(union bpf_attr *attr)
 {
@@ -229,6 +244,11 @@ static void __user *u64_to_ptr(__u64 val)
        return (void __user *) (unsigned long) val;
 }
 
+int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
+{
+       return -ENOTSUPP;
+}
+
 /* last field in 'union bpf_attr' used by this command */
 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
 
@@ -239,6 +259,7 @@ static int map_lookup_elem(union bpf_attr *attr)
        int ufd = attr->map_fd;
        struct bpf_map *map;
        void *key, *value, *ptr;
+       u32 value_size;
        struct fd f;
        int err;
 
@@ -259,23 +280,37 @@ static int map_lookup_elem(union bpf_attr *attr)
        if (copy_from_user(key, ukey, map->key_size) != 0)
                goto free_key;
 
+       if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
+           map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
+               value_size = round_up(map->value_size, 8) * num_possible_cpus();
+       else
+               value_size = map->value_size;
+
        err = -ENOMEM;
-       value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
+       value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
        if (!value)
                goto free_key;
 
-       rcu_read_lock();
-       ptr = map->ops->map_lookup_elem(map, key);
-       if (ptr)
-               memcpy(value, ptr, map->value_size);
-       rcu_read_unlock();
+       if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH) {
+               err = bpf_percpu_hash_copy(map, key, value);
+       } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
+               err = bpf_percpu_array_copy(map, key, value);
+       } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
+               err = bpf_stackmap_copy(map, key, value);
+       } else {
+               rcu_read_lock();
+               ptr = map->ops->map_lookup_elem(map, key);
+               if (ptr)
+                       memcpy(value, ptr, value_size);
+               rcu_read_unlock();
+               err = ptr ? 0 : -ENOENT;
+       }
 
-       err = -ENOENT;
-       if (!ptr)
+       if (err)
                goto free_value;
 
        err = -EFAULT;
-       if (copy_to_user(uvalue, value, map->value_size) != 0)
+       if (copy_to_user(uvalue, value, value_size) != 0)
                goto free_value;
 
        err = 0;
@@ -298,6 +333,7 @@ static int map_update_elem(union bpf_attr *attr)
        int ufd = attr->map_fd;
        struct bpf_map *map;
        void *key, *value;
+       u32 value_size;
        struct fd f;
        int err;
 
@@ -318,21 +354,37 @@ static int map_update_elem(union bpf_attr *attr)
        if (copy_from_user(key, ukey, map->key_size) != 0)
                goto free_key;
 
+       if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
+           map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
+               value_size = round_up(map->value_size, 8) * num_possible_cpus();
+       else
+               value_size = map->value_size;
+
        err = -ENOMEM;
-       value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
+       value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
        if (!value)
                goto free_key;
 
        err = -EFAULT;
-       if (copy_from_user(value, uvalue, map->value_size) != 0)
+       if (copy_from_user(value, uvalue, value_size) != 0)
                goto free_value;
 
-       /* eBPF program that use maps are running under rcu_read_lock(),
-        * therefore all map accessors rely on this fact, so do the same here
+       /* must increment bpf_prog_active to avoid kprobe+bpf triggering from
+        * inside bpf map update or delete otherwise deadlocks are possible
         */
-       rcu_read_lock();
-       err = map->ops->map_update_elem(map, key, value, attr->flags);
-       rcu_read_unlock();
+       preempt_disable();
+       __this_cpu_inc(bpf_prog_active);
+       if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH) {
+               err = bpf_percpu_hash_update(map, key, value, attr->flags);
+       } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
+               err = bpf_percpu_array_update(map, key, value, attr->flags);
+       } else {
+               rcu_read_lock();
+               err = map->ops->map_update_elem(map, key, value, attr->flags);
+               rcu_read_unlock();
+       }
+       __this_cpu_dec(bpf_prog_active);
+       preempt_enable();
 
 free_value:
        kfree(value);
@@ -371,9 +423,13 @@ static int map_delete_elem(union bpf_attr *attr)
        if (copy_from_user(key, ukey, map->key_size) != 0)
                goto free_key;
 
+       preempt_disable();
+       __this_cpu_inc(bpf_prog_active);
        rcu_read_lock();
        err = map->ops->map_delete_elem(map, key);
        rcu_read_unlock();
+       __this_cpu_dec(bpf_prog_active);
+       preempt_enable();
 
 free_key:
        kfree(key);