| 1 | /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com |
| 2 | * |
| 3 | * This program is free software; you can redistribute it and/or |
| 4 | * modify it under the terms of version 2 of the GNU General Public |
| 5 | * License as published by the Free Software Foundation. |
| 6 | */ |
| 7 | #ifndef _LINUX_BPF_H |
| 8 | #define _LINUX_BPF_H 1 |
| 9 | |
| 10 | #include <uapi/linux/bpf.h> |
| 11 | #include <linux/workqueue.h> |
| 12 | #include <linux/file.h> |
| 13 | #include <linux/percpu.h> |
| 14 | |
| 15 | struct bpf_map; |
| 16 | |
| 17 | /* map is generic key/value storage optionally accesible by eBPF programs */ |
| 18 | struct bpf_map_ops { |
| 19 | /* funcs callable from userspace (via syscall) */ |
| 20 | struct bpf_map *(*map_alloc)(union bpf_attr *attr); |
| 21 | void (*map_free)(struct bpf_map *); |
| 22 | int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); |
| 23 | |
| 24 | /* funcs callable from userspace and from eBPF programs */ |
| 25 | void *(*map_lookup_elem)(struct bpf_map *map, void *key); |
| 26 | int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); |
| 27 | int (*map_delete_elem)(struct bpf_map *map, void *key); |
| 28 | |
| 29 | /* funcs called by prog_array and perf_event_array map */ |
| 30 | void *(*map_fd_get_ptr) (struct bpf_map *map, int fd); |
| 31 | void (*map_fd_put_ptr) (void *ptr); |
| 32 | }; |
| 33 | |
| 34 | struct bpf_map { |
| 35 | atomic_t refcnt; |
| 36 | enum bpf_map_type map_type; |
| 37 | u32 key_size; |
| 38 | u32 value_size; |
| 39 | u32 max_entries; |
| 40 | u32 map_flags; |
| 41 | u32 pages; |
| 42 | struct user_struct *user; |
| 43 | const struct bpf_map_ops *ops; |
| 44 | struct work_struct work; |
| 45 | atomic_t usercnt; |
| 46 | }; |
| 47 | |
| 48 | struct bpf_map_type_list { |
| 49 | struct list_head list_node; |
| 50 | const struct bpf_map_ops *ops; |
| 51 | enum bpf_map_type type; |
| 52 | }; |
| 53 | |
| 54 | /* function argument constraints */ |
| 55 | enum bpf_arg_type { |
| 56 | ARG_DONTCARE = 0, /* unused argument in helper function */ |
| 57 | |
| 58 | /* the following constraints used to prototype |
| 59 | * bpf_map_lookup/update/delete_elem() functions |
| 60 | */ |
| 61 | ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ |
| 62 | ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ |
| 63 | ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ |
| 64 | |
| 65 | /* the following constraints used to prototype bpf_memcmp() and other |
| 66 | * functions that access data on eBPF program stack |
| 67 | */ |
| 68 | ARG_PTR_TO_STACK, /* any pointer to eBPF program stack */ |
| 69 | ARG_PTR_TO_RAW_STACK, /* any pointer to eBPF program stack, area does not |
| 70 | * need to be initialized, helper function must fill |
| 71 | * all bytes or clear them in error case. |
| 72 | */ |
| 73 | |
| 74 | ARG_CONST_STACK_SIZE, /* number of bytes accessed from stack */ |
| 75 | ARG_CONST_STACK_SIZE_OR_ZERO, /* number of bytes accessed from stack or 0 */ |
| 76 | |
| 77 | ARG_PTR_TO_CTX, /* pointer to context */ |
| 78 | ARG_ANYTHING, /* any (initialized) argument is ok */ |
| 79 | }; |
| 80 | |
| 81 | /* type of values returned from helper functions */ |
| 82 | enum bpf_return_type { |
| 83 | RET_INTEGER, /* function returns integer */ |
| 84 | RET_VOID, /* function doesn't return anything */ |
| 85 | RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */ |
| 86 | }; |
| 87 | |
| 88 | /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs |
| 89 | * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL |
| 90 | * instructions after verifying |
| 91 | */ |
| 92 | struct bpf_func_proto { |
| 93 | u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); |
| 94 | bool gpl_only; |
| 95 | enum bpf_return_type ret_type; |
| 96 | enum bpf_arg_type arg1_type; |
| 97 | enum bpf_arg_type arg2_type; |
| 98 | enum bpf_arg_type arg3_type; |
| 99 | enum bpf_arg_type arg4_type; |
| 100 | enum bpf_arg_type arg5_type; |
| 101 | }; |
| 102 | |
| 103 | /* bpf_context is intentionally undefined structure. Pointer to bpf_context is |
| 104 | * the first argument to eBPF programs. |
| 105 | * For socket filters: 'struct bpf_context *' == 'struct sk_buff *' |
| 106 | */ |
| 107 | struct bpf_context; |
| 108 | |
| 109 | enum bpf_access_type { |
| 110 | BPF_READ = 1, |
| 111 | BPF_WRITE = 2 |
| 112 | }; |
| 113 | |
| 114 | struct bpf_prog; |
| 115 | |
| 116 | struct bpf_verifier_ops { |
| 117 | /* return eBPF function prototype for verification */ |
| 118 | const struct bpf_func_proto *(*get_func_proto)(enum bpf_func_id func_id); |
| 119 | |
| 120 | /* return true if 'size' wide access at offset 'off' within bpf_context |
| 121 | * with 'type' (read or write) is allowed |
| 122 | */ |
| 123 | bool (*is_valid_access)(int off, int size, enum bpf_access_type type); |
| 124 | |
| 125 | u32 (*convert_ctx_access)(enum bpf_access_type type, int dst_reg, |
| 126 | int src_reg, int ctx_off, |
| 127 | struct bpf_insn *insn, struct bpf_prog *prog); |
| 128 | }; |
| 129 | |
| 130 | struct bpf_prog_type_list { |
| 131 | struct list_head list_node; |
| 132 | const struct bpf_verifier_ops *ops; |
| 133 | enum bpf_prog_type type; |
| 134 | }; |
| 135 | |
| 136 | struct bpf_prog_aux { |
| 137 | atomic_t refcnt; |
| 138 | u32 used_map_cnt; |
| 139 | u32 max_ctx_offset; |
| 140 | const struct bpf_verifier_ops *ops; |
| 141 | struct bpf_map **used_maps; |
| 142 | struct bpf_prog *prog; |
| 143 | struct user_struct *user; |
| 144 | union { |
| 145 | struct work_struct work; |
| 146 | struct rcu_head rcu; |
| 147 | }; |
| 148 | }; |
| 149 | |
| 150 | struct bpf_array { |
| 151 | struct bpf_map map; |
| 152 | u32 elem_size; |
| 153 | /* 'ownership' of prog_array is claimed by the first program that |
| 154 | * is going to use this map or by the first program which FD is stored |
| 155 | * in the map to make sure that all callers and callees have the same |
| 156 | * prog_type and JITed flag |
| 157 | */ |
| 158 | enum bpf_prog_type owner_prog_type; |
| 159 | bool owner_jited; |
| 160 | union { |
| 161 | char value[0] __aligned(8); |
| 162 | void *ptrs[0] __aligned(8); |
| 163 | void __percpu *pptrs[0] __aligned(8); |
| 164 | }; |
| 165 | }; |
| 166 | #define MAX_TAIL_CALL_CNT 32 |
| 167 | |
| 168 | u64 bpf_tail_call(u64 ctx, u64 r2, u64 index, u64 r4, u64 r5); |
| 169 | u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); |
| 170 | void bpf_fd_array_map_clear(struct bpf_map *map); |
| 171 | bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp); |
| 172 | |
| 173 | const struct bpf_func_proto *bpf_get_trace_printk_proto(void); |
| 174 | const struct bpf_func_proto *bpf_get_event_output_proto(void); |
| 175 | |
| 176 | #ifdef CONFIG_BPF_SYSCALL |
| 177 | DECLARE_PER_CPU(int, bpf_prog_active); |
| 178 | |
| 179 | void bpf_register_prog_type(struct bpf_prog_type_list *tl); |
| 180 | void bpf_register_map_type(struct bpf_map_type_list *tl); |
| 181 | |
| 182 | struct bpf_prog *bpf_prog_get(u32 ufd); |
| 183 | struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog); |
| 184 | void bpf_prog_put(struct bpf_prog *prog); |
| 185 | void bpf_prog_put_rcu(struct bpf_prog *prog); |
| 186 | |
| 187 | struct bpf_map *bpf_map_get_with_uref(u32 ufd); |
| 188 | struct bpf_map *__bpf_map_get(struct fd f); |
| 189 | struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref); |
| 190 | void bpf_map_put_with_uref(struct bpf_map *map); |
| 191 | void bpf_map_put(struct bpf_map *map); |
| 192 | int bpf_map_precharge_memlock(u32 pages); |
| 193 | |
| 194 | extern int sysctl_unprivileged_bpf_disabled; |
| 195 | |
| 196 | int bpf_map_new_fd(struct bpf_map *map); |
| 197 | int bpf_prog_new_fd(struct bpf_prog *prog); |
| 198 | |
| 199 | int bpf_obj_pin_user(u32 ufd, const char __user *pathname); |
| 200 | int bpf_obj_get_user(const char __user *pathname); |
| 201 | |
| 202 | int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); |
| 203 | int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); |
| 204 | int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, |
| 205 | u64 flags); |
| 206 | int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, |
| 207 | u64 flags); |
| 208 | int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); |
| 209 | |
| 210 | /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and |
| 211 | * forced to use 'long' read/writes to try to atomically copy long counters. |
| 212 | * Best-effort only. No barriers here, since it _will_ race with concurrent |
| 213 | * updates from BPF programs. Called from bpf syscall and mostly used with |
| 214 | * size 8 or 16 bytes, so ask compiler to inline it. |
| 215 | */ |
| 216 | static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) |
| 217 | { |
| 218 | const long *lsrc = src; |
| 219 | long *ldst = dst; |
| 220 | |
| 221 | size /= sizeof(long); |
| 222 | while (size--) |
| 223 | *ldst++ = *lsrc++; |
| 224 | } |
| 225 | |
| 226 | /* verify correctness of eBPF program */ |
| 227 | int bpf_check(struct bpf_prog **fp, union bpf_attr *attr); |
| 228 | #else |
| 229 | static inline void bpf_register_prog_type(struct bpf_prog_type_list *tl) |
| 230 | { |
| 231 | } |
| 232 | |
| 233 | static inline struct bpf_prog *bpf_prog_get(u32 ufd) |
| 234 | { |
| 235 | return ERR_PTR(-EOPNOTSUPP); |
| 236 | } |
| 237 | |
| 238 | static inline void bpf_prog_put(struct bpf_prog *prog) |
| 239 | { |
| 240 | } |
| 241 | #endif /* CONFIG_BPF_SYSCALL */ |
| 242 | |
| 243 | /* verifier prototypes for helper functions called from eBPF programs */ |
| 244 | extern const struct bpf_func_proto bpf_map_lookup_elem_proto; |
| 245 | extern const struct bpf_func_proto bpf_map_update_elem_proto; |
| 246 | extern const struct bpf_func_proto bpf_map_delete_elem_proto; |
| 247 | |
| 248 | extern const struct bpf_func_proto bpf_get_prandom_u32_proto; |
| 249 | extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; |
| 250 | extern const struct bpf_func_proto bpf_tail_call_proto; |
| 251 | extern const struct bpf_func_proto bpf_ktime_get_ns_proto; |
| 252 | extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; |
| 253 | extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; |
| 254 | extern const struct bpf_func_proto bpf_get_current_comm_proto; |
| 255 | extern const struct bpf_func_proto bpf_skb_vlan_push_proto; |
| 256 | extern const struct bpf_func_proto bpf_skb_vlan_pop_proto; |
| 257 | extern const struct bpf_func_proto bpf_get_stackid_proto; |
| 258 | |
| 259 | /* Shared helpers among cBPF and eBPF. */ |
| 260 | void bpf_user_rnd_init_once(void); |
| 261 | u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); |
| 262 | |
| 263 | #endif /* _LINUX_BPF_H */ |