1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful, but
8 * WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10 * General Public License for more details.
12 #include <linux/bpf.h>
13 #include <linux/err.h>
14 #include <linux/vmalloc.h>
15 #include <linux/slab.h>
17 #include <linux/filter.h>
18 #include <linux/perf_event.h>
20 /* Called from syscall */
21 static struct bpf_map
*array_map_alloc(union bpf_attr
*attr
)
23 struct bpf_array
*array
;
24 u32 elem_size
, array_size
;
26 /* check sanity of attributes */
27 if (attr
->max_entries
== 0 || attr
->key_size
!= 4 ||
28 attr
->value_size
== 0)
29 return ERR_PTR(-EINVAL
);
31 elem_size
= round_up(attr
->value_size
, 8);
33 /* check round_up into zero and u32 overflow */
35 attr
->max_entries
> (U32_MAX
- sizeof(*array
)) / elem_size
)
36 return ERR_PTR(-ENOMEM
);
38 array_size
= sizeof(*array
) + attr
->max_entries
* elem_size
;
40 /* allocate all map elements and zero-initialize them */
41 array
= kzalloc(array_size
, GFP_USER
| __GFP_NOWARN
);
43 array
= vzalloc(array_size
);
45 return ERR_PTR(-ENOMEM
);
48 /* copy mandatory map attributes */
49 array
->map
.key_size
= attr
->key_size
;
50 array
->map
.value_size
= attr
->value_size
;
51 array
->map
.max_entries
= attr
->max_entries
;
52 array
->map
.pages
= round_up(array_size
, PAGE_SIZE
) >> PAGE_SHIFT
;
53 array
->elem_size
= elem_size
;
58 /* Called from syscall or from eBPF program */
59 static void *array_map_lookup_elem(struct bpf_map
*map
, void *key
)
61 struct bpf_array
*array
= container_of(map
, struct bpf_array
, map
);
62 u32 index
= *(u32
*)key
;
64 if (index
>= array
->map
.max_entries
)
67 return array
->value
+ array
->elem_size
* index
;
70 /* Called from syscall */
71 static int array_map_get_next_key(struct bpf_map
*map
, void *key
, void *next_key
)
73 struct bpf_array
*array
= container_of(map
, struct bpf_array
, map
);
74 u32 index
= *(u32
*)key
;
75 u32
*next
= (u32
*)next_key
;
77 if (index
>= array
->map
.max_entries
) {
82 if (index
== array
->map
.max_entries
- 1)
89 /* Called from syscall or from eBPF program */
90 static int array_map_update_elem(struct bpf_map
*map
, void *key
, void *value
,
93 struct bpf_array
*array
= container_of(map
, struct bpf_array
, map
);
94 u32 index
= *(u32
*)key
;
96 if (map_flags
> BPF_EXIST
)
100 if (index
>= array
->map
.max_entries
)
101 /* all elements were pre-allocated, cannot insert a new one */
104 if (map_flags
== BPF_NOEXIST
)
105 /* all elements already exist */
108 memcpy(array
->value
+ array
->elem_size
* index
, value
, array
->elem_size
);
112 /* Called from syscall or from eBPF program */
113 static int array_map_delete_elem(struct bpf_map
*map
, void *key
)
118 /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
119 static void array_map_free(struct bpf_map
*map
)
121 struct bpf_array
*array
= container_of(map
, struct bpf_array
, map
);
123 /* at this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,
124 * so the programs (can be more than one that used this map) were
125 * disconnected from events. Wait for outstanding programs to complete
133 static const struct bpf_map_ops array_ops
= {
134 .map_alloc
= array_map_alloc
,
135 .map_free
= array_map_free
,
136 .map_get_next_key
= array_map_get_next_key
,
137 .map_lookup_elem
= array_map_lookup_elem
,
138 .map_update_elem
= array_map_update_elem
,
139 .map_delete_elem
= array_map_delete_elem
,
142 static struct bpf_map_type_list array_type __read_mostly
= {
144 .type
= BPF_MAP_TYPE_ARRAY
,
147 static int __init
register_array_map(void)
149 bpf_register_map_type(&array_type
);
152 late_initcall(register_array_map
);
154 static struct bpf_map
*fd_array_map_alloc(union bpf_attr
*attr
)
156 /* only file descriptors can be stored in this type of map */
157 if (attr
->value_size
!= sizeof(u32
))
158 return ERR_PTR(-EINVAL
);
159 return array_map_alloc(attr
);
162 static void fd_array_map_free(struct bpf_map
*map
)
164 struct bpf_array
*array
= container_of(map
, struct bpf_array
, map
);
169 /* make sure it's empty */
170 for (i
= 0; i
< array
->map
.max_entries
; i
++)
171 BUG_ON(array
->ptrs
[i
] != NULL
);
175 static void *fd_array_map_lookup_elem(struct bpf_map
*map
, void *key
)
180 /* only called from syscall */
181 static int fd_array_map_update_elem(struct bpf_map
*map
, void *key
,
182 void *value
, u64 map_flags
)
184 struct bpf_array
*array
= container_of(map
, struct bpf_array
, map
);
185 void *new_ptr
, *old_ptr
;
186 u32 index
= *(u32
*)key
, ufd
;
188 if (map_flags
!= BPF_ANY
)
191 if (index
>= array
->map
.max_entries
)
195 new_ptr
= map
->ops
->map_fd_get_ptr(map
, ufd
);
197 return PTR_ERR(new_ptr
);
199 old_ptr
= xchg(array
->ptrs
+ index
, new_ptr
);
201 map
->ops
->map_fd_put_ptr(old_ptr
);
206 static int fd_array_map_delete_elem(struct bpf_map
*map
, void *key
)
208 struct bpf_array
*array
= container_of(map
, struct bpf_array
, map
);
210 u32 index
= *(u32
*)key
;
212 if (index
>= array
->map
.max_entries
)
215 old_ptr
= xchg(array
->ptrs
+ index
, NULL
);
217 map
->ops
->map_fd_put_ptr(old_ptr
);
224 static void *prog_fd_array_get_ptr(struct bpf_map
*map
, int fd
)
226 struct bpf_array
*array
= container_of(map
, struct bpf_array
, map
);
227 struct bpf_prog
*prog
= bpf_prog_get(fd
);
231 if (!bpf_prog_array_compatible(array
, prog
)) {
233 return ERR_PTR(-EINVAL
);
238 static void prog_fd_array_put_ptr(void *ptr
)
240 struct bpf_prog
*prog
= ptr
;
242 bpf_prog_put_rcu(prog
);
245 /* decrement refcnt of all bpf_progs that are stored in this map */
246 void bpf_fd_array_map_clear(struct bpf_map
*map
)
248 struct bpf_array
*array
= container_of(map
, struct bpf_array
, map
);
251 for (i
= 0; i
< array
->map
.max_entries
; i
++)
252 fd_array_map_delete_elem(map
, &i
);
255 static const struct bpf_map_ops prog_array_ops
= {
256 .map_alloc
= fd_array_map_alloc
,
257 .map_free
= fd_array_map_free
,
258 .map_get_next_key
= array_map_get_next_key
,
259 .map_lookup_elem
= fd_array_map_lookup_elem
,
260 .map_update_elem
= fd_array_map_update_elem
,
261 .map_delete_elem
= fd_array_map_delete_elem
,
262 .map_fd_get_ptr
= prog_fd_array_get_ptr
,
263 .map_fd_put_ptr
= prog_fd_array_put_ptr
,
266 static struct bpf_map_type_list prog_array_type __read_mostly
= {
267 .ops
= &prog_array_ops
,
268 .type
= BPF_MAP_TYPE_PROG_ARRAY
,
271 static int __init
register_prog_array_map(void)
273 bpf_register_map_type(&prog_array_type
);
276 late_initcall(register_prog_array_map
);
278 static void perf_event_array_map_free(struct bpf_map
*map
)
280 bpf_fd_array_map_clear(map
);
281 fd_array_map_free(map
);
284 static void *perf_event_fd_array_get_ptr(struct bpf_map
*map
, int fd
)
286 struct perf_event
*event
;
287 const struct perf_event_attr
*attr
;
289 event
= perf_event_get(fd
);
293 attr
= perf_event_attrs(event
);
297 if (attr
->type
!= PERF_TYPE_RAW
&&
298 attr
->type
!= PERF_TYPE_HARDWARE
) {
299 perf_event_release_kernel(event
);
300 return ERR_PTR(-EINVAL
);
305 static void perf_event_fd_array_put_ptr(void *ptr
)
307 struct perf_event
*event
= ptr
;
309 perf_event_release_kernel(event
);
312 static const struct bpf_map_ops perf_event_array_ops
= {
313 .map_alloc
= fd_array_map_alloc
,
314 .map_free
= perf_event_array_map_free
,
315 .map_get_next_key
= array_map_get_next_key
,
316 .map_lookup_elem
= fd_array_map_lookup_elem
,
317 .map_update_elem
= fd_array_map_update_elem
,
318 .map_delete_elem
= fd_array_map_delete_elem
,
319 .map_fd_get_ptr
= perf_event_fd_array_get_ptr
,
320 .map_fd_put_ptr
= perf_event_fd_array_put_ptr
,
323 static struct bpf_map_type_list perf_event_array_type __read_mostly
= {
324 .ops
= &perf_event_array_ops
,
325 .type
= BPF_MAP_TYPE_PERF_EVENT_ARRAY
,
328 static int __init
register_perf_event_array_map(void)
330 bpf_register_map_type(&perf_event_array_type
);
333 late_initcall(register_perf_event_array_map
);