2 * Berkeley Packet Filter based traffic classifier
4 * Might be used to classify traffic through flexible, user-defined and
5 * possibly JIT-ed BPF filters for traffic control as an alternative to
8 * (C) 2013 Daniel Borkmann <dborkman@redhat.com>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
15 #include <linux/module.h>
16 #include <linux/types.h>
17 #include <linux/skbuff.h>
18 #include <linux/filter.h>
19 #include <linux/bpf.h>
21 #include <net/rtnetlink.h>
22 #include <net/pkt_cls.h>
25 MODULE_LICENSE("GPL");
26 MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>");
27 MODULE_DESCRIPTION("TC BPF based classifier");
29 #define CLS_BPF_NAME_LEN 256
32 struct list_head plist
;
38 struct bpf_prog
*filter
;
39 struct list_head link
;
40 struct tcf_result res
;
48 struct sock_filter
*bpf_ops
;
54 static const struct nla_policy bpf_policy
[TCA_BPF_MAX
+ 1] = {
55 [TCA_BPF_CLASSID
] = { .type
= NLA_U32
},
56 [TCA_BPF_FLAGS
] = { .type
= NLA_U32
},
57 [TCA_BPF_FD
] = { .type
= NLA_U32
},
58 [TCA_BPF_NAME
] = { .type
= NLA_NUL_STRING
, .len
= CLS_BPF_NAME_LEN
},
59 [TCA_BPF_OPS_LEN
] = { .type
= NLA_U16
},
60 [TCA_BPF_OPS
] = { .type
= NLA_BINARY
,
61 .len
= sizeof(struct sock_filter
) * BPF_MAXINSNS
},
64 static int cls_bpf_exec_opcode(int code
)
78 static int cls_bpf_classify(struct sk_buff
*skb
, const struct tcf_proto
*tp
,
79 struct tcf_result
*res
)
81 struct cls_bpf_head
*head
= rcu_dereference_bh(tp
->root
);
82 bool at_ingress
= skb_at_tc_ingress(skb
);
83 struct cls_bpf_prog
*prog
;
86 if (unlikely(!skb_mac_header_was_set(skb
)))
89 /* Needed here for accessing maps. */
91 list_for_each_entry_rcu(prog
, &head
->plist
, link
) {
94 qdisc_skb_cb(skb
)->tc_classid
= prog
->res
.classid
;
97 /* It is safe to push/pull even if skb_shared() */
98 __skb_push(skb
, skb
->mac_len
);
99 filter_res
= BPF_PROG_RUN(prog
->filter
, skb
);
100 __skb_pull(skb
, skb
->mac_len
);
102 filter_res
= BPF_PROG_RUN(prog
->filter
, skb
);
105 if (prog
->exts_integrated
) {
107 res
->classid
= TC_H_MAJ(prog
->res
.classid
) |
108 qdisc_skb_cb(skb
)->tc_classid
;
110 ret
= cls_bpf_exec_opcode(filter_res
);
111 if (ret
== TC_ACT_UNSPEC
)
118 if (filter_res
!= -1) {
120 res
->classid
= filter_res
;
125 ret
= tcf_exts_exec(skb
, &prog
->exts
, res
);
136 static bool cls_bpf_is_ebpf(const struct cls_bpf_prog
*prog
)
138 return !prog
->bpf_ops
;
141 static int cls_bpf_init(struct tcf_proto
*tp
)
143 struct cls_bpf_head
*head
;
145 head
= kzalloc(sizeof(*head
), GFP_KERNEL
);
149 INIT_LIST_HEAD_RCU(&head
->plist
);
150 rcu_assign_pointer(tp
->root
, head
);
155 static void cls_bpf_delete_prog(struct tcf_proto
*tp
, struct cls_bpf_prog
*prog
)
157 tcf_exts_destroy(&prog
->exts
);
159 if (cls_bpf_is_ebpf(prog
))
160 bpf_prog_put(prog
->filter
);
162 bpf_prog_destroy(prog
->filter
);
164 kfree(prog
->bpf_name
);
165 kfree(prog
->bpf_ops
);
169 static void __cls_bpf_delete_prog(struct rcu_head
*rcu
)
171 struct cls_bpf_prog
*prog
= container_of(rcu
, struct cls_bpf_prog
, rcu
);
173 cls_bpf_delete_prog(prog
->tp
, prog
);
176 static int cls_bpf_delete(struct tcf_proto
*tp
, unsigned long arg
)
178 struct cls_bpf_prog
*prog
= (struct cls_bpf_prog
*) arg
;
180 list_del_rcu(&prog
->link
);
181 tcf_unbind_filter(tp
, &prog
->res
);
182 call_rcu(&prog
->rcu
, __cls_bpf_delete_prog
);
187 static bool cls_bpf_destroy(struct tcf_proto
*tp
, bool force
)
189 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
190 struct cls_bpf_prog
*prog
, *tmp
;
192 if (!force
&& !list_empty(&head
->plist
))
195 list_for_each_entry_safe(prog
, tmp
, &head
->plist
, link
) {
196 list_del_rcu(&prog
->link
);
197 tcf_unbind_filter(tp
, &prog
->res
);
198 call_rcu(&prog
->rcu
, __cls_bpf_delete_prog
);
201 RCU_INIT_POINTER(tp
->root
, NULL
);
202 kfree_rcu(head
, rcu
);
206 static unsigned long cls_bpf_get(struct tcf_proto
*tp
, u32 handle
)
208 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
209 struct cls_bpf_prog
*prog
;
210 unsigned long ret
= 0UL;
215 list_for_each_entry(prog
, &head
->plist
, link
) {
216 if (prog
->handle
== handle
) {
217 ret
= (unsigned long) prog
;
225 static int cls_bpf_prog_from_ops(struct nlattr
**tb
, struct cls_bpf_prog
*prog
)
227 struct sock_filter
*bpf_ops
;
228 struct sock_fprog_kern fprog_tmp
;
230 u16 bpf_size
, bpf_num_ops
;
233 bpf_num_ops
= nla_get_u16(tb
[TCA_BPF_OPS_LEN
]);
234 if (bpf_num_ops
> BPF_MAXINSNS
|| bpf_num_ops
== 0)
237 bpf_size
= bpf_num_ops
* sizeof(*bpf_ops
);
238 if (bpf_size
!= nla_len(tb
[TCA_BPF_OPS
]))
241 bpf_ops
= kzalloc(bpf_size
, GFP_KERNEL
);
245 memcpy(bpf_ops
, nla_data(tb
[TCA_BPF_OPS
]), bpf_size
);
247 fprog_tmp
.len
= bpf_num_ops
;
248 fprog_tmp
.filter
= bpf_ops
;
250 ret
= bpf_prog_create(&fp
, &fprog_tmp
);
256 prog
->bpf_ops
= bpf_ops
;
257 prog
->bpf_num_ops
= bpf_num_ops
;
258 prog
->bpf_name
= NULL
;
264 static int cls_bpf_prog_from_efd(struct nlattr
**tb
, struct cls_bpf_prog
*prog
,
265 const struct tcf_proto
*tp
)
271 bpf_fd
= nla_get_u32(tb
[TCA_BPF_FD
]);
273 fp
= bpf_prog_get(bpf_fd
);
277 if (fp
->type
!= BPF_PROG_TYPE_SCHED_CLS
) {
282 if (tb
[TCA_BPF_NAME
]) {
283 name
= kmemdup(nla_data(tb
[TCA_BPF_NAME
]),
284 nla_len(tb
[TCA_BPF_NAME
]),
292 prog
->bpf_ops
= NULL
;
293 prog
->bpf_fd
= bpf_fd
;
294 prog
->bpf_name
= name
;
297 if (fp
->dst_needed
&& !(tp
->q
->flags
& TCQ_F_INGRESS
))
298 netif_keep_dst(qdisc_dev(tp
->q
));
303 static int cls_bpf_modify_existing(struct net
*net
, struct tcf_proto
*tp
,
304 struct cls_bpf_prog
*prog
,
305 unsigned long base
, struct nlattr
**tb
,
306 struct nlattr
*est
, bool ovr
)
308 bool is_bpf
, is_ebpf
, have_exts
= false;
309 struct tcf_exts exts
;
312 is_bpf
= tb
[TCA_BPF_OPS_LEN
] && tb
[TCA_BPF_OPS
];
313 is_ebpf
= tb
[TCA_BPF_FD
];
314 if ((!is_bpf
&& !is_ebpf
) || (is_bpf
&& is_ebpf
))
317 tcf_exts_init(&exts
, TCA_BPF_ACT
, TCA_BPF_POLICE
);
318 ret
= tcf_exts_validate(net
, tp
, tb
, est
, &exts
, ovr
);
322 if (tb
[TCA_BPF_FLAGS
]) {
323 u32 bpf_flags
= nla_get_u32(tb
[TCA_BPF_FLAGS
]);
325 if (bpf_flags
& ~TCA_BPF_FLAG_ACT_DIRECT
) {
326 tcf_exts_destroy(&exts
);
330 have_exts
= bpf_flags
& TCA_BPF_FLAG_ACT_DIRECT
;
333 prog
->exts_integrated
= have_exts
;
335 ret
= is_bpf
? cls_bpf_prog_from_ops(tb
, prog
) :
336 cls_bpf_prog_from_efd(tb
, prog
, tp
);
338 tcf_exts_destroy(&exts
);
342 if (tb
[TCA_BPF_CLASSID
]) {
343 prog
->res
.classid
= nla_get_u32(tb
[TCA_BPF_CLASSID
]);
344 tcf_bind_filter(tp
, &prog
->res
, base
);
347 tcf_exts_change(tp
, &prog
->exts
, &exts
);
351 static u32
cls_bpf_grab_new_handle(struct tcf_proto
*tp
,
352 struct cls_bpf_head
*head
)
354 unsigned int i
= 0x80000000;
358 if (++head
->hgen
== 0x7FFFFFFF)
360 } while (--i
> 0 && cls_bpf_get(tp
, head
->hgen
));
362 if (unlikely(i
== 0)) {
363 pr_err("Insufficient number of handles\n");
372 static int cls_bpf_change(struct net
*net
, struct sk_buff
*in_skb
,
373 struct tcf_proto
*tp
, unsigned long base
,
374 u32 handle
, struct nlattr
**tca
,
375 unsigned long *arg
, bool ovr
)
377 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
378 struct cls_bpf_prog
*oldprog
= (struct cls_bpf_prog
*) *arg
;
379 struct nlattr
*tb
[TCA_BPF_MAX
+ 1];
380 struct cls_bpf_prog
*prog
;
383 if (tca
[TCA_OPTIONS
] == NULL
)
386 ret
= nla_parse_nested(tb
, TCA_BPF_MAX
, tca
[TCA_OPTIONS
], bpf_policy
);
390 prog
= kzalloc(sizeof(*prog
), GFP_KERNEL
);
394 tcf_exts_init(&prog
->exts
, TCA_BPF_ACT
, TCA_BPF_POLICE
);
397 if (handle
&& oldprog
->handle
!= handle
) {
404 prog
->handle
= cls_bpf_grab_new_handle(tp
, head
);
406 prog
->handle
= handle
;
407 if (prog
->handle
== 0) {
412 ret
= cls_bpf_modify_existing(net
, tp
, prog
, base
, tb
, tca
[TCA_RATE
], ovr
);
417 list_replace_rcu(&oldprog
->link
, &prog
->link
);
418 tcf_unbind_filter(tp
, &oldprog
->res
);
419 call_rcu(&oldprog
->rcu
, __cls_bpf_delete_prog
);
421 list_add_rcu(&prog
->link
, &head
->plist
);
424 *arg
= (unsigned long) prog
;
432 static int cls_bpf_dump_bpf_info(const struct cls_bpf_prog
*prog
,
437 if (nla_put_u16(skb
, TCA_BPF_OPS_LEN
, prog
->bpf_num_ops
))
440 nla
= nla_reserve(skb
, TCA_BPF_OPS
, prog
->bpf_num_ops
*
441 sizeof(struct sock_filter
));
445 memcpy(nla_data(nla
), prog
->bpf_ops
, nla_len(nla
));
450 static int cls_bpf_dump_ebpf_info(const struct cls_bpf_prog
*prog
,
453 if (nla_put_u32(skb
, TCA_BPF_FD
, prog
->bpf_fd
))
456 if (prog
->bpf_name
&&
457 nla_put_string(skb
, TCA_BPF_NAME
, prog
->bpf_name
))
463 static int cls_bpf_dump(struct net
*net
, struct tcf_proto
*tp
, unsigned long fh
,
464 struct sk_buff
*skb
, struct tcmsg
*tm
)
466 struct cls_bpf_prog
*prog
= (struct cls_bpf_prog
*) fh
;
474 tm
->tcm_handle
= prog
->handle
;
476 nest
= nla_nest_start(skb
, TCA_OPTIONS
);
478 goto nla_put_failure
;
480 if (prog
->res
.classid
&&
481 nla_put_u32(skb
, TCA_BPF_CLASSID
, prog
->res
.classid
))
482 goto nla_put_failure
;
484 if (cls_bpf_is_ebpf(prog
))
485 ret
= cls_bpf_dump_ebpf_info(prog
, skb
);
487 ret
= cls_bpf_dump_bpf_info(prog
, skb
);
489 goto nla_put_failure
;
491 if (tcf_exts_dump(skb
, &prog
->exts
) < 0)
492 goto nla_put_failure
;
494 if (prog
->exts_integrated
)
495 bpf_flags
|= TCA_BPF_FLAG_ACT_DIRECT
;
496 if (bpf_flags
&& nla_put_u32(skb
, TCA_BPF_FLAGS
, bpf_flags
))
497 goto nla_put_failure
;
499 nla_nest_end(skb
, nest
);
501 if (tcf_exts_dump_stats(skb
, &prog
->exts
) < 0)
502 goto nla_put_failure
;
507 nla_nest_cancel(skb
, nest
);
511 static void cls_bpf_walk(struct tcf_proto
*tp
, struct tcf_walker
*arg
)
513 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
514 struct cls_bpf_prog
*prog
;
516 list_for_each_entry(prog
, &head
->plist
, link
) {
517 if (arg
->count
< arg
->skip
)
519 if (arg
->fn(tp
, (unsigned long) prog
, arg
) < 0) {
528 static struct tcf_proto_ops cls_bpf_ops __read_mostly
= {
530 .owner
= THIS_MODULE
,
531 .classify
= cls_bpf_classify
,
532 .init
= cls_bpf_init
,
533 .destroy
= cls_bpf_destroy
,
535 .change
= cls_bpf_change
,
536 .delete = cls_bpf_delete
,
537 .walk
= cls_bpf_walk
,
538 .dump
= cls_bpf_dump
,
541 static int __init
cls_bpf_init_mod(void)
543 return register_tcf_proto_ops(&cls_bpf_ops
);
546 static void __exit
cls_bpf_exit_mod(void)
548 unregister_tcf_proto_ops(&cls_bpf_ops
);
551 module_init(cls_bpf_init_mod
);
552 module_exit(cls_bpf_exit_mod
);