Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * net/sched/cls_u32.c Ugly (or Universal) 32bit key Packet Classifier. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation; either version | |
7 | * 2 of the License, or (at your option) any later version. | |
8 | * | |
9 | * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> | |
10 | * | |
11 | * The filters are packed to hash tables of key nodes | |
12 | * with a set of 32bit key/mask pairs at every node. | |
13 | * Nodes reference next level hash tables etc. | |
14 | * | |
15 | * This scheme is the best universal classifier I managed to | |
16 | * invent; it is not super-fast, but it is not slow (provided you | |
17 | * program it correctly), and general enough. And its relative | |
18 | * speed grows as the number of rules becomes larger. | |
19 | * | |
20 | * It seems that it represents the best middle point between | |
21 | * speed and manageability both by human and by machine. | |
22 | * | |
23 | * It is especially useful for link sharing combined with QoS; | |
24 | * pure RSVP doesn't need such a general approach and can use | |
25 | * much simpler (and faster) schemes, sort of cls_rsvp.c. | |
26 | * | |
27 | * JHS: We should remove the CONFIG_NET_CLS_IND from here | |
28 | * eventually when the meta match extension is made available | |
29 | * | |
30 | * nfmark match added by Catalin(ux aka Dino) BOIE <catab at umbrella.ro> | |
31 | */ | |
32 | ||
1da177e4 | 33 | #include <linux/module.h> |
5a0e3ad6 | 34 | #include <linux/slab.h> |
1da177e4 LT |
35 | #include <linux/types.h> |
36 | #include <linux/kernel.h> | |
1da177e4 | 37 | #include <linux/string.h> |
1da177e4 | 38 | #include <linux/errno.h> |
1da177e4 | 39 | #include <linux/rtnetlink.h> |
1da177e4 | 40 | #include <linux/skbuff.h> |
7801db8a | 41 | #include <linux/bitmap.h> |
0ba48053 | 42 | #include <net/netlink.h> |
1da177e4 LT |
43 | #include <net/act_api.h> |
44 | #include <net/pkt_cls.h> | |
45 | ||
cc7ec456 | 46 | struct tc_u_knode { |
1da177e4 LT |
47 | struct tc_u_knode *next; |
48 | u32 handle; | |
49 | struct tc_u_hnode *ht_up; | |
50 | struct tcf_exts exts; | |
51 | #ifdef CONFIG_NET_CLS_IND | |
2519a602 | 52 | int ifindex; |
1da177e4 LT |
53 | #endif |
54 | u8 fshift; | |
55 | struct tcf_result res; | |
56 | struct tc_u_hnode *ht_down; | |
57 | #ifdef CONFIG_CLS_U32_PERF | |
f4f64050 | 58 | struct tc_u32_pcnt __percpu *pf; |
1da177e4 LT |
59 | #endif |
60 | #ifdef CONFIG_CLS_U32_MARK | |
f4f64050 JF |
61 | u32 val; |
62 | u32 mask; | |
63 | u32 __percpu *pcpu_success; | |
1da177e4 LT |
64 | #endif |
65 | struct tc_u32_sel sel; | |
66 | }; | |
67 | ||
cc7ec456 | 68 | struct tc_u_hnode { |
1da177e4 LT |
69 | struct tc_u_hnode *next; |
70 | u32 handle; | |
71 | u32 prio; | |
72 | struct tc_u_common *tp_c; | |
73 | int refcnt; | |
cc7ec456 | 74 | unsigned int divisor; |
1da177e4 LT |
75 | struct tc_u_knode *ht[1]; |
76 | }; | |
77 | ||
cc7ec456 | 78 | struct tc_u_common { |
1da177e4 LT |
79 | struct tc_u_hnode *hlist; |
80 | struct Qdisc *q; | |
81 | int refcnt; | |
82 | u32 hgenerator; | |
83 | }; | |
84 | ||
cc7ec456 ED |
85 | static inline unsigned int u32_hash_fold(__be32 key, |
86 | const struct tc_u32_sel *sel, | |
87 | u8 fshift) | |
1da177e4 | 88 | { |
cc7ec456 | 89 | unsigned int h = ntohl(key & sel->hmask) >> fshift; |
1da177e4 LT |
90 | |
91 | return h; | |
92 | } | |
93 | ||
dc7f9f6e | 94 | static int u32_classify(struct sk_buff *skb, const struct tcf_proto *tp, struct tcf_result *res) |
1da177e4 LT |
95 | { |
96 | struct { | |
97 | struct tc_u_knode *knode; | |
fbc2e7d9 | 98 | unsigned int off; |
1da177e4 LT |
99 | } stack[TC_U32_MAXDEPTH]; |
100 | ||
a8701a6c | 101 | struct tc_u_hnode *ht = tp->root; |
fbc2e7d9 | 102 | unsigned int off = skb_network_offset(skb); |
1da177e4 LT |
103 | struct tc_u_knode *n; |
104 | int sdepth = 0; | |
105 | int off2 = 0; | |
106 | int sel = 0; | |
107 | #ifdef CONFIG_CLS_U32_PERF | |
108 | int j; | |
109 | #endif | |
110 | int i, r; | |
111 | ||
112 | next_ht: | |
113 | n = ht->ht[sel]; | |
114 | ||
115 | next_knode: | |
116 | if (n) { | |
117 | struct tc_u32_key *key = n->sel.keys; | |
118 | ||
119 | #ifdef CONFIG_CLS_U32_PERF | |
f4f64050 | 120 | __this_cpu_inc(n->pf->rcnt); |
1da177e4 LT |
121 | j = 0; |
122 | #endif | |
123 | ||
124 | #ifdef CONFIG_CLS_U32_MARK | |
f4f64050 | 125 | if ((skb->mark & n->mask) != n->val) { |
1da177e4 LT |
126 | n = n->next; |
127 | goto next_knode; | |
128 | } else { | |
f4f64050 | 129 | __this_cpu_inc(*n->pcpu_success); |
1da177e4 LT |
130 | } |
131 | #endif | |
132 | ||
cc7ec456 | 133 | for (i = n->sel.nkeys; i > 0; i--, key++) { |
66d50d25 | 134 | int toff = off + key->off + (off2 & key->offmask); |
86fce3ba | 135 | __be32 *data, hdata; |
fbc2e7d9 | 136 | |
4e18b3ed | 137 | if (skb_headroom(skb) + toff > INT_MAX) |
66d50d25 | 138 | goto out; |
139 | ||
86fce3ba | 140 | data = skb_header_pointer(skb, toff, 4, &hdata); |
fbc2e7d9 CG |
141 | if (!data) |
142 | goto out; | |
143 | if ((*data ^ key->val) & key->mask) { | |
1da177e4 LT |
144 | n = n->next; |
145 | goto next_knode; | |
146 | } | |
147 | #ifdef CONFIG_CLS_U32_PERF | |
f4f64050 | 148 | __this_cpu_inc(n->pf->kcnts[j]); |
1da177e4 LT |
149 | j++; |
150 | #endif | |
151 | } | |
152 | if (n->ht_down == NULL) { | |
153 | check_terminal: | |
cc7ec456 | 154 | if (n->sel.flags & TC_U32_TERMINAL) { |
1da177e4 LT |
155 | |
156 | *res = n->res; | |
157 | #ifdef CONFIG_NET_CLS_IND | |
2519a602 | 158 | if (!tcf_match_indev(skb, n->ifindex)) { |
1da177e4 LT |
159 | n = n->next; |
160 | goto next_knode; | |
161 | } | |
162 | #endif | |
163 | #ifdef CONFIG_CLS_U32_PERF | |
f4f64050 | 164 | __this_cpu_inc(n->pf->rhit); |
1da177e4 LT |
165 | #endif |
166 | r = tcf_exts_exec(skb, &n->exts, res); | |
167 | if (r < 0) { | |
168 | n = n->next; | |
169 | goto next_knode; | |
170 | } | |
171 | ||
172 | return r; | |
173 | } | |
174 | n = n->next; | |
175 | goto next_knode; | |
176 | } | |
177 | ||
178 | /* PUSH */ | |
179 | if (sdepth >= TC_U32_MAXDEPTH) | |
180 | goto deadloop; | |
181 | stack[sdepth].knode = n; | |
fbc2e7d9 | 182 | stack[sdepth].off = off; |
1da177e4 LT |
183 | sdepth++; |
184 | ||
185 | ht = n->ht_down; | |
186 | sel = 0; | |
fbc2e7d9 | 187 | if (ht->divisor) { |
86fce3ba | 188 | __be32 *data, hdata; |
fbc2e7d9 CG |
189 | |
190 | data = skb_header_pointer(skb, off + n->sel.hoff, 4, | |
86fce3ba | 191 | &hdata); |
fbc2e7d9 CG |
192 | if (!data) |
193 | goto out; | |
194 | sel = ht->divisor & u32_hash_fold(*data, &n->sel, | |
195 | n->fshift); | |
196 | } | |
cc7ec456 | 197 | if (!(n->sel.flags & (TC_U32_VAROFFSET | TC_U32_OFFSET | TC_U32_EAT))) |
1da177e4 LT |
198 | goto next_ht; |
199 | ||
cc7ec456 | 200 | if (n->sel.flags & (TC_U32_OFFSET | TC_U32_VAROFFSET)) { |
1da177e4 | 201 | off2 = n->sel.off + 3; |
fbc2e7d9 | 202 | if (n->sel.flags & TC_U32_VAROFFSET) { |
86fce3ba | 203 | __be16 *data, hdata; |
fbc2e7d9 CG |
204 | |
205 | data = skb_header_pointer(skb, | |
206 | off + n->sel.offoff, | |
86fce3ba | 207 | 2, &hdata); |
fbc2e7d9 CG |
208 | if (!data) |
209 | goto out; | |
210 | off2 += ntohs(n->sel.offmask & *data) >> | |
211 | n->sel.offshift; | |
212 | } | |
1da177e4 LT |
213 | off2 &= ~3; |
214 | } | |
cc7ec456 | 215 | if (n->sel.flags & TC_U32_EAT) { |
fbc2e7d9 | 216 | off += off2; |
1da177e4 LT |
217 | off2 = 0; |
218 | } | |
219 | ||
fbc2e7d9 | 220 | if (off < skb->len) |
1da177e4 LT |
221 | goto next_ht; |
222 | } | |
223 | ||
224 | /* POP */ | |
225 | if (sdepth--) { | |
226 | n = stack[sdepth].knode; | |
227 | ht = n->ht_up; | |
fbc2e7d9 | 228 | off = stack[sdepth].off; |
1da177e4 LT |
229 | goto check_terminal; |
230 | } | |
fbc2e7d9 | 231 | out: |
1da177e4 LT |
232 | return -1; |
233 | ||
234 | deadloop: | |
e87cc472 | 235 | net_warn_ratelimited("cls_u32: dead loop\n"); |
1da177e4 LT |
236 | return -1; |
237 | } | |
238 | ||
cc7ec456 | 239 | static struct tc_u_hnode * |
1da177e4 LT |
240 | u32_lookup_ht(struct tc_u_common *tp_c, u32 handle) |
241 | { | |
242 | struct tc_u_hnode *ht; | |
243 | ||
244 | for (ht = tp_c->hlist; ht; ht = ht->next) | |
245 | if (ht->handle == handle) | |
246 | break; | |
247 | ||
248 | return ht; | |
249 | } | |
250 | ||
cc7ec456 | 251 | static struct tc_u_knode * |
1da177e4 LT |
252 | u32_lookup_key(struct tc_u_hnode *ht, u32 handle) |
253 | { | |
cc7ec456 | 254 | unsigned int sel; |
1da177e4 LT |
255 | struct tc_u_knode *n = NULL; |
256 | ||
257 | sel = TC_U32_HASH(handle); | |
258 | if (sel > ht->divisor) | |
259 | goto out; | |
260 | ||
261 | for (n = ht->ht[sel]; n; n = n->next) | |
262 | if (n->handle == handle) | |
263 | break; | |
264 | out: | |
265 | return n; | |
266 | } | |
267 | ||
268 | ||
269 | static unsigned long u32_get(struct tcf_proto *tp, u32 handle) | |
270 | { | |
271 | struct tc_u_hnode *ht; | |
272 | struct tc_u_common *tp_c = tp->data; | |
273 | ||
274 | if (TC_U32_HTID(handle) == TC_U32_ROOT) | |
275 | ht = tp->root; | |
276 | else | |
277 | ht = u32_lookup_ht(tp_c, TC_U32_HTID(handle)); | |
278 | ||
279 | if (!ht) | |
280 | return 0; | |
281 | ||
282 | if (TC_U32_KEY(handle) == 0) | |
283 | return (unsigned long)ht; | |
284 | ||
285 | return (unsigned long)u32_lookup_key(ht, handle); | |
286 | } | |
287 | ||
288 | static void u32_put(struct tcf_proto *tp, unsigned long f) | |
289 | { | |
290 | } | |
291 | ||
292 | static u32 gen_new_htid(struct tc_u_common *tp_c) | |
293 | { | |
294 | int i = 0x800; | |
295 | ||
296 | do { | |
297 | if (++tp_c->hgenerator == 0x7FF) | |
298 | tp_c->hgenerator = 1; | |
cc7ec456 | 299 | } while (--i > 0 && u32_lookup_ht(tp_c, (tp_c->hgenerator|0x800)<<20)); |
1da177e4 LT |
300 | |
301 | return i > 0 ? (tp_c->hgenerator|0x800)<<20 : 0; | |
302 | } | |
303 | ||
304 | static int u32_init(struct tcf_proto *tp) | |
305 | { | |
306 | struct tc_u_hnode *root_ht; | |
307 | struct tc_u_common *tp_c; | |
308 | ||
72b25a91 | 309 | tp_c = tp->q->u32_node; |
1da177e4 | 310 | |
0da974f4 | 311 | root_ht = kzalloc(sizeof(*root_ht), GFP_KERNEL); |
1da177e4 LT |
312 | if (root_ht == NULL) |
313 | return -ENOBUFS; | |
314 | ||
1da177e4 LT |
315 | root_ht->divisor = 0; |
316 | root_ht->refcnt++; | |
317 | root_ht->handle = tp_c ? gen_new_htid(tp_c) : 0x80000000; | |
318 | root_ht->prio = tp->prio; | |
319 | ||
320 | if (tp_c == NULL) { | |
0da974f4 | 321 | tp_c = kzalloc(sizeof(*tp_c), GFP_KERNEL); |
1da177e4 LT |
322 | if (tp_c == NULL) { |
323 | kfree(root_ht); | |
324 | return -ENOBUFS; | |
325 | } | |
1da177e4 | 326 | tp_c->q = tp->q; |
72b25a91 | 327 | tp->q->u32_node = tp_c; |
1da177e4 LT |
328 | } |
329 | ||
330 | tp_c->refcnt++; | |
331 | root_ht->next = tp_c->hlist; | |
332 | tp_c->hlist = root_ht; | |
333 | root_ht->tp_c = tp_c; | |
334 | ||
335 | tp->root = root_ht; | |
336 | tp->data = tp_c; | |
337 | return 0; | |
338 | } | |
339 | ||
340 | static int u32_destroy_key(struct tcf_proto *tp, struct tc_u_knode *n) | |
341 | { | |
342 | tcf_unbind_filter(tp, &n->res); | |
343 | tcf_exts_destroy(tp, &n->exts); | |
344 | if (n->ht_down) | |
345 | n->ht_down->refcnt--; | |
346 | #ifdef CONFIG_CLS_U32_PERF | |
f4f64050 | 347 | free_percpu(n->pf); |
1da177e4 LT |
348 | #endif |
349 | kfree(n); | |
350 | return 0; | |
351 | } | |
352 | ||
82d567c2 | 353 | static int u32_delete_key(struct tcf_proto *tp, struct tc_u_knode *key) |
1da177e4 LT |
354 | { |
355 | struct tc_u_knode **kp; | |
356 | struct tc_u_hnode *ht = key->ht_up; | |
357 | ||
358 | if (ht) { | |
359 | for (kp = &ht->ht[TC_U32_HASH(key->handle)]; *kp; kp = &(*kp)->next) { | |
360 | if (*kp == key) { | |
361 | tcf_tree_lock(tp); | |
362 | *kp = key->next; | |
363 | tcf_tree_unlock(tp); | |
364 | ||
365 | u32_destroy_key(tp, key); | |
366 | return 0; | |
367 | } | |
368 | } | |
369 | } | |
547b792c | 370 | WARN_ON(1); |
1da177e4 LT |
371 | return 0; |
372 | } | |
373 | ||
374 | static void u32_clear_hnode(struct tcf_proto *tp, struct tc_u_hnode *ht) | |
375 | { | |
376 | struct tc_u_knode *n; | |
cc7ec456 | 377 | unsigned int h; |
1da177e4 | 378 | |
cc7ec456 | 379 | for (h = 0; h <= ht->divisor; h++) { |
1da177e4 LT |
380 | while ((n = ht->ht[h]) != NULL) { |
381 | ht->ht[h] = n->next; | |
382 | ||
383 | u32_destroy_key(tp, n); | |
384 | } | |
385 | } | |
386 | } | |
387 | ||
388 | static int u32_destroy_hnode(struct tcf_proto *tp, struct tc_u_hnode *ht) | |
389 | { | |
390 | struct tc_u_common *tp_c = tp->data; | |
391 | struct tc_u_hnode **hn; | |
392 | ||
547b792c | 393 | WARN_ON(ht->refcnt); |
1da177e4 LT |
394 | |
395 | u32_clear_hnode(tp, ht); | |
396 | ||
397 | for (hn = &tp_c->hlist; *hn; hn = &(*hn)->next) { | |
398 | if (*hn == ht) { | |
399 | *hn = ht->next; | |
400 | kfree(ht); | |
401 | return 0; | |
402 | } | |
403 | } | |
404 | ||
547b792c | 405 | WARN_ON(1); |
1da177e4 LT |
406 | return -ENOENT; |
407 | } | |
408 | ||
409 | static void u32_destroy(struct tcf_proto *tp) | |
410 | { | |
411 | struct tc_u_common *tp_c = tp->data; | |
47a1a1d4 | 412 | struct tc_u_hnode *root_ht = tp->root; |
1da177e4 | 413 | |
547b792c | 414 | WARN_ON(root_ht == NULL); |
1da177e4 LT |
415 | |
416 | if (root_ht && --root_ht->refcnt == 0) | |
417 | u32_destroy_hnode(tp, root_ht); | |
418 | ||
419 | if (--tp_c->refcnt == 0) { | |
420 | struct tc_u_hnode *ht; | |
1da177e4 | 421 | |
72b25a91 | 422 | tp->q->u32_node = NULL; |
1da177e4 | 423 | |
e56cfad1 JP |
424 | for (ht = tp_c->hlist; ht; ht = ht->next) { |
425 | ht->refcnt--; | |
1da177e4 | 426 | u32_clear_hnode(tp, ht); |
e56cfad1 | 427 | } |
1da177e4 LT |
428 | |
429 | while ((ht = tp_c->hlist) != NULL) { | |
430 | tp_c->hlist = ht->next; | |
431 | ||
547b792c | 432 | WARN_ON(ht->refcnt != 0); |
1da177e4 LT |
433 | |
434 | kfree(ht); | |
3ff50b79 | 435 | } |
1da177e4 LT |
436 | |
437 | kfree(tp_c); | |
438 | } | |
439 | ||
440 | tp->data = NULL; | |
441 | } | |
442 | ||
443 | static int u32_delete(struct tcf_proto *tp, unsigned long arg) | |
444 | { | |
cc7ec456 | 445 | struct tc_u_hnode *ht = (struct tc_u_hnode *)arg; |
1da177e4 LT |
446 | |
447 | if (ht == NULL) | |
448 | return 0; | |
449 | ||
450 | if (TC_U32_KEY(ht->handle)) | |
cc7ec456 | 451 | return u32_delete_key(tp, (struct tc_u_knode *)ht); |
1da177e4 LT |
452 | |
453 | if (tp->root == ht) | |
454 | return -EINVAL; | |
455 | ||
e56cfad1 JP |
456 | if (ht->refcnt == 1) { |
457 | ht->refcnt--; | |
1da177e4 | 458 | u32_destroy_hnode(tp, ht); |
e56cfad1 JP |
459 | } else { |
460 | return -EBUSY; | |
461 | } | |
1da177e4 LT |
462 | |
463 | return 0; | |
464 | } | |
465 | ||
7801db8a | 466 | #define NR_U32_NODE (1<<12) |
1da177e4 LT |
467 | static u32 gen_new_kid(struct tc_u_hnode *ht, u32 handle) |
468 | { | |
469 | struct tc_u_knode *n; | |
7801db8a CW |
470 | unsigned long i; |
471 | unsigned long *bitmap = kzalloc(BITS_TO_LONGS(NR_U32_NODE) * sizeof(unsigned long), | |
472 | GFP_KERNEL); | |
473 | if (!bitmap) | |
474 | return handle | 0xFFF; | |
1da177e4 | 475 | |
cc7ec456 | 476 | for (n = ht->ht[TC_U32_HASH(handle)]; n; n = n->next) |
7801db8a | 477 | set_bit(TC_U32_NODE(n->handle), bitmap); |
1da177e4 | 478 | |
7801db8a CW |
479 | i = find_next_zero_bit(bitmap, NR_U32_NODE, 0x800); |
480 | if (i >= NR_U32_NODE) | |
481 | i = find_next_zero_bit(bitmap, NR_U32_NODE, 1); | |
482 | ||
483 | kfree(bitmap); | |
484 | return handle | (i >= NR_U32_NODE ? 0xFFF : i); | |
1da177e4 LT |
485 | } |
486 | ||
6fa8c014 PM |
487 | static const struct nla_policy u32_policy[TCA_U32_MAX + 1] = { |
488 | [TCA_U32_CLASSID] = { .type = NLA_U32 }, | |
489 | [TCA_U32_HASH] = { .type = NLA_U32 }, | |
490 | [TCA_U32_LINK] = { .type = NLA_U32 }, | |
491 | [TCA_U32_DIVISOR] = { .type = NLA_U32 }, | |
492 | [TCA_U32_SEL] = { .len = sizeof(struct tc_u32_sel) }, | |
493 | [TCA_U32_INDEV] = { .type = NLA_STRING, .len = IFNAMSIZ }, | |
494 | [TCA_U32_MARK] = { .len = sizeof(struct tc_u32_mark) }, | |
495 | }; | |
496 | ||
c1b52739 BL |
497 | static int u32_set_parms(struct net *net, struct tcf_proto *tp, |
498 | unsigned long base, struct tc_u_hnode *ht, | |
add93b61 | 499 | struct tc_u_knode *n, struct nlattr **tb, |
2f7ef2f8 | 500 | struct nlattr *est, bool ovr) |
1da177e4 LT |
501 | { |
502 | int err; | |
503 | struct tcf_exts e; | |
504 | ||
5da57f42 | 505 | tcf_exts_init(&e, TCA_U32_ACT, TCA_U32_POLICE); |
2f7ef2f8 | 506 | err = tcf_exts_validate(net, tp, tb, est, &e, ovr); |
1da177e4 LT |
507 | if (err < 0) |
508 | return err; | |
509 | ||
510 | err = -EINVAL; | |
add93b61 | 511 | if (tb[TCA_U32_LINK]) { |
1587bac4 | 512 | u32 handle = nla_get_u32(tb[TCA_U32_LINK]); |
47a1a1d4 | 513 | struct tc_u_hnode *ht_down = NULL, *ht_old; |
1da177e4 LT |
514 | |
515 | if (TC_U32_KEY(handle)) | |
516 | goto errout; | |
517 | ||
518 | if (handle) { | |
519 | ht_down = u32_lookup_ht(ht->tp_c, handle); | |
520 | ||
521 | if (ht_down == NULL) | |
522 | goto errout; | |
523 | ht_down->refcnt++; | |
524 | } | |
525 | ||
526 | tcf_tree_lock(tp); | |
47a1a1d4 PM |
527 | ht_old = n->ht_down; |
528 | n->ht_down = ht_down; | |
1da177e4 LT |
529 | tcf_tree_unlock(tp); |
530 | ||
47a1a1d4 PM |
531 | if (ht_old) |
532 | ht_old->refcnt--; | |
1da177e4 | 533 | } |
add93b61 | 534 | if (tb[TCA_U32_CLASSID]) { |
1587bac4 | 535 | n->res.classid = nla_get_u32(tb[TCA_U32_CLASSID]); |
1da177e4 LT |
536 | tcf_bind_filter(tp, &n->res, base); |
537 | } | |
538 | ||
539 | #ifdef CONFIG_NET_CLS_IND | |
add93b61 | 540 | if (tb[TCA_U32_INDEV]) { |
2519a602 WC |
541 | int ret; |
542 | ret = tcf_change_indev(net, tb[TCA_U32_INDEV]); | |
543 | if (ret < 0) | |
1da177e4 | 544 | goto errout; |
2519a602 | 545 | n->ifindex = ret; |
1da177e4 LT |
546 | } |
547 | #endif | |
548 | tcf_exts_change(tp, &n->exts, &e); | |
549 | ||
550 | return 0; | |
551 | errout: | |
552 | tcf_exts_destroy(tp, &e); | |
553 | return err; | |
554 | } | |
555 | ||
c1b52739 | 556 | static int u32_change(struct net *net, struct sk_buff *in_skb, |
af4c6641 | 557 | struct tcf_proto *tp, unsigned long base, u32 handle, |
add93b61 | 558 | struct nlattr **tca, |
2f7ef2f8 | 559 | unsigned long *arg, bool ovr) |
1da177e4 LT |
560 | { |
561 | struct tc_u_common *tp_c = tp->data; | |
562 | struct tc_u_hnode *ht; | |
563 | struct tc_u_knode *n; | |
564 | struct tc_u32_sel *s; | |
add93b61 PM |
565 | struct nlattr *opt = tca[TCA_OPTIONS]; |
566 | struct nlattr *tb[TCA_U32_MAX + 1]; | |
1da177e4 LT |
567 | u32 htid; |
568 | int err; | |
f4f64050 JF |
569 | #ifdef CONFIG_CLS_U32_PERF |
570 | size_t size; | |
571 | #endif | |
1da177e4 LT |
572 | |
573 | if (opt == NULL) | |
574 | return handle ? -EINVAL : 0; | |
575 | ||
6fa8c014 | 576 | err = nla_parse_nested(tb, TCA_U32_MAX, opt, u32_policy); |
cee63723 PM |
577 | if (err < 0) |
578 | return err; | |
1da177e4 | 579 | |
cc7ec456 ED |
580 | n = (struct tc_u_knode *)*arg; |
581 | if (n) { | |
1da177e4 LT |
582 | if (TC_U32_KEY(n->handle) == 0) |
583 | return -EINVAL; | |
584 | ||
c1b52739 | 585 | return u32_set_parms(net, tp, base, n->ht_up, n, tb, |
2f7ef2f8 | 586 | tca[TCA_RATE], ovr); |
1da177e4 LT |
587 | } |
588 | ||
add93b61 | 589 | if (tb[TCA_U32_DIVISOR]) { |
cc7ec456 | 590 | unsigned int divisor = nla_get_u32(tb[TCA_U32_DIVISOR]); |
1da177e4 LT |
591 | |
592 | if (--divisor > 0x100) | |
593 | return -EINVAL; | |
594 | if (TC_U32_KEY(handle)) | |
595 | return -EINVAL; | |
596 | if (handle == 0) { | |
597 | handle = gen_new_htid(tp->data); | |
598 | if (handle == 0) | |
599 | return -ENOMEM; | |
600 | } | |
cc7ec456 | 601 | ht = kzalloc(sizeof(*ht) + divisor*sizeof(void *), GFP_KERNEL); |
1da177e4 LT |
602 | if (ht == NULL) |
603 | return -ENOBUFS; | |
1da177e4 | 604 | ht->tp_c = tp_c; |
e56cfad1 | 605 | ht->refcnt = 1; |
1da177e4 LT |
606 | ht->divisor = divisor; |
607 | ht->handle = handle; | |
608 | ht->prio = tp->prio; | |
609 | ht->next = tp_c->hlist; | |
610 | tp_c->hlist = ht; | |
611 | *arg = (unsigned long)ht; | |
612 | return 0; | |
613 | } | |
614 | ||
add93b61 | 615 | if (tb[TCA_U32_HASH]) { |
1587bac4 | 616 | htid = nla_get_u32(tb[TCA_U32_HASH]); |
1da177e4 LT |
617 | if (TC_U32_HTID(htid) == TC_U32_ROOT) { |
618 | ht = tp->root; | |
619 | htid = ht->handle; | |
620 | } else { | |
621 | ht = u32_lookup_ht(tp->data, TC_U32_HTID(htid)); | |
622 | if (ht == NULL) | |
623 | return -EINVAL; | |
624 | } | |
625 | } else { | |
626 | ht = tp->root; | |
627 | htid = ht->handle; | |
628 | } | |
629 | ||
630 | if (ht->divisor < TC_U32_HASH(htid)) | |
631 | return -EINVAL; | |
632 | ||
633 | if (handle) { | |
634 | if (TC_U32_HTID(handle) && TC_U32_HTID(handle^htid)) | |
635 | return -EINVAL; | |
636 | handle = htid | TC_U32_NODE(handle); | |
637 | } else | |
638 | handle = gen_new_kid(ht, htid); | |
639 | ||
6fa8c014 | 640 | if (tb[TCA_U32_SEL] == NULL) |
1da177e4 LT |
641 | return -EINVAL; |
642 | ||
add93b61 | 643 | s = nla_data(tb[TCA_U32_SEL]); |
1da177e4 | 644 | |
0da974f4 | 645 | n = kzalloc(sizeof(*n) + s->nkeys*sizeof(struct tc_u32_key), GFP_KERNEL); |
1da177e4 LT |
646 | if (n == NULL) |
647 | return -ENOBUFS; | |
648 | ||
1da177e4 | 649 | #ifdef CONFIG_CLS_U32_PERF |
f4f64050 JF |
650 | size = sizeof(struct tc_u32_pcnt) + s->nkeys * sizeof(u64); |
651 | n->pf = __alloc_percpu(size, __alignof__(struct tc_u32_pcnt)); | |
652 | if (!n->pf) { | |
1da177e4 LT |
653 | kfree(n); |
654 | return -ENOBUFS; | |
655 | } | |
1da177e4 LT |
656 | #endif |
657 | ||
658 | memcpy(&n->sel, s, sizeof(*s) + s->nkeys*sizeof(struct tc_u32_key)); | |
659 | n->ht_up = ht; | |
660 | n->handle = handle; | |
b2268016 | 661 | n->fshift = s->hmask ? ffs(ntohl(s->hmask)) - 1 : 0; |
5da57f42 | 662 | tcf_exts_init(&n->exts, TCA_U32_ACT, TCA_U32_POLICE); |
1da177e4 LT |
663 | |
664 | #ifdef CONFIG_CLS_U32_MARK | |
f4f64050 JF |
665 | n->pcpu_success = alloc_percpu(u32); |
666 | ||
add93b61 | 667 | if (tb[TCA_U32_MARK]) { |
1da177e4 LT |
668 | struct tc_u32_mark *mark; |
669 | ||
add93b61 | 670 | mark = nla_data(tb[TCA_U32_MARK]); |
f4f64050 JF |
671 | n->val = mark->val; |
672 | n->mask = mark->mask; | |
1da177e4 LT |
673 | } |
674 | #endif | |
675 | ||
2f7ef2f8 | 676 | err = u32_set_parms(net, tp, base, ht, n, tb, tca[TCA_RATE], ovr); |
1da177e4 LT |
677 | if (err == 0) { |
678 | struct tc_u_knode **ins; | |
679 | for (ins = &ht->ht[TC_U32_HASH(handle)]; *ins; ins = &(*ins)->next) | |
680 | if (TC_U32_NODE(handle) < TC_U32_NODE((*ins)->handle)) | |
681 | break; | |
682 | ||
683 | n->next = *ins; | |
6f573214 | 684 | tcf_tree_lock(tp); |
1da177e4 | 685 | *ins = n; |
6f573214 | 686 | tcf_tree_unlock(tp); |
1da177e4 LT |
687 | |
688 | *arg = (unsigned long)n; | |
689 | return 0; | |
690 | } | |
691 | #ifdef CONFIG_CLS_U32_PERF | |
1ae39a43 | 692 | kfree(n->pf); |
1da177e4 LT |
693 | #endif |
694 | kfree(n); | |
695 | return err; | |
696 | } | |
697 | ||
698 | static void u32_walk(struct tcf_proto *tp, struct tcf_walker *arg) | |
699 | { | |
700 | struct tc_u_common *tp_c = tp->data; | |
701 | struct tc_u_hnode *ht; | |
702 | struct tc_u_knode *n; | |
cc7ec456 | 703 | unsigned int h; |
1da177e4 LT |
704 | |
705 | if (arg->stop) | |
706 | return; | |
707 | ||
708 | for (ht = tp_c->hlist; ht; ht = ht->next) { | |
709 | if (ht->prio != tp->prio) | |
710 | continue; | |
711 | if (arg->count >= arg->skip) { | |
712 | if (arg->fn(tp, (unsigned long)ht, arg) < 0) { | |
713 | arg->stop = 1; | |
714 | return; | |
715 | } | |
716 | } | |
717 | arg->count++; | |
718 | for (h = 0; h <= ht->divisor; h++) { | |
719 | for (n = ht->ht[h]; n; n = n->next) { | |
720 | if (arg->count < arg->skip) { | |
721 | arg->count++; | |
722 | continue; | |
723 | } | |
724 | if (arg->fn(tp, (unsigned long)n, arg) < 0) { | |
725 | arg->stop = 1; | |
726 | return; | |
727 | } | |
728 | arg->count++; | |
729 | } | |
730 | } | |
731 | } | |
732 | } | |
733 | ||
832d1d5b | 734 | static int u32_dump(struct net *net, struct tcf_proto *tp, unsigned long fh, |
1da177e4 LT |
735 | struct sk_buff *skb, struct tcmsg *t) |
736 | { | |
cc7ec456 | 737 | struct tc_u_knode *n = (struct tc_u_knode *)fh; |
4b3550ef | 738 | struct nlattr *nest; |
1da177e4 LT |
739 | |
740 | if (n == NULL) | |
741 | return skb->len; | |
742 | ||
743 | t->tcm_handle = n->handle; | |
744 | ||
4b3550ef PM |
745 | nest = nla_nest_start(skb, TCA_OPTIONS); |
746 | if (nest == NULL) | |
747 | goto nla_put_failure; | |
1da177e4 LT |
748 | |
749 | if (TC_U32_KEY(n->handle) == 0) { | |
cc7ec456 ED |
750 | struct tc_u_hnode *ht = (struct tc_u_hnode *)fh; |
751 | u32 divisor = ht->divisor + 1; | |
752 | ||
1b34ec43 DM |
753 | if (nla_put_u32(skb, TCA_U32_DIVISOR, divisor)) |
754 | goto nla_put_failure; | |
1da177e4 | 755 | } else { |
f4f64050 JF |
756 | #ifdef CONFIG_CLS_U32_PERF |
757 | struct tc_u32_pcnt *gpf; | |
758 | #endif | |
759 | int cpu; | |
760 | ||
1b34ec43 DM |
761 | if (nla_put(skb, TCA_U32_SEL, |
762 | sizeof(n->sel) + n->sel.nkeys*sizeof(struct tc_u32_key), | |
763 | &n->sel)) | |
764 | goto nla_put_failure; | |
1da177e4 LT |
765 | if (n->ht_up) { |
766 | u32 htid = n->handle & 0xFFFFF000; | |
1b34ec43 DM |
767 | if (nla_put_u32(skb, TCA_U32_HASH, htid)) |
768 | goto nla_put_failure; | |
1da177e4 | 769 | } |
1b34ec43 DM |
770 | if (n->res.classid && |
771 | nla_put_u32(skb, TCA_U32_CLASSID, n->res.classid)) | |
772 | goto nla_put_failure; | |
773 | if (n->ht_down && | |
774 | nla_put_u32(skb, TCA_U32_LINK, n->ht_down->handle)) | |
775 | goto nla_put_failure; | |
1da177e4 LT |
776 | |
777 | #ifdef CONFIG_CLS_U32_MARK | |
f4f64050 JF |
778 | if ((n->val || n->mask)) { |
779 | struct tc_u32_mark mark = {.val = n->val, | |
780 | .mask = n->mask, | |
781 | .success = 0}; | |
782 | ||
783 | for_each_possible_cpu(cpu) { | |
784 | __u32 cnt = *per_cpu_ptr(n->pcpu_success, cpu); | |
785 | ||
786 | mark.success += cnt; | |
787 | } | |
788 | ||
789 | if (nla_put(skb, TCA_U32_MARK, sizeof(mark), &mark)) | |
790 | goto nla_put_failure; | |
791 | } | |
1da177e4 LT |
792 | #endif |
793 | ||
5da57f42 | 794 | if (tcf_exts_dump(skb, &n->exts) < 0) |
add93b61 | 795 | goto nla_put_failure; |
1da177e4 LT |
796 | |
797 | #ifdef CONFIG_NET_CLS_IND | |
2519a602 WC |
798 | if (n->ifindex) { |
799 | struct net_device *dev; | |
800 | dev = __dev_get_by_index(net, n->ifindex); | |
801 | if (dev && nla_put_string(skb, TCA_U32_INDEV, dev->name)) | |
802 | goto nla_put_failure; | |
803 | } | |
1da177e4 LT |
804 | #endif |
805 | #ifdef CONFIG_CLS_U32_PERF | |
f4f64050 JF |
806 | gpf = kzalloc(sizeof(struct tc_u32_pcnt) + |
807 | n->sel.nkeys * sizeof(u64), | |
808 | GFP_KERNEL); | |
809 | if (!gpf) | |
810 | goto nla_put_failure; | |
811 | ||
812 | for_each_possible_cpu(cpu) { | |
813 | int i; | |
814 | struct tc_u32_pcnt *pf = per_cpu_ptr(n->pf, cpu); | |
815 | ||
816 | gpf->rcnt += pf->rcnt; | |
817 | gpf->rhit += pf->rhit; | |
818 | for (i = 0; i < n->sel.nkeys; i++) | |
819 | gpf->kcnts[i] += pf->kcnts[i]; | |
820 | } | |
821 | ||
1b34ec43 DM |
822 | if (nla_put(skb, TCA_U32_PCNT, |
823 | sizeof(struct tc_u32_pcnt) + n->sel.nkeys*sizeof(u64), | |
f4f64050 JF |
824 | gpf)) { |
825 | kfree(gpf); | |
1b34ec43 | 826 | goto nla_put_failure; |
f4f64050 JF |
827 | } |
828 | kfree(gpf); | |
1da177e4 LT |
829 | #endif |
830 | } | |
831 | ||
4b3550ef PM |
832 | nla_nest_end(skb, nest); |
833 | ||
1da177e4 | 834 | if (TC_U32_KEY(n->handle)) |
5da57f42 | 835 | if (tcf_exts_dump_stats(skb, &n->exts) < 0) |
add93b61 | 836 | goto nla_put_failure; |
1da177e4 LT |
837 | return skb->len; |
838 | ||
add93b61 | 839 | nla_put_failure: |
4b3550ef | 840 | nla_nest_cancel(skb, nest); |
1da177e4 LT |
841 | return -1; |
842 | } | |
843 | ||
2eb9d75c | 844 | static struct tcf_proto_ops cls_u32_ops __read_mostly = { |
1da177e4 LT |
845 | .kind = "u32", |
846 | .classify = u32_classify, | |
847 | .init = u32_init, | |
848 | .destroy = u32_destroy, | |
849 | .get = u32_get, | |
850 | .put = u32_put, | |
851 | .change = u32_change, | |
852 | .delete = u32_delete, | |
853 | .walk = u32_walk, | |
854 | .dump = u32_dump, | |
855 | .owner = THIS_MODULE, | |
856 | }; | |
857 | ||
858 | static int __init init_u32(void) | |
859 | { | |
6ff9c364 | 860 | pr_info("u32 classifier\n"); |
1da177e4 | 861 | #ifdef CONFIG_CLS_U32_PERF |
6ff9c364 | 862 | pr_info(" Performance counters on\n"); |
1da177e4 | 863 | #endif |
1da177e4 | 864 | #ifdef CONFIG_NET_CLS_IND |
6ff9c364 | 865 | pr_info(" input device check on\n"); |
1da177e4 LT |
866 | #endif |
867 | #ifdef CONFIG_NET_CLS_ACT | |
6ff9c364 | 868 | pr_info(" Actions configured\n"); |
1da177e4 LT |
869 | #endif |
870 | return register_tcf_proto_ops(&cls_u32_ops); | |
871 | } | |
872 | ||
10297b99 | 873 | static void __exit exit_u32(void) |
1da177e4 LT |
874 | { |
875 | unregister_tcf_proto_ops(&cls_u32_ops); | |
876 | } | |
877 | ||
878 | module_init(init_u32) | |
879 | module_exit(exit_u32) | |
880 | MODULE_LICENSE("GPL"); |