net/core/flow_dissector.c

   1 #include <linux/kernel.h>
   2 #include <linux/skbuff.h>
   3 #include <linux/export.h>
   4 #include <linux/ip.h>
   5 #include <linux/ipv6.h>
   6 #include <linux/if_vlan.h>
   7 #include <net/ip.h>
   8 #include <net/ipv6.h>
   9 #include <linux/igmp.h>
  10 #include <linux/icmp.h>
  11 #include <linux/sctp.h>
  12 #include <linux/dccp.h>
  13 #include <linux/if_tunnel.h>
  14 #include <linux/if_pppox.h>
  15 #include <linux/ppp_defs.h>
  16 #include <linux/stddef.h>
  17 #include <linux/if_ether.h>
  18 #include <net/flow_dissector.h>
  19 #include <scsi/fc/fc_fcoe.h>
  20
  21 static bool skb_flow_dissector_uses_key(struct flow_dissector *flow_dissector,
  22                                         enum flow_dissector_key_id key_id)
  23 {
  24         return flow_dissector->used_keys & (1 << key_id);
  25 }
  26
  27 static void skb_flow_dissector_set_key(struct flow_dissector *flow_dissector,
  28                                        enum flow_dissector_key_id key_id)
  29 {
  30         flow_dissector->used_keys |= (1 << key_id);
  31 }
  32
  33 static void *skb_flow_dissector_target(struct flow_dissector *flow_dissector,
  34                                        enum flow_dissector_key_id key_id,
  35                                        void *target_container)
  36 {
  37         return ((char *) target_container) + flow_dissector->offset[key_id];
  38 }
  39
  40 void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
  41                              const struct flow_dissector_key *key,
  42                              unsigned int key_count)
  43 {
  44         unsigned int i;
  45
  46         memset(flow_dissector, 0, sizeof(*flow_dissector));
  47
  48         for (i = 0; i < key_count; i++, key++) {
  49                 /* User should make sure that every key target offset is withing
  50                  * boundaries of unsigned short.
  51                  */
  52                 BUG_ON(key->offset > USHRT_MAX);
  53                 BUG_ON(skb_flow_dissector_uses_key(flow_dissector,
  54                                                    key->key_id));
  55
  56                 skb_flow_dissector_set_key(flow_dissector, key->key_id);
  57                 flow_dissector->offset[key->key_id] = key->offset;
  58         }
  59
  60         /* Ensure that the dissector always includes control and basic key.
  61          * That way we are able to avoid handling lack of these in fast path.
  62          */
  63         BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
  64                                             FLOW_DISSECTOR_KEY_CONTROL));
  65         BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
  66                                             FLOW_DISSECTOR_KEY_BASIC));
  67 }
  68 EXPORT_SYMBOL(skb_flow_dissector_init);
  69
  70 /**
  71  * __skb_flow_get_ports - extract the upper layer ports and return them
  72  * @skb: sk_buff to extract the ports from
  73  * @thoff: transport header offset
  74  * @ip_proto: protocol for which to get port offset
  75  * @data: raw buffer pointer to the packet, if NULL use skb->data
  76  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
  77  *
  78  * The function will try to retrieve the ports at offset thoff + poff where poff
  79  * is the protocol port offset returned from proto_ports_offset
  80  */
  81 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
  82                             void *data, int hlen)
  83 {
  84         int poff = proto_ports_offset(ip_proto);
  85
  86         if (!data) {
  87                 data = skb->data;
  88                 hlen = skb_headlen(skb);
  89         }
  90
  91         if (poff >= 0) {
  92                 __be32 *ports, _ports;
  93
  94                 ports = __skb_header_pointer(skb, thoff + poff,
  95                                              sizeof(_ports), data, hlen, &_ports);
  96                 if (ports)
  97                         return *ports;
  98         }
  99
 100         return 0;
 101 }
 102 EXPORT_SYMBOL(__skb_flow_get_ports);
 103
 104 /**
 105  * __skb_flow_dissect - extract the flow_keys struct and return it
 106  * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
 107  * @flow_dissector: list of keys to dissect
 108  * @target_container: target structure to put dissected values into
 109  * @data: raw buffer pointer to the packet, if NULL use skb->data
 110  * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
 111  * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
 112  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
 113  *
 114  * The function will try to retrieve individual keys into target specified
 115  * by flow_dissector from either the skbuff or a raw buffer specified by the
 116  * rest parameters.
 117  *
 118  * Caller must take care of zeroing target container memory.
 119  */
 120 bool __skb_flow_dissect(const struct sk_buff *skb,
 121                         struct flow_dissector *flow_dissector,
 122                         void *target_container,
 123                         void *data, __be16 proto, int nhoff, int hlen)
 124 {
 125         struct flow_dissector_key_control *key_control;
 126         struct flow_dissector_key_basic *key_basic;
 127         struct flow_dissector_key_addrs *key_addrs;
 128         struct flow_dissector_key_ports *key_ports;
 129         u8 ip_proto;
 130
 131         if (!data) {
 132                 data = skb->data;
 133                 proto = skb->protocol;
 134                 nhoff = skb_network_offset(skb);
 135                 hlen = skb_headlen(skb);
 136         }
 137
 138         /* It is ensured by skb_flow_dissector_init() that control key will
 139          * be always present.
 140          */
 141         key_control = skb_flow_dissector_target(flow_dissector,
 142                                                 FLOW_DISSECTOR_KEY_CONTROL,
 143                                                 target_container);
 144
 145         /* It is ensured by skb_flow_dissector_init() that basic key will
 146          * be always present.
 147          */
 148         key_basic = skb_flow_dissector_target(flow_dissector,
 149                                               FLOW_DISSECTOR_KEY_BASIC,
 150                                               target_container);
 151
 152         if (skb_flow_dissector_uses_key(flow_dissector,
 153                                         FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
 154                 struct ethhdr *eth = eth_hdr(skb);
 155                 struct flow_dissector_key_eth_addrs *key_eth_addrs;
 156
 157                 key_eth_addrs = skb_flow_dissector_target(flow_dissector,
 158                                                           FLOW_DISSECTOR_KEY_ETH_ADDRS,
 159                                                           target_container);
 160                 memcpy(key_eth_addrs, &eth->h_dest, sizeof(*key_eth_addrs));
 161         }
 162
 163 again:
 164         switch (proto) {
 165         case htons(ETH_P_IP): {
 166                 const struct iphdr *iph;
 167                 struct iphdr _iph;
 168 ip:
 169                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
 170                 if (!iph || iph->ihl < 5)
 171                         return false;
 172                 nhoff += iph->ihl * 4;
 173
 174                 ip_proto = iph->protocol;
 175                 if (ip_is_fragment(iph))
 176                         ip_proto = 0;
 177
 178                 if (!skb_flow_dissector_uses_key(flow_dissector,
 179                                                  FLOW_DISSECTOR_KEY_IPV4_ADDRS))
 180                         break;
 181                 key_addrs = skb_flow_dissector_target(flow_dissector,
 182                                                       FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 183                                                       target_container);
 184                 memcpy(key_addrs, &iph->saddr, sizeof(*key_addrs));
 185                 break;
 186         }
 187         case htons(ETH_P_IPV6): {
 188                 const struct ipv6hdr *iph;
 189                 struct ipv6hdr _iph;
 190                 __be32 flow_label;
 191
 192 ipv6:
 193                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
 194                 if (!iph)
 195                         return false;
 196
 197                 ip_proto = iph->nexthdr;
 198                 nhoff += sizeof(struct ipv6hdr);
 199
 200                 if (skb_flow_dissector_uses_key(flow_dissector,
 201                                                 FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
 202                         key_addrs = skb_flow_dissector_target(flow_dissector,
 203                                                               FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
 204                                                               target_container);
 205
 206                         key_addrs->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
 207                         key_addrs->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
 208                         goto flow_label;
 209                 }
 210                 if (skb_flow_dissector_uses_key(flow_dissector,
 211                                                 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 212                         struct flow_dissector_key_ipv6_addrs *key_ipv6_addrs;
 213
 214                         key_ipv6_addrs = skb_flow_dissector_target(flow_dissector,
 215                                                                    FLOW_DISSECTOR_KEY_IPV6_ADDRS,
 216                                                                    target_container);
 217
 218                         memcpy(key_ipv6_addrs, &iph->saddr, sizeof(*key_ipv6_addrs));
 219                         goto flow_label;
 220                 }
 221                 break;
 222 flow_label:
 223                 flow_label = ip6_flowlabel(iph);
 224                 if (flow_label) {
 225                         /* Awesome, IPv6 packet has a flow label so we can
 226                          * use that to represent the ports without any
 227                          * further dissection.
 228                          */
 229
 230                         key_basic->n_proto = proto;
 231                         key_basic->ip_proto = ip_proto;
 232                         key_control->thoff = (u16)nhoff;
 233
 234                         if (skb_flow_dissector_uses_key(flow_dissector,
 235                                                         FLOW_DISSECTOR_KEY_PORTS)) {
 236                                 key_ports = skb_flow_dissector_target(flow_dissector,
 237                                                                       FLOW_DISSECTOR_KEY_PORTS,
 238                                                                       target_container);
 239                                 key_ports->ports = flow_label;
 240                         }
 241
 242                         return true;
 243                 }
 244
 245                 break;
 246         }
 247         case htons(ETH_P_8021AD):
 248         case htons(ETH_P_8021Q): {
 249                 const struct vlan_hdr *vlan;
 250                 struct vlan_hdr _vlan;
 251
 252                 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan);
 253                 if (!vlan)
 254                         return false;
 255
 256                 proto = vlan->h_vlan_encapsulated_proto;
 257                 nhoff += sizeof(*vlan);
 258                 goto again;
 259         }
 260         case htons(ETH_P_PPP_SES): {
 261                 struct {
 262                         struct pppoe_hdr hdr;
 263                         __be16 proto;
 264                 } *hdr, _hdr;
 265                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 266                 if (!hdr)
 267                         return false;
 268                 proto = hdr->proto;
 269                 nhoff += PPPOE_SES_HLEN;
 270                 switch (proto) {
 271                 case htons(PPP_IP):
 272                         goto ip;
 273                 case htons(PPP_IPV6):
 274                         goto ipv6;
 275                 default:
 276                         return false;
 277                 }
 278         }
 279         case htons(ETH_P_TIPC): {
 280                 struct {
 281                         __be32 pre[3];
 282                         __be32 srcnode;
 283                 } *hdr, _hdr;
 284                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 285                 if (!hdr)
 286                         return false;
 287                 key_basic->n_proto = proto;
 288                 key_control->thoff = (u16)nhoff;
 289
 290                 if (skb_flow_dissector_uses_key(flow_dissector,
 291                                                 FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
 292                         key_addrs = skb_flow_dissector_target(flow_dissector,
 293                                                               FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
 294                                                               target_container);
 295                         key_addrs->src = hdr->srcnode;
 296                         key_addrs->dst = 0;
 297                 }
 298                 return true;
 299         }
 300         case htons(ETH_P_FCOE):
 301                 key_control->thoff = (u16)(nhoff + FCOE_HEADER_LEN);
 302                 /* fall through */
 303         default:
 304                 return false;
 305         }
 306
 307         switch (ip_proto) {
 308         case IPPROTO_GRE: {
 309                 struct gre_hdr {
 310                         __be16 flags;
 311                         __be16 proto;
 312                 } *hdr, _hdr;
 313
 314                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 315                 if (!hdr)
 316                         return false;
 317                 /*
 318                  * Only look inside GRE if version zero and no
 319                  * routing
 320                  */
 321                 if (hdr->flags & (GRE_VERSION | GRE_ROUTING))
 322                         break;
 323
 324                 proto = hdr->proto;
 325                 nhoff += 4;
 326                 if (hdr->flags & GRE_CSUM)
 327                         nhoff += 4;
 328                 if (hdr->flags & GRE_KEY)
 329                         nhoff += 4;
 330                 if (hdr->flags & GRE_SEQ)
 331                         nhoff += 4;
 332                 if (proto == htons(ETH_P_TEB)) {
 333                         const struct ethhdr *eth;
 334                         struct ethhdr _eth;
 335
 336                         eth = __skb_header_pointer(skb, nhoff,
 337                                                    sizeof(_eth),
 338                                                    data, hlen, &_eth);
 339                         if (!eth)
 340                                 return false;
 341                         proto = eth->h_proto;
 342                         nhoff += sizeof(*eth);
 343                 }
 344                 goto again;
 345         }
 346         case IPPROTO_IPIP:
 347                 proto = htons(ETH_P_IP);
 348                 goto ip;
 349         case IPPROTO_IPV6:
 350                 proto = htons(ETH_P_IPV6);
 351                 goto ipv6;
 352         default:
 353                 break;
 354         }
 355
 356         key_basic->n_proto = proto;
 357         key_basic->ip_proto = ip_proto;
 358         key_control->thoff = (u16)nhoff;
 359
 360         if (skb_flow_dissector_uses_key(flow_dissector,
 361                                         FLOW_DISSECTOR_KEY_PORTS)) {
 362                 key_ports = skb_flow_dissector_target(flow_dissector,
 363                                                       FLOW_DISSECTOR_KEY_PORTS,
 364                                                       target_container);
 365                 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
 366                                                         data, hlen);
 367         }
 368
 369         return true;
 370 }
 371 EXPORT_SYMBOL(__skb_flow_dissect);
 372
 373 static u32 hashrnd __read_mostly;
 374 static __always_inline void __flow_hash_secret_init(void)
 375 {
 376         net_get_random_once(&hashrnd, sizeof(hashrnd));
 377 }
 378
 379 static __always_inline u32 __flow_hash_words(u32 *words, u32 length, u32 keyval)
 380 {
 381         return jhash2(words, length, keyval);
 382 }
 383
 384 static inline void *flow_keys_hash_start(struct flow_keys *flow)
 385 {
 386         BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % sizeof(u32));
 387         return (void *)flow + FLOW_KEYS_HASH_OFFSET;
 388 }
 389
 390 static inline size_t flow_keys_hash_length(struct flow_keys *flow)
 391 {
 392         BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32));
 393         return (sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) / sizeof(u32);
 394 }
 395
 396 static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
 397 {
 398         u32 hash;
 399
 400         /* get a consistent hash (same value on both flow directions) */
 401         if (((__force u32)keys->addrs.dst < (__force u32)keys->addrs.src) ||
 402             (((__force u32)keys->addrs.dst == (__force u32)keys->addrs.src) &&
 403              ((__force u16)keys->ports.dst < (__force u16)keys->ports.src))) {
 404                 swap(keys->addrs.dst, keys->addrs.src);
 405                 swap(keys->ports.src, keys->ports.dst);
 406         }
 407
 408         hash = __flow_hash_words((u32 *)flow_keys_hash_start(keys),
 409                                  flow_keys_hash_length(keys), keyval);
 410         if (!hash)
 411                 hash = 1;
 412
 413         return hash;
 414 }
 415
 416 u32 flow_hash_from_keys(struct flow_keys *keys)
 417 {
 418         __flow_hash_secret_init();
 419         return __flow_hash_from_keys(keys, hashrnd);
 420 }
 421 EXPORT_SYMBOL(flow_hash_from_keys);
 422
 423 static inline u32 ___skb_get_hash(const struct sk_buff *skb,
 424                                   struct flow_keys *keys, u32 keyval)
 425 {
 426         if (!skb_flow_dissect_flow_keys(skb, keys))
 427                 return 0;
 428
 429         return __flow_hash_from_keys(keys, keyval);
 430 }
 431
 432 struct _flow_keys_digest_data {
 433         __be16  n_proto;
 434         u8      ip_proto;
 435         u8      padding;
 436         __be32  ports;
 437         __be32  src;
 438         __be32  dst;
 439 };
 440
 441 void make_flow_keys_digest(struct flow_keys_digest *digest,
 442                            const struct flow_keys *flow)
 443 {
 444         struct _flow_keys_digest_data *data =
 445             (struct _flow_keys_digest_data *)digest;
 446
 447         BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));
 448
 449         memset(digest, 0, sizeof(*digest));
 450
 451         data->n_proto = flow->basic.n_proto;
 452         data->ip_proto = flow->basic.ip_proto;
 453         data->ports = flow->ports.ports;
 454         data->src = flow->addrs.src;
 455         data->dst = flow->addrs.dst;
 456 }
 457 EXPORT_SYMBOL(make_flow_keys_digest);
 458
 459 /**
 460  * __skb_get_hash: calculate a flow hash
 461  * @skb: sk_buff to calculate flow hash from
 462  *
 463  * This function calculates a flow hash based on src/dst addresses
 464  * and src/dst port numbers.  Sets hash in skb to non-zero hash value
 465  * on success, zero indicates no valid hash.  Also, sets l4_hash in skb
 466  * if hash is a canonical 4-tuple hash over transport ports.
 467  */
 468 void __skb_get_hash(struct sk_buff *skb)
 469 {
 470         struct flow_keys keys;
 471         u32 hash;
 472
 473         __flow_hash_secret_init();
 474
 475         hash = ___skb_get_hash(skb, &keys, hashrnd);
 476         if (!hash)
 477                 return;
 478         if (keys.ports.ports)
 479                 skb->l4_hash = 1;
 480         skb->sw_hash = 1;
 481         skb->hash = hash;
 482 }
 483 EXPORT_SYMBOL(__skb_get_hash);
 484
 485 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb)
 486 {
 487         struct flow_keys keys;
 488
 489         return ___skb_get_hash(skb, &keys, perturb);
 490 }
 491 EXPORT_SYMBOL(skb_get_hash_perturb);
 492
 493 u32 __skb_get_poff(const struct sk_buff *skb, void *data,
 494                    const struct flow_keys *keys, int hlen)
 495 {
 496         u32 poff = keys->control.thoff;
 497
 498         switch (keys->basic.ip_proto) {
 499         case IPPROTO_TCP: {
 500                 /* access doff as u8 to avoid unaligned access */
 501                 const u8 *doff;
 502                 u8 _doff;
 503
 504                 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
 505                                             data, hlen, &_doff);
 506                 if (!doff)
 507                         return poff;
 508
 509                 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
 510                 break;
 511         }
 512         case IPPROTO_UDP:
 513         case IPPROTO_UDPLITE:
 514                 poff += sizeof(struct udphdr);
 515                 break;
 516         /* For the rest, we do not really care about header
 517          * extensions at this point for now.
 518          */
 519         case IPPROTO_ICMP:
 520                 poff += sizeof(struct icmphdr);
 521                 break;
 522         case IPPROTO_ICMPV6:
 523                 poff += sizeof(struct icmp6hdr);
 524                 break;
 525         case IPPROTO_IGMP:
 526                 poff += sizeof(struct igmphdr);
 527                 break;
 528         case IPPROTO_DCCP:
 529                 poff += sizeof(struct dccp_hdr);
 530                 break;
 531         case IPPROTO_SCTP:
 532                 poff += sizeof(struct sctphdr);
 533                 break;
 534         }
 535
 536         return poff;
 537 }
 538
 539 /**
 540  * skb_get_poff - get the offset to the payload
 541  * @skb: sk_buff to get the payload offset from
 542  *
 543  * The function will get the offset to the payload as far as it could
 544  * be dissected.  The main user is currently BPF, so that we can dynamically
 545  * truncate packets without needing to push actual payload to the user
 546  * space and can analyze headers only, instead.
 547  */
 548 u32 skb_get_poff(const struct sk_buff *skb)
 549 {
 550         struct flow_keys keys;
 551
 552         if (!skb_flow_dissect_flow_keys(skb, &keys))
 553                 return 0;
 554
 555         return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
 556 }
 557
 558 static const struct flow_dissector_key flow_keys_dissector_keys[] = {
 559         {
 560                 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
 561                 .offset = offsetof(struct flow_keys, control),
 562         },
 563         {
 564                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
 565                 .offset = offsetof(struct flow_keys, basic),
 566         },
 567         {
 568                 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 569                 .offset = offsetof(struct flow_keys, addrs),
 570         },
 571         {
 572                 .key_id = FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
 573                 .offset = offsetof(struct flow_keys, addrs),
 574         },
 575         {
 576                 .key_id = FLOW_DISSECTOR_KEY_PORTS,
 577                 .offset = offsetof(struct flow_keys, ports),
 578         },
 579 };
 580
 581 static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = {
 582         {
 583                 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
 584                 .offset = offsetof(struct flow_keys, control),
 585         },
 586         {
 587                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
 588                 .offset = offsetof(struct flow_keys, basic),
 589         },
 590 };
 591
 592 struct flow_dissector flow_keys_dissector __read_mostly;
 593 EXPORT_SYMBOL(flow_keys_dissector);
 594
 595 struct flow_dissector flow_keys_buf_dissector __read_mostly;
 596
 597 static int __init init_default_flow_dissectors(void)
 598 {
 599         skb_flow_dissector_init(&flow_keys_dissector,
 600                                 flow_keys_dissector_keys,
 601                                 ARRAY_SIZE(flow_keys_dissector_keys));
 602         skb_flow_dissector_init(&flow_keys_buf_dissector,
 603                                 flow_keys_buf_dissector_keys,
 604                                 ARRAY_SIZE(flow_keys_buf_dissector_keys));
 605         return 0;
 606 }
 607
 608 late_initcall_sync(init_default_flow_dissectors);