1 #include <linux/skbuff.h>
2 #include <linux/export.h>
4 #include <linux/ipv6.h>
5 #include <linux/if_vlan.h>
8 #include <linux/igmp.h>
9 #include <linux/icmp.h>
10 #include <linux/sctp.h>
11 #include <linux/dccp.h>
12 #include <linux/if_tunnel.h>
13 #include <linux/if_pppox.h>
14 #include <linux/ppp_defs.h>
15 #include <net/flow_keys.h>
16 #include <scsi/fc/fc_fcoe.h>
18 /* copy saddr & daddr, possibly using 64bit load/store
19 * Equivalent to : flow->src = iph->saddr;
20 * flow->dst = iph->daddr;
22 static void iph_to_flow_copy_addrs(struct flow_keys
*flow
, const struct iphdr
*iph
)
24 BUILD_BUG_ON(offsetof(typeof(*flow
), dst
) !=
25 offsetof(typeof(*flow
), src
) + sizeof(flow
->src
));
26 memcpy(&flow
->src
, &iph
->saddr
, sizeof(flow
->src
) + sizeof(flow
->dst
));
30 * __skb_flow_get_ports - extract the upper layer ports and return them
31 * @skb: sk_buff to extract the ports from
32 * @thoff: transport header offset
33 * @ip_proto: protocol for which to get port offset
34 * @data: raw buffer pointer to the packet, if NULL use skb->data
35 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
37 * The function will try to retrieve the ports at offset thoff + poff where poff
38 * is the protocol port offset returned from proto_ports_offset
40 __be32
__skb_flow_get_ports(const struct sk_buff
*skb
, int thoff
, u8 ip_proto
,
43 int poff
= proto_ports_offset(ip_proto
);
47 hlen
= skb_headlen(skb
);
51 __be32
*ports
, _ports
;
53 ports
= __skb_header_pointer(skb
, thoff
+ poff
,
54 sizeof(_ports
), data
, hlen
, &_ports
);
61 EXPORT_SYMBOL(__skb_flow_get_ports
);
64 * __skb_flow_dissect - extract the flow_keys struct and return it
65 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
66 * @data: raw buffer pointer to the packet, if NULL use skb->data
67 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
68 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
69 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
71 * The function will try to retrieve the struct flow_keys from either the skbuff
72 * or a raw buffer specified by the rest parameters
74 bool __skb_flow_dissect(const struct sk_buff
*skb
, struct flow_keys
*flow
,
75 void *data
, __be16 proto
, int nhoff
, int hlen
)
81 proto
= skb
->protocol
;
82 nhoff
= skb_network_offset(skb
);
83 hlen
= skb_headlen(skb
);
86 memset(flow
, 0, sizeof(*flow
));
90 case htons(ETH_P_IP
): {
91 const struct iphdr
*iph
;
94 iph
= __skb_header_pointer(skb
, nhoff
, sizeof(_iph
), data
, hlen
, &_iph
);
95 if (!iph
|| iph
->ihl
< 5)
97 nhoff
+= iph
->ihl
* 4;
99 ip_proto
= iph
->protocol
;
100 if (ip_is_fragment(iph
))
103 /* skip the address processing if skb is NULL. The assumption
104 * here is that if there is no skb we are not looking for flow
105 * info but lengths and protocols.
110 iph_to_flow_copy_addrs(flow
, iph
);
113 case htons(ETH_P_IPV6
): {
114 const struct ipv6hdr
*iph
;
119 iph
= __skb_header_pointer(skb
, nhoff
, sizeof(_iph
), data
, hlen
, &_iph
);
123 ip_proto
= iph
->nexthdr
;
124 nhoff
+= sizeof(struct ipv6hdr
);
126 /* see comment above in IPv4 section */
130 flow
->src
= (__force __be32
)ipv6_addr_hash(&iph
->saddr
);
131 flow
->dst
= (__force __be32
)ipv6_addr_hash(&iph
->daddr
);
133 flow_label
= ip6_flowlabel(iph
);
135 /* Awesome, IPv6 packet has a flow label so we can
136 * use that to represent the ports without any
137 * further dissection.
139 flow
->n_proto
= proto
;
140 flow
->ip_proto
= ip_proto
;
141 flow
->ports
= flow_label
;
142 flow
->thoff
= (u16
)nhoff
;
149 case htons(ETH_P_8021AD
):
150 case htons(ETH_P_8021Q
): {
151 const struct vlan_hdr
*vlan
;
152 struct vlan_hdr _vlan
;
154 vlan
= __skb_header_pointer(skb
, nhoff
, sizeof(_vlan
), data
, hlen
, &_vlan
);
158 proto
= vlan
->h_vlan_encapsulated_proto
;
159 nhoff
+= sizeof(*vlan
);
162 case htons(ETH_P_PPP_SES
): {
164 struct pppoe_hdr hdr
;
167 hdr
= __skb_header_pointer(skb
, nhoff
, sizeof(_hdr
), data
, hlen
, &_hdr
);
171 nhoff
+= PPPOE_SES_HLEN
;
175 case htons(PPP_IPV6
):
181 case htons(ETH_P_FCOE
):
182 flow
->thoff
= (u16
)(nhoff
+ FCOE_HEADER_LEN
);
195 hdr
= __skb_header_pointer(skb
, nhoff
, sizeof(_hdr
), data
, hlen
, &_hdr
);
199 * Only look inside GRE if version zero and no
202 if (!(hdr
->flags
& (GRE_VERSION
|GRE_ROUTING
))) {
205 if (hdr
->flags
& GRE_CSUM
)
207 if (hdr
->flags
& GRE_KEY
)
209 if (hdr
->flags
& GRE_SEQ
)
211 if (proto
== htons(ETH_P_TEB
)) {
212 const struct ethhdr
*eth
;
215 eth
= __skb_header_pointer(skb
, nhoff
,
220 proto
= eth
->h_proto
;
221 nhoff
+= sizeof(*eth
);
228 proto
= htons(ETH_P_IP
);
231 proto
= htons(ETH_P_IPV6
);
237 flow
->n_proto
= proto
;
238 flow
->ip_proto
= ip_proto
;
239 flow
->thoff
= (u16
) nhoff
;
241 /* unless skb is set we don't need to record port info */
243 flow
->ports
= __skb_flow_get_ports(skb
, nhoff
, ip_proto
,
248 EXPORT_SYMBOL(__skb_flow_dissect
);
250 static u32 hashrnd __read_mostly
;
251 static __always_inline
void __flow_hash_secret_init(void)
253 net_get_random_once(&hashrnd
, sizeof(hashrnd
));
256 static __always_inline u32
__flow_hash_3words(u32 a
, u32 b
, u32 c
)
258 __flow_hash_secret_init();
259 return jhash_3words(a
, b
, c
, hashrnd
);
262 static inline u32
__flow_hash_from_keys(struct flow_keys
*keys
)
266 /* get a consistent hash (same value on both flow directions) */
267 if (((__force u32
)keys
->dst
< (__force u32
)keys
->src
) ||
268 (((__force u32
)keys
->dst
== (__force u32
)keys
->src
) &&
269 ((__force u16
)keys
->port16
[1] < (__force u16
)keys
->port16
[0]))) {
270 swap(keys
->dst
, keys
->src
);
271 swap(keys
->port16
[0], keys
->port16
[1]);
274 hash
= __flow_hash_3words((__force u32
)keys
->dst
,
275 (__force u32
)keys
->src
,
276 (__force u32
)keys
->ports
);
283 u32
flow_hash_from_keys(struct flow_keys
*keys
)
285 return __flow_hash_from_keys(keys
);
287 EXPORT_SYMBOL(flow_hash_from_keys
);
290 * __skb_get_hash: calculate a flow hash based on src/dst addresses
291 * and src/dst port numbers. Sets hash in skb to non-zero hash value
292 * on success, zero indicates no valid hash. Also, sets l4_hash in skb
293 * if hash is a canonical 4-tuple hash over transport ports.
295 void __skb_get_hash(struct sk_buff
*skb
)
297 struct flow_keys keys
;
299 if (!skb_flow_dissect(skb
, &keys
))
307 skb
->hash
= __flow_hash_from_keys(&keys
);
309 EXPORT_SYMBOL(__skb_get_hash
);
312 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
313 * to be used as a distribution range.
315 u16
__skb_tx_hash(const struct net_device
*dev
, struct sk_buff
*skb
,
316 unsigned int num_tx_queues
)
320 u16 qcount
= num_tx_queues
;
322 if (skb_rx_queue_recorded(skb
)) {
323 hash
= skb_get_rx_queue(skb
);
324 while (unlikely(hash
>= num_tx_queues
))
325 hash
-= num_tx_queues
;
330 u8 tc
= netdev_get_prio_tc_map(dev
, skb
->priority
);
331 qoffset
= dev
->tc_to_txq
[tc
].offset
;
332 qcount
= dev
->tc_to_txq
[tc
].count
;
335 return (u16
) reciprocal_scale(skb_get_hash(skb
), qcount
) + qoffset
;
337 EXPORT_SYMBOL(__skb_tx_hash
);
339 u32
__skb_get_poff(const struct sk_buff
*skb
, void *data
,
340 const struct flow_keys
*keys
, int hlen
)
342 u32 poff
= keys
->thoff
;
344 switch (keys
->ip_proto
) {
346 /* access doff as u8 to avoid unaligned access */
350 doff
= __skb_header_pointer(skb
, poff
+ 12, sizeof(_doff
),
355 poff
+= max_t(u32
, sizeof(struct tcphdr
), (*doff
& 0xF0) >> 2);
359 case IPPROTO_UDPLITE
:
360 poff
+= sizeof(struct udphdr
);
362 /* For the rest, we do not really care about header
363 * extensions at this point for now.
366 poff
+= sizeof(struct icmphdr
);
369 poff
+= sizeof(struct icmp6hdr
);
372 poff
+= sizeof(struct igmphdr
);
375 poff
+= sizeof(struct dccp_hdr
);
378 poff
+= sizeof(struct sctphdr
);
385 /* skb_get_poff() returns the offset to the payload as far as it could
386 * be dissected. The main user is currently BPF, so that we can dynamically
387 * truncate packets without needing to push actual payload to the user
388 * space and can analyze headers only, instead.
390 u32
skb_get_poff(const struct sk_buff
*skb
)
392 struct flow_keys keys
;
394 if (!skb_flow_dissect(skb
, &keys
))
397 return __skb_get_poff(skb
, skb
->data
, &keys
, skb_headlen(skb
));
400 static inline int get_xps_queue(struct net_device
*dev
, struct sk_buff
*skb
)
403 struct xps_dev_maps
*dev_maps
;
405 int queue_index
= -1;
408 dev_maps
= rcu_dereference(dev
->xps_maps
);
410 map
= rcu_dereference(
411 dev_maps
->cpu_map
[raw_smp_processor_id()]);
414 queue_index
= map
->queues
[0];
416 queue_index
= map
->queues
[reciprocal_scale(skb_get_hash(skb
),
418 if (unlikely(queue_index
>= dev
->real_num_tx_queues
))
430 static u16
__netdev_pick_tx(struct net_device
*dev
, struct sk_buff
*skb
)
432 struct sock
*sk
= skb
->sk
;
433 int queue_index
= sk_tx_queue_get(sk
);
435 if (queue_index
< 0 || skb
->ooo_okay
||
436 queue_index
>= dev
->real_num_tx_queues
) {
437 int new_index
= get_xps_queue(dev
, skb
);
439 new_index
= skb_tx_hash(dev
, skb
);
441 if (queue_index
!= new_index
&& sk
&&
442 rcu_access_pointer(sk
->sk_dst_cache
))
443 sk_tx_queue_set(sk
, new_index
);
445 queue_index
= new_index
;
451 struct netdev_queue
*netdev_pick_tx(struct net_device
*dev
,
457 if (dev
->real_num_tx_queues
!= 1) {
458 const struct net_device_ops
*ops
= dev
->netdev_ops
;
459 if (ops
->ndo_select_queue
)
460 queue_index
= ops
->ndo_select_queue(dev
, skb
, accel_priv
,
463 queue_index
= __netdev_pick_tx(dev
, skb
);
466 queue_index
= netdev_cap_txqueue(dev
, queue_index
);
469 skb_set_queue_mapping(skb
, queue_index
);
470 return netdev_get_tx_queue(dev
, queue_index
);