1 #include <linux/skbuff.h>
2 #include <linux/export.h>
4 #include <linux/ipv6.h>
5 #include <linux/if_vlan.h>
8 #include <linux/igmp.h>
9 #include <linux/icmp.h>
10 #include <linux/sctp.h>
11 #include <linux/dccp.h>
12 #include <linux/if_tunnel.h>
13 #include <linux/if_pppox.h>
14 #include <linux/ppp_defs.h>
15 #include <net/flow_keys.h>
16 #include <scsi/fc/fc_fcoe.h>
18 /* copy saddr & daddr, possibly using 64bit load/store
19 * Equivalent to : flow->src = iph->saddr;
20 * flow->dst = iph->daddr;
22 static void iph_to_flow_copy_addrs(struct flow_keys
*flow
, const struct iphdr
*iph
)
24 BUILD_BUG_ON(offsetof(typeof(*flow
), dst
) !=
25 offsetof(typeof(*flow
), src
) + sizeof(flow
->src
));
26 memcpy(&flow
->src
, &iph
->saddr
, sizeof(flow
->src
) + sizeof(flow
->dst
));
30 * __skb_flow_get_ports - extract the upper layer ports and return them
31 * @skb: sk_buff to extract the ports from
32 * @thoff: transport header offset
33 * @ip_proto: protocol for which to get port offset
34 * @data: raw buffer pointer to the packet, if NULL use skb->data
35 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
37 * The function will try to retrieve the ports at offset thoff + poff where poff
38 * is the protocol port offset returned from proto_ports_offset
40 __be32
__skb_flow_get_ports(const struct sk_buff
*skb
, int thoff
, u8 ip_proto
,
43 int poff
= proto_ports_offset(ip_proto
);
47 hlen
= skb_headlen(skb
);
51 __be32
*ports
, _ports
;
53 ports
= __skb_header_pointer(skb
, thoff
+ poff
,
54 sizeof(_ports
), data
, hlen
, &_ports
);
61 EXPORT_SYMBOL(__skb_flow_get_ports
);
64 * __skb_flow_dissect - extract the flow_keys struct and return it
65 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
66 * @data: raw buffer pointer to the packet, if NULL use skb->data
67 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
68 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
69 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
71 * The function will try to retrieve the struct flow_keys from either the skbuff
72 * or a raw buffer specified by the rest parameters
74 bool __skb_flow_dissect(const struct sk_buff
*skb
, struct flow_keys
*flow
,
75 void *data
, __be16 proto
, int nhoff
, int hlen
)
81 proto
= skb
->protocol
;
82 nhoff
= skb_network_offset(skb
);
83 hlen
= skb_headlen(skb
);
86 memset(flow
, 0, sizeof(*flow
));
90 case htons(ETH_P_IP
): {
91 const struct iphdr
*iph
;
94 iph
= __skb_header_pointer(skb
, nhoff
, sizeof(_iph
), data
, hlen
, &_iph
);
95 if (!iph
|| iph
->ihl
< 5)
97 nhoff
+= iph
->ihl
* 4;
99 ip_proto
= iph
->protocol
;
100 if (ip_is_fragment(iph
))
103 iph_to_flow_copy_addrs(flow
, iph
);
106 case htons(ETH_P_IPV6
): {
107 const struct ipv6hdr
*iph
;
112 iph
= __skb_header_pointer(skb
, nhoff
, sizeof(_iph
), data
, hlen
, &_iph
);
116 ip_proto
= iph
->nexthdr
;
117 flow
->src
= (__force __be32
)ipv6_addr_hash(&iph
->saddr
);
118 flow
->dst
= (__force __be32
)ipv6_addr_hash(&iph
->daddr
);
119 nhoff
+= sizeof(struct ipv6hdr
);
121 /* skip the flow label processing if skb is NULL. The
122 * assumption here is that if there is no skb we are not
123 * looking for flow info as much as we are length.
128 flow_label
= ip6_flowlabel(iph
);
130 /* Awesome, IPv6 packet has a flow label so we can
131 * use that to represent the ports without any
132 * further dissection.
134 flow
->n_proto
= proto
;
135 flow
->ip_proto
= ip_proto
;
136 flow
->ports
= flow_label
;
137 flow
->thoff
= (u16
)nhoff
;
144 case htons(ETH_P_8021AD
):
145 case htons(ETH_P_8021Q
): {
146 const struct vlan_hdr
*vlan
;
147 struct vlan_hdr _vlan
;
149 vlan
= __skb_header_pointer(skb
, nhoff
, sizeof(_vlan
), data
, hlen
, &_vlan
);
153 proto
= vlan
->h_vlan_encapsulated_proto
;
154 nhoff
+= sizeof(*vlan
);
157 case htons(ETH_P_PPP_SES
): {
159 struct pppoe_hdr hdr
;
162 hdr
= __skb_header_pointer(skb
, nhoff
, sizeof(_hdr
), data
, hlen
, &_hdr
);
166 nhoff
+= PPPOE_SES_HLEN
;
170 case htons(PPP_IPV6
):
176 case htons(ETH_P_FCOE
):
177 flow
->thoff
= (u16
)(nhoff
+ FCOE_HEADER_LEN
);
190 hdr
= __skb_header_pointer(skb
, nhoff
, sizeof(_hdr
), data
, hlen
, &_hdr
);
194 * Only look inside GRE if version zero and no
197 if (!(hdr
->flags
& (GRE_VERSION
|GRE_ROUTING
))) {
200 if (hdr
->flags
& GRE_CSUM
)
202 if (hdr
->flags
& GRE_KEY
)
204 if (hdr
->flags
& GRE_SEQ
)
206 if (proto
== htons(ETH_P_TEB
)) {
207 const struct ethhdr
*eth
;
210 eth
= __skb_header_pointer(skb
, nhoff
,
215 proto
= eth
->h_proto
;
216 nhoff
+= sizeof(*eth
);
223 proto
= htons(ETH_P_IP
);
226 proto
= htons(ETH_P_IPV6
);
232 flow
->n_proto
= proto
;
233 flow
->ip_proto
= ip_proto
;
234 flow
->ports
= __skb_flow_get_ports(skb
, nhoff
, ip_proto
, data
, hlen
);
235 flow
->thoff
= (u16
) nhoff
;
239 EXPORT_SYMBOL(__skb_flow_dissect
);
241 static u32 hashrnd __read_mostly
;
242 static __always_inline
void __flow_hash_secret_init(void)
244 net_get_random_once(&hashrnd
, sizeof(hashrnd
));
247 static __always_inline u32
__flow_hash_3words(u32 a
, u32 b
, u32 c
)
249 __flow_hash_secret_init();
250 return jhash_3words(a
, b
, c
, hashrnd
);
253 static inline u32
__flow_hash_from_keys(struct flow_keys
*keys
)
257 /* get a consistent hash (same value on both flow directions) */
258 if (((__force u32
)keys
->dst
< (__force u32
)keys
->src
) ||
259 (((__force u32
)keys
->dst
== (__force u32
)keys
->src
) &&
260 ((__force u16
)keys
->port16
[1] < (__force u16
)keys
->port16
[0]))) {
261 swap(keys
->dst
, keys
->src
);
262 swap(keys
->port16
[0], keys
->port16
[1]);
265 hash
= __flow_hash_3words((__force u32
)keys
->dst
,
266 (__force u32
)keys
->src
,
267 (__force u32
)keys
->ports
);
274 u32
flow_hash_from_keys(struct flow_keys
*keys
)
276 return __flow_hash_from_keys(keys
);
278 EXPORT_SYMBOL(flow_hash_from_keys
);
281 * __skb_get_hash: calculate a flow hash based on src/dst addresses
282 * and src/dst port numbers. Sets hash in skb to non-zero hash value
283 * on success, zero indicates no valid hash. Also, sets l4_hash in skb
284 * if hash is a canonical 4-tuple hash over transport ports.
286 void __skb_get_hash(struct sk_buff
*skb
)
288 struct flow_keys keys
;
290 if (!skb_flow_dissect(skb
, &keys
))
298 skb
->hash
= __flow_hash_from_keys(&keys
);
300 EXPORT_SYMBOL(__skb_get_hash
);
303 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
304 * to be used as a distribution range.
306 u16
__skb_tx_hash(const struct net_device
*dev
, struct sk_buff
*skb
,
307 unsigned int num_tx_queues
)
311 u16 qcount
= num_tx_queues
;
313 if (skb_rx_queue_recorded(skb
)) {
314 hash
= skb_get_rx_queue(skb
);
315 while (unlikely(hash
>= num_tx_queues
))
316 hash
-= num_tx_queues
;
321 u8 tc
= netdev_get_prio_tc_map(dev
, skb
->priority
);
322 qoffset
= dev
->tc_to_txq
[tc
].offset
;
323 qcount
= dev
->tc_to_txq
[tc
].count
;
326 return (u16
) reciprocal_scale(skb_get_hash(skb
), qcount
) + qoffset
;
328 EXPORT_SYMBOL(__skb_tx_hash
);
330 u32
__skb_get_poff(const struct sk_buff
*skb
, void *data
,
331 const struct flow_keys
*keys
, int hlen
)
333 u32 poff
= keys
->thoff
;
335 switch (keys
->ip_proto
) {
337 const struct tcphdr
*tcph
;
340 tcph
= __skb_header_pointer(skb
, poff
, sizeof(_tcph
),
345 poff
+= max_t(u32
, sizeof(struct tcphdr
), tcph
->doff
* 4);
349 case IPPROTO_UDPLITE
:
350 poff
+= sizeof(struct udphdr
);
352 /* For the rest, we do not really care about header
353 * extensions at this point for now.
356 poff
+= sizeof(struct icmphdr
);
359 poff
+= sizeof(struct icmp6hdr
);
362 poff
+= sizeof(struct igmphdr
);
365 poff
+= sizeof(struct dccp_hdr
);
368 poff
+= sizeof(struct sctphdr
);
375 /* skb_get_poff() returns the offset to the payload as far as it could
376 * be dissected. The main user is currently BPF, so that we can dynamically
377 * truncate packets without needing to push actual payload to the user
378 * space and can analyze headers only, instead.
380 u32
skb_get_poff(const struct sk_buff
*skb
)
382 struct flow_keys keys
;
384 if (!skb_flow_dissect(skb
, &keys
))
387 return __skb_get_poff(skb
, skb
->data
, &keys
, skb_headlen(skb
));
390 static inline int get_xps_queue(struct net_device
*dev
, struct sk_buff
*skb
)
393 struct xps_dev_maps
*dev_maps
;
395 int queue_index
= -1;
398 dev_maps
= rcu_dereference(dev
->xps_maps
);
400 map
= rcu_dereference(
401 dev_maps
->cpu_map
[raw_smp_processor_id()]);
404 queue_index
= map
->queues
[0];
406 queue_index
= map
->queues
[reciprocal_scale(skb_get_hash(skb
),
408 if (unlikely(queue_index
>= dev
->real_num_tx_queues
))
420 static u16
__netdev_pick_tx(struct net_device
*dev
, struct sk_buff
*skb
)
422 struct sock
*sk
= skb
->sk
;
423 int queue_index
= sk_tx_queue_get(sk
);
425 if (queue_index
< 0 || skb
->ooo_okay
||
426 queue_index
>= dev
->real_num_tx_queues
) {
427 int new_index
= get_xps_queue(dev
, skb
);
429 new_index
= skb_tx_hash(dev
, skb
);
431 if (queue_index
!= new_index
&& sk
&&
432 rcu_access_pointer(sk
->sk_dst_cache
))
433 sk_tx_queue_set(sk
, new_index
);
435 queue_index
= new_index
;
441 struct netdev_queue
*netdev_pick_tx(struct net_device
*dev
,
447 if (dev
->real_num_tx_queues
!= 1) {
448 const struct net_device_ops
*ops
= dev
->netdev_ops
;
449 if (ops
->ndo_select_queue
)
450 queue_index
= ops
->ndo_select_queue(dev
, skb
, accel_priv
,
453 queue_index
= __netdev_pick_tx(dev
, skb
);
456 queue_index
= netdev_cap_txqueue(dev
, queue_index
);
459 skb_set_queue_mapping(skb
, queue_index
);
460 return netdev_get_tx_queue(dev
, queue_index
);