2 * IPV4 GSO/GRO offload support
3 * Linux INET implementation
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version
8 * 2 of the License, or (at your option) any later version.
13 #include <linux/skbuff.h>
15 #include <net/protocol.h>
17 static DEFINE_SPINLOCK(udp_offload_lock
);
18 static struct udp_offload_priv __rcu
*udp_offload_base __read_mostly
;
20 #define udp_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&udp_offload_lock))
22 struct udp_offload_priv
{
23 struct udp_offload
*offload
;
26 struct udp_offload_priv __rcu
*next
;
29 static struct sk_buff
*__skb_udp_tunnel_segment(struct sk_buff
*skb
,
30 netdev_features_t features
,
31 struct sk_buff
*(*gso_inner_segment
)(struct sk_buff
*skb
,
32 netdev_features_t features
),
33 __be16 new_protocol
, bool is_ipv6
)
35 int tnl_hlen
= skb_inner_mac_header(skb
) - skb_transport_header(skb
);
36 bool remcsum
, need_csum
, offload_csum
, ufo
;
37 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
38 struct udphdr
*uh
= udp_hdr(skb
);
39 u16 mac_offset
= skb
->mac_header
;
40 __be16 protocol
= skb
->protocol
;
41 u16 mac_len
= skb
->mac_len
;
42 int udp_offset
, outer_hlen
;
45 if (unlikely(!pskb_may_pull(skb
, tnl_hlen
)))
48 /* Adjust partial header checksum to negate old length.
49 * We cannot rely on the value contained in uh->len as it is
50 * possible that the actual value exceeds the boundaries of the
51 * 16 bit length field due to the header being added outside of an
52 * IP or IPv6 frame that was already limited to 64K - 1.
54 partial
= csum_sub(csum_unfold(uh
->check
),
55 (__force __wsum
)htonl(skb
->len
));
57 /* setup inner skb. */
58 skb
->encapsulation
= 0;
59 __skb_pull(skb
, tnl_hlen
);
60 skb_reset_mac_header(skb
);
61 skb_set_network_header(skb
, skb_inner_network_offset(skb
));
62 skb
->mac_len
= skb_inner_network_offset(skb
);
63 skb
->protocol
= new_protocol
;
65 need_csum
= !!(skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_TUNNEL_CSUM
);
66 skb
->encap_hdr_csum
= need_csum
;
68 remcsum
= !!(skb_shinfo(skb
)->gso_type
& SKB_GSO_TUNNEL_REMCSUM
);
69 skb
->remcsum_offload
= remcsum
;
71 ufo
= !!(skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP
);
73 /* Try to offload checksum if possible */
74 offload_csum
= !!(need_csum
&&
76 (is_ipv6
? (NETIF_F_HW_CSUM
| NETIF_F_IPV6_CSUM
) :
77 (NETIF_F_HW_CSUM
| NETIF_F_IP_CSUM
))));
79 features
&= skb
->dev
->hw_enc_features
;
81 /* The only checksum offload we care about from here on out is the
82 * outer one so strip the existing checksum feature flags and
83 * instead set the flag based on our outer checksum offload value.
86 features
&= ~NETIF_F_CSUM_MASK
;
87 if (!need_csum
|| offload_csum
)
88 features
|= NETIF_F_HW_CSUM
;
91 /* segment inner packet. */
92 segs
= gso_inner_segment(skb
, features
);
93 if (IS_ERR_OR_NULL(segs
)) {
94 skb_gso_error_unwind(skb
, protocol
, tnl_hlen
, mac_offset
,
99 outer_hlen
= skb_tnl_header_len(skb
);
100 udp_offset
= outer_hlen
- tnl_hlen
;
106 skb
->ip_summed
= CHECKSUM_NONE
;
108 /* Set up inner headers if we are offloading inner checksum */
109 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
110 skb_reset_inner_headers(skb
);
111 skb
->encapsulation
= 1;
114 skb
->mac_len
= mac_len
;
115 skb
->protocol
= protocol
;
117 __skb_push(skb
, outer_hlen
);
118 skb_reset_mac_header(skb
);
119 skb_set_network_header(skb
, mac_len
);
120 skb_set_transport_header(skb
, udp_offset
);
121 len
= htons(skb
->len
- udp_offset
);
128 uh
->check
= ~csum_fold(csum_add(partial
, (__force __wsum
)len
));
130 if (skb
->encapsulation
|| !offload_csum
) {
131 uh
->check
= gso_make_checksum(skb
, ~uh
->check
);
133 uh
->check
= CSUM_MANGLED_0
;
135 skb
->ip_summed
= CHECKSUM_PARTIAL
;
136 skb
->csum_start
= skb_transport_header(skb
) - skb
->head
;
137 skb
->csum_offset
= offsetof(struct udphdr
, check
);
139 } while ((skb
= skb
->next
));
144 struct sk_buff
*skb_udp_tunnel_segment(struct sk_buff
*skb
,
145 netdev_features_t features
,
148 __be16 protocol
= skb
->protocol
;
149 const struct net_offload
**offloads
;
150 const struct net_offload
*ops
;
151 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
152 struct sk_buff
*(*gso_inner_segment
)(struct sk_buff
*skb
,
153 netdev_features_t features
);
157 switch (skb
->inner_protocol_type
) {
158 case ENCAP_TYPE_ETHER
:
159 protocol
= skb
->inner_protocol
;
160 gso_inner_segment
= skb_mac_gso_segment
;
162 case ENCAP_TYPE_IPPROTO
:
163 offloads
= is_ipv6
? inet6_offloads
: inet_offloads
;
164 ops
= rcu_dereference(offloads
[skb
->inner_ipproto
]);
165 if (!ops
|| !ops
->callbacks
.gso_segment
)
167 gso_inner_segment
= ops
->callbacks
.gso_segment
;
173 segs
= __skb_udp_tunnel_segment(skb
, features
, gso_inner_segment
,
182 static struct sk_buff
*udp4_ufo_fragment(struct sk_buff
*skb
,
183 netdev_features_t features
)
185 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
191 if (skb
->encapsulation
&&
192 (skb_shinfo(skb
)->gso_type
&
193 (SKB_GSO_UDP_TUNNEL
|SKB_GSO_UDP_TUNNEL_CSUM
))) {
194 segs
= skb_udp_tunnel_segment(skb
, features
, false);
198 if (!pskb_may_pull(skb
, sizeof(struct udphdr
)))
201 mss
= skb_shinfo(skb
)->gso_size
;
202 if (unlikely(skb
->len
<= mss
))
205 if (skb_gso_ok(skb
, features
| NETIF_F_GSO_ROBUST
)) {
206 /* Packet is from an untrusted source, reset gso_segs. */
207 int type
= skb_shinfo(skb
)->gso_type
;
209 if (unlikely(type
& ~(SKB_GSO_UDP
| SKB_GSO_DODGY
|
211 SKB_GSO_UDP_TUNNEL_CSUM
|
212 SKB_GSO_TUNNEL_REMCSUM
|
214 SKB_GSO_GRE
| SKB_GSO_GRE_CSUM
) ||
215 !(type
& (SKB_GSO_UDP
))))
218 skb_shinfo(skb
)->gso_segs
= DIV_ROUND_UP(skb
->len
, mss
);
224 /* Do software UFO. Complete and fill in the UDP checksum as
225 * HW cannot do checksum of UDP packets sent as multiple
233 csum
= skb_checksum(skb
, 0, skb
->len
, 0);
234 uh
->check
= udp_v4_check(skb
->len
, iph
->saddr
, iph
->daddr
, csum
);
236 uh
->check
= CSUM_MANGLED_0
;
238 skb
->ip_summed
= CHECKSUM_NONE
;
240 /* If there is no outer header we can fake a checksum offload
241 * due to the fact that we have already done the checksum in
242 * software prior to segmenting the frame.
244 if (!skb
->encap_hdr_csum
)
245 features
|= NETIF_F_HW_CSUM
;
247 /* Fragment the skb. IP headers of the fragments are updated in
250 segs
= skb_segment(skb
, features
);
255 int udp_add_offload(struct net
*net
, struct udp_offload
*uo
)
257 struct udp_offload_priv
*new_offload
= kzalloc(sizeof(*new_offload
), GFP_ATOMIC
);
262 write_pnet(&new_offload
->net
, net
);
263 new_offload
->offload
= uo
;
265 spin_lock(&udp_offload_lock
);
266 new_offload
->next
= udp_offload_base
;
267 rcu_assign_pointer(udp_offload_base
, new_offload
);
268 spin_unlock(&udp_offload_lock
);
272 EXPORT_SYMBOL(udp_add_offload
);
274 static void udp_offload_free_routine(struct rcu_head
*head
)
276 struct udp_offload_priv
*ou_priv
= container_of(head
, struct udp_offload_priv
, rcu
);
280 void udp_del_offload(struct udp_offload
*uo
)
282 struct udp_offload_priv __rcu
**head
= &udp_offload_base
;
283 struct udp_offload_priv
*uo_priv
;
285 spin_lock(&udp_offload_lock
);
287 uo_priv
= udp_deref_protected(*head
);
288 for (; uo_priv
!= NULL
;
289 uo_priv
= udp_deref_protected(*head
)) {
290 if (uo_priv
->offload
== uo
) {
291 rcu_assign_pointer(*head
,
292 udp_deref_protected(uo_priv
->next
));
295 head
= &uo_priv
->next
;
297 pr_warn("udp_del_offload: didn't find offload for port %d\n", ntohs(uo
->port
));
299 spin_unlock(&udp_offload_lock
);
301 call_rcu(&uo_priv
->rcu
, udp_offload_free_routine
);
303 EXPORT_SYMBOL(udp_del_offload
);
305 struct sk_buff
**udp_gro_receive(struct sk_buff
**head
, struct sk_buff
*skb
,
308 struct udp_offload_priv
*uo_priv
;
309 struct sk_buff
*p
, **pp
= NULL
;
311 unsigned int off
= skb_gro_offset(skb
);
314 if (NAPI_GRO_CB(skb
)->udp_mark
||
315 (skb
->ip_summed
!= CHECKSUM_PARTIAL
&&
316 NAPI_GRO_CB(skb
)->csum_cnt
== 0 &&
317 !NAPI_GRO_CB(skb
)->csum_valid
))
320 /* mark that this skb passed once through the udp gro layer */
321 NAPI_GRO_CB(skb
)->udp_mark
= 1;
324 uo_priv
= rcu_dereference(udp_offload_base
);
325 for (; uo_priv
!= NULL
; uo_priv
= rcu_dereference(uo_priv
->next
)) {
326 if (net_eq(read_pnet(&uo_priv
->net
), dev_net(skb
->dev
)) &&
327 uo_priv
->offload
->port
== uh
->dest
&&
328 uo_priv
->offload
->callbacks
.gro_receive
)
336 for (p
= *head
; p
; p
= p
->next
) {
337 if (!NAPI_GRO_CB(p
)->same_flow
)
340 uh2
= (struct udphdr
*)(p
->data
+ off
);
342 /* Match ports and either checksums are either both zero
345 if ((*(u32
*)&uh
->source
!= *(u32
*)&uh2
->source
) ||
346 (!uh
->check
^ !uh2
->check
)) {
347 NAPI_GRO_CB(p
)->same_flow
= 0;
352 skb_gro_pull(skb
, sizeof(struct udphdr
)); /* pull encapsulating udp header */
353 skb_gro_postpull_rcsum(skb
, uh
, sizeof(struct udphdr
));
354 NAPI_GRO_CB(skb
)->proto
= uo_priv
->offload
->ipproto
;
355 pp
= uo_priv
->offload
->callbacks
.gro_receive(head
, skb
,
361 NAPI_GRO_CB(skb
)->flush
|= flush
;
365 static struct sk_buff
**udp4_gro_receive(struct sk_buff
**head
,
368 struct udphdr
*uh
= udp_gro_udphdr(skb
);
373 /* Don't bother verifying checksum if we're going to flush anyway. */
374 if (NAPI_GRO_CB(skb
)->flush
)
377 if (skb_gro_checksum_validate_zero_check(skb
, IPPROTO_UDP
, uh
->check
,
378 inet_gro_compute_pseudo
))
381 skb_gro_checksum_try_convert(skb
, IPPROTO_UDP
, uh
->check
,
382 inet_gro_compute_pseudo
);
384 NAPI_GRO_CB(skb
)->is_ipv6
= 0;
385 return udp_gro_receive(head
, skb
, uh
);
388 NAPI_GRO_CB(skb
)->flush
= 1;
392 int udp_gro_complete(struct sk_buff
*skb
, int nhoff
)
394 struct udp_offload_priv
*uo_priv
;
395 __be16 newlen
= htons(skb
->len
- nhoff
);
396 struct udphdr
*uh
= (struct udphdr
*)(skb
->data
+ nhoff
);
403 uo_priv
= rcu_dereference(udp_offload_base
);
404 for (; uo_priv
!= NULL
; uo_priv
= rcu_dereference(uo_priv
->next
)) {
405 if (net_eq(read_pnet(&uo_priv
->net
), dev_net(skb
->dev
)) &&
406 uo_priv
->offload
->port
== uh
->dest
&&
407 uo_priv
->offload
->callbacks
.gro_complete
)
412 NAPI_GRO_CB(skb
)->proto
= uo_priv
->offload
->ipproto
;
413 err
= uo_priv
->offload
->callbacks
.gro_complete(skb
,
414 nhoff
+ sizeof(struct udphdr
),
420 if (skb
->remcsum_offload
)
421 skb_shinfo(skb
)->gso_type
|= SKB_GSO_TUNNEL_REMCSUM
;
423 skb
->encapsulation
= 1;
424 skb_set_inner_mac_header(skb
, nhoff
+ sizeof(struct udphdr
));
429 static int udp4_gro_complete(struct sk_buff
*skb
, int nhoff
)
431 const struct iphdr
*iph
= ip_hdr(skb
);
432 struct udphdr
*uh
= (struct udphdr
*)(skb
->data
+ nhoff
);
435 skb_shinfo(skb
)->gso_type
|= SKB_GSO_UDP_TUNNEL_CSUM
;
436 uh
->check
= ~udp_v4_check(skb
->len
- nhoff
, iph
->saddr
,
439 skb_shinfo(skb
)->gso_type
|= SKB_GSO_UDP_TUNNEL
;
442 return udp_gro_complete(skb
, nhoff
);
445 static const struct net_offload udpv4_offload
= {
447 .gso_segment
= udp4_ufo_fragment
,
448 .gro_receive
= udp4_gro_receive
,
449 .gro_complete
= udp4_gro_complete
,
453 int __init
udpv4_offload_init(void)
455 return inet_add_offload(&udpv4_offload
, IPPROTO_UDP
);