2 * IPV4 GSO/GRO offload support
3 * Linux INET implementation
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version
8 * 2 of the License, or (at your option) any later version.
13 #include <linux/skbuff.h>
15 #include <net/protocol.h>
17 static DEFINE_SPINLOCK(udp_offload_lock
);
18 static struct udp_offload_priv __rcu
*udp_offload_base __read_mostly
;
20 #define udp_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&udp_offload_lock))
22 struct udp_offload_priv
{
23 struct udp_offload
*offload
;
25 struct udp_offload_priv __rcu
*next
;
28 static int udp4_ufo_send_check(struct sk_buff
*skb
)
33 struct sk_buff
*skb_udp_tunnel_segment(struct sk_buff
*skb
,
34 netdev_features_t features
)
36 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
37 u16 mac_offset
= skb
->mac_header
;
38 int mac_len
= skb
->mac_len
;
39 int tnl_hlen
= skb_inner_mac_header(skb
) - skb_transport_header(skb
);
40 __be16 protocol
= skb
->protocol
;
41 netdev_features_t enc_features
;
42 int udp_offset
, outer_hlen
;
46 oldlen
= (u16
)~skb
->len
;
48 if (unlikely(!pskb_may_pull(skb
, tnl_hlen
)))
51 skb
->encapsulation
= 0;
52 __skb_pull(skb
, tnl_hlen
);
53 skb_reset_mac_header(skb
);
54 skb_set_network_header(skb
, skb_inner_network_offset(skb
));
55 skb
->mac_len
= skb_inner_network_offset(skb
);
56 skb
->protocol
= htons(ETH_P_TEB
);
58 need_csum
= !!(skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_TUNNEL_CSUM
);
60 skb
->encap_hdr_csum
= 1;
62 /* segment inner packet. */
63 enc_features
= skb
->dev
->hw_enc_features
& netif_skb_features(skb
);
64 segs
= skb_mac_gso_segment(skb
, enc_features
);
65 if (IS_ERR_OR_NULL(segs
)) {
66 skb_gso_error_unwind(skb
, protocol
, tnl_hlen
, mac_offset
,
71 outer_hlen
= skb_tnl_header_len(skb
);
72 udp_offset
= outer_hlen
- tnl_hlen
;
78 skb_reset_inner_headers(skb
);
79 skb
->encapsulation
= 1;
81 skb
->mac_len
= mac_len
;
83 skb_push(skb
, outer_hlen
);
84 skb_reset_mac_header(skb
);
85 skb_set_network_header(skb
, mac_len
);
86 skb_set_transport_header(skb
, udp_offset
);
87 len
= skb
->len
- udp_offset
;
92 __be32 delta
= htonl(oldlen
+ len
);
94 uh
->check
= ~csum_fold((__force __wsum
)
95 ((__force u32
)uh
->check
+
97 uh
->check
= gso_make_checksum(skb
, ~uh
->check
);
100 uh
->check
= CSUM_MANGLED_0
;
103 skb
->protocol
= protocol
;
104 } while ((skb
= skb
->next
));
109 static struct sk_buff
*udp4_ufo_fragment(struct sk_buff
*skb
,
110 netdev_features_t features
)
112 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
118 if (skb
->encapsulation
&&
119 (skb_shinfo(skb
)->gso_type
&
120 (SKB_GSO_UDP_TUNNEL
|SKB_GSO_UDP_TUNNEL_CSUM
))) {
121 segs
= skb_udp_tunnel_segment(skb
, features
);
125 if (!pskb_may_pull(skb
, sizeof(struct udphdr
)))
128 mss
= skb_shinfo(skb
)->gso_size
;
129 if (unlikely(skb
->len
<= mss
))
132 if (skb_gso_ok(skb
, features
| NETIF_F_GSO_ROBUST
)) {
133 /* Packet is from an untrusted source, reset gso_segs. */
134 int type
= skb_shinfo(skb
)->gso_type
;
136 if (unlikely(type
& ~(SKB_GSO_UDP
| SKB_GSO_DODGY
|
138 SKB_GSO_UDP_TUNNEL_CSUM
|
140 SKB_GSO_GRE
| SKB_GSO_GRE_CSUM
|
142 !(type
& (SKB_GSO_UDP
))))
145 skb_shinfo(skb
)->gso_segs
= DIV_ROUND_UP(skb
->len
, mss
);
151 /* Do software UFO. Complete and fill in the UDP checksum as
152 * HW cannot do checksum of UDP packets sent as multiple
160 csum
= skb_checksum(skb
, 0, skb
->len
, 0);
161 uh
->check
= udp_v4_check(skb
->len
, iph
->saddr
, iph
->daddr
, csum
);
163 uh
->check
= CSUM_MANGLED_0
;
165 skb
->ip_summed
= CHECKSUM_NONE
;
167 /* Fragment the skb. IP headers of the fragments are updated in
170 segs
= skb_segment(skb
, features
);
175 int udp_add_offload(struct udp_offload
*uo
)
177 struct udp_offload_priv
*new_offload
= kzalloc(sizeof(*new_offload
), GFP_ATOMIC
);
182 new_offload
->offload
= uo
;
184 spin_lock(&udp_offload_lock
);
185 new_offload
->next
= udp_offload_base
;
186 rcu_assign_pointer(udp_offload_base
, new_offload
);
187 spin_unlock(&udp_offload_lock
);
191 EXPORT_SYMBOL(udp_add_offload
);
193 static void udp_offload_free_routine(struct rcu_head
*head
)
195 struct udp_offload_priv
*ou_priv
= container_of(head
, struct udp_offload_priv
, rcu
);
199 void udp_del_offload(struct udp_offload
*uo
)
201 struct udp_offload_priv __rcu
**head
= &udp_offload_base
;
202 struct udp_offload_priv
*uo_priv
;
204 spin_lock(&udp_offload_lock
);
206 uo_priv
= udp_deref_protected(*head
);
207 for (; uo_priv
!= NULL
;
208 uo_priv
= udp_deref_protected(*head
)) {
209 if (uo_priv
->offload
== uo
) {
210 rcu_assign_pointer(*head
,
211 udp_deref_protected(uo_priv
->next
));
214 head
= &uo_priv
->next
;
216 pr_warn("udp_del_offload: didn't find offload for port %d\n", ntohs(uo
->port
));
218 spin_unlock(&udp_offload_lock
);
220 call_rcu(&uo_priv
->rcu
, udp_offload_free_routine
);
222 EXPORT_SYMBOL(udp_del_offload
);
224 struct sk_buff
**udp_gro_receive(struct sk_buff
**head
, struct sk_buff
*skb
,
227 struct udp_offload_priv
*uo_priv
;
228 struct sk_buff
*p
, **pp
= NULL
;
230 unsigned int off
= skb_gro_offset(skb
);
233 if (NAPI_GRO_CB(skb
)->udp_mark
||
234 (skb
->ip_summed
!= CHECKSUM_PARTIAL
&&
235 NAPI_GRO_CB(skb
)->csum_cnt
== 0 &&
236 !NAPI_GRO_CB(skb
)->csum_valid
))
239 /* mark that this skb passed once through the udp gro layer */
240 NAPI_GRO_CB(skb
)->udp_mark
= 1;
243 uo_priv
= rcu_dereference(udp_offload_base
);
244 for (; uo_priv
!= NULL
; uo_priv
= rcu_dereference(uo_priv
->next
)) {
245 if (uo_priv
->offload
->port
== uh
->dest
&&
246 uo_priv
->offload
->callbacks
.gro_receive
)
254 for (p
= *head
; p
; p
= p
->next
) {
255 if (!NAPI_GRO_CB(p
)->same_flow
)
258 uh2
= (struct udphdr
*)(p
->data
+ off
);
260 /* Match ports and either checksums are either both zero
263 if ((*(u32
*)&uh
->source
!= *(u32
*)&uh2
->source
) ||
264 (!uh
->check
^ !uh2
->check
)) {
265 NAPI_GRO_CB(p
)->same_flow
= 0;
270 skb_gro_pull(skb
, sizeof(struct udphdr
)); /* pull encapsulating udp header */
271 skb_gro_postpull_rcsum(skb
, uh
, sizeof(struct udphdr
));
272 NAPI_GRO_CB(skb
)->proto
= uo_priv
->offload
->ipproto
;
273 pp
= uo_priv
->offload
->callbacks
.gro_receive(head
, skb
);
278 NAPI_GRO_CB(skb
)->flush
|= flush
;
282 static struct sk_buff
**udp4_gro_receive(struct sk_buff
**head
,
285 struct udphdr
*uh
= udp_gro_udphdr(skb
);
290 /* Don't bother verifying checksum if we're going to flush anyway. */
291 if (NAPI_GRO_CB(skb
)->flush
)
294 if (skb_gro_checksum_validate_zero_check(skb
, IPPROTO_UDP
, uh
->check
,
295 inet_gro_compute_pseudo
))
298 skb_gro_checksum_try_convert(skb
, IPPROTO_UDP
, uh
->check
,
299 inet_gro_compute_pseudo
);
301 return udp_gro_receive(head
, skb
, uh
);
304 NAPI_GRO_CB(skb
)->flush
= 1;
308 int udp_gro_complete(struct sk_buff
*skb
, int nhoff
)
310 struct udp_offload_priv
*uo_priv
;
311 __be16 newlen
= htons(skb
->len
- nhoff
);
312 struct udphdr
*uh
= (struct udphdr
*)(skb
->data
+ nhoff
);
319 uo_priv
= rcu_dereference(udp_offload_base
);
320 for (; uo_priv
!= NULL
; uo_priv
= rcu_dereference(uo_priv
->next
)) {
321 if (uo_priv
->offload
->port
== uh
->dest
&&
322 uo_priv
->offload
->callbacks
.gro_complete
)
326 if (uo_priv
!= NULL
) {
327 NAPI_GRO_CB(skb
)->proto
= uo_priv
->offload
->ipproto
;
328 err
= uo_priv
->offload
->callbacks
.gro_complete(skb
, nhoff
+ sizeof(struct udphdr
));
335 static int udp4_gro_complete(struct sk_buff
*skb
, int nhoff
)
337 const struct iphdr
*iph
= ip_hdr(skb
);
338 struct udphdr
*uh
= (struct udphdr
*)(skb
->data
+ nhoff
);
341 uh
->check
= ~udp_v4_check(skb
->len
- nhoff
, iph
->saddr
,
344 return udp_gro_complete(skb
, nhoff
);
347 static const struct net_offload udpv4_offload
= {
349 .gso_send_check
= udp4_ufo_send_check
,
350 .gso_segment
= udp4_ufo_fragment
,
351 .gro_receive
= udp4_gro_receive
,
352 .gro_complete
= udp4_gro_complete
,
356 int __init
udpv4_offload_init(void)
358 return inet_add_offload(&udpv4_offload
, IPPROTO_UDP
);