2 * IPV4 GSO/GRO offload support
3 * Linux INET implementation
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version
8 * 2 of the License, or (at your option) any later version.
13 #include <linux/skbuff.h>
14 #include <net/protocol.h>
17 static int gre_gso_send_check(struct sk_buff
*skb
)
19 if (!skb
->encapsulation
)
24 static struct sk_buff
*gre_gso_segment(struct sk_buff
*skb
,
25 netdev_features_t features
)
27 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
28 netdev_features_t enc_features
;
30 struct gre_base_hdr
*greh
;
31 u16 mac_offset
= skb
->mac_header
;
32 int mac_len
= skb
->mac_len
;
33 __be16 protocol
= skb
->protocol
;
37 if (unlikely(skb_shinfo(skb
)->gso_type
&
47 if (unlikely(!pskb_may_pull(skb
, sizeof(*greh
))))
50 greh
= (struct gre_base_hdr
*)skb_transport_header(skb
);
52 ghl
= skb_inner_network_header(skb
) - skb_transport_header(skb
);
53 if (unlikely(ghl
< sizeof(*greh
)))
56 csum
= !!(greh
->flags
& GRE_CSUM
);
58 if (unlikely(!pskb_may_pull(skb
, ghl
)))
61 /* setup inner skb. */
62 skb
->protocol
= greh
->protocol
;
63 skb
->encapsulation
= 0;
66 skb_reset_mac_header(skb
);
67 skb_set_network_header(skb
, skb_inner_network_offset(skb
));
68 skb
->mac_len
= skb_inner_network_offset(skb
);
70 /* segment inner packet. */
71 enc_features
= skb
->dev
->hw_enc_features
& netif_skb_features(skb
);
72 segs
= skb_mac_gso_segment(skb
, enc_features
);
73 if (!segs
|| IS_ERR(segs
)) {
74 skb_gso_error_unwind(skb
, protocol
, ghl
, mac_offset
, mac_len
);
79 tnl_hlen
= skb_tnl_header_len(skb
);
85 if (skb_has_shared_frag(skb
)) {
88 err
= __skb_linearize(skb
);
96 greh
= (struct gre_base_hdr
*)(skb
->data
);
97 pcsum
= (__be32
*)(greh
+ 1);
99 *(__sum16
*)pcsum
= csum_fold(skb_checksum(skb
, 0, skb
->len
, 0));
101 __skb_push(skb
, tnl_hlen
- ghl
);
103 skb_reset_inner_headers(skb
);
104 skb
->encapsulation
= 1;
106 skb_reset_mac_header(skb
);
107 skb_set_network_header(skb
, mac_len
);
108 skb
->mac_len
= mac_len
;
109 skb
->protocol
= protocol
;
110 } while ((skb
= skb
->next
));
115 /* Compute the whole skb csum in s/w and store it, then verify GRO csum
116 * starting from gro_offset.
118 static __sum16
gro_skb_checksum(struct sk_buff
*skb
)
122 skb
->csum
= skb_checksum(skb
, 0, skb
->len
, 0);
123 NAPI_GRO_CB(skb
)->csum
= csum_sub(skb
->csum
,
124 csum_partial(skb
->data
, skb_gro_offset(skb
), 0));
125 sum
= csum_fold(NAPI_GRO_CB(skb
)->csum
);
126 if (unlikely(skb
->ip_summed
== CHECKSUM_COMPLETE
)) {
128 netdev_rx_csum_fault(skb
->dev
);
130 skb
->ip_summed
= CHECKSUM_COMPLETE
;
135 static struct sk_buff
**gre_gro_receive(struct sk_buff
**head
,
138 struct sk_buff
**pp
= NULL
;
140 const struct gre_base_hdr
*greh
;
141 unsigned int hlen
, grehlen
;
144 struct packet_offload
*ptype
;
147 off
= skb_gro_offset(skb
);
148 hlen
= off
+ sizeof(*greh
);
149 greh
= skb_gro_header_fast(skb
, off
);
150 if (skb_gro_header_hard(skb
, hlen
)) {
151 greh
= skb_gro_header_slow(skb
, hlen
, off
);
156 /* Only support version 0 and K (key), C (csum) flags. Note that
157 * although the support for the S (seq#) flag can be added easily
158 * for GRO, this is problematic for GSO hence can not be enabled
159 * here because a GRO pkt may end up in the forwarding path, thus
160 * requiring GSO support to break it up correctly.
162 if ((greh
->flags
& ~(GRE_KEY
|GRE_CSUM
)) != 0)
165 type
= greh
->protocol
;
168 ptype
= gro_find_receive_by_type(type
);
172 grehlen
= GRE_HEADER_SECTION
;
174 if (greh
->flags
& GRE_KEY
)
175 grehlen
+= GRE_HEADER_SECTION
;
177 if (greh
->flags
& GRE_CSUM
)
178 grehlen
+= GRE_HEADER_SECTION
;
180 hlen
= off
+ grehlen
;
181 if (skb_gro_header_hard(skb
, hlen
)) {
182 greh
= skb_gro_header_slow(skb
, hlen
, off
);
186 if (greh
->flags
& GRE_CSUM
) { /* Need to verify GRE csum first */
189 if (skb
->ip_summed
== CHECKSUM_COMPLETE
)
190 csum
= csum_fold(NAPI_GRO_CB(skb
)->csum
);
191 /* Don't trust csum error calculated/reported by h/w */
192 if (skb
->ip_summed
== CHECKSUM_NONE
|| csum
!= 0)
193 csum
= gro_skb_checksum(skb
);
195 /* GRE CSUM is the 1's complement of the 1's complement sum
196 * of the GRE hdr plus payload so it should add up to 0xffff
197 * (and 0 after csum_fold()) just like the IPv4 hdr csum.
204 for (p
= *head
; p
; p
= p
->next
) {
205 const struct gre_base_hdr
*greh2
;
207 if (!NAPI_GRO_CB(p
)->same_flow
)
210 /* The following checks are needed to ensure only pkts
211 * from the same tunnel are considered for aggregation.
212 * The criteria for "the same tunnel" includes:
213 * 1) same version (we only support version 0 here)
214 * 2) same protocol (we only support ETH_P_IP for now)
215 * 3) same set of flags
216 * 4) same key if the key field is present.
218 greh2
= (struct gre_base_hdr
*)(p
->data
+ off
);
220 if (greh2
->flags
!= greh
->flags
||
221 greh2
->protocol
!= greh
->protocol
) {
222 NAPI_GRO_CB(p
)->same_flow
= 0;
225 if (greh
->flags
& GRE_KEY
) {
227 if (*(__be32
*)(greh2
+1) != *(__be32
*)(greh
+1)) {
228 NAPI_GRO_CB(p
)->same_flow
= 0;
234 skb_gro_pull(skb
, grehlen
);
236 /* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/
237 skb_gro_postpull_rcsum(skb
, greh
, grehlen
);
239 pp
= ptype
->callbacks
.gro_receive(head
, skb
);
244 NAPI_GRO_CB(skb
)->flush
|= flush
;
249 static int gre_gro_complete(struct sk_buff
*skb
, int nhoff
)
251 struct gre_base_hdr
*greh
= (struct gre_base_hdr
*)(skb
->data
+ nhoff
);
252 struct packet_offload
*ptype
;
253 unsigned int grehlen
= sizeof(*greh
);
257 type
= greh
->protocol
;
258 if (greh
->flags
& GRE_KEY
)
259 grehlen
+= GRE_HEADER_SECTION
;
261 if (greh
->flags
& GRE_CSUM
)
262 grehlen
+= GRE_HEADER_SECTION
;
265 ptype
= gro_find_complete_by_type(type
);
267 err
= ptype
->callbacks
.gro_complete(skb
, nhoff
+ grehlen
);
273 static const struct net_offload gre_offload
= {
275 .gso_send_check
= gre_gso_send_check
,
276 .gso_segment
= gre_gso_segment
,
277 .gro_receive
= gre_gro_receive
,
278 .gro_complete
= gre_gro_complete
,
282 static int __init
gre_offload_init(void)
284 return inet_add_offload(&gre_offload
, IPPROTO_GRE
);
287 static void __exit
gre_offload_exit(void)
289 inet_del_offload(&gre_offload
, IPPROTO_GRE
);
292 module_init(gre_offload_init
);
293 module_exit(gre_offload_exit
);