2 * IPV4 GSO/GRO offload support
3 * Linux INET implementation
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version
8 * 2 of the License, or (at your option) any later version.
10 * TCPv4 GSO/GRO support
13 #include <linux/skbuff.h>
15 #include <net/protocol.h>
17 static void tcp_gso_tstamp(struct sk_buff
*skb
, unsigned int ts_seq
,
18 unsigned int seq
, unsigned int mss
)
21 if (before(ts_seq
, seq
+ mss
)) {
22 skb_shinfo(skb
)->tx_flags
|= SKBTX_SW_TSTAMP
;
23 skb_shinfo(skb
)->tskey
= ts_seq
;
32 static struct sk_buff
*tcp4_gso_segment(struct sk_buff
*skb
,
33 netdev_features_t features
)
35 if (!pskb_may_pull(skb
, sizeof(struct tcphdr
)))
36 return ERR_PTR(-EINVAL
);
38 if (unlikely(skb
->ip_summed
!= CHECKSUM_PARTIAL
)) {
39 const struct iphdr
*iph
= ip_hdr(skb
);
40 struct tcphdr
*th
= tcp_hdr(skb
);
42 /* Set up checksum pseudo header, usually expect stack to
43 * have done this already.
47 skb
->ip_summed
= CHECKSUM_PARTIAL
;
48 __tcp_v4_send_check(skb
, iph
->saddr
, iph
->daddr
);
51 return tcp_gso_segment(skb
, features
);
54 struct sk_buff
*tcp_gso_segment(struct sk_buff
*skb
,
55 netdev_features_t features
)
57 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
58 unsigned int sum_truesize
= 0;
65 struct sk_buff
*gso_skb
= skb
;
67 bool ooo_okay
, copy_destructor
;
71 if (thlen
< sizeof(*th
))
74 if (!pskb_may_pull(skb
, thlen
))
77 oldlen
= (u16
)~skb
->len
;
78 __skb_pull(skb
, thlen
);
80 mss
= skb_shinfo(skb
)->gso_size
;
81 if (unlikely(skb
->len
<= mss
))
84 if (skb_gso_ok(skb
, features
| NETIF_F_GSO_ROBUST
)) {
85 /* Packet is from an untrusted source, reset gso_segs. */
86 int type
= skb_shinfo(skb
)->gso_type
;
99 SKB_GSO_UDP_TUNNEL_CSUM
|
100 SKB_GSO_TUNNEL_REMCSUM
|
102 !(type
& (SKB_GSO_TCPV4
|
106 skb_shinfo(skb
)->gso_segs
= DIV_ROUND_UP(skb
->len
, mss
);
112 copy_destructor
= gso_skb
->destructor
== tcp_wfree
;
113 ooo_okay
= gso_skb
->ooo_okay
;
114 /* All segments but the first should have ooo_okay cleared */
117 segs
= skb_segment(skb
, features
);
121 /* Only first segment might have ooo_okay set */
122 segs
->ooo_okay
= ooo_okay
;
124 delta
= htonl(oldlen
+ (thlen
+ mss
));
128 seq
= ntohl(th
->seq
);
130 if (unlikely(skb_shinfo(gso_skb
)->tx_flags
& SKBTX_SW_TSTAMP
))
131 tcp_gso_tstamp(segs
, skb_shinfo(gso_skb
)->tskey
, seq
, mss
);
133 newcheck
= ~csum_fold((__force __wsum
)((__force u32
)th
->check
+
134 (__force u32
)delta
));
137 th
->fin
= th
->psh
= 0;
138 th
->check
= newcheck
;
140 if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
141 gso_reset_checksum(skb
, ~th
->check
);
143 th
->check
= gso_make_checksum(skb
, ~th
->check
);
146 if (copy_destructor
) {
147 skb
->destructor
= gso_skb
->destructor
;
148 skb
->sk
= gso_skb
->sk
;
149 sum_truesize
+= skb
->truesize
;
154 th
->seq
= htonl(seq
);
158 /* Following permits TCP Small Queues to work well with GSO :
159 * The callback to TCP stack will be called at the time last frag
160 * is freed at TX completion, and not right now when gso_skb
161 * is freed by GSO engine
163 if (copy_destructor
) {
164 swap(gso_skb
->sk
, skb
->sk
);
165 swap(gso_skb
->destructor
, skb
->destructor
);
166 sum_truesize
+= skb
->truesize
;
167 atomic_add(sum_truesize
- gso_skb
->truesize
,
168 &skb
->sk
->sk_wmem_alloc
);
171 delta
= htonl(oldlen
+ (skb_tail_pointer(skb
) -
172 skb_transport_header(skb
)) +
174 th
->check
= ~csum_fold((__force __wsum
)((__force u32
)th
->check
+
175 (__force u32
)delta
));
176 if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
177 gso_reset_checksum(skb
, ~th
->check
);
179 th
->check
= gso_make_checksum(skb
, ~th
->check
);
184 struct sk_buff
**tcp_gro_receive(struct sk_buff
**head
, struct sk_buff
*skb
)
186 struct sk_buff
**pp
= NULL
;
193 unsigned int mss
= 1;
199 off
= skb_gro_offset(skb
);
200 hlen
= off
+ sizeof(*th
);
201 th
= skb_gro_header_fast(skb
, off
);
202 if (skb_gro_header_hard(skb
, hlen
)) {
203 th
= skb_gro_header_slow(skb
, hlen
, off
);
208 thlen
= th
->doff
* 4;
209 if (thlen
< sizeof(*th
))
213 if (skb_gro_header_hard(skb
, hlen
)) {
214 th
= skb_gro_header_slow(skb
, hlen
, off
);
219 skb_gro_pull(skb
, thlen
);
221 len
= skb_gro_len(skb
);
222 flags
= tcp_flag_word(th
);
224 for (; (p
= *head
); head
= &p
->next
) {
225 if (!NAPI_GRO_CB(p
)->same_flow
)
230 if (*(u32
*)&th
->source
^ *(u32
*)&th2
->source
) {
231 NAPI_GRO_CB(p
)->same_flow
= 0;
238 goto out_check_final
;
241 /* Include the IP ID check below from the inner most IP hdr */
242 flush
= NAPI_GRO_CB(p
)->flush
| NAPI_GRO_CB(p
)->flush_id
;
243 flush
|= (__force
int)(flags
& TCP_FLAG_CWR
);
244 flush
|= (__force
int)((flags
^ tcp_flag_word(th2
)) &
245 ~(TCP_FLAG_CWR
| TCP_FLAG_FIN
| TCP_FLAG_PSH
));
246 flush
|= (__force
int)(th
->ack_seq
^ th2
->ack_seq
);
247 for (i
= sizeof(*th
); i
< thlen
; i
+= 4)
248 flush
|= *(u32
*)((u8
*)th
+ i
) ^
249 *(u32
*)((u8
*)th2
+ i
);
251 mss
= skb_shinfo(p
)->gso_size
;
253 flush
|= (len
- 1) >= mss
;
254 flush
|= (ntohl(th2
->seq
) + skb_gro_len(p
)) ^ ntohl(th
->seq
);
256 if (flush
|| skb_gro_receive(head
, skb
)) {
258 goto out_check_final
;
263 tcp_flag_word(th2
) |= flags
& (TCP_FLAG_FIN
| TCP_FLAG_PSH
);
267 flush
|= (__force
int)(flags
& (TCP_FLAG_URG
| TCP_FLAG_PSH
|
268 TCP_FLAG_RST
| TCP_FLAG_SYN
|
271 if (p
&& (!NAPI_GRO_CB(skb
)->same_flow
|| flush
))
275 NAPI_GRO_CB(skb
)->flush
|= (flush
!= 0);
280 int tcp_gro_complete(struct sk_buff
*skb
)
282 struct tcphdr
*th
= tcp_hdr(skb
);
284 skb
->csum_start
= (unsigned char *)th
- skb
->head
;
285 skb
->csum_offset
= offsetof(struct tcphdr
, check
);
286 skb
->ip_summed
= CHECKSUM_PARTIAL
;
288 skb_shinfo(skb
)->gso_segs
= NAPI_GRO_CB(skb
)->count
;
291 skb_shinfo(skb
)->gso_type
|= SKB_GSO_TCP_ECN
;
295 EXPORT_SYMBOL(tcp_gro_complete
);
297 static struct sk_buff
**tcp4_gro_receive(struct sk_buff
**head
, struct sk_buff
*skb
)
299 /* Don't bother verifying checksum if we're going to flush anyway. */
300 if (!NAPI_GRO_CB(skb
)->flush
&&
301 skb_gro_checksum_validate(skb
, IPPROTO_TCP
,
302 inet_gro_compute_pseudo
)) {
303 NAPI_GRO_CB(skb
)->flush
= 1;
307 return tcp_gro_receive(head
, skb
);
310 static int tcp4_gro_complete(struct sk_buff
*skb
, int thoff
)
312 const struct iphdr
*iph
= ip_hdr(skb
);
313 struct tcphdr
*th
= tcp_hdr(skb
);
315 th
->check
= ~tcp_v4_check(skb
->len
- thoff
, iph
->saddr
,
317 skb_shinfo(skb
)->gso_type
|= SKB_GSO_TCPV4
;
319 return tcp_gro_complete(skb
);
322 static const struct net_offload tcpv4_offload
= {
324 .gso_segment
= tcp4_gso_segment
,
325 .gro_receive
= tcp4_gro_receive
,
326 .gro_complete
= tcp4_gro_complete
,
330 int __init
tcpv4_offload_init(void)
332 return inet_add_offload(&tcpv4_offload
, IPPROTO_TCP
);