Commit | Line | Data |
---|---|---|
cf80e0e4 | 1 | #include <linux/crypto.h> |
10467163 | 2 | #include <linux/err.h> |
2100c8d2 YC |
3 | #include <linux/init.h> |
4 | #include <linux/kernel.h> | |
10467163 JC |
5 | #include <linux/list.h> |
6 | #include <linux/tcp.h> | |
7 | #include <linux/rcupdate.h> | |
8 | #include <linux/rculist.h> | |
9 | #include <net/inetpeer.h> | |
10 | #include <net/tcp.h> | |
2100c8d2 | 11 | |
0d41cca4 | 12 | int sysctl_tcp_fastopen __read_mostly = TFO_CLIENT_ENABLE; |
10467163 JC |
13 | |
14 | struct tcp_fastopen_context __rcu *tcp_fastopen_ctx; | |
15 | ||
16 | static DEFINE_SPINLOCK(tcp_fastopen_ctx_lock); | |
17 | ||
222e83d2 HFS |
18 | void tcp_fastopen_init_key_once(bool publish) |
19 | { | |
20 | static u8 key[TCP_FASTOPEN_KEY_LENGTH]; | |
21 | ||
22 | /* tcp_fastopen_reset_cipher publishes the new context | |
23 | * atomically, so we allow this race happening here. | |
24 | * | |
25 | * All call sites of tcp_fastopen_cookie_gen also check | |
26 | * for a valid cookie, so this is an acceptable risk. | |
27 | */ | |
28 | if (net_get_random_once(key, sizeof(key)) && publish) | |
29 | tcp_fastopen_reset_cipher(key, sizeof(key)); | |
30 | } | |
31 | ||
10467163 JC |
32 | static void tcp_fastopen_ctx_free(struct rcu_head *head) |
33 | { | |
34 | struct tcp_fastopen_context *ctx = | |
35 | container_of(head, struct tcp_fastopen_context, rcu); | |
36 | crypto_free_cipher(ctx->tfm); | |
37 | kfree(ctx); | |
38 | } | |
39 | ||
40 | int tcp_fastopen_reset_cipher(void *key, unsigned int len) | |
41 | { | |
42 | int err; | |
43 | struct tcp_fastopen_context *ctx, *octx; | |
44 | ||
45 | ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); | |
46 | if (!ctx) | |
47 | return -ENOMEM; | |
48 | ctx->tfm = crypto_alloc_cipher("aes", 0, 0); | |
49 | ||
50 | if (IS_ERR(ctx->tfm)) { | |
51 | err = PTR_ERR(ctx->tfm); | |
52 | error: kfree(ctx); | |
53 | pr_err("TCP: TFO aes cipher alloc error: %d\n", err); | |
54 | return err; | |
55 | } | |
56 | err = crypto_cipher_setkey(ctx->tfm, key, len); | |
57 | if (err) { | |
58 | pr_err("TCP: TFO cipher key error: %d\n", err); | |
59 | crypto_free_cipher(ctx->tfm); | |
60 | goto error; | |
61 | } | |
62 | memcpy(ctx->key, key, len); | |
63 | ||
64 | spin_lock(&tcp_fastopen_ctx_lock); | |
65 | ||
66 | octx = rcu_dereference_protected(tcp_fastopen_ctx, | |
67 | lockdep_is_held(&tcp_fastopen_ctx_lock)); | |
68 | rcu_assign_pointer(tcp_fastopen_ctx, ctx); | |
69 | spin_unlock(&tcp_fastopen_ctx_lock); | |
70 | ||
71 | if (octx) | |
72 | call_rcu(&octx->rcu, tcp_fastopen_ctx_free); | |
73 | return err; | |
74 | } | |
75 | ||
3a19ce0e DL |
76 | static bool __tcp_fastopen_cookie_gen(const void *path, |
77 | struct tcp_fastopen_cookie *foc) | |
10467163 | 78 | { |
10467163 | 79 | struct tcp_fastopen_context *ctx; |
3a19ce0e | 80 | bool ok = false; |
10467163 JC |
81 | |
82 | rcu_read_lock(); | |
83 | ctx = rcu_dereference(tcp_fastopen_ctx); | |
84 | if (ctx) { | |
3a19ce0e | 85 | crypto_cipher_encrypt_one(ctx->tfm, foc->val, path); |
10467163 | 86 | foc->len = TCP_FASTOPEN_COOKIE_SIZE; |
3a19ce0e | 87 | ok = true; |
10467163 JC |
88 | } |
89 | rcu_read_unlock(); | |
3a19ce0e DL |
90 | return ok; |
91 | } | |
92 | ||
93 | /* Generate the fastopen cookie by doing aes128 encryption on both | |
94 | * the source and destination addresses. Pad 0s for IPv4 or IPv4-mapped-IPv6 | |
95 | * addresses. For the longer IPv6 addresses use CBC-MAC. | |
96 | * | |
97 | * XXX (TFO) - refactor when TCP_FASTOPEN_COOKIE_SIZE != AES_BLOCK_SIZE. | |
98 | */ | |
99 | static bool tcp_fastopen_cookie_gen(struct request_sock *req, | |
100 | struct sk_buff *syn, | |
101 | struct tcp_fastopen_cookie *foc) | |
102 | { | |
103 | if (req->rsk_ops->family == AF_INET) { | |
104 | const struct iphdr *iph = ip_hdr(syn); | |
105 | ||
106 | __be32 path[4] = { iph->saddr, iph->daddr, 0, 0 }; | |
107 | return __tcp_fastopen_cookie_gen(path, foc); | |
108 | } | |
109 | ||
110 | #if IS_ENABLED(CONFIG_IPV6) | |
111 | if (req->rsk_ops->family == AF_INET6) { | |
112 | const struct ipv6hdr *ip6h = ipv6_hdr(syn); | |
113 | struct tcp_fastopen_cookie tmp; | |
114 | ||
115 | if (__tcp_fastopen_cookie_gen(&ip6h->saddr, &tmp)) { | |
116 | struct in6_addr *buf = (struct in6_addr *) tmp.val; | |
41c91996 | 117 | int i; |
3a19ce0e DL |
118 | |
119 | for (i = 0; i < 4; i++) | |
120 | buf->s6_addr32[i] ^= ip6h->daddr.s6_addr32[i]; | |
121 | return __tcp_fastopen_cookie_gen(buf, foc); | |
122 | } | |
123 | } | |
124 | #endif | |
125 | return false; | |
10467163 | 126 | } |
5b7ed089 | 127 | |
61d2bcae ED |
128 | |
129 | /* If an incoming SYN or SYNACK frame contains a payload and/or FIN, | |
130 | * queue this additional data / FIN. | |
131 | */ | |
132 | void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb) | |
133 | { | |
134 | struct tcp_sock *tp = tcp_sk(sk); | |
135 | ||
136 | if (TCP_SKB_CB(skb)->end_seq == tp->rcv_nxt) | |
137 | return; | |
138 | ||
139 | skb = skb_clone(skb, GFP_ATOMIC); | |
140 | if (!skb) | |
141 | return; | |
142 | ||
143 | skb_dst_drop(skb); | |
a44d6eac MKL |
144 | /* segs_in has been initialized to 1 in tcp_create_openreq_child(). |
145 | * Hence, reset segs_in to 0 before calling tcp_segs_in() | |
146 | * to avoid double counting. Also, tcp_segs_in() expects | |
147 | * skb->len to include the tcp_hdrlen. Hence, it should | |
148 | * be called before __skb_pull(). | |
149 | */ | |
150 | tp->segs_in = 0; | |
151 | tcp_segs_in(tp, skb); | |
61d2bcae ED |
152 | __skb_pull(skb, tcp_hdrlen(skb)); |
153 | skb_set_owner_r(skb, sk); | |
154 | ||
9d691539 ED |
155 | TCP_SKB_CB(skb)->seq++; |
156 | TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_SYN; | |
157 | ||
61d2bcae ED |
158 | tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; |
159 | __skb_queue_tail(&sk->sk_receive_queue, skb); | |
160 | tp->syn_data_acked = 1; | |
161 | ||
162 | /* u64_stats_update_begin(&tp->syncp) not needed here, | |
163 | * as we certainly are not changing upper 32bit value (0) | |
164 | */ | |
165 | tp->bytes_received = skb->len; | |
e3e17b77 ED |
166 | |
167 | if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) | |
168 | tcp_fin(sk); | |
61d2bcae ED |
169 | } |
170 | ||
7c85af88 ED |
171 | static struct sock *tcp_fastopen_create_child(struct sock *sk, |
172 | struct sk_buff *skb, | |
173 | struct dst_entry *dst, | |
174 | struct request_sock *req) | |
5b7ed089 | 175 | { |
17846376 | 176 | struct tcp_sock *tp; |
5b7ed089 | 177 | struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue; |
5b7ed089 | 178 | struct sock *child; |
5e0724d0 | 179 | bool own_req; |
5b7ed089 YC |
180 | |
181 | req->num_retrans = 0; | |
182 | req->num_timeout = 0; | |
183 | req->sk = NULL; | |
184 | ||
5e0724d0 ED |
185 | child = inet_csk(sk)->icsk_af_ops->syn_recv_sock(sk, skb, req, NULL, |
186 | NULL, &own_req); | |
51456b29 | 187 | if (!child) |
7c85af88 | 188 | return NULL; |
5b7ed089 | 189 | |
0536fcc0 ED |
190 | spin_lock(&queue->fastopenq.lock); |
191 | queue->fastopenq.qlen++; | |
192 | spin_unlock(&queue->fastopenq.lock); | |
5b7ed089 YC |
193 | |
194 | /* Initialize the child socket. Have to fix some values to take | |
195 | * into account the child is a Fast Open socket and is created | |
196 | * only out of the bits carried in the SYN packet. | |
197 | */ | |
198 | tp = tcp_sk(child); | |
199 | ||
200 | tp->fastopen_rsk = req; | |
9439ce00 | 201 | tcp_rsk(req)->tfo_listener = true; |
5b7ed089 YC |
202 | |
203 | /* RFC1323: The window in SYN & SYN/ACK segments is never | |
204 | * scaled. So correct it appropriately. | |
205 | */ | |
206 | tp->snd_wnd = ntohs(tcp_hdr(skb)->window); | |
207 | ||
208 | /* Activate the retrans timer so that SYNACK can be retransmitted. | |
ca6fb065 | 209 | * The request socket is not added to the ehash |
5b7ed089 YC |
210 | * because it's been added to the accept queue directly. |
211 | */ | |
212 | inet_csk_reset_xmit_timer(child, ICSK_TIME_RETRANS, | |
213 | TCP_TIMEOUT_INIT, TCP_RTO_MAX); | |
214 | ||
ca6fb065 | 215 | atomic_set(&req->rsk_refcnt, 2); |
5b7ed089 YC |
216 | |
217 | /* Now finish processing the fastopen child socket. */ | |
218 | inet_csk(child)->icsk_af_ops->rebuild_header(child); | |
219 | tcp_init_congestion_control(child); | |
220 | tcp_mtup_init(child); | |
221 | tcp_init_metrics(child); | |
222 | tcp_init_buffer_space(child); | |
223 | ||
61d2bcae ED |
224 | tp->rcv_nxt = TCP_SKB_CB(skb)->seq + 1; |
225 | ||
226 | tcp_fastopen_add_skb(child, skb); | |
227 | ||
228 | tcp_rsk(req)->rcv_nxt = tp->rcv_nxt; | |
7656d842 ED |
229 | /* tcp_conn_request() is sending the SYNACK, |
230 | * and queues the child into listener accept queue. | |
7c85af88 | 231 | */ |
7c85af88 | 232 | return child; |
5b7ed089 | 233 | } |
5b7ed089 YC |
234 | |
235 | static bool tcp_fastopen_queue_check(struct sock *sk) | |
236 | { | |
237 | struct fastopen_queue *fastopenq; | |
238 | ||
239 | /* Make sure the listener has enabled fastopen, and we don't | |
240 | * exceed the max # of pending TFO requests allowed before trying | |
241 | * to validating the cookie in order to avoid burning CPU cycles | |
242 | * unnecessarily. | |
243 | * | |
244 | * XXX (TFO) - The implication of checking the max_qlen before | |
245 | * processing a cookie request is that clients can't differentiate | |
246 | * between qlen overflow causing Fast Open to be disabled | |
247 | * temporarily vs a server not supporting Fast Open at all. | |
248 | */ | |
0536fcc0 ED |
249 | fastopenq = &inet_csk(sk)->icsk_accept_queue.fastopenq; |
250 | if (fastopenq->max_qlen == 0) | |
5b7ed089 YC |
251 | return false; |
252 | ||
253 | if (fastopenq->qlen >= fastopenq->max_qlen) { | |
254 | struct request_sock *req1; | |
255 | spin_lock(&fastopenq->lock); | |
256 | req1 = fastopenq->rskq_rst_head; | |
fa76ce73 | 257 | if (!req1 || time_after(req1->rsk_timer.expires, jiffies)) { |
5b7ed089 YC |
258 | spin_unlock(&fastopenq->lock); |
259 | NET_INC_STATS_BH(sock_net(sk), | |
260 | LINUX_MIB_TCPFASTOPENLISTENOVERFLOW); | |
261 | return false; | |
262 | } | |
263 | fastopenq->rskq_rst_head = req1->dl_next; | |
264 | fastopenq->qlen--; | |
265 | spin_unlock(&fastopenq->lock); | |
13854e5a | 266 | reqsk_put(req1); |
5b7ed089 YC |
267 | } |
268 | return true; | |
269 | } | |
270 | ||
89278c9d YC |
271 | /* Returns true if we should perform Fast Open on the SYN. The cookie (foc) |
272 | * may be updated and return the client in the SYN-ACK later. E.g., Fast Open | |
273 | * cookie request (foc->len == 0). | |
274 | */ | |
7c85af88 ED |
275 | struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, |
276 | struct request_sock *req, | |
277 | struct tcp_fastopen_cookie *foc, | |
278 | struct dst_entry *dst) | |
5b7ed089 | 279 | { |
89278c9d YC |
280 | struct tcp_fastopen_cookie valid_foc = { .len = -1 }; |
281 | bool syn_data = TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq + 1; | |
7c85af88 | 282 | struct sock *child; |
5b7ed089 | 283 | |
531c94a9 YC |
284 | if (foc->len == 0) /* Client requests a cookie */ |
285 | NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPFASTOPENCOOKIEREQD); | |
286 | ||
89278c9d YC |
287 | if (!((sysctl_tcp_fastopen & TFO_SERVER_ENABLE) && |
288 | (syn_data || foc->len >= 0) && | |
289 | tcp_fastopen_queue_check(sk))) { | |
290 | foc->len = -1; | |
7c85af88 | 291 | return NULL; |
5b7ed089 YC |
292 | } |
293 | ||
89278c9d YC |
294 | if (syn_data && (sysctl_tcp_fastopen & TFO_SERVER_COOKIE_NOT_REQD)) |
295 | goto fastopen; | |
296 | ||
531c94a9 YC |
297 | if (foc->len >= 0 && /* Client presents or requests a cookie */ |
298 | tcp_fastopen_cookie_gen(req, skb, &valid_foc) && | |
3a19ce0e | 299 | foc->len == TCP_FASTOPEN_COOKIE_SIZE && |
89278c9d YC |
300 | foc->len == valid_foc.len && |
301 | !memcmp(foc->val, valid_foc.val, foc->len)) { | |
843f4a55 YC |
302 | /* Cookie is valid. Create a (full) child socket to accept |
303 | * the data in SYN before returning a SYN-ACK to ack the | |
304 | * data. If we fail to create the socket, fall back and | |
305 | * ack the ISN only but includes the same cookie. | |
306 | * | |
307 | * Note: Data-less SYN with valid cookie is allowed to send | |
308 | * data in SYN_RECV state. | |
309 | */ | |
89278c9d | 310 | fastopen: |
7c85af88 ED |
311 | child = tcp_fastopen_create_child(sk, skb, dst, req); |
312 | if (child) { | |
843f4a55 YC |
313 | foc->len = -1; |
314 | NET_INC_STATS_BH(sock_net(sk), | |
315 | LINUX_MIB_TCPFASTOPENPASSIVE); | |
7c85af88 | 316 | return child; |
843f4a55 | 317 | } |
531c94a9 YC |
318 | NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPFASTOPENPASSIVEFAIL); |
319 | } else if (foc->len > 0) /* Client presents an invalid cookie */ | |
320 | NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPFASTOPENPASSIVEFAIL); | |
89278c9d | 321 | |
7f9b838b | 322 | valid_foc.exp = foc->exp; |
89278c9d | 323 | *foc = valid_foc; |
7c85af88 | 324 | return NULL; |
5b7ed089 | 325 | } |