Commit | Line | Data |
---|---|---|
70041088 AG |
1 | /* |
2 | * Copyright (c) 2006 Oracle. All rights reserved. | |
3 | * | |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | * | |
32 | */ | |
33 | #include <linux/kernel.h> | |
34 | #include <linux/in.h> | |
35 | #include <net/tcp.h> | |
36 | ||
0cb43965 | 37 | #include "rds_single_path.h" |
70041088 AG |
38 | #include "rds.h" |
39 | #include "tcp.h" | |
40 | ||
41 | static void rds_tcp_cork(struct socket *sock, int val) | |
42 | { | |
43 | mm_segment_t oldfs; | |
44 | ||
45 | oldfs = get_fs(); | |
46 | set_fs(KERNEL_DS); | |
47 | sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (char __user *)&val, | |
48 | sizeof(val)); | |
49 | set_fs(oldfs); | |
50 | } | |
51 | ||
226f7a7d | 52 | void rds_tcp_xmit_path_prepare(struct rds_conn_path *cp) |
70041088 | 53 | { |
226f7a7d | 54 | struct rds_tcp_connection *tc = cp->cp_transport_data; |
70041088 AG |
55 | |
56 | rds_tcp_cork(tc->t_sock, 1); | |
57 | } | |
58 | ||
226f7a7d | 59 | void rds_tcp_xmit_path_complete(struct rds_conn_path *cp) |
70041088 | 60 | { |
226f7a7d | 61 | struct rds_tcp_connection *tc = cp->cp_transport_data; |
70041088 AG |
62 | |
63 | rds_tcp_cork(tc->t_sock, 0); | |
64 | } | |
65 | ||
66 | /* the core send_sem serializes this with other xmit and shutdown */ | |
ff51bf84 | 67 | static int rds_tcp_sendmsg(struct socket *sock, void *data, unsigned int len) |
70041088 AG |
68 | { |
69 | struct kvec vec = { | |
5c3da57d JH |
70 | .iov_base = data, |
71 | .iov_len = len, | |
72 | }; | |
73 | struct msghdr msg = { | |
74 | .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL, | |
70041088 | 75 | }; |
70041088 AG |
76 | |
77 | return kernel_sendmsg(sock, &msg, &vec, 1, vec.iov_len); | |
78 | } | |
79 | ||
70041088 AG |
80 | /* the core send_sem serializes this with other xmit and shutdown */ |
81 | int rds_tcp_xmit(struct rds_connection *conn, struct rds_message *rm, | |
5c3da57d | 82 | unsigned int hdr_off, unsigned int sg, unsigned int off) |
70041088 | 83 | { |
5916e2c1 SV |
84 | struct rds_conn_path *cp = rm->m_inc.i_conn_path; |
85 | struct rds_tcp_connection *tc = cp->cp_transport_data; | |
70041088 AG |
86 | int done = 0; |
87 | int ret = 0; | |
76b29ef1 | 88 | int more; |
70041088 AG |
89 | |
90 | if (hdr_off == 0) { | |
91 | /* | |
92 | * m_ack_seq is set to the sequence number of the last byte of | |
93 | * header and data. see rds_tcp_is_acked(). | |
94 | */ | |
95 | tc->t_last_sent_nxt = rds_tcp_snd_nxt(tc); | |
96 | rm->m_ack_seq = tc->t_last_sent_nxt + | |
97 | sizeof(struct rds_header) + | |
98 | be32_to_cpu(rm->m_inc.i_hdr.h_len) - 1; | |
4e857c58 | 99 | smp_mb__before_atomic(); |
70041088 AG |
100 | set_bit(RDS_MSG_HAS_ACK_SEQ, &rm->m_flags); |
101 | tc->t_last_expected_una = rm->m_ack_seq + 1; | |
102 | ||
103 | rdsdebug("rm %p tcp nxt %u ack_seq %llu\n", | |
104 | rm, rds_tcp_snd_nxt(tc), | |
105 | (unsigned long long)rm->m_ack_seq); | |
106 | } | |
107 | ||
108 | if (hdr_off < sizeof(struct rds_header)) { | |
109 | /* see rds_tcp_write_space() */ | |
110 | set_bit(SOCK_NOSPACE, &tc->t_sock->sk->sk_socket->flags); | |
111 | ||
112 | ret = rds_tcp_sendmsg(tc->t_sock, | |
113 | (void *)&rm->m_inc.i_hdr + hdr_off, | |
114 | sizeof(rm->m_inc.i_hdr) - hdr_off); | |
115 | if (ret < 0) | |
116 | goto out; | |
117 | done += ret; | |
118 | if (hdr_off + done != sizeof(struct rds_header)) | |
119 | goto out; | |
120 | } | |
121 | ||
76b29ef1 | 122 | more = rm->data.op_nents > 1 ? (MSG_MORE | MSG_SENDPAGE_NOTLAST) : 0; |
6c7cc6e4 | 123 | while (sg < rm->data.op_nents) { |
76b29ef1 SV |
124 | int flags = MSG_DONTWAIT | MSG_NOSIGNAL | more; |
125 | ||
70041088 | 126 | ret = tc->t_sock->ops->sendpage(tc->t_sock, |
6c7cc6e4 AG |
127 | sg_page(&rm->data.op_sg[sg]), |
128 | rm->data.op_sg[sg].offset + off, | |
129 | rm->data.op_sg[sg].length - off, | |
76b29ef1 | 130 | flags); |
6c7cc6e4 AG |
131 | rdsdebug("tcp sendpage %p:%u:%u ret %d\n", (void *)sg_page(&rm->data.op_sg[sg]), |
132 | rm->data.op_sg[sg].offset + off, rm->data.op_sg[sg].length - off, | |
70041088 AG |
133 | ret); |
134 | if (ret <= 0) | |
135 | break; | |
136 | ||
137 | off += ret; | |
138 | done += ret; | |
6c7cc6e4 | 139 | if (off == rm->data.op_sg[sg].length) { |
70041088 AG |
140 | off = 0; |
141 | sg++; | |
142 | } | |
76b29ef1 SV |
143 | if (sg == rm->data.op_nents - 1) |
144 | more = 0; | |
70041088 AG |
145 | } |
146 | ||
147 | out: | |
148 | if (ret <= 0) { | |
149 | /* write_space will hit after EAGAIN, all else fatal */ | |
150 | if (ret == -EAGAIN) { | |
151 | rds_tcp_stats_inc(s_tcp_sndbuf_full); | |
152 | ret = 0; | |
153 | } else { | |
5916e2c1 SV |
154 | /* No need to disconnect/reconnect if path_drop |
155 | * has already been triggered, because, e.g., of | |
156 | * an incoming RST. | |
157 | */ | |
158 | if (rds_conn_path_up(cp)) { | |
159 | pr_warn("RDS/tcp: send to %pI4 on cp [%d]" | |
160 | "returned %d, " | |
161 | "disconnecting and reconnecting\n", | |
162 | &conn->c_faddr, cp->cp_index, ret); | |
163 | rds_conn_path_drop(cp); | |
164 | } | |
70041088 AG |
165 | } |
166 | } | |
167 | if (done == 0) | |
168 | done = ret; | |
169 | return done; | |
170 | } | |
171 | ||
172 | /* | |
173 | * rm->m_ack_seq is set to the tcp sequence number that corresponds to the | |
174 | * last byte of the message, including the header. This means that the | |
175 | * entire message has been received if rm->m_ack_seq is "before" the next | |
176 | * unacked byte of the TCP sequence space. We have to do very careful | |
177 | * wrapping 32bit comparisons here. | |
178 | */ | |
179 | static int rds_tcp_is_acked(struct rds_message *rm, uint64_t ack) | |
180 | { | |
181 | if (!test_bit(RDS_MSG_HAS_ACK_SEQ, &rm->m_flags)) | |
182 | return 0; | |
183 | return (__s32)((u32)rm->m_ack_seq - (u32)ack) < 0; | |
184 | } | |
185 | ||
186 | void rds_tcp_write_space(struct sock *sk) | |
187 | { | |
188 | void (*write_space)(struct sock *sk); | |
ea3b1ea5 | 189 | struct rds_conn_path *cp; |
70041088 AG |
190 | struct rds_tcp_connection *tc; |
191 | ||
38036629 | 192 | read_lock_bh(&sk->sk_callback_lock); |
ea3b1ea5 SV |
193 | cp = sk->sk_user_data; |
194 | if (!cp) { | |
70041088 AG |
195 | write_space = sk->sk_write_space; |
196 | goto out; | |
197 | } | |
198 | ||
ea3b1ea5 | 199 | tc = cp->cp_transport_data; |
70041088 AG |
200 | rdsdebug("write_space for tc %p\n", tc); |
201 | write_space = tc->t_orig_write_space; | |
202 | rds_tcp_stats_inc(s_tcp_write_space_calls); | |
203 | ||
204 | rdsdebug("tcp una %u\n", rds_tcp_snd_una(tc)); | |
205 | tc->t_last_seen_una = rds_tcp_snd_una(tc); | |
ea3b1ea5 | 206 | rds_send_path_drop_acked(cp, rds_tcp_snd_una(tc), rds_tcp_is_acked); |
70041088 | 207 | |
5c3da57d | 208 | if ((atomic_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) |
ea3b1ea5 | 209 | queue_delayed_work(rds_wq, &cp->cp_send_w, 0); |
8e82376e | 210 | |
70041088 | 211 | out: |
38036629 | 212 | read_unlock_bh(&sk->sk_callback_lock); |
70041088 AG |
213 | |
214 | /* | |
215 | * write_space is only called when data leaves tcp's send queue if | |
216 | * SOCK_NOSPACE is set. We set SOCK_NOSPACE every time we put | |
217 | * data in tcp's send queue because we use write_space to parse the | |
218 | * sequence numbers and notice that rds messages have been fully | |
219 | * received. | |
220 | * | |
221 | * tcp's write_space clears SOCK_NOSPACE if the send queue has more | |
222 | * than a certain amount of space. So we need to set it again *after* | |
223 | * we call tcp's write_space or else we might only get called on the | |
224 | * first of a series of incoming tcp acks. | |
225 | */ | |
226 | write_space(sk); | |
227 | ||
228 | if (sk->sk_socket) | |
229 | set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); | |
230 | } |