Commit | Line | Data |
---|---|---|
70041088 AG |
1 | /* |
2 | * Copyright (c) 2006 Oracle. All rights reserved. | |
3 | * | |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | * | |
32 | */ | |
33 | #include <linux/kernel.h> | |
34 | #include <linux/in.h> | |
35 | #include <net/tcp.h> | |
36 | ||
37 | #include "rds.h" | |
38 | #include "tcp.h" | |
39 | ||
40 | static void rds_tcp_cork(struct socket *sock, int val) | |
41 | { | |
42 | mm_segment_t oldfs; | |
43 | ||
44 | oldfs = get_fs(); | |
45 | set_fs(KERNEL_DS); | |
46 | sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (char __user *)&val, | |
47 | sizeof(val)); | |
48 | set_fs(oldfs); | |
49 | } | |
50 | ||
51 | void rds_tcp_xmit_prepare(struct rds_connection *conn) | |
52 | { | |
53 | struct rds_tcp_connection *tc = conn->c_transport_data; | |
54 | ||
55 | rds_tcp_cork(tc->t_sock, 1); | |
56 | } | |
57 | ||
58 | void rds_tcp_xmit_complete(struct rds_connection *conn) | |
59 | { | |
60 | struct rds_tcp_connection *tc = conn->c_transport_data; | |
61 | ||
62 | rds_tcp_cork(tc->t_sock, 0); | |
63 | } | |
64 | ||
65 | /* the core send_sem serializes this with other xmit and shutdown */ | |
ff51bf84 | 66 | static int rds_tcp_sendmsg(struct socket *sock, void *data, unsigned int len) |
70041088 AG |
67 | { |
68 | struct kvec vec = { | |
69 | .iov_base = data, | |
70 | .iov_len = len, | |
71 | }; | |
72 | struct msghdr msg = { | |
73 | .msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL, | |
74 | }; | |
75 | ||
76 | return kernel_sendmsg(sock, &msg, &vec, 1, vec.iov_len); | |
77 | } | |
78 | ||
70041088 AG |
79 | /* the core send_sem serializes this with other xmit and shutdown */ |
80 | int rds_tcp_xmit(struct rds_connection *conn, struct rds_message *rm, | |
81 | unsigned int hdr_off, unsigned int sg, unsigned int off) | |
82 | { | |
83 | struct rds_tcp_connection *tc = conn->c_transport_data; | |
84 | int done = 0; | |
85 | int ret = 0; | |
86 | ||
87 | if (hdr_off == 0) { | |
88 | /* | |
89 | * m_ack_seq is set to the sequence number of the last byte of | |
90 | * header and data. see rds_tcp_is_acked(). | |
91 | */ | |
92 | tc->t_last_sent_nxt = rds_tcp_snd_nxt(tc); | |
93 | rm->m_ack_seq = tc->t_last_sent_nxt + | |
94 | sizeof(struct rds_header) + | |
95 | be32_to_cpu(rm->m_inc.i_hdr.h_len) - 1; | |
96 | smp_mb__before_clear_bit(); | |
97 | set_bit(RDS_MSG_HAS_ACK_SEQ, &rm->m_flags); | |
98 | tc->t_last_expected_una = rm->m_ack_seq + 1; | |
99 | ||
100 | rdsdebug("rm %p tcp nxt %u ack_seq %llu\n", | |
101 | rm, rds_tcp_snd_nxt(tc), | |
102 | (unsigned long long)rm->m_ack_seq); | |
103 | } | |
104 | ||
105 | if (hdr_off < sizeof(struct rds_header)) { | |
106 | /* see rds_tcp_write_space() */ | |
107 | set_bit(SOCK_NOSPACE, &tc->t_sock->sk->sk_socket->flags); | |
108 | ||
109 | ret = rds_tcp_sendmsg(tc->t_sock, | |
110 | (void *)&rm->m_inc.i_hdr + hdr_off, | |
111 | sizeof(rm->m_inc.i_hdr) - hdr_off); | |
112 | if (ret < 0) | |
113 | goto out; | |
114 | done += ret; | |
115 | if (hdr_off + done != sizeof(struct rds_header)) | |
116 | goto out; | |
117 | } | |
118 | ||
6c7cc6e4 | 119 | while (sg < rm->data.op_nents) { |
70041088 | 120 | ret = tc->t_sock->ops->sendpage(tc->t_sock, |
6c7cc6e4 AG |
121 | sg_page(&rm->data.op_sg[sg]), |
122 | rm->data.op_sg[sg].offset + off, | |
123 | rm->data.op_sg[sg].length - off, | |
70041088 | 124 | MSG_DONTWAIT|MSG_NOSIGNAL); |
6c7cc6e4 AG |
125 | rdsdebug("tcp sendpage %p:%u:%u ret %d\n", (void *)sg_page(&rm->data.op_sg[sg]), |
126 | rm->data.op_sg[sg].offset + off, rm->data.op_sg[sg].length - off, | |
70041088 AG |
127 | ret); |
128 | if (ret <= 0) | |
129 | break; | |
130 | ||
131 | off += ret; | |
132 | done += ret; | |
6c7cc6e4 | 133 | if (off == rm->data.op_sg[sg].length) { |
70041088 AG |
134 | off = 0; |
135 | sg++; | |
136 | } | |
137 | } | |
138 | ||
139 | out: | |
140 | if (ret <= 0) { | |
141 | /* write_space will hit after EAGAIN, all else fatal */ | |
142 | if (ret == -EAGAIN) { | |
143 | rds_tcp_stats_inc(s_tcp_sndbuf_full); | |
144 | ret = 0; | |
145 | } else { | |
6884b348 | 146 | printk(KERN_WARNING "RDS/tcp: send to %pI4 " |
70041088 | 147 | "returned %d, disconnecting and reconnecting\n", |
6884b348 | 148 | &conn->c_faddr, ret); |
70041088 AG |
149 | rds_conn_drop(conn); |
150 | } | |
151 | } | |
152 | if (done == 0) | |
153 | done = ret; | |
154 | return done; | |
155 | } | |
156 | ||
157 | /* | |
158 | * rm->m_ack_seq is set to the tcp sequence number that corresponds to the | |
159 | * last byte of the message, including the header. This means that the | |
160 | * entire message has been received if rm->m_ack_seq is "before" the next | |
161 | * unacked byte of the TCP sequence space. We have to do very careful | |
162 | * wrapping 32bit comparisons here. | |
163 | */ | |
164 | static int rds_tcp_is_acked(struct rds_message *rm, uint64_t ack) | |
165 | { | |
166 | if (!test_bit(RDS_MSG_HAS_ACK_SEQ, &rm->m_flags)) | |
167 | return 0; | |
168 | return (__s32)((u32)rm->m_ack_seq - (u32)ack) < 0; | |
169 | } | |
170 | ||
171 | void rds_tcp_write_space(struct sock *sk) | |
172 | { | |
173 | void (*write_space)(struct sock *sk); | |
174 | struct rds_connection *conn; | |
175 | struct rds_tcp_connection *tc; | |
176 | ||
f064af1e | 177 | read_lock_bh(&sk->sk_callback_lock); |
70041088 | 178 | conn = sk->sk_user_data; |
8690bfa1 | 179 | if (!conn) { |
70041088 AG |
180 | write_space = sk->sk_write_space; |
181 | goto out; | |
182 | } | |
183 | ||
184 | tc = conn->c_transport_data; | |
185 | rdsdebug("write_space for tc %p\n", tc); | |
186 | write_space = tc->t_orig_write_space; | |
187 | rds_tcp_stats_inc(s_tcp_write_space_calls); | |
188 | ||
189 | rdsdebug("tcp una %u\n", rds_tcp_snd_una(tc)); | |
190 | tc->t_last_seen_una = rds_tcp_snd_una(tc); | |
191 | rds_send_drop_acked(conn, rds_tcp_snd_una(tc), rds_tcp_is_acked); | |
192 | ||
8e82376e AG |
193 | if ((atomic_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) |
194 | queue_delayed_work(rds_wq, &conn->c_send_w, 0); | |
195 | ||
70041088 | 196 | out: |
f064af1e | 197 | read_unlock_bh(&sk->sk_callback_lock); |
70041088 AG |
198 | |
199 | /* | |
200 | * write_space is only called when data leaves tcp's send queue if | |
201 | * SOCK_NOSPACE is set. We set SOCK_NOSPACE every time we put | |
202 | * data in tcp's send queue because we use write_space to parse the | |
203 | * sequence numbers and notice that rds messages have been fully | |
204 | * received. | |
205 | * | |
206 | * tcp's write_space clears SOCK_NOSPACE if the send queue has more | |
207 | * than a certain amount of space. So we need to set it again *after* | |
208 | * we call tcp's write_space or else we might only get called on the | |
209 | * first of a series of incoming tcp acks. | |
210 | */ | |
211 | write_space(sk); | |
212 | ||
213 | if (sk->sk_socket) | |
214 | set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); | |
215 | } |