2 * Copyright (c) 2006 Oracle. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/kernel.h>
34 #include <linux/slab.h>
37 #include <linux/export.h>
38 #include <linux/time.h>
39 #include <linux/rds.h>
43 void rds_inc_init(struct rds_incoming
*inc
, struct rds_connection
*conn
,
46 atomic_set(&inc
->i_refcount
, 1);
47 INIT_LIST_HEAD(&inc
->i_item
);
50 inc
->i_rdma_cookie
= 0;
51 inc
->i_rx_tstamp
.tv_sec
= 0;
52 inc
->i_rx_tstamp
.tv_usec
= 0;
54 EXPORT_SYMBOL_GPL(rds_inc_init
);
56 void rds_inc_path_init(struct rds_incoming
*inc
, struct rds_conn_path
*cp
,
59 atomic_set(&inc
->i_refcount
, 1);
60 INIT_LIST_HEAD(&inc
->i_item
);
61 inc
->i_conn
= cp
->cp_conn
;
62 inc
->i_conn_path
= cp
;
64 inc
->i_rdma_cookie
= 0;
65 inc
->i_rx_tstamp
.tv_sec
= 0;
66 inc
->i_rx_tstamp
.tv_usec
= 0;
68 EXPORT_SYMBOL_GPL(rds_inc_path_init
);
70 static void rds_inc_addref(struct rds_incoming
*inc
)
72 rdsdebug("addref inc %p ref %d\n", inc
, atomic_read(&inc
->i_refcount
));
73 atomic_inc(&inc
->i_refcount
);
76 void rds_inc_put(struct rds_incoming
*inc
)
78 rdsdebug("put inc %p ref %d\n", inc
, atomic_read(&inc
->i_refcount
));
79 if (atomic_dec_and_test(&inc
->i_refcount
)) {
80 BUG_ON(!list_empty(&inc
->i_item
));
82 inc
->i_conn
->c_trans
->inc_free(inc
);
85 EXPORT_SYMBOL_GPL(rds_inc_put
);
87 static void rds_recv_rcvbuf_delta(struct rds_sock
*rs
, struct sock
*sk
,
88 struct rds_cong_map
*map
,
89 int delta
, __be16 port
)
96 rs
->rs_rcv_bytes
+= delta
;
97 now_congested
= rs
->rs_rcv_bytes
> rds_sk_rcvbuf(rs
);
99 rdsdebug("rs %p (%pI4:%u) recv bytes %d buf %d "
100 "now_cong %d delta %d\n",
101 rs
, &rs
->rs_bound_addr
,
102 ntohs(rs
->rs_bound_port
), rs
->rs_rcv_bytes
,
103 rds_sk_rcvbuf(rs
), now_congested
, delta
);
105 /* wasn't -> am congested */
106 if (!rs
->rs_congested
&& now_congested
) {
107 rs
->rs_congested
= 1;
108 rds_cong_set_bit(map
, port
);
109 rds_cong_queue_updates(map
);
111 /* was -> aren't congested */
112 /* Require more free space before reporting uncongested to prevent
113 bouncing cong/uncong state too often */
114 else if (rs
->rs_congested
&& (rs
->rs_rcv_bytes
< (rds_sk_rcvbuf(rs
)/2))) {
115 rs
->rs_congested
= 0;
116 rds_cong_clear_bit(map
, port
);
117 rds_cong_queue_updates(map
);
120 /* do nothing if no change in cong state */
124 * Process all extension headers that come with this message.
126 static void rds_recv_incoming_exthdrs(struct rds_incoming
*inc
, struct rds_sock
*rs
)
128 struct rds_header
*hdr
= &inc
->i_hdr
;
129 unsigned int pos
= 0, type
, len
;
131 struct rds_ext_header_version version
;
132 struct rds_ext_header_rdma rdma
;
133 struct rds_ext_header_rdma_dest rdma_dest
;
137 len
= sizeof(buffer
);
138 type
= rds_message_next_extension(hdr
, &pos
, &buffer
, &len
);
139 if (type
== RDS_EXTHDR_NONE
)
141 /* Process extension header here */
143 case RDS_EXTHDR_RDMA
:
144 rds_rdma_unuse(rs
, be32_to_cpu(buffer
.rdma
.h_rdma_rkey
), 0);
147 case RDS_EXTHDR_RDMA_DEST
:
148 /* We ignore the size for now. We could stash it
149 * somewhere and use it for error checking. */
150 inc
->i_rdma_cookie
= rds_rdma_make_cookie(
151 be32_to_cpu(buffer
.rdma_dest
.h_rdma_rkey
),
152 be32_to_cpu(buffer
.rdma_dest
.h_rdma_offset
));
160 * The transport must make sure that this is serialized against other
161 * rx and conn reset on this specific conn.
163 * We currently assert that only one fragmented message will be sent
164 * down a connection at a time. This lets us reassemble in the conn
165 * instead of per-flow which means that we don't have to go digging through
166 * flows to tear down partial reassembly progress on conn failure and
167 * we save flow lookup and locking for each frag arrival. It does mean
168 * that small messages will wait behind large ones. Fragmenting at all
169 * is only to reduce the memory consumption of pre-posted buffers.
171 * The caller passes in saddr and daddr instead of us getting it from the
172 * conn. This lets loopback, who only has one conn for both directions,
173 * tell us which roles the addrs in the conn are playing for this message.
175 void rds_recv_incoming(struct rds_connection
*conn
, __be32 saddr
, __be32 daddr
,
176 struct rds_incoming
*inc
, gfp_t gfp
)
178 struct rds_sock
*rs
= NULL
;
181 struct rds_conn_path
*cp
;
184 inc
->i_rx_jiffies
= jiffies
;
185 if (conn
->c_trans
->t_mp_capable
)
186 cp
= inc
->i_conn_path
;
188 cp
= &conn
->c_path
[0];
190 rdsdebug("conn %p next %llu inc %p seq %llu len %u sport %u dport %u "
191 "flags 0x%x rx_jiffies %lu\n", conn
,
192 (unsigned long long)cp
->cp_next_rx_seq
,
194 (unsigned long long)be64_to_cpu(inc
->i_hdr
.h_sequence
),
195 be32_to_cpu(inc
->i_hdr
.h_len
),
196 be16_to_cpu(inc
->i_hdr
.h_sport
),
197 be16_to_cpu(inc
->i_hdr
.h_dport
),
202 * Sequence numbers should only increase. Messages get their
203 * sequence number as they're queued in a sending conn. They
204 * can be dropped, though, if the sending socket is closed before
205 * they hit the wire. So sequence numbers can skip forward
206 * under normal operation. They can also drop back in the conn
207 * failover case as previously sent messages are resent down the
208 * new instance of a conn. We drop those, otherwise we have
209 * to assume that the next valid seq does not come after a
210 * hole in the fragment stream.
212 * The headers don't give us a way to realize if fragments of
213 * a message have been dropped. We assume that frags that arrive
214 * to a flow are part of the current message on the flow that is
215 * being reassembled. This means that senders can't drop messages
216 * from the sending conn until all their frags are sent.
218 * XXX we could spend more on the wire to get more robust failure
219 * detection, arguably worth it to avoid data corruption.
221 if (be64_to_cpu(inc
->i_hdr
.h_sequence
) < cp
->cp_next_rx_seq
&&
222 (inc
->i_hdr
.h_flags
& RDS_FLAG_RETRANSMITTED
)) {
223 rds_stats_inc(s_recv_drop_old_seq
);
226 cp
->cp_next_rx_seq
= be64_to_cpu(inc
->i_hdr
.h_sequence
) + 1;
228 if (rds_sysctl_ping_enable
&& inc
->i_hdr
.h_dport
== 0) {
229 if (inc
->i_hdr
.h_sport
== 0) {
230 rdsdebug("ignore ping with 0 sport from 0x%x\n", saddr
);
233 rds_stats_inc(s_recv_ping
);
234 rds_send_pong(cp
, inc
->i_hdr
.h_sport
);
238 rs
= rds_find_bound(daddr
, inc
->i_hdr
.h_dport
);
240 rds_stats_inc(s_recv_drop_no_sock
);
244 /* Process extension headers */
245 rds_recv_incoming_exthdrs(inc
, rs
);
247 /* We can be racing with rds_release() which marks the socket dead. */
248 sk
= rds_rs_to_sk(rs
);
250 /* serialize with rds_release -> sock_orphan */
251 write_lock_irqsave(&rs
->rs_recv_lock
, flags
);
252 if (!sock_flag(sk
, SOCK_DEAD
)) {
253 rdsdebug("adding inc %p to rs %p's recv queue\n", inc
, rs
);
254 rds_stats_inc(s_recv_queued
);
255 rds_recv_rcvbuf_delta(rs
, sk
, inc
->i_conn
->c_lcong
,
256 be32_to_cpu(inc
->i_hdr
.h_len
),
258 if (sock_flag(sk
, SOCK_RCVTSTAMP
))
259 do_gettimeofday(&inc
->i_rx_tstamp
);
261 list_add_tail(&inc
->i_item
, &rs
->rs_recv_queue
);
262 __rds_wake_sk_sleep(sk
);
264 rds_stats_inc(s_recv_drop_dead_sock
);
266 write_unlock_irqrestore(&rs
->rs_recv_lock
, flags
);
272 EXPORT_SYMBOL_GPL(rds_recv_incoming
);
275 * be very careful here. This is being called as the condition in
276 * wait_event_*() needs to cope with being called many times.
278 static int rds_next_incoming(struct rds_sock
*rs
, struct rds_incoming
**inc
)
283 read_lock_irqsave(&rs
->rs_recv_lock
, flags
);
284 if (!list_empty(&rs
->rs_recv_queue
)) {
285 *inc
= list_entry(rs
->rs_recv_queue
.next
,
288 rds_inc_addref(*inc
);
290 read_unlock_irqrestore(&rs
->rs_recv_lock
, flags
);
296 static int rds_still_queued(struct rds_sock
*rs
, struct rds_incoming
*inc
,
299 struct sock
*sk
= rds_rs_to_sk(rs
);
303 write_lock_irqsave(&rs
->rs_recv_lock
, flags
);
304 if (!list_empty(&inc
->i_item
)) {
307 /* XXX make sure this i_conn is reliable */
308 rds_recv_rcvbuf_delta(rs
, sk
, inc
->i_conn
->c_lcong
,
309 -be32_to_cpu(inc
->i_hdr
.h_len
),
311 list_del_init(&inc
->i_item
);
315 write_unlock_irqrestore(&rs
->rs_recv_lock
, flags
);
317 rdsdebug("inc %p rs %p still %d dropped %d\n", inc
, rs
, ret
, drop
);
322 * Pull errors off the error queue.
323 * If msghdr is NULL, we will just purge the error queue.
325 int rds_notify_queue_get(struct rds_sock
*rs
, struct msghdr
*msghdr
)
327 struct rds_notifier
*notifier
;
328 struct rds_rdma_notify cmsg
= { 0 }; /* fill holes with zero */
329 unsigned int count
= 0, max_messages
= ~0U;
335 /* put_cmsg copies to user space and thus may sleep. We can't do this
336 * with rs_lock held, so first grab as many notifications as we can stuff
337 * in the user provided cmsg buffer. We don't try to copy more, to avoid
338 * losing notifications - except when the buffer is so small that it wouldn't
339 * even hold a single notification. Then we give him as much of this single
340 * msg as we can squeeze in, and set MSG_CTRUNC.
343 max_messages
= msghdr
->msg_controllen
/ CMSG_SPACE(sizeof(cmsg
));
348 spin_lock_irqsave(&rs
->rs_lock
, flags
);
349 while (!list_empty(&rs
->rs_notify_queue
) && count
< max_messages
) {
350 notifier
= list_entry(rs
->rs_notify_queue
.next
,
351 struct rds_notifier
, n_list
);
352 list_move(¬ifier
->n_list
, ©
);
355 spin_unlock_irqrestore(&rs
->rs_lock
, flags
);
360 while (!list_empty(©
)) {
361 notifier
= list_entry(copy
.next
, struct rds_notifier
, n_list
);
364 cmsg
.user_token
= notifier
->n_user_token
;
365 cmsg
.status
= notifier
->n_status
;
367 err
= put_cmsg(msghdr
, SOL_RDS
, RDS_CMSG_RDMA_STATUS
,
368 sizeof(cmsg
), &cmsg
);
373 list_del_init(¬ifier
->n_list
);
377 /* If we bailed out because of an error in put_cmsg,
378 * we may be left with one or more notifications that we
379 * didn't process. Return them to the head of the list. */
380 if (!list_empty(©
)) {
381 spin_lock_irqsave(&rs
->rs_lock
, flags
);
382 list_splice(©
, &rs
->rs_notify_queue
);
383 spin_unlock_irqrestore(&rs
->rs_lock
, flags
);
390 * Queue a congestion notification
392 static int rds_notify_cong(struct rds_sock
*rs
, struct msghdr
*msghdr
)
394 uint64_t notify
= rs
->rs_cong_notify
;
398 err
= put_cmsg(msghdr
, SOL_RDS
, RDS_CMSG_CONG_UPDATE
,
399 sizeof(notify
), ¬ify
);
403 spin_lock_irqsave(&rs
->rs_lock
, flags
);
404 rs
->rs_cong_notify
&= ~notify
;
405 spin_unlock_irqrestore(&rs
->rs_lock
, flags
);
411 * Receive any control messages.
413 static int rds_cmsg_recv(struct rds_incoming
*inc
, struct msghdr
*msg
,
418 if (inc
->i_rdma_cookie
) {
419 ret
= put_cmsg(msg
, SOL_RDS
, RDS_CMSG_RDMA_DEST
,
420 sizeof(inc
->i_rdma_cookie
), &inc
->i_rdma_cookie
);
425 if ((inc
->i_rx_tstamp
.tv_sec
!= 0) &&
426 sock_flag(rds_rs_to_sk(rs
), SOCK_RCVTSTAMP
)) {
427 ret
= put_cmsg(msg
, SOL_SOCKET
, SCM_TIMESTAMP
,
428 sizeof(struct timeval
),
437 int rds_recvmsg(struct socket
*sock
, struct msghdr
*msg
, size_t size
,
440 struct sock
*sk
= sock
->sk
;
441 struct rds_sock
*rs
= rds_sk_to_rs(sk
);
443 int ret
= 0, nonblock
= msg_flags
& MSG_DONTWAIT
;
444 DECLARE_SOCKADDR(struct sockaddr_in
*, sin
, msg
->msg_name
);
445 struct rds_incoming
*inc
= NULL
;
447 /* udp_recvmsg()->sock_recvtimeo() gets away without locking too.. */
448 timeo
= sock_rcvtimeo(sk
, nonblock
);
450 rdsdebug("size %zu flags 0x%x timeo %ld\n", size
, msg_flags
, timeo
);
452 if (msg_flags
& MSG_OOB
)
456 struct iov_iter save
;
457 /* If there are pending notifications, do those - and nothing else */
458 if (!list_empty(&rs
->rs_notify_queue
)) {
459 ret
= rds_notify_queue_get(rs
, msg
);
463 if (rs
->rs_cong_notify
) {
464 ret
= rds_notify_cong(rs
, msg
);
468 if (!rds_next_incoming(rs
, &inc
)) {
474 timeo
= wait_event_interruptible_timeout(*sk_sleep(sk
),
475 (!list_empty(&rs
->rs_notify_queue
) ||
476 rs
->rs_cong_notify
||
477 rds_next_incoming(rs
, &inc
)), timeo
);
478 rdsdebug("recvmsg woke inc %p timeo %ld\n", inc
,
480 if (timeo
> 0 || timeo
== MAX_SCHEDULE_TIMEOUT
)
489 rdsdebug("copying inc %p from %pI4:%u to user\n", inc
,
490 &inc
->i_conn
->c_faddr
,
491 ntohs(inc
->i_hdr
.h_sport
));
492 save
= msg
->msg_iter
;
493 ret
= inc
->i_conn
->c_trans
->inc_copy_to_user(inc
, &msg
->msg_iter
);
498 * if the message we just copied isn't at the head of the
499 * recv queue then someone else raced us to return it, try
500 * to get the next message.
502 if (!rds_still_queued(rs
, inc
, !(msg_flags
& MSG_PEEK
))) {
505 rds_stats_inc(s_recv_deliver_raced
);
506 msg
->msg_iter
= save
;
510 if (ret
< be32_to_cpu(inc
->i_hdr
.h_len
)) {
511 if (msg_flags
& MSG_TRUNC
)
512 ret
= be32_to_cpu(inc
->i_hdr
.h_len
);
513 msg
->msg_flags
|= MSG_TRUNC
;
516 if (rds_cmsg_recv(inc
, msg
, rs
)) {
521 rds_stats_inc(s_recv_delivered
);
524 sin
->sin_family
= AF_INET
;
525 sin
->sin_port
= inc
->i_hdr
.h_sport
;
526 sin
->sin_addr
.s_addr
= inc
->i_saddr
;
527 memset(sin
->sin_zero
, 0, sizeof(sin
->sin_zero
));
528 msg
->msg_namelen
= sizeof(*sin
);
541 * The socket is being shut down and we're asked to drop messages that were
542 * queued for recvmsg. The caller has unbound the socket so the receive path
543 * won't queue any more incoming fragments or messages on the socket.
545 void rds_clear_recv_queue(struct rds_sock
*rs
)
547 struct sock
*sk
= rds_rs_to_sk(rs
);
548 struct rds_incoming
*inc
, *tmp
;
551 write_lock_irqsave(&rs
->rs_recv_lock
, flags
);
552 list_for_each_entry_safe(inc
, tmp
, &rs
->rs_recv_queue
, i_item
) {
553 rds_recv_rcvbuf_delta(rs
, sk
, inc
->i_conn
->c_lcong
,
554 -be32_to_cpu(inc
->i_hdr
.h_len
),
556 list_del_init(&inc
->i_item
);
559 write_unlock_irqrestore(&rs
->rs_recv_lock
, flags
);
563 * inc->i_saddr isn't used here because it is only set in the receive
566 void rds_inc_info_copy(struct rds_incoming
*inc
,
567 struct rds_info_iterator
*iter
,
568 __be32 saddr
, __be32 daddr
, int flip
)
570 struct rds_info_message minfo
;
572 minfo
.seq
= be64_to_cpu(inc
->i_hdr
.h_sequence
);
573 minfo
.len
= be32_to_cpu(inc
->i_hdr
.h_len
);
578 minfo
.lport
= inc
->i_hdr
.h_dport
;
579 minfo
.fport
= inc
->i_hdr
.h_sport
;
583 minfo
.lport
= inc
->i_hdr
.h_sport
;
584 minfo
.fport
= inc
->i_hdr
.h_dport
;
589 rds_info_copy(iter
, &minfo
, sizeof(minfo
));