2 * Copyright (c) 2006 Oracle. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/kernel.h>
34 #include <linux/slab.h>
36 #include <linux/module.h>
38 #include <net/net_namespace.h>
39 #include <net/netns/generic.h>
44 /* only for info exporting */
45 static DEFINE_SPINLOCK(rds_tcp_tc_list_lock
);
46 static LIST_HEAD(rds_tcp_tc_list
);
47 static unsigned int rds_tcp_tc_count
;
49 /* Track rds_tcp_connection structs so they can be cleaned up */
50 static DEFINE_SPINLOCK(rds_tcp_conn_lock
);
51 static LIST_HEAD(rds_tcp_conn_list
);
53 static struct kmem_cache
*rds_tcp_conn_slab
;
55 static int rds_tcp_skbuf_handler(struct ctl_table
*ctl
, int write
,
56 void __user
*buffer
, size_t *lenp
,
59 int rds_tcp_min_sndbuf
= SOCK_MIN_SNDBUF
;
60 int rds_tcp_min_rcvbuf
= SOCK_MIN_RCVBUF
;
62 static struct ctl_table rds_tcp_sysctl_table
[] = {
63 #define RDS_TCP_SNDBUF 0
65 .procname
= "rds_tcp_sndbuf",
66 /* data is per-net pointer */
67 .maxlen
= sizeof(int),
69 .proc_handler
= rds_tcp_skbuf_handler
,
70 .extra1
= &rds_tcp_min_sndbuf
,
72 #define RDS_TCP_RCVBUF 1
74 .procname
= "rds_tcp_rcvbuf",
75 /* data is per-net pointer */
76 .maxlen
= sizeof(int),
78 .proc_handler
= rds_tcp_skbuf_handler
,
79 .extra1
= &rds_tcp_min_rcvbuf
,
84 /* doing it this way avoids calling tcp_sk() */
85 void rds_tcp_nonagle(struct socket
*sock
)
87 mm_segment_t oldfs
= get_fs();
91 sock
->ops
->setsockopt(sock
, SOL_TCP
, TCP_NODELAY
, (char __user
*)&val
,
96 u32
rds_tcp_snd_nxt(struct rds_tcp_connection
*tc
)
98 return tcp_sk(tc
->t_sock
->sk
)->snd_nxt
;
101 u32
rds_tcp_snd_una(struct rds_tcp_connection
*tc
)
103 return tcp_sk(tc
->t_sock
->sk
)->snd_una
;
106 void rds_tcp_restore_callbacks(struct socket
*sock
,
107 struct rds_tcp_connection
*tc
)
109 rdsdebug("restoring sock %p callbacks from tc %p\n", sock
, tc
);
110 write_lock_bh(&sock
->sk
->sk_callback_lock
);
112 /* done under the callback_lock to serialize with write_space */
113 spin_lock(&rds_tcp_tc_list_lock
);
114 list_del_init(&tc
->t_list_item
);
116 spin_unlock(&rds_tcp_tc_list_lock
);
120 sock
->sk
->sk_write_space
= tc
->t_orig_write_space
;
121 sock
->sk
->sk_data_ready
= tc
->t_orig_data_ready
;
122 sock
->sk
->sk_state_change
= tc
->t_orig_state_change
;
123 sock
->sk
->sk_user_data
= NULL
;
125 write_unlock_bh(&sock
->sk
->sk_callback_lock
);
129 * This is the only path that sets tc->t_sock. Send and receive trust that
130 * it is set. The RDS_CONN_UP bit protects those paths from being
131 * called while it isn't set.
133 void rds_tcp_set_callbacks(struct socket
*sock
, struct rds_connection
*conn
)
135 struct rds_tcp_connection
*tc
= conn
->c_transport_data
;
137 rdsdebug("setting sock %p callbacks to tc %p\n", sock
, tc
);
138 write_lock_bh(&sock
->sk
->sk_callback_lock
);
140 /* done under the callback_lock to serialize with write_space */
141 spin_lock(&rds_tcp_tc_list_lock
);
142 list_add_tail(&tc
->t_list_item
, &rds_tcp_tc_list
);
144 spin_unlock(&rds_tcp_tc_list_lock
);
146 /* accepted sockets need our listen data ready undone */
147 if (sock
->sk
->sk_data_ready
== rds_tcp_listen_data_ready
)
148 sock
->sk
->sk_data_ready
= sock
->sk
->sk_user_data
;
152 tc
->t_orig_data_ready
= sock
->sk
->sk_data_ready
;
153 tc
->t_orig_write_space
= sock
->sk
->sk_write_space
;
154 tc
->t_orig_state_change
= sock
->sk
->sk_state_change
;
156 sock
->sk
->sk_user_data
= conn
;
157 sock
->sk
->sk_data_ready
= rds_tcp_data_ready
;
158 sock
->sk
->sk_write_space
= rds_tcp_write_space
;
159 sock
->sk
->sk_state_change
= rds_tcp_state_change
;
161 write_unlock_bh(&sock
->sk
->sk_callback_lock
);
164 static void rds_tcp_tc_info(struct socket
*sock
, unsigned int len
,
165 struct rds_info_iterator
*iter
,
166 struct rds_info_lengths
*lens
)
168 struct rds_info_tcp_socket tsinfo
;
169 struct rds_tcp_connection
*tc
;
171 struct sockaddr_in sin
;
174 spin_lock_irqsave(&rds_tcp_tc_list_lock
, flags
);
176 if (len
/ sizeof(tsinfo
) < rds_tcp_tc_count
)
179 list_for_each_entry(tc
, &rds_tcp_tc_list
, t_list_item
) {
181 sock
->ops
->getname(sock
, (struct sockaddr
*)&sin
, &sinlen
, 0);
182 tsinfo
.local_addr
= sin
.sin_addr
.s_addr
;
183 tsinfo
.local_port
= sin
.sin_port
;
184 sock
->ops
->getname(sock
, (struct sockaddr
*)&sin
, &sinlen
, 1);
185 tsinfo
.peer_addr
= sin
.sin_addr
.s_addr
;
186 tsinfo
.peer_port
= sin
.sin_port
;
188 tsinfo
.hdr_rem
= tc
->t_tinc_hdr_rem
;
189 tsinfo
.data_rem
= tc
->t_tinc_data_rem
;
190 tsinfo
.last_sent_nxt
= tc
->t_last_sent_nxt
;
191 tsinfo
.last_expected_una
= tc
->t_last_expected_una
;
192 tsinfo
.last_seen_una
= tc
->t_last_seen_una
;
194 rds_info_copy(iter
, &tsinfo
, sizeof(tsinfo
));
198 lens
->nr
= rds_tcp_tc_count
;
199 lens
->each
= sizeof(tsinfo
);
201 spin_unlock_irqrestore(&rds_tcp_tc_list_lock
, flags
);
204 static int rds_tcp_laddr_check(struct net
*net
, __be32 addr
)
206 if (inet_addr_type(net
, addr
) == RTN_LOCAL
)
208 return -EADDRNOTAVAIL
;
211 static int rds_tcp_conn_alloc(struct rds_connection
*conn
, gfp_t gfp
)
213 struct rds_tcp_connection
*tc
;
215 tc
= kmem_cache_alloc(rds_tcp_conn_slab
, gfp
);
219 mutex_init(&tc
->t_conn_lock
);
222 tc
->t_tinc_hdr_rem
= sizeof(struct rds_header
);
223 tc
->t_tinc_data_rem
= 0;
225 conn
->c_transport_data
= tc
;
227 spin_lock_irq(&rds_tcp_conn_lock
);
228 list_add_tail(&tc
->t_tcp_node
, &rds_tcp_conn_list
);
229 spin_unlock_irq(&rds_tcp_conn_lock
);
231 rdsdebug("alloced tc %p\n", conn
->c_transport_data
);
235 static void rds_tcp_conn_free(void *arg
)
237 struct rds_tcp_connection
*tc
= arg
;
239 rdsdebug("freeing tc %p\n", tc
);
241 spin_lock_irqsave(&rds_tcp_conn_lock
, flags
);
242 list_del(&tc
->t_tcp_node
);
243 spin_unlock_irqrestore(&rds_tcp_conn_lock
, flags
);
245 kmem_cache_free(rds_tcp_conn_slab
, tc
);
248 static void rds_tcp_destroy_conns(void)
250 struct rds_tcp_connection
*tc
, *_tc
;
253 /* avoid calling conn_destroy with irqs off */
254 spin_lock_irq(&rds_tcp_conn_lock
);
255 list_splice(&rds_tcp_conn_list
, &tmp_list
);
256 INIT_LIST_HEAD(&rds_tcp_conn_list
);
257 spin_unlock_irq(&rds_tcp_conn_lock
);
259 list_for_each_entry_safe(tc
, _tc
, &tmp_list
, t_tcp_node
) {
260 if (tc
->conn
->c_passive
)
261 rds_conn_destroy(tc
->conn
->c_passive
);
262 rds_conn_destroy(tc
->conn
);
266 static void rds_tcp_exit(void);
268 struct rds_transport rds_tcp_transport
= {
269 .laddr_check
= rds_tcp_laddr_check
,
270 .xmit_prepare
= rds_tcp_xmit_prepare
,
271 .xmit_complete
= rds_tcp_xmit_complete
,
272 .xmit
= rds_tcp_xmit
,
273 .recv
= rds_tcp_recv
,
274 .conn_alloc
= rds_tcp_conn_alloc
,
275 .conn_free
= rds_tcp_conn_free
,
276 .conn_connect
= rds_tcp_conn_connect
,
277 .conn_shutdown
= rds_tcp_conn_shutdown
,
278 .inc_copy_to_user
= rds_tcp_inc_copy_to_user
,
279 .inc_free
= rds_tcp_inc_free
,
280 .stats_info_copy
= rds_tcp_stats_info_copy
,
281 .exit
= rds_tcp_exit
,
282 .t_owner
= THIS_MODULE
,
284 .t_type
= RDS_TRANS_TCP
,
285 .t_prefer_loopback
= 1,
288 static int rds_tcp_netid
;
290 /* per-network namespace private data for this module */
292 struct socket
*rds_tcp_listen_sock
;
293 struct work_struct rds_tcp_accept_w
;
294 struct ctl_table_header
*rds_tcp_sysctl
;
295 struct ctl_table
*ctl_table
;
300 /* All module specific customizations to the RDS-TCP socket should be done in
301 * rds_tcp_tune() and applied after socket creation.
303 void rds_tcp_tune(struct socket
*sock
)
305 struct sock
*sk
= sock
->sk
;
306 struct net
*net
= sock_net(sk
);
307 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
309 rds_tcp_nonagle(sock
);
311 if (rtn
->sndbuf_size
> 0) {
312 sk
->sk_sndbuf
= rtn
->sndbuf_size
;
313 sk
->sk_userlocks
|= SOCK_SNDBUF_LOCK
;
315 if (rtn
->rcvbuf_size
> 0) {
316 sk
->sk_sndbuf
= rtn
->rcvbuf_size
;
317 sk
->sk_userlocks
|= SOCK_RCVBUF_LOCK
;
322 static void rds_tcp_accept_worker(struct work_struct
*work
)
324 struct rds_tcp_net
*rtn
= container_of(work
,
328 while (rds_tcp_accept_one(rtn
->rds_tcp_listen_sock
) == 0)
332 void rds_tcp_accept_work(struct sock
*sk
)
334 struct net
*net
= sock_net(sk
);
335 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
337 queue_work(rds_wq
, &rtn
->rds_tcp_accept_w
);
340 static __net_init
int rds_tcp_init_net(struct net
*net
)
342 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
343 struct ctl_table
*tbl
;
346 memset(rtn
, 0, sizeof(*rtn
));
348 /* {snd, rcv}buf_size default to 0, which implies we let the
349 * stack pick the value, and permit auto-tuning of buffer size.
351 if (net
== &init_net
) {
352 tbl
= rds_tcp_sysctl_table
;
354 tbl
= kmemdup(rds_tcp_sysctl_table
,
355 sizeof(rds_tcp_sysctl_table
), GFP_KERNEL
);
357 pr_warn("could not set allocate syctl table\n");
360 rtn
->ctl_table
= tbl
;
362 tbl
[RDS_TCP_SNDBUF
].data
= &rtn
->sndbuf_size
;
363 tbl
[RDS_TCP_RCVBUF
].data
= &rtn
->rcvbuf_size
;
364 rtn
->rds_tcp_sysctl
= register_net_sysctl(net
, "net/rds/tcp", tbl
);
365 if (!rtn
->rds_tcp_sysctl
) {
366 pr_warn("could not register sysctl\n");
370 rtn
->rds_tcp_listen_sock
= rds_tcp_listen_init(net
);
371 if (!rtn
->rds_tcp_listen_sock
) {
372 pr_warn("could not set up listen sock\n");
373 unregister_net_sysctl_table(rtn
->rds_tcp_sysctl
);
374 rtn
->rds_tcp_sysctl
= NULL
;
378 INIT_WORK(&rtn
->rds_tcp_accept_w
, rds_tcp_accept_worker
);
382 if (net
!= &init_net
)
387 static void __net_exit
rds_tcp_exit_net(struct net
*net
)
389 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
391 if (rtn
->rds_tcp_sysctl
)
392 unregister_net_sysctl_table(rtn
->rds_tcp_sysctl
);
394 if (net
!= &init_net
&& rtn
->ctl_table
)
395 kfree(rtn
->ctl_table
);
397 /* If rds_tcp_exit_net() is called as a result of netns deletion,
398 * the rds_tcp_kill_sock() device notifier would already have cleaned
399 * up the listen socket, thus there is no work to do in this function.
401 * If rds_tcp_exit_net() is called as a result of module unload,
402 * i.e., due to rds_tcp_exit() -> unregister_pernet_subsys(), then
403 * we do need to clean up the listen socket here.
405 if (rtn
->rds_tcp_listen_sock
) {
406 rds_tcp_listen_stop(rtn
->rds_tcp_listen_sock
);
407 rtn
->rds_tcp_listen_sock
= NULL
;
408 flush_work(&rtn
->rds_tcp_accept_w
);
412 static struct pernet_operations rds_tcp_net_ops
= {
413 .init
= rds_tcp_init_net
,
414 .exit
= rds_tcp_exit_net
,
415 .id
= &rds_tcp_netid
,
416 .size
= sizeof(struct rds_tcp_net
),
419 static void rds_tcp_kill_sock(struct net
*net
)
421 struct rds_tcp_connection
*tc
, *_tc
;
424 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
426 rds_tcp_listen_stop(rtn
->rds_tcp_listen_sock
);
427 rtn
->rds_tcp_listen_sock
= NULL
;
428 flush_work(&rtn
->rds_tcp_accept_w
);
429 spin_lock_irq(&rds_tcp_conn_lock
);
430 list_for_each_entry_safe(tc
, _tc
, &rds_tcp_conn_list
, t_tcp_node
) {
431 struct net
*c_net
= read_pnet(&tc
->conn
->c_net
);
433 if (net
!= c_net
|| !tc
->t_sock
)
435 list_move_tail(&tc
->t_tcp_node
, &tmp_list
);
437 spin_unlock_irq(&rds_tcp_conn_lock
);
438 list_for_each_entry_safe(tc
, _tc
, &tmp_list
, t_tcp_node
) {
440 sk
->sk_prot
->disconnect(sk
, 0);
442 if (tc
->conn
->c_passive
)
443 rds_conn_destroy(tc
->conn
->c_passive
);
444 rds_conn_destroy(tc
->conn
);
448 static int rds_tcp_dev_event(struct notifier_block
*this,
449 unsigned long event
, void *ptr
)
451 struct net_device
*dev
= netdev_notifier_info_to_dev(ptr
);
453 /* rds-tcp registers as a pernet subys, so the ->exit will only
454 * get invoked after network acitivity has quiesced. We need to
455 * clean up all sockets to quiesce network activity, and use
456 * the unregistration of the per-net loopback device as a trigger
457 * to start that cleanup.
459 if (event
== NETDEV_UNREGISTER_FINAL
&&
460 dev
->ifindex
== LOOPBACK_IFINDEX
)
461 rds_tcp_kill_sock(dev_net(dev
));
466 static struct notifier_block rds_tcp_dev_notifier
= {
467 .notifier_call
= rds_tcp_dev_event
,
468 .priority
= -10, /* must be called after other network notifiers */
471 /* when sysctl is used to modify some kernel socket parameters,this
472 * function resets the RDS connections in that netns so that we can
473 * restart with new parameters. The assumption is that such reset
474 * events are few and far-between.
476 static void rds_tcp_sysctl_reset(struct net
*net
)
478 struct rds_tcp_connection
*tc
, *_tc
;
480 spin_lock_irq(&rds_tcp_conn_lock
);
481 list_for_each_entry_safe(tc
, _tc
, &rds_tcp_conn_list
, t_tcp_node
) {
482 struct net
*c_net
= read_pnet(&tc
->conn
->c_net
);
484 if (net
!= c_net
|| !tc
->t_sock
)
487 rds_conn_drop(tc
->conn
); /* reconnect with new parameters */
489 spin_unlock_irq(&rds_tcp_conn_lock
);
492 static int rds_tcp_skbuf_handler(struct ctl_table
*ctl
, int write
,
493 void __user
*buffer
, size_t *lenp
,
496 struct net
*net
= current
->nsproxy
->net_ns
;
499 err
= proc_dointvec_minmax(ctl
, write
, buffer
, lenp
, fpos
);
501 pr_warn("Invalid input. Must be >= %d\n",
502 *(int *)(ctl
->extra1
));
506 rds_tcp_sysctl_reset(net
);
510 static void rds_tcp_exit(void)
512 rds_info_deregister_func(RDS_INFO_TCP_SOCKETS
, rds_tcp_tc_info
);
513 unregister_pernet_subsys(&rds_tcp_net_ops
);
514 if (unregister_netdevice_notifier(&rds_tcp_dev_notifier
))
515 pr_warn("could not unregister rds_tcp_dev_notifier\n");
516 rds_tcp_destroy_conns();
517 rds_trans_unregister(&rds_tcp_transport
);
519 kmem_cache_destroy(rds_tcp_conn_slab
);
521 module_exit(rds_tcp_exit
);
523 static int rds_tcp_init(void)
527 rds_tcp_conn_slab
= kmem_cache_create("rds_tcp_connection",
528 sizeof(struct rds_tcp_connection
),
530 if (!rds_tcp_conn_slab
) {
535 ret
= register_netdevice_notifier(&rds_tcp_dev_notifier
);
537 pr_warn("could not register rds_tcp_dev_notifier\n");
541 ret
= register_pernet_subsys(&rds_tcp_net_ops
);
545 ret
= rds_tcp_recv_init();
549 ret
= rds_trans_register(&rds_tcp_transport
);
553 rds_info_register_func(RDS_INFO_TCP_SOCKETS
, rds_tcp_tc_info
);
560 unregister_pernet_subsys(&rds_tcp_net_ops
);
561 kmem_cache_destroy(rds_tcp_conn_slab
);
565 module_init(rds_tcp_init
);
567 MODULE_AUTHOR("Oracle Corporation <rds-devel@oss.oracle.com>");
568 MODULE_DESCRIPTION("RDS: TCP transport");
569 MODULE_LICENSE("Dual BSD/GPL");