2 * Copyright (c) 2006 Oracle. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/kernel.h>
34 #include <linux/slab.h>
36 #include <linux/module.h>
38 #include <net/net_namespace.h>
39 #include <net/netns/generic.h>
44 /* only for info exporting */
45 static DEFINE_SPINLOCK(rds_tcp_tc_list_lock
);
46 static LIST_HEAD(rds_tcp_tc_list
);
47 static unsigned int rds_tcp_tc_count
;
49 /* Track rds_tcp_connection structs so they can be cleaned up */
50 static DEFINE_SPINLOCK(rds_tcp_conn_lock
);
51 static LIST_HEAD(rds_tcp_conn_list
);
53 static struct kmem_cache
*rds_tcp_conn_slab
;
55 #define RDS_TCP_DEFAULT_BUFSIZE (128 * 1024)
57 static int rds_tcp_skbuf_handler(struct ctl_table
*ctl
, int write
,
58 void __user
*buffer
, size_t *lenp
,
61 int rds_tcp_min_sndbuf
= SOCK_MIN_SNDBUF
;
62 int rds_tcp_min_rcvbuf
= SOCK_MIN_RCVBUF
;
64 static struct ctl_table rds_tcp_sysctl_table
[] = {
65 #define RDS_TCP_SNDBUF 0
67 .procname
= "rds_tcp_sndbuf",
68 /* data is per-net pointer */
69 .maxlen
= sizeof(int),
71 .proc_handler
= rds_tcp_skbuf_handler
,
72 .extra1
= &rds_tcp_min_sndbuf
,
74 #define RDS_TCP_RCVBUF 1
76 .procname
= "rds_tcp_rcvbuf",
77 /* data is per-net pointer */
78 .maxlen
= sizeof(int),
80 .proc_handler
= rds_tcp_skbuf_handler
,
81 .extra1
= &rds_tcp_min_rcvbuf
,
86 /* doing it this way avoids calling tcp_sk() */
87 void rds_tcp_nonagle(struct socket
*sock
)
89 mm_segment_t oldfs
= get_fs();
93 sock
->ops
->setsockopt(sock
, SOL_TCP
, TCP_NODELAY
, (char __user
*)&val
,
98 u32
rds_tcp_snd_nxt(struct rds_tcp_connection
*tc
)
100 return tcp_sk(tc
->t_sock
->sk
)->snd_nxt
;
103 u32
rds_tcp_snd_una(struct rds_tcp_connection
*tc
)
105 return tcp_sk(tc
->t_sock
->sk
)->snd_una
;
108 void rds_tcp_restore_callbacks(struct socket
*sock
,
109 struct rds_tcp_connection
*tc
)
111 rdsdebug("restoring sock %p callbacks from tc %p\n", sock
, tc
);
112 write_lock_bh(&sock
->sk
->sk_callback_lock
);
114 /* done under the callback_lock to serialize with write_space */
115 spin_lock(&rds_tcp_tc_list_lock
);
116 list_del_init(&tc
->t_list_item
);
118 spin_unlock(&rds_tcp_tc_list_lock
);
122 sock
->sk
->sk_write_space
= tc
->t_orig_write_space
;
123 sock
->sk
->sk_data_ready
= tc
->t_orig_data_ready
;
124 sock
->sk
->sk_state_change
= tc
->t_orig_state_change
;
125 sock
->sk
->sk_user_data
= NULL
;
127 write_unlock_bh(&sock
->sk
->sk_callback_lock
);
131 * This is the only path that sets tc->t_sock. Send and receive trust that
132 * it is set. The RDS_CONN_CONNECTED bit protects those paths from being
133 * called while it isn't set.
135 void rds_tcp_set_callbacks(struct socket
*sock
, struct rds_connection
*conn
)
137 struct rds_tcp_connection
*tc
= conn
->c_transport_data
;
139 rdsdebug("setting sock %p callbacks to tc %p\n", sock
, tc
);
140 write_lock_bh(&sock
->sk
->sk_callback_lock
);
142 /* done under the callback_lock to serialize with write_space */
143 spin_lock(&rds_tcp_tc_list_lock
);
144 list_add_tail(&tc
->t_list_item
, &rds_tcp_tc_list
);
146 spin_unlock(&rds_tcp_tc_list_lock
);
148 /* accepted sockets need our listen data ready undone */
149 if (sock
->sk
->sk_data_ready
== rds_tcp_listen_data_ready
)
150 sock
->sk
->sk_data_ready
= sock
->sk
->sk_user_data
;
154 tc
->t_orig_data_ready
= sock
->sk
->sk_data_ready
;
155 tc
->t_orig_write_space
= sock
->sk
->sk_write_space
;
156 tc
->t_orig_state_change
= sock
->sk
->sk_state_change
;
158 sock
->sk
->sk_user_data
= conn
;
159 sock
->sk
->sk_data_ready
= rds_tcp_data_ready
;
160 sock
->sk
->sk_write_space
= rds_tcp_write_space
;
161 sock
->sk
->sk_state_change
= rds_tcp_state_change
;
163 write_unlock_bh(&sock
->sk
->sk_callback_lock
);
166 static void rds_tcp_tc_info(struct socket
*sock
, unsigned int len
,
167 struct rds_info_iterator
*iter
,
168 struct rds_info_lengths
*lens
)
170 struct rds_info_tcp_socket tsinfo
;
171 struct rds_tcp_connection
*tc
;
173 struct sockaddr_in sin
;
176 spin_lock_irqsave(&rds_tcp_tc_list_lock
, flags
);
178 if (len
/ sizeof(tsinfo
) < rds_tcp_tc_count
)
181 list_for_each_entry(tc
, &rds_tcp_tc_list
, t_list_item
) {
183 sock
->ops
->getname(sock
, (struct sockaddr
*)&sin
, &sinlen
, 0);
184 tsinfo
.local_addr
= sin
.sin_addr
.s_addr
;
185 tsinfo
.local_port
= sin
.sin_port
;
186 sock
->ops
->getname(sock
, (struct sockaddr
*)&sin
, &sinlen
, 1);
187 tsinfo
.peer_addr
= sin
.sin_addr
.s_addr
;
188 tsinfo
.peer_port
= sin
.sin_port
;
190 tsinfo
.hdr_rem
= tc
->t_tinc_hdr_rem
;
191 tsinfo
.data_rem
= tc
->t_tinc_data_rem
;
192 tsinfo
.last_sent_nxt
= tc
->t_last_sent_nxt
;
193 tsinfo
.last_expected_una
= tc
->t_last_expected_una
;
194 tsinfo
.last_seen_una
= tc
->t_last_seen_una
;
196 rds_info_copy(iter
, &tsinfo
, sizeof(tsinfo
));
200 lens
->nr
= rds_tcp_tc_count
;
201 lens
->each
= sizeof(tsinfo
);
203 spin_unlock_irqrestore(&rds_tcp_tc_list_lock
, flags
);
206 static int rds_tcp_laddr_check(struct net
*net
, __be32 addr
)
208 if (inet_addr_type(net
, addr
) == RTN_LOCAL
)
210 return -EADDRNOTAVAIL
;
213 static int rds_tcp_conn_alloc(struct rds_connection
*conn
, gfp_t gfp
)
215 struct rds_tcp_connection
*tc
;
217 tc
= kmem_cache_alloc(rds_tcp_conn_slab
, gfp
);
223 tc
->t_tinc_hdr_rem
= sizeof(struct rds_header
);
224 tc
->t_tinc_data_rem
= 0;
226 conn
->c_transport_data
= tc
;
228 spin_lock_irq(&rds_tcp_conn_lock
);
229 list_add_tail(&tc
->t_tcp_node
, &rds_tcp_conn_list
);
230 spin_unlock_irq(&rds_tcp_conn_lock
);
232 rdsdebug("alloced tc %p\n", conn
->c_transport_data
);
236 static void rds_tcp_conn_free(void *arg
)
238 struct rds_tcp_connection
*tc
= arg
;
240 rdsdebug("freeing tc %p\n", tc
);
242 spin_lock_irqsave(&rds_tcp_conn_lock
, flags
);
243 list_del(&tc
->t_tcp_node
);
244 spin_unlock_irqrestore(&rds_tcp_conn_lock
, flags
);
246 kmem_cache_free(rds_tcp_conn_slab
, tc
);
249 static void rds_tcp_destroy_conns(void)
251 struct rds_tcp_connection
*tc
, *_tc
;
254 /* avoid calling conn_destroy with irqs off */
255 spin_lock_irq(&rds_tcp_conn_lock
);
256 list_splice(&rds_tcp_conn_list
, &tmp_list
);
257 INIT_LIST_HEAD(&rds_tcp_conn_list
);
258 spin_unlock_irq(&rds_tcp_conn_lock
);
260 list_for_each_entry_safe(tc
, _tc
, &tmp_list
, t_tcp_node
) {
261 if (tc
->conn
->c_passive
)
262 rds_conn_destroy(tc
->conn
->c_passive
);
263 rds_conn_destroy(tc
->conn
);
267 static void rds_tcp_exit(void);
269 struct rds_transport rds_tcp_transport
= {
270 .laddr_check
= rds_tcp_laddr_check
,
271 .xmit_prepare
= rds_tcp_xmit_prepare
,
272 .xmit_complete
= rds_tcp_xmit_complete
,
273 .xmit
= rds_tcp_xmit
,
274 .recv
= rds_tcp_recv
,
275 .conn_alloc
= rds_tcp_conn_alloc
,
276 .conn_free
= rds_tcp_conn_free
,
277 .conn_connect
= rds_tcp_conn_connect
,
278 .conn_shutdown
= rds_tcp_conn_shutdown
,
279 .inc_copy_to_user
= rds_tcp_inc_copy_to_user
,
280 .inc_free
= rds_tcp_inc_free
,
281 .stats_info_copy
= rds_tcp_stats_info_copy
,
282 .exit
= rds_tcp_exit
,
283 .t_owner
= THIS_MODULE
,
285 .t_type
= RDS_TRANS_TCP
,
286 .t_prefer_loopback
= 1,
289 static int rds_tcp_netid
;
291 /* per-network namespace private data for this module */
293 struct socket
*rds_tcp_listen_sock
;
294 struct work_struct rds_tcp_accept_w
;
295 struct ctl_table_header
*rds_tcp_sysctl
;
296 struct ctl_table
*ctl_table
;
301 /* All module specific customizations to the RDS-TCP socket should be done in
302 * rds_tcp_tune() and applied after socket creation.
304 void rds_tcp_tune(struct socket
*sock
)
306 struct sock
*sk
= sock
->sk
;
307 struct net
*net
= sock_net(sk
);
308 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
310 rds_tcp_nonagle(sock
);
312 if (rtn
->sndbuf_size
> 0) {
313 sk
->sk_sndbuf
= rtn
->sndbuf_size
;
314 sk
->sk_userlocks
|= SOCK_SNDBUF_LOCK
;
316 if (rtn
->rcvbuf_size
> 0) {
317 sk
->sk_sndbuf
= rtn
->rcvbuf_size
;
318 sk
->sk_userlocks
|= SOCK_RCVBUF_LOCK
;
323 static void rds_tcp_accept_worker(struct work_struct
*work
)
325 struct rds_tcp_net
*rtn
= container_of(work
,
329 while (rds_tcp_accept_one(rtn
->rds_tcp_listen_sock
) == 0)
333 void rds_tcp_accept_work(struct sock
*sk
)
335 struct net
*net
= sock_net(sk
);
336 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
338 queue_work(rds_wq
, &rtn
->rds_tcp_accept_w
);
341 static __net_init
int rds_tcp_init_net(struct net
*net
)
343 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
344 struct ctl_table
*tbl
;
347 memset(rtn
, 0, sizeof(*rtn
));
349 /* {snd, rcv}buf_size default to 0, which implies we let the
350 * stack pick the value, and permit auto-tuning of buffer size.
352 if (net
== &init_net
) {
353 tbl
= rds_tcp_sysctl_table
;
355 tbl
= kmemdup(rds_tcp_sysctl_table
,
356 sizeof(rds_tcp_sysctl_table
), GFP_KERNEL
);
358 pr_warn("could not set allocate syctl table\n");
361 rtn
->ctl_table
= tbl
;
363 tbl
[RDS_TCP_SNDBUF
].data
= &rtn
->sndbuf_size
;
364 tbl
[RDS_TCP_RCVBUF
].data
= &rtn
->rcvbuf_size
;
365 rtn
->rds_tcp_sysctl
= register_net_sysctl(net
, "net/rds/tcp", tbl
);
366 if (!rtn
->rds_tcp_sysctl
) {
367 pr_warn("could not register sysctl\n");
371 rtn
->rds_tcp_listen_sock
= rds_tcp_listen_init(net
);
372 if (!rtn
->rds_tcp_listen_sock
) {
373 pr_warn("could not set up listen sock\n");
374 unregister_net_sysctl_table(rtn
->rds_tcp_sysctl
);
375 rtn
->rds_tcp_sysctl
= NULL
;
379 INIT_WORK(&rtn
->rds_tcp_accept_w
, rds_tcp_accept_worker
);
383 if (net
!= &init_net
)
388 static void __net_exit
rds_tcp_exit_net(struct net
*net
)
390 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
392 if (rtn
->rds_tcp_sysctl
)
393 unregister_net_sysctl_table(rtn
->rds_tcp_sysctl
);
395 if (net
!= &init_net
&& rtn
->ctl_table
)
396 kfree(rtn
->ctl_table
);
398 /* If rds_tcp_exit_net() is called as a result of netns deletion,
399 * the rds_tcp_kill_sock() device notifier would already have cleaned
400 * up the listen socket, thus there is no work to do in this function.
402 * If rds_tcp_exit_net() is called as a result of module unload,
403 * i.e., due to rds_tcp_exit() -> unregister_pernet_subsys(), then
404 * we do need to clean up the listen socket here.
406 if (rtn
->rds_tcp_listen_sock
) {
407 rds_tcp_listen_stop(rtn
->rds_tcp_listen_sock
);
408 rtn
->rds_tcp_listen_sock
= NULL
;
409 flush_work(&rtn
->rds_tcp_accept_w
);
413 static struct pernet_operations rds_tcp_net_ops
= {
414 .init
= rds_tcp_init_net
,
415 .exit
= rds_tcp_exit_net
,
416 .id
= &rds_tcp_netid
,
417 .size
= sizeof(struct rds_tcp_net
),
420 static void rds_tcp_kill_sock(struct net
*net
)
422 struct rds_tcp_connection
*tc
, *_tc
;
425 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
427 rds_tcp_listen_stop(rtn
->rds_tcp_listen_sock
);
428 rtn
->rds_tcp_listen_sock
= NULL
;
429 flush_work(&rtn
->rds_tcp_accept_w
);
430 spin_lock_irq(&rds_tcp_conn_lock
);
431 list_for_each_entry_safe(tc
, _tc
, &rds_tcp_conn_list
, t_tcp_node
) {
432 struct net
*c_net
= read_pnet(&tc
->conn
->c_net
);
434 if (net
!= c_net
|| !tc
->t_sock
)
436 list_move_tail(&tc
->t_tcp_node
, &tmp_list
);
438 spin_unlock_irq(&rds_tcp_conn_lock
);
439 list_for_each_entry_safe(tc
, _tc
, &tmp_list
, t_tcp_node
) {
441 sk
->sk_prot
->disconnect(sk
, 0);
443 if (tc
->conn
->c_passive
)
444 rds_conn_destroy(tc
->conn
->c_passive
);
445 rds_conn_destroy(tc
->conn
);
449 static int rds_tcp_dev_event(struct notifier_block
*this,
450 unsigned long event
, void *ptr
)
452 struct net_device
*dev
= netdev_notifier_info_to_dev(ptr
);
454 /* rds-tcp registers as a pernet subys, so the ->exit will only
455 * get invoked after network acitivity has quiesced. We need to
456 * clean up all sockets to quiesce network activity, and use
457 * the unregistration of the per-net loopback device as a trigger
458 * to start that cleanup.
460 if (event
== NETDEV_UNREGISTER_FINAL
&&
461 dev
->ifindex
== LOOPBACK_IFINDEX
)
462 rds_tcp_kill_sock(dev_net(dev
));
467 static struct notifier_block rds_tcp_dev_notifier
= {
468 .notifier_call
= rds_tcp_dev_event
,
469 .priority
= -10, /* must be called after other network notifiers */
472 /* when sysctl is used to modify some kernel socket parameters,this
473 * function resets the RDS connections in that netns so that we can
474 * restart with new parameters. The assumption is that such reset
475 * events are few and far-between.
477 static void rds_tcp_sysctl_reset(struct net
*net
)
479 struct rds_tcp_connection
*tc
, *_tc
;
481 spin_lock_irq(&rds_tcp_conn_lock
);
482 list_for_each_entry_safe(tc
, _tc
, &rds_tcp_conn_list
, t_tcp_node
) {
483 struct net
*c_net
= read_pnet(&tc
->conn
->c_net
);
485 if (net
!= c_net
|| !tc
->t_sock
)
488 rds_conn_drop(tc
->conn
); /* reconnect with new parameters */
490 spin_unlock_irq(&rds_tcp_conn_lock
);
493 static int rds_tcp_skbuf_handler(struct ctl_table
*ctl
, int write
,
494 void __user
*buffer
, size_t *lenp
,
497 struct net
*net
= current
->nsproxy
->net_ns
;
500 err
= proc_dointvec_minmax(ctl
, write
, buffer
, lenp
, fpos
);
502 pr_warn("Invalid input. Must be >= %d\n",
503 *(int *)(ctl
->extra1
));
507 rds_tcp_sysctl_reset(net
);
511 static void rds_tcp_exit(void)
513 rds_info_deregister_func(RDS_INFO_TCP_SOCKETS
, rds_tcp_tc_info
);
514 unregister_pernet_subsys(&rds_tcp_net_ops
);
515 if (unregister_netdevice_notifier(&rds_tcp_dev_notifier
))
516 pr_warn("could not unregister rds_tcp_dev_notifier\n");
517 rds_tcp_destroy_conns();
518 rds_trans_unregister(&rds_tcp_transport
);
520 kmem_cache_destroy(rds_tcp_conn_slab
);
522 module_exit(rds_tcp_exit
);
524 static int rds_tcp_init(void)
528 rds_tcp_conn_slab
= kmem_cache_create("rds_tcp_connection",
529 sizeof(struct rds_tcp_connection
),
531 if (!rds_tcp_conn_slab
) {
536 ret
= register_netdevice_notifier(&rds_tcp_dev_notifier
);
538 pr_warn("could not register rds_tcp_dev_notifier\n");
542 ret
= register_pernet_subsys(&rds_tcp_net_ops
);
546 ret
= rds_tcp_recv_init();
550 ret
= rds_trans_register(&rds_tcp_transport
);
554 rds_info_register_func(RDS_INFO_TCP_SOCKETS
, rds_tcp_tc_info
);
561 unregister_pernet_subsys(&rds_tcp_net_ops
);
562 kmem_cache_destroy(rds_tcp_conn_slab
);
566 module_init(rds_tcp_init
);
568 MODULE_AUTHOR("Oracle Corporation <rds-devel@oss.oracle.com>");
569 MODULE_DESCRIPTION("RDS: TCP transport");
570 MODULE_LICENSE("Dual BSD/GPL");