2 * Copyright (c) 2006 Oracle. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/kernel.h>
34 #include <linux/slab.h>
36 #include <linux/module.h>
38 #include <net/net_namespace.h>
39 #include <net/netns/generic.h>
44 /* only for info exporting */
45 static DEFINE_SPINLOCK(rds_tcp_tc_list_lock
);
46 static LIST_HEAD(rds_tcp_tc_list
);
47 static unsigned int rds_tcp_tc_count
;
49 /* Track rds_tcp_connection structs so they can be cleaned up */
50 static DEFINE_SPINLOCK(rds_tcp_conn_lock
);
51 static LIST_HEAD(rds_tcp_conn_list
);
53 static struct kmem_cache
*rds_tcp_conn_slab
;
55 #define RDS_TCP_DEFAULT_BUFSIZE (128 * 1024)
57 /* doing it this way avoids calling tcp_sk() */
58 void rds_tcp_nonagle(struct socket
*sock
)
60 mm_segment_t oldfs
= get_fs();
64 sock
->ops
->setsockopt(sock
, SOL_TCP
, TCP_NODELAY
, (char __user
*)&val
,
69 /* All module specific customizations to the RDS-TCP socket should be done in
70 * rds_tcp_tune() and applied after socket creation. In general these
71 * customizations should be tunable via module_param()
73 void rds_tcp_tune(struct socket
*sock
)
75 rds_tcp_nonagle(sock
);
78 u32
rds_tcp_snd_nxt(struct rds_tcp_connection
*tc
)
80 return tcp_sk(tc
->t_sock
->sk
)->snd_nxt
;
83 u32
rds_tcp_snd_una(struct rds_tcp_connection
*tc
)
85 return tcp_sk(tc
->t_sock
->sk
)->snd_una
;
88 void rds_tcp_restore_callbacks(struct socket
*sock
,
89 struct rds_tcp_connection
*tc
)
91 rdsdebug("restoring sock %p callbacks from tc %p\n", sock
, tc
);
92 write_lock_bh(&sock
->sk
->sk_callback_lock
);
94 /* done under the callback_lock to serialize with write_space */
95 spin_lock(&rds_tcp_tc_list_lock
);
96 list_del_init(&tc
->t_list_item
);
98 spin_unlock(&rds_tcp_tc_list_lock
);
102 sock
->sk
->sk_write_space
= tc
->t_orig_write_space
;
103 sock
->sk
->sk_data_ready
= tc
->t_orig_data_ready
;
104 sock
->sk
->sk_state_change
= tc
->t_orig_state_change
;
105 sock
->sk
->sk_user_data
= NULL
;
107 write_unlock_bh(&sock
->sk
->sk_callback_lock
);
111 * This is the only path that sets tc->t_sock. Send and receive trust that
112 * it is set. The RDS_CONN_CONNECTED bit protects those paths from being
113 * called while it isn't set.
115 void rds_tcp_set_callbacks(struct socket
*sock
, struct rds_connection
*conn
)
117 struct rds_tcp_connection
*tc
= conn
->c_transport_data
;
119 rdsdebug("setting sock %p callbacks to tc %p\n", sock
, tc
);
120 write_lock_bh(&sock
->sk
->sk_callback_lock
);
122 /* done under the callback_lock to serialize with write_space */
123 spin_lock(&rds_tcp_tc_list_lock
);
124 list_add_tail(&tc
->t_list_item
, &rds_tcp_tc_list
);
126 spin_unlock(&rds_tcp_tc_list_lock
);
128 /* accepted sockets need our listen data ready undone */
129 if (sock
->sk
->sk_data_ready
== rds_tcp_listen_data_ready
)
130 sock
->sk
->sk_data_ready
= sock
->sk
->sk_user_data
;
134 tc
->t_orig_data_ready
= sock
->sk
->sk_data_ready
;
135 tc
->t_orig_write_space
= sock
->sk
->sk_write_space
;
136 tc
->t_orig_state_change
= sock
->sk
->sk_state_change
;
138 sock
->sk
->sk_user_data
= conn
;
139 sock
->sk
->sk_data_ready
= rds_tcp_data_ready
;
140 sock
->sk
->sk_write_space
= rds_tcp_write_space
;
141 sock
->sk
->sk_state_change
= rds_tcp_state_change
;
143 write_unlock_bh(&sock
->sk
->sk_callback_lock
);
146 static void rds_tcp_tc_info(struct socket
*sock
, unsigned int len
,
147 struct rds_info_iterator
*iter
,
148 struct rds_info_lengths
*lens
)
150 struct rds_info_tcp_socket tsinfo
;
151 struct rds_tcp_connection
*tc
;
153 struct sockaddr_in sin
;
156 spin_lock_irqsave(&rds_tcp_tc_list_lock
, flags
);
158 if (len
/ sizeof(tsinfo
) < rds_tcp_tc_count
)
161 list_for_each_entry(tc
, &rds_tcp_tc_list
, t_list_item
) {
163 sock
->ops
->getname(sock
, (struct sockaddr
*)&sin
, &sinlen
, 0);
164 tsinfo
.local_addr
= sin
.sin_addr
.s_addr
;
165 tsinfo
.local_port
= sin
.sin_port
;
166 sock
->ops
->getname(sock
, (struct sockaddr
*)&sin
, &sinlen
, 1);
167 tsinfo
.peer_addr
= sin
.sin_addr
.s_addr
;
168 tsinfo
.peer_port
= sin
.sin_port
;
170 tsinfo
.hdr_rem
= tc
->t_tinc_hdr_rem
;
171 tsinfo
.data_rem
= tc
->t_tinc_data_rem
;
172 tsinfo
.last_sent_nxt
= tc
->t_last_sent_nxt
;
173 tsinfo
.last_expected_una
= tc
->t_last_expected_una
;
174 tsinfo
.last_seen_una
= tc
->t_last_seen_una
;
176 rds_info_copy(iter
, &tsinfo
, sizeof(tsinfo
));
180 lens
->nr
= rds_tcp_tc_count
;
181 lens
->each
= sizeof(tsinfo
);
183 spin_unlock_irqrestore(&rds_tcp_tc_list_lock
, flags
);
186 static int rds_tcp_laddr_check(struct net
*net
, __be32 addr
)
188 if (inet_addr_type(net
, addr
) == RTN_LOCAL
)
190 return -EADDRNOTAVAIL
;
193 static int rds_tcp_conn_alloc(struct rds_connection
*conn
, gfp_t gfp
)
195 struct rds_tcp_connection
*tc
;
197 tc
= kmem_cache_alloc(rds_tcp_conn_slab
, gfp
);
203 tc
->t_tinc_hdr_rem
= sizeof(struct rds_header
);
204 tc
->t_tinc_data_rem
= 0;
206 conn
->c_transport_data
= tc
;
208 spin_lock_irq(&rds_tcp_conn_lock
);
209 list_add_tail(&tc
->t_tcp_node
, &rds_tcp_conn_list
);
210 spin_unlock_irq(&rds_tcp_conn_lock
);
212 rdsdebug("alloced tc %p\n", conn
->c_transport_data
);
216 static void rds_tcp_conn_free(void *arg
)
218 struct rds_tcp_connection
*tc
= arg
;
220 rdsdebug("freeing tc %p\n", tc
);
222 spin_lock_irqsave(&rds_tcp_conn_lock
, flags
);
223 list_del(&tc
->t_tcp_node
);
224 spin_unlock_irqrestore(&rds_tcp_conn_lock
, flags
);
226 kmem_cache_free(rds_tcp_conn_slab
, tc
);
229 static void rds_tcp_destroy_conns(void)
231 struct rds_tcp_connection
*tc
, *_tc
;
234 /* avoid calling conn_destroy with irqs off */
235 spin_lock_irq(&rds_tcp_conn_lock
);
236 list_splice(&rds_tcp_conn_list
, &tmp_list
);
237 INIT_LIST_HEAD(&rds_tcp_conn_list
);
238 spin_unlock_irq(&rds_tcp_conn_lock
);
240 list_for_each_entry_safe(tc
, _tc
, &tmp_list
, t_tcp_node
) {
241 if (tc
->conn
->c_passive
)
242 rds_conn_destroy(tc
->conn
->c_passive
);
243 rds_conn_destroy(tc
->conn
);
247 static void rds_tcp_exit(void);
249 struct rds_transport rds_tcp_transport
= {
250 .laddr_check
= rds_tcp_laddr_check
,
251 .xmit_prepare
= rds_tcp_xmit_prepare
,
252 .xmit_complete
= rds_tcp_xmit_complete
,
253 .xmit
= rds_tcp_xmit
,
254 .recv
= rds_tcp_recv
,
255 .conn_alloc
= rds_tcp_conn_alloc
,
256 .conn_free
= rds_tcp_conn_free
,
257 .conn_connect
= rds_tcp_conn_connect
,
258 .conn_shutdown
= rds_tcp_conn_shutdown
,
259 .inc_copy_to_user
= rds_tcp_inc_copy_to_user
,
260 .inc_free
= rds_tcp_inc_free
,
261 .stats_info_copy
= rds_tcp_stats_info_copy
,
262 .exit
= rds_tcp_exit
,
263 .t_owner
= THIS_MODULE
,
265 .t_type
= RDS_TRANS_TCP
,
266 .t_prefer_loopback
= 1,
269 static int rds_tcp_netid
;
271 /* per-network namespace private data for this module */
273 struct socket
*rds_tcp_listen_sock
;
274 struct work_struct rds_tcp_accept_w
;
277 static void rds_tcp_accept_worker(struct work_struct
*work
)
279 struct rds_tcp_net
*rtn
= container_of(work
,
283 while (rds_tcp_accept_one(rtn
->rds_tcp_listen_sock
) == 0)
287 void rds_tcp_accept_work(struct sock
*sk
)
289 struct net
*net
= sock_net(sk
);
290 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
292 queue_work(rds_wq
, &rtn
->rds_tcp_accept_w
);
295 static __net_init
int rds_tcp_init_net(struct net
*net
)
297 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
299 rtn
->rds_tcp_listen_sock
= rds_tcp_listen_init(net
);
300 if (!rtn
->rds_tcp_listen_sock
) {
301 pr_warn("could not set up listen sock\n");
302 return -EAFNOSUPPORT
;
304 INIT_WORK(&rtn
->rds_tcp_accept_w
, rds_tcp_accept_worker
);
308 static void __net_exit
rds_tcp_exit_net(struct net
*net
)
310 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
312 /* If rds_tcp_exit_net() is called as a result of netns deletion,
313 * the rds_tcp_kill_sock() device notifier would already have cleaned
314 * up the listen socket, thus there is no work to do in this function.
316 * If rds_tcp_exit_net() is called as a result of module unload,
317 * i.e., due to rds_tcp_exit() -> unregister_pernet_subsys(), then
318 * we do need to clean up the listen socket here.
320 if (rtn
->rds_tcp_listen_sock
) {
321 rds_tcp_listen_stop(rtn
->rds_tcp_listen_sock
);
322 rtn
->rds_tcp_listen_sock
= NULL
;
323 flush_work(&rtn
->rds_tcp_accept_w
);
327 static struct pernet_operations rds_tcp_net_ops
= {
328 .init
= rds_tcp_init_net
,
329 .exit
= rds_tcp_exit_net
,
330 .id
= &rds_tcp_netid
,
331 .size
= sizeof(struct rds_tcp_net
),
334 static void rds_tcp_kill_sock(struct net
*net
)
336 struct rds_tcp_connection
*tc
, *_tc
;
339 struct rds_tcp_net
*rtn
= net_generic(net
, rds_tcp_netid
);
341 rds_tcp_listen_stop(rtn
->rds_tcp_listen_sock
);
342 rtn
->rds_tcp_listen_sock
= NULL
;
343 flush_work(&rtn
->rds_tcp_accept_w
);
344 spin_lock_irq(&rds_tcp_conn_lock
);
345 list_for_each_entry_safe(tc
, _tc
, &rds_tcp_conn_list
, t_tcp_node
) {
346 struct net
*c_net
= read_pnet(&tc
->conn
->c_net
);
348 if (net
!= c_net
|| !tc
->t_sock
)
350 list_move_tail(&tc
->t_tcp_node
, &tmp_list
);
352 spin_unlock_irq(&rds_tcp_conn_lock
);
353 list_for_each_entry_safe(tc
, _tc
, &tmp_list
, t_tcp_node
) {
355 sk
->sk_prot
->disconnect(sk
, 0);
357 if (tc
->conn
->c_passive
)
358 rds_conn_destroy(tc
->conn
->c_passive
);
359 rds_conn_destroy(tc
->conn
);
363 static int rds_tcp_dev_event(struct notifier_block
*this,
364 unsigned long event
, void *ptr
)
366 struct net_device
*dev
= netdev_notifier_info_to_dev(ptr
);
368 /* rds-tcp registers as a pernet subys, so the ->exit will only
369 * get invoked after network acitivity has quiesced. We need to
370 * clean up all sockets to quiesce network activity, and use
371 * the unregistration of the per-net loopback device as a trigger
372 * to start that cleanup.
374 if (event
== NETDEV_UNREGISTER_FINAL
&&
375 dev
->ifindex
== LOOPBACK_IFINDEX
)
376 rds_tcp_kill_sock(dev_net(dev
));
381 static struct notifier_block rds_tcp_dev_notifier
= {
382 .notifier_call
= rds_tcp_dev_event
,
383 .priority
= -10, /* must be called after other network notifiers */
386 static void rds_tcp_exit(void)
388 rds_info_deregister_func(RDS_INFO_TCP_SOCKETS
, rds_tcp_tc_info
);
389 unregister_pernet_subsys(&rds_tcp_net_ops
);
390 if (unregister_netdevice_notifier(&rds_tcp_dev_notifier
))
391 pr_warn("could not unregister rds_tcp_dev_notifier\n");
392 rds_tcp_destroy_conns();
393 rds_trans_unregister(&rds_tcp_transport
);
395 kmem_cache_destroy(rds_tcp_conn_slab
);
397 module_exit(rds_tcp_exit
);
399 static int rds_tcp_init(void)
403 rds_tcp_conn_slab
= kmem_cache_create("rds_tcp_connection",
404 sizeof(struct rds_tcp_connection
),
406 if (!rds_tcp_conn_slab
) {
411 ret
= register_netdevice_notifier(&rds_tcp_dev_notifier
);
413 pr_warn("could not register rds_tcp_dev_notifier\n");
417 ret
= register_pernet_subsys(&rds_tcp_net_ops
);
421 ret
= rds_tcp_recv_init();
425 ret
= rds_trans_register(&rds_tcp_transport
);
429 rds_info_register_func(RDS_INFO_TCP_SOCKETS
, rds_tcp_tc_info
);
436 unregister_pernet_subsys(&rds_tcp_net_ops
);
437 kmem_cache_destroy(rds_tcp_conn_slab
);
441 module_init(rds_tcp_init
);
443 MODULE_AUTHOR("Oracle Corporation <rds-devel@oss.oracle.com>");
444 MODULE_DESCRIPTION("RDS: TCP transport");
445 MODULE_LICENSE("Dual BSD/GPL");