2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * Generic TIME_WAIT sockets functions
8 * From code orinally in TCP
11 #include <linux/kernel.h>
12 #include <linux/kmemcheck.h>
13 #include <net/inet_hashtables.h>
14 #include <net/inet_timewait_sock.h>
19 * unhash a timewait socket from established hash
20 * lock must be hold by caller
22 int inet_twsk_unhash(struct inet_timewait_sock
*tw
)
24 if (hlist_nulls_unhashed(&tw
->tw_node
))
27 hlist_nulls_del_rcu(&tw
->tw_node
);
28 sk_nulls_node_init(&tw
->tw_node
);
33 * unhash a timewait socket from bind hash
34 * lock must be hold by caller
36 int inet_twsk_bind_unhash(struct inet_timewait_sock
*tw
,
37 struct inet_hashinfo
*hashinfo
)
39 struct inet_bind_bucket
*tb
= tw
->tw_tb
;
44 __hlist_del(&tw
->tw_bind_node
);
46 inet_bind_bucket_destroy(hashinfo
->bind_bucket_cachep
, tb
);
50 /* Must be called with locally disabled BHs. */
51 static void __inet_twsk_kill(struct inet_timewait_sock
*tw
,
52 struct inet_hashinfo
*hashinfo
)
54 struct inet_bind_hashbucket
*bhead
;
56 /* Unlink from established hashes. */
57 spinlock_t
*lock
= inet_ehash_lockp(hashinfo
, tw
->tw_hash
);
60 refcnt
= inet_twsk_unhash(tw
);
63 /* Disassociate with bind bucket. */
64 bhead
= &hashinfo
->bhash
[inet_bhashfn(twsk_net(tw
), tw
->tw_num
,
65 hashinfo
->bhash_size
)];
67 spin_lock(&bhead
->lock
);
68 refcnt
+= inet_twsk_bind_unhash(tw
, hashinfo
);
69 spin_unlock(&bhead
->lock
);
71 #ifdef SOCK_REFCNT_DEBUG
72 if (atomic_read(&tw
->tw_refcnt
) != 1) {
73 printk(KERN_DEBUG
"%s timewait_sock %p refcnt=%d\n",
74 tw
->tw_prot
->name
, tw
, atomic_read(&tw
->tw_refcnt
));
83 static noinline
void inet_twsk_free(struct inet_timewait_sock
*tw
)
85 struct module
*owner
= tw
->tw_prot
->owner
;
86 twsk_destructor((struct sock
*)tw
);
87 #ifdef SOCK_REFCNT_DEBUG
88 pr_debug("%s timewait_sock %p released\n", tw
->tw_prot
->name
, tw
);
90 release_net(twsk_net(tw
));
91 kmem_cache_free(tw
->tw_prot
->twsk_prot
->twsk_slab
, tw
);
95 void inet_twsk_put(struct inet_timewait_sock
*tw
)
97 if (atomic_dec_and_test(&tw
->tw_refcnt
))
100 EXPORT_SYMBOL_GPL(inet_twsk_put
);
103 * Enter the time wait state. This is called with locally disabled BH.
104 * Essentially we whip up a timewait bucket, copy the relevant info into it
105 * from the SK, and mess with hash chains and list linkage.
107 void __inet_twsk_hashdance(struct inet_timewait_sock
*tw
, struct sock
*sk
,
108 struct inet_hashinfo
*hashinfo
)
110 const struct inet_sock
*inet
= inet_sk(sk
);
111 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
112 struct inet_ehash_bucket
*ehead
= inet_ehash_bucket(hashinfo
, sk
->sk_hash
);
113 spinlock_t
*lock
= inet_ehash_lockp(hashinfo
, sk
->sk_hash
);
114 struct inet_bind_hashbucket
*bhead
;
115 /* Step 1: Put TW into bind hash. Original socket stays there too.
116 Note, that any socket with inet->num != 0 MUST be bound in
117 binding cache, even if it is closed.
119 bhead
= &hashinfo
->bhash
[inet_bhashfn(twsk_net(tw
), inet
->inet_num
,
120 hashinfo
->bhash_size
)];
121 spin_lock(&bhead
->lock
);
122 tw
->tw_tb
= icsk
->icsk_bind_hash
;
123 WARN_ON(!icsk
->icsk_bind_hash
);
124 inet_twsk_add_bind_node(tw
, &tw
->tw_tb
->owners
);
125 spin_unlock(&bhead
->lock
);
130 * Step 2: Hash TW into TIMEWAIT chain.
131 * Should be done before removing sk from established chain
132 * because readers are lockless and search established first.
134 inet_twsk_add_node_rcu(tw
, &ehead
->twchain
);
136 /* Step 3: Remove SK from established hash. */
137 if (__sk_nulls_del_node_init_rcu(sk
))
138 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, -1);
142 * - We initially set tw_refcnt to 0 in inet_twsk_alloc()
143 * - We add one reference for the bhash link
144 * - We add one reference for the ehash link
145 * - We want this refcnt update done before allowing other
146 * threads to find this tw in ehash chain.
148 atomic_add(1 + 1 + 1, &tw
->tw_refcnt
);
153 EXPORT_SYMBOL_GPL(__inet_twsk_hashdance
);
155 struct inet_timewait_sock
*inet_twsk_alloc(const struct sock
*sk
, const int state
)
157 struct inet_timewait_sock
*tw
=
158 kmem_cache_alloc(sk
->sk_prot_creator
->twsk_prot
->twsk_slab
,
161 const struct inet_sock
*inet
= inet_sk(sk
);
163 kmemcheck_annotate_bitfield(tw
, flags
);
165 /* Give us an identity. */
166 tw
->tw_daddr
= inet
->inet_daddr
;
167 tw
->tw_rcv_saddr
= inet
->inet_rcv_saddr
;
168 tw
->tw_bound_dev_if
= sk
->sk_bound_dev_if
;
169 tw
->tw_num
= inet
->inet_num
;
170 tw
->tw_state
= TCP_TIME_WAIT
;
171 tw
->tw_substate
= state
;
172 tw
->tw_sport
= inet
->inet_sport
;
173 tw
->tw_dport
= inet
->inet_dport
;
174 tw
->tw_family
= sk
->sk_family
;
175 tw
->tw_reuse
= sk
->sk_reuse
;
176 tw
->tw_hash
= sk
->sk_hash
;
178 tw
->tw_transparent
= inet
->transparent
;
179 tw
->tw_prot
= sk
->sk_prot_creator
;
180 twsk_net_set(tw
, hold_net(sock_net(sk
)));
182 * Because we use RCU lookups, we should not set tw_refcnt
183 * to a non null value before everything is setup for this
186 atomic_set(&tw
->tw_refcnt
, 0);
187 inet_twsk_dead_node_init(tw
);
188 __module_get(tw
->tw_prot
->owner
);
194 EXPORT_SYMBOL_GPL(inet_twsk_alloc
);
196 /* Returns non-zero if quota exceeded. */
197 static int inet_twdr_do_twkill_work(struct inet_timewait_death_row
*twdr
,
200 struct inet_timewait_sock
*tw
;
201 struct hlist_node
*node
;
205 /* NOTE: compare this to previous version where lock
206 * was released after detaching chain. It was racy,
207 * because tw buckets are scheduled in not serialized context
208 * in 2.3 (with netfilter), and with softnet it is common, because
209 * soft irqs are not sequenced.
214 inet_twsk_for_each_inmate(tw
, node
, &twdr
->cells
[slot
]) {
215 __inet_twsk_del_dead_node(tw
);
216 spin_unlock(&twdr
->death_lock
);
217 __inet_twsk_kill(tw
, twdr
->hashinfo
);
219 NET_INC_STATS_BH(twsk_net(tw
), LINUX_MIB_TIMEWAITED
);
223 spin_lock(&twdr
->death_lock
);
224 if (killed
> INET_TWDR_TWKILL_QUOTA
) {
229 /* While we dropped twdr->death_lock, another cpu may have
230 * killed off the next TW bucket in the list, therefore
231 * do a fresh re-read of the hlist head node with the
232 * lock reacquired. We still use the hlist traversal
233 * macro in order to get the prefetches.
238 twdr
->tw_count
-= killed
;
239 #ifndef CONFIG_NET_NS
240 NET_ADD_STATS_BH(&init_net
, LINUX_MIB_TIMEWAITED
, killed
);
245 void inet_twdr_hangman(unsigned long data
)
247 struct inet_timewait_death_row
*twdr
;
248 int unsigned need_timer
;
250 twdr
= (struct inet_timewait_death_row
*)data
;
251 spin_lock(&twdr
->death_lock
);
253 if (twdr
->tw_count
== 0)
257 if (inet_twdr_do_twkill_work(twdr
, twdr
->slot
)) {
258 twdr
->thread_slots
|= (1 << twdr
->slot
);
259 schedule_work(&twdr
->twkill_work
);
262 /* We purged the entire slot, anything left? */
265 twdr
->slot
= ((twdr
->slot
+ 1) & (INET_TWDR_TWKILL_SLOTS
- 1));
268 mod_timer(&twdr
->tw_timer
, jiffies
+ twdr
->period
);
270 spin_unlock(&twdr
->death_lock
);
273 EXPORT_SYMBOL_GPL(inet_twdr_hangman
);
275 void inet_twdr_twkill_work(struct work_struct
*work
)
277 struct inet_timewait_death_row
*twdr
=
278 container_of(work
, struct inet_timewait_death_row
, twkill_work
);
281 BUILD_BUG_ON((INET_TWDR_TWKILL_SLOTS
- 1) >
282 (sizeof(twdr
->thread_slots
) * 8));
284 while (twdr
->thread_slots
) {
285 spin_lock_bh(&twdr
->death_lock
);
286 for (i
= 0; i
< INET_TWDR_TWKILL_SLOTS
; i
++) {
287 if (!(twdr
->thread_slots
& (1 << i
)))
290 while (inet_twdr_do_twkill_work(twdr
, i
) != 0) {
291 if (need_resched()) {
292 spin_unlock_bh(&twdr
->death_lock
);
294 spin_lock_bh(&twdr
->death_lock
);
298 twdr
->thread_slots
&= ~(1 << i
);
300 spin_unlock_bh(&twdr
->death_lock
);
304 EXPORT_SYMBOL_GPL(inet_twdr_twkill_work
);
306 /* These are always called from BH context. See callers in
307 * tcp_input.c to verify this.
310 /* This is for handling early-kills of TIME_WAIT sockets. */
311 void inet_twsk_deschedule(struct inet_timewait_sock
*tw
,
312 struct inet_timewait_death_row
*twdr
)
314 spin_lock(&twdr
->death_lock
);
315 if (inet_twsk_del_dead_node(tw
)) {
317 if (--twdr
->tw_count
== 0)
318 del_timer(&twdr
->tw_timer
);
320 spin_unlock(&twdr
->death_lock
);
321 __inet_twsk_kill(tw
, twdr
->hashinfo
);
324 EXPORT_SYMBOL(inet_twsk_deschedule
);
326 void inet_twsk_schedule(struct inet_timewait_sock
*tw
,
327 struct inet_timewait_death_row
*twdr
,
328 const int timeo
, const int timewait_len
)
330 struct hlist_head
*list
;
333 /* timeout := RTO * 3.5
335 * 3.5 = 1+2+0.5 to wait for two retransmits.
337 * RATIONALE: if FIN arrived and we entered TIME-WAIT state,
338 * our ACK acking that FIN can be lost. If N subsequent retransmitted
339 * FINs (or previous seqments) are lost (probability of such event
340 * is p^(N+1), where p is probability to lose single packet and
341 * time to detect the loss is about RTO*(2^N - 1) with exponential
342 * backoff). Normal timewait length is calculated so, that we
343 * waited at least for one retransmitted FIN (maximal RTO is 120sec).
344 * [ BTW Linux. following BSD, violates this requirement waiting
345 * only for 60sec, we should wait at least for 240 secs.
346 * Well, 240 consumes too much of resources 8)
348 * This interval is not reduced to catch old duplicate and
349 * responces to our wandering segments living for two MSLs.
350 * However, if we use PAWS to detect
351 * old duplicates, we can reduce the interval to bounds required
352 * by RTO, rather than MSL. So, if peer understands PAWS, we
353 * kill tw bucket after 3.5*RTO (it is important that this number
354 * is greater than TS tick!) and detect old duplicates with help
357 slot
= (timeo
+ (1 << INET_TWDR_RECYCLE_TICK
) - 1) >> INET_TWDR_RECYCLE_TICK
;
359 spin_lock(&twdr
->death_lock
);
361 /* Unlink it, if it was scheduled */
362 if (inet_twsk_del_dead_node(tw
))
365 atomic_inc(&tw
->tw_refcnt
);
367 if (slot
>= INET_TWDR_RECYCLE_SLOTS
) {
368 /* Schedule to slow timer */
369 if (timeo
>= timewait_len
) {
370 slot
= INET_TWDR_TWKILL_SLOTS
- 1;
372 slot
= DIV_ROUND_UP(timeo
, twdr
->period
);
373 if (slot
>= INET_TWDR_TWKILL_SLOTS
)
374 slot
= INET_TWDR_TWKILL_SLOTS
- 1;
376 tw
->tw_ttd
= jiffies
+ timeo
;
377 slot
= (twdr
->slot
+ slot
) & (INET_TWDR_TWKILL_SLOTS
- 1);
378 list
= &twdr
->cells
[slot
];
380 tw
->tw_ttd
= jiffies
+ (slot
<< INET_TWDR_RECYCLE_TICK
);
382 if (twdr
->twcal_hand
< 0) {
383 twdr
->twcal_hand
= 0;
384 twdr
->twcal_jiffie
= jiffies
;
385 twdr
->twcal_timer
.expires
= twdr
->twcal_jiffie
+
386 (slot
<< INET_TWDR_RECYCLE_TICK
);
387 add_timer(&twdr
->twcal_timer
);
389 if (time_after(twdr
->twcal_timer
.expires
,
390 jiffies
+ (slot
<< INET_TWDR_RECYCLE_TICK
)))
391 mod_timer(&twdr
->twcal_timer
,
392 jiffies
+ (slot
<< INET_TWDR_RECYCLE_TICK
));
393 slot
= (twdr
->twcal_hand
+ slot
) & (INET_TWDR_RECYCLE_SLOTS
- 1);
395 list
= &twdr
->twcal_row
[slot
];
398 hlist_add_head(&tw
->tw_death_node
, list
);
400 if (twdr
->tw_count
++ == 0)
401 mod_timer(&twdr
->tw_timer
, jiffies
+ twdr
->period
);
402 spin_unlock(&twdr
->death_lock
);
405 EXPORT_SYMBOL_GPL(inet_twsk_schedule
);
407 void inet_twdr_twcal_tick(unsigned long data
)
409 struct inet_timewait_death_row
*twdr
;
412 unsigned long now
= jiffies
;
416 twdr
= (struct inet_timewait_death_row
*)data
;
418 spin_lock(&twdr
->death_lock
);
419 if (twdr
->twcal_hand
< 0)
422 slot
= twdr
->twcal_hand
;
423 j
= twdr
->twcal_jiffie
;
425 for (n
= 0; n
< INET_TWDR_RECYCLE_SLOTS
; n
++) {
426 if (time_before_eq(j
, now
)) {
427 struct hlist_node
*node
, *safe
;
428 struct inet_timewait_sock
*tw
;
430 inet_twsk_for_each_inmate_safe(tw
, node
, safe
,
431 &twdr
->twcal_row
[slot
]) {
432 __inet_twsk_del_dead_node(tw
);
433 __inet_twsk_kill(tw
, twdr
->hashinfo
);
435 NET_INC_STATS_BH(twsk_net(tw
), LINUX_MIB_TIMEWAITKILLED
);
443 twdr
->twcal_jiffie
= j
;
444 twdr
->twcal_hand
= slot
;
447 if (!hlist_empty(&twdr
->twcal_row
[slot
])) {
448 mod_timer(&twdr
->twcal_timer
, j
);
452 j
+= 1 << INET_TWDR_RECYCLE_TICK
;
453 slot
= (slot
+ 1) & (INET_TWDR_RECYCLE_SLOTS
- 1);
455 twdr
->twcal_hand
= -1;
458 if ((twdr
->tw_count
-= killed
) == 0)
459 del_timer(&twdr
->tw_timer
);
460 #ifndef CONFIG_NET_NS
461 NET_ADD_STATS_BH(&init_net
, LINUX_MIB_TIMEWAITKILLED
, killed
);
463 spin_unlock(&twdr
->death_lock
);
466 EXPORT_SYMBOL_GPL(inet_twdr_twcal_tick
);
468 void inet_twsk_purge(struct inet_hashinfo
*hashinfo
,
469 struct inet_timewait_death_row
*twdr
, int family
)
471 struct inet_timewait_sock
*tw
;
473 struct hlist_nulls_node
*node
;
476 for (slot
= 0; slot
<= hashinfo
->ehash_mask
; slot
++) {
477 struct inet_ehash_bucket
*head
= &hashinfo
->ehash
[slot
];
481 sk_nulls_for_each_rcu(sk
, node
, &head
->twchain
) {
483 if ((tw
->tw_family
!= family
) ||
484 atomic_read(&twsk_net(tw
)->count
))
487 if (unlikely(!atomic_inc_not_zero(&tw
->tw_refcnt
)))
490 if (unlikely((tw
->tw_family
!= family
) ||
491 atomic_read(&twsk_net(tw
)->count
))) {
497 inet_twsk_deschedule(tw
, twdr
);
501 /* If the nulls value we got at the end of this lookup is
502 * not the expected one, we must restart lookup.
503 * We probably met an item that was moved to another chain.
505 if (get_nulls_value(node
) != slot
)
510 EXPORT_SYMBOL_GPL(inet_twsk_purge
);
This page took 0.068758 seconds and 5 git commands to generate.