projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
kcm: Sendpage support
[deliverable/linux.git] / net / kcm / kcmsock.c
CommitLineData
ab7ac4eb
TH		 1#include <linux/bpf.h>
2#include <linux/errno.h>
3#include <linux/errqueue.h>
4#include <linux/file.h>
5#include <linux/in.h>
6#include <linux/kernel.h>
7#include <linux/module.h>
8#include <linux/net.h>
9#include <linux/netdevice.h>
10#include <linux/poll.h>
11#include <linux/rculist.h>
12#include <linux/skbuff.h>
13#include <linux/socket.h>
14#include <linux/uaccess.h>
15#include <linux/workqueue.h>
16#include <net/kcm.h>
17#include <net/netns/generic.h>
18#include <net/sock.h>
19#include <net/tcp.h>
20#include <uapi/linux/kcm.h>
21
22unsigned int kcm_net_id;
23
24static struct kmem_cache *kcm_psockp __read_mostly;
25static struct kmem_cache *kcm_muxp __read_mostly;
26static struct workqueue_struct *kcm_wq;
27
28static inline struct kcm_sock *kcm_sk(const struct sock *sk)
29{
30 return (struct kcm_sock *)sk;
31}
32
33static inline struct kcm_tx_msg *kcm_tx_msg(struct sk_buff *skb)
34{
35 return (struct kcm_tx_msg *)skb->cb;
36}
37
38static inline struct kcm_rx_msg *kcm_rx_msg(struct sk_buff *skb)
39{
40 return (struct kcm_rx_msg *)((void *)skb->cb +
41 offsetof(struct qdisc_skb_cb, data));
42}
43
44static void report_csk_error(struct sock *csk, int err)
45{
46 csk->sk_err = EPIPE;
47 csk->sk_error_report(csk);
48}
49
50/* Callback lock held */
51static void kcm_abort_rx_psock(struct kcm_psock *psock, int err,
52 struct sk_buff *skb)
53{
54 struct sock *csk = psock->sk;
55
56 /* Unrecoverable error in receive */
57
58 if (psock->rx_stopped)
59 return;
60
61 psock->rx_stopped = 1;
cd6e111b 62 KCM_STATS_INCR(psock->stats.rx_aborts);
ab7ac4eb
TH		 63
64 /* Report an error on the lower socket */
65 report_csk_error(csk, err);
66}
67
68static void kcm_abort_tx_psock(struct kcm_psock *psock, int err,
69 bool wakeup_kcm)
70{
71 struct sock *csk = psock->sk;
72 struct kcm_mux *mux = psock->mux;
73
74 /* Unrecoverable error in transmit */
75
76 spin_lock_bh(&mux->lock);
77
78 if (psock->tx_stopped) {
79 spin_unlock_bh(&mux->lock);
80 return;
81 }
82
83 psock->tx_stopped = 1;
cd6e111b 84 KCM_STATS_INCR(psock->stats.tx_aborts);
ab7ac4eb
TH		 85
86 if (!psock->tx_kcm) {
87 /* Take off psocks_avail list */
88 list_del(&psock->psock_avail_list);
89 } else if (wakeup_kcm) {
90 /* In this case psock is being aborted while outside of
91 * write_msgs and psock is reserved. Schedule tx_work
92 * to handle the failure there. Need to commit tx_stopped
93 * before queuing work.
94 */
95 smp_mb();
96
97 queue_work(kcm_wq, &psock->tx_kcm->tx_work);
98 }
99
100 spin_unlock_bh(&mux->lock);
101
102 /* Report error on lower socket */
103 report_csk_error(csk, err);
104}
105
cd6e111b
TH		 106/* RX mux lock held. */
107static void kcm_update_rx_mux_stats(struct kcm_mux *mux,
108 struct kcm_psock *psock)
109{
110 KCM_STATS_ADD(mux->stats.rx_bytes,
111 psock->stats.rx_bytes - psock->saved_rx_bytes);
112 mux->stats.rx_msgs +=
113 psock->stats.rx_msgs - psock->saved_rx_msgs;
114 psock->saved_rx_msgs = psock->stats.rx_msgs;
115 psock->saved_rx_bytes = psock->stats.rx_bytes;
116}
117
118static void kcm_update_tx_mux_stats(struct kcm_mux *mux,
119 struct kcm_psock *psock)
120{
121 KCM_STATS_ADD(mux->stats.tx_bytes,
122 psock->stats.tx_bytes - psock->saved_tx_bytes);
123 mux->stats.tx_msgs +=
124 psock->stats.tx_msgs - psock->saved_tx_msgs;
125 psock->saved_tx_msgs = psock->stats.tx_msgs;
126 psock->saved_tx_bytes = psock->stats.tx_bytes;
127}
128
ab7ac4eb
TH		 129static int kcm_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
130
131/* KCM is ready to receive messages on its queue-- either the KCM is new or
132 * has become unblocked after being blocked on full socket buffer. Queue any
133 * pending ready messages on a psock. RX mux lock held.
134 */
135static void kcm_rcv_ready(struct kcm_sock *kcm)
136{
137 struct kcm_mux *mux = kcm->mux;
138 struct kcm_psock *psock;
139 struct sk_buff *skb;
140
141 if (unlikely(kcm->rx_wait || kcm->rx_psock || kcm->rx_disabled))
142 return;
143
144 while (unlikely((skb = __skb_dequeue(&mux->rx_hold_queue)))) {
145 if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
146 /* Assuming buffer limit has been reached */
147 skb_queue_head(&mux->rx_hold_queue, skb);
148 WARN_ON(!sk_rmem_alloc_get(&kcm->sk));
149 return;
150 }
151 }
152
153 while (!list_empty(&mux->psocks_ready)) {
154 psock = list_first_entry(&mux->psocks_ready, struct kcm_psock,
155 psock_ready_list);
156
157 if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) {
158 /* Assuming buffer limit has been reached */
159 WARN_ON(!sk_rmem_alloc_get(&kcm->sk));
160 return;
161 }
162
163 /* Consumed the ready message on the psock. Schedule rx_work to
164 * get more messages.
165 */
166 list_del(&psock->psock_ready_list);
167 psock->ready_rx_msg = NULL;
168
169 /* Commit clearing of ready_rx_msg for queuing work */
170 smp_mb();
171
172 queue_work(kcm_wq, &psock->rx_work);
173 }
174
175 /* Buffer limit is okay now, add to ready list */
176 list_add_tail(&kcm->wait_rx_list,
177 &kcm->mux->kcm_rx_waiters);
178 kcm->rx_wait = true;
179}
180
181static void kcm_rfree(struct sk_buff *skb)
182{
183 struct sock *sk = skb->sk;
184 struct kcm_sock *kcm = kcm_sk(sk);
185 struct kcm_mux *mux = kcm->mux;
186 unsigned int len = skb->truesize;
187
188 sk_mem_uncharge(sk, len);
189 atomic_sub(len, &sk->sk_rmem_alloc);
190
191 /* For reading rx_wait and rx_psock without holding lock */
192 smp_mb__after_atomic();
193
194 if (!kcm->rx_wait && !kcm->rx_psock &&
195 sk_rmem_alloc_get(sk) < sk->sk_rcvlowat) {
196 spin_lock_bh(&mux->rx_lock);
197 kcm_rcv_ready(kcm);
198 spin_unlock_bh(&mux->rx_lock);
199 }
200}
201
202static int kcm_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
203{
204 struct sk_buff_head *list = &sk->sk_receive_queue;
205
206 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
207 return -ENOMEM;
208
209 if (!sk_rmem_schedule(sk, skb, skb->truesize))
210 return -ENOBUFS;
211
212 skb->dev = NULL;
213
214 skb_orphan(skb);
215 skb->sk = sk;
216 skb->destructor = kcm_rfree;
217 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
218 sk_mem_charge(sk, skb->truesize);
219
220 skb_queue_tail(list, skb);
221
222 if (!sock_flag(sk, SOCK_DEAD))
223 sk->sk_data_ready(sk);
224
225 return 0;
226}
227
228/* Requeue received messages for a kcm socket to other kcm sockets. This is
229 * called with a kcm socket is receive disabled.
230 * RX mux lock held.
231 */
232static void requeue_rx_msgs(struct kcm_mux *mux, struct sk_buff_head *head)
233{
234 struct sk_buff *skb;
235 struct kcm_sock *kcm;
236
237 while ((skb = __skb_dequeue(head))) {
238 /* Reset destructor to avoid calling kcm_rcv_ready */
239 skb->destructor = sock_rfree;
240 skb_orphan(skb);
241try_again:
242 if (list_empty(&mux->kcm_rx_waiters)) {
243 skb_queue_tail(&mux->rx_hold_queue, skb);
244 continue;
245 }
246
247 kcm = list_first_entry(&mux->kcm_rx_waiters,
248 struct kcm_sock, wait_rx_list);
249
250 if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
251 /* Should mean socket buffer full */
252 list_del(&kcm->wait_rx_list);
253 kcm->rx_wait = false;
254
255 /* Commit rx_wait to read in kcm_free */
256 smp_wmb();
257
258 goto try_again;
259 }
260 }
261}
262
263/* Lower sock lock held */
264static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock,
265 struct sk_buff *head)
266{
267 struct kcm_mux *mux = psock->mux;
268 struct kcm_sock *kcm;
269
270 WARN_ON(psock->ready_rx_msg);
271
272 if (psock->rx_kcm)
273 return psock->rx_kcm;
274
275 spin_lock_bh(&mux->rx_lock);
276
277 if (psock->rx_kcm) {
278 spin_unlock_bh(&mux->rx_lock);
279 return psock->rx_kcm;
280 }
281
cd6e111b
TH		 282 kcm_update_rx_mux_stats(mux, psock);
283
ab7ac4eb
TH		 284 if (list_empty(&mux->kcm_rx_waiters)) {
285 psock->ready_rx_msg = head;
286 list_add_tail(&psock->psock_ready_list,
287 &mux->psocks_ready);
288 spin_unlock_bh(&mux->rx_lock);
289 return NULL;
290 }
291
292 kcm = list_first_entry(&mux->kcm_rx_waiters,
293 struct kcm_sock, wait_rx_list);
294 list_del(&kcm->wait_rx_list);
295 kcm->rx_wait = false;
296
297 psock->rx_kcm = kcm;
298 kcm->rx_psock = psock;
299
300 spin_unlock_bh(&mux->rx_lock);
301
302 return kcm;
303}
304
305static void kcm_done(struct kcm_sock *kcm);
306
307static void kcm_done_work(struct work_struct *w)
308{
309 kcm_done(container_of(w, struct kcm_sock, done_work));
310}
311
312/* Lower sock held */
313static void unreserve_rx_kcm(struct kcm_psock *psock,
314 bool rcv_ready)
315{
316 struct kcm_sock *kcm = psock->rx_kcm;
317 struct kcm_mux *mux = psock->mux;
318
319 if (!kcm)
320 return;
321
322 spin_lock_bh(&mux->rx_lock);
323
324 psock->rx_kcm = NULL;
325 kcm->rx_psock = NULL;
326
327 /* Commit kcm->rx_psock before sk_rmem_alloc_get to sync with
328 * kcm_rfree
329 */
330 smp_mb();
331
332 if (unlikely(kcm->done)) {
333 spin_unlock_bh(&mux->rx_lock);
334
335 /* Need to run kcm_done in a task since we need to qcquire
336 * callback locks which may already be held here.
337 */
338 INIT_WORK(&kcm->done_work, kcm_done_work);
339 schedule_work(&kcm->done_work);
340 return;
341 }
342
343 if (unlikely(kcm->rx_disabled)) {
344 requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);
345 } else if (rcv_ready || unlikely(!sk_rmem_alloc_get(&kcm->sk))) {
346 /* Check for degenerative race with rx_wait that all
347 * data was dequeued (accounted for in kcm_rfree).
348 */
349 kcm_rcv_ready(kcm);
350 }
351 spin_unlock_bh(&mux->rx_lock);
352}
353
354/* Macro to invoke filter function. */
355#define KCM_RUN_FILTER(prog, ctx) \
356 (*prog->bpf_func)(ctx, prog->insnsi)
357
358/* Lower socket lock held */
359static int kcm_tcp_recv(read_descriptor_t *desc, struct sk_buff *orig_skb,
360 unsigned int orig_offset, size_t orig_len)
361{
362 struct kcm_psock *psock = (struct kcm_psock *)desc->arg.data;
363 struct kcm_rx_msg *rxm;
364 struct kcm_sock *kcm;
365 struct sk_buff *head, *skb;
366 size_t eaten = 0, cand_len;
367 ssize_t extra;
368 int err;
369 bool cloned_orig = false;
370
371 if (psock->ready_rx_msg)
372 return 0;
373
374 head = psock->rx_skb_head;
375 if (head) {
376 /* Message already in progress */
377
378 if (unlikely(orig_offset)) {
379 /* Getting data with a non-zero offset when a message is
380 * in progress is not expected. If it does happen, we
381 * need to clone and pull since we can't deal with
382 * offsets in the skbs for a message expect in the head.
383 */
384 orig_skb = skb_clone(orig_skb, GFP_ATOMIC);
385 if (!orig_skb) {
cd6e111b 386 KCM_STATS_INCR(psock->stats.rx_mem_fail);
ab7ac4eb
TH		 387 desc->error = -ENOMEM;
388 return 0;
389 }
390 if (!pskb_pull(orig_skb, orig_offset)) {
cd6e111b 391 KCM_STATS_INCR(psock->stats.rx_mem_fail);
ab7ac4eb
TH		 392 kfree_skb(orig_skb);
393 desc->error = -ENOMEM;
394 return 0;
395 }
396 cloned_orig = true;
397 orig_offset = 0;
398 }
399
400 if (!psock->rx_skb_nextp) {
401 /* We are going to append to the frags_list of head.
402 * Need to unshare the frag_list.
403 */
404 err = skb_unclone(head, GFP_ATOMIC);
405 if (err) {
cd6e111b 406 KCM_STATS_INCR(psock->stats.rx_mem_fail);
ab7ac4eb
TH		 407 desc->error = err;
408 return 0;
409 }
410
411 if (unlikely(skb_shinfo(head)->frag_list)) {
412 /* We can't append to an sk_buff that already
413 * has a frag_list. We create a new head, point
414 * the frag_list of that to the old head, and
415 * then are able to use the old head->next for
416 * appending to the message.
417 */
418 if (WARN_ON(head->next)) {
419 desc->error = -EINVAL;
420 return 0;
421 }
422
423 skb = alloc_skb(0, GFP_ATOMIC);
424 if (!skb) {
cd6e111b 425 KCM_STATS_INCR(psock->stats.rx_mem_fail);
ab7ac4eb
TH		 426 desc->error = -ENOMEM;
427 return 0;
428 }
429 skb->len = head->len;
430 skb->data_len = head->len;
431 skb->truesize = head->truesize;
432 *kcm_rx_msg(skb) = *kcm_rx_msg(head);
433 psock->rx_skb_nextp = &head->next;
434 skb_shinfo(skb)->frag_list = head;
435 psock->rx_skb_head = skb;
436 head = skb;
437 } else {
438 psock->rx_skb_nextp =
439 &skb_shinfo(head)->frag_list;
440 }
441 }
442 }
443
444 while (eaten < orig_len) {
445 /* Always clone since we will consume something */
446 skb = skb_clone(orig_skb, GFP_ATOMIC);
447 if (!skb) {
cd6e111b 448 KCM_STATS_INCR(psock->stats.rx_mem_fail);
ab7ac4eb
TH		 449 desc->error = -ENOMEM;
450 break;
451 }
452
453 cand_len = orig_len - eaten;
454
455 head = psock->rx_skb_head;
456 if (!head) {
457 head = skb;
458 psock->rx_skb_head = head;
459 /* Will set rx_skb_nextp on next packet if needed */
460 psock->rx_skb_nextp = NULL;
461 rxm = kcm_rx_msg(head);
462 memset(rxm, 0, sizeof(*rxm));
463 rxm->offset = orig_offset + eaten;
464 } else {
465 /* Unclone since we may be appending to an skb that we
466 * already share a frag_list with.
467 */
468 err = skb_unclone(skb, GFP_ATOMIC);
469 if (err) {
cd6e111b 470 KCM_STATS_INCR(psock->stats.rx_mem_fail);
ab7ac4eb
TH		 471 desc->error = err;
472 break;
473 }
474
475 rxm = kcm_rx_msg(head);
476 *psock->rx_skb_nextp = skb;
477 psock->rx_skb_nextp = &skb->next;
478 head->data_len += skb->len;
479 head->len += skb->len;
480 head->truesize += skb->truesize;
481 }
482
483 if (!rxm->full_len) {
484 ssize_t len;
485
486 len = KCM_RUN_FILTER(psock->bpf_prog, head);
487
488 if (!len) {
489 /* Need more header to determine length */
490 rxm->accum_len += cand_len;
491 eaten += cand_len;
cd6e111b 492 KCM_STATS_INCR(psock->stats.rx_need_more_hdr);
ab7ac4eb
TH		 493 WARN_ON(eaten != orig_len);
494 break;
495 } else if (len <= (ssize_t)head->len -
496 skb->len - rxm->offset) {
497 /* Length must be into new skb (and also
498 * greater than zero)
499 */
cd6e111b 500 KCM_STATS_INCR(psock->stats.rx_bad_hdr_len);
ab7ac4eb
TH		 501 desc->error = -EPROTO;
502 psock->rx_skb_head = NULL;
503 kcm_abort_rx_psock(psock, EPROTO, head);
504 break;
505 }
506
507 rxm->full_len = len;
508 }
509
510 extra = (ssize_t)(rxm->accum_len + cand_len) - rxm->full_len;
511
512 if (extra < 0) {
513 /* Message not complete yet. */
514 rxm->accum_len += cand_len;
515 eaten += cand_len;
516 WARN_ON(eaten != orig_len);
517 break;
518 }
519
520 /* Positive extra indicates ore bytes than needed for the
521 * message
522 */
523
524 WARN_ON(extra > cand_len);
525
526 eaten += (cand_len - extra);
527
528 /* Hurray, we have a new message! */
529 psock->rx_skb_head = NULL;
cd6e111b 530 KCM_STATS_INCR(psock->stats.rx_msgs);
ab7ac4eb
TH		 531
532try_queue:
533 kcm = reserve_rx_kcm(psock, head);
534 if (!kcm) {
535 /* Unable to reserve a KCM, message is held in psock. */
536 break;
537 }
538
539 if (kcm_queue_rcv_skb(&kcm->sk, head)) {
540 /* Should mean socket buffer full */
541 unreserve_rx_kcm(psock, false);
542 goto try_queue;
543 }
544 }
545
546 if (cloned_orig)
547 kfree_skb(orig_skb);
548
cd6e111b
TH		 549 KCM_STATS_ADD(psock->stats.rx_bytes, eaten);
550
ab7ac4eb
TH		 551 return eaten;
552}
553
554/* Called with lock held on lower socket */
555static int psock_tcp_read_sock(struct kcm_psock *psock)
556{
557 read_descriptor_t desc;
558
559 desc.arg.data = psock;
560 desc.error = 0;
561 desc.count = 1; /* give more than one skb per call */
562
563 /* sk should be locked here, so okay to do tcp_read_sock */
564 tcp_read_sock(psock->sk, &desc, kcm_tcp_recv);
565
566 unreserve_rx_kcm(psock, true);
567
568 return desc.error;
569}
570
571/* Lower sock lock held */
572static void psock_tcp_data_ready(struct sock *sk)
573{
574 struct kcm_psock *psock;
575
576 read_lock_bh(&sk->sk_callback_lock);
577
578 psock = (struct kcm_psock *)sk->sk_user_data;
579 if (unlikely(!psock || psock->rx_stopped))
580 goto out;
581
582 if (psock->ready_rx_msg)
583 goto out;
584
585 if (psock_tcp_read_sock(psock) == -ENOMEM)
586 queue_delayed_work(kcm_wq, &psock->rx_delayed_work, 0);
587
588out:
589 read_unlock_bh(&sk->sk_callback_lock);
590}
591
592static void do_psock_rx_work(struct kcm_psock *psock)
593{
594 read_descriptor_t rd_desc;
595 struct sock *csk = psock->sk;
596
597 /* We need the read lock to synchronize with psock_tcp_data_ready. We
598 * need the socket lock for calling tcp_read_sock.
599 */
600 lock_sock(csk);
601 read_lock_bh(&csk->sk_callback_lock);
602
603 if (unlikely(csk->sk_user_data != psock))
604 goto out;
605
606 if (unlikely(psock->rx_stopped))
607 goto out;
608
609 if (psock->ready_rx_msg)
610 goto out;
611
612 rd_desc.arg.data = psock;
613
614 if (psock_tcp_read_sock(psock) == -ENOMEM)
615 queue_delayed_work(kcm_wq, &psock->rx_delayed_work, 0);
616
617out:
618 read_unlock_bh(&csk->sk_callback_lock);
619 release_sock(csk);
620}
621
622static void psock_rx_work(struct work_struct *w)
623{
624 do_psock_rx_work(container_of(w, struct kcm_psock, rx_work));
625}
626
627static void psock_rx_delayed_work(struct work_struct *w)
628{
629 do_psock_rx_work(container_of(w, struct kcm_psock,
630 rx_delayed_work.work));
631}
632
633static void psock_tcp_state_change(struct sock *sk)
634{
635 /* TCP only does a POLLIN for a half close. Do a POLLHUP here
636 * since application will normally not poll with POLLIN
637 * on the TCP sockets.
638 */
639
640 report_csk_error(sk, EPIPE);
641}
642
643static void psock_tcp_write_space(struct sock *sk)
644{
645 struct kcm_psock *psock;
646 struct kcm_mux *mux;
647 struct kcm_sock *kcm;
648
649 read_lock_bh(&sk->sk_callback_lock);
650
651 psock = (struct kcm_psock *)sk->sk_user_data;
652 if (unlikely(!psock))
653 goto out;
654
655 mux = psock->mux;
656
657 spin_lock_bh(&mux->lock);
658
659 /* Check if the socket is reserved so someone is waiting for sending. */
660 kcm = psock->tx_kcm;
661 if (kcm)
662 queue_work(kcm_wq, &kcm->tx_work);
663
664 spin_unlock_bh(&mux->lock);
665out:
666 read_unlock_bh(&sk->sk_callback_lock);
667}
668
669static void unreserve_psock(struct kcm_sock *kcm);
670
671/* kcm sock is locked. */
672static struct kcm_psock *reserve_psock(struct kcm_sock *kcm)
673{
674 struct kcm_mux *mux = kcm->mux;
675 struct kcm_psock *psock;
676
677 psock = kcm->tx_psock;
678
679 smp_rmb(); /* Must read tx_psock before tx_wait */
680
681 if (psock) {
682 WARN_ON(kcm->tx_wait);
683 if (unlikely(psock->tx_stopped))
684 unreserve_psock(kcm);
685 else
686 return kcm->tx_psock;
687 }
688
689 spin_lock_bh(&mux->lock);
690
691 /* Check again under lock to see if psock was reserved for this
692 * psock via psock_unreserve.
693 */
694 psock = kcm->tx_psock;
695 if (unlikely(psock)) {
696 WARN_ON(kcm->tx_wait);
697 spin_unlock_bh(&mux->lock);
698 return kcm->tx_psock;
699 }
700
701 if (!list_empty(&mux->psocks_avail)) {
702 psock = list_first_entry(&mux->psocks_avail,
703 struct kcm_psock,
704 psock_avail_list);
705 list_del(&psock->psock_avail_list);
706 if (kcm->tx_wait) {
707 list_del(&kcm->wait_psock_list);
708 kcm->tx_wait = false;
709 }
710 kcm->tx_psock = psock;
711 psock->tx_kcm = kcm;
cd6e111b 712 KCM_STATS_INCR(psock->stats.reserved);
ab7ac4eb
TH		 713 } else if (!kcm->tx_wait) {
714 list_add_tail(&kcm->wait_psock_list,
715 &mux->kcm_tx_waiters);
716 kcm->tx_wait = true;
717 }
718
719 spin_unlock_bh(&mux->lock);
720
721 return psock;
722}
723
724/* mux lock held */
725static void psock_now_avail(struct kcm_psock *psock)
726{
727 struct kcm_mux *mux = psock->mux;
728 struct kcm_sock *kcm;
729
730 if (list_empty(&mux->kcm_tx_waiters)) {
731 list_add_tail(&psock->psock_avail_list,
732 &mux->psocks_avail);
733 } else {
734 kcm = list_first_entry(&mux->kcm_tx_waiters,
735 struct kcm_sock,
736 wait_psock_list);
737 list_del(&kcm->wait_psock_list);
738 kcm->tx_wait = false;
739 psock->tx_kcm = kcm;
740
741 /* Commit before changing tx_psock since that is read in
742 * reserve_psock before queuing work.
743 */
744 smp_mb();
745
746 kcm->tx_psock = psock;
cd6e111b 747 KCM_STATS_INCR(psock->stats.reserved);
ab7ac4eb
TH		 748 queue_work(kcm_wq, &kcm->tx_work);
749 }
750}
751
752/* kcm sock is locked. */
753static void unreserve_psock(struct kcm_sock *kcm)
754{
755 struct kcm_psock *psock;
756 struct kcm_mux *mux = kcm->mux;
757
758 spin_lock_bh(&mux->lock);
759
760 psock = kcm->tx_psock;
761
762 if (WARN_ON(!psock)) {
763 spin_unlock_bh(&mux->lock);
764 return;
765 }
766
767 smp_rmb(); /* Read tx_psock before tx_wait */
768
cd6e111b
TH		 769 kcm_update_tx_mux_stats(mux, psock);
770
ab7ac4eb
TH		 771 WARN_ON(kcm->tx_wait);
772
773 kcm->tx_psock = NULL;
774 psock->tx_kcm = NULL;
cd6e111b 775 KCM_STATS_INCR(psock->stats.unreserved);
ab7ac4eb
TH		 776
777 if (unlikely(psock->tx_stopped)) {
778 if (psock->done) {
779 /* Deferred free */
780 list_del(&psock->psock_list);
781 mux->psocks_cnt--;
782 sock_put(psock->sk);
783 fput(psock->sk->sk_socket->file);
784 kmem_cache_free(kcm_psockp, psock);
785 }
786
787 /* Don't put back on available list */
788
789 spin_unlock_bh(&mux->lock);
790
791 return;
792 }
793
794 psock_now_avail(psock);
795
796 spin_unlock_bh(&mux->lock);
797}
798
cd6e111b
TH		 799static void kcm_report_tx_retry(struct kcm_sock *kcm)
800{
801 struct kcm_mux *mux = kcm->mux;
802
803 spin_lock_bh(&mux->lock);
804 KCM_STATS_INCR(mux->stats.tx_retries);
805 spin_unlock_bh(&mux->lock);
806}
807
ab7ac4eb
TH		 808/* Write any messages ready on the kcm socket. Called with kcm sock lock
809 * held. Return bytes actually sent or error.
810 */
811static int kcm_write_msgs(struct kcm_sock *kcm)
812{
813 struct sock *sk = &kcm->sk;
814 struct kcm_psock *psock;
815 struct sk_buff *skb, *head;
816 struct kcm_tx_msg *txm;
817 unsigned short fragidx, frag_offset;
818 unsigned int sent, total_sent = 0;
819 int ret = 0;
820
821 kcm->tx_wait_more = false;
822 psock = kcm->tx_psock;
823 if (unlikely(psock && psock->tx_stopped)) {
824 /* A reserved psock was aborted asynchronously. Unreserve
825 * it and we'll retry the message.
826 */
827 unreserve_psock(kcm);
cd6e111b 828 kcm_report_tx_retry(kcm);
ab7ac4eb
TH		 829 if (skb_queue_empty(&sk->sk_write_queue))
830 return 0;
831
832 kcm_tx_msg(skb_peek(&sk->sk_write_queue))->sent = 0;
833
834 } else if (skb_queue_empty(&sk->sk_write_queue)) {
835 return 0;
836 }
837
838 head = skb_peek(&sk->sk_write_queue);
839 txm = kcm_tx_msg(head);
840
841 if (txm->sent) {
842 /* Send of first skbuff in queue already in progress */
843 if (WARN_ON(!psock)) {
844 ret = -EINVAL;
845 goto out;
846 }
847 sent = txm->sent;
848 frag_offset = txm->frag_offset;
849 fragidx = txm->fragidx;
850 skb = txm->frag_skb;
851
852 goto do_frag;
853 }
854
855try_again:
856 psock = reserve_psock(kcm);
857 if (!psock)
858 goto out;
859
860 do {
861 skb = head;
862 txm = kcm_tx_msg(head);
863 sent = 0;
864
865do_frag_list:
866 if (WARN_ON(!skb_shinfo(skb)->nr_frags)) {
867 ret = -EINVAL;
868 goto out;
869 }
870
871 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags;
872 fragidx++) {
873 skb_frag_t *frag;
874
875 frag_offset = 0;
876do_frag:
877 frag = &skb_shinfo(skb)->frags[fragidx];
878 if (WARN_ON(!frag->size)) {
879 ret = -EINVAL;
880 goto out;
881 }
882
883 ret = kernel_sendpage(psock->sk->sk_socket,
884 frag->page.p,
885 frag->page_offset + frag_offset,
886 frag->size - frag_offset,
887 MSG_DONTWAIT);
888 if (ret <= 0) {
889 if (ret == -EAGAIN) {
890 /* Save state to try again when there's
891 * write space on the socket
892 */
893 txm->sent = sent;
894 txm->frag_offset = frag_offset;
895 txm->fragidx = fragidx;
896 txm->frag_skb = skb;
897
898 ret = 0;
899 goto out;
900 }
901
902 /* Hard failure in sending message, abort this
903 * psock since it has lost framing
904 * synchonization and retry sending the
905 * message from the beginning.
906 */
907 kcm_abort_tx_psock(psock, ret ? -ret : EPIPE,
908 true);
909 unreserve_psock(kcm);
910
911 txm->sent = 0;
cd6e111b 912 kcm_report_tx_retry(kcm);
ab7ac4eb
TH		 913 ret = 0;
914
915 goto try_again;
916 }
917
918 sent += ret;
919 frag_offset += ret;
cd6e111b 920 KCM_STATS_ADD(psock->stats.tx_bytes, ret);
ab7ac4eb
TH		 921 if (frag_offset < frag->size) {
922 /* Not finished with this frag */
923 goto do_frag;
924 }
925 }
926
927 if (skb == head) {
928 if (skb_has_frag_list(skb)) {
929 skb = skb_shinfo(skb)->frag_list;
930 goto do_frag_list;
931 }
932 } else if (skb->next) {
933 skb = skb->next;
934 goto do_frag_list;
935 }
936
937 /* Successfully sent the whole packet, account for it. */
938 skb_dequeue(&sk->sk_write_queue);
939 kfree_skb(head);
940 sk->sk_wmem_queued -= sent;
941 total_sent += sent;
cd6e111b 942 KCM_STATS_INCR(psock->stats.tx_msgs);
ab7ac4eb
TH		 943 } while ((head = skb_peek(&sk->sk_write_queue)));
944out:
945 if (!head) {
946 /* Done with all queued messages. */
947 WARN_ON(!skb_queue_empty(&sk->sk_write_queue));
948 unreserve_psock(kcm);
949 }
950
951 /* Check if write space is available */
952 sk->sk_write_space(sk);
953
954 return total_sent ? : ret;
955}
956
957static void kcm_tx_work(struct work_struct *w)
958{
959 struct kcm_sock *kcm = container_of(w, struct kcm_sock, tx_work);
960 struct sock *sk = &kcm->sk;
961 int err;
962
963 lock_sock(sk);
964
965 /* Primarily for SOCK_DGRAM sockets, also handle asynchronous tx
966 * aborts
967 */
968 err = kcm_write_msgs(kcm);
969 if (err < 0) {
970 /* Hard failure in write, report error on KCM socket */
971 pr_warn("KCM: Hard failure on kcm_write_msgs %d\n", err);
972 report_csk_error(&kcm->sk, -err);
973 goto out;
974 }
975
976 /* Primarily for SOCK_SEQPACKET sockets */
977 if (likely(sk->sk_socket) &&
978 test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
979 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
980 sk->sk_write_space(sk);
981 }
982
983out:
984 release_sock(sk);
985}
986
987static void kcm_push(struct kcm_sock *kcm)
988{
989 if (kcm->tx_wait_more)
990 kcm_write_msgs(kcm);
991}
992
f29698fc
TH		 993static ssize_t kcm_sendpage(struct socket *sock, struct page *page,
994 int offset, size_t size, int flags)
995
996{
997 struct sock *sk = sock->sk;
998 struct kcm_sock *kcm = kcm_sk(sk);
999 struct sk_buff *skb = NULL, *head = NULL;
1000 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1001 bool eor;
1002 int err = 0;
1003 int i;
1004
1005 if (flags & MSG_SENDPAGE_NOTLAST)
1006 flags |= MSG_MORE;
1007
1008 /* No MSG_EOR from splice, only look at MSG_MORE */
1009 eor = !(flags & MSG_MORE);
1010
1011 lock_sock(sk);
1012
1013 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1014
1015 err = -EPIPE;
1016 if (sk->sk_err)
1017 goto out_error;
1018
1019 if (kcm->seq_skb) {
1020 /* Previously opened message */
1021 head = kcm->seq_skb;
1022 skb = kcm_tx_msg(head)->last_skb;
1023 i = skb_shinfo(skb)->nr_frags;
1024
1025 if (skb_can_coalesce(skb, i, page, offset)) {
1026 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
1027 skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
1028 goto coalesced;
1029 }
1030
1031 if (i >= MAX_SKB_FRAGS) {
1032 struct sk_buff *tskb;
1033
1034 tskb = alloc_skb(0, sk->sk_allocation);
1035 while (!tskb) {
1036 kcm_push(kcm);
1037 err = sk_stream_wait_memory(sk, &timeo);
1038 if (err)
1039 goto out_error;
1040 }
1041
1042 if (head == skb)
1043 skb_shinfo(head)->frag_list = tskb;
1044 else
1045 skb->next = tskb;
1046
1047 skb = tskb;
1048 skb->ip_summed = CHECKSUM_UNNECESSARY;
1049 i = 0;
1050 }
1051 } else {
1052 /* Call the sk_stream functions to manage the sndbuf mem. */
1053 if (!sk_stream_memory_free(sk)) {
1054 kcm_push(kcm);
1055 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1056 err = sk_stream_wait_memory(sk, &timeo);
1057 if (err)
1058 goto out_error;
1059 }
1060
1061 head = alloc_skb(0, sk->sk_allocation);
1062 while (!head) {
1063 kcm_push(kcm);
1064 err = sk_stream_wait_memory(sk, &timeo);
1065 if (err)
1066 goto out_error;
1067 }
1068
1069 skb = head;
1070 i = 0;
1071 }
1072
1073 get_page(page);
1074 skb_fill_page_desc(skb, i, page, offset, size);
1075 skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
1076
1077coalesced:
1078 skb->len += size;
1079 skb->data_len += size;
1080 skb->truesize += size;
1081 sk->sk_wmem_queued += size;
1082 sk_mem_charge(sk, size);
1083
1084 if (head != skb) {
1085 head->len += size;
1086 head->data_len += size;
1087 head->truesize += size;
1088 }
1089
1090 if (eor) {
1091 bool not_busy = skb_queue_empty(&sk->sk_write_queue);
1092
1093 /* Message complete, queue it on send buffer */
1094 __skb_queue_tail(&sk->sk_write_queue, head);
1095 kcm->seq_skb = NULL;
1096 KCM_STATS_INCR(kcm->stats.tx_msgs);
1097
1098 if (flags & MSG_BATCH) {
1099 kcm->tx_wait_more = true;
1100 } else if (kcm->tx_wait_more || not_busy) {
1101 err = kcm_write_msgs(kcm);
1102 if (err < 0) {
1103 /* We got a hard error in write_msgs but have
1104 * already queued this message. Report an error
1105 * in the socket, but don't affect return value
1106 * from sendmsg
1107 */
1108 pr_warn("KCM: Hard failure on kcm_write_msgs\n");
1109 report_csk_error(&kcm->sk, -err);
1110 }
1111 }
1112 } else {
1113 /* Message not complete, save state */
1114 kcm->seq_skb = head;
1115 kcm_tx_msg(head)->last_skb = skb;
1116 }
1117
1118 KCM_STATS_ADD(kcm->stats.tx_bytes, size);
1119
1120 release_sock(sk);
1121 return size;
1122
1123out_error:
1124 kcm_push(kcm);
1125
1126 err = sk_stream_error(sk, flags, err);
1127
1128 /* make sure we wake any epoll edge trigger waiter */
1129 if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
1130 sk->sk_write_space(sk);
1131
1132 release_sock(sk);
1133 return err;
1134}
1135
ab7ac4eb
TH		 1136static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1137{
1138 struct sock *sk = sock->sk;
1139 struct kcm_sock *kcm = kcm_sk(sk);
1140 struct sk_buff *skb = NULL, *head = NULL;
1141 size_t copy, copied = 0;
1142 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
1143 int eor = (sock->type == SOCK_DGRAM) ?
1144 !(msg->msg_flags & MSG_MORE) : !!(msg->msg_flags & MSG_EOR);
1145 int err = -EPIPE;
1146
1147 lock_sock(sk);
1148
1149 /* Per tcp_sendmsg this should be in poll */
1150 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1151
1152 if (sk->sk_err)
1153 goto out_error;
1154
1155 if (kcm->seq_skb) {
1156 /* Previously opened message */
1157 head = kcm->seq_skb;
1158 skb = kcm_tx_msg(head)->last_skb;
1159 goto start;
1160 }
1161
1162 /* Call the sk_stream functions to manage the sndbuf mem. */
1163 if (!sk_stream_memory_free(sk)) {
1164 kcm_push(kcm);
1165 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1166 err = sk_stream_wait_memory(sk, &timeo);
1167 if (err)
1168 goto out_error;
1169 }
1170
1171 /* New message, alloc head skb */
1172 head = alloc_skb(0, sk->sk_allocation);
1173 while (!head) {
1174 kcm_push(kcm);
1175 err = sk_stream_wait_memory(sk, &timeo);
1176 if (err)
1177 goto out_error;
1178
1179 head = alloc_skb(0, sk->sk_allocation);
1180 }
1181
1182 skb = head;
1183
1184 /* Set ip_summed to CHECKSUM_UNNECESSARY to avoid calling
1185 * csum_and_copy_from_iter from skb_do_copy_data_nocache.
1186 */
1187 skb->ip_summed = CHECKSUM_UNNECESSARY;
1188
1189start:
1190 while (msg_data_left(msg)) {
1191 bool merge = true;
1192 int i = skb_shinfo(skb)->nr_frags;
1193 struct page_frag *pfrag = sk_page_frag(sk);
1194
1195 if (!sk_page_frag_refill(sk, pfrag))
1196 goto wait_for_memory;
1197
1198 if (!skb_can_coalesce(skb, i, pfrag->page,
1199 pfrag->offset)) {
1200 if (i == MAX_SKB_FRAGS) {
1201 struct sk_buff *tskb;
1202
1203 tskb = alloc_skb(0, sk->sk_allocation);
1204 if (!tskb)
1205 goto wait_for_memory;
1206
1207 if (head == skb)
1208 skb_shinfo(head)->frag_list = tskb;
1209 else
1210 skb->next = tskb;
1211
1212 skb = tskb;
1213 skb->ip_summed = CHECKSUM_UNNECESSARY;
1214 continue;
1215 }
1216 merge = false;
1217 }
1218
1219 copy = min_t(int, msg_data_left(msg),
1220 pfrag->size - pfrag->offset);
1221
1222 if (!sk_wmem_schedule(sk, copy))
1223 goto wait_for_memory;
1224
1225 err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
1226 pfrag->page,
1227 pfrag->offset,
1228 copy);
1229 if (err)
1230 goto out_error;
1231
1232 /* Update the skb. */
1233 if (merge) {
1234 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1235 } else {
1236 skb_fill_page_desc(skb, i, pfrag->page,
1237 pfrag->offset, copy);
1238 get_page(pfrag->page);
1239 }
1240
1241 pfrag->offset += copy;
1242 copied += copy;
1243 if (head != skb) {
1244 head->len += copy;
1245 head->data_len += copy;
1246 }
1247
1248 continue;
1249
1250wait_for_memory:
1251 kcm_push(kcm);
1252 err = sk_stream_wait_memory(sk, &timeo);
1253 if (err)
1254 goto out_error;
1255 }
1256
1257 if (eor) {
1258 bool not_busy = skb_queue_empty(&sk->sk_write_queue);
1259
1260 /* Message complete, queue it on send buffer */
1261 __skb_queue_tail(&sk->sk_write_queue, head);
1262 kcm->seq_skb = NULL;
cd6e111b 1263 KCM_STATS_INCR(kcm->stats.tx_msgs);
ab7ac4eb
TH		 1264
1265 if (msg->msg_flags & MSG_BATCH) {
1266 kcm->tx_wait_more = true;
1267 } else if (kcm->tx_wait_more || not_busy) {
1268 err = kcm_write_msgs(kcm);
1269 if (err < 0) {
1270 /* We got a hard error in write_msgs but have
1271 * already queued this message. Report an error
1272 * in the socket, but don't affect return value
1273 * from sendmsg
1274 */
1275 pr_warn("KCM: Hard failure on kcm_write_msgs\n");
1276 report_csk_error(&kcm->sk, -err);
1277 }
1278 }
1279 } else {
1280 /* Message not complete, save state */
1281partial_message:
1282 kcm->seq_skb = head;
1283 kcm_tx_msg(head)->last_skb = skb;
1284 }
1285
cd6e111b
TH		 1286 KCM_STATS_ADD(kcm->stats.tx_bytes, copied);
1287
ab7ac4eb
TH		 1288 release_sock(sk);
1289 return copied;
1290
1291out_error:
1292 kcm_push(kcm);
1293
1294 if (copied && sock->type == SOCK_SEQPACKET) {
1295 /* Wrote some bytes before encountering an
1296 * error, return partial success.
1297 */
1298 goto partial_message;
1299 }
1300
1301 if (head != kcm->seq_skb)
1302 kfree_skb(head);
1303
1304 err = sk_stream_error(sk, msg->msg_flags, err);
1305
1306 /* make sure we wake any epoll edge trigger waiter */
1307 if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
1308 sk->sk_write_space(sk);
1309
1310 release_sock(sk);
1311 return err;
1312}
1313
1314static struct sk_buff *kcm_wait_data(struct sock *sk, int flags,
1315 long timeo, int *err)
1316{
1317 struct sk_buff *skb;
1318
1319 while (!(skb = skb_peek(&sk->sk_receive_queue))) {
1320 if (sk->sk_err) {
1321 *err = sock_error(sk);
1322 return NULL;
1323 }
1324
1325 if (sock_flag(sk, SOCK_DONE))
1326 return NULL;
1327
1328 if ((flags & MSG_DONTWAIT) || !timeo) {
1329 *err = -EAGAIN;
1330 return NULL;
1331 }
1332
1333 sk_wait_data(sk, &timeo, NULL);
1334
1335 /* Handle signals */
1336 if (signal_pending(current)) {
1337 *err = sock_intr_errno(timeo);
1338 return NULL;
1339 }
1340 }
1341
1342 return skb;
1343}
1344
1345static int kcm_recvmsg(struct socket *sock, struct msghdr *msg,
1346 size_t len, int flags)
1347{
1348 struct sock *sk = sock->sk;
cd6e111b 1349 struct kcm_sock *kcm = kcm_sk(sk);
ab7ac4eb
TH		 1350 int err = 0;
1351 long timeo;
1352 struct kcm_rx_msg *rxm;
1353 int copied = 0;
1354 struct sk_buff *skb;
1355
1356 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1357
1358 lock_sock(sk);
1359
1360 skb = kcm_wait_data(sk, flags, timeo, &err);
1361 if (!skb)
1362 goto out;
1363
1364 /* Okay, have a message on the receive queue */
1365
1366 rxm = kcm_rx_msg(skb);
1367
1368 if (len > rxm->full_len)
1369 len = rxm->full_len;
1370
1371 err = skb_copy_datagram_msg(skb, rxm->offset, msg, len);
1372 if (err < 0)
1373 goto out;
1374
1375 copied = len;
1376 if (likely(!(flags & MSG_PEEK))) {
cd6e111b 1377 KCM_STATS_ADD(kcm->stats.rx_bytes, copied);
ab7ac4eb
TH		 1378 if (copied < rxm->full_len) {
1379 if (sock->type == SOCK_DGRAM) {
1380 /* Truncated message */
1381 msg->msg_flags |= MSG_TRUNC;
1382 goto msg_finished;
1383 }
1384 rxm->offset += copied;
1385 rxm->full_len -= copied;
1386 } else {
1387msg_finished:
1388 /* Finished with message */
1389 msg->msg_flags |= MSG_EOR;
cd6e111b 1390 KCM_STATS_INCR(kcm->stats.rx_msgs);
ab7ac4eb
TH		 1391 skb_unlink(skb, &sk->sk_receive_queue);
1392 kfree_skb(skb);
1393 }
1394 }
1395
1396out:
1397 release_sock(sk);
1398
1399 return copied ? : err;
1400}
1401
91687355
TH		 1402static ssize_t kcm_sock_splice(struct sock *sk,
1403 struct pipe_inode_info *pipe,
1404 struct splice_pipe_desc *spd)
1405{
1406 int ret;
1407
1408 release_sock(sk);
1409 ret = splice_to_pipe(pipe, spd);
1410 lock_sock(sk);
1411
1412 return ret;
1413}
1414
1415static ssize_t kcm_splice_read(struct socket *sock, loff_t *ppos,
1416 struct pipe_inode_info *pipe, size_t len,
1417 unsigned int flags)
1418{
1419 struct sock *sk = sock->sk;
1420 struct kcm_sock *kcm = kcm_sk(sk);
1421 long timeo;
1422 struct kcm_rx_msg *rxm;
1423 int err = 0;
1424 size_t copied;
1425 struct sk_buff *skb;
1426
1427 /* Only support splice for SOCKSEQPACKET */
1428
1429 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1430
1431 lock_sock(sk);
1432
1433 skb = kcm_wait_data(sk, flags, timeo, &err);
1434 if (!skb)
1435 goto err_out;
1436
1437 /* Okay, have a message on the receive queue */
1438
1439 rxm = kcm_rx_msg(skb);
1440
1441 if (len > rxm->full_len)
1442 len = rxm->full_len;
1443
1444 copied = skb_splice_bits(skb, sk, rxm->offset, pipe, len, flags,
1445 kcm_sock_splice);
1446 if (copied < 0) {
1447 err = copied;
1448 goto err_out;
1449 }
1450
1451 KCM_STATS_ADD(kcm->stats.rx_bytes, copied);
1452
1453 rxm->offset += copied;
1454 rxm->full_len -= copied;
1455
1456 /* We have no way to return MSG_EOR. If all the bytes have been
1457 * read we still leave the message in the receive socket buffer.
1458 * A subsequent recvmsg needs to be done to return MSG_EOR and
1459 * finish reading the message.
1460 */
1461
1462 release_sock(sk);
1463
1464 return copied;
1465
1466err_out:
1467 release_sock(sk);
1468
1469 return err;
1470}
1471
ab7ac4eb
TH		 1472/* kcm sock lock held */
1473static void kcm_recv_disable(struct kcm_sock *kcm)
1474{
1475 struct kcm_mux *mux = kcm->mux;
1476
1477 if (kcm->rx_disabled)
1478 return;
1479
1480 spin_lock_bh(&mux->rx_lock);
1481
1482 kcm->rx_disabled = 1;
1483
1484 /* If a psock is reserved we'll do cleanup in unreserve */
1485 if (!kcm->rx_psock) {
1486 if (kcm->rx_wait) {
1487 list_del(&kcm->wait_rx_list);
1488 kcm->rx_wait = false;
1489 }
1490
1491 requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);
1492 }
1493
1494 spin_unlock_bh(&mux->rx_lock);
1495}
1496
1497/* kcm sock lock held */
1498static void kcm_recv_enable(struct kcm_sock *kcm)
1499{
1500 struct kcm_mux *mux = kcm->mux;
1501
1502 if (!kcm->rx_disabled)
1503 return;
1504
1505 spin_lock_bh(&mux->rx_lock);
1506
1507 kcm->rx_disabled = 0;
1508 kcm_rcv_ready(kcm);
1509
1510 spin_unlock_bh(&mux->rx_lock);
1511}
1512
1513static int kcm_setsockopt(struct socket *sock, int level, int optname,
1514 char __user *optval, unsigned int optlen)
1515{
1516 struct kcm_sock *kcm = kcm_sk(sock->sk);
1517 int val, valbool;
1518 int err = 0;
1519
1520 if (level != SOL_KCM)
1521 return -ENOPROTOOPT;
1522
1523 if (optlen < sizeof(int))
1524 return -EINVAL;
1525
1526 if (get_user(val, (int __user *)optval))
1527 return -EINVAL;
1528
1529 valbool = val ? 1 : 0;
1530
1531 switch (optname) {
1532 case KCM_RECV_DISABLE:
1533 lock_sock(&kcm->sk);
1534 if (valbool)
1535 kcm_recv_disable(kcm);
1536 else
1537 kcm_recv_enable(kcm);
1538 release_sock(&kcm->sk);
1539 break;
1540 default:
1541 err = -ENOPROTOOPT;
1542 }
1543
1544 return err;
1545}
1546
1547static int kcm_getsockopt(struct socket *sock, int level, int optname,
1548 char __user *optval, int __user *optlen)
1549{
1550 struct kcm_sock *kcm = kcm_sk(sock->sk);
1551 int val, len;
1552
1553 if (level != SOL_KCM)
1554 return -ENOPROTOOPT;
1555
1556 if (get_user(len, optlen))
1557 return -EFAULT;
1558
1559 len = min_t(unsigned int, len, sizeof(int));
1560 if (len < 0)
1561 return -EINVAL;
1562
1563 switch (optname) {
1564 case KCM_RECV_DISABLE:
1565 val = kcm->rx_disabled;
1566 break;
1567 default:
1568 return -ENOPROTOOPT;
1569 }
1570
1571 if (put_user(len, optlen))
1572 return -EFAULT;
1573 if (copy_to_user(optval, &val, len))
1574 return -EFAULT;
1575 return 0;
1576}
1577
1578static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)
1579{
1580 struct kcm_sock *tkcm;
1581 struct list_head *head;
1582 int index = 0;
1583
1584 /* For SOCK_SEQPACKET sock type, datagram_poll checks the sk_state, so
1585 * we set sk_state, otherwise epoll_wait always returns right away with
1586 * POLLHUP
1587 */
1588 kcm->sk.sk_state = TCP_ESTABLISHED;
1589
1590 /* Add to mux's kcm sockets list */
1591 kcm->mux = mux;
1592 spin_lock_bh(&mux->lock);
1593
1594 head = &mux->kcm_socks;
1595 list_for_each_entry(tkcm, &mux->kcm_socks, kcm_sock_list) {
1596 if (tkcm->index != index)
1597 break;
1598 head = &tkcm->kcm_sock_list;
1599 index++;
1600 }
1601
1602 list_add(&kcm->kcm_sock_list, head);
1603 kcm->index = index;
1604
1605 mux->kcm_socks_cnt++;
1606 spin_unlock_bh(&mux->lock);
1607
1608 INIT_WORK(&kcm->tx_work, kcm_tx_work);
1609
1610 spin_lock_bh(&mux->rx_lock);
1611 kcm_rcv_ready(kcm);
1612 spin_unlock_bh(&mux->rx_lock);
1613}
1614
1615static int kcm_attach(struct socket *sock, struct socket *csock,
1616 struct bpf_prog *prog)
1617{
1618 struct kcm_sock *kcm = kcm_sk(sock->sk);
1619 struct kcm_mux *mux = kcm->mux;
1620 struct sock *csk;
1621 struct kcm_psock *psock = NULL, *tpsock;
1622 struct list_head *head;
1623 int index = 0;
1624
1625 if (csock->ops->family != PF_INET &&
1626 csock->ops->family != PF_INET6)
1627 return -EINVAL;
1628
1629 csk = csock->sk;
1630 if (!csk)
1631 return -EINVAL;
1632
1633 /* Only support TCP for now */
1634 if (csk->sk_protocol != IPPROTO_TCP)
1635 return -EINVAL;
1636
1637 psock = kmem_cache_zalloc(kcm_psockp, GFP_KERNEL);
1638 if (!psock)
1639 return -ENOMEM;
1640
1641 psock->mux = mux;
1642 psock->sk = csk;
1643 psock->bpf_prog = prog;
1644 INIT_WORK(&psock->rx_work, psock_rx_work);
1645 INIT_DELAYED_WORK(&psock->rx_delayed_work, psock_rx_delayed_work);
1646
1647 sock_hold(csk);
1648
1649 write_lock_bh(&csk->sk_callback_lock);
1650 psock->save_data_ready = csk->sk_data_ready;
1651 psock->save_write_space = csk->sk_write_space;
1652 psock->save_state_change = csk->sk_state_change;
1653 csk->sk_user_data = psock;
1654 csk->sk_data_ready = psock_tcp_data_ready;
1655 csk->sk_write_space = psock_tcp_write_space;
1656 csk->sk_state_change = psock_tcp_state_change;
1657 write_unlock_bh(&csk->sk_callback_lock);
1658
1659 /* Finished initialization, now add the psock to the MUX. */
1660 spin_lock_bh(&mux->lock);
1661 head = &mux->psocks;
1662 list_for_each_entry(tpsock, &mux->psocks, psock_list) {
1663 if (tpsock->index != index)
1664 break;
1665 head = &tpsock->psock_list;
1666 index++;
1667 }
1668
1669 list_add(&psock->psock_list, head);
1670 psock->index = index;
1671
cd6e111b 1672 KCM_STATS_INCR(mux->stats.psock_attach);
ab7ac4eb
TH		 1673 mux->psocks_cnt++;
1674 psock_now_avail(psock);
1675 spin_unlock_bh(&mux->lock);
1676
1677 /* Schedule RX work in case there are already bytes queued */
1678 queue_work(kcm_wq, &psock->rx_work);
1679
1680 return 0;
1681}
1682
1683static int kcm_attach_ioctl(struct socket *sock, struct kcm_attach *info)
1684{
1685 struct socket *csock;
1686 struct bpf_prog *prog;
1687 int err;
1688
1689 csock = sockfd_lookup(info->fd, &err);
1690 if (!csock)
1691 return -ENOENT;
1692
1693 prog = bpf_prog_get(info->bpf_fd);
1694 if (IS_ERR(prog)) {
1695 err = PTR_ERR(prog);
1696 goto out;
1697 }
1698
1699 if (prog->type != BPF_PROG_TYPE_SOCKET_FILTER) {
1700 bpf_prog_put(prog);
1701 err = -EINVAL;
1702 goto out;
1703 }
1704
1705 err = kcm_attach(sock, csock, prog);
1706 if (err) {
1707 bpf_prog_put(prog);
1708 goto out;
1709 }
1710
1711 /* Keep reference on file also */
1712
1713 return 0;
1714out:
1715 fput(csock->file);
1716 return err;
1717}
1718
1719static void kcm_unattach(struct kcm_psock *psock)
1720{
1721 struct sock *csk = psock->sk;
1722 struct kcm_mux *mux = psock->mux;
1723
1724 /* Stop getting callbacks from TCP socket. After this there should
1725 * be no way to reserve a kcm for this psock.
1726 */
1727 write_lock_bh(&csk->sk_callback_lock);
1728 csk->sk_user_data = NULL;
1729 csk->sk_data_ready = psock->save_data_ready;
1730 csk->sk_write_space = psock->save_write_space;
1731 csk->sk_state_change = psock->save_state_change;
1732 psock->rx_stopped = 1;
1733
1734 if (WARN_ON(psock->rx_kcm)) {
1735 write_unlock_bh(&csk->sk_callback_lock);
1736 return;
1737 }
1738
1739 spin_lock_bh(&mux->rx_lock);
1740
1741 /* Stop receiver activities. After this point psock should not be
1742 * able to get onto ready list either through callbacks or work.
1743 */
1744 if (psock->ready_rx_msg) {
1745 list_del(&psock->psock_ready_list);
1746 kfree_skb(psock->ready_rx_msg);
1747 psock->ready_rx_msg = NULL;
cd6e111b 1748 KCM_STATS_INCR(mux->stats.rx_ready_drops);
ab7ac4eb
TH		 1749 }
1750
1751 spin_unlock_bh(&mux->rx_lock);
1752
1753 write_unlock_bh(&csk->sk_callback_lock);
1754
1755 cancel_work_sync(&psock->rx_work);
1756 cancel_delayed_work_sync(&psock->rx_delayed_work);
1757
1758 bpf_prog_put(psock->bpf_prog);
1759
1760 kfree_skb(psock->rx_skb_head);
1761 psock->rx_skb_head = NULL;
1762
1763 spin_lock_bh(&mux->lock);
1764
cd6e111b
TH		 1765 aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats);
1766
1767 KCM_STATS_INCR(mux->stats.psock_unattach);
1768
ab7ac4eb
TH		 1769 if (psock->tx_kcm) {
1770 /* psock was reserved. Just mark it finished and we will clean
1771 * up in the kcm paths, we need kcm lock which can not be
1772 * acquired here.
1773 */
cd6e111b 1774 KCM_STATS_INCR(mux->stats.psock_unattach_rsvd);
ab7ac4eb
TH		 1775 spin_unlock_bh(&mux->lock);
1776
1777 /* We are unattaching a socket that is reserved. Abort the
1778 * socket since we may be out of sync in sending on it. We need
1779 * to do this without the mux lock.
1780 */
1781 kcm_abort_tx_psock(psock, EPIPE, false);
1782
1783 spin_lock_bh(&mux->lock);
1784 if (!psock->tx_kcm) {
1785 /* psock now unreserved in window mux was unlocked */
1786 goto no_reserved;
1787 }
1788 psock->done = 1;
1789
1790 /* Commit done before queuing work to process it */
1791 smp_mb();
1792
1793 /* Queue tx work to make sure psock->done is handled */
1794 queue_work(kcm_wq, &psock->tx_kcm->tx_work);
1795 spin_unlock_bh(&mux->lock);
1796 } else {
1797no_reserved:
1798 if (!psock->tx_stopped)
1799 list_del(&psock->psock_avail_list);
1800 list_del(&psock->psock_list);
1801 mux->psocks_cnt--;
1802 spin_unlock_bh(&mux->lock);
1803
1804 sock_put(csk);
1805 fput(csk->sk_socket->file);
1806 kmem_cache_free(kcm_psockp, psock);
1807 }
1808}
1809
1810static int kcm_unattach_ioctl(struct socket *sock, struct kcm_unattach *info)
1811{
1812 struct kcm_sock *kcm = kcm_sk(sock->sk);
1813 struct kcm_mux *mux = kcm->mux;
1814 struct kcm_psock *psock;
1815 struct socket *csock;
1816 struct sock *csk;
1817 int err;
1818
1819 csock = sockfd_lookup(info->fd, &err);
1820 if (!csock)
1821 return -ENOENT;
1822
1823 csk = csock->sk;
1824 if (!csk) {
1825 err = -EINVAL;
1826 goto out;
1827 }
1828
1829 err = -ENOENT;
1830
1831 spin_lock_bh(&mux->lock);
1832
1833 list_for_each_entry(psock, &mux->psocks, psock_list) {
1834 if (psock->sk != csk)
1835 continue;
1836
1837 /* Found the matching psock */
1838
1839 if (psock->unattaching || WARN_ON(psock->done)) {
1840 err = -EALREADY;
1841 break;
1842 }
1843
1844 psock->unattaching = 1;
1845
1846 spin_unlock_bh(&mux->lock);
1847
1848 kcm_unattach(psock);
1849
1850 err = 0;
1851 goto out;
1852 }
1853
1854 spin_unlock_bh(&mux->lock);
1855
1856out:
1857 fput(csock->file);
1858 return err;
1859}
1860
1861static struct proto kcm_proto = {
1862 .name = "KCM",
1863 .owner = THIS_MODULE,
1864 .obj_size = sizeof(struct kcm_sock),
1865};
1866
1867/* Clone a kcm socket. */
1868static int kcm_clone(struct socket *osock, struct kcm_clone *info,
1869 struct socket **newsockp)
1870{
1871 struct socket *newsock;
1872 struct sock *newsk;
1873 struct file *newfile;
1874 int err, newfd;
1875
1876 err = -ENFILE;
1877 newsock = sock_alloc();
1878 if (!newsock)
1879 goto out;
1880
1881 newsock->type = osock->type;
1882 newsock->ops = osock->ops;
1883
1884 __module_get(newsock->ops->owner);
1885
1886 newfd = get_unused_fd_flags(0);
1887 if (unlikely(newfd < 0)) {
1888 err = newfd;
1889 goto out_fd_fail;
1890 }
1891
1892 newfile = sock_alloc_file(newsock, 0, osock->sk->sk_prot_creator->name);
1893 if (unlikely(IS_ERR(newfile))) {
1894 err = PTR_ERR(newfile);
1895 goto out_sock_alloc_fail;
1896 }
1897
1898 newsk = sk_alloc(sock_net(osock->sk), PF_KCM, GFP_KERNEL,
1899 &kcm_proto, true);
1900 if (!newsk) {
1901 err = -ENOMEM;
1902 goto out_sk_alloc_fail;
1903 }
1904
1905 sock_init_data(newsock, newsk);
1906 init_kcm_sock(kcm_sk(newsk), kcm_sk(osock->sk)->mux);
1907
1908 fd_install(newfd, newfile);
1909 *newsockp = newsock;
1910 info->fd = newfd;
1911
1912 return 0;
1913
1914out_sk_alloc_fail:
1915 fput(newfile);
1916out_sock_alloc_fail:
1917 put_unused_fd(newfd);
1918out_fd_fail:
1919 sock_release(newsock);
1920out:
1921 return err;
1922}
1923
1924static int kcm_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
1925{
1926 int err;
1927
1928 switch (cmd) {
1929 case SIOCKCMATTACH: {
1930 struct kcm_attach info;
1931
1932 if (copy_from_user(&info, (void __user *)arg, sizeof(info)))
1933 err = -EFAULT;
1934
1935 err = kcm_attach_ioctl(sock, &info);
1936
1937 break;
1938 }
1939 case SIOCKCMUNATTACH: {
1940 struct kcm_unattach info;
1941
1942 if (copy_from_user(&info, (void __user *)arg, sizeof(info)))
1943 err = -EFAULT;
1944
1945 err = kcm_unattach_ioctl(sock, &info);
1946
1947 break;
1948 }
1949 case SIOCKCMCLONE: {
1950 struct kcm_clone info;
1951 struct socket *newsock = NULL;
1952
1953 if (copy_from_user(&info, (void __user *)arg, sizeof(info)))
1954 err = -EFAULT;
1955
1956 err = kcm_clone(sock, &info, &newsock);
1957
1958 if (!err) {
1959 if (copy_to_user((void __user *)arg, &info,
1960 sizeof(info))) {
1961 err = -EFAULT;
1962 sock_release(newsock);
1963 }
1964 }
1965
1966 break;
1967 }
1968 default:
1969 err = -ENOIOCTLCMD;
1970 break;
1971 }
1972
1973 return err;
1974}
1975
1976static void free_mux(struct rcu_head *rcu)
1977{
1978 struct kcm_mux *mux = container_of(rcu,
1979 struct kcm_mux, rcu);
1980
1981 kmem_cache_free(kcm_muxp, mux);
1982}
1983
1984static void release_mux(struct kcm_mux *mux)
1985{
1986 struct kcm_net *knet = mux->knet;
1987 struct kcm_psock *psock, *tmp_psock;
1988
1989 /* Release psocks */
1990 list_for_each_entry_safe(psock, tmp_psock,
1991 &mux->psocks, psock_list) {
1992 if (!WARN_ON(psock->unattaching))
1993 kcm_unattach(psock);
1994 }
1995
1996 if (WARN_ON(mux->psocks_cnt))
1997 return;
1998
1999 __skb_queue_purge(&mux->rx_hold_queue);
2000
2001 mutex_lock(&knet->mutex);
cd6e111b
TH		 2002 aggregate_mux_stats(&mux->stats, &knet->aggregate_mux_stats);
2003 aggregate_psock_stats(&mux->aggregate_psock_stats,
2004 &knet->aggregate_psock_stats);
ab7ac4eb
TH		 2005 list_del_rcu(&mux->kcm_mux_list);
2006 knet->count--;
2007 mutex_unlock(&knet->mutex);
2008
2009 call_rcu(&mux->rcu, free_mux);
2010}
2011
2012static void kcm_done(struct kcm_sock *kcm)
2013{
2014 struct kcm_mux *mux = kcm->mux;
2015 struct sock *sk = &kcm->sk;
2016 int socks_cnt;
2017
2018 spin_lock_bh(&mux->rx_lock);
2019 if (kcm->rx_psock) {
2020 /* Cleanup in unreserve_rx_kcm */
2021 WARN_ON(kcm->done);
2022 kcm->rx_disabled = 1;
2023 kcm->done = 1;
2024 spin_unlock_bh(&mux->rx_lock);
2025 return;
2026 }
2027
2028 if (kcm->rx_wait) {
2029 list_del(&kcm->wait_rx_list);
2030 kcm->rx_wait = false;
2031 }
2032 /* Move any pending receive messages to other kcm sockets */
2033 requeue_rx_msgs(mux, &sk->sk_receive_queue);
2034
2035 spin_unlock_bh(&mux->rx_lock);
2036
2037 if (WARN_ON(sk_rmem_alloc_get(sk)))
2038 return;
2039
2040 /* Detach from MUX */
2041 spin_lock_bh(&mux->lock);
2042
2043 list_del(&kcm->kcm_sock_list);
2044 mux->kcm_socks_cnt--;
2045 socks_cnt = mux->kcm_socks_cnt;
2046
2047 spin_unlock_bh(&mux->lock);
2048
2049 if (!socks_cnt) {
2050 /* We are done with the mux now. */
2051 release_mux(mux);
2052 }
2053
2054 WARN_ON(kcm->rx_wait);
2055
2056 sock_put(&kcm->sk);
2057}
2058
2059/* Called by kcm_release to close a KCM socket.
2060 * If this is the last KCM socket on the MUX, destroy the MUX.
2061 */
2062static int kcm_release(struct socket *sock)
2063{
2064 struct sock *sk = sock->sk;
2065 struct kcm_sock *kcm;
2066 struct kcm_mux *mux;
2067 struct kcm_psock *psock;
2068
2069 if (!sk)
2070 return 0;
2071
2072 kcm = kcm_sk(sk);
2073 mux = kcm->mux;
2074
2075 sock_orphan(sk);
2076 kfree_skb(kcm->seq_skb);
2077
2078 lock_sock(sk);
2079 /* Purge queue under lock to avoid race condition with tx_work trying
2080 * to act when queue is nonempty. If tx_work runs after this point
2081 * it will just return.
2082 */
2083 __skb_queue_purge(&sk->sk_write_queue);
2084 release_sock(sk);
2085
2086 spin_lock_bh(&mux->lock);
2087 if (kcm->tx_wait) {
2088 /* Take of tx_wait list, after this point there should be no way
2089 * that a psock will be assigned to this kcm.
2090 */
2091 list_del(&kcm->wait_psock_list);
2092 kcm->tx_wait = false;
2093 }
2094 spin_unlock_bh(&mux->lock);
2095
2096 /* Cancel work. After this point there should be no outside references
2097 * to the kcm socket.
2098 */
2099 cancel_work_sync(&kcm->tx_work);
2100
2101 lock_sock(sk);
2102 psock = kcm->tx_psock;
2103 if (psock) {
2104 /* A psock was reserved, so we need to kill it since it
2105 * may already have some bytes queued from a message. We
2106 * need to do this after removing kcm from tx_wait list.
2107 */
2108 kcm_abort_tx_psock(psock, EPIPE, false);
2109 unreserve_psock(kcm);
2110 }
2111 release_sock(sk);
2112
2113 WARN_ON(kcm->tx_wait);
2114 WARN_ON(kcm->tx_psock);
2115
2116 sock->sk = NULL;
2117
2118 kcm_done(kcm);
2119
2120 return 0;
2121}
2122
91687355 2123static const struct proto_ops kcm_dgram_ops = {
ab7ac4eb
TH		 2124 .family = PF_KCM,
2125 .owner = THIS_MODULE,
2126 .release = kcm_release,
2127 .bind = sock_no_bind,
2128 .connect = sock_no_connect,
2129 .socketpair = sock_no_socketpair,
2130 .accept = sock_no_accept,
2131 .getname = sock_no_getname,
2132 .poll = datagram_poll,
2133 .ioctl = kcm_ioctl,
2134 .listen = sock_no_listen,
2135 .shutdown = sock_no_shutdown,
2136 .setsockopt = kcm_setsockopt,
2137 .getsockopt = kcm_getsockopt,
2138 .sendmsg = kcm_sendmsg,
2139 .recvmsg = kcm_recvmsg,
2140 .mmap = sock_no_mmap,
f29698fc 2141 .sendpage = kcm_sendpage,
ab7ac4eb
TH		 2142};
2143
91687355
TH		 2144static const struct proto_ops kcm_seqpacket_ops = {
2145 .family = PF_KCM,
2146 .owner = THIS_MODULE,
2147 .release = kcm_release,
2148 .bind = sock_no_bind,
2149 .connect = sock_no_connect,
2150 .socketpair = sock_no_socketpair,
2151 .accept = sock_no_accept,
2152 .getname = sock_no_getname,
2153 .poll = datagram_poll,
2154 .ioctl = kcm_ioctl,
2155 .listen = sock_no_listen,
2156 .shutdown = sock_no_shutdown,
2157 .setsockopt = kcm_setsockopt,
2158 .getsockopt = kcm_getsockopt,
2159 .sendmsg = kcm_sendmsg,
2160 .recvmsg = kcm_recvmsg,
2161 .mmap = sock_no_mmap,
f29698fc 2162 .sendpage = kcm_sendpage,
91687355
TH		 2163 .splice_read = kcm_splice_read,
2164};
2165
ab7ac4eb
TH		 2166/* Create proto operation for kcm sockets */
2167static int kcm_create(struct net *net, struct socket *sock,
2168 int protocol, int kern)
2169{
2170 struct kcm_net *knet = net_generic(net, kcm_net_id);
2171 struct sock *sk;
2172 struct kcm_mux *mux;
2173
2174 switch (sock->type) {
2175 case SOCK_DGRAM:
91687355
TH		 2176 sock->ops = &kcm_dgram_ops;
2177 break;
ab7ac4eb 2178 case SOCK_SEQPACKET:
91687355 2179 sock->ops = &kcm_seqpacket_ops;
ab7ac4eb
TH		 2180 break;
2181 default:
2182 return -ESOCKTNOSUPPORT;
2183 }
2184
2185 if (protocol != KCMPROTO_CONNECTED)
2186 return -EPROTONOSUPPORT;
2187
2188 sk = sk_alloc(net, PF_KCM, GFP_KERNEL, &kcm_proto, kern);
2189 if (!sk)
2190 return -ENOMEM;
2191
2192 /* Allocate a kcm mux, shared between KCM sockets */
2193 mux = kmem_cache_zalloc(kcm_muxp, GFP_KERNEL);
2194 if (!mux) {
2195 sk_free(sk);
2196 return -ENOMEM;
2197 }
2198
2199 spin_lock_init(&mux->lock);
2200 spin_lock_init(&mux->rx_lock);
2201 INIT_LIST_HEAD(&mux->kcm_socks);
2202 INIT_LIST_HEAD(&mux->kcm_rx_waiters);
2203 INIT_LIST_HEAD(&mux->kcm_tx_waiters);
2204
2205 INIT_LIST_HEAD(&mux->psocks);
2206 INIT_LIST_HEAD(&mux->psocks_ready);
2207 INIT_LIST_HEAD(&mux->psocks_avail);
2208
2209 mux->knet = knet;
2210
2211 /* Add new MUX to list */
2212 mutex_lock(&knet->mutex);
2213 list_add_rcu(&mux->kcm_mux_list, &knet->mux_list);
2214 knet->count++;
2215 mutex_unlock(&knet->mutex);
2216
2217 skb_queue_head_init(&mux->rx_hold_queue);
2218
2219 /* Init KCM socket */
2220 sock_init_data(sock, sk);
2221 init_kcm_sock(kcm_sk(sk), mux);
2222
2223 return 0;
2224}
2225
2226static struct net_proto_family kcm_family_ops = {
2227 .family = PF_KCM,
2228 .create = kcm_create,
2229 .owner = THIS_MODULE,
2230};
2231
2232static __net_init int kcm_init_net(struct net *net)
2233{
2234 struct kcm_net *knet = net_generic(net, kcm_net_id);
2235
2236 INIT_LIST_HEAD_RCU(&knet->mux_list);
2237 mutex_init(&knet->mutex);
2238
2239 return 0;
2240}
2241
2242static __net_exit void kcm_exit_net(struct net *net)
2243{
2244 struct kcm_net *knet = net_generic(net, kcm_net_id);
2245
2246 /* All KCM sockets should be closed at this point, which should mean
2247 * that all multiplexors and psocks have been destroyed.
2248 */
2249 WARN_ON(!list_empty(&knet->mux_list));
2250}
2251
2252static struct pernet_operations kcm_net_ops = {
2253 .init = kcm_init_net,
2254 .exit = kcm_exit_net,
2255 .id = &kcm_net_id,
2256 .size = sizeof(struct kcm_net),
2257};
2258
2259static int __init kcm_init(void)
2260{
2261 int err = -ENOMEM;
2262
2263 kcm_muxp = kmem_cache_create("kcm_mux_cache",
2264 sizeof(struct kcm_mux), 0,
2265 SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
2266 if (!kcm_muxp)
2267 goto fail;
2268
2269 kcm_psockp = kmem_cache_create("kcm_psock_cache",
2270 sizeof(struct kcm_psock), 0,
2271 SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
2272 if (!kcm_psockp)
2273 goto fail;
2274
2275 kcm_wq = create_singlethread_workqueue("kkcmd");
2276 if (!kcm_wq)
2277 goto fail;
2278
2279 err = proto_register(&kcm_proto, 1);
2280 if (err)
2281 goto fail;
2282
2283 err = sock_register(&kcm_family_ops);
2284 if (err)
2285 goto sock_register_fail;
2286
2287 err = register_pernet_device(&kcm_net_ops);
2288 if (err)
2289 goto net_ops_fail;
2290
cd6e111b
TH		 2291 err = kcm_proc_init();
2292 if (err)
2293 goto proc_init_fail;
2294
ab7ac4eb
TH		 2295 return 0;
2296
cd6e111b
TH		 2297proc_init_fail:
2298 unregister_pernet_device(&kcm_net_ops);
2299
ab7ac4eb
TH		 2300net_ops_fail:
2301 sock_unregister(PF_KCM);
2302
2303sock_register_fail:
2304 proto_unregister(&kcm_proto);
2305
2306fail:
2307 kmem_cache_destroy(kcm_muxp);
2308 kmem_cache_destroy(kcm_psockp);
2309
2310 if (kcm_wq)
2311 destroy_workqueue(kcm_wq);
2312
2313 return err;
2314}
2315
2316static void __exit kcm_exit(void)
2317{
cd6e111b 2318 kcm_proc_exit();
ab7ac4eb
TH		 2319 unregister_pernet_device(&kcm_net_ops);
2320 sock_unregister(PF_KCM);
2321 proto_unregister(&kcm_proto);
2322 destroy_workqueue(kcm_wq);
2323
2324 kmem_cache_destroy(kcm_muxp);
2325 kmem_cache_destroy(kcm_psockp);
2326}
2327
2328module_init(kcm_init);
2329module_exit(kcm_exit);
2330
2331MODULE_LICENSE("GPL");
2332MODULE_ALIAS_NETPROTO(PF_KCM);
2333
Linux kernel - Deliverables
This page took 0.113258 seconds and 5 git commands to generate.