projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: relax rcvbuf limits
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
76#include <asm/system.h>
77#include <asm/uaccess.h>
78#include <asm/ioctls.h>
79#include <asm/page.h>
a1f8e7f7 80#include <asm/cacheflush.h>
1da177e4
LT		 81#include <asm/io.h>
82#include <linux/proc_fs.h>
83#include <linux/seq_file.h>
84#include <linux/poll.h>
85#include <linux/module.h>
86#include <linux/init.h>
905db440 87#include <linux/mutex.h>
05423b24 88#include <linux/if_vlan.h>
bfd5f4a3 89#include <linux/virtio_net.h>
ed85b565 90#include <linux/errqueue.h>
614f60fa 91#include <linux/net_tstamp.h>
1da177e4
LT		 92
93#ifdef CONFIG_INET
94#include <net/inet_common.h>
95#endif
96
1da177e4
LT		 97/*
98 Assumptions:
99 - if device has no dev->hard_header routine, it adds and removes ll header
100 inside itself. In this case ll header is invisible outside of device,
101 but higher levels still should reserve dev->hard_header_len.
102 Some devices are enough clever to reallocate skb, when header
103 will not fit to reserved space (tunnel), another ones are silly
104 (PPP).
105 - packet socket receives packets with pulled ll header,
106 so that SOCK_RAW should push it back.
107
108On receive:
109-----------
110
111Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 112 mac_header -> ll header
113 data -> data
1da177e4
LT		 114
115Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 116 mac_header -> ll header
117 data -> ll header
1da177e4
LT		 118
119Incoming, dev->hard_header==NULL
b0e380b1
ACM		 120 mac_header -> UNKNOWN position. It is very likely, that it points to ll
121 header. PPP makes it, that is wrong, because introduce
db0c58f9 122 assymetry between rx and tx paths.
b0e380b1 123 data -> data
1da177e4
LT		 124
125Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 126 mac_header -> data. ll header is still not built!
127 data -> data
1da177e4
LT		 128
129Resume
130 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
131
132
133On transmit:
134------------
135
136dev->hard_header != NULL
b0e380b1
ACM		 137 mac_header -> ll header
138 data -> ll header
1da177e4
LT		 139
140dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 141 mac_header -> data
142 data -> data
1da177e4
LT		 143
144 We should set nh.raw on output to correct posistion,
145 packet classifier depends on it.
146 */
147
1da177e4
LT		 148/* Private packet socket structures. */
149
40d4e3df 150struct packet_mclist {
1da177e4
LT		 151 struct packet_mclist *next;
152 int ifindex;
153 int count;
154 unsigned short type;
155 unsigned short alen;
0fb375fb
EB		 156 unsigned char addr[MAX_ADDR_LEN];
157};
158/* identical to struct packet_mreq except it has
159 * a longer address field.
160 */
40d4e3df 161struct packet_mreq_max {
0fb375fb
EB		 162 int mr_ifindex;
163 unsigned short mr_type;
164 unsigned short mr_alen;
165 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 166};
a2efcfa0 167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171
172#define V3_ALIGNMENT (8)
173
bc59ba39 174#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 175
176#define BLK_PLUS_PRIV(sz_of_priv) \
177 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
178
179/* kbdq - kernel block descriptor queue */
bc59ba39 180struct tpacket_kbdq_core {
f6fb8f10 181 struct pgv *pkbdq;
182 unsigned int feature_req_word;
183 unsigned int hdrlen;
184 unsigned char reset_pending_on_curr_blk;
185 unsigned char delete_blk_timer;
186 unsigned short kactive_blk_num;
187 unsigned short blk_sizeof_priv;
188
189 /* last_kactive_blk_num:
190 * trick to see if user-space has caught up
191 * in order to avoid refreshing timer when every single pkt arrives.
192 */
193 unsigned short last_kactive_blk_num;
194
195 char *pkblk_start;
196 char *pkblk_end;
197 int kblk_size;
198 unsigned int knum_blocks;
199 uint64_t knxt_seq_num;
200 char *prev;
201 char *nxt_offset;
202 struct sk_buff *skb;
203
204 atomic_t blk_fill_in_prog;
205
206 /* Default is set to 8ms */
207#define DEFAULT_PRB_RETIRE_TOV (8)
208
209 unsigned short retire_blk_tov;
210 unsigned short version;
211 unsigned long tov_in_jiffies;
212
213 /* timer to retire an outstanding block */
214 struct timer_list retire_blk_timer;
215};
216
217#define PGV_FROM_VMALLOC 1
0e3125c7
NH		 218struct pgv {
219 char *buffer;
0e3125c7
NH		 220};
221
69e3c75f 222struct packet_ring_buffer {
0e3125c7 223 struct pgv *pg_vec;
69e3c75f
JB		 224 unsigned int head;
225 unsigned int frames_per_block;
226 unsigned int frame_size;
227 unsigned int frame_max;
228
229 unsigned int pg_vec_order;
230 unsigned int pg_vec_pages;
231 unsigned int pg_vec_len;
232
bc59ba39 233 struct tpacket_kbdq_core prb_bdqc;
69e3c75f
JB		 234 atomic_t pending;
235};
236
f6fb8f10 237#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
238#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
239#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
240#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
241#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
242#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
243#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
244
69e3c75f
JB		 245struct packet_sock;
246static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
1da177e4 247
f6fb8f10 248static void *packet_previous_frame(struct packet_sock *po,
249 struct packet_ring_buffer *rb,
250 int status);
251static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 252static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
253 struct tpacket_block_desc *);
254static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 255 struct packet_sock *);
bc59ba39 256static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 257 struct packet_sock *, unsigned int status);
bc59ba39 258static int prb_queue_frozen(struct tpacket_kbdq_core *);
259static void prb_open_block(struct tpacket_kbdq_core *,
260 struct tpacket_block_desc *);
f6fb8f10 261static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 262static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
263static void prb_init_blk_timer(struct packet_sock *,
264 struct tpacket_kbdq_core *,
265 void (*func) (unsigned long));
266static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
267static void prb_clear_rxhash(struct tpacket_kbdq_core *,
268 struct tpacket3_hdr *);
269static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
270 struct tpacket3_hdr *);
1da177e4
LT		 271static void packet_flush_mclist(struct sock *sk);
272
dc99f600 273struct packet_fanout;
1da177e4
LT		 274struct packet_sock {
275 /* struct sock has to be the first member of packet_sock */
276 struct sock sk;
dc99f600 277 struct packet_fanout *fanout;
1da177e4 278 struct tpacket_stats stats;
f6fb8f10 279 union tpacket_stats_u stats_u;
69e3c75f
JB		 280 struct packet_ring_buffer rx_ring;
281 struct packet_ring_buffer tx_ring;
1da177e4 282 int copy_thresh;
1da177e4 283 spinlock_t bind_lock;
905db440 284 struct mutex pg_vec_lock;
8dc41944 285 unsigned int running:1, /* prot_hook is attached*/
80feaacb 286 auxdata:1,
bfd5f4a3
SS		 287 origdev:1,
288 has_vnet_hdr:1;
1da177e4 289 int ifindex; /* bound device */
0e11c91e 290 __be16 num;
1da177e4 291 struct packet_mclist *mclist;
1da177e4 292 atomic_t mapped;
bbd6ef87
PM		 293 enum tpacket_versions tp_version;
294 unsigned int tp_hdrlen;
8913336a 295 unsigned int tp_reserve;
69e3c75f 296 unsigned int tp_loss:1;
614f60fa 297 unsigned int tp_tstamp;
94b05952 298 struct packet_type prot_hook ____cacheline_aligned_in_smp;
1da177e4
LT		 299};
300
dc99f600
DM		 301#define PACKET_FANOUT_MAX 256
302
303struct packet_fanout {
304#ifdef CONFIG_NET_NS
305 struct net *net;
306#endif
307 unsigned int num_members;
308 u16 id;
309 u8 type;
7736d33f 310 u8 defrag;
dc99f600
DM		 311 atomic_t rr_cur;
312 struct list_head list;
313 struct sock *arr[PACKET_FANOUT_MAX];
314 spinlock_t lock;
315 atomic_t sk_ref;
316 struct packet_type prot_hook ____cacheline_aligned_in_smp;
317};
318
ffbc6111
HX		 319struct packet_skb_cb {
320 unsigned int origlen;
321 union {
322 struct sockaddr_pkt pkt;
323 struct sockaddr_ll ll;
324 } sa;
325};
326
327#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 328
bc59ba39 329#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 330#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 331 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 332#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 333 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 334#define GET_NEXT_PRB_BLK_NUM(x) \
335 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
336 ((x)->kactive_blk_num+1) : 0)
337
eea49cc9 338static struct packet_sock *pkt_sk(struct sock *sk)
ce06b03e
DM		 339{
340 return (struct packet_sock *)sk;
341}
342
dc99f600
DM		 343static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
344static void __fanout_link(struct sock *sk, struct packet_sock *po);
345
ce06b03e
DM		 346/* register_prot_hook must be invoked with the po->bind_lock held,
347 * or from a context in which asynchronous accesses to the packet
348 * socket is not possible (packet_create()).
349 */
350static void register_prot_hook(struct sock *sk)
351{
352 struct packet_sock *po = pkt_sk(sk);
353 if (!po->running) {
dc99f600
DM		 354 if (po->fanout)
355 __fanout_link(sk, po);
356 else
357 dev_add_pack(&po->prot_hook);
ce06b03e
DM		 358 sock_hold(sk);
359 po->running = 1;
360 }
361}
362
363/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
364 * held. If the sync parameter is true, we will temporarily drop
365 * the po->bind_lock and do a synchronize_net to make sure no
366 * asynchronous packet processing paths still refer to the elements
367 * of po->prot_hook. If the sync parameter is false, it is the
368 * callers responsibility to take care of this.
369 */
370static void __unregister_prot_hook(struct sock *sk, bool sync)
371{
372 struct packet_sock *po = pkt_sk(sk);
373
374 po->running = 0;
dc99f600
DM		 375 if (po->fanout)
376 __fanout_unlink(sk, po);
377 else
378 __dev_remove_pack(&po->prot_hook);
ce06b03e
DM		 379 __sock_put(sk);
380
381 if (sync) {
382 spin_unlock(&po->bind_lock);
383 synchronize_net();
384 spin_lock(&po->bind_lock);
385 }
386}
387
388static void unregister_prot_hook(struct sock *sk, bool sync)
389{
390 struct packet_sock *po = pkt_sk(sk);
391
392 if (po->running)
393 __unregister_prot_hook(sk, sync);
394}
395
f6dafa95 396static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 397{
398 if (is_vmalloc_addr(addr))
399 return vmalloc_to_page(addr);
400 return virt_to_page(addr);
401}
402
69e3c75f 403static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 404{
bbd6ef87
PM		 405 union {
406 struct tpacket_hdr *h1;
407 struct tpacket2_hdr *h2;
408 void *raw;
409 } h;
1da177e4 410
69e3c75f 411 h.raw = frame;
bbd6ef87
PM		 412 switch (po->tp_version) {
413 case TPACKET_V1:
69e3c75f 414 h.h1->tp_status = status;
0af55bb5 415 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 416 break;
417 case TPACKET_V2:
69e3c75f 418 h.h2->tp_status = status;
0af55bb5 419 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 420 break;
f6fb8f10 421 case TPACKET_V3:
69e3c75f 422 default:
f6fb8f10 423 WARN(1, "TPACKET version not supported.\n");
69e3c75f 424 BUG();
bbd6ef87 425 }
69e3c75f
JB		 426
427 smp_wmb();
bbd6ef87
PM		 428}
429
69e3c75f 430static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87
PM		 431{
432 union {
433 struct tpacket_hdr *h1;
434 struct tpacket2_hdr *h2;
435 void *raw;
436 } h;
437
69e3c75f
JB		 438 smp_rmb();
439
bbd6ef87
PM		 440 h.raw = frame;
441 switch (po->tp_version) {
442 case TPACKET_V1:
0af55bb5 443 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 444 return h.h1->tp_status;
bbd6ef87 445 case TPACKET_V2:
0af55bb5 446 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 447 return h.h2->tp_status;
f6fb8f10 448 case TPACKET_V3:
69e3c75f 449 default:
f6fb8f10 450 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 451 BUG();
452 return 0;
bbd6ef87 453 }
1da177e4 454}
69e3c75f
JB		 455
456static void *packet_lookup_frame(struct packet_sock *po,
457 struct packet_ring_buffer *rb,
458 unsigned int position,
459 int status)
460{
461 unsigned int pg_vec_pos, frame_offset;
462 union {
463 struct tpacket_hdr *h1;
464 struct tpacket2_hdr *h2;
465 void *raw;
466 } h;
467
468 pg_vec_pos = position / rb->frames_per_block;
469 frame_offset = position % rb->frames_per_block;
470
0e3125c7
NH		 471 h.raw = rb->pg_vec[pg_vec_pos].buffer +
472 (frame_offset * rb->frame_size);
69e3c75f
JB		 473
474 if (status != __packet_get_status(po, h.raw))
475 return NULL;
476
477 return h.raw;
478}
479
eea49cc9 480static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 481 struct packet_ring_buffer *rb,
482 int status)
483{
484 return packet_lookup_frame(po, rb, rb->head, status);
485}
486
bc59ba39 487static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 488{
489 del_timer_sync(&pkc->retire_blk_timer);
490}
491
492static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
493 int tx_ring,
494 struct sk_buff_head *rb_queue)
495{
bc59ba39 496 struct tpacket_kbdq_core *pkc;
f6fb8f10 497
498 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
499
500 spin_lock(&rb_queue->lock);
501 pkc->delete_blk_timer = 1;
502 spin_unlock(&rb_queue->lock);
503
504 prb_del_retire_blk_timer(pkc);
505}
506
507static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 508 struct tpacket_kbdq_core *pkc,
f6fb8f10 509 void (*func) (unsigned long))
510{
511 init_timer(&pkc->retire_blk_timer);
512 pkc->retire_blk_timer.data = (long)po;
513 pkc->retire_blk_timer.function = func;
514 pkc->retire_blk_timer.expires = jiffies;
515}
516
517static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
518{
bc59ba39 519 struct tpacket_kbdq_core *pkc;
f6fb8f10 520
521 if (tx_ring)
522 BUG();
523
524 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
525 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
526}
527
528static int prb_calc_retire_blk_tmo(struct packet_sock *po,
529 int blk_size_in_bytes)
530{
531 struct net_device *dev;
532 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 533 struct ethtool_cmd ecmd;
534 int err;
f6fb8f10 535
4bc71cb9
JP		 536 rtnl_lock();
537 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
538 if (unlikely(!dev)) {
539 rtnl_unlock();
f6fb8f10 540 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 541 }
542 err = __ethtool_get_settings(dev, &ecmd);
543 rtnl_unlock();
544 if (!err) {
545 switch (ecmd.speed) {
546 case SPEED_10000:
547 msec = 1;
548 div = 10000/1000;
549 break;
550 case SPEED_1000:
551 msec = 1;
552 div = 1000/1000;
553 break;
554 /*
555 * If the link speed is so slow you don't really
556 * need to worry about perf anyways
557 */
558 case SPEED_100:
559 case SPEED_10:
560 default:
561 return DEFAULT_PRB_RETIRE_TOV;
f6fb8f10 562 }
563 }
564
565 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
566
567 if (div)
568 mbits /= div;
569
570 tmo = mbits * msec;
571
572 if (div)
573 return tmo+1;
574 return tmo;
575}
576
bc59ba39 577static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 578 union tpacket_req_u *req_u)
579{
580 p1->feature_req_word = req_u->req3.tp_feature_req_word;
581}
582
583static void init_prb_bdqc(struct packet_sock *po,
584 struct packet_ring_buffer *rb,
585 struct pgv *pg_vec,
586 union tpacket_req_u *req_u, int tx_ring)
587{
bc59ba39 588 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
589 struct tpacket_block_desc *pbd;
f6fb8f10 590
591 memset(p1, 0x0, sizeof(*p1));
592
593 p1->knxt_seq_num = 1;
594 p1->pkbdq = pg_vec;
bc59ba39 595 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
f6fb8f10 596 p1->pkblk_start = (char *)pg_vec[0].buffer;
597 p1->kblk_size = req_u->req3.tp_block_size;
598 p1->knum_blocks = req_u->req3.tp_block_nr;
599 p1->hdrlen = po->tp_hdrlen;
600 p1->version = po->tp_version;
601 p1->last_kactive_blk_num = 0;
602 po->stats_u.stats3.tp_freeze_q_cnt = 0;
603 if (req_u->req3.tp_retire_blk_tov)
604 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
605 else
606 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
607 req_u->req3.tp_block_size);
608 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
609 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
610
611 prb_init_ft_ops(p1, req_u);
612 prb_setup_retire_blk_timer(po, tx_ring);
613 prb_open_block(p1, pbd);
614}
615
616/* Do NOT update the last_blk_num first.
617 * Assumes sk_buff_head lock is held.
618 */
bc59ba39 619static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 620{
621 mod_timer(&pkc->retire_blk_timer,
622 jiffies + pkc->tov_in_jiffies);
623 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
624}
625
626/*
627 * Timer logic:
628 * 1) We refresh the timer only when we open a block.
629 * By doing this we don't waste cycles refreshing the timer
630 * on packet-by-packet basis.
631 *
632 * With a 1MB block-size, on a 1Gbps line, it will take
633 * i) ~8 ms to fill a block + ii) memcpy etc.
634 * In this cut we are not accounting for the memcpy time.
635 *
636 * So, if the user sets the 'tmo' to 10ms then the timer
637 * will never fire while the block is still getting filled
638 * (which is what we want). However, the user could choose
639 * to close a block early and that's fine.
640 *
641 * But when the timer does fire, we check whether or not to refresh it.
642 * Since the tmo granularity is in msecs, it is not too expensive
643 * to refresh the timer, lets say every '8' msecs.
644 * Either the user can set the 'tmo' or we can derive it based on
645 * a) line-speed and b) block-size.
646 * prb_calc_retire_blk_tmo() calculates the tmo.
647 *
648 */
649static void prb_retire_rx_blk_timer_expired(unsigned long data)
650{
651 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 652 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 653 unsigned int frozen;
bc59ba39 654 struct tpacket_block_desc *pbd;
f6fb8f10 655
656 spin_lock(&po->sk.sk_receive_queue.lock);
657
658 frozen = prb_queue_frozen(pkc);
659 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
660
661 if (unlikely(pkc->delete_blk_timer))
662 goto out;
663
664 /* We only need to plug the race when the block is partially filled.
665 * tpacket_rcv:
666 * lock(); increment BLOCK_NUM_PKTS; unlock()
667 * copy_bits() is in progress ...
668 * timer fires on other cpu:
669 * we can't retire the current block because copy_bits
670 * is in progress.
671 *
672 */
673 if (BLOCK_NUM_PKTS(pbd)) {
674 while (atomic_read(&pkc->blk_fill_in_prog)) {
675 /* Waiting for skb_copy_bits to finish... */
676 cpu_relax();
677 }
678 }
679
680 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
681 if (!frozen) {
682 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
683 if (!prb_dispatch_next_block(pkc, po))
684 goto refresh_timer;
685 else
686 goto out;
687 } else {
688 /* Case 1. Queue was frozen because user-space was
689 * lagging behind.
690 */
691 if (prb_curr_blk_in_use(pkc, pbd)) {
692 /*
693 * Ok, user-space is still behind.
694 * So just refresh the timer.
695 */
696 goto refresh_timer;
697 } else {
698 /* Case 2. queue was frozen,user-space caught up,
699 * now the link went idle && the timer fired.
700 * We don't have a block to close.So we open this
701 * block and restart the timer.
702 * opening a block thaws the queue,restarts timer
703 * Thawing/timer-refresh is a side effect.
704 */
705 prb_open_block(pkc, pbd);
706 goto out;
707 }
708 }
709 }
710
711refresh_timer:
712 _prb_refresh_rx_retire_blk_timer(pkc);
713
714out:
715 spin_unlock(&po->sk.sk_receive_queue.lock);
716}
717
eea49cc9 718static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 719 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 720{
721 /* Flush everything minus the block header */
722
723#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
724 u8 *start, *end;
725
726 start = (u8 *)pbd1;
727
728 /* Skip the block header(we know header WILL fit in 4K) */
729 start += PAGE_SIZE;
730
731 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
732 for (; start < end; start += PAGE_SIZE)
733 flush_dcache_page(pgv_to_page(start));
734
735 smp_wmb();
736#endif
737
738 /* Now update the block status. */
739
740 BLOCK_STATUS(pbd1) = status;
741
742 /* Flush the block header */
743
744#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
745 start = (u8 *)pbd1;
746 flush_dcache_page(pgv_to_page(start));
747
748 smp_wmb();
749#endif
750}
751
752/*
753 * Side effect:
754 *
755 * 1) flush the block
756 * 2) Increment active_blk_num
757 *
758 * Note:We DONT refresh the timer on purpose.
759 * Because almost always the next block will be opened.
760 */
bc59ba39 761static void prb_close_block(struct tpacket_kbdq_core *pkc1,
762 struct tpacket_block_desc *pbd1,
f6fb8f10 763 struct packet_sock *po, unsigned int stat)
764{
765 __u32 status = TP_STATUS_USER | stat;
766
767 struct tpacket3_hdr *last_pkt;
bc59ba39 768 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 769
770 if (po->stats.tp_drops)
771 status |= TP_STATUS_LOSING;
772
773 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
774 last_pkt->tp_next_offset = 0;
775
776 /* Get the ts of the last pkt */
777 if (BLOCK_NUM_PKTS(pbd1)) {
778 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
779 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
780 } else {
781 /* Ok, we tmo'd - so get the current time */
782 struct timespec ts;
783 getnstimeofday(&ts);
784 h1->ts_last_pkt.ts_sec = ts.tv_sec;
785 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
786 }
787
788 smp_wmb();
789
790 /* Flush the block */
791 prb_flush_block(pkc1, pbd1, status);
792
793 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
794}
795
eea49cc9 796static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 797{
798 pkc->reset_pending_on_curr_blk = 0;
799}
800
801/*
802 * Side effect of opening a block:
803 *
804 * 1) prb_queue is thawed.
805 * 2) retire_blk_timer is refreshed.
806 *
807 */
bc59ba39 808static void prb_open_block(struct tpacket_kbdq_core *pkc1,
809 struct tpacket_block_desc *pbd1)
f6fb8f10 810{
811 struct timespec ts;
bc59ba39 812 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 813
814 smp_rmb();
815
816 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd1))) {
817
818 /* We could have just memset this but we will lose the
819 * flexibility of making the priv area sticky
820 */
821 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
822 BLOCK_NUM_PKTS(pbd1) = 0;
823 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
824 getnstimeofday(&ts);
825 h1->ts_first_pkt.ts_sec = ts.tv_sec;
826 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
827 pkc1->pkblk_start = (char *)pbd1;
828 pkc1->nxt_offset = (char *)(pkc1->pkblk_start +
829 BLK_PLUS_PRIV(pkc1->blk_sizeof_priv));
830 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
831 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
832 pbd1->version = pkc1->version;
833 pkc1->prev = pkc1->nxt_offset;
834 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
835 prb_thaw_queue(pkc1);
836 _prb_refresh_rx_retire_blk_timer(pkc1);
837
838 smp_wmb();
839
840 return;
841 }
842
843 WARN(1, "ERROR block:%p is NOT FREE status:%d kactive_blk_num:%d\n",
844 pbd1, BLOCK_STATUS(pbd1), pkc1->kactive_blk_num);
845 dump_stack();
846 BUG();
847}
848
849/*
850 * Queue freeze logic:
851 * 1) Assume tp_block_nr = 8 blocks.
852 * 2) At time 't0', user opens Rx ring.
853 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
854 * 4) user-space is either sleeping or processing block '0'.
855 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
856 * it will close block-7,loop around and try to fill block '0'.
857 * call-flow:
858 * __packet_lookup_frame_in_block
859 * prb_retire_current_block()
860 * prb_dispatch_next_block()
861 * |->(BLOCK_STATUS == USER) evaluates to true
862 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
863 * 6) Now there are two cases:
864 * 6.1) Link goes idle right after the queue is frozen.
865 * But remember, the last open_block() refreshed the timer.
866 * When this timer expires,it will refresh itself so that we can
867 * re-open block-0 in near future.
868 * 6.2) Link is busy and keeps on receiving packets. This is a simple
869 * case and __packet_lookup_frame_in_block will check if block-0
870 * is free and can now be re-used.
871 */
eea49cc9 872static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 873 struct packet_sock *po)
874{
875 pkc->reset_pending_on_curr_blk = 1;
876 po->stats_u.stats3.tp_freeze_q_cnt++;
877}
878
879#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
880
881/*
882 * If the next block is free then we will dispatch it
883 * and return a good offset.
884 * Else, we will freeze the queue.
885 * So, caller must check the return value.
886 */
bc59ba39 887static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 888 struct packet_sock *po)
889{
bc59ba39 890 struct tpacket_block_desc *pbd;
f6fb8f10 891
892 smp_rmb();
893
894 /* 1. Get current block num */
895 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
896
897 /* 2. If this block is currently in_use then freeze the queue */
898 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
899 prb_freeze_queue(pkc, po);
900 return NULL;
901 }
902
903 /*
904 * 3.
905 * open this block and return the offset where the first packet
906 * needs to get stored.
907 */
908 prb_open_block(pkc, pbd);
909 return (void *)pkc->nxt_offset;
910}
911
bc59ba39 912static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 913 struct packet_sock *po, unsigned int status)
914{
bc59ba39 915 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 916
917 /* retire/close the current block */
918 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
919 /*
920 * Plug the case where copy_bits() is in progress on
921 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
922 * have space to copy the pkt in the current block and
923 * called prb_retire_current_block()
924 *
925 * We don't need to worry about the TMO case because
926 * the timer-handler already handled this case.
927 */
928 if (!(status & TP_STATUS_BLK_TMO)) {
929 while (atomic_read(&pkc->blk_fill_in_prog)) {
930 /* Waiting for skb_copy_bits to finish... */
931 cpu_relax();
932 }
933 }
934 prb_close_block(pkc, pbd, po, status);
935 return;
936 }
937
938 WARN(1, "ERROR-pbd[%d]:%p\n", pkc->kactive_blk_num, pbd);
939 dump_stack();
940 BUG();
941}
942
eea49cc9 943static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 944 struct tpacket_block_desc *pbd)
f6fb8f10 945{
946 return TP_STATUS_USER & BLOCK_STATUS(pbd);
947}
948
eea49cc9 949static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 950{
951 return pkc->reset_pending_on_curr_blk;
952}
953
eea49cc9 954static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 955{
bc59ba39 956 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 957 atomic_dec(&pkc->blk_fill_in_prog);
958}
959
eea49cc9 960static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 961 struct tpacket3_hdr *ppd)
962{
963 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
964}
965
eea49cc9 966static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 967 struct tpacket3_hdr *ppd)
968{
969 ppd->hv1.tp_rxhash = 0;
970}
971
eea49cc9 972static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 973 struct tpacket3_hdr *ppd)
974{
975 if (vlan_tx_tag_present(pkc->skb)) {
976 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
977 ppd->tp_status = TP_STATUS_VLAN_VALID;
978 } else {
979 ppd->hv1.tp_vlan_tci = ppd->tp_status = 0;
980 }
981}
982
bc59ba39 983static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 984 struct tpacket3_hdr *ppd)
985{
986 prb_fill_vlan_info(pkc, ppd);
987
988 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
989 prb_fill_rxhash(pkc, ppd);
990 else
991 prb_clear_rxhash(pkc, ppd);
992}
993
eea49cc9 994static void prb_fill_curr_block(char *curr,
bc59ba39 995 struct tpacket_kbdq_core *pkc,
996 struct tpacket_block_desc *pbd,
f6fb8f10 997 unsigned int len)
998{
999 struct tpacket3_hdr *ppd;
1000
1001 ppd = (struct tpacket3_hdr *)curr;
1002 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1003 pkc->prev = curr;
1004 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1005 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1006 BLOCK_NUM_PKTS(pbd) += 1;
1007 atomic_inc(&pkc->blk_fill_in_prog);
1008 prb_run_all_ft_ops(pkc, ppd);
1009}
1010
1011/* Assumes caller has the sk->rx_queue.lock */
1012static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1013 struct sk_buff *skb,
1014 int status,
1015 unsigned int len
1016 )
1017{
bc59ba39 1018 struct tpacket_kbdq_core *pkc;
1019 struct tpacket_block_desc *pbd;
f6fb8f10 1020 char *curr, *end;
1021
1022 pkc = GET_PBDQC_FROM_RB(((struct packet_ring_buffer *)&po->rx_ring));
1023 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1024
1025 /* Queue is frozen when user space is lagging behind */
1026 if (prb_queue_frozen(pkc)) {
1027 /*
1028 * Check if that last block which caused the queue to freeze,
1029 * is still in_use by user-space.
1030 */
1031 if (prb_curr_blk_in_use(pkc, pbd)) {
1032 /* Can't record this packet */
1033 return NULL;
1034 } else {
1035 /*
1036 * Ok, the block was released by user-space.
1037 * Now let's open that block.
1038 * opening a block also thaws the queue.
1039 * Thawing is a side effect.
1040 */
1041 prb_open_block(pkc, pbd);
1042 }
1043 }
1044
1045 smp_mb();
1046 curr = pkc->nxt_offset;
1047 pkc->skb = skb;
1048 end = (char *) ((char *)pbd + pkc->kblk_size);
1049
1050 /* first try the current block */
1051 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1052 prb_fill_curr_block(curr, pkc, pbd, len);
1053 return (void *)curr;
1054 }
1055
1056 /* Ok, close the current block */
1057 prb_retire_current_block(pkc, po, 0);
1058
1059 /* Now, try to dispatch the next block */
1060 curr = (char *)prb_dispatch_next_block(pkc, po);
1061 if (curr) {
1062 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1063 prb_fill_curr_block(curr, pkc, pbd, len);
1064 return (void *)curr;
1065 }
1066
1067 /*
1068 * No free blocks are available.user_space hasn't caught up yet.
1069 * Queue was just frozen and now this packet will get dropped.
1070 */
1071 return NULL;
1072}
1073
eea49cc9 1074static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1075 struct sk_buff *skb,
1076 int status, unsigned int len)
1077{
1078 char *curr = NULL;
1079 switch (po->tp_version) {
1080 case TPACKET_V1:
1081 case TPACKET_V2:
1082 curr = packet_lookup_frame(po, &po->rx_ring,
1083 po->rx_ring.head, status);
1084 return curr;
1085 case TPACKET_V3:
1086 return __packet_lookup_frame_in_block(po, skb, status, len);
1087 default:
1088 WARN(1, "TPACKET version not supported\n");
1089 BUG();
1090 return 0;
1091 }
1092}
1093
eea49cc9 1094static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1095 struct packet_ring_buffer *rb,
1096 unsigned int previous,
1097 int status)
1098{
bc59ba39 1099 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
1100 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, previous);
f6fb8f10 1101
1102 if (status != BLOCK_STATUS(pbd))
1103 return NULL;
1104 return pbd;
1105}
1106
eea49cc9 1107static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1108{
1109 unsigned int prev;
1110 if (rb->prb_bdqc.kactive_blk_num)
1111 prev = rb->prb_bdqc.kactive_blk_num-1;
1112 else
1113 prev = rb->prb_bdqc.knum_blocks-1;
1114 return prev;
1115}
1116
1117/* Assumes caller has held the rx_queue.lock */
eea49cc9 1118static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1119 struct packet_ring_buffer *rb,
1120 int status)
1121{
1122 unsigned int previous = prb_previous_blk_num(rb);
1123 return prb_lookup_block(po, rb, previous, status);
1124}
1125
eea49cc9 1126static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1127 struct packet_ring_buffer *rb,
1128 int status)
1129{
1130 if (po->tp_version <= TPACKET_V2)
1131 return packet_previous_frame(po, rb, status);
1132
1133 return __prb_previous_block(po, rb, status);
1134}
1135
eea49cc9 1136static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1137 struct packet_ring_buffer *rb)
1138{
1139 switch (po->tp_version) {
1140 case TPACKET_V1:
1141 case TPACKET_V2:
1142 return packet_increment_head(rb);
1143 case TPACKET_V3:
1144 default:
1145 WARN(1, "TPACKET version not supported.\n");
1146 BUG();
1147 return;
1148 }
1149}
1150
eea49cc9 1151static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1152 struct packet_ring_buffer *rb,
1153 int status)
1154{
1155 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1156 return packet_lookup_frame(po, rb, previous, status);
1157}
1158
eea49cc9 1159static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1160{
1161 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1162}
1163
1da177e4
LT		 1164static void packet_sock_destruct(struct sock *sk)
1165{
ed85b565
RC		 1166 skb_queue_purge(&sk->sk_error_queue);
1167
547b792c
IJ		 1168 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1169 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1170
1171 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1172 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1173 return;
1174 }
1175
17ab56a2 1176 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1177}
1178
dc99f600
DM		 1179static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1180{
1181 int x = atomic_read(&f->rr_cur) + 1;
1182
1183 if (x >= num)
1184 x = 0;
1185
1186 return x;
1187}
1188
1189static struct sock *fanout_demux_hash(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
1190{
1191 u32 idx, hash = skb->rxhash;
1192
1193 idx = ((u64)hash * num) >> 32;
1194
1195 return f->arr[idx];
1196}
1197
1198static struct sock *fanout_demux_lb(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
1199{
1200 int cur, old;
1201
1202 cur = atomic_read(&f->rr_cur);
1203 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1204 fanout_rr_next(f, num))) != cur)
1205 cur = old;
1206 return f->arr[cur];
1207}
1208
95ec3eb4
DM		 1209static struct sock *fanout_demux_cpu(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
1210{
1211 unsigned int cpu = smp_processor_id();
1212
1213 return f->arr[cpu % num];
1214}
1215
95ec3eb4
DM		 1216static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1217 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1218{
1219 struct packet_fanout *f = pt->af_packet_priv;
1220 unsigned int num = f->num_members;
1221 struct packet_sock *po;
1222 struct sock *sk;
1223
1224 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1225 !num) {
1226 kfree_skb(skb);
1227 return 0;
1228 }
1229
95ec3eb4
DM		 1230 switch (f->type) {
1231 case PACKET_FANOUT_HASH:
1232 default:
1233 if (f->defrag) {
bc416d97 1234 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1235 if (!skb)
1236 return 0;
1237 }
1238 skb_get_rxhash(skb);
1239 sk = fanout_demux_hash(f, skb, num);
1240 break;
1241 case PACKET_FANOUT_LB:
1242 sk = fanout_demux_lb(f, skb, num);
1243 break;
1244 case PACKET_FANOUT_CPU:
1245 sk = fanout_demux_cpu(f, skb, num);
1246 break;
dc99f600
DM		 1247 }
1248
dc99f600
DM		 1249 po = pkt_sk(sk);
1250
1251 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1252}
1253
1254static DEFINE_MUTEX(fanout_mutex);
1255static LIST_HEAD(fanout_list);
1256
1257static void __fanout_link(struct sock *sk, struct packet_sock *po)
1258{
1259 struct packet_fanout *f = po->fanout;
1260
1261 spin_lock(&f->lock);
1262 f->arr[f->num_members] = sk;
1263 smp_wmb();
1264 f->num_members++;
1265 spin_unlock(&f->lock);
1266}
1267
1268static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1269{
1270 struct packet_fanout *f = po->fanout;
1271 int i;
1272
1273 spin_lock(&f->lock);
1274 for (i = 0; i < f->num_members; i++) {
1275 if (f->arr[i] == sk)
1276 break;
1277 }
1278 BUG_ON(i >= f->num_members);
1279 f->arr[i] = f->arr[f->num_members - 1];
1280 f->num_members--;
1281 spin_unlock(&f->lock);
1282}
1283
7736d33f 1284static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1285{
1286 struct packet_sock *po = pkt_sk(sk);
1287 struct packet_fanout *f, *match;
7736d33f
DM		 1288 u8 type = type_flags & 0xff;
1289 u8 defrag = (type_flags & PACKET_FANOUT_FLAG_DEFRAG) ? 1 : 0;
dc99f600
DM		 1290 int err;
1291
1292 switch (type) {
1293 case PACKET_FANOUT_HASH:
1294 case PACKET_FANOUT_LB:
95ec3eb4 1295 case PACKET_FANOUT_CPU:
dc99f600
DM		 1296 break;
1297 default:
1298 return -EINVAL;
1299 }
1300
1301 if (!po->running)
1302 return -EINVAL;
1303
1304 if (po->fanout)
1305 return -EALREADY;
1306
1307 mutex_lock(&fanout_mutex);
1308 match = NULL;
1309 list_for_each_entry(f, &fanout_list, list) {
1310 if (f->id == id &&
1311 read_pnet(&f->net) == sock_net(sk)) {
1312 match = f;
1313 break;
1314 }
1315 }
afe62c68 1316 err = -EINVAL;
7736d33f 1317 if (match && match->defrag != defrag)
afe62c68 1318 goto out;
dc99f600 1319 if (!match) {
afe62c68 1320 err = -ENOMEM;
dc99f600 1321 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1322 if (!match)
1323 goto out;
1324 write_pnet(&match->net, sock_net(sk));
1325 match->id = id;
1326 match->type = type;
1327 match->defrag = defrag;
1328 atomic_set(&match->rr_cur, 0);
1329 INIT_LIST_HEAD(&match->list);
1330 spin_lock_init(&match->lock);
1331 atomic_set(&match->sk_ref, 0);
1332 match->prot_hook.type = po->prot_hook.type;
1333 match->prot_hook.dev = po->prot_hook.dev;
1334 match->prot_hook.func = packet_rcv_fanout;
1335 match->prot_hook.af_packet_priv = match;
1336 dev_add_pack(&match->prot_hook);
1337 list_add(&match->list, &fanout_list);
dc99f600 1338 }
afe62c68
ED		 1339 err = -EINVAL;
1340 if (match->type == type &&
1341 match->prot_hook.type == po->prot_hook.type &&
1342 match->prot_hook.dev == po->prot_hook.dev) {
1343 err = -ENOSPC;
1344 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1345 __dev_remove_pack(&po->prot_hook);
1346 po->fanout = match;
1347 atomic_inc(&match->sk_ref);
1348 __fanout_link(sk, po);
1349 err = 0;
dc99f600
DM		 1350 }
1351 }
afe62c68 1352out:
dc99f600
DM		 1353 mutex_unlock(&fanout_mutex);
1354 return err;
1355}
1356
1357static void fanout_release(struct sock *sk)
1358{
1359 struct packet_sock *po = pkt_sk(sk);
1360 struct packet_fanout *f;
1361
1362 f = po->fanout;
1363 if (!f)
1364 return;
1365
1366 po->fanout = NULL;
1367
1368 mutex_lock(&fanout_mutex);
1369 if (atomic_dec_and_test(&f->sk_ref)) {
1370 list_del(&f->list);
1371 dev_remove_pack(&f->prot_hook);
1372 kfree(f);
1373 }
1374 mutex_unlock(&fanout_mutex);
1375}
1da177e4 1376
90ddc4f0 1377static const struct proto_ops packet_ops;
1da177e4 1378
90ddc4f0 1379static const struct proto_ops packet_ops_spkt;
1da177e4 1380
40d4e3df
ED		 1381static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1382 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1383{
1384 struct sock *sk;
1385 struct sockaddr_pkt *spkt;
1386
1387 /*
1388 * When we registered the protocol we saved the socket in the data
1389 * field for just this event.
1390 */
1391
1392 sk = pt->af_packet_priv;
1ce4f28b 1393
1da177e4
LT		 1394 /*
1395 * Yank back the headers [hope the device set this
1396 * right or kerboom...]
1397 *
1398 * Incoming packets have ll header pulled,
1399 * push it back.
1400 *
98e399f8 1401 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1402 * so that this procedure is noop.
1403 */
1404
1405 if (skb->pkt_type == PACKET_LOOPBACK)
1406 goto out;
1407
09ad9bc7 1408 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1409 goto out;
1410
40d4e3df
ED		 1411 skb = skb_share_check(skb, GFP_ATOMIC);
1412 if (skb == NULL)
1da177e4
LT		 1413 goto oom;
1414
1415 /* drop any routing info */
adf30907 1416 skb_dst_drop(skb);
1da177e4 1417
84531c24
PO		 1418 /* drop conntrack reference */
1419 nf_reset(skb);
1420
ffbc6111 1421 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1422
98e399f8 1423 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1424
1425 /*
1426 * The SOCK_PACKET socket receives _all_ frames.
1427 */
1428
1429 spkt->spkt_family = dev->type;
1430 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1431 spkt->spkt_protocol = skb->protocol;
1432
1433 /*
1434 * Charge the memory to the socket. This is done specifically
1435 * to prevent sockets using all the memory up.
1436 */
1437
40d4e3df 1438 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1439 return 0;
1440
1441out:
1442 kfree_skb(skb);
1443oom:
1444 return 0;
1445}
1446
1447
1448/*
1449 * Output a raw packet to a device layer. This bypasses all the other
1450 * protocol layers and you must therefore supply it with a complete frame
1451 */
1ce4f28b 1452
1da177e4
LT		 1453static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1454 struct msghdr *msg, size_t len)
1455{
1456 struct sock *sk = sock->sk;
40d4e3df 1457 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1458 struct sk_buff *skb = NULL;
1da177e4 1459 struct net_device *dev;
40d4e3df 1460 __be16 proto = 0;
1da177e4 1461 int err;
1ce4f28b 1462
1da177e4 1463 /*
1ce4f28b 1464 * Get and verify the address.
1da177e4
LT		 1465 */
1466
40d4e3df 1467 if (saddr) {
1da177e4 1468 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1469 return -EINVAL;
1470 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1471 proto = saddr->spkt_protocol;
1472 } else
1473 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1474
1475 /*
1ce4f28b 1476 * Find the device first to size check it
1da177e4
LT		 1477 */
1478
1479 saddr->spkt_device[13] = 0;
1a35ca80 1480retry:
654d1f8a
ED		 1481 rcu_read_lock();
1482 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1483 err = -ENODEV;
1484 if (dev == NULL)
1485 goto out_unlock;
1ce4f28b 1486
d5e76b0a
DM		 1487 err = -ENETDOWN;
1488 if (!(dev->flags & IFF_UP))
1489 goto out_unlock;
1490
1da177e4 1491 /*
40d4e3df
ED		 1492 * You may not queue a frame bigger than the mtu. This is the lowest level
1493 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1494 */
1ce4f28b 1495
1da177e4 1496 err = -EMSGSIZE;
57f89bfa 1497 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN)
1da177e4
LT		 1498 goto out_unlock;
1499
1a35ca80
ED		 1500 if (!skb) {
1501 size_t reserved = LL_RESERVED_SPACE(dev);
1502 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1503
1504 rcu_read_unlock();
1505 skb = sock_wmalloc(sk, len + reserved, 0, GFP_KERNEL);
1506 if (skb == NULL)
1507 return -ENOBUFS;
1508 /* FIXME: Save some space for broken drivers that write a hard
1509 * header at transmission time by themselves. PPP is the notable
1510 * one here. This should really be fixed at the driver level.
1511 */
1512 skb_reserve(skb, reserved);
1513 skb_reset_network_header(skb);
1514
1515 /* Try to align data part correctly */
1516 if (hhlen) {
1517 skb->data -= hhlen;
1518 skb->tail -= hhlen;
1519 if (len < hhlen)
1520 skb_reset_network_header(skb);
1521 }
1522 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1523 if (err)
1524 goto out_free;
1525 goto retry;
1da177e4
LT		 1526 }
1527
57f89bfa
BG		 1528 if (len > (dev->mtu + dev->hard_header_len)) {
1529 /* Earlier code assumed this would be a VLAN pkt,
1530 * double-check this now that we have the actual
1531 * packet in hand.
1532 */
1533 struct ethhdr *ehdr;
1534 skb_reset_mac_header(skb);
1535 ehdr = eth_hdr(skb);
1536 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1537 err = -EMSGSIZE;
1538 goto out_unlock;
1539 }
1540 }
1a35ca80 1541
1da177e4
LT		 1542 skb->protocol = proto;
1543 skb->dev = dev;
1544 skb->priority = sk->sk_priority;
2d37a186 1545 skb->mark = sk->sk_mark;
2244d07b 1546 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
ed85b565
RC		 1547 if (err < 0)
1548 goto out_unlock;
1da177e4
LT		 1549
1550 dev_queue_xmit(skb);
654d1f8a 1551 rcu_read_unlock();
40d4e3df 1552 return len;
1da177e4 1553
1da177e4 1554out_unlock:
654d1f8a 1555 rcu_read_unlock();
1a35ca80
ED		 1556out_free:
1557 kfree_skb(skb);
1da177e4
LT		 1558 return err;
1559}
1da177e4 1560
eea49cc9 1561static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1562 const struct sock *sk,
dbcb5855 1563 unsigned int res)
1da177e4
LT		 1564{
1565 struct sk_filter *filter;
fda9ef5d 1566
80f8f102
ED		 1567 rcu_read_lock();
1568 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1569 if (filter != NULL)
0a14842f 1570 res = SK_RUN_FILTER(filter, skb);
80f8f102 1571 rcu_read_unlock();
1da177e4 1572
dbcb5855 1573 return res;
1da177e4
LT		 1574}
1575
1576/*
62ab0812
ED		 1577 * This function makes lazy skb cloning in hope that most of packets
1578 * are discarded by BPF.
1579 *
1580 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1581 * and skb->cb are mangled. It works because (and until) packets
1582 * falling here are owned by current CPU. Output packets are cloned
1583 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1584 * sequencially, so that if we return skb to original state on exit,
1585 * we will not harm anyone.
1da177e4
LT		 1586 */
1587
40d4e3df
ED		 1588static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1589 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1590{
1591 struct sock *sk;
1592 struct sockaddr_ll *sll;
1593 struct packet_sock *po;
40d4e3df 1594 u8 *skb_head = skb->data;
1da177e4 1595 int skb_len = skb->len;
dbcb5855 1596 unsigned int snaplen, res;
1da177e4
LT		 1597
1598 if (skb->pkt_type == PACKET_LOOPBACK)
1599 goto drop;
1600
1601 sk = pt->af_packet_priv;
1602 po = pkt_sk(sk);
1603
09ad9bc7 1604 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1605 goto drop;
1606
1da177e4
LT		 1607 skb->dev = dev;
1608
3b04ddde 1609 if (dev->header_ops) {
1da177e4 1610 /* The device has an explicit notion of ll header,
62ab0812
ED		 1611 * exported to higher levels.
1612 *
1613 * Otherwise, the device hides details of its frame
1614 * structure, so that corresponding packet head is
1615 * never delivered to user.
1da177e4
LT		 1616 */
1617 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1618 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1619 else if (skb->pkt_type == PACKET_OUTGOING) {
1620 /* Special case: outgoing packets have ll header at head */
bbe735e4 1621 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1622 }
1623 }
1624
1625 snaplen = skb->len;
1626
dbcb5855
DM		 1627 res = run_filter(skb, sk, snaplen);
1628 if (!res)
fda9ef5d 1629 goto drop_n_restore;
dbcb5855
DM		 1630 if (snaplen > res)
1631 snaplen = res;
1da177e4 1632
0fd7bac6 1633 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1634 goto drop_n_acct;
1635
1636 if (skb_shared(skb)) {
1637 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1638 if (nskb == NULL)
1639 goto drop_n_acct;
1640
1641 if (skb_head != skb->data) {
1642 skb->data = skb_head;
1643 skb->len = skb_len;
1644 }
1645 kfree_skb(skb);
1646 skb = nskb;
1647 }
1648
ffbc6111
HX		 1649 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1650 sizeof(skb->cb));
1651
1652 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1653 sll->sll_family = AF_PACKET;
1654 sll->sll_hatype = dev->type;
1655 sll->sll_protocol = skb->protocol;
1656 sll->sll_pkttype = skb->pkt_type;
8032b464 1657 if (unlikely(po->origdev))
80feaacb
PWJ		 1658 sll->sll_ifindex = orig_dev->ifindex;
1659 else
1660 sll->sll_ifindex = dev->ifindex;
1da177e4 1661
b95cce35 1662 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1663
ffbc6111 1664 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1665
1da177e4
LT		 1666 if (pskb_trim(skb, snaplen))
1667 goto drop_n_acct;
1668
1669 skb_set_owner_r(skb, sk);
1670 skb->dev = NULL;
adf30907 1671 skb_dst_drop(skb);
1da177e4 1672
84531c24
PO		 1673 /* drop conntrack reference */
1674 nf_reset(skb);
1675
1da177e4
LT		 1676 spin_lock(&sk->sk_receive_queue.lock);
1677 po->stats.tp_packets++;
3b885787 1678 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1679 __skb_queue_tail(&sk->sk_receive_queue, skb);
1680 spin_unlock(&sk->sk_receive_queue.lock);
1681 sk->sk_data_ready(sk, skb->len);
1682 return 0;
1683
1684drop_n_acct:
7091fbd8
WB		 1685 spin_lock(&sk->sk_receive_queue.lock);
1686 po->stats.tp_drops++;
1687 atomic_inc(&sk->sk_drops);
1688 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1689
1690drop_n_restore:
1691 if (skb_head != skb->data && skb_shared(skb)) {
1692 skb->data = skb_head;
1693 skb->len = skb_len;
1694 }
1695drop:
ead2ceb0 1696 consume_skb(skb);
1da177e4
LT		 1697 return 0;
1698}
1699
40d4e3df
ED		 1700static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1701 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1702{
1703 struct sock *sk;
1704 struct packet_sock *po;
1705 struct sockaddr_ll *sll;
bbd6ef87
PM		 1706 union {
1707 struct tpacket_hdr *h1;
1708 struct tpacket2_hdr *h2;
f6fb8f10 1709 struct tpacket3_hdr *h3;
bbd6ef87
PM		 1710 void *raw;
1711 } h;
40d4e3df 1712 u8 *skb_head = skb->data;
1da177e4 1713 int skb_len = skb->len;
dbcb5855 1714 unsigned int snaplen, res;
f6fb8f10 1715 unsigned long status = TP_STATUS_USER;
bbd6ef87 1716 unsigned short macoff, netoff, hdrlen;
1da177e4 1717 struct sk_buff *copy_skb = NULL;
b7aa0bf7 1718 struct timeval tv;
bbd6ef87 1719 struct timespec ts;
614f60fa 1720 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
1da177e4
LT		 1721
1722 if (skb->pkt_type == PACKET_LOOPBACK)
1723 goto drop;
1724
1725 sk = pt->af_packet_priv;
1726 po = pkt_sk(sk);
1727
09ad9bc7 1728 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1729 goto drop;
1730
3b04ddde 1731 if (dev->header_ops) {
1da177e4 1732 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1733 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1734 else if (skb->pkt_type == PACKET_OUTGOING) {
1735 /* Special case: outgoing packets have ll header at head */
bbe735e4 1736 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1737 }
1738 }
1739
8dc41944
HX		 1740 if (skb->ip_summed == CHECKSUM_PARTIAL)
1741 status |= TP_STATUS_CSUMNOTREADY;
1742
1da177e4
LT		 1743 snaplen = skb->len;
1744
dbcb5855
DM		 1745 res = run_filter(skb, sk, snaplen);
1746 if (!res)
fda9ef5d 1747 goto drop_n_restore;
dbcb5855
DM		 1748 if (snaplen > res)
1749 snaplen = res;
1da177e4
LT		 1750
1751 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1752 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1753 po->tp_reserve;
1da177e4 1754 } else {
bbe735e4 1755 unsigned maclen = skb_network_offset(skb);
bbd6ef87 1756 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1757 (maclen < 16 ? 16 : maclen)) +
1758 po->tp_reserve;
1da177e4
LT		 1759 macoff = netoff - maclen;
1760 }
f6fb8f10 1761 if (po->tp_version <= TPACKET_V2) {
1762 if (macoff + snaplen > po->rx_ring.frame_size) {
1763 if (po->copy_thresh &&
0fd7bac6 1764 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1765 if (skb_shared(skb)) {
1766 copy_skb = skb_clone(skb, GFP_ATOMIC);
1767 } else {
1768 copy_skb = skb_get(skb);
1769 skb_head = skb->data;
1770 }
1771 if (copy_skb)
1772 skb_set_owner_r(copy_skb, sk);
1da177e4 1773 }
f6fb8f10 1774 snaplen = po->rx_ring.frame_size - macoff;
1775 if ((int)snaplen < 0)
1776 snaplen = 0;
1da177e4 1777 }
1da177e4 1778 }
1da177e4 1779 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1780 h.raw = packet_current_rx_frame(po, skb,
1781 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1782 if (!h.raw)
1da177e4 1783 goto ring_is_full;
f6fb8f10 1784 if (po->tp_version <= TPACKET_V2) {
1785 packet_increment_rx_head(po, &po->rx_ring);
1786 /*
1787 * LOSING will be reported till you read the stats,
1788 * because it's COR - Clear On Read.
1789 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1790 * at packet level.
1791 */
1792 if (po->stats.tp_drops)
1793 status |= TP_STATUS_LOSING;
1794 }
1da177e4
LT		 1795 po->stats.tp_packets++;
1796 if (copy_skb) {
1797 status |= TP_STATUS_COPY;
1798 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1799 }
1da177e4
LT		 1800 spin_unlock(&sk->sk_receive_queue.lock);
1801
bbd6ef87 1802 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
1da177e4 1803
bbd6ef87
PM		 1804 switch (po->tp_version) {
1805 case TPACKET_V1:
1806 h.h1->tp_len = skb->len;
1807 h.h1->tp_snaplen = snaplen;
1808 h.h1->tp_mac = macoff;
1809 h.h1->tp_net = netoff;
614f60fa
SM		 1810 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1811 && shhwtstamps->syststamp.tv64)
1812 tv = ktime_to_timeval(shhwtstamps->syststamp);
1813 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1814 && shhwtstamps->hwtstamp.tv64)
1815 tv = ktime_to_timeval(shhwtstamps->hwtstamp);
1816 else if (skb->tstamp.tv64)
bbd6ef87
PM		 1817 tv = ktime_to_timeval(skb->tstamp);
1818 else
1819 do_gettimeofday(&tv);
1820 h.h1->tp_sec = tv.tv_sec;
1821 h.h1->tp_usec = tv.tv_usec;
1822 hdrlen = sizeof(*h.h1);
1823 break;
1824 case TPACKET_V2:
1825 h.h2->tp_len = skb->len;
1826 h.h2->tp_snaplen = snaplen;
1827 h.h2->tp_mac = macoff;
1828 h.h2->tp_net = netoff;
614f60fa
SM		 1829 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1830 && shhwtstamps->syststamp.tv64)
1831 ts = ktime_to_timespec(shhwtstamps->syststamp);
1832 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1833 && shhwtstamps->hwtstamp.tv64)
1834 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
1835 else if (skb->tstamp.tv64)
bbd6ef87
PM		 1836 ts = ktime_to_timespec(skb->tstamp);
1837 else
1838 getnstimeofday(&ts);
1839 h.h2->tp_sec = ts.tv_sec;
1840 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1841 if (vlan_tx_tag_present(skb)) {
1842 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1843 status |= TP_STATUS_VLAN_VALID;
1844 } else {
1845 h.h2->tp_vlan_tci = 0;
1846 }
13fcb7bd 1847 h.h2->tp_padding = 0;
bbd6ef87
PM		 1848 hdrlen = sizeof(*h.h2);
1849 break;
f6fb8f10 1850 case TPACKET_V3:
1851 /* tp_nxt_offset,vlan are already populated above.
1852 * So DONT clear those fields here
1853 */
1854 h.h3->tp_status |= status;
1855 h.h3->tp_len = skb->len;
1856 h.h3->tp_snaplen = snaplen;
1857 h.h3->tp_mac = macoff;
1858 h.h3->tp_net = netoff;
1859 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1860 && shhwtstamps->syststamp.tv64)
1861 ts = ktime_to_timespec(shhwtstamps->syststamp);
1862 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1863 && shhwtstamps->hwtstamp.tv64)
1864 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
1865 else if (skb->tstamp.tv64)
1866 ts = ktime_to_timespec(skb->tstamp);
1867 else
1868 getnstimeofday(&ts);
1869 h.h3->tp_sec = ts.tv_sec;
1870 h.h3->tp_nsec = ts.tv_nsec;
1871 hdrlen = sizeof(*h.h3);
1872 break;
bbd6ef87
PM		 1873 default:
1874 BUG();
1875 }
1da177e4 1876
bbd6ef87 1877 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1878 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1879 sll->sll_family = AF_PACKET;
1880 sll->sll_hatype = dev->type;
1881 sll->sll_protocol = skb->protocol;
1882 sll->sll_pkttype = skb->pkt_type;
8032b464 1883 if (unlikely(po->origdev))
80feaacb
PWJ		 1884 sll->sll_ifindex = orig_dev->ifindex;
1885 else
1886 sll->sll_ifindex = dev->ifindex;
1da177e4 1887
e16aa207 1888 smp_mb();
f6dafa95 1889#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1890 {
0af55bb5
CG		 1891 u8 *start, *end;
1892
f6fb8f10 1893 if (po->tp_version <= TPACKET_V2) {
1894 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1895 + macoff + snaplen);
1896 for (start = h.raw; start < end; start += PAGE_SIZE)
1897 flush_dcache_page(pgv_to_page(start));
1898 }
cc9f01b2 1899 smp_wmb();
1da177e4 1900 }
f6dafa95 1901#endif
f6fb8f10 1902 if (po->tp_version <= TPACKET_V2)
1903 __packet_set_status(po, h.raw, status);
1904 else
1905 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1906
1907 sk->sk_data_ready(sk, 0);
1908
1909drop_n_restore:
1910 if (skb_head != skb->data && skb_shared(skb)) {
1911 skb->data = skb_head;
1912 skb->len = skb_len;
1913 }
1914drop:
1ce4f28b 1915 kfree_skb(skb);
1da177e4
LT		 1916 return 0;
1917
1918ring_is_full:
1919 po->stats.tp_drops++;
1920 spin_unlock(&sk->sk_receive_queue.lock);
1921
1922 sk->sk_data_ready(sk, 0);
acb5d75b 1923 kfree_skb(copy_skb);
1da177e4
LT		 1924 goto drop_n_restore;
1925}
1926
69e3c75f
JB		 1927static void tpacket_destruct_skb(struct sk_buff *skb)
1928{
1929 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1930 void *ph;
1da177e4 1931
69e3c75f
JB		 1932 if (likely(po->tx_ring.pg_vec)) {
1933 ph = skb_shinfo(skb)->destructor_arg;
1934 BUG_ON(__packet_get_status(po, ph) != TP_STATUS_SENDING);
1935 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1936 atomic_dec(&po->tx_ring.pending);
1937 __packet_set_status(po, ph, TP_STATUS_AVAILABLE);
1938 }
1939
1940 sock_wfree(skb);
1941}
1942
40d4e3df
ED		 1943static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1944 void *frame, struct net_device *dev, int size_max,
1945 __be16 proto, unsigned char *addr)
69e3c75f
JB		 1946{
1947 union {
1948 struct tpacket_hdr *h1;
1949 struct tpacket2_hdr *h2;
1950 void *raw;
1951 } ph;
1952 int to_write, offset, len, tp_len, nr_frags, len_max;
1953 struct socket *sock = po->sk.sk_socket;
1954 struct page *page;
1955 void *data;
1956 int err;
1957
1958 ph.raw = frame;
1959
1960 skb->protocol = proto;
1961 skb->dev = dev;
1962 skb->priority = po->sk.sk_priority;
2d37a186 1963 skb->mark = po->sk.sk_mark;
69e3c75f
JB		 1964 skb_shinfo(skb)->destructor_arg = ph.raw;
1965
1966 switch (po->tp_version) {
1967 case TPACKET_V2:
1968 tp_len = ph.h2->tp_len;
1969 break;
1970 default:
1971 tp_len = ph.h1->tp_len;
1972 break;
1973 }
1974 if (unlikely(tp_len > size_max)) {
40d4e3df 1975 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
69e3c75f
JB		 1976 return -EMSGSIZE;
1977 }
1978
1979 skb_reserve(skb, LL_RESERVED_SPACE(dev));
1980 skb_reset_network_header(skb);
1981
1982 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
1983 to_write = tp_len;
1984
1985 if (sock->type == SOCK_DGRAM) {
1986 err = dev_hard_header(skb, dev, ntohs(proto), addr,
1987 NULL, tp_len);
1988 if (unlikely(err < 0))
1989 return -EINVAL;
40d4e3df 1990 } else if (dev->hard_header_len) {
69e3c75f
JB		 1991 /* net device doesn't like empty head */
1992 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 1993 pr_err("packet size is too short (%d < %d)\n",
1994 tp_len, dev->hard_header_len);
69e3c75f
JB		 1995 return -EINVAL;
1996 }
1997
1998 skb_push(skb, dev->hard_header_len);
1999 err = skb_store_bits(skb, 0, data,
2000 dev->hard_header_len);
2001 if (unlikely(err))
2002 return err;
2003
2004 data += dev->hard_header_len;
2005 to_write -= dev->hard_header_len;
2006 }
2007
2008 err = -EFAULT;
69e3c75f
JB		 2009 offset = offset_in_page(data);
2010 len_max = PAGE_SIZE - offset;
2011 len = ((to_write > len_max) ? len_max : to_write);
2012
2013 skb->data_len = to_write;
2014 skb->len += to_write;
2015 skb->truesize += to_write;
2016 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2017
2018 while (likely(to_write)) {
2019 nr_frags = skb_shinfo(skb)->nr_frags;
2020
2021 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2022 pr_err("Packet exceed the number of skb frags(%lu)\n",
2023 MAX_SKB_FRAGS);
69e3c75f
JB		 2024 return -EFAULT;
2025 }
2026
0af55bb5
CG		 2027 page = pgv_to_page(data);
2028 data += len;
69e3c75f
JB		 2029 flush_dcache_page(page);
2030 get_page(page);
0af55bb5 2031 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2032 to_write -= len;
2033 offset = 0;
2034 len_max = PAGE_SIZE;
2035 len = ((to_write > len_max) ? len_max : to_write);
2036 }
2037
2038 return tp_len;
2039}
2040
2041static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2042{
69e3c75f
JB		 2043 struct sk_buff *skb;
2044 struct net_device *dev;
2045 __be16 proto;
827d9780
BG		 2046 bool need_rls_dev = false;
2047 int err, reserve = 0;
40d4e3df
ED		 2048 void *ph;
2049 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2050 int tp_len, size_max;
2051 unsigned char *addr;
2052 int len_sum = 0;
2053 int status = 0;
2054
69e3c75f
JB		 2055 mutex_lock(&po->pg_vec_lock);
2056
2057 err = -EBUSY;
2058 if (saddr == NULL) {
827d9780 2059 dev = po->prot_hook.dev;
69e3c75f
JB		 2060 proto = po->num;
2061 addr = NULL;
2062 } else {
2063 err = -EINVAL;
2064 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2065 goto out;
2066 if (msg->msg_namelen < (saddr->sll_halen
2067 + offsetof(struct sockaddr_ll,
2068 sll_addr)))
2069 goto out;
69e3c75f
JB		 2070 proto = saddr->sll_protocol;
2071 addr = saddr->sll_addr;
827d9780
BG		 2072 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2073 need_rls_dev = true;
69e3c75f
JB		 2074 }
2075
69e3c75f
JB		 2076 err = -ENXIO;
2077 if (unlikely(dev == NULL))
2078 goto out;
2079
2080 reserve = dev->hard_header_len;
2081
2082 err = -ENETDOWN;
2083 if (unlikely(!(dev->flags & IFF_UP)))
2084 goto out_put;
2085
2086 size_max = po->tx_ring.frame_size
b5dd884e 2087 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f
JB		 2088
2089 if (size_max > dev->mtu + reserve)
2090 size_max = dev->mtu + reserve;
2091
2092 do {
2093 ph = packet_current_frame(po, &po->tx_ring,
2094 TP_STATUS_SEND_REQUEST);
2095
2096 if (unlikely(ph == NULL)) {
2097 schedule();
2098 continue;
2099 }
2100
2101 status = TP_STATUS_SEND_REQUEST;
2102 skb = sock_alloc_send_skb(&po->sk,
2103 LL_ALLOCATED_SPACE(dev)
2104 + sizeof(struct sockaddr_ll),
2105 0, &err);
2106
2107 if (unlikely(skb == NULL))
2108 goto out_status;
2109
2110 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
2111 addr);
2112
2113 if (unlikely(tp_len < 0)) {
2114 if (po->tp_loss) {
2115 __packet_set_status(po, ph,
2116 TP_STATUS_AVAILABLE);
2117 packet_increment_head(&po->tx_ring);
2118 kfree_skb(skb);
2119 continue;
2120 } else {
2121 status = TP_STATUS_WRONG_FORMAT;
2122 err = tp_len;
2123 goto out_status;
2124 }
2125 }
2126
2127 skb->destructor = tpacket_destruct_skb;
2128 __packet_set_status(po, ph, TP_STATUS_SENDING);
2129 atomic_inc(&po->tx_ring.pending);
2130
2131 status = TP_STATUS_SEND_REQUEST;
2132 err = dev_queue_xmit(skb);
eb70df13
JP		 2133 if (unlikely(err > 0)) {
2134 err = net_xmit_errno(err);
2135 if (err && __packet_get_status(po, ph) ==
2136 TP_STATUS_AVAILABLE) {
2137 /* skb was destructed already */
2138 skb = NULL;
2139 goto out_status;
2140 }
2141 /*
2142 * skb was dropped but not destructed yet;
2143 * let's treat it like congestion or err < 0
2144 */
2145 err = 0;
2146 }
69e3c75f
JB		 2147 packet_increment_head(&po->tx_ring);
2148 len_sum += tp_len;
f64f9e71
JP		 2149 } while (likely((ph != NULL) ||
2150 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2151 (atomic_read(&po->tx_ring.pending))))
2152 );
69e3c75f
JB		 2153
2154 err = len_sum;
2155 goto out_put;
2156
69e3c75f
JB		 2157out_status:
2158 __packet_set_status(po, ph, status);
2159 kfree_skb(skb);
2160out_put:
827d9780
BG		 2161 if (need_rls_dev)
2162 dev_put(dev);
69e3c75f
JB		 2163out:
2164 mutex_unlock(&po->pg_vec_lock);
2165 return err;
2166}
69e3c75f 2167
eea49cc9
OJ		 2168static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2169 size_t reserve, size_t len,
2170 size_t linear, int noblock,
2171 int *err)
bfd5f4a3
SS		 2172{
2173 struct sk_buff *skb;
2174
2175 /* Under a page? Don't bother with paged skb. */
2176 if (prepad + len < PAGE_SIZE || !linear)
2177 linear = len;
2178
2179 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2180 err);
2181 if (!skb)
2182 return NULL;
2183
2184 skb_reserve(skb, reserve);
2185 skb_put(skb, linear);
2186 skb->data_len = len - linear;
2187 skb->len += len - linear;
2188
2189 return skb;
2190}
2191
69e3c75f 2192static int packet_snd(struct socket *sock,
1da177e4
LT		 2193 struct msghdr *msg, size_t len)
2194{
2195 struct sock *sk = sock->sk;
40d4e3df 2196 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2197 struct sk_buff *skb;
2198 struct net_device *dev;
0e11c91e 2199 __be16 proto;
827d9780 2200 bool need_rls_dev = false;
1da177e4 2201 unsigned char *addr;
827d9780 2202 int err, reserve = 0;
bfd5f4a3
SS		 2203 struct virtio_net_hdr vnet_hdr = { 0 };
2204 int offset = 0;
2205 int vnet_hdr_len;
2206 struct packet_sock *po = pkt_sk(sk);
2207 unsigned short gso_type = 0;
1da177e4
LT		 2208
2209 /*
1ce4f28b 2210 * Get and verify the address.
1da177e4 2211 */
1ce4f28b 2212
1da177e4 2213 if (saddr == NULL) {
827d9780 2214 dev = po->prot_hook.dev;
1da177e4
LT		 2215 proto = po->num;
2216 addr = NULL;
2217 } else {
2218 err = -EINVAL;
2219 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2220 goto out;
0fb375fb
EB		 2221 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2222 goto out;
1da177e4
LT		 2223 proto = saddr->sll_protocol;
2224 addr = saddr->sll_addr;
827d9780
BG		 2225 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2226 need_rls_dev = true;
1da177e4
LT		 2227 }
2228
1da177e4
LT		 2229 err = -ENXIO;
2230 if (dev == NULL)
2231 goto out_unlock;
2232 if (sock->type == SOCK_RAW)
2233 reserve = dev->hard_header_len;
2234
d5e76b0a
DM		 2235 err = -ENETDOWN;
2236 if (!(dev->flags & IFF_UP))
2237 goto out_unlock;
2238
bfd5f4a3
SS		 2239 if (po->has_vnet_hdr) {
2240 vnet_hdr_len = sizeof(vnet_hdr);
2241
2242 err = -EINVAL;
2243 if (len < vnet_hdr_len)
2244 goto out_unlock;
2245
2246 len -= vnet_hdr_len;
2247
2248 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2249 vnet_hdr_len);
2250 if (err < 0)
2251 goto out_unlock;
2252
2253 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2254 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2255 vnet_hdr.hdr_len))
2256 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2257 vnet_hdr.csum_offset + 2;
2258
2259 err = -EINVAL;
2260 if (vnet_hdr.hdr_len > len)
2261 goto out_unlock;
2262
2263 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2264 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2265 case VIRTIO_NET_HDR_GSO_TCPV4:
2266 gso_type = SKB_GSO_TCPV4;
2267 break;
2268 case VIRTIO_NET_HDR_GSO_TCPV6:
2269 gso_type = SKB_GSO_TCPV6;
2270 break;
2271 case VIRTIO_NET_HDR_GSO_UDP:
2272 gso_type = SKB_GSO_UDP;
2273 break;
2274 default:
2275 goto out_unlock;
2276 }
2277
2278 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2279 gso_type |= SKB_GSO_TCP_ECN;
2280
2281 if (vnet_hdr.gso_size == 0)
2282 goto out_unlock;
2283
2284 }
2285 }
2286
1da177e4 2287 err = -EMSGSIZE;
57f89bfa 2288 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN))
1da177e4
LT		 2289 goto out_unlock;
2290
bfd5f4a3
SS		 2291 err = -ENOBUFS;
2292 skb = packet_alloc_skb(sk, LL_ALLOCATED_SPACE(dev),
2293 LL_RESERVED_SPACE(dev), len, vnet_hdr.hdr_len,
2294 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2295 if (skb == NULL)
1da177e4
LT		 2296 goto out_unlock;
2297
bfd5f4a3 2298 skb_set_network_header(skb, reserve);
1da177e4 2299
0c4e8581
SH		 2300 err = -EINVAL;
2301 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2302 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2303 goto out_free;
1da177e4
LT		 2304
2305 /* Returns -EFAULT on error */
bfd5f4a3 2306 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2307 if (err)
2308 goto out_free;
2244d07b 2309 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
ed85b565
RC		 2310 if (err < 0)
2311 goto out_free;
1da177e4 2312
57f89bfa
BG		 2313 if (!gso_type && (len > dev->mtu + reserve)) {
2314 /* Earlier code assumed this would be a VLAN pkt,
2315 * double-check this now that we have the actual
2316 * packet in hand.
2317 */
2318 struct ethhdr *ehdr;
2319 skb_reset_mac_header(skb);
2320 ehdr = eth_hdr(skb);
2321 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2322 err = -EMSGSIZE;
2323 goto out_free;
2324 }
2325 }
2326
1da177e4
LT		 2327 skb->protocol = proto;
2328 skb->dev = dev;
2329 skb->priority = sk->sk_priority;
2d37a186 2330 skb->mark = sk->sk_mark;
1da177e4 2331
bfd5f4a3
SS		 2332 if (po->has_vnet_hdr) {
2333 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2334 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2335 vnet_hdr.csum_offset)) {
2336 err = -EINVAL;
2337 goto out_free;
2338 }
2339 }
2340
2341 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2342 skb_shinfo(skb)->gso_type = gso_type;
2343
2344 /* Header must be checked, and gso_segs computed. */
2345 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2346 skb_shinfo(skb)->gso_segs = 0;
2347
2348 len += vnet_hdr_len;
2349 }
2350
1da177e4
LT		 2351 /*
2352 * Now send it
2353 */
2354
2355 err = dev_queue_xmit(skb);
2356 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2357 goto out_unlock;
2358
827d9780
BG		 2359 if (need_rls_dev)
2360 dev_put(dev);
1da177e4 2361
40d4e3df 2362 return len;
1da177e4
LT		 2363
2364out_free:
2365 kfree_skb(skb);
2366out_unlock:
827d9780 2367 if (dev && need_rls_dev)
1da177e4
LT		 2368 dev_put(dev);
2369out:
2370 return err;
2371}
2372
69e3c75f
JB		 2373static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2374 struct msghdr *msg, size_t len)
2375{
69e3c75f
JB		 2376 struct sock *sk = sock->sk;
2377 struct packet_sock *po = pkt_sk(sk);
2378 if (po->tx_ring.pg_vec)
2379 return tpacket_snd(po, msg);
2380 else
69e3c75f
JB		 2381 return packet_snd(sock, msg, len);
2382}
2383
1da177e4
LT		 2384/*
2385 * Close a PACKET socket. This is fairly simple. We immediately go
2386 * to 'closed' state and remove our protocol entry in the device list.
2387 */
2388
2389static int packet_release(struct socket *sock)
2390{
2391 struct sock *sk = sock->sk;
2392 struct packet_sock *po;
d12d01d6 2393 struct net *net;
f6fb8f10 2394 union tpacket_req_u req_u;
1da177e4
LT		 2395
2396 if (!sk)
2397 return 0;
2398
3b1e0a65 2399 net = sock_net(sk);
1da177e4
LT		 2400 po = pkt_sk(sk);
2401
808f5114 2402 spin_lock_bh(&net->packet.sklist_lock);
2403 sk_del_node_init_rcu(sk);
920de804 2404 sock_prot_inuse_add(net, sk->sk_prot, -1);
808f5114 2405 spin_unlock_bh(&net->packet.sklist_lock);
1da177e4 2406
808f5114 2407 spin_lock(&po->bind_lock);
ce06b03e 2408 unregister_prot_hook(sk, false);
160ff18a
BG		 2409 if (po->prot_hook.dev) {
2410 dev_put(po->prot_hook.dev);
2411 po->prot_hook.dev = NULL;
2412 }
808f5114 2413 spin_unlock(&po->bind_lock);
1da177e4 2414
1da177e4 2415 packet_flush_mclist(sk);
1da177e4 2416
f6fb8f10 2417 memset(&req_u, 0, sizeof(req_u));
69e3c75f
JB		 2418
2419 if (po->rx_ring.pg_vec)
f6fb8f10 2420 packet_set_ring(sk, &req_u, 1, 0);
69e3c75f
JB		 2421
2422 if (po->tx_ring.pg_vec)
f6fb8f10 2423 packet_set_ring(sk, &req_u, 1, 1);
1da177e4 2424
dc99f600
DM		 2425 fanout_release(sk);
2426
808f5114 2427 synchronize_net();
1da177e4
LT		 2428 /*
2429 * Now the socket is dead. No more input will appear.
2430 */
1da177e4
LT		 2431 sock_orphan(sk);
2432 sock->sk = NULL;
2433
2434 /* Purge queues */
2435
2436 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2437 sk_refcnt_debug_release(sk);
1da177e4
LT		 2438
2439 sock_put(sk);
2440 return 0;
2441}
2442
2443/*
2444 * Attach a packet hook.
2445 */
2446
0e11c91e 2447static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2448{
2449 struct packet_sock *po = pkt_sk(sk);
dc99f600
DM		 2450
2451 if (po->fanout)
2452 return -EINVAL;
1da177e4
LT		 2453
2454 lock_sock(sk);
2455
2456 spin_lock(&po->bind_lock);
ce06b03e 2457 unregister_prot_hook(sk, true);
1da177e4
LT		 2458 po->num = protocol;
2459 po->prot_hook.type = protocol;
160ff18a
BG		 2460 if (po->prot_hook.dev)
2461 dev_put(po->prot_hook.dev);
1da177e4
LT		 2462 po->prot_hook.dev = dev;
2463
2464 po->ifindex = dev ? dev->ifindex : 0;
2465
2466 if (protocol == 0)
2467 goto out_unlock;
2468
be85d4ad 2469 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2470 register_prot_hook(sk);
be85d4ad
UT		 2471 } else {
2472 sk->sk_err = ENETDOWN;
2473 if (!sock_flag(sk, SOCK_DEAD))
2474 sk->sk_error_report(sk);
1da177e4
LT		 2475 }
2476
2477out_unlock:
2478 spin_unlock(&po->bind_lock);
2479 release_sock(sk);
2480 return 0;
2481}
2482
2483/*
2484 * Bind a packet socket to a device
2485 */
2486
40d4e3df
ED		 2487static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2488 int addr_len)
1da177e4 2489{
40d4e3df 2490 struct sock *sk = sock->sk;
1da177e4
LT		 2491 char name[15];
2492 struct net_device *dev;
2493 int err = -ENODEV;
1ce4f28b 2494
1da177e4
LT		 2495 /*
2496 * Check legality
2497 */
1ce4f28b 2498
8ae55f04 2499 if (addr_len != sizeof(struct sockaddr))
1da177e4 2500 return -EINVAL;
40d4e3df 2501 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2502
3b1e0a65 2503 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2504 if (dev)
1da177e4 2505 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2506 return err;
2507}
1da177e4
LT		 2508
2509static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2510{
40d4e3df
ED		 2511 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2512 struct sock *sk = sock->sk;
1da177e4
LT		 2513 struct net_device *dev = NULL;
2514 int err;
2515
2516
2517 /*
2518 * Check legality
2519 */
1ce4f28b 2520
1da177e4
LT		 2521 if (addr_len < sizeof(struct sockaddr_ll))
2522 return -EINVAL;
2523 if (sll->sll_family != AF_PACKET)
2524 return -EINVAL;
2525
2526 if (sll->sll_ifindex) {
2527 err = -ENODEV;
3b1e0a65 2528 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2529 if (dev == NULL)
2530 goto out;
2531 }
2532 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2533
2534out:
2535 return err;
2536}
2537
2538static struct proto packet_proto = {
2539 .name = "PACKET",
2540 .owner = THIS_MODULE,
2541 .obj_size = sizeof(struct packet_sock),
2542};
2543
2544/*
1ce4f28b 2545 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2546 */
2547
3f378b68
EP		 2548static int packet_create(struct net *net, struct socket *sock, int protocol,
2549 int kern)
1da177e4
LT		 2550{
2551 struct sock *sk;
2552 struct packet_sock *po;
0e11c91e 2553 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2554 int err;
2555
2556 if (!capable(CAP_NET_RAW))
2557 return -EPERM;
be02097c
DM		 2558 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2559 sock->type != SOCK_PACKET)
1da177e4
LT		 2560 return -ESOCKTNOSUPPORT;
2561
2562 sock->state = SS_UNCONNECTED;
2563
2564 err = -ENOBUFS;
6257ff21 2565 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2566 if (sk == NULL)
2567 goto out;
2568
2569 sock->ops = &packet_ops;
1da177e4
LT		 2570 if (sock->type == SOCK_PACKET)
2571 sock->ops = &packet_ops_spkt;
be02097c 2572
1da177e4
LT		 2573 sock_init_data(sock, sk);
2574
2575 po = pkt_sk(sk);
2576 sk->sk_family = PF_PACKET;
0e11c91e 2577 po->num = proto;
1da177e4
LT		 2578
2579 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2580 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2581
2582 /*
2583 * Attach a protocol block
2584 */
2585
2586 spin_lock_init(&po->bind_lock);
905db440 2587 mutex_init(&po->pg_vec_lock);
1da177e4 2588 po->prot_hook.func = packet_rcv;
be02097c 2589
1da177e4
LT		 2590 if (sock->type == SOCK_PACKET)
2591 po->prot_hook.func = packet_rcv_spkt;
be02097c 2592
1da177e4
LT		 2593 po->prot_hook.af_packet_priv = sk;
2594
0e11c91e
AV		 2595 if (proto) {
2596 po->prot_hook.type = proto;
ce06b03e 2597 register_prot_hook(sk);
1da177e4
LT		 2598 }
2599
808f5114 2600 spin_lock_bh(&net->packet.sklist_lock);
2601 sk_add_node_rcu(sk, &net->packet.sklist);
3680453c 2602 sock_prot_inuse_add(net, &packet_proto, 1);
808f5114 2603 spin_unlock_bh(&net->packet.sklist_lock);
2604
40d4e3df 2605 return 0;
1da177e4
LT		 2606out:
2607 return err;
2608}
2609
ed85b565
RC		 2610static int packet_recv_error(struct sock *sk, struct msghdr *msg, int len)
2611{
2612 struct sock_exterr_skb *serr;
2613 struct sk_buff *skb, *skb2;
2614 int copied, err;
2615
2616 err = -EAGAIN;
2617 skb = skb_dequeue(&sk->sk_error_queue);
2618 if (skb == NULL)
2619 goto out;
2620
2621 copied = skb->len;
2622 if (copied > len) {
2623 msg->msg_flags |= MSG_TRUNC;
2624 copied = len;
2625 }
2626 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2627 if (err)
2628 goto out_free_skb;
2629
2630 sock_recv_timestamp(msg, sk, skb);
2631
2632 serr = SKB_EXT_ERR(skb);
2633 put_cmsg(msg, SOL_PACKET, PACKET_TX_TIMESTAMP,
2634 sizeof(serr->ee), &serr->ee);
2635
2636 msg->msg_flags |= MSG_ERRQUEUE;
2637 err = copied;
2638
2639 /* Reset and regenerate socket error */
2640 spin_lock_bh(&sk->sk_error_queue.lock);
2641 sk->sk_err = 0;
2642 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) {
2643 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno;
2644 spin_unlock_bh(&sk->sk_error_queue.lock);
2645 sk->sk_error_report(sk);
2646 } else
2647 spin_unlock_bh(&sk->sk_error_queue.lock);
2648
2649out_free_skb:
2650 kfree_skb(skb);
2651out:
2652 return err;
2653}
2654
1da177e4
LT		 2655/*
2656 * Pull a packet from our receive queue and hand it to the user.
2657 * If necessary we block.
2658 */
2659
2660static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2661 struct msghdr *msg, size_t len, int flags)
2662{
2663 struct sock *sk = sock->sk;
2664 struct sk_buff *skb;
2665 int copied, err;
0fb375fb 2666 struct sockaddr_ll *sll;
bfd5f4a3 2667 int vnet_hdr_len = 0;
1da177e4
LT		 2668
2669 err = -EINVAL;
ed85b565 2670 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2671 goto out;
2672
2673#if 0
2674 /* What error should we return now? EUNATTACH? */
2675 if (pkt_sk(sk)->ifindex < 0)
2676 return -ENODEV;
2677#endif
2678
ed85b565
RC		 2679 if (flags & MSG_ERRQUEUE) {
2680 err = packet_recv_error(sk, msg, len);
2681 goto out;
2682 }
2683
1da177e4
LT		 2684 /*
2685 * Call the generic datagram receiver. This handles all sorts
2686 * of horrible races and re-entrancy so we can forget about it
2687 * in the protocol layers.
2688 *
2689 * Now it will return ENETDOWN, if device have just gone down,
2690 * but then it will block.
2691 */
2692
40d4e3df 2693 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2694
2695 /*
1ce4f28b 2696 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2697 * handles the blocking we don't see and worry about blocking
2698 * retries.
2699 */
2700
8ae55f04 2701 if (skb == NULL)
1da177e4
LT		 2702 goto out;
2703
bfd5f4a3
SS		 2704 if (pkt_sk(sk)->has_vnet_hdr) {
2705 struct virtio_net_hdr vnet_hdr = { 0 };
2706
2707 err = -EINVAL;
2708 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2709 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2710 goto out_free;
2711
1f18b717
MK		 2712 len -= vnet_hdr_len;
2713
bfd5f4a3
SS		 2714 if (skb_is_gso(skb)) {
2715 struct skb_shared_info *sinfo = skb_shinfo(skb);
2716
2717 /* This is a hint as to how much should be linear. */
2718 vnet_hdr.hdr_len = skb_headlen(skb);
2719 vnet_hdr.gso_size = sinfo->gso_size;
2720 if (sinfo->gso_type & SKB_GSO_TCPV4)
2721 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2722 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2723 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2724 else if (sinfo->gso_type & SKB_GSO_UDP)
2725 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2726 else if (sinfo->gso_type & SKB_GSO_FCOE)
2727 goto out_free;
2728 else
2729 BUG();
2730 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2731 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2732 } else
2733 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2734
2735 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2736 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2737 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2738 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2739 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2740 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2741 } /* else everything is zero */
2742
2743 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2744 vnet_hdr_len);
2745 if (err < 0)
2746 goto out_free;
2747 }
2748
0fb375fb
EB		 2749 /*
2750 * If the address length field is there to be filled in, we fill
2751 * it in now.
2752 */
2753
ffbc6111 2754 sll = &PACKET_SKB_CB(skb)->sa.ll;
0fb375fb
EB		 2755 if (sock->type == SOCK_PACKET)
2756 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2757 else
2758 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
2759
1da177e4
LT		 2760 /*
2761 * You lose any data beyond the buffer you gave. If it worries a
2762 * user program they can ask the device for its MTU anyway.
2763 */
2764
2765 copied = skb->len;
40d4e3df
ED		 2766 if (copied > len) {
2767 copied = len;
2768 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2769 }
2770
2771 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2772 if (err)
2773 goto out_free;
2774
3b885787 2775 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4
LT		 2776
2777 if (msg->msg_name)
ffbc6111
HX		 2778 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2779 msg->msg_namelen);
1da177e4 2780
8dc41944 2781 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2782 struct tpacket_auxdata aux;
2783
2784 aux.tp_status = TP_STATUS_USER;
2785 if (skb->ip_summed == CHECKSUM_PARTIAL)
2786 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2787 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2788 aux.tp_snaplen = skb->len;
2789 aux.tp_mac = 0;
bbe735e4 2790 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2791 if (vlan_tx_tag_present(skb)) {
2792 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2793 aux.tp_status |= TP_STATUS_VLAN_VALID;
2794 } else {
2795 aux.tp_vlan_tci = 0;
2796 }
13fcb7bd 2797 aux.tp_padding = 0;
ffbc6111 2798 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2799 }
2800
1da177e4
LT		 2801 /*
2802 * Free or return the buffer as appropriate. Again this
2803 * hides all the races and re-entrancy issues from us.
2804 */
bfd5f4a3 2805 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2806
2807out_free:
2808 skb_free_datagram(sk, skb);
2809out:
2810 return err;
2811}
2812
1da177e4
LT		 2813static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2814 int *uaddr_len, int peer)
2815{
2816 struct net_device *dev;
2817 struct sock *sk = sock->sk;
2818
2819 if (peer)
2820 return -EOPNOTSUPP;
2821
2822 uaddr->sa_family = AF_PACKET;
654d1f8a
ED		 2823 rcu_read_lock();
2824 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2825 if (dev)
67286640 2826 strncpy(uaddr->sa_data, dev->name, 14);
654d1f8a 2827 else
1da177e4 2828 memset(uaddr->sa_data, 0, 14);
654d1f8a 2829 rcu_read_unlock();
1da177e4
LT		 2830 *uaddr_len = sizeof(*uaddr);
2831
2832 return 0;
2833}
1da177e4
LT		 2834
2835static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2836 int *uaddr_len, int peer)
2837{
2838 struct net_device *dev;
2839 struct sock *sk = sock->sk;
2840 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2841 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2842
2843 if (peer)
2844 return -EOPNOTSUPP;
2845
2846 sll->sll_family = AF_PACKET;
2847 sll->sll_ifindex = po->ifindex;
2848 sll->sll_protocol = po->num;
67286640 2849 sll->sll_pkttype = 0;
654d1f8a
ED		 2850 rcu_read_lock();
2851 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2852 if (dev) {
2853 sll->sll_hatype = dev->type;
2854 sll->sll_halen = dev->addr_len;
2855 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2856 } else {
2857 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2858 sll->sll_halen = 0;
2859 }
654d1f8a 2860 rcu_read_unlock();
0fb375fb 2861 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2862
2863 return 0;
2864}
2865
2aeb0b88
WC		 2866static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2867 int what)
1da177e4
LT		 2868{
2869 switch (i->type) {
2870 case PACKET_MR_MULTICAST:
1162563f
JP		 2871 if (i->alen != dev->addr_len)
2872 return -EINVAL;
1da177e4 2873 if (what > 0)
22bedad3 2874 return dev_mc_add(dev, i->addr);
1da177e4 2875 else
22bedad3 2876 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2877 break;
2878 case PACKET_MR_PROMISC:
2aeb0b88 2879 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2880 break;
2881 case PACKET_MR_ALLMULTI:
2aeb0b88 2882 return dev_set_allmulti(dev, what);
1da177e4 2883 break;
d95ed927 2884 case PACKET_MR_UNICAST:
1162563f
JP		 2885 if (i->alen != dev->addr_len)
2886 return -EINVAL;
d95ed927 2887 if (what > 0)
a748ee24 2888 return dev_uc_add(dev, i->addr);
d95ed927 2889 else
a748ee24 2890 return dev_uc_del(dev, i->addr);
d95ed927 2891 break;
40d4e3df
ED		 2892 default:
2893 break;
1da177e4 2894 }
2aeb0b88 2895 return 0;
1da177e4
LT		 2896}
2897
2898static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2899{
40d4e3df 2900 for ( ; i; i = i->next) {
1da177e4
LT		 2901 if (i->ifindex == dev->ifindex)
2902 packet_dev_mc(dev, i, what);
2903 }
2904}
2905
0fb375fb 2906static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2907{
2908 struct packet_sock *po = pkt_sk(sk);
2909 struct packet_mclist *ml, *i;
2910 struct net_device *dev;
2911 int err;
2912
2913 rtnl_lock();
2914
2915 err = -ENODEV;
3b1e0a65 2916 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2917 if (!dev)
2918 goto done;
2919
2920 err = -EINVAL;
1162563f 2921 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2922 goto done;
2923
2924 err = -ENOBUFS;
8b3a7005 2925 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 2926 if (i == NULL)
2927 goto done;
2928
2929 err = 0;
2930 for (ml = po->mclist; ml; ml = ml->next) {
2931 if (ml->ifindex == mreq->mr_ifindex &&
2932 ml->type == mreq->mr_type &&
2933 ml->alen == mreq->mr_alen &&
2934 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2935 ml->count++;
2936 /* Free the new element ... */
2937 kfree(i);
2938 goto done;
2939 }
2940 }
2941
2942 i->type = mreq->mr_type;
2943 i->ifindex = mreq->mr_ifindex;
2944 i->alen = mreq->mr_alen;
2945 memcpy(i->addr, mreq->mr_address, i->alen);
2946 i->count = 1;
2947 i->next = po->mclist;
2948 po->mclist = i;
2aeb0b88
WC		 2949 err = packet_dev_mc(dev, i, 1);
2950 if (err) {
2951 po->mclist = i->next;
2952 kfree(i);
2953 }
1da177e4
LT		 2954
2955done:
2956 rtnl_unlock();
2957 return err;
2958}
2959
0fb375fb 2960static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2961{
2962 struct packet_mclist *ml, **mlp;
2963
2964 rtnl_lock();
2965
2966 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
2967 if (ml->ifindex == mreq->mr_ifindex &&
2968 ml->type == mreq->mr_type &&
2969 ml->alen == mreq->mr_alen &&
2970 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2971 if (--ml->count == 0) {
2972 struct net_device *dev;
2973 *mlp = ml->next;
ad959e76
ED		 2974 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2975 if (dev)
1da177e4 2976 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 2977 kfree(ml);
2978 }
2979 rtnl_unlock();
2980 return 0;
2981 }
2982 }
2983 rtnl_unlock();
2984 return -EADDRNOTAVAIL;
2985}
2986
2987static void packet_flush_mclist(struct sock *sk)
2988{
2989 struct packet_sock *po = pkt_sk(sk);
2990 struct packet_mclist *ml;
2991
2992 if (!po->mclist)
2993 return;
2994
2995 rtnl_lock();
2996 while ((ml = po->mclist) != NULL) {
2997 struct net_device *dev;
2998
2999 po->mclist = ml->next;
ad959e76
ED		 3000 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3001 if (dev != NULL)
1da177e4 3002 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3003 kfree(ml);
3004 }
3005 rtnl_unlock();
3006}
1da177e4
LT		 3007
3008static int
b7058842 3009packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3010{
3011 struct sock *sk = sock->sk;
8dc41944 3012 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3013 int ret;
3014
3015 if (level != SOL_PACKET)
3016 return -ENOPROTOOPT;
3017
69e3c75f 3018 switch (optname) {
1ce4f28b 3019 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3020 case PACKET_DROP_MEMBERSHIP:
3021 {
0fb375fb
EB		 3022 struct packet_mreq_max mreq;
3023 int len = optlen;
3024 memset(&mreq, 0, sizeof(mreq));
3025 if (len < sizeof(struct packet_mreq))
1da177e4 3026 return -EINVAL;
0fb375fb
EB		 3027 if (len > sizeof(mreq))
3028 len = sizeof(mreq);
40d4e3df 3029 if (copy_from_user(&mreq, optval, len))
1da177e4 3030 return -EFAULT;
0fb375fb
EB		 3031 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3032 return -EINVAL;
1da177e4
LT		 3033 if (optname == PACKET_ADD_MEMBERSHIP)
3034 ret = packet_mc_add(sk, &mreq);
3035 else
3036 ret = packet_mc_drop(sk, &mreq);
3037 return ret;
3038 }
a2efcfa0 3039
1da177e4 3040 case PACKET_RX_RING:
69e3c75f 3041 case PACKET_TX_RING:
1da177e4 3042 {
f6fb8f10 3043 union tpacket_req_u req_u;
3044 int len;
1da177e4 3045
f6fb8f10 3046 switch (po->tp_version) {
3047 case TPACKET_V1:
3048 case TPACKET_V2:
3049 len = sizeof(req_u.req);
3050 break;
3051 case TPACKET_V3:
3052 default:
3053 len = sizeof(req_u.req3);
3054 break;
3055 }
3056 if (optlen < len)
1da177e4 3057 return -EINVAL;
bfd5f4a3
SS		 3058 if (pkt_sk(sk)->has_vnet_hdr)
3059 return -EINVAL;
f6fb8f10 3060 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3061 return -EFAULT;
f6fb8f10 3062 return packet_set_ring(sk, &req_u, 0,
3063 optname == PACKET_TX_RING);
1da177e4
LT		 3064 }
3065 case PACKET_COPY_THRESH:
3066 {
3067 int val;
3068
40d4e3df 3069 if (optlen != sizeof(val))
1da177e4 3070 return -EINVAL;
40d4e3df 3071 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3072 return -EFAULT;
3073
3074 pkt_sk(sk)->copy_thresh = val;
3075 return 0;
3076 }
bbd6ef87
PM		 3077 case PACKET_VERSION:
3078 {
3079 int val;
3080
3081 if (optlen != sizeof(val))
3082 return -EINVAL;
69e3c75f 3083 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3084 return -EBUSY;
3085 if (copy_from_user(&val, optval, sizeof(val)))
3086 return -EFAULT;
3087 switch (val) {
3088 case TPACKET_V1:
3089 case TPACKET_V2:
f6fb8f10 3090 case TPACKET_V3:
bbd6ef87
PM		 3091 po->tp_version = val;
3092 return 0;
3093 default:
3094 return -EINVAL;
3095 }
3096 }
8913336a
PM		 3097 case PACKET_RESERVE:
3098 {
3099 unsigned int val;
3100
3101 if (optlen != sizeof(val))
3102 return -EINVAL;
69e3c75f 3103 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3104 return -EBUSY;
3105 if (copy_from_user(&val, optval, sizeof(val)))
3106 return -EFAULT;
3107 po->tp_reserve = val;
3108 return 0;
3109 }
69e3c75f
JB		 3110 case PACKET_LOSS:
3111 {
3112 unsigned int val;
3113
3114 if (optlen != sizeof(val))
3115 return -EINVAL;
3116 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3117 return -EBUSY;
3118 if (copy_from_user(&val, optval, sizeof(val)))
3119 return -EFAULT;
3120 po->tp_loss = !!val;
3121 return 0;
3122 }
8dc41944
HX		 3123 case PACKET_AUXDATA:
3124 {
3125 int val;
3126
3127 if (optlen < sizeof(val))
3128 return -EINVAL;
3129 if (copy_from_user(&val, optval, sizeof(val)))
3130 return -EFAULT;
3131
3132 po->auxdata = !!val;
3133 return 0;
3134 }
80feaacb
PWJ		 3135 case PACKET_ORIGDEV:
3136 {
3137 int val;
3138
3139 if (optlen < sizeof(val))
3140 return -EINVAL;
3141 if (copy_from_user(&val, optval, sizeof(val)))
3142 return -EFAULT;
3143
3144 po->origdev = !!val;
3145 return 0;
3146 }
bfd5f4a3
SS		 3147 case PACKET_VNET_HDR:
3148 {
3149 int val;
3150
3151 if (sock->type != SOCK_RAW)
3152 return -EINVAL;
3153 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3154 return -EBUSY;
3155 if (optlen < sizeof(val))
3156 return -EINVAL;
3157 if (copy_from_user(&val, optval, sizeof(val)))
3158 return -EFAULT;
3159
3160 po->has_vnet_hdr = !!val;
3161 return 0;
3162 }
614f60fa
SM		 3163 case PACKET_TIMESTAMP:
3164 {
3165 int val;
3166
3167 if (optlen != sizeof(val))
3168 return -EINVAL;
3169 if (copy_from_user(&val, optval, sizeof(val)))
3170 return -EFAULT;
3171
3172 po->tp_tstamp = val;
3173 return 0;
3174 }
dc99f600
DM		 3175 case PACKET_FANOUT:
3176 {
3177 int val;
3178
3179 if (optlen != sizeof(val))
3180 return -EINVAL;
3181 if (copy_from_user(&val, optval, sizeof(val)))
3182 return -EFAULT;
3183
3184 return fanout_add(sk, val & 0xffff, val >> 16);
3185 }
1da177e4
LT		 3186 default:
3187 return -ENOPROTOOPT;
3188 }
3189}
3190
3191static int packet_getsockopt(struct socket *sock, int level, int optname,
3192 char __user *optval, int __user *optlen)
3193{
3194 int len;
8dc41944 3195 int val;
1da177e4
LT		 3196 struct sock *sk = sock->sk;
3197 struct packet_sock *po = pkt_sk(sk);
8dc41944
HX		 3198 void *data;
3199 struct tpacket_stats st;
f6fb8f10 3200 union tpacket_stats_u st_u;
1da177e4
LT		 3201
3202 if (level != SOL_PACKET)
3203 return -ENOPROTOOPT;
3204
8ae55f04
KK		 3205 if (get_user(len, optlen))
3206 return -EFAULT;
1da177e4
LT		 3207
3208 if (len < 0)
3209 return -EINVAL;
1ce4f28b 3210
69e3c75f 3211 switch (optname) {
1da177e4 3212 case PACKET_STATISTICS:
f6fb8f10 3213 if (po->tp_version == TPACKET_V3) {
3214 len = sizeof(struct tpacket_stats_v3);
3215 } else {
3216 if (len > sizeof(struct tpacket_stats))
3217 len = sizeof(struct tpacket_stats);
3218 }
1da177e4 3219 spin_lock_bh(&sk->sk_receive_queue.lock);
f6fb8f10 3220 if (po->tp_version == TPACKET_V3) {
3221 memcpy(&st_u.stats3, &po->stats,
3222 sizeof(struct tpacket_stats));
3223 st_u.stats3.tp_freeze_q_cnt =
3224 po->stats_u.stats3.tp_freeze_q_cnt;
3225 st_u.stats3.tp_packets += po->stats.tp_drops;
3226 data = &st_u.stats3;
3227 } else {
3228 st = po->stats;
3229 st.tp_packets += st.tp_drops;
3230 data = &st;
3231 }
1da177e4
LT		 3232 memset(&po->stats, 0, sizeof(st));
3233 spin_unlock_bh(&sk->sk_receive_queue.lock);
8dc41944
HX		 3234 break;
3235 case PACKET_AUXDATA:
3236 if (len > sizeof(int))
3237 len = sizeof(int);
3238 val = po->auxdata;
3239
80feaacb
PWJ		 3240 data = &val;
3241 break;
3242 case PACKET_ORIGDEV:
3243 if (len > sizeof(int))
3244 len = sizeof(int);
3245 val = po->origdev;
3246
bfd5f4a3
SS		 3247 data = &val;
3248 break;
3249 case PACKET_VNET_HDR:
3250 if (len > sizeof(int))
3251 len = sizeof(int);
3252 val = po->has_vnet_hdr;
3253
8dc41944 3254 data = &val;
1da177e4 3255 break;
bbd6ef87
PM		 3256 case PACKET_VERSION:
3257 if (len > sizeof(int))
3258 len = sizeof(int);
3259 val = po->tp_version;
3260 data = &val;
3261 break;
3262 case PACKET_HDRLEN:
3263 if (len > sizeof(int))
3264 len = sizeof(int);
3265 if (copy_from_user(&val, optval, len))
3266 return -EFAULT;
3267 switch (val) {
3268 case TPACKET_V1:
3269 val = sizeof(struct tpacket_hdr);
3270 break;
3271 case TPACKET_V2:
3272 val = sizeof(struct tpacket2_hdr);
3273 break;
f6fb8f10 3274 case TPACKET_V3:
3275 val = sizeof(struct tpacket3_hdr);
3276 break;
bbd6ef87
PM		 3277 default:
3278 return -EINVAL;
3279 }
3280 data = &val;
3281 break;
8913336a
PM		 3282 case PACKET_RESERVE:
3283 if (len > sizeof(unsigned int))
3284 len = sizeof(unsigned int);
3285 val = po->tp_reserve;
3286 data = &val;
3287 break;
69e3c75f
JB		 3288 case PACKET_LOSS:
3289 if (len > sizeof(unsigned int))
3290 len = sizeof(unsigned int);
3291 val = po->tp_loss;
3292 data = &val;
3293 break;
614f60fa
SM		 3294 case PACKET_TIMESTAMP:
3295 if (len > sizeof(int))
3296 len = sizeof(int);
3297 val = po->tp_tstamp;
3298 data = &val;
3299 break;
dc99f600
DM		 3300 case PACKET_FANOUT:
3301 if (len > sizeof(int))
3302 len = sizeof(int);
3303 val = (po->fanout ?
3304 ((u32)po->fanout->id |
3305 ((u32)po->fanout->type << 16)) :
3306 0);
3307 data = &val;
3308 break;
1da177e4
LT		 3309 default:
3310 return -ENOPROTOOPT;
3311 }
3312
8ae55f04
KK		 3313 if (put_user(len, optlen))
3314 return -EFAULT;
8dc41944
HX		 3315 if (copy_to_user(optval, data, len))
3316 return -EFAULT;
8ae55f04 3317 return 0;
1da177e4
LT		 3318}
3319
3320
3321static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
3322{
3323 struct sock *sk;
3324 struct hlist_node *node;
ad930650 3325 struct net_device *dev = data;
c346dca1 3326 struct net *net = dev_net(dev);
1da177e4 3327
808f5114 3328 rcu_read_lock();
3329 sk_for_each_rcu(sk, node, &net->packet.sklist) {
1da177e4
LT		 3330 struct packet_sock *po = pkt_sk(sk);
3331
3332 switch (msg) {
3333 case NETDEV_UNREGISTER:
1da177e4
LT		 3334 if (po->mclist)
3335 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3336 /* fallthrough */
3337
1da177e4
LT		 3338 case NETDEV_DOWN:
3339 if (dev->ifindex == po->ifindex) {
3340 spin_lock(&po->bind_lock);
3341 if (po->running) {
ce06b03e 3342 __unregister_prot_hook(sk, false);
1da177e4
LT		 3343 sk->sk_err = ENETDOWN;
3344 if (!sock_flag(sk, SOCK_DEAD))
3345 sk->sk_error_report(sk);
3346 }
3347 if (msg == NETDEV_UNREGISTER) {
3348 po->ifindex = -1;
160ff18a
BG		 3349 if (po->prot_hook.dev)
3350 dev_put(po->prot_hook.dev);
1da177e4
LT		 3351 po->prot_hook.dev = NULL;
3352 }
3353 spin_unlock(&po->bind_lock);
3354 }
3355 break;
3356 case NETDEV_UP:
808f5114 3357 if (dev->ifindex == po->ifindex) {
3358 spin_lock(&po->bind_lock);
ce06b03e
DM		 3359 if (po->num)
3360 register_prot_hook(sk);
808f5114 3361 spin_unlock(&po->bind_lock);
1da177e4 3362 }
1da177e4
LT		 3363 break;
3364 }
3365 }
808f5114 3366 rcu_read_unlock();
1da177e4
LT		 3367 return NOTIFY_DONE;
3368}
3369
3370
3371static int packet_ioctl(struct socket *sock, unsigned int cmd,
3372 unsigned long arg)
3373{
3374 struct sock *sk = sock->sk;
3375
69e3c75f 3376 switch (cmd) {
40d4e3df
ED		 3377 case SIOCOUTQ:
3378 {
3379 int amount = sk_wmem_alloc_get(sk);
31e6d363 3380
40d4e3df
ED		 3381 return put_user(amount, (int __user *)arg);
3382 }
3383 case SIOCINQ:
3384 {
3385 struct sk_buff *skb;
3386 int amount = 0;
3387
3388 spin_lock_bh(&sk->sk_receive_queue.lock);
3389 skb = skb_peek(&sk->sk_receive_queue);
3390 if (skb)
3391 amount = skb->len;
3392 spin_unlock_bh(&sk->sk_receive_queue.lock);
3393 return put_user(amount, (int __user *)arg);
3394 }
3395 case SIOCGSTAMP:
3396 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3397 case SIOCGSTAMPNS:
3398 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3399
1da177e4 3400#ifdef CONFIG_INET
40d4e3df
ED		 3401 case SIOCADDRT:
3402 case SIOCDELRT:
3403 case SIOCDARP:
3404 case SIOCGARP:
3405 case SIOCSARP:
3406 case SIOCGIFADDR:
3407 case SIOCSIFADDR:
3408 case SIOCGIFBRDADDR:
3409 case SIOCSIFBRDADDR:
3410 case SIOCGIFNETMASK:
3411 case SIOCSIFNETMASK:
3412 case SIOCGIFDSTADDR:
3413 case SIOCSIFDSTADDR:
3414 case SIOCSIFFLAGS:
40d4e3df 3415 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3416#endif
3417
40d4e3df
ED		 3418 default:
3419 return -ENOIOCTLCMD;
1da177e4
LT		 3420 }
3421 return 0;
3422}
3423
40d4e3df 3424static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3425 poll_table *wait)
3426{
3427 struct sock *sk = sock->sk;
3428 struct packet_sock *po = pkt_sk(sk);
3429 unsigned int mask = datagram_poll(file, sock, wait);
3430
3431 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3432 if (po->rx_ring.pg_vec) {
f6fb8f10 3433 if (!packet_previous_rx_frame(po, &po->rx_ring,
3434 TP_STATUS_KERNEL))
1da177e4
LT		 3435 mask |= POLLIN | POLLRDNORM;
3436 }
3437 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3438 spin_lock_bh(&sk->sk_write_queue.lock);
3439 if (po->tx_ring.pg_vec) {
3440 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3441 mask |= POLLOUT | POLLWRNORM;
3442 }
3443 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3444 return mask;
3445}
3446
3447
3448/* Dirty? Well, I still did not learn better way to account
3449 * for user mmaps.
3450 */
3451
3452static void packet_mm_open(struct vm_area_struct *vma)
3453{
3454 struct file *file = vma->vm_file;
40d4e3df 3455 struct socket *sock = file->private_data;
1da177e4 3456 struct sock *sk = sock->sk;
1ce4f28b 3457
1da177e4
LT		 3458 if (sk)
3459 atomic_inc(&pkt_sk(sk)->mapped);
3460}
3461
3462static void packet_mm_close(struct vm_area_struct *vma)
3463{
3464 struct file *file = vma->vm_file;
40d4e3df 3465 struct socket *sock = file->private_data;
1da177e4 3466 struct sock *sk = sock->sk;
1ce4f28b 3467
1da177e4
LT		 3468 if (sk)
3469 atomic_dec(&pkt_sk(sk)->mapped);
3470}
3471
f0f37e2f 3472static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3473 .open = packet_mm_open,
3474 .close = packet_mm_close,
1da177e4
LT		 3475};
3476
0e3125c7
NH		 3477static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3478 unsigned int len)
1da177e4
LT		 3479{
3480 int i;
3481
4ebf0ae2 3482 for (i = 0; i < len; i++) {
0e3125c7 3483 if (likely(pg_vec[i].buffer)) {
c56b4d90 3484 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3485 vfree(pg_vec[i].buffer);
3486 else
3487 free_pages((unsigned long)pg_vec[i].buffer,
3488 order);
3489 pg_vec[i].buffer = NULL;
3490 }
1da177e4
LT		 3491 }
3492 kfree(pg_vec);
3493}
3494
eea49cc9 3495static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3496{
0e3125c7
NH		 3497 char *buffer = NULL;
3498 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3499 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3500
3501 buffer = (char *) __get_free_pages(gfp_flags, order);
3502
3503 if (buffer)
3504 return buffer;
3505
3506 /*
3507 * __get_free_pages failed, fall back to vmalloc
3508 */
bbce5a59 3509 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3510
0e3125c7
NH		 3511 if (buffer)
3512 return buffer;
3513
3514 /*
3515 * vmalloc failed, lets dig into swap here
3516 */
0e3125c7
NH		 3517 gfp_flags &= ~__GFP_NORETRY;
3518 buffer = (char *)__get_free_pages(gfp_flags, order);
3519 if (buffer)
3520 return buffer;
3521
3522 /*
3523 * complete and utter failure
3524 */
3525 return NULL;
4ebf0ae2
DM		 3526}
3527
0e3125c7 3528static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3529{
3530 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3531 struct pgv *pg_vec;
4ebf0ae2
DM		 3532 int i;
3533
0e3125c7 3534 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3535 if (unlikely(!pg_vec))
3536 goto out;
3537
3538 for (i = 0; i < block_nr; i++) {
c56b4d90 3539 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3540 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3541 goto out_free_pgvec;
3542 }
3543
3544out:
3545 return pg_vec;
3546
3547out_free_pgvec:
3548 free_pg_vec(pg_vec, order, block_nr);
3549 pg_vec = NULL;
3550 goto out;
3551}
1da177e4 3552
f6fb8f10 3553static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3554 int closing, int tx_ring)
1da177e4 3555{
0e3125c7 3556 struct pgv *pg_vec = NULL;
1da177e4 3557 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3558 int was_running, order = 0;
69e3c75f
JB		 3559 struct packet_ring_buffer *rb;
3560 struct sk_buff_head *rb_queue;
0e11c91e 3561 __be16 num;
f6fb8f10 3562 int err = -EINVAL;
3563 /* Added to avoid minimal code churn */
3564 struct tpacket_req *req = &req_u->req;
3565
3566 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3567 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3568 WARN(1, "Tx-ring is not supported.\n");
3569 goto out;
3570 }
1ce4f28b 3571
69e3c75f
JB		 3572 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3573 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3574
69e3c75f
JB		 3575 err = -EBUSY;
3576 if (!closing) {
3577 if (atomic_read(&po->mapped))
3578 goto out;
3579 if (atomic_read(&rb->pending))
3580 goto out;
3581 }
1da177e4 3582
69e3c75f
JB		 3583 if (req->tp_block_nr) {
3584 /* Sanity tests and some calculations */
3585 err = -EBUSY;
3586 if (unlikely(rb->pg_vec))
3587 goto out;
1da177e4 3588
bbd6ef87
PM		 3589 switch (po->tp_version) {
3590 case TPACKET_V1:
3591 po->tp_hdrlen = TPACKET_HDRLEN;
3592 break;
3593 case TPACKET_V2:
3594 po->tp_hdrlen = TPACKET2_HDRLEN;
3595 break;
f6fb8f10 3596 case TPACKET_V3:
3597 po->tp_hdrlen = TPACKET3_HDRLEN;
3598 break;
bbd6ef87
PM		 3599 }
3600
69e3c75f 3601 err = -EINVAL;
4ebf0ae2 3602 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3603 goto out;
4ebf0ae2 3604 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3605 goto out;
8913336a 3606 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3607 po->tp_reserve))
3608 goto out;
4ebf0ae2 3609 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3610 goto out;
1da177e4 3611
69e3c75f
JB		 3612 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3613 if (unlikely(rb->frames_per_block <= 0))
3614 goto out;
3615 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3616 req->tp_frame_nr))
3617 goto out;
1da177e4
LT		 3618
3619 err = -ENOMEM;
4ebf0ae2
DM		 3620 order = get_order(req->tp_block_size);
3621 pg_vec = alloc_pg_vec(req, order);
3622 if (unlikely(!pg_vec))
1da177e4 3623 goto out;
f6fb8f10 3624 switch (po->tp_version) {
3625 case TPACKET_V3:
3626 /* Transmit path is not supported. We checked
3627 * it above but just being paranoid
3628 */
3629 if (!tx_ring)
3630 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3631 break;
3632 default:
3633 break;
3634 }
69e3c75f
JB		 3635 }
3636 /* Done */
3637 else {
3638 err = -EINVAL;
4ebf0ae2 3639 if (unlikely(req->tp_frame_nr))
69e3c75f 3640 goto out;
1da177e4
LT		 3641 }
3642
3643 lock_sock(sk);
3644
3645 /* Detach socket from network */
3646 spin_lock(&po->bind_lock);
3647 was_running = po->running;
3648 num = po->num;
3649 if (was_running) {
1da177e4 3650 po->num = 0;
ce06b03e 3651 __unregister_prot_hook(sk, false);
1da177e4
LT		 3652 }
3653 spin_unlock(&po->bind_lock);
1ce4f28b 3654
1da177e4
LT		 3655 synchronize_net();
3656
3657 err = -EBUSY;
905db440 3658 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3659 if (closing || atomic_read(&po->mapped) == 0) {
3660 err = 0;
69e3c75f 3661 spin_lock_bh(&rb_queue->lock);
c053fd96 3662 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3663 rb->frame_max = (req->tp_frame_nr - 1);
3664 rb->head = 0;
3665 rb->frame_size = req->tp_frame_size;
3666 spin_unlock_bh(&rb_queue->lock);
3667
c053fd96
CG		 3668 swap(rb->pg_vec_order, order);
3669 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3670
3671 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3672 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3673 tpacket_rcv : packet_rcv;
3674 skb_queue_purge(rb_queue);
1da177e4 3675 if (atomic_read(&po->mapped))
40d4e3df
ED		 3676 pr_err("packet_mmap: vma is busy: %d\n",
3677 atomic_read(&po->mapped));
1da177e4 3678 }
905db440 3679 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3680
3681 spin_lock(&po->bind_lock);
ce06b03e 3682 if (was_running) {
1da177e4 3683 po->num = num;
ce06b03e 3684 register_prot_hook(sk);
1da177e4
LT		 3685 }
3686 spin_unlock(&po->bind_lock);
f6fb8f10 3687 if (closing && (po->tp_version > TPACKET_V2)) {
3688 /* Because we don't support block-based V3 on tx-ring */
3689 if (!tx_ring)
3690 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3691 }
1da177e4
LT		 3692 release_sock(sk);
3693
1da177e4
LT		 3694 if (pg_vec)
3695 free_pg_vec(pg_vec, order, req->tp_block_nr);
3696out:
3697 return err;
3698}
3699
69e3c75f
JB		 3700static int packet_mmap(struct file *file, struct socket *sock,
3701 struct vm_area_struct *vma)
1da177e4
LT		 3702{
3703 struct sock *sk = sock->sk;
3704 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3705 unsigned long size, expected_size;
3706 struct packet_ring_buffer *rb;
1da177e4
LT		 3707 unsigned long start;
3708 int err = -EINVAL;
3709 int i;
3710
3711 if (vma->vm_pgoff)
3712 return -EINVAL;
3713
905db440 3714 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3715
3716 expected_size = 0;
3717 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3718 if (rb->pg_vec) {
3719 expected_size += rb->pg_vec_len
3720 * rb->pg_vec_pages
3721 * PAGE_SIZE;
3722 }
3723 }
3724
3725 if (expected_size == 0)
1da177e4 3726 goto out;
69e3c75f
JB		 3727
3728 size = vma->vm_end - vma->vm_start;
3729 if (size != expected_size)
1da177e4
LT		 3730 goto out;
3731
1da177e4 3732 start = vma->vm_start;
69e3c75f
JB		 3733 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3734 if (rb->pg_vec == NULL)
3735 continue;
3736
3737 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3738 struct page *page;
3739 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3740 int pg_num;
3741
c56b4d90
CG		 3742 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3743 page = pgv_to_page(kaddr);
69e3c75f
JB		 3744 err = vm_insert_page(vma, start, page);
3745 if (unlikely(err))
3746 goto out;
3747 start += PAGE_SIZE;
0e3125c7 3748 kaddr += PAGE_SIZE;
69e3c75f 3749 }
4ebf0ae2 3750 }
1da177e4 3751 }
69e3c75f 3752
4ebf0ae2 3753 atomic_inc(&po->mapped);
1da177e4
LT		 3754 vma->vm_ops = &packet_mmap_ops;
3755 err = 0;
3756
3757out:
905db440 3758 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3759 return err;
3760}
1da177e4 3761
90ddc4f0 3762static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3763 .family = PF_PACKET,
3764 .owner = THIS_MODULE,
3765 .release = packet_release,
3766 .bind = packet_bind_spkt,
3767 .connect = sock_no_connect,
3768 .socketpair = sock_no_socketpair,
3769 .accept = sock_no_accept,
3770 .getname = packet_getname_spkt,
3771 .poll = datagram_poll,
3772 .ioctl = packet_ioctl,
3773 .listen = sock_no_listen,
3774 .shutdown = sock_no_shutdown,
3775 .setsockopt = sock_no_setsockopt,
3776 .getsockopt = sock_no_getsockopt,
3777 .sendmsg = packet_sendmsg_spkt,
3778 .recvmsg = packet_recvmsg,
3779 .mmap = sock_no_mmap,
3780 .sendpage = sock_no_sendpage,
3781};
1da177e4 3782
90ddc4f0 3783static const struct proto_ops packet_ops = {
1da177e4
LT		 3784 .family = PF_PACKET,
3785 .owner = THIS_MODULE,
3786 .release = packet_release,
3787 .bind = packet_bind,
3788 .connect = sock_no_connect,
3789 .socketpair = sock_no_socketpair,
3790 .accept = sock_no_accept,
1ce4f28b 3791 .getname = packet_getname,
1da177e4
LT		 3792 .poll = packet_poll,
3793 .ioctl = packet_ioctl,
3794 .listen = sock_no_listen,
3795 .shutdown = sock_no_shutdown,
3796 .setsockopt = packet_setsockopt,
3797 .getsockopt = packet_getsockopt,
3798 .sendmsg = packet_sendmsg,
3799 .recvmsg = packet_recvmsg,
3800 .mmap = packet_mmap,
3801 .sendpage = sock_no_sendpage,
3802};
3803
ec1b4cf7 3804static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3805 .family = PF_PACKET,
3806 .create = packet_create,
3807 .owner = THIS_MODULE,
3808};
3809
3810static struct notifier_block packet_netdev_notifier = {
40d4e3df 3811 .notifier_call = packet_notifier,
1da177e4
LT		 3812};
3813
3814#ifdef CONFIG_PROC_FS
1da177e4
LT		 3815
3816static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3817 __acquires(RCU)
1da177e4 3818{
e372c414 3819 struct net *net = seq_file_net(seq);
808f5114 3820
3821 rcu_read_lock();
3822 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3823}
3824
3825static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3826{
1bf40954 3827 struct net *net = seq_file_net(seq);
808f5114 3828 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3829}
3830
3831static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3832 __releases(RCU)
1da177e4 3833{
808f5114 3834 rcu_read_unlock();
1da177e4
LT		 3835}
3836
1ce4f28b 3837static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3838{
3839 if (v == SEQ_START_TOKEN)
3840 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3841 else {
b7ceabd9 3842 struct sock *s = sk_entry(v);
1da177e4
LT		 3843 const struct packet_sock *po = pkt_sk(s);
3844
3845 seq_printf(seq,
71338aa7 3846 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3847 s,
3848 atomic_read(&s->sk_refcnt),
3849 s->sk_type,
3850 ntohs(po->num),
3851 po->ifindex,
3852 po->running,
3853 atomic_read(&s->sk_rmem_alloc),
3854 sock_i_uid(s),
40d4e3df 3855 sock_i_ino(s));
1da177e4
LT		 3856 }
3857
3858 return 0;
3859}
3860
56b3d975 3861static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3862 .start = packet_seq_start,
3863 .next = packet_seq_next,
3864 .stop = packet_seq_stop,
3865 .show = packet_seq_show,
3866};
3867
3868static int packet_seq_open(struct inode *inode, struct file *file)
3869{
e372c414
DL		 3870 return seq_open_net(inode, file, &packet_seq_ops,
3871 sizeof(struct seq_net_private));
1da177e4
LT		 3872}
3873
da7071d7 3874static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3875 .owner = THIS_MODULE,
3876 .open = packet_seq_open,
3877 .read = seq_read,
3878 .llseek = seq_lseek,
e372c414 3879 .release = seq_release_net,
1da177e4
LT		 3880};
3881
3882#endif
3883
2c8c1e72 3884static int __net_init packet_net_init(struct net *net)
d12d01d6 3885{
808f5114 3886 spin_lock_init(&net->packet.sklist_lock);
2aaef4e4 3887 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6
DL		 3888
3889 if (!proc_net_fops_create(net, "packet", 0, &packet_seq_fops))
3890 return -ENOMEM;
3891
3892 return 0;
3893}
3894
2c8c1e72 3895static void __net_exit packet_net_exit(struct net *net)
d12d01d6
DL		 3896{
3897 proc_net_remove(net, "packet");
3898}
3899
3900static struct pernet_operations packet_net_ops = {
3901 .init = packet_net_init,
3902 .exit = packet_net_exit,
3903};
3904
3905
1da177e4
LT		 3906static void __exit packet_exit(void)
3907{
1da177e4 3908 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3909 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3910 sock_unregister(PF_PACKET);
3911 proto_unregister(&packet_proto);
3912}
3913
3914static int __init packet_init(void)
3915{
3916 int rc = proto_register(&packet_proto, 0);
3917
3918 if (rc != 0)
3919 goto out;
3920
3921 sock_register(&packet_family_ops);
d12d01d6 3922 register_pernet_subsys(&packet_net_ops);
1da177e4 3923 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3924out:
3925 return rc;
3926}
3927
3928module_init(packet_init);
3929module_exit(packet_exit);
3930MODULE_LICENSE("GPL");
3931MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 0.914336 seconds and 5 git commands to generate.