projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
packet: Add needed_tailroom to packet_sendmsg_spkt
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
76#include <asm/system.h>
77#include <asm/uaccess.h>
78#include <asm/ioctls.h>
79#include <asm/page.h>
a1f8e7f7 80#include <asm/cacheflush.h>
1da177e4
LT		 81#include <asm/io.h>
82#include <linux/proc_fs.h>
83#include <linux/seq_file.h>
84#include <linux/poll.h>
85#include <linux/module.h>
86#include <linux/init.h>
905db440 87#include <linux/mutex.h>
05423b24 88#include <linux/if_vlan.h>
bfd5f4a3 89#include <linux/virtio_net.h>
ed85b565 90#include <linux/errqueue.h>
614f60fa 91#include <linux/net_tstamp.h>
1da177e4
LT		 92
93#ifdef CONFIG_INET
94#include <net/inet_common.h>
95#endif
96
1da177e4
LT		 97/*
98 Assumptions:
99 - if device has no dev->hard_header routine, it adds and removes ll header
100 inside itself. In this case ll header is invisible outside of device,
101 but higher levels still should reserve dev->hard_header_len.
102 Some devices are enough clever to reallocate skb, when header
103 will not fit to reserved space (tunnel), another ones are silly
104 (PPP).
105 - packet socket receives packets with pulled ll header,
106 so that SOCK_RAW should push it back.
107
108On receive:
109-----------
110
111Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 112 mac_header -> ll header
113 data -> data
1da177e4
LT		 114
115Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 116 mac_header -> ll header
117 data -> ll header
1da177e4
LT		 118
119Incoming, dev->hard_header==NULL
b0e380b1
ACM		 120 mac_header -> UNKNOWN position. It is very likely, that it points to ll
121 header. PPP makes it, that is wrong, because introduce
db0c58f9 122 assymetry between rx and tx paths.
b0e380b1 123 data -> data
1da177e4
LT		 124
125Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 126 mac_header -> data. ll header is still not built!
127 data -> data
1da177e4
LT		 128
129Resume
130 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
131
132
133On transmit:
134------------
135
136dev->hard_header != NULL
b0e380b1
ACM		 137 mac_header -> ll header
138 data -> ll header
1da177e4
LT		 139
140dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 141 mac_header -> data
142 data -> data
1da177e4
LT		 143
144 We should set nh.raw on output to correct posistion,
145 packet classifier depends on it.
146 */
147
1da177e4
LT		 148/* Private packet socket structures. */
149
40d4e3df 150struct packet_mclist {
1da177e4
LT		 151 struct packet_mclist *next;
152 int ifindex;
153 int count;
154 unsigned short type;
155 unsigned short alen;
0fb375fb
EB		 156 unsigned char addr[MAX_ADDR_LEN];
157};
158/* identical to struct packet_mreq except it has
159 * a longer address field.
160 */
40d4e3df 161struct packet_mreq_max {
0fb375fb
EB		 162 int mr_ifindex;
163 unsigned short mr_type;
164 unsigned short mr_alen;
165 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 166};
a2efcfa0 167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171
172#define V3_ALIGNMENT (8)
173
bc59ba39 174#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 175
176#define BLK_PLUS_PRIV(sz_of_priv) \
177 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
178
179/* kbdq - kernel block descriptor queue */
bc59ba39 180struct tpacket_kbdq_core {
f6fb8f10 181 struct pgv *pkbdq;
182 unsigned int feature_req_word;
183 unsigned int hdrlen;
184 unsigned char reset_pending_on_curr_blk;
185 unsigned char delete_blk_timer;
186 unsigned short kactive_blk_num;
187 unsigned short blk_sizeof_priv;
188
189 /* last_kactive_blk_num:
190 * trick to see if user-space has caught up
191 * in order to avoid refreshing timer when every single pkt arrives.
192 */
193 unsigned short last_kactive_blk_num;
194
195 char *pkblk_start;
196 char *pkblk_end;
197 int kblk_size;
198 unsigned int knum_blocks;
199 uint64_t knxt_seq_num;
200 char *prev;
201 char *nxt_offset;
202 struct sk_buff *skb;
203
204 atomic_t blk_fill_in_prog;
205
206 /* Default is set to 8ms */
207#define DEFAULT_PRB_RETIRE_TOV (8)
208
209 unsigned short retire_blk_tov;
210 unsigned short version;
211 unsigned long tov_in_jiffies;
212
213 /* timer to retire an outstanding block */
214 struct timer_list retire_blk_timer;
215};
216
217#define PGV_FROM_VMALLOC 1
0e3125c7
NH		 218struct pgv {
219 char *buffer;
0e3125c7
NH		 220};
221
69e3c75f 222struct packet_ring_buffer {
0e3125c7 223 struct pgv *pg_vec;
69e3c75f
JB		 224 unsigned int head;
225 unsigned int frames_per_block;
226 unsigned int frame_size;
227 unsigned int frame_max;
228
229 unsigned int pg_vec_order;
230 unsigned int pg_vec_pages;
231 unsigned int pg_vec_len;
232
bc59ba39 233 struct tpacket_kbdq_core prb_bdqc;
69e3c75f
JB		 234 atomic_t pending;
235};
236
f6fb8f10 237#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
238#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
239#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
240#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
241#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
242#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
243#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
244
69e3c75f
JB		 245struct packet_sock;
246static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
1da177e4 247
f6fb8f10 248static void *packet_previous_frame(struct packet_sock *po,
249 struct packet_ring_buffer *rb,
250 int status);
251static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 252static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
253 struct tpacket_block_desc *);
254static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 255 struct packet_sock *);
bc59ba39 256static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 257 struct packet_sock *, unsigned int status);
bc59ba39 258static int prb_queue_frozen(struct tpacket_kbdq_core *);
259static void prb_open_block(struct tpacket_kbdq_core *,
260 struct tpacket_block_desc *);
f6fb8f10 261static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 262static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
263static void prb_init_blk_timer(struct packet_sock *,
264 struct tpacket_kbdq_core *,
265 void (*func) (unsigned long));
266static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
267static void prb_clear_rxhash(struct tpacket_kbdq_core *,
268 struct tpacket3_hdr *);
269static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
270 struct tpacket3_hdr *);
1da177e4
LT		 271static void packet_flush_mclist(struct sock *sk);
272
dc99f600 273struct packet_fanout;
1da177e4
LT		 274struct packet_sock {
275 /* struct sock has to be the first member of packet_sock */
276 struct sock sk;
dc99f600 277 struct packet_fanout *fanout;
1da177e4 278 struct tpacket_stats stats;
f6fb8f10 279 union tpacket_stats_u stats_u;
69e3c75f
JB		 280 struct packet_ring_buffer rx_ring;
281 struct packet_ring_buffer tx_ring;
1da177e4 282 int copy_thresh;
1da177e4 283 spinlock_t bind_lock;
905db440 284 struct mutex pg_vec_lock;
8dc41944 285 unsigned int running:1, /* prot_hook is attached*/
80feaacb 286 auxdata:1,
bfd5f4a3
SS		 287 origdev:1,
288 has_vnet_hdr:1;
1da177e4 289 int ifindex; /* bound device */
0e11c91e 290 __be16 num;
1da177e4 291 struct packet_mclist *mclist;
1da177e4 292 atomic_t mapped;
bbd6ef87
PM		 293 enum tpacket_versions tp_version;
294 unsigned int tp_hdrlen;
8913336a 295 unsigned int tp_reserve;
69e3c75f 296 unsigned int tp_loss:1;
614f60fa 297 unsigned int tp_tstamp;
94b05952 298 struct packet_type prot_hook ____cacheline_aligned_in_smp;
1da177e4
LT		 299};
300
dc99f600
DM		 301#define PACKET_FANOUT_MAX 256
302
303struct packet_fanout {
304#ifdef CONFIG_NET_NS
305 struct net *net;
306#endif
307 unsigned int num_members;
308 u16 id;
309 u8 type;
7736d33f 310 u8 defrag;
dc99f600
DM		 311 atomic_t rr_cur;
312 struct list_head list;
313 struct sock *arr[PACKET_FANOUT_MAX];
314 spinlock_t lock;
315 atomic_t sk_ref;
316 struct packet_type prot_hook ____cacheline_aligned_in_smp;
317};
318
ffbc6111
HX		 319struct packet_skb_cb {
320 unsigned int origlen;
321 union {
322 struct sockaddr_pkt pkt;
323 struct sockaddr_ll ll;
324 } sa;
325};
326
327#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 328
bc59ba39 329#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 330#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 331 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 332#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 333 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 334#define GET_NEXT_PRB_BLK_NUM(x) \
335 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
336 ((x)->kactive_blk_num+1) : 0)
337
eea49cc9 338static struct packet_sock *pkt_sk(struct sock *sk)
ce06b03e
DM		 339{
340 return (struct packet_sock *)sk;
341}
342
dc99f600
DM		 343static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
344static void __fanout_link(struct sock *sk, struct packet_sock *po);
345
ce06b03e
DM		 346/* register_prot_hook must be invoked with the po->bind_lock held,
347 * or from a context in which asynchronous accesses to the packet
348 * socket is not possible (packet_create()).
349 */
350static void register_prot_hook(struct sock *sk)
351{
352 struct packet_sock *po = pkt_sk(sk);
353 if (!po->running) {
dc99f600
DM		 354 if (po->fanout)
355 __fanout_link(sk, po);
356 else
357 dev_add_pack(&po->prot_hook);
ce06b03e
DM		 358 sock_hold(sk);
359 po->running = 1;
360 }
361}
362
363/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
364 * held. If the sync parameter is true, we will temporarily drop
365 * the po->bind_lock and do a synchronize_net to make sure no
366 * asynchronous packet processing paths still refer to the elements
367 * of po->prot_hook. If the sync parameter is false, it is the
368 * callers responsibility to take care of this.
369 */
370static void __unregister_prot_hook(struct sock *sk, bool sync)
371{
372 struct packet_sock *po = pkt_sk(sk);
373
374 po->running = 0;
dc99f600
DM		 375 if (po->fanout)
376 __fanout_unlink(sk, po);
377 else
378 __dev_remove_pack(&po->prot_hook);
ce06b03e
DM		 379 __sock_put(sk);
380
381 if (sync) {
382 spin_unlock(&po->bind_lock);
383 synchronize_net();
384 spin_lock(&po->bind_lock);
385 }
386}
387
388static void unregister_prot_hook(struct sock *sk, bool sync)
389{
390 struct packet_sock *po = pkt_sk(sk);
391
392 if (po->running)
393 __unregister_prot_hook(sk, sync);
394}
395
f6dafa95 396static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 397{
398 if (is_vmalloc_addr(addr))
399 return vmalloc_to_page(addr);
400 return virt_to_page(addr);
401}
402
69e3c75f 403static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 404{
bbd6ef87
PM		 405 union {
406 struct tpacket_hdr *h1;
407 struct tpacket2_hdr *h2;
408 void *raw;
409 } h;
1da177e4 410
69e3c75f 411 h.raw = frame;
bbd6ef87
PM		 412 switch (po->tp_version) {
413 case TPACKET_V1:
69e3c75f 414 h.h1->tp_status = status;
0af55bb5 415 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 416 break;
417 case TPACKET_V2:
69e3c75f 418 h.h2->tp_status = status;
0af55bb5 419 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 420 break;
f6fb8f10 421 case TPACKET_V3:
69e3c75f 422 default:
f6fb8f10 423 WARN(1, "TPACKET version not supported.\n");
69e3c75f 424 BUG();
bbd6ef87 425 }
69e3c75f
JB		 426
427 smp_wmb();
bbd6ef87
PM		 428}
429
69e3c75f 430static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87
PM		 431{
432 union {
433 struct tpacket_hdr *h1;
434 struct tpacket2_hdr *h2;
435 void *raw;
436 } h;
437
69e3c75f
JB		 438 smp_rmb();
439
bbd6ef87
PM		 440 h.raw = frame;
441 switch (po->tp_version) {
442 case TPACKET_V1:
0af55bb5 443 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 444 return h.h1->tp_status;
bbd6ef87 445 case TPACKET_V2:
0af55bb5 446 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 447 return h.h2->tp_status;
f6fb8f10 448 case TPACKET_V3:
69e3c75f 449 default:
f6fb8f10 450 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 451 BUG();
452 return 0;
bbd6ef87 453 }
1da177e4 454}
69e3c75f
JB		 455
456static void *packet_lookup_frame(struct packet_sock *po,
457 struct packet_ring_buffer *rb,
458 unsigned int position,
459 int status)
460{
461 unsigned int pg_vec_pos, frame_offset;
462 union {
463 struct tpacket_hdr *h1;
464 struct tpacket2_hdr *h2;
465 void *raw;
466 } h;
467
468 pg_vec_pos = position / rb->frames_per_block;
469 frame_offset = position % rb->frames_per_block;
470
0e3125c7
NH		 471 h.raw = rb->pg_vec[pg_vec_pos].buffer +
472 (frame_offset * rb->frame_size);
69e3c75f
JB		 473
474 if (status != __packet_get_status(po, h.raw))
475 return NULL;
476
477 return h.raw;
478}
479
eea49cc9 480static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 481 struct packet_ring_buffer *rb,
482 int status)
483{
484 return packet_lookup_frame(po, rb, rb->head, status);
485}
486
bc59ba39 487static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 488{
489 del_timer_sync(&pkc->retire_blk_timer);
490}
491
492static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
493 int tx_ring,
494 struct sk_buff_head *rb_queue)
495{
bc59ba39 496 struct tpacket_kbdq_core *pkc;
f6fb8f10 497
498 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
499
500 spin_lock(&rb_queue->lock);
501 pkc->delete_blk_timer = 1;
502 spin_unlock(&rb_queue->lock);
503
504 prb_del_retire_blk_timer(pkc);
505}
506
507static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 508 struct tpacket_kbdq_core *pkc,
f6fb8f10 509 void (*func) (unsigned long))
510{
511 init_timer(&pkc->retire_blk_timer);
512 pkc->retire_blk_timer.data = (long)po;
513 pkc->retire_blk_timer.function = func;
514 pkc->retire_blk_timer.expires = jiffies;
515}
516
517static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
518{
bc59ba39 519 struct tpacket_kbdq_core *pkc;
f6fb8f10 520
521 if (tx_ring)
522 BUG();
523
524 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
525 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
526}
527
528static int prb_calc_retire_blk_tmo(struct packet_sock *po,
529 int blk_size_in_bytes)
530{
531 struct net_device *dev;
532 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 533 struct ethtool_cmd ecmd;
534 int err;
f6fb8f10 535
4bc71cb9
JP		 536 rtnl_lock();
537 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
538 if (unlikely(!dev)) {
539 rtnl_unlock();
f6fb8f10 540 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 541 }
542 err = __ethtool_get_settings(dev, &ecmd);
543 rtnl_unlock();
544 if (!err) {
545 switch (ecmd.speed) {
546 case SPEED_10000:
547 msec = 1;
548 div = 10000/1000;
549 break;
550 case SPEED_1000:
551 msec = 1;
552 div = 1000/1000;
553 break;
554 /*
555 * If the link speed is so slow you don't really
556 * need to worry about perf anyways
557 */
558 case SPEED_100:
559 case SPEED_10:
560 default:
561 return DEFAULT_PRB_RETIRE_TOV;
f6fb8f10 562 }
563 }
564
565 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
566
567 if (div)
568 mbits /= div;
569
570 tmo = mbits * msec;
571
572 if (div)
573 return tmo+1;
574 return tmo;
575}
576
bc59ba39 577static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 578 union tpacket_req_u *req_u)
579{
580 p1->feature_req_word = req_u->req3.tp_feature_req_word;
581}
582
583static void init_prb_bdqc(struct packet_sock *po,
584 struct packet_ring_buffer *rb,
585 struct pgv *pg_vec,
586 union tpacket_req_u *req_u, int tx_ring)
587{
bc59ba39 588 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
589 struct tpacket_block_desc *pbd;
f6fb8f10 590
591 memset(p1, 0x0, sizeof(*p1));
592
593 p1->knxt_seq_num = 1;
594 p1->pkbdq = pg_vec;
bc59ba39 595 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
f6fb8f10 596 p1->pkblk_start = (char *)pg_vec[0].buffer;
597 p1->kblk_size = req_u->req3.tp_block_size;
598 p1->knum_blocks = req_u->req3.tp_block_nr;
599 p1->hdrlen = po->tp_hdrlen;
600 p1->version = po->tp_version;
601 p1->last_kactive_blk_num = 0;
602 po->stats_u.stats3.tp_freeze_q_cnt = 0;
603 if (req_u->req3.tp_retire_blk_tov)
604 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
605 else
606 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
607 req_u->req3.tp_block_size);
608 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
609 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
610
611 prb_init_ft_ops(p1, req_u);
612 prb_setup_retire_blk_timer(po, tx_ring);
613 prb_open_block(p1, pbd);
614}
615
616/* Do NOT update the last_blk_num first.
617 * Assumes sk_buff_head lock is held.
618 */
bc59ba39 619static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 620{
621 mod_timer(&pkc->retire_blk_timer,
622 jiffies + pkc->tov_in_jiffies);
623 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
624}
625
626/*
627 * Timer logic:
628 * 1) We refresh the timer only when we open a block.
629 * By doing this we don't waste cycles refreshing the timer
630 * on packet-by-packet basis.
631 *
632 * With a 1MB block-size, on a 1Gbps line, it will take
633 * i) ~8 ms to fill a block + ii) memcpy etc.
634 * In this cut we are not accounting for the memcpy time.
635 *
636 * So, if the user sets the 'tmo' to 10ms then the timer
637 * will never fire while the block is still getting filled
638 * (which is what we want). However, the user could choose
639 * to close a block early and that's fine.
640 *
641 * But when the timer does fire, we check whether or not to refresh it.
642 * Since the tmo granularity is in msecs, it is not too expensive
643 * to refresh the timer, lets say every '8' msecs.
644 * Either the user can set the 'tmo' or we can derive it based on
645 * a) line-speed and b) block-size.
646 * prb_calc_retire_blk_tmo() calculates the tmo.
647 *
648 */
649static void prb_retire_rx_blk_timer_expired(unsigned long data)
650{
651 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 652 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 653 unsigned int frozen;
bc59ba39 654 struct tpacket_block_desc *pbd;
f6fb8f10 655
656 spin_lock(&po->sk.sk_receive_queue.lock);
657
658 frozen = prb_queue_frozen(pkc);
659 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
660
661 if (unlikely(pkc->delete_blk_timer))
662 goto out;
663
664 /* We only need to plug the race when the block is partially filled.
665 * tpacket_rcv:
666 * lock(); increment BLOCK_NUM_PKTS; unlock()
667 * copy_bits() is in progress ...
668 * timer fires on other cpu:
669 * we can't retire the current block because copy_bits
670 * is in progress.
671 *
672 */
673 if (BLOCK_NUM_PKTS(pbd)) {
674 while (atomic_read(&pkc->blk_fill_in_prog)) {
675 /* Waiting for skb_copy_bits to finish... */
676 cpu_relax();
677 }
678 }
679
680 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
681 if (!frozen) {
682 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
683 if (!prb_dispatch_next_block(pkc, po))
684 goto refresh_timer;
685 else
686 goto out;
687 } else {
688 /* Case 1. Queue was frozen because user-space was
689 * lagging behind.
690 */
691 if (prb_curr_blk_in_use(pkc, pbd)) {
692 /*
693 * Ok, user-space is still behind.
694 * So just refresh the timer.
695 */
696 goto refresh_timer;
697 } else {
698 /* Case 2. queue was frozen,user-space caught up,
699 * now the link went idle && the timer fired.
700 * We don't have a block to close.So we open this
701 * block and restart the timer.
702 * opening a block thaws the queue,restarts timer
703 * Thawing/timer-refresh is a side effect.
704 */
705 prb_open_block(pkc, pbd);
706 goto out;
707 }
708 }
709 }
710
711refresh_timer:
712 _prb_refresh_rx_retire_blk_timer(pkc);
713
714out:
715 spin_unlock(&po->sk.sk_receive_queue.lock);
716}
717
eea49cc9 718static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 719 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 720{
721 /* Flush everything minus the block header */
722
723#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
724 u8 *start, *end;
725
726 start = (u8 *)pbd1;
727
728 /* Skip the block header(we know header WILL fit in 4K) */
729 start += PAGE_SIZE;
730
731 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
732 for (; start < end; start += PAGE_SIZE)
733 flush_dcache_page(pgv_to_page(start));
734
735 smp_wmb();
736#endif
737
738 /* Now update the block status. */
739
740 BLOCK_STATUS(pbd1) = status;
741
742 /* Flush the block header */
743
744#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
745 start = (u8 *)pbd1;
746 flush_dcache_page(pgv_to_page(start));
747
748 smp_wmb();
749#endif
750}
751
752/*
753 * Side effect:
754 *
755 * 1) flush the block
756 * 2) Increment active_blk_num
757 *
758 * Note:We DONT refresh the timer on purpose.
759 * Because almost always the next block will be opened.
760 */
bc59ba39 761static void prb_close_block(struct tpacket_kbdq_core *pkc1,
762 struct tpacket_block_desc *pbd1,
f6fb8f10 763 struct packet_sock *po, unsigned int stat)
764{
765 __u32 status = TP_STATUS_USER | stat;
766
767 struct tpacket3_hdr *last_pkt;
bc59ba39 768 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 769
770 if (po->stats.tp_drops)
771 status |= TP_STATUS_LOSING;
772
773 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
774 last_pkt->tp_next_offset = 0;
775
776 /* Get the ts of the last pkt */
777 if (BLOCK_NUM_PKTS(pbd1)) {
778 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
779 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
780 } else {
781 /* Ok, we tmo'd - so get the current time */
782 struct timespec ts;
783 getnstimeofday(&ts);
784 h1->ts_last_pkt.ts_sec = ts.tv_sec;
785 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
786 }
787
788 smp_wmb();
789
790 /* Flush the block */
791 prb_flush_block(pkc1, pbd1, status);
792
793 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
794}
795
eea49cc9 796static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 797{
798 pkc->reset_pending_on_curr_blk = 0;
799}
800
801/*
802 * Side effect of opening a block:
803 *
804 * 1) prb_queue is thawed.
805 * 2) retire_blk_timer is refreshed.
806 *
807 */
bc59ba39 808static void prb_open_block(struct tpacket_kbdq_core *pkc1,
809 struct tpacket_block_desc *pbd1)
f6fb8f10 810{
811 struct timespec ts;
bc59ba39 812 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 813
814 smp_rmb();
815
816 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd1))) {
817
818 /* We could have just memset this but we will lose the
819 * flexibility of making the priv area sticky
820 */
821 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
822 BLOCK_NUM_PKTS(pbd1) = 0;
823 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
824 getnstimeofday(&ts);
825 h1->ts_first_pkt.ts_sec = ts.tv_sec;
826 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
827 pkc1->pkblk_start = (char *)pbd1;
828 pkc1->nxt_offset = (char *)(pkc1->pkblk_start +
829 BLK_PLUS_PRIV(pkc1->blk_sizeof_priv));
830 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
831 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
832 pbd1->version = pkc1->version;
833 pkc1->prev = pkc1->nxt_offset;
834 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
835 prb_thaw_queue(pkc1);
836 _prb_refresh_rx_retire_blk_timer(pkc1);
837
838 smp_wmb();
839
840 return;
841 }
842
843 WARN(1, "ERROR block:%p is NOT FREE status:%d kactive_blk_num:%d\n",
844 pbd1, BLOCK_STATUS(pbd1), pkc1->kactive_blk_num);
845 dump_stack();
846 BUG();
847}
848
849/*
850 * Queue freeze logic:
851 * 1) Assume tp_block_nr = 8 blocks.
852 * 2) At time 't0', user opens Rx ring.
853 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
854 * 4) user-space is either sleeping or processing block '0'.
855 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
856 * it will close block-7,loop around and try to fill block '0'.
857 * call-flow:
858 * __packet_lookup_frame_in_block
859 * prb_retire_current_block()
860 * prb_dispatch_next_block()
861 * |->(BLOCK_STATUS == USER) evaluates to true
862 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
863 * 6) Now there are two cases:
864 * 6.1) Link goes idle right after the queue is frozen.
865 * But remember, the last open_block() refreshed the timer.
866 * When this timer expires,it will refresh itself so that we can
867 * re-open block-0 in near future.
868 * 6.2) Link is busy and keeps on receiving packets. This is a simple
869 * case and __packet_lookup_frame_in_block will check if block-0
870 * is free and can now be re-used.
871 */
eea49cc9 872static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 873 struct packet_sock *po)
874{
875 pkc->reset_pending_on_curr_blk = 1;
876 po->stats_u.stats3.tp_freeze_q_cnt++;
877}
878
879#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
880
881/*
882 * If the next block is free then we will dispatch it
883 * and return a good offset.
884 * Else, we will freeze the queue.
885 * So, caller must check the return value.
886 */
bc59ba39 887static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 888 struct packet_sock *po)
889{
bc59ba39 890 struct tpacket_block_desc *pbd;
f6fb8f10 891
892 smp_rmb();
893
894 /* 1. Get current block num */
895 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
896
897 /* 2. If this block is currently in_use then freeze the queue */
898 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
899 prb_freeze_queue(pkc, po);
900 return NULL;
901 }
902
903 /*
904 * 3.
905 * open this block and return the offset where the first packet
906 * needs to get stored.
907 */
908 prb_open_block(pkc, pbd);
909 return (void *)pkc->nxt_offset;
910}
911
bc59ba39 912static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 913 struct packet_sock *po, unsigned int status)
914{
bc59ba39 915 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 916
917 /* retire/close the current block */
918 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
919 /*
920 * Plug the case where copy_bits() is in progress on
921 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
922 * have space to copy the pkt in the current block and
923 * called prb_retire_current_block()
924 *
925 * We don't need to worry about the TMO case because
926 * the timer-handler already handled this case.
927 */
928 if (!(status & TP_STATUS_BLK_TMO)) {
929 while (atomic_read(&pkc->blk_fill_in_prog)) {
930 /* Waiting for skb_copy_bits to finish... */
931 cpu_relax();
932 }
933 }
934 prb_close_block(pkc, pbd, po, status);
935 return;
936 }
937
938 WARN(1, "ERROR-pbd[%d]:%p\n", pkc->kactive_blk_num, pbd);
939 dump_stack();
940 BUG();
941}
942
eea49cc9 943static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 944 struct tpacket_block_desc *pbd)
f6fb8f10 945{
946 return TP_STATUS_USER & BLOCK_STATUS(pbd);
947}
948
eea49cc9 949static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 950{
951 return pkc->reset_pending_on_curr_blk;
952}
953
eea49cc9 954static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 955{
bc59ba39 956 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 957 atomic_dec(&pkc->blk_fill_in_prog);
958}
959
eea49cc9 960static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 961 struct tpacket3_hdr *ppd)
962{
963 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
964}
965
eea49cc9 966static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 967 struct tpacket3_hdr *ppd)
968{
969 ppd->hv1.tp_rxhash = 0;
970}
971
eea49cc9 972static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 973 struct tpacket3_hdr *ppd)
974{
975 if (vlan_tx_tag_present(pkc->skb)) {
976 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
977 ppd->tp_status = TP_STATUS_VLAN_VALID;
978 } else {
979 ppd->hv1.tp_vlan_tci = ppd->tp_status = 0;
980 }
981}
982
bc59ba39 983static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 984 struct tpacket3_hdr *ppd)
985{
986 prb_fill_vlan_info(pkc, ppd);
987
988 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
989 prb_fill_rxhash(pkc, ppd);
990 else
991 prb_clear_rxhash(pkc, ppd);
992}
993
eea49cc9 994static void prb_fill_curr_block(char *curr,
bc59ba39 995 struct tpacket_kbdq_core *pkc,
996 struct tpacket_block_desc *pbd,
f6fb8f10 997 unsigned int len)
998{
999 struct tpacket3_hdr *ppd;
1000
1001 ppd = (struct tpacket3_hdr *)curr;
1002 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1003 pkc->prev = curr;
1004 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1005 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1006 BLOCK_NUM_PKTS(pbd) += 1;
1007 atomic_inc(&pkc->blk_fill_in_prog);
1008 prb_run_all_ft_ops(pkc, ppd);
1009}
1010
1011/* Assumes caller has the sk->rx_queue.lock */
1012static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1013 struct sk_buff *skb,
1014 int status,
1015 unsigned int len
1016 )
1017{
bc59ba39 1018 struct tpacket_kbdq_core *pkc;
1019 struct tpacket_block_desc *pbd;
f6fb8f10 1020 char *curr, *end;
1021
1022 pkc = GET_PBDQC_FROM_RB(((struct packet_ring_buffer *)&po->rx_ring));
1023 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1024
1025 /* Queue is frozen when user space is lagging behind */
1026 if (prb_queue_frozen(pkc)) {
1027 /*
1028 * Check if that last block which caused the queue to freeze,
1029 * is still in_use by user-space.
1030 */
1031 if (prb_curr_blk_in_use(pkc, pbd)) {
1032 /* Can't record this packet */
1033 return NULL;
1034 } else {
1035 /*
1036 * Ok, the block was released by user-space.
1037 * Now let's open that block.
1038 * opening a block also thaws the queue.
1039 * Thawing is a side effect.
1040 */
1041 prb_open_block(pkc, pbd);
1042 }
1043 }
1044
1045 smp_mb();
1046 curr = pkc->nxt_offset;
1047 pkc->skb = skb;
1048 end = (char *) ((char *)pbd + pkc->kblk_size);
1049
1050 /* first try the current block */
1051 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1052 prb_fill_curr_block(curr, pkc, pbd, len);
1053 return (void *)curr;
1054 }
1055
1056 /* Ok, close the current block */
1057 prb_retire_current_block(pkc, po, 0);
1058
1059 /* Now, try to dispatch the next block */
1060 curr = (char *)prb_dispatch_next_block(pkc, po);
1061 if (curr) {
1062 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1063 prb_fill_curr_block(curr, pkc, pbd, len);
1064 return (void *)curr;
1065 }
1066
1067 /*
1068 * No free blocks are available.user_space hasn't caught up yet.
1069 * Queue was just frozen and now this packet will get dropped.
1070 */
1071 return NULL;
1072}
1073
eea49cc9 1074static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1075 struct sk_buff *skb,
1076 int status, unsigned int len)
1077{
1078 char *curr = NULL;
1079 switch (po->tp_version) {
1080 case TPACKET_V1:
1081 case TPACKET_V2:
1082 curr = packet_lookup_frame(po, &po->rx_ring,
1083 po->rx_ring.head, status);
1084 return curr;
1085 case TPACKET_V3:
1086 return __packet_lookup_frame_in_block(po, skb, status, len);
1087 default:
1088 WARN(1, "TPACKET version not supported\n");
1089 BUG();
1090 return 0;
1091 }
1092}
1093
eea49cc9 1094static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1095 struct packet_ring_buffer *rb,
1096 unsigned int previous,
1097 int status)
1098{
bc59ba39 1099 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
1100 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, previous);
f6fb8f10 1101
1102 if (status != BLOCK_STATUS(pbd))
1103 return NULL;
1104 return pbd;
1105}
1106
eea49cc9 1107static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1108{
1109 unsigned int prev;
1110 if (rb->prb_bdqc.kactive_blk_num)
1111 prev = rb->prb_bdqc.kactive_blk_num-1;
1112 else
1113 prev = rb->prb_bdqc.knum_blocks-1;
1114 return prev;
1115}
1116
1117/* Assumes caller has held the rx_queue.lock */
eea49cc9 1118static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1119 struct packet_ring_buffer *rb,
1120 int status)
1121{
1122 unsigned int previous = prb_previous_blk_num(rb);
1123 return prb_lookup_block(po, rb, previous, status);
1124}
1125
eea49cc9 1126static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1127 struct packet_ring_buffer *rb,
1128 int status)
1129{
1130 if (po->tp_version <= TPACKET_V2)
1131 return packet_previous_frame(po, rb, status);
1132
1133 return __prb_previous_block(po, rb, status);
1134}
1135
eea49cc9 1136static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1137 struct packet_ring_buffer *rb)
1138{
1139 switch (po->tp_version) {
1140 case TPACKET_V1:
1141 case TPACKET_V2:
1142 return packet_increment_head(rb);
1143 case TPACKET_V3:
1144 default:
1145 WARN(1, "TPACKET version not supported.\n");
1146 BUG();
1147 return;
1148 }
1149}
1150
eea49cc9 1151static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1152 struct packet_ring_buffer *rb,
1153 int status)
1154{
1155 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1156 return packet_lookup_frame(po, rb, previous, status);
1157}
1158
eea49cc9 1159static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1160{
1161 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1162}
1163
1da177e4
LT		 1164static void packet_sock_destruct(struct sock *sk)
1165{
ed85b565
RC		 1166 skb_queue_purge(&sk->sk_error_queue);
1167
547b792c
IJ		 1168 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1169 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1170
1171 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1172 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1173 return;
1174 }
1175
17ab56a2 1176 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1177}
1178
dc99f600
DM		 1179static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1180{
1181 int x = atomic_read(&f->rr_cur) + 1;
1182
1183 if (x >= num)
1184 x = 0;
1185
1186 return x;
1187}
1188
1189static struct sock *fanout_demux_hash(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
1190{
1191 u32 idx, hash = skb->rxhash;
1192
1193 idx = ((u64)hash * num) >> 32;
1194
1195 return f->arr[idx];
1196}
1197
1198static struct sock *fanout_demux_lb(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
1199{
1200 int cur, old;
1201
1202 cur = atomic_read(&f->rr_cur);
1203 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1204 fanout_rr_next(f, num))) != cur)
1205 cur = old;
1206 return f->arr[cur];
1207}
1208
95ec3eb4
DM		 1209static struct sock *fanout_demux_cpu(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
1210{
1211 unsigned int cpu = smp_processor_id();
1212
1213 return f->arr[cpu % num];
1214}
1215
95ec3eb4
DM		 1216static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1217 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1218{
1219 struct packet_fanout *f = pt->af_packet_priv;
1220 unsigned int num = f->num_members;
1221 struct packet_sock *po;
1222 struct sock *sk;
1223
1224 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1225 !num) {
1226 kfree_skb(skb);
1227 return 0;
1228 }
1229
95ec3eb4
DM		 1230 switch (f->type) {
1231 case PACKET_FANOUT_HASH:
1232 default:
1233 if (f->defrag) {
bc416d97 1234 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1235 if (!skb)
1236 return 0;
1237 }
1238 skb_get_rxhash(skb);
1239 sk = fanout_demux_hash(f, skb, num);
1240 break;
1241 case PACKET_FANOUT_LB:
1242 sk = fanout_demux_lb(f, skb, num);
1243 break;
1244 case PACKET_FANOUT_CPU:
1245 sk = fanout_demux_cpu(f, skb, num);
1246 break;
dc99f600
DM		 1247 }
1248
dc99f600
DM		 1249 po = pkt_sk(sk);
1250
1251 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1252}
1253
1254static DEFINE_MUTEX(fanout_mutex);
1255static LIST_HEAD(fanout_list);
1256
1257static void __fanout_link(struct sock *sk, struct packet_sock *po)
1258{
1259 struct packet_fanout *f = po->fanout;
1260
1261 spin_lock(&f->lock);
1262 f->arr[f->num_members] = sk;
1263 smp_wmb();
1264 f->num_members++;
1265 spin_unlock(&f->lock);
1266}
1267
1268static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1269{
1270 struct packet_fanout *f = po->fanout;
1271 int i;
1272
1273 spin_lock(&f->lock);
1274 for (i = 0; i < f->num_members; i++) {
1275 if (f->arr[i] == sk)
1276 break;
1277 }
1278 BUG_ON(i >= f->num_members);
1279 f->arr[i] = f->arr[f->num_members - 1];
1280 f->num_members--;
1281 spin_unlock(&f->lock);
1282}
1283
7736d33f 1284static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1285{
1286 struct packet_sock *po = pkt_sk(sk);
1287 struct packet_fanout *f, *match;
7736d33f
DM		 1288 u8 type = type_flags & 0xff;
1289 u8 defrag = (type_flags & PACKET_FANOUT_FLAG_DEFRAG) ? 1 : 0;
dc99f600
DM		 1290 int err;
1291
1292 switch (type) {
1293 case PACKET_FANOUT_HASH:
1294 case PACKET_FANOUT_LB:
95ec3eb4 1295 case PACKET_FANOUT_CPU:
dc99f600
DM		 1296 break;
1297 default:
1298 return -EINVAL;
1299 }
1300
1301 if (!po->running)
1302 return -EINVAL;
1303
1304 if (po->fanout)
1305 return -EALREADY;
1306
1307 mutex_lock(&fanout_mutex);
1308 match = NULL;
1309 list_for_each_entry(f, &fanout_list, list) {
1310 if (f->id == id &&
1311 read_pnet(&f->net) == sock_net(sk)) {
1312 match = f;
1313 break;
1314 }
1315 }
afe62c68 1316 err = -EINVAL;
7736d33f 1317 if (match && match->defrag != defrag)
afe62c68 1318 goto out;
dc99f600 1319 if (!match) {
afe62c68 1320 err = -ENOMEM;
dc99f600 1321 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1322 if (!match)
1323 goto out;
1324 write_pnet(&match->net, sock_net(sk));
1325 match->id = id;
1326 match->type = type;
1327 match->defrag = defrag;
1328 atomic_set(&match->rr_cur, 0);
1329 INIT_LIST_HEAD(&match->list);
1330 spin_lock_init(&match->lock);
1331 atomic_set(&match->sk_ref, 0);
1332 match->prot_hook.type = po->prot_hook.type;
1333 match->prot_hook.dev = po->prot_hook.dev;
1334 match->prot_hook.func = packet_rcv_fanout;
1335 match->prot_hook.af_packet_priv = match;
1336 dev_add_pack(&match->prot_hook);
1337 list_add(&match->list, &fanout_list);
dc99f600 1338 }
afe62c68
ED		 1339 err = -EINVAL;
1340 if (match->type == type &&
1341 match->prot_hook.type == po->prot_hook.type &&
1342 match->prot_hook.dev == po->prot_hook.dev) {
1343 err = -ENOSPC;
1344 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1345 __dev_remove_pack(&po->prot_hook);
1346 po->fanout = match;
1347 atomic_inc(&match->sk_ref);
1348 __fanout_link(sk, po);
1349 err = 0;
dc99f600
DM		 1350 }
1351 }
afe62c68 1352out:
dc99f600
DM		 1353 mutex_unlock(&fanout_mutex);
1354 return err;
1355}
1356
1357static void fanout_release(struct sock *sk)
1358{
1359 struct packet_sock *po = pkt_sk(sk);
1360 struct packet_fanout *f;
1361
1362 f = po->fanout;
1363 if (!f)
1364 return;
1365
1366 po->fanout = NULL;
1367
1368 mutex_lock(&fanout_mutex);
1369 if (atomic_dec_and_test(&f->sk_ref)) {
1370 list_del(&f->list);
1371 dev_remove_pack(&f->prot_hook);
1372 kfree(f);
1373 }
1374 mutex_unlock(&fanout_mutex);
1375}
1da177e4 1376
90ddc4f0 1377static const struct proto_ops packet_ops;
1da177e4 1378
90ddc4f0 1379static const struct proto_ops packet_ops_spkt;
1da177e4 1380
40d4e3df
ED		 1381static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1382 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1383{
1384 struct sock *sk;
1385 struct sockaddr_pkt *spkt;
1386
1387 /*
1388 * When we registered the protocol we saved the socket in the data
1389 * field for just this event.
1390 */
1391
1392 sk = pt->af_packet_priv;
1ce4f28b 1393
1da177e4
LT		 1394 /*
1395 * Yank back the headers [hope the device set this
1396 * right or kerboom...]
1397 *
1398 * Incoming packets have ll header pulled,
1399 * push it back.
1400 *
98e399f8 1401 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1402 * so that this procedure is noop.
1403 */
1404
1405 if (skb->pkt_type == PACKET_LOOPBACK)
1406 goto out;
1407
09ad9bc7 1408 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1409 goto out;
1410
40d4e3df
ED		 1411 skb = skb_share_check(skb, GFP_ATOMIC);
1412 if (skb == NULL)
1da177e4
LT		 1413 goto oom;
1414
1415 /* drop any routing info */
adf30907 1416 skb_dst_drop(skb);
1da177e4 1417
84531c24
PO		 1418 /* drop conntrack reference */
1419 nf_reset(skb);
1420
ffbc6111 1421 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1422
98e399f8 1423 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1424
1425 /*
1426 * The SOCK_PACKET socket receives _all_ frames.
1427 */
1428
1429 spkt->spkt_family = dev->type;
1430 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1431 spkt->spkt_protocol = skb->protocol;
1432
1433 /*
1434 * Charge the memory to the socket. This is done specifically
1435 * to prevent sockets using all the memory up.
1436 */
1437
40d4e3df 1438 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1439 return 0;
1440
1441out:
1442 kfree_skb(skb);
1443oom:
1444 return 0;
1445}
1446
1447
1448/*
1449 * Output a raw packet to a device layer. This bypasses all the other
1450 * protocol layers and you must therefore supply it with a complete frame
1451 */
1ce4f28b 1452
1da177e4
LT		 1453static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1454 struct msghdr *msg, size_t len)
1455{
1456 struct sock *sk = sock->sk;
40d4e3df 1457 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1458 struct sk_buff *skb = NULL;
1da177e4 1459 struct net_device *dev;
40d4e3df 1460 __be16 proto = 0;
1da177e4 1461 int err;
1ce4f28b 1462
1da177e4 1463 /*
1ce4f28b 1464 * Get and verify the address.
1da177e4
LT		 1465 */
1466
40d4e3df 1467 if (saddr) {
1da177e4 1468 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1469 return -EINVAL;
1470 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1471 proto = saddr->spkt_protocol;
1472 } else
1473 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1474
1475 /*
1ce4f28b 1476 * Find the device first to size check it
1da177e4
LT		 1477 */
1478
1479 saddr->spkt_device[13] = 0;
1a35ca80 1480retry:
654d1f8a
ED		 1481 rcu_read_lock();
1482 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1483 err = -ENODEV;
1484 if (dev == NULL)
1485 goto out_unlock;
1ce4f28b 1486
d5e76b0a
DM		 1487 err = -ENETDOWN;
1488 if (!(dev->flags & IFF_UP))
1489 goto out_unlock;
1490
1da177e4 1491 /*
40d4e3df
ED		 1492 * You may not queue a frame bigger than the mtu. This is the lowest level
1493 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1494 */
1ce4f28b 1495
1da177e4 1496 err = -EMSGSIZE;
57f89bfa 1497 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN)
1da177e4
LT		 1498 goto out_unlock;
1499
1a35ca80
ED		 1500 if (!skb) {
1501 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1502 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1503 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1504
1505 rcu_read_unlock();
4ce40912 1506 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1507 if (skb == NULL)
1508 return -ENOBUFS;
1509 /* FIXME: Save some space for broken drivers that write a hard
1510 * header at transmission time by themselves. PPP is the notable
1511 * one here. This should really be fixed at the driver level.
1512 */
1513 skb_reserve(skb, reserved);
1514 skb_reset_network_header(skb);
1515
1516 /* Try to align data part correctly */
1517 if (hhlen) {
1518 skb->data -= hhlen;
1519 skb->tail -= hhlen;
1520 if (len < hhlen)
1521 skb_reset_network_header(skb);
1522 }
1523 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1524 if (err)
1525 goto out_free;
1526 goto retry;
1da177e4
LT		 1527 }
1528
57f89bfa
BG		 1529 if (len > (dev->mtu + dev->hard_header_len)) {
1530 /* Earlier code assumed this would be a VLAN pkt,
1531 * double-check this now that we have the actual
1532 * packet in hand.
1533 */
1534 struct ethhdr *ehdr;
1535 skb_reset_mac_header(skb);
1536 ehdr = eth_hdr(skb);
1537 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1538 err = -EMSGSIZE;
1539 goto out_unlock;
1540 }
1541 }
1a35ca80 1542
1da177e4
LT		 1543 skb->protocol = proto;
1544 skb->dev = dev;
1545 skb->priority = sk->sk_priority;
2d37a186 1546 skb->mark = sk->sk_mark;
2244d07b 1547 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
ed85b565
RC		 1548 if (err < 0)
1549 goto out_unlock;
1da177e4
LT		 1550
1551 dev_queue_xmit(skb);
654d1f8a 1552 rcu_read_unlock();
40d4e3df 1553 return len;
1da177e4 1554
1da177e4 1555out_unlock:
654d1f8a 1556 rcu_read_unlock();
1a35ca80
ED		 1557out_free:
1558 kfree_skb(skb);
1da177e4
LT		 1559 return err;
1560}
1da177e4 1561
eea49cc9 1562static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1563 const struct sock *sk,
dbcb5855 1564 unsigned int res)
1da177e4
LT		 1565{
1566 struct sk_filter *filter;
fda9ef5d 1567
80f8f102
ED		 1568 rcu_read_lock();
1569 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1570 if (filter != NULL)
0a14842f 1571 res = SK_RUN_FILTER(filter, skb);
80f8f102 1572 rcu_read_unlock();
1da177e4 1573
dbcb5855 1574 return res;
1da177e4
LT		 1575}
1576
1577/*
62ab0812
ED		 1578 * This function makes lazy skb cloning in hope that most of packets
1579 * are discarded by BPF.
1580 *
1581 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1582 * and skb->cb are mangled. It works because (and until) packets
1583 * falling here are owned by current CPU. Output packets are cloned
1584 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1585 * sequencially, so that if we return skb to original state on exit,
1586 * we will not harm anyone.
1da177e4
LT		 1587 */
1588
40d4e3df
ED		 1589static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1590 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1591{
1592 struct sock *sk;
1593 struct sockaddr_ll *sll;
1594 struct packet_sock *po;
40d4e3df 1595 u8 *skb_head = skb->data;
1da177e4 1596 int skb_len = skb->len;
dbcb5855 1597 unsigned int snaplen, res;
1da177e4
LT		 1598
1599 if (skb->pkt_type == PACKET_LOOPBACK)
1600 goto drop;
1601
1602 sk = pt->af_packet_priv;
1603 po = pkt_sk(sk);
1604
09ad9bc7 1605 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1606 goto drop;
1607
1da177e4
LT		 1608 skb->dev = dev;
1609
3b04ddde 1610 if (dev->header_ops) {
1da177e4 1611 /* The device has an explicit notion of ll header,
62ab0812
ED		 1612 * exported to higher levels.
1613 *
1614 * Otherwise, the device hides details of its frame
1615 * structure, so that corresponding packet head is
1616 * never delivered to user.
1da177e4
LT		 1617 */
1618 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1619 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1620 else if (skb->pkt_type == PACKET_OUTGOING) {
1621 /* Special case: outgoing packets have ll header at head */
bbe735e4 1622 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1623 }
1624 }
1625
1626 snaplen = skb->len;
1627
dbcb5855
DM		 1628 res = run_filter(skb, sk, snaplen);
1629 if (!res)
fda9ef5d 1630 goto drop_n_restore;
dbcb5855
DM		 1631 if (snaplen > res)
1632 snaplen = res;
1da177e4
LT		 1633
1634 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
1635 (unsigned)sk->sk_rcvbuf)
1636 goto drop_n_acct;
1637
1638 if (skb_shared(skb)) {
1639 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1640 if (nskb == NULL)
1641 goto drop_n_acct;
1642
1643 if (skb_head != skb->data) {
1644 skb->data = skb_head;
1645 skb->len = skb_len;
1646 }
1647 kfree_skb(skb);
1648 skb = nskb;
1649 }
1650
ffbc6111
HX		 1651 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1652 sizeof(skb->cb));
1653
1654 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1655 sll->sll_family = AF_PACKET;
1656 sll->sll_hatype = dev->type;
1657 sll->sll_protocol = skb->protocol;
1658 sll->sll_pkttype = skb->pkt_type;
8032b464 1659 if (unlikely(po->origdev))
80feaacb
PWJ		 1660 sll->sll_ifindex = orig_dev->ifindex;
1661 else
1662 sll->sll_ifindex = dev->ifindex;
1da177e4 1663
b95cce35 1664 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1665
ffbc6111 1666 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1667
1da177e4
LT		 1668 if (pskb_trim(skb, snaplen))
1669 goto drop_n_acct;
1670
1671 skb_set_owner_r(skb, sk);
1672 skb->dev = NULL;
adf30907 1673 skb_dst_drop(skb);
1da177e4 1674
84531c24
PO		 1675 /* drop conntrack reference */
1676 nf_reset(skb);
1677
1da177e4
LT		 1678 spin_lock(&sk->sk_receive_queue.lock);
1679 po->stats.tp_packets++;
3b885787 1680 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1681 __skb_queue_tail(&sk->sk_receive_queue, skb);
1682 spin_unlock(&sk->sk_receive_queue.lock);
1683 sk->sk_data_ready(sk, skb->len);
1684 return 0;
1685
1686drop_n_acct:
7091fbd8
WB		 1687 spin_lock(&sk->sk_receive_queue.lock);
1688 po->stats.tp_drops++;
1689 atomic_inc(&sk->sk_drops);
1690 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1691
1692drop_n_restore:
1693 if (skb_head != skb->data && skb_shared(skb)) {
1694 skb->data = skb_head;
1695 skb->len = skb_len;
1696 }
1697drop:
ead2ceb0 1698 consume_skb(skb);
1da177e4
LT		 1699 return 0;
1700}
1701
40d4e3df
ED		 1702static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1703 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1704{
1705 struct sock *sk;
1706 struct packet_sock *po;
1707 struct sockaddr_ll *sll;
bbd6ef87
PM		 1708 union {
1709 struct tpacket_hdr *h1;
1710 struct tpacket2_hdr *h2;
f6fb8f10 1711 struct tpacket3_hdr *h3;
bbd6ef87
PM		 1712 void *raw;
1713 } h;
40d4e3df 1714 u8 *skb_head = skb->data;
1da177e4 1715 int skb_len = skb->len;
dbcb5855 1716 unsigned int snaplen, res;
f6fb8f10 1717 unsigned long status = TP_STATUS_USER;
bbd6ef87 1718 unsigned short macoff, netoff, hdrlen;
1da177e4 1719 struct sk_buff *copy_skb = NULL;
b7aa0bf7 1720 struct timeval tv;
bbd6ef87 1721 struct timespec ts;
614f60fa 1722 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
1da177e4
LT		 1723
1724 if (skb->pkt_type == PACKET_LOOPBACK)
1725 goto drop;
1726
1727 sk = pt->af_packet_priv;
1728 po = pkt_sk(sk);
1729
09ad9bc7 1730 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1731 goto drop;
1732
3b04ddde 1733 if (dev->header_ops) {
1da177e4 1734 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1735 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1736 else if (skb->pkt_type == PACKET_OUTGOING) {
1737 /* Special case: outgoing packets have ll header at head */
bbe735e4 1738 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1739 }
1740 }
1741
8dc41944
HX		 1742 if (skb->ip_summed == CHECKSUM_PARTIAL)
1743 status |= TP_STATUS_CSUMNOTREADY;
1744
1da177e4
LT		 1745 snaplen = skb->len;
1746
dbcb5855
DM		 1747 res = run_filter(skb, sk, snaplen);
1748 if (!res)
fda9ef5d 1749 goto drop_n_restore;
dbcb5855
DM		 1750 if (snaplen > res)
1751 snaplen = res;
1da177e4
LT		 1752
1753 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1754 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1755 po->tp_reserve;
1da177e4 1756 } else {
bbe735e4 1757 unsigned maclen = skb_network_offset(skb);
bbd6ef87 1758 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1759 (maclen < 16 ? 16 : maclen)) +
1760 po->tp_reserve;
1da177e4
LT		 1761 macoff = netoff - maclen;
1762 }
f6fb8f10 1763 if (po->tp_version <= TPACKET_V2) {
1764 if (macoff + snaplen > po->rx_ring.frame_size) {
1765 if (po->copy_thresh &&
1766 atomic_read(&sk->sk_rmem_alloc) + skb->truesize
1767 < (unsigned)sk->sk_rcvbuf) {
1768 if (skb_shared(skb)) {
1769 copy_skb = skb_clone(skb, GFP_ATOMIC);
1770 } else {
1771 copy_skb = skb_get(skb);
1772 skb_head = skb->data;
1773 }
1774 if (copy_skb)
1775 skb_set_owner_r(copy_skb, sk);
1da177e4 1776 }
f6fb8f10 1777 snaplen = po->rx_ring.frame_size - macoff;
1778 if ((int)snaplen < 0)
1779 snaplen = 0;
1da177e4 1780 }
1da177e4 1781 }
1da177e4 1782 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1783 h.raw = packet_current_rx_frame(po, skb,
1784 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1785 if (!h.raw)
1da177e4 1786 goto ring_is_full;
f6fb8f10 1787 if (po->tp_version <= TPACKET_V2) {
1788 packet_increment_rx_head(po, &po->rx_ring);
1789 /*
1790 * LOSING will be reported till you read the stats,
1791 * because it's COR - Clear On Read.
1792 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1793 * at packet level.
1794 */
1795 if (po->stats.tp_drops)
1796 status |= TP_STATUS_LOSING;
1797 }
1da177e4
LT		 1798 po->stats.tp_packets++;
1799 if (copy_skb) {
1800 status |= TP_STATUS_COPY;
1801 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1802 }
1da177e4
LT		 1803 spin_unlock(&sk->sk_receive_queue.lock);
1804
bbd6ef87 1805 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
1da177e4 1806
bbd6ef87
PM		 1807 switch (po->tp_version) {
1808 case TPACKET_V1:
1809 h.h1->tp_len = skb->len;
1810 h.h1->tp_snaplen = snaplen;
1811 h.h1->tp_mac = macoff;
1812 h.h1->tp_net = netoff;
614f60fa
SM		 1813 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1814 && shhwtstamps->syststamp.tv64)
1815 tv = ktime_to_timeval(shhwtstamps->syststamp);
1816 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1817 && shhwtstamps->hwtstamp.tv64)
1818 tv = ktime_to_timeval(shhwtstamps->hwtstamp);
1819 else if (skb->tstamp.tv64)
bbd6ef87
PM		 1820 tv = ktime_to_timeval(skb->tstamp);
1821 else
1822 do_gettimeofday(&tv);
1823 h.h1->tp_sec = tv.tv_sec;
1824 h.h1->tp_usec = tv.tv_usec;
1825 hdrlen = sizeof(*h.h1);
1826 break;
1827 case TPACKET_V2:
1828 h.h2->tp_len = skb->len;
1829 h.h2->tp_snaplen = snaplen;
1830 h.h2->tp_mac = macoff;
1831 h.h2->tp_net = netoff;
614f60fa
SM		 1832 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1833 && shhwtstamps->syststamp.tv64)
1834 ts = ktime_to_timespec(shhwtstamps->syststamp);
1835 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1836 && shhwtstamps->hwtstamp.tv64)
1837 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
1838 else if (skb->tstamp.tv64)
bbd6ef87
PM		 1839 ts = ktime_to_timespec(skb->tstamp);
1840 else
1841 getnstimeofday(&ts);
1842 h.h2->tp_sec = ts.tv_sec;
1843 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1844 if (vlan_tx_tag_present(skb)) {
1845 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1846 status |= TP_STATUS_VLAN_VALID;
1847 } else {
1848 h.h2->tp_vlan_tci = 0;
1849 }
13fcb7bd 1850 h.h2->tp_padding = 0;
bbd6ef87
PM		 1851 hdrlen = sizeof(*h.h2);
1852 break;
f6fb8f10 1853 case TPACKET_V3:
1854 /* tp_nxt_offset,vlan are already populated above.
1855 * So DONT clear those fields here
1856 */
1857 h.h3->tp_status |= status;
1858 h.h3->tp_len = skb->len;
1859 h.h3->tp_snaplen = snaplen;
1860 h.h3->tp_mac = macoff;
1861 h.h3->tp_net = netoff;
1862 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE)
1863 && shhwtstamps->syststamp.tv64)
1864 ts = ktime_to_timespec(shhwtstamps->syststamp);
1865 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE)
1866 && shhwtstamps->hwtstamp.tv64)
1867 ts = ktime_to_timespec(shhwtstamps->hwtstamp);
1868 else if (skb->tstamp.tv64)
1869 ts = ktime_to_timespec(skb->tstamp);
1870 else
1871 getnstimeofday(&ts);
1872 h.h3->tp_sec = ts.tv_sec;
1873 h.h3->tp_nsec = ts.tv_nsec;
1874 hdrlen = sizeof(*h.h3);
1875 break;
bbd6ef87
PM		 1876 default:
1877 BUG();
1878 }
1da177e4 1879
bbd6ef87 1880 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1881 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1882 sll->sll_family = AF_PACKET;
1883 sll->sll_hatype = dev->type;
1884 sll->sll_protocol = skb->protocol;
1885 sll->sll_pkttype = skb->pkt_type;
8032b464 1886 if (unlikely(po->origdev))
80feaacb
PWJ		 1887 sll->sll_ifindex = orig_dev->ifindex;
1888 else
1889 sll->sll_ifindex = dev->ifindex;
1da177e4 1890
e16aa207 1891 smp_mb();
f6dafa95 1892#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1893 {
0af55bb5
CG		 1894 u8 *start, *end;
1895
f6fb8f10 1896 if (po->tp_version <= TPACKET_V2) {
1897 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1898 + macoff + snaplen);
1899 for (start = h.raw; start < end; start += PAGE_SIZE)
1900 flush_dcache_page(pgv_to_page(start));
1901 }
cc9f01b2 1902 smp_wmb();
1da177e4 1903 }
f6dafa95 1904#endif
f6fb8f10 1905 if (po->tp_version <= TPACKET_V2)
1906 __packet_set_status(po, h.raw, status);
1907 else
1908 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1909
1910 sk->sk_data_ready(sk, 0);
1911
1912drop_n_restore:
1913 if (skb_head != skb->data && skb_shared(skb)) {
1914 skb->data = skb_head;
1915 skb->len = skb_len;
1916 }
1917drop:
1ce4f28b 1918 kfree_skb(skb);
1da177e4
LT		 1919 return 0;
1920
1921ring_is_full:
1922 po->stats.tp_drops++;
1923 spin_unlock(&sk->sk_receive_queue.lock);
1924
1925 sk->sk_data_ready(sk, 0);
acb5d75b 1926 kfree_skb(copy_skb);
1da177e4
LT		 1927 goto drop_n_restore;
1928}
1929
69e3c75f
JB		 1930static void tpacket_destruct_skb(struct sk_buff *skb)
1931{
1932 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1933 void *ph;
1da177e4 1934
69e3c75f
JB		 1935 if (likely(po->tx_ring.pg_vec)) {
1936 ph = skb_shinfo(skb)->destructor_arg;
1937 BUG_ON(__packet_get_status(po, ph) != TP_STATUS_SENDING);
1938 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1939 atomic_dec(&po->tx_ring.pending);
1940 __packet_set_status(po, ph, TP_STATUS_AVAILABLE);
1941 }
1942
1943 sock_wfree(skb);
1944}
1945
40d4e3df
ED		 1946static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1947 void *frame, struct net_device *dev, int size_max,
ae641949 1948 __be16 proto, unsigned char *addr, int hlen)
69e3c75f
JB		 1949{
1950 union {
1951 struct tpacket_hdr *h1;
1952 struct tpacket2_hdr *h2;
1953 void *raw;
1954 } ph;
1955 int to_write, offset, len, tp_len, nr_frags, len_max;
1956 struct socket *sock = po->sk.sk_socket;
1957 struct page *page;
1958 void *data;
1959 int err;
1960
1961 ph.raw = frame;
1962
1963 skb->protocol = proto;
1964 skb->dev = dev;
1965 skb->priority = po->sk.sk_priority;
2d37a186 1966 skb->mark = po->sk.sk_mark;
69e3c75f
JB		 1967 skb_shinfo(skb)->destructor_arg = ph.raw;
1968
1969 switch (po->tp_version) {
1970 case TPACKET_V2:
1971 tp_len = ph.h2->tp_len;
1972 break;
1973 default:
1974 tp_len = ph.h1->tp_len;
1975 break;
1976 }
1977 if (unlikely(tp_len > size_max)) {
40d4e3df 1978 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
69e3c75f
JB		 1979 return -EMSGSIZE;
1980 }
1981
ae641949 1982 skb_reserve(skb, hlen);
69e3c75f
JB		 1983 skb_reset_network_header(skb);
1984
1985 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
1986 to_write = tp_len;
1987
1988 if (sock->type == SOCK_DGRAM) {
1989 err = dev_hard_header(skb, dev, ntohs(proto), addr,
1990 NULL, tp_len);
1991 if (unlikely(err < 0))
1992 return -EINVAL;
40d4e3df 1993 } else if (dev->hard_header_len) {
69e3c75f
JB		 1994 /* net device doesn't like empty head */
1995 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 1996 pr_err("packet size is too short (%d < %d)\n",
1997 tp_len, dev->hard_header_len);
69e3c75f
JB		 1998 return -EINVAL;
1999 }
2000
2001 skb_push(skb, dev->hard_header_len);
2002 err = skb_store_bits(skb, 0, data,
2003 dev->hard_header_len);
2004 if (unlikely(err))
2005 return err;
2006
2007 data += dev->hard_header_len;
2008 to_write -= dev->hard_header_len;
2009 }
2010
2011 err = -EFAULT;
69e3c75f
JB		 2012 offset = offset_in_page(data);
2013 len_max = PAGE_SIZE - offset;
2014 len = ((to_write > len_max) ? len_max : to_write);
2015
2016 skb->data_len = to_write;
2017 skb->len += to_write;
2018 skb->truesize += to_write;
2019 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2020
2021 while (likely(to_write)) {
2022 nr_frags = skb_shinfo(skb)->nr_frags;
2023
2024 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2025 pr_err("Packet exceed the number of skb frags(%lu)\n",
2026 MAX_SKB_FRAGS);
69e3c75f
JB		 2027 return -EFAULT;
2028 }
2029
0af55bb5
CG		 2030 page = pgv_to_page(data);
2031 data += len;
69e3c75f
JB		 2032 flush_dcache_page(page);
2033 get_page(page);
0af55bb5 2034 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2035 to_write -= len;
2036 offset = 0;
2037 len_max = PAGE_SIZE;
2038 len = ((to_write > len_max) ? len_max : to_write);
2039 }
2040
2041 return tp_len;
2042}
2043
2044static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2045{
69e3c75f
JB		 2046 struct sk_buff *skb;
2047 struct net_device *dev;
2048 __be16 proto;
827d9780
BG		 2049 bool need_rls_dev = false;
2050 int err, reserve = 0;
40d4e3df
ED		 2051 void *ph;
2052 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2053 int tp_len, size_max;
2054 unsigned char *addr;
2055 int len_sum = 0;
2056 int status = 0;
ae641949 2057 int hlen, tlen;
69e3c75f 2058
69e3c75f
JB		 2059 mutex_lock(&po->pg_vec_lock);
2060
2061 err = -EBUSY;
2062 if (saddr == NULL) {
827d9780 2063 dev = po->prot_hook.dev;
69e3c75f
JB		 2064 proto = po->num;
2065 addr = NULL;
2066 } else {
2067 err = -EINVAL;
2068 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2069 goto out;
2070 if (msg->msg_namelen < (saddr->sll_halen
2071 + offsetof(struct sockaddr_ll,
2072 sll_addr)))
2073 goto out;
69e3c75f
JB		 2074 proto = saddr->sll_protocol;
2075 addr = saddr->sll_addr;
827d9780
BG		 2076 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2077 need_rls_dev = true;
69e3c75f
JB		 2078 }
2079
69e3c75f
JB		 2080 err = -ENXIO;
2081 if (unlikely(dev == NULL))
2082 goto out;
2083
2084 reserve = dev->hard_header_len;
2085
2086 err = -ENETDOWN;
2087 if (unlikely(!(dev->flags & IFF_UP)))
2088 goto out_put;
2089
2090 size_max = po->tx_ring.frame_size
b5dd884e 2091 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f
JB		 2092
2093 if (size_max > dev->mtu + reserve)
2094 size_max = dev->mtu + reserve;
2095
2096 do {
2097 ph = packet_current_frame(po, &po->tx_ring,
2098 TP_STATUS_SEND_REQUEST);
2099
2100 if (unlikely(ph == NULL)) {
2101 schedule();
2102 continue;
2103 }
2104
2105 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2106 hlen = LL_RESERVED_SPACE(dev);
2107 tlen = dev->needed_tailroom;
69e3c75f 2108 skb = sock_alloc_send_skb(&po->sk,
ae641949 2109 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2110 0, &err);
2111
2112 if (unlikely(skb == NULL))
2113 goto out_status;
2114
2115 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2116 addr, hlen);
69e3c75f
JB		 2117
2118 if (unlikely(tp_len < 0)) {
2119 if (po->tp_loss) {
2120 __packet_set_status(po, ph,
2121 TP_STATUS_AVAILABLE);
2122 packet_increment_head(&po->tx_ring);
2123 kfree_skb(skb);
2124 continue;
2125 } else {
2126 status = TP_STATUS_WRONG_FORMAT;
2127 err = tp_len;
2128 goto out_status;
2129 }
2130 }
2131
2132 skb->destructor = tpacket_destruct_skb;
2133 __packet_set_status(po, ph, TP_STATUS_SENDING);
2134 atomic_inc(&po->tx_ring.pending);
2135
2136 status = TP_STATUS_SEND_REQUEST;
2137 err = dev_queue_xmit(skb);
eb70df13
JP		 2138 if (unlikely(err > 0)) {
2139 err = net_xmit_errno(err);
2140 if (err && __packet_get_status(po, ph) ==
2141 TP_STATUS_AVAILABLE) {
2142 /* skb was destructed already */
2143 skb = NULL;
2144 goto out_status;
2145 }
2146 /*
2147 * skb was dropped but not destructed yet;
2148 * let's treat it like congestion or err < 0
2149 */
2150 err = 0;
2151 }
69e3c75f
JB		 2152 packet_increment_head(&po->tx_ring);
2153 len_sum += tp_len;
f64f9e71
JP		 2154 } while (likely((ph != NULL) ||
2155 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2156 (atomic_read(&po->tx_ring.pending))))
2157 );
69e3c75f
JB		 2158
2159 err = len_sum;
2160 goto out_put;
2161
69e3c75f
JB		 2162out_status:
2163 __packet_set_status(po, ph, status);
2164 kfree_skb(skb);
2165out_put:
827d9780
BG		 2166 if (need_rls_dev)
2167 dev_put(dev);
69e3c75f
JB		 2168out:
2169 mutex_unlock(&po->pg_vec_lock);
2170 return err;
2171}
69e3c75f 2172
eea49cc9
OJ		 2173static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2174 size_t reserve, size_t len,
2175 size_t linear, int noblock,
2176 int *err)
bfd5f4a3
SS		 2177{
2178 struct sk_buff *skb;
2179
2180 /* Under a page? Don't bother with paged skb. */
2181 if (prepad + len < PAGE_SIZE || !linear)
2182 linear = len;
2183
2184 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2185 err);
2186 if (!skb)
2187 return NULL;
2188
2189 skb_reserve(skb, reserve);
2190 skb_put(skb, linear);
2191 skb->data_len = len - linear;
2192 skb->len += len - linear;
2193
2194 return skb;
2195}
2196
69e3c75f 2197static int packet_snd(struct socket *sock,
1da177e4
LT		 2198 struct msghdr *msg, size_t len)
2199{
2200 struct sock *sk = sock->sk;
40d4e3df 2201 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2202 struct sk_buff *skb;
2203 struct net_device *dev;
0e11c91e 2204 __be16 proto;
827d9780 2205 bool need_rls_dev = false;
1da177e4 2206 unsigned char *addr;
827d9780 2207 int err, reserve = 0;
bfd5f4a3
SS		 2208 struct virtio_net_hdr vnet_hdr = { 0 };
2209 int offset = 0;
2210 int vnet_hdr_len;
2211 struct packet_sock *po = pkt_sk(sk);
2212 unsigned short gso_type = 0;
ae641949 2213 int hlen, tlen;
1da177e4
LT		 2214
2215 /*
1ce4f28b 2216 * Get and verify the address.
1da177e4 2217 */
1ce4f28b 2218
1da177e4 2219 if (saddr == NULL) {
827d9780 2220 dev = po->prot_hook.dev;
1da177e4
LT		 2221 proto = po->num;
2222 addr = NULL;
2223 } else {
2224 err = -EINVAL;
2225 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2226 goto out;
0fb375fb
EB		 2227 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2228 goto out;
1da177e4
LT		 2229 proto = saddr->sll_protocol;
2230 addr = saddr->sll_addr;
827d9780
BG		 2231 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2232 need_rls_dev = true;
1da177e4
LT		 2233 }
2234
1da177e4
LT		 2235 err = -ENXIO;
2236 if (dev == NULL)
2237 goto out_unlock;
2238 if (sock->type == SOCK_RAW)
2239 reserve = dev->hard_header_len;
2240
d5e76b0a
DM		 2241 err = -ENETDOWN;
2242 if (!(dev->flags & IFF_UP))
2243 goto out_unlock;
2244
bfd5f4a3
SS		 2245 if (po->has_vnet_hdr) {
2246 vnet_hdr_len = sizeof(vnet_hdr);
2247
2248 err = -EINVAL;
2249 if (len < vnet_hdr_len)
2250 goto out_unlock;
2251
2252 len -= vnet_hdr_len;
2253
2254 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2255 vnet_hdr_len);
2256 if (err < 0)
2257 goto out_unlock;
2258
2259 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2260 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2261 vnet_hdr.hdr_len))
2262 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2263 vnet_hdr.csum_offset + 2;
2264
2265 err = -EINVAL;
2266 if (vnet_hdr.hdr_len > len)
2267 goto out_unlock;
2268
2269 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2270 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2271 case VIRTIO_NET_HDR_GSO_TCPV4:
2272 gso_type = SKB_GSO_TCPV4;
2273 break;
2274 case VIRTIO_NET_HDR_GSO_TCPV6:
2275 gso_type = SKB_GSO_TCPV6;
2276 break;
2277 case VIRTIO_NET_HDR_GSO_UDP:
2278 gso_type = SKB_GSO_UDP;
2279 break;
2280 default:
2281 goto out_unlock;
2282 }
2283
2284 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2285 gso_type |= SKB_GSO_TCP_ECN;
2286
2287 if (vnet_hdr.gso_size == 0)
2288 goto out_unlock;
2289
2290 }
2291 }
2292
1da177e4 2293 err = -EMSGSIZE;
57f89bfa 2294 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN))
1da177e4
LT		 2295 goto out_unlock;
2296
bfd5f4a3 2297 err = -ENOBUFS;
ae641949
HX		 2298 hlen = LL_RESERVED_SPACE(dev);
2299 tlen = dev->needed_tailroom;
2300 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2301 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2302 if (skb == NULL)
1da177e4
LT		 2303 goto out_unlock;
2304
bfd5f4a3 2305 skb_set_network_header(skb, reserve);
1da177e4 2306
0c4e8581
SH		 2307 err = -EINVAL;
2308 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2309 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2310 goto out_free;
1da177e4
LT		 2311
2312 /* Returns -EFAULT on error */
bfd5f4a3 2313 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2314 if (err)
2315 goto out_free;
2244d07b 2316 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
ed85b565
RC		 2317 if (err < 0)
2318 goto out_free;
1da177e4 2319
57f89bfa
BG		 2320 if (!gso_type && (len > dev->mtu + reserve)) {
2321 /* Earlier code assumed this would be a VLAN pkt,
2322 * double-check this now that we have the actual
2323 * packet in hand.
2324 */
2325 struct ethhdr *ehdr;
2326 skb_reset_mac_header(skb);
2327 ehdr = eth_hdr(skb);
2328 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2329 err = -EMSGSIZE;
2330 goto out_free;
2331 }
2332 }
2333
1da177e4
LT		 2334 skb->protocol = proto;
2335 skb->dev = dev;
2336 skb->priority = sk->sk_priority;
2d37a186 2337 skb->mark = sk->sk_mark;
1da177e4 2338
bfd5f4a3
SS		 2339 if (po->has_vnet_hdr) {
2340 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2341 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2342 vnet_hdr.csum_offset)) {
2343 err = -EINVAL;
2344 goto out_free;
2345 }
2346 }
2347
2348 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2349 skb_shinfo(skb)->gso_type = gso_type;
2350
2351 /* Header must be checked, and gso_segs computed. */
2352 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2353 skb_shinfo(skb)->gso_segs = 0;
2354
2355 len += vnet_hdr_len;
2356 }
2357
1da177e4
LT		 2358 /*
2359 * Now send it
2360 */
2361
2362 err = dev_queue_xmit(skb);
2363 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2364 goto out_unlock;
2365
827d9780
BG		 2366 if (need_rls_dev)
2367 dev_put(dev);
1da177e4 2368
40d4e3df 2369 return len;
1da177e4
LT		 2370
2371out_free:
2372 kfree_skb(skb);
2373out_unlock:
827d9780 2374 if (dev && need_rls_dev)
1da177e4
LT		 2375 dev_put(dev);
2376out:
2377 return err;
2378}
2379
69e3c75f
JB		 2380static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2381 struct msghdr *msg, size_t len)
2382{
69e3c75f
JB		 2383 struct sock *sk = sock->sk;
2384 struct packet_sock *po = pkt_sk(sk);
2385 if (po->tx_ring.pg_vec)
2386 return tpacket_snd(po, msg);
2387 else
69e3c75f
JB		 2388 return packet_snd(sock, msg, len);
2389}
2390
1da177e4
LT		 2391/*
2392 * Close a PACKET socket. This is fairly simple. We immediately go
2393 * to 'closed' state and remove our protocol entry in the device list.
2394 */
2395
2396static int packet_release(struct socket *sock)
2397{
2398 struct sock *sk = sock->sk;
2399 struct packet_sock *po;
d12d01d6 2400 struct net *net;
f6fb8f10 2401 union tpacket_req_u req_u;
1da177e4
LT		 2402
2403 if (!sk)
2404 return 0;
2405
3b1e0a65 2406 net = sock_net(sk);
1da177e4
LT		 2407 po = pkt_sk(sk);
2408
808f5114 2409 spin_lock_bh(&net->packet.sklist_lock);
2410 sk_del_node_init_rcu(sk);
920de804 2411 sock_prot_inuse_add(net, sk->sk_prot, -1);
808f5114 2412 spin_unlock_bh(&net->packet.sklist_lock);
1da177e4 2413
808f5114 2414 spin_lock(&po->bind_lock);
ce06b03e 2415 unregister_prot_hook(sk, false);
160ff18a
BG		 2416 if (po->prot_hook.dev) {
2417 dev_put(po->prot_hook.dev);
2418 po->prot_hook.dev = NULL;
2419 }
808f5114 2420 spin_unlock(&po->bind_lock);
1da177e4 2421
1da177e4 2422 packet_flush_mclist(sk);
1da177e4 2423
f6fb8f10 2424 memset(&req_u, 0, sizeof(req_u));
69e3c75f
JB		 2425
2426 if (po->rx_ring.pg_vec)
f6fb8f10 2427 packet_set_ring(sk, &req_u, 1, 0);
69e3c75f
JB		 2428
2429 if (po->tx_ring.pg_vec)
f6fb8f10 2430 packet_set_ring(sk, &req_u, 1, 1);
1da177e4 2431
dc99f600
DM		 2432 fanout_release(sk);
2433
808f5114 2434 synchronize_net();
1da177e4
LT		 2435 /*
2436 * Now the socket is dead. No more input will appear.
2437 */
1da177e4
LT		 2438 sock_orphan(sk);
2439 sock->sk = NULL;
2440
2441 /* Purge queues */
2442
2443 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2444 sk_refcnt_debug_release(sk);
1da177e4
LT		 2445
2446 sock_put(sk);
2447 return 0;
2448}
2449
2450/*
2451 * Attach a packet hook.
2452 */
2453
0e11c91e 2454static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2455{
2456 struct packet_sock *po = pkt_sk(sk);
dc99f600
DM		 2457
2458 if (po->fanout)
2459 return -EINVAL;
1da177e4
LT		 2460
2461 lock_sock(sk);
2462
2463 spin_lock(&po->bind_lock);
ce06b03e 2464 unregister_prot_hook(sk, true);
1da177e4
LT		 2465 po->num = protocol;
2466 po->prot_hook.type = protocol;
160ff18a
BG		 2467 if (po->prot_hook.dev)
2468 dev_put(po->prot_hook.dev);
1da177e4
LT		 2469 po->prot_hook.dev = dev;
2470
2471 po->ifindex = dev ? dev->ifindex : 0;
2472
2473 if (protocol == 0)
2474 goto out_unlock;
2475
be85d4ad 2476 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2477 register_prot_hook(sk);
be85d4ad
UT		 2478 } else {
2479 sk->sk_err = ENETDOWN;
2480 if (!sock_flag(sk, SOCK_DEAD))
2481 sk->sk_error_report(sk);
1da177e4
LT		 2482 }
2483
2484out_unlock:
2485 spin_unlock(&po->bind_lock);
2486 release_sock(sk);
2487 return 0;
2488}
2489
2490/*
2491 * Bind a packet socket to a device
2492 */
2493
40d4e3df
ED		 2494static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2495 int addr_len)
1da177e4 2496{
40d4e3df 2497 struct sock *sk = sock->sk;
1da177e4
LT		 2498 char name[15];
2499 struct net_device *dev;
2500 int err = -ENODEV;
1ce4f28b 2501
1da177e4
LT		 2502 /*
2503 * Check legality
2504 */
1ce4f28b 2505
8ae55f04 2506 if (addr_len != sizeof(struct sockaddr))
1da177e4 2507 return -EINVAL;
40d4e3df 2508 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2509
3b1e0a65 2510 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2511 if (dev)
1da177e4 2512 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2513 return err;
2514}
1da177e4
LT		 2515
2516static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2517{
40d4e3df
ED		 2518 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2519 struct sock *sk = sock->sk;
1da177e4
LT		 2520 struct net_device *dev = NULL;
2521 int err;
2522
2523
2524 /*
2525 * Check legality
2526 */
1ce4f28b 2527
1da177e4
LT		 2528 if (addr_len < sizeof(struct sockaddr_ll))
2529 return -EINVAL;
2530 if (sll->sll_family != AF_PACKET)
2531 return -EINVAL;
2532
2533 if (sll->sll_ifindex) {
2534 err = -ENODEV;
3b1e0a65 2535 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2536 if (dev == NULL)
2537 goto out;
2538 }
2539 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2540
2541out:
2542 return err;
2543}
2544
2545static struct proto packet_proto = {
2546 .name = "PACKET",
2547 .owner = THIS_MODULE,
2548 .obj_size = sizeof(struct packet_sock),
2549};
2550
2551/*
1ce4f28b 2552 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2553 */
2554
3f378b68
EP		 2555static int packet_create(struct net *net, struct socket *sock, int protocol,
2556 int kern)
1da177e4
LT		 2557{
2558 struct sock *sk;
2559 struct packet_sock *po;
0e11c91e 2560 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2561 int err;
2562
2563 if (!capable(CAP_NET_RAW))
2564 return -EPERM;
be02097c
DM		 2565 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2566 sock->type != SOCK_PACKET)
1da177e4
LT		 2567 return -ESOCKTNOSUPPORT;
2568
2569 sock->state = SS_UNCONNECTED;
2570
2571 err = -ENOBUFS;
6257ff21 2572 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2573 if (sk == NULL)
2574 goto out;
2575
2576 sock->ops = &packet_ops;
1da177e4
LT		 2577 if (sock->type == SOCK_PACKET)
2578 sock->ops = &packet_ops_spkt;
be02097c 2579
1da177e4
LT		 2580 sock_init_data(sock, sk);
2581
2582 po = pkt_sk(sk);
2583 sk->sk_family = PF_PACKET;
0e11c91e 2584 po->num = proto;
1da177e4
LT		 2585
2586 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2587 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2588
2589 /*
2590 * Attach a protocol block
2591 */
2592
2593 spin_lock_init(&po->bind_lock);
905db440 2594 mutex_init(&po->pg_vec_lock);
1da177e4 2595 po->prot_hook.func = packet_rcv;
be02097c 2596
1da177e4
LT		 2597 if (sock->type == SOCK_PACKET)
2598 po->prot_hook.func = packet_rcv_spkt;
be02097c 2599
1da177e4
LT		 2600 po->prot_hook.af_packet_priv = sk;
2601
0e11c91e
AV		 2602 if (proto) {
2603 po->prot_hook.type = proto;
ce06b03e 2604 register_prot_hook(sk);
1da177e4
LT		 2605 }
2606
808f5114 2607 spin_lock_bh(&net->packet.sklist_lock);
2608 sk_add_node_rcu(sk, &net->packet.sklist);
3680453c 2609 sock_prot_inuse_add(net, &packet_proto, 1);
808f5114 2610 spin_unlock_bh(&net->packet.sklist_lock);
2611
40d4e3df 2612 return 0;
1da177e4
LT		 2613out:
2614 return err;
2615}
2616
ed85b565
RC		 2617static int packet_recv_error(struct sock *sk, struct msghdr *msg, int len)
2618{
2619 struct sock_exterr_skb *serr;
2620 struct sk_buff *skb, *skb2;
2621 int copied, err;
2622
2623 err = -EAGAIN;
2624 skb = skb_dequeue(&sk->sk_error_queue);
2625 if (skb == NULL)
2626 goto out;
2627
2628 copied = skb->len;
2629 if (copied > len) {
2630 msg->msg_flags |= MSG_TRUNC;
2631 copied = len;
2632 }
2633 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2634 if (err)
2635 goto out_free_skb;
2636
2637 sock_recv_timestamp(msg, sk, skb);
2638
2639 serr = SKB_EXT_ERR(skb);
2640 put_cmsg(msg, SOL_PACKET, PACKET_TX_TIMESTAMP,
2641 sizeof(serr->ee), &serr->ee);
2642
2643 msg->msg_flags |= MSG_ERRQUEUE;
2644 err = copied;
2645
2646 /* Reset and regenerate socket error */
2647 spin_lock_bh(&sk->sk_error_queue.lock);
2648 sk->sk_err = 0;
2649 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) {
2650 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno;
2651 spin_unlock_bh(&sk->sk_error_queue.lock);
2652 sk->sk_error_report(sk);
2653 } else
2654 spin_unlock_bh(&sk->sk_error_queue.lock);
2655
2656out_free_skb:
2657 kfree_skb(skb);
2658out:
2659 return err;
2660}
2661
1da177e4
LT		 2662/*
2663 * Pull a packet from our receive queue and hand it to the user.
2664 * If necessary we block.
2665 */
2666
2667static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2668 struct msghdr *msg, size_t len, int flags)
2669{
2670 struct sock *sk = sock->sk;
2671 struct sk_buff *skb;
2672 int copied, err;
0fb375fb 2673 struct sockaddr_ll *sll;
bfd5f4a3 2674 int vnet_hdr_len = 0;
1da177e4
LT		 2675
2676 err = -EINVAL;
ed85b565 2677 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2678 goto out;
2679
2680#if 0
2681 /* What error should we return now? EUNATTACH? */
2682 if (pkt_sk(sk)->ifindex < 0)
2683 return -ENODEV;
2684#endif
2685
ed85b565
RC		 2686 if (flags & MSG_ERRQUEUE) {
2687 err = packet_recv_error(sk, msg, len);
2688 goto out;
2689 }
2690
1da177e4
LT		 2691 /*
2692 * Call the generic datagram receiver. This handles all sorts
2693 * of horrible races and re-entrancy so we can forget about it
2694 * in the protocol layers.
2695 *
2696 * Now it will return ENETDOWN, if device have just gone down,
2697 * but then it will block.
2698 */
2699
40d4e3df 2700 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2701
2702 /*
1ce4f28b 2703 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2704 * handles the blocking we don't see and worry about blocking
2705 * retries.
2706 */
2707
8ae55f04 2708 if (skb == NULL)
1da177e4
LT		 2709 goto out;
2710
bfd5f4a3
SS		 2711 if (pkt_sk(sk)->has_vnet_hdr) {
2712 struct virtio_net_hdr vnet_hdr = { 0 };
2713
2714 err = -EINVAL;
2715 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2716 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2717 goto out_free;
2718
1f18b717
MK		 2719 len -= vnet_hdr_len;
2720
bfd5f4a3
SS		 2721 if (skb_is_gso(skb)) {
2722 struct skb_shared_info *sinfo = skb_shinfo(skb);
2723
2724 /* This is a hint as to how much should be linear. */
2725 vnet_hdr.hdr_len = skb_headlen(skb);
2726 vnet_hdr.gso_size = sinfo->gso_size;
2727 if (sinfo->gso_type & SKB_GSO_TCPV4)
2728 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2729 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2730 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2731 else if (sinfo->gso_type & SKB_GSO_UDP)
2732 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2733 else if (sinfo->gso_type & SKB_GSO_FCOE)
2734 goto out_free;
2735 else
2736 BUG();
2737 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2738 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2739 } else
2740 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2741
2742 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2743 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2744 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2745 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2746 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2747 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2748 } /* else everything is zero */
2749
2750 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2751 vnet_hdr_len);
2752 if (err < 0)
2753 goto out_free;
2754 }
2755
0fb375fb
EB		 2756 /*
2757 * If the address length field is there to be filled in, we fill
2758 * it in now.
2759 */
2760
ffbc6111 2761 sll = &PACKET_SKB_CB(skb)->sa.ll;
0fb375fb
EB		 2762 if (sock->type == SOCK_PACKET)
2763 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2764 else
2765 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
2766
1da177e4
LT		 2767 /*
2768 * You lose any data beyond the buffer you gave. If it worries a
2769 * user program they can ask the device for its MTU anyway.
2770 */
2771
2772 copied = skb->len;
40d4e3df
ED		 2773 if (copied > len) {
2774 copied = len;
2775 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2776 }
2777
2778 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2779 if (err)
2780 goto out_free;
2781
3b885787 2782 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4
LT		 2783
2784 if (msg->msg_name)
ffbc6111
HX		 2785 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2786 msg->msg_namelen);
1da177e4 2787
8dc41944 2788 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2789 struct tpacket_auxdata aux;
2790
2791 aux.tp_status = TP_STATUS_USER;
2792 if (skb->ip_summed == CHECKSUM_PARTIAL)
2793 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2794 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2795 aux.tp_snaplen = skb->len;
2796 aux.tp_mac = 0;
bbe735e4 2797 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2798 if (vlan_tx_tag_present(skb)) {
2799 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2800 aux.tp_status |= TP_STATUS_VLAN_VALID;
2801 } else {
2802 aux.tp_vlan_tci = 0;
2803 }
13fcb7bd 2804 aux.tp_padding = 0;
ffbc6111 2805 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2806 }
2807
1da177e4
LT		 2808 /*
2809 * Free or return the buffer as appropriate. Again this
2810 * hides all the races and re-entrancy issues from us.
2811 */
bfd5f4a3 2812 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2813
2814out_free:
2815 skb_free_datagram(sk, skb);
2816out:
2817 return err;
2818}
2819
1da177e4
LT		 2820static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2821 int *uaddr_len, int peer)
2822{
2823 struct net_device *dev;
2824 struct sock *sk = sock->sk;
2825
2826 if (peer)
2827 return -EOPNOTSUPP;
2828
2829 uaddr->sa_family = AF_PACKET;
654d1f8a
ED		 2830 rcu_read_lock();
2831 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2832 if (dev)
67286640 2833 strncpy(uaddr->sa_data, dev->name, 14);
654d1f8a 2834 else
1da177e4 2835 memset(uaddr->sa_data, 0, 14);
654d1f8a 2836 rcu_read_unlock();
1da177e4
LT		 2837 *uaddr_len = sizeof(*uaddr);
2838
2839 return 0;
2840}
1da177e4
LT		 2841
2842static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2843 int *uaddr_len, int peer)
2844{
2845 struct net_device *dev;
2846 struct sock *sk = sock->sk;
2847 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2848 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2849
2850 if (peer)
2851 return -EOPNOTSUPP;
2852
2853 sll->sll_family = AF_PACKET;
2854 sll->sll_ifindex = po->ifindex;
2855 sll->sll_protocol = po->num;
67286640 2856 sll->sll_pkttype = 0;
654d1f8a
ED		 2857 rcu_read_lock();
2858 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2859 if (dev) {
2860 sll->sll_hatype = dev->type;
2861 sll->sll_halen = dev->addr_len;
2862 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2863 } else {
2864 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2865 sll->sll_halen = 0;
2866 }
654d1f8a 2867 rcu_read_unlock();
0fb375fb 2868 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2869
2870 return 0;
2871}
2872
2aeb0b88
WC		 2873static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2874 int what)
1da177e4
LT		 2875{
2876 switch (i->type) {
2877 case PACKET_MR_MULTICAST:
1162563f
JP		 2878 if (i->alen != dev->addr_len)
2879 return -EINVAL;
1da177e4 2880 if (what > 0)
22bedad3 2881 return dev_mc_add(dev, i->addr);
1da177e4 2882 else
22bedad3 2883 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2884 break;
2885 case PACKET_MR_PROMISC:
2aeb0b88 2886 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2887 break;
2888 case PACKET_MR_ALLMULTI:
2aeb0b88 2889 return dev_set_allmulti(dev, what);
1da177e4 2890 break;
d95ed927 2891 case PACKET_MR_UNICAST:
1162563f
JP		 2892 if (i->alen != dev->addr_len)
2893 return -EINVAL;
d95ed927 2894 if (what > 0)
a748ee24 2895 return dev_uc_add(dev, i->addr);
d95ed927 2896 else
a748ee24 2897 return dev_uc_del(dev, i->addr);
d95ed927 2898 break;
40d4e3df
ED		 2899 default:
2900 break;
1da177e4 2901 }
2aeb0b88 2902 return 0;
1da177e4
LT		 2903}
2904
2905static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2906{
40d4e3df 2907 for ( ; i; i = i->next) {
1da177e4
LT		 2908 if (i->ifindex == dev->ifindex)
2909 packet_dev_mc(dev, i, what);
2910 }
2911}
2912
0fb375fb 2913static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2914{
2915 struct packet_sock *po = pkt_sk(sk);
2916 struct packet_mclist *ml, *i;
2917 struct net_device *dev;
2918 int err;
2919
2920 rtnl_lock();
2921
2922 err = -ENODEV;
3b1e0a65 2923 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2924 if (!dev)
2925 goto done;
2926
2927 err = -EINVAL;
1162563f 2928 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2929 goto done;
2930
2931 err = -ENOBUFS;
8b3a7005 2932 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 2933 if (i == NULL)
2934 goto done;
2935
2936 err = 0;
2937 for (ml = po->mclist; ml; ml = ml->next) {
2938 if (ml->ifindex == mreq->mr_ifindex &&
2939 ml->type == mreq->mr_type &&
2940 ml->alen == mreq->mr_alen &&
2941 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2942 ml->count++;
2943 /* Free the new element ... */
2944 kfree(i);
2945 goto done;
2946 }
2947 }
2948
2949 i->type = mreq->mr_type;
2950 i->ifindex = mreq->mr_ifindex;
2951 i->alen = mreq->mr_alen;
2952 memcpy(i->addr, mreq->mr_address, i->alen);
2953 i->count = 1;
2954 i->next = po->mclist;
2955 po->mclist = i;
2aeb0b88
WC		 2956 err = packet_dev_mc(dev, i, 1);
2957 if (err) {
2958 po->mclist = i->next;
2959 kfree(i);
2960 }
1da177e4
LT		 2961
2962done:
2963 rtnl_unlock();
2964 return err;
2965}
2966
0fb375fb 2967static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2968{
2969 struct packet_mclist *ml, **mlp;
2970
2971 rtnl_lock();
2972
2973 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
2974 if (ml->ifindex == mreq->mr_ifindex &&
2975 ml->type == mreq->mr_type &&
2976 ml->alen == mreq->mr_alen &&
2977 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2978 if (--ml->count == 0) {
2979 struct net_device *dev;
2980 *mlp = ml->next;
ad959e76
ED		 2981 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2982 if (dev)
1da177e4 2983 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 2984 kfree(ml);
2985 }
2986 rtnl_unlock();
2987 return 0;
2988 }
2989 }
2990 rtnl_unlock();
2991 return -EADDRNOTAVAIL;
2992}
2993
2994static void packet_flush_mclist(struct sock *sk)
2995{
2996 struct packet_sock *po = pkt_sk(sk);
2997 struct packet_mclist *ml;
2998
2999 if (!po->mclist)
3000 return;
3001
3002 rtnl_lock();
3003 while ((ml = po->mclist) != NULL) {
3004 struct net_device *dev;
3005
3006 po->mclist = ml->next;
ad959e76
ED		 3007 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3008 if (dev != NULL)
1da177e4 3009 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3010 kfree(ml);
3011 }
3012 rtnl_unlock();
3013}
1da177e4
LT		 3014
3015static int
b7058842 3016packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3017{
3018 struct sock *sk = sock->sk;
8dc41944 3019 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3020 int ret;
3021
3022 if (level != SOL_PACKET)
3023 return -ENOPROTOOPT;
3024
69e3c75f 3025 switch (optname) {
1ce4f28b 3026 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3027 case PACKET_DROP_MEMBERSHIP:
3028 {
0fb375fb
EB		 3029 struct packet_mreq_max mreq;
3030 int len = optlen;
3031 memset(&mreq, 0, sizeof(mreq));
3032 if (len < sizeof(struct packet_mreq))
1da177e4 3033 return -EINVAL;
0fb375fb
EB		 3034 if (len > sizeof(mreq))
3035 len = sizeof(mreq);
40d4e3df 3036 if (copy_from_user(&mreq, optval, len))
1da177e4 3037 return -EFAULT;
0fb375fb
EB		 3038 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3039 return -EINVAL;
1da177e4
LT		 3040 if (optname == PACKET_ADD_MEMBERSHIP)
3041 ret = packet_mc_add(sk, &mreq);
3042 else
3043 ret = packet_mc_drop(sk, &mreq);
3044 return ret;
3045 }
a2efcfa0 3046
1da177e4 3047 case PACKET_RX_RING:
69e3c75f 3048 case PACKET_TX_RING:
1da177e4 3049 {
f6fb8f10 3050 union tpacket_req_u req_u;
3051 int len;
1da177e4 3052
f6fb8f10 3053 switch (po->tp_version) {
3054 case TPACKET_V1:
3055 case TPACKET_V2:
3056 len = sizeof(req_u.req);
3057 break;
3058 case TPACKET_V3:
3059 default:
3060 len = sizeof(req_u.req3);
3061 break;
3062 }
3063 if (optlen < len)
1da177e4 3064 return -EINVAL;
bfd5f4a3
SS		 3065 if (pkt_sk(sk)->has_vnet_hdr)
3066 return -EINVAL;
f6fb8f10 3067 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3068 return -EFAULT;
f6fb8f10 3069 return packet_set_ring(sk, &req_u, 0,
3070 optname == PACKET_TX_RING);
1da177e4
LT		 3071 }
3072 case PACKET_COPY_THRESH:
3073 {
3074 int val;
3075
40d4e3df 3076 if (optlen != sizeof(val))
1da177e4 3077 return -EINVAL;
40d4e3df 3078 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3079 return -EFAULT;
3080
3081 pkt_sk(sk)->copy_thresh = val;
3082 return 0;
3083 }
bbd6ef87
PM		 3084 case PACKET_VERSION:
3085 {
3086 int val;
3087
3088 if (optlen != sizeof(val))
3089 return -EINVAL;
69e3c75f 3090 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3091 return -EBUSY;
3092 if (copy_from_user(&val, optval, sizeof(val)))
3093 return -EFAULT;
3094 switch (val) {
3095 case TPACKET_V1:
3096 case TPACKET_V2:
f6fb8f10 3097 case TPACKET_V3:
bbd6ef87
PM		 3098 po->tp_version = val;
3099 return 0;
3100 default:
3101 return -EINVAL;
3102 }
3103 }
8913336a
PM		 3104 case PACKET_RESERVE:
3105 {
3106 unsigned int val;
3107
3108 if (optlen != sizeof(val))
3109 return -EINVAL;
69e3c75f 3110 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3111 return -EBUSY;
3112 if (copy_from_user(&val, optval, sizeof(val)))
3113 return -EFAULT;
3114 po->tp_reserve = val;
3115 return 0;
3116 }
69e3c75f
JB		 3117 case PACKET_LOSS:
3118 {
3119 unsigned int val;
3120
3121 if (optlen != sizeof(val))
3122 return -EINVAL;
3123 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3124 return -EBUSY;
3125 if (copy_from_user(&val, optval, sizeof(val)))
3126 return -EFAULT;
3127 po->tp_loss = !!val;
3128 return 0;
3129 }
8dc41944
HX		 3130 case PACKET_AUXDATA:
3131 {
3132 int val;
3133
3134 if (optlen < sizeof(val))
3135 return -EINVAL;
3136 if (copy_from_user(&val, optval, sizeof(val)))
3137 return -EFAULT;
3138
3139 po->auxdata = !!val;
3140 return 0;
3141 }
80feaacb
PWJ		 3142 case PACKET_ORIGDEV:
3143 {
3144 int val;
3145
3146 if (optlen < sizeof(val))
3147 return -EINVAL;
3148 if (copy_from_user(&val, optval, sizeof(val)))
3149 return -EFAULT;
3150
3151 po->origdev = !!val;
3152 return 0;
3153 }
bfd5f4a3
SS		 3154 case PACKET_VNET_HDR:
3155 {
3156 int val;
3157
3158 if (sock->type != SOCK_RAW)
3159 return -EINVAL;
3160 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3161 return -EBUSY;
3162 if (optlen < sizeof(val))
3163 return -EINVAL;
3164 if (copy_from_user(&val, optval, sizeof(val)))
3165 return -EFAULT;
3166
3167 po->has_vnet_hdr = !!val;
3168 return 0;
3169 }
614f60fa
SM		 3170 case PACKET_TIMESTAMP:
3171 {
3172 int val;
3173
3174 if (optlen != sizeof(val))
3175 return -EINVAL;
3176 if (copy_from_user(&val, optval, sizeof(val)))
3177 return -EFAULT;
3178
3179 po->tp_tstamp = val;
3180 return 0;
3181 }
dc99f600
DM		 3182 case PACKET_FANOUT:
3183 {
3184 int val;
3185
3186 if (optlen != sizeof(val))
3187 return -EINVAL;
3188 if (copy_from_user(&val, optval, sizeof(val)))
3189 return -EFAULT;
3190
3191 return fanout_add(sk, val & 0xffff, val >> 16);
3192 }
1da177e4
LT		 3193 default:
3194 return -ENOPROTOOPT;
3195 }
3196}
3197
3198static int packet_getsockopt(struct socket *sock, int level, int optname,
3199 char __user *optval, int __user *optlen)
3200{
3201 int len;
8dc41944 3202 int val;
1da177e4
LT		 3203 struct sock *sk = sock->sk;
3204 struct packet_sock *po = pkt_sk(sk);
8dc41944
HX		 3205 void *data;
3206 struct tpacket_stats st;
f6fb8f10 3207 union tpacket_stats_u st_u;
1da177e4
LT		 3208
3209 if (level != SOL_PACKET)
3210 return -ENOPROTOOPT;
3211
8ae55f04
KK		 3212 if (get_user(len, optlen))
3213 return -EFAULT;
1da177e4
LT		 3214
3215 if (len < 0)
3216 return -EINVAL;
1ce4f28b 3217
69e3c75f 3218 switch (optname) {
1da177e4 3219 case PACKET_STATISTICS:
f6fb8f10 3220 if (po->tp_version == TPACKET_V3) {
3221 len = sizeof(struct tpacket_stats_v3);
3222 } else {
3223 if (len > sizeof(struct tpacket_stats))
3224 len = sizeof(struct tpacket_stats);
3225 }
1da177e4 3226 spin_lock_bh(&sk->sk_receive_queue.lock);
f6fb8f10 3227 if (po->tp_version == TPACKET_V3) {
3228 memcpy(&st_u.stats3, &po->stats,
3229 sizeof(struct tpacket_stats));
3230 st_u.stats3.tp_freeze_q_cnt =
3231 po->stats_u.stats3.tp_freeze_q_cnt;
3232 st_u.stats3.tp_packets += po->stats.tp_drops;
3233 data = &st_u.stats3;
3234 } else {
3235 st = po->stats;
3236 st.tp_packets += st.tp_drops;
3237 data = &st;
3238 }
1da177e4
LT		 3239 memset(&po->stats, 0, sizeof(st));
3240 spin_unlock_bh(&sk->sk_receive_queue.lock);
8dc41944
HX		 3241 break;
3242 case PACKET_AUXDATA:
3243 if (len > sizeof(int))
3244 len = sizeof(int);
3245 val = po->auxdata;
3246
80feaacb
PWJ		 3247 data = &val;
3248 break;
3249 case PACKET_ORIGDEV:
3250 if (len > sizeof(int))
3251 len = sizeof(int);
3252 val = po->origdev;
3253
bfd5f4a3
SS		 3254 data = &val;
3255 break;
3256 case PACKET_VNET_HDR:
3257 if (len > sizeof(int))
3258 len = sizeof(int);
3259 val = po->has_vnet_hdr;
3260
8dc41944 3261 data = &val;
1da177e4 3262 break;
bbd6ef87
PM		 3263 case PACKET_VERSION:
3264 if (len > sizeof(int))
3265 len = sizeof(int);
3266 val = po->tp_version;
3267 data = &val;
3268 break;
3269 case PACKET_HDRLEN:
3270 if (len > sizeof(int))
3271 len = sizeof(int);
3272 if (copy_from_user(&val, optval, len))
3273 return -EFAULT;
3274 switch (val) {
3275 case TPACKET_V1:
3276 val = sizeof(struct tpacket_hdr);
3277 break;
3278 case TPACKET_V2:
3279 val = sizeof(struct tpacket2_hdr);
3280 break;
f6fb8f10 3281 case TPACKET_V3:
3282 val = sizeof(struct tpacket3_hdr);
3283 break;
bbd6ef87
PM		 3284 default:
3285 return -EINVAL;
3286 }
3287 data = &val;
3288 break;
8913336a
PM		 3289 case PACKET_RESERVE:
3290 if (len > sizeof(unsigned int))
3291 len = sizeof(unsigned int);
3292 val = po->tp_reserve;
3293 data = &val;
3294 break;
69e3c75f
JB		 3295 case PACKET_LOSS:
3296 if (len > sizeof(unsigned int))
3297 len = sizeof(unsigned int);
3298 val = po->tp_loss;
3299 data = &val;
3300 break;
614f60fa
SM		 3301 case PACKET_TIMESTAMP:
3302 if (len > sizeof(int))
3303 len = sizeof(int);
3304 val = po->tp_tstamp;
3305 data = &val;
3306 break;
dc99f600
DM		 3307 case PACKET_FANOUT:
3308 if (len > sizeof(int))
3309 len = sizeof(int);
3310 val = (po->fanout ?
3311 ((u32)po->fanout->id |
3312 ((u32)po->fanout->type << 16)) :
3313 0);
3314 data = &val;
3315 break;
1da177e4
LT		 3316 default:
3317 return -ENOPROTOOPT;
3318 }
3319
8ae55f04
KK		 3320 if (put_user(len, optlen))
3321 return -EFAULT;
8dc41944
HX		 3322 if (copy_to_user(optval, data, len))
3323 return -EFAULT;
8ae55f04 3324 return 0;
1da177e4
LT		 3325}
3326
3327
3328static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
3329{
3330 struct sock *sk;
3331 struct hlist_node *node;
ad930650 3332 struct net_device *dev = data;
c346dca1 3333 struct net *net = dev_net(dev);
1da177e4 3334
808f5114 3335 rcu_read_lock();
3336 sk_for_each_rcu(sk, node, &net->packet.sklist) {
1da177e4
LT		 3337 struct packet_sock *po = pkt_sk(sk);
3338
3339 switch (msg) {
3340 case NETDEV_UNREGISTER:
1da177e4
LT		 3341 if (po->mclist)
3342 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3343 /* fallthrough */
3344
1da177e4
LT		 3345 case NETDEV_DOWN:
3346 if (dev->ifindex == po->ifindex) {
3347 spin_lock(&po->bind_lock);
3348 if (po->running) {
ce06b03e 3349 __unregister_prot_hook(sk, false);
1da177e4
LT		 3350 sk->sk_err = ENETDOWN;
3351 if (!sock_flag(sk, SOCK_DEAD))
3352 sk->sk_error_report(sk);
3353 }
3354 if (msg == NETDEV_UNREGISTER) {
3355 po->ifindex = -1;
160ff18a
BG		 3356 if (po->prot_hook.dev)
3357 dev_put(po->prot_hook.dev);
1da177e4
LT		 3358 po->prot_hook.dev = NULL;
3359 }
3360 spin_unlock(&po->bind_lock);
3361 }
3362 break;
3363 case NETDEV_UP:
808f5114 3364 if (dev->ifindex == po->ifindex) {
3365 spin_lock(&po->bind_lock);
ce06b03e
DM		 3366 if (po->num)
3367 register_prot_hook(sk);
808f5114 3368 spin_unlock(&po->bind_lock);
1da177e4 3369 }
1da177e4
LT		 3370 break;
3371 }
3372 }
808f5114 3373 rcu_read_unlock();
1da177e4
LT		 3374 return NOTIFY_DONE;
3375}
3376
3377
3378static int packet_ioctl(struct socket *sock, unsigned int cmd,
3379 unsigned long arg)
3380{
3381 struct sock *sk = sock->sk;
3382
69e3c75f 3383 switch (cmd) {
40d4e3df
ED		 3384 case SIOCOUTQ:
3385 {
3386 int amount = sk_wmem_alloc_get(sk);
31e6d363 3387
40d4e3df
ED		 3388 return put_user(amount, (int __user *)arg);
3389 }
3390 case SIOCINQ:
3391 {
3392 struct sk_buff *skb;
3393 int amount = 0;
3394
3395 spin_lock_bh(&sk->sk_receive_queue.lock);
3396 skb = skb_peek(&sk->sk_receive_queue);
3397 if (skb)
3398 amount = skb->len;
3399 spin_unlock_bh(&sk->sk_receive_queue.lock);
3400 return put_user(amount, (int __user *)arg);
3401 }
3402 case SIOCGSTAMP:
3403 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3404 case SIOCGSTAMPNS:
3405 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3406
1da177e4 3407#ifdef CONFIG_INET
40d4e3df
ED		 3408 case SIOCADDRT:
3409 case SIOCDELRT:
3410 case SIOCDARP:
3411 case SIOCGARP:
3412 case SIOCSARP:
3413 case SIOCGIFADDR:
3414 case SIOCSIFADDR:
3415 case SIOCGIFBRDADDR:
3416 case SIOCSIFBRDADDR:
3417 case SIOCGIFNETMASK:
3418 case SIOCSIFNETMASK:
3419 case SIOCGIFDSTADDR:
3420 case SIOCSIFDSTADDR:
3421 case SIOCSIFFLAGS:
40d4e3df 3422 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3423#endif
3424
40d4e3df
ED		 3425 default:
3426 return -ENOIOCTLCMD;
1da177e4
LT		 3427 }
3428 return 0;
3429}
3430
40d4e3df 3431static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3432 poll_table *wait)
3433{
3434 struct sock *sk = sock->sk;
3435 struct packet_sock *po = pkt_sk(sk);
3436 unsigned int mask = datagram_poll(file, sock, wait);
3437
3438 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3439 if (po->rx_ring.pg_vec) {
f6fb8f10 3440 if (!packet_previous_rx_frame(po, &po->rx_ring,
3441 TP_STATUS_KERNEL))
1da177e4
LT		 3442 mask |= POLLIN | POLLRDNORM;
3443 }
3444 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3445 spin_lock_bh(&sk->sk_write_queue.lock);
3446 if (po->tx_ring.pg_vec) {
3447 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3448 mask |= POLLOUT | POLLWRNORM;
3449 }
3450 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3451 return mask;
3452}
3453
3454
3455/* Dirty? Well, I still did not learn better way to account
3456 * for user mmaps.
3457 */
3458
3459static void packet_mm_open(struct vm_area_struct *vma)
3460{
3461 struct file *file = vma->vm_file;
40d4e3df 3462 struct socket *sock = file->private_data;
1da177e4 3463 struct sock *sk = sock->sk;
1ce4f28b 3464
1da177e4
LT		 3465 if (sk)
3466 atomic_inc(&pkt_sk(sk)->mapped);
3467}
3468
3469static void packet_mm_close(struct vm_area_struct *vma)
3470{
3471 struct file *file = vma->vm_file;
40d4e3df 3472 struct socket *sock = file->private_data;
1da177e4 3473 struct sock *sk = sock->sk;
1ce4f28b 3474
1da177e4
LT		 3475 if (sk)
3476 atomic_dec(&pkt_sk(sk)->mapped);
3477}
3478
f0f37e2f 3479static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3480 .open = packet_mm_open,
3481 .close = packet_mm_close,
1da177e4
LT		 3482};
3483
0e3125c7
NH		 3484static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3485 unsigned int len)
1da177e4
LT		 3486{
3487 int i;
3488
4ebf0ae2 3489 for (i = 0; i < len; i++) {
0e3125c7 3490 if (likely(pg_vec[i].buffer)) {
c56b4d90 3491 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3492 vfree(pg_vec[i].buffer);
3493 else
3494 free_pages((unsigned long)pg_vec[i].buffer,
3495 order);
3496 pg_vec[i].buffer = NULL;
3497 }
1da177e4
LT		 3498 }
3499 kfree(pg_vec);
3500}
3501
eea49cc9 3502static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3503{
0e3125c7
NH		 3504 char *buffer = NULL;
3505 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3506 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3507
3508 buffer = (char *) __get_free_pages(gfp_flags, order);
3509
3510 if (buffer)
3511 return buffer;
3512
3513 /*
3514 * __get_free_pages failed, fall back to vmalloc
3515 */
bbce5a59 3516 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3517
0e3125c7
NH		 3518 if (buffer)
3519 return buffer;
3520
3521 /*
3522 * vmalloc failed, lets dig into swap here
3523 */
0e3125c7
NH		 3524 gfp_flags &= ~__GFP_NORETRY;
3525 buffer = (char *)__get_free_pages(gfp_flags, order);
3526 if (buffer)
3527 return buffer;
3528
3529 /*
3530 * complete and utter failure
3531 */
3532 return NULL;
4ebf0ae2
DM		 3533}
3534
0e3125c7 3535static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3536{
3537 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3538 struct pgv *pg_vec;
4ebf0ae2
DM		 3539 int i;
3540
0e3125c7 3541 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3542 if (unlikely(!pg_vec))
3543 goto out;
3544
3545 for (i = 0; i < block_nr; i++) {
c56b4d90 3546 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3547 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3548 goto out_free_pgvec;
3549 }
3550
3551out:
3552 return pg_vec;
3553
3554out_free_pgvec:
3555 free_pg_vec(pg_vec, order, block_nr);
3556 pg_vec = NULL;
3557 goto out;
3558}
1da177e4 3559
f6fb8f10 3560static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3561 int closing, int tx_ring)
1da177e4 3562{
0e3125c7 3563 struct pgv *pg_vec = NULL;
1da177e4 3564 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3565 int was_running, order = 0;
69e3c75f
JB		 3566 struct packet_ring_buffer *rb;
3567 struct sk_buff_head *rb_queue;
0e11c91e 3568 __be16 num;
f6fb8f10 3569 int err = -EINVAL;
3570 /* Added to avoid minimal code churn */
3571 struct tpacket_req *req = &req_u->req;
3572
3573 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3574 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3575 WARN(1, "Tx-ring is not supported.\n");
3576 goto out;
3577 }
1ce4f28b 3578
69e3c75f
JB		 3579 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3580 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3581
69e3c75f
JB		 3582 err = -EBUSY;
3583 if (!closing) {
3584 if (atomic_read(&po->mapped))
3585 goto out;
3586 if (atomic_read(&rb->pending))
3587 goto out;
3588 }
1da177e4 3589
69e3c75f
JB		 3590 if (req->tp_block_nr) {
3591 /* Sanity tests and some calculations */
3592 err = -EBUSY;
3593 if (unlikely(rb->pg_vec))
3594 goto out;
1da177e4 3595
bbd6ef87
PM		 3596 switch (po->tp_version) {
3597 case TPACKET_V1:
3598 po->tp_hdrlen = TPACKET_HDRLEN;
3599 break;
3600 case TPACKET_V2:
3601 po->tp_hdrlen = TPACKET2_HDRLEN;
3602 break;
f6fb8f10 3603 case TPACKET_V3:
3604 po->tp_hdrlen = TPACKET3_HDRLEN;
3605 break;
bbd6ef87
PM		 3606 }
3607
69e3c75f 3608 err = -EINVAL;
4ebf0ae2 3609 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3610 goto out;
4ebf0ae2 3611 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3612 goto out;
8913336a 3613 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3614 po->tp_reserve))
3615 goto out;
4ebf0ae2 3616 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3617 goto out;
1da177e4 3618
69e3c75f
JB		 3619 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3620 if (unlikely(rb->frames_per_block <= 0))
3621 goto out;
3622 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3623 req->tp_frame_nr))
3624 goto out;
1da177e4
LT		 3625
3626 err = -ENOMEM;
4ebf0ae2
DM		 3627 order = get_order(req->tp_block_size);
3628 pg_vec = alloc_pg_vec(req, order);
3629 if (unlikely(!pg_vec))
1da177e4 3630 goto out;
f6fb8f10 3631 switch (po->tp_version) {
3632 case TPACKET_V3:
3633 /* Transmit path is not supported. We checked
3634 * it above but just being paranoid
3635 */
3636 if (!tx_ring)
3637 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3638 break;
3639 default:
3640 break;
3641 }
69e3c75f
JB		 3642 }
3643 /* Done */
3644 else {
3645 err = -EINVAL;
4ebf0ae2 3646 if (unlikely(req->tp_frame_nr))
69e3c75f 3647 goto out;
1da177e4
LT		 3648 }
3649
3650 lock_sock(sk);
3651
3652 /* Detach socket from network */
3653 spin_lock(&po->bind_lock);
3654 was_running = po->running;
3655 num = po->num;
3656 if (was_running) {
1da177e4 3657 po->num = 0;
ce06b03e 3658 __unregister_prot_hook(sk, false);
1da177e4
LT		 3659 }
3660 spin_unlock(&po->bind_lock);
1ce4f28b 3661
1da177e4
LT		 3662 synchronize_net();
3663
3664 err = -EBUSY;
905db440 3665 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3666 if (closing || atomic_read(&po->mapped) == 0) {
3667 err = 0;
69e3c75f 3668 spin_lock_bh(&rb_queue->lock);
c053fd96 3669 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3670 rb->frame_max = (req->tp_frame_nr - 1);
3671 rb->head = 0;
3672 rb->frame_size = req->tp_frame_size;
3673 spin_unlock_bh(&rb_queue->lock);
3674
c053fd96
CG		 3675 swap(rb->pg_vec_order, order);
3676 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3677
3678 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3679 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3680 tpacket_rcv : packet_rcv;
3681 skb_queue_purge(rb_queue);
1da177e4 3682 if (atomic_read(&po->mapped))
40d4e3df
ED		 3683 pr_err("packet_mmap: vma is busy: %d\n",
3684 atomic_read(&po->mapped));
1da177e4 3685 }
905db440 3686 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3687
3688 spin_lock(&po->bind_lock);
ce06b03e 3689 if (was_running) {
1da177e4 3690 po->num = num;
ce06b03e 3691 register_prot_hook(sk);
1da177e4
LT		 3692 }
3693 spin_unlock(&po->bind_lock);
f6fb8f10 3694 if (closing && (po->tp_version > TPACKET_V2)) {
3695 /* Because we don't support block-based V3 on tx-ring */
3696 if (!tx_ring)
3697 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3698 }
1da177e4
LT		 3699 release_sock(sk);
3700
1da177e4
LT		 3701 if (pg_vec)
3702 free_pg_vec(pg_vec, order, req->tp_block_nr);
3703out:
3704 return err;
3705}
3706
69e3c75f
JB		 3707static int packet_mmap(struct file *file, struct socket *sock,
3708 struct vm_area_struct *vma)
1da177e4
LT		 3709{
3710 struct sock *sk = sock->sk;
3711 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3712 unsigned long size, expected_size;
3713 struct packet_ring_buffer *rb;
1da177e4
LT		 3714 unsigned long start;
3715 int err = -EINVAL;
3716 int i;
3717
3718 if (vma->vm_pgoff)
3719 return -EINVAL;
3720
905db440 3721 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3722
3723 expected_size = 0;
3724 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3725 if (rb->pg_vec) {
3726 expected_size += rb->pg_vec_len
3727 * rb->pg_vec_pages
3728 * PAGE_SIZE;
3729 }
3730 }
3731
3732 if (expected_size == 0)
1da177e4 3733 goto out;
69e3c75f
JB		 3734
3735 size = vma->vm_end - vma->vm_start;
3736 if (size != expected_size)
1da177e4
LT		 3737 goto out;
3738
1da177e4 3739 start = vma->vm_start;
69e3c75f
JB		 3740 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3741 if (rb->pg_vec == NULL)
3742 continue;
3743
3744 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3745 struct page *page;
3746 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3747 int pg_num;
3748
c56b4d90
CG		 3749 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3750 page = pgv_to_page(kaddr);
69e3c75f
JB		 3751 err = vm_insert_page(vma, start, page);
3752 if (unlikely(err))
3753 goto out;
3754 start += PAGE_SIZE;
0e3125c7 3755 kaddr += PAGE_SIZE;
69e3c75f 3756 }
4ebf0ae2 3757 }
1da177e4 3758 }
69e3c75f 3759
4ebf0ae2 3760 atomic_inc(&po->mapped);
1da177e4
LT		 3761 vma->vm_ops = &packet_mmap_ops;
3762 err = 0;
3763
3764out:
905db440 3765 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3766 return err;
3767}
1da177e4 3768
90ddc4f0 3769static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3770 .family = PF_PACKET,
3771 .owner = THIS_MODULE,
3772 .release = packet_release,
3773 .bind = packet_bind_spkt,
3774 .connect = sock_no_connect,
3775 .socketpair = sock_no_socketpair,
3776 .accept = sock_no_accept,
3777 .getname = packet_getname_spkt,
3778 .poll = datagram_poll,
3779 .ioctl = packet_ioctl,
3780 .listen = sock_no_listen,
3781 .shutdown = sock_no_shutdown,
3782 .setsockopt = sock_no_setsockopt,
3783 .getsockopt = sock_no_getsockopt,
3784 .sendmsg = packet_sendmsg_spkt,
3785 .recvmsg = packet_recvmsg,
3786 .mmap = sock_no_mmap,
3787 .sendpage = sock_no_sendpage,
3788};
1da177e4 3789
90ddc4f0 3790static const struct proto_ops packet_ops = {
1da177e4
LT		 3791 .family = PF_PACKET,
3792 .owner = THIS_MODULE,
3793 .release = packet_release,
3794 .bind = packet_bind,
3795 .connect = sock_no_connect,
3796 .socketpair = sock_no_socketpair,
3797 .accept = sock_no_accept,
1ce4f28b 3798 .getname = packet_getname,
1da177e4
LT		 3799 .poll = packet_poll,
3800 .ioctl = packet_ioctl,
3801 .listen = sock_no_listen,
3802 .shutdown = sock_no_shutdown,
3803 .setsockopt = packet_setsockopt,
3804 .getsockopt = packet_getsockopt,
3805 .sendmsg = packet_sendmsg,
3806 .recvmsg = packet_recvmsg,
3807 .mmap = packet_mmap,
3808 .sendpage = sock_no_sendpage,
3809};
3810
ec1b4cf7 3811static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3812 .family = PF_PACKET,
3813 .create = packet_create,
3814 .owner = THIS_MODULE,
3815};
3816
3817static struct notifier_block packet_netdev_notifier = {
40d4e3df 3818 .notifier_call = packet_notifier,
1da177e4
LT		 3819};
3820
3821#ifdef CONFIG_PROC_FS
1da177e4
LT		 3822
3823static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3824 __acquires(RCU)
1da177e4 3825{
e372c414 3826 struct net *net = seq_file_net(seq);
808f5114 3827
3828 rcu_read_lock();
3829 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3830}
3831
3832static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3833{
1bf40954 3834 struct net *net = seq_file_net(seq);
808f5114 3835 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3836}
3837
3838static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3839 __releases(RCU)
1da177e4 3840{
808f5114 3841 rcu_read_unlock();
1da177e4
LT		 3842}
3843
1ce4f28b 3844static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3845{
3846 if (v == SEQ_START_TOKEN)
3847 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3848 else {
b7ceabd9 3849 struct sock *s = sk_entry(v);
1da177e4
LT		 3850 const struct packet_sock *po = pkt_sk(s);
3851
3852 seq_printf(seq,
71338aa7 3853 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3854 s,
3855 atomic_read(&s->sk_refcnt),
3856 s->sk_type,
3857 ntohs(po->num),
3858 po->ifindex,
3859 po->running,
3860 atomic_read(&s->sk_rmem_alloc),
3861 sock_i_uid(s),
40d4e3df 3862 sock_i_ino(s));
1da177e4
LT		 3863 }
3864
3865 return 0;
3866}
3867
56b3d975 3868static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3869 .start = packet_seq_start,
3870 .next = packet_seq_next,
3871 .stop = packet_seq_stop,
3872 .show = packet_seq_show,
3873};
3874
3875static int packet_seq_open(struct inode *inode, struct file *file)
3876{
e372c414
DL		 3877 return seq_open_net(inode, file, &packet_seq_ops,
3878 sizeof(struct seq_net_private));
1da177e4
LT		 3879}
3880
da7071d7 3881static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3882 .owner = THIS_MODULE,
3883 .open = packet_seq_open,
3884 .read = seq_read,
3885 .llseek = seq_lseek,
e372c414 3886 .release = seq_release_net,
1da177e4
LT		 3887};
3888
3889#endif
3890
2c8c1e72 3891static int __net_init packet_net_init(struct net *net)
d12d01d6 3892{
808f5114 3893 spin_lock_init(&net->packet.sklist_lock);
2aaef4e4 3894 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6
DL		 3895
3896 if (!proc_net_fops_create(net, "packet", 0, &packet_seq_fops))
3897 return -ENOMEM;
3898
3899 return 0;
3900}
3901
2c8c1e72 3902static void __net_exit packet_net_exit(struct net *net)
d12d01d6
DL		 3903{
3904 proc_net_remove(net, "packet");
3905}
3906
3907static struct pernet_operations packet_net_ops = {
3908 .init = packet_net_init,
3909 .exit = packet_net_exit,
3910};
3911
3912
1da177e4
LT		 3913static void __exit packet_exit(void)
3914{
1da177e4 3915 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3916 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3917 sock_unregister(PF_PACKET);
3918 proto_unregister(&packet_proto);
3919}
3920
3921static int __init packet_init(void)
3922{
3923 int rc = proto_register(&packet_proto, 0);
3924
3925 if (rc != 0)
3926 goto out;
3927
3928 sock_register(&packet_family_ops);
d12d01d6 3929 register_pernet_subsys(&packet_net_ops);
1da177e4 3930 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3931out:
3932 return rc;
3933}
3934
3935module_init(packet_init);
3936module_exit(packet_exit);
3937MODULE_LICENSE("GPL");
3938MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 0.885887 seconds and 5 git commands to generate.