projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
macvtap: Do not double-count received packets
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
5df0ddfb 91#include <linux/reciprocal_div.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
ce06b03e
DM		 240/* register_prot_hook must be invoked with the po->bind_lock held,
241 * or from a context in which asynchronous accesses to the packet
242 * socket is not possible (packet_create()).
243 */
244static void register_prot_hook(struct sock *sk)
245{
246 struct packet_sock *po = pkt_sk(sk);
e40526cb 247
ce06b03e 248 if (!po->running) {
e40526cb 249 if (po->fanout) {
dc99f600 250 __fanout_link(sk, po);
e40526cb 251 } else {
dc99f600 252 dev_add_pack(&po->prot_hook);
e40526cb
DB		 253 rcu_assign_pointer(po->cached_dev, po->prot_hook.dev);
254 }
255
ce06b03e
DM		 256 sock_hold(sk);
257 po->running = 1;
258 }
259}
260
261/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
262 * held. If the sync parameter is true, we will temporarily drop
263 * the po->bind_lock and do a synchronize_net to make sure no
264 * asynchronous packet processing paths still refer to the elements
265 * of po->prot_hook. If the sync parameter is false, it is the
266 * callers responsibility to take care of this.
267 */
268static void __unregister_prot_hook(struct sock *sk, bool sync)
269{
270 struct packet_sock *po = pkt_sk(sk);
271
272 po->running = 0;
e40526cb 273 if (po->fanout) {
dc99f600 274 __fanout_unlink(sk, po);
e40526cb 275 } else {
dc99f600 276 __dev_remove_pack(&po->prot_hook);
e40526cb
DB		 277 RCU_INIT_POINTER(po->cached_dev, NULL);
278 }
279
ce06b03e
DM		 280 __sock_put(sk);
281
282 if (sync) {
283 spin_unlock(&po->bind_lock);
284 synchronize_net();
285 spin_lock(&po->bind_lock);
286 }
287}
288
289static void unregister_prot_hook(struct sock *sk, bool sync)
290{
291 struct packet_sock *po = pkt_sk(sk);
292
293 if (po->running)
294 __unregister_prot_hook(sk, sync);
295}
296
f6dafa95 297static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 298{
299 if (is_vmalloc_addr(addr))
300 return vmalloc_to_page(addr);
301 return virt_to_page(addr);
302}
303
69e3c75f 304static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 305{
184f489e 306 union tpacket_uhdr h;
1da177e4 307
69e3c75f 308 h.raw = frame;
bbd6ef87
PM		 309 switch (po->tp_version) {
310 case TPACKET_V1:
69e3c75f 311 h.h1->tp_status = status;
0af55bb5 312 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 313 break;
314 case TPACKET_V2:
69e3c75f 315 h.h2->tp_status = status;
0af55bb5 316 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 317 break;
f6fb8f10 318 case TPACKET_V3:
69e3c75f 319 default:
f6fb8f10 320 WARN(1, "TPACKET version not supported.\n");
69e3c75f 321 BUG();
bbd6ef87 322 }
69e3c75f
JB		 323
324 smp_wmb();
bbd6ef87
PM		 325}
326
69e3c75f 327static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 328{
184f489e 329 union tpacket_uhdr h;
bbd6ef87 330
69e3c75f
JB		 331 smp_rmb();
332
bbd6ef87
PM		 333 h.raw = frame;
334 switch (po->tp_version) {
335 case TPACKET_V1:
0af55bb5 336 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 337 return h.h1->tp_status;
bbd6ef87 338 case TPACKET_V2:
0af55bb5 339 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 340 return h.h2->tp_status;
f6fb8f10 341 case TPACKET_V3:
69e3c75f 342 default:
f6fb8f10 343 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 344 BUG();
345 return 0;
bbd6ef87 346 }
1da177e4 347}
69e3c75f 348
b9c32fb2
DB		 349static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
350 unsigned int flags)
7a51384c
DB		 351{
352 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
353
354 if (shhwtstamps) {
355 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
356 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
b9c32fb2 357 return TP_STATUS_TS_SYS_HARDWARE;
7a51384c
DB		 358 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
359 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
b9c32fb2 360 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 361 }
362
363 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 364 return TP_STATUS_TS_SOFTWARE;
7a51384c 365
b9c32fb2 366 return 0;
7a51384c
DB		 367}
368
b9c32fb2
DB		 369static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
370 struct sk_buff *skb)
2e31396f
WB		 371{
372 union tpacket_uhdr h;
373 struct timespec ts;
b9c32fb2 374 __u32 ts_status;
2e31396f 375
b9c32fb2
DB		 376 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
377 return 0;
2e31396f
WB		 378
379 h.raw = frame;
380 switch (po->tp_version) {
381 case TPACKET_V1:
382 h.h1->tp_sec = ts.tv_sec;
383 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
384 break;
385 case TPACKET_V2:
386 h.h2->tp_sec = ts.tv_sec;
387 h.h2->tp_nsec = ts.tv_nsec;
388 break;
389 case TPACKET_V3:
390 default:
391 WARN(1, "TPACKET version not supported.\n");
392 BUG();
393 }
394
395 /* one flush is safe, as both fields always lie on the same cacheline */
396 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
397 smp_wmb();
b9c32fb2
DB		 398
399 return ts_status;
2e31396f
WB		 400}
401
69e3c75f
JB		 402static void *packet_lookup_frame(struct packet_sock *po,
403 struct packet_ring_buffer *rb,
404 unsigned int position,
405 int status)
406{
407 unsigned int pg_vec_pos, frame_offset;
184f489e 408 union tpacket_uhdr h;
69e3c75f
JB		 409
410 pg_vec_pos = position / rb->frames_per_block;
411 frame_offset = position % rb->frames_per_block;
412
0e3125c7
NH		 413 h.raw = rb->pg_vec[pg_vec_pos].buffer +
414 (frame_offset * rb->frame_size);
69e3c75f
JB		 415
416 if (status != __packet_get_status(po, h.raw))
417 return NULL;
418
419 return h.raw;
420}
421
eea49cc9 422static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 423 struct packet_ring_buffer *rb,
424 int status)
425{
426 return packet_lookup_frame(po, rb, rb->head, status);
427}
428
bc59ba39 429static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 430{
431 del_timer_sync(&pkc->retire_blk_timer);
432}
433
434static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
435 int tx_ring,
436 struct sk_buff_head *rb_queue)
437{
bc59ba39 438 struct tpacket_kbdq_core *pkc;
f6fb8f10 439
440 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
441
442 spin_lock(&rb_queue->lock);
443 pkc->delete_blk_timer = 1;
444 spin_unlock(&rb_queue->lock);
445
446 prb_del_retire_blk_timer(pkc);
447}
448
449static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 450 struct tpacket_kbdq_core *pkc,
f6fb8f10 451 void (*func) (unsigned long))
452{
453 init_timer(&pkc->retire_blk_timer);
454 pkc->retire_blk_timer.data = (long)po;
455 pkc->retire_blk_timer.function = func;
456 pkc->retire_blk_timer.expires = jiffies;
457}
458
459static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
460{
bc59ba39 461 struct tpacket_kbdq_core *pkc;
f6fb8f10 462
463 if (tx_ring)
464 BUG();
465
466 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
467 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
468}
469
470static int prb_calc_retire_blk_tmo(struct packet_sock *po,
471 int blk_size_in_bytes)
472{
473 struct net_device *dev;
474 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 475 struct ethtool_cmd ecmd;
476 int err;
e440cf2c 477 u32 speed;
f6fb8f10 478
4bc71cb9
JP		 479 rtnl_lock();
480 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
481 if (unlikely(!dev)) {
482 rtnl_unlock();
f6fb8f10 483 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 484 }
485 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 486 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 487 rtnl_unlock();
488 if (!err) {
4bc71cb9
JP		 489 /*
490 * If the link speed is so slow you don't really
491 * need to worry about perf anyways
492 */
e440cf2c 493 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 494 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 495 } else {
496 msec = 1;
497 div = speed / 1000;
f6fb8f10 498 }
499 }
500
501 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
502
503 if (div)
504 mbits /= div;
505
506 tmo = mbits * msec;
507
508 if (div)
509 return tmo+1;
510 return tmo;
511}
512
bc59ba39 513static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 514 union tpacket_req_u *req_u)
515{
516 p1->feature_req_word = req_u->req3.tp_feature_req_word;
517}
518
519static void init_prb_bdqc(struct packet_sock *po,
520 struct packet_ring_buffer *rb,
521 struct pgv *pg_vec,
522 union tpacket_req_u *req_u, int tx_ring)
523{
bc59ba39 524 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
525 struct tpacket_block_desc *pbd;
f6fb8f10 526
527 memset(p1, 0x0, sizeof(*p1));
528
529 p1->knxt_seq_num = 1;
530 p1->pkbdq = pg_vec;
bc59ba39 531 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 532 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 533 p1->kblk_size = req_u->req3.tp_block_size;
534 p1->knum_blocks = req_u->req3.tp_block_nr;
535 p1->hdrlen = po->tp_hdrlen;
536 p1->version = po->tp_version;
537 p1->last_kactive_blk_num = 0;
ee80fbf3 538 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 539 if (req_u->req3.tp_retire_blk_tov)
540 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
541 else
542 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
543 req_u->req3.tp_block_size);
544 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
545 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
546
547 prb_init_ft_ops(p1, req_u);
548 prb_setup_retire_blk_timer(po, tx_ring);
549 prb_open_block(p1, pbd);
550}
551
552/* Do NOT update the last_blk_num first.
553 * Assumes sk_buff_head lock is held.
554 */
bc59ba39 555static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 556{
557 mod_timer(&pkc->retire_blk_timer,
558 jiffies + pkc->tov_in_jiffies);
559 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
560}
561
562/*
563 * Timer logic:
564 * 1) We refresh the timer only when we open a block.
565 * By doing this we don't waste cycles refreshing the timer
566 * on packet-by-packet basis.
567 *
568 * With a 1MB block-size, on a 1Gbps line, it will take
569 * i) ~8 ms to fill a block + ii) memcpy etc.
570 * In this cut we are not accounting for the memcpy time.
571 *
572 * So, if the user sets the 'tmo' to 10ms then the timer
573 * will never fire while the block is still getting filled
574 * (which is what we want). However, the user could choose
575 * to close a block early and that's fine.
576 *
577 * But when the timer does fire, we check whether or not to refresh it.
578 * Since the tmo granularity is in msecs, it is not too expensive
579 * to refresh the timer, lets say every '8' msecs.
580 * Either the user can set the 'tmo' or we can derive it based on
581 * a) line-speed and b) block-size.
582 * prb_calc_retire_blk_tmo() calculates the tmo.
583 *
584 */
585static void prb_retire_rx_blk_timer_expired(unsigned long data)
586{
587 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 588 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 589 unsigned int frozen;
bc59ba39 590 struct tpacket_block_desc *pbd;
f6fb8f10 591
592 spin_lock(&po->sk.sk_receive_queue.lock);
593
594 frozen = prb_queue_frozen(pkc);
595 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
596
597 if (unlikely(pkc->delete_blk_timer))
598 goto out;
599
600 /* We only need to plug the race when the block is partially filled.
601 * tpacket_rcv:
602 * lock(); increment BLOCK_NUM_PKTS; unlock()
603 * copy_bits() is in progress ...
604 * timer fires on other cpu:
605 * we can't retire the current block because copy_bits
606 * is in progress.
607 *
608 */
609 if (BLOCK_NUM_PKTS(pbd)) {
610 while (atomic_read(&pkc->blk_fill_in_prog)) {
611 /* Waiting for skb_copy_bits to finish... */
612 cpu_relax();
613 }
614 }
615
616 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
617 if (!frozen) {
618 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
619 if (!prb_dispatch_next_block(pkc, po))
620 goto refresh_timer;
621 else
622 goto out;
623 } else {
624 /* Case 1. Queue was frozen because user-space was
625 * lagging behind.
626 */
627 if (prb_curr_blk_in_use(pkc, pbd)) {
628 /*
629 * Ok, user-space is still behind.
630 * So just refresh the timer.
631 */
632 goto refresh_timer;
633 } else {
634 /* Case 2. queue was frozen,user-space caught up,
635 * now the link went idle && the timer fired.
636 * We don't have a block to close.So we open this
637 * block and restart the timer.
638 * opening a block thaws the queue,restarts timer
639 * Thawing/timer-refresh is a side effect.
640 */
641 prb_open_block(pkc, pbd);
642 goto out;
643 }
644 }
645 }
646
647refresh_timer:
648 _prb_refresh_rx_retire_blk_timer(pkc);
649
650out:
651 spin_unlock(&po->sk.sk_receive_queue.lock);
652}
653
eea49cc9 654static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 655 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 656{
657 /* Flush everything minus the block header */
658
659#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
660 u8 *start, *end;
661
662 start = (u8 *)pbd1;
663
664 /* Skip the block header(we know header WILL fit in 4K) */
665 start += PAGE_SIZE;
666
667 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
668 for (; start < end; start += PAGE_SIZE)
669 flush_dcache_page(pgv_to_page(start));
670
671 smp_wmb();
672#endif
673
674 /* Now update the block status. */
675
676 BLOCK_STATUS(pbd1) = status;
677
678 /* Flush the block header */
679
680#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
681 start = (u8 *)pbd1;
682 flush_dcache_page(pgv_to_page(start));
683
684 smp_wmb();
685#endif
686}
687
688/*
689 * Side effect:
690 *
691 * 1) flush the block
692 * 2) Increment active_blk_num
693 *
694 * Note:We DONT refresh the timer on purpose.
695 * Because almost always the next block will be opened.
696 */
bc59ba39 697static void prb_close_block(struct tpacket_kbdq_core *pkc1,
698 struct tpacket_block_desc *pbd1,
f6fb8f10 699 struct packet_sock *po, unsigned int stat)
700{
701 __u32 status = TP_STATUS_USER | stat;
702
703 struct tpacket3_hdr *last_pkt;
bc59ba39 704 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 705
ee80fbf3 706 if (po->stats.stats3.tp_drops)
f6fb8f10 707 status |= TP_STATUS_LOSING;
708
709 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
710 last_pkt->tp_next_offset = 0;
711
712 /* Get the ts of the last pkt */
713 if (BLOCK_NUM_PKTS(pbd1)) {
714 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
715 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
716 } else {
717 /* Ok, we tmo'd - so get the current time */
718 struct timespec ts;
719 getnstimeofday(&ts);
720 h1->ts_last_pkt.ts_sec = ts.tv_sec;
721 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
722 }
723
724 smp_wmb();
725
726 /* Flush the block */
727 prb_flush_block(pkc1, pbd1, status);
728
729 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
730}
731
eea49cc9 732static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 733{
734 pkc->reset_pending_on_curr_blk = 0;
735}
736
737/*
738 * Side effect of opening a block:
739 *
740 * 1) prb_queue is thawed.
741 * 2) retire_blk_timer is refreshed.
742 *
743 */
bc59ba39 744static void prb_open_block(struct tpacket_kbdq_core *pkc1,
745 struct tpacket_block_desc *pbd1)
f6fb8f10 746{
747 struct timespec ts;
bc59ba39 748 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 749
750 smp_rmb();
751
8da3056c
DB		 752 /* We could have just memset this but we will lose the
753 * flexibility of making the priv area sticky
754 */
f6fb8f10 755
8da3056c
DB		 756 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
757 BLOCK_NUM_PKTS(pbd1) = 0;
758 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 759
8da3056c
DB		 760 getnstimeofday(&ts);
761
762 h1->ts_first_pkt.ts_sec = ts.tv_sec;
763 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 764
8da3056c
DB		 765 pkc1->pkblk_start = (char *)pbd1;
766 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
767
768 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
769 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
770
771 pbd1->version = pkc1->version;
772 pkc1->prev = pkc1->nxt_offset;
773 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
774
775 prb_thaw_queue(pkc1);
776 _prb_refresh_rx_retire_blk_timer(pkc1);
777
778 smp_wmb();
f6fb8f10 779}
780
781/*
782 * Queue freeze logic:
783 * 1) Assume tp_block_nr = 8 blocks.
784 * 2) At time 't0', user opens Rx ring.
785 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
786 * 4) user-space is either sleeping or processing block '0'.
787 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
788 * it will close block-7,loop around and try to fill block '0'.
789 * call-flow:
790 * __packet_lookup_frame_in_block
791 * prb_retire_current_block()
792 * prb_dispatch_next_block()
793 * |->(BLOCK_STATUS == USER) evaluates to true
794 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
795 * 6) Now there are two cases:
796 * 6.1) Link goes idle right after the queue is frozen.
797 * But remember, the last open_block() refreshed the timer.
798 * When this timer expires,it will refresh itself so that we can
799 * re-open block-0 in near future.
800 * 6.2) Link is busy and keeps on receiving packets. This is a simple
801 * case and __packet_lookup_frame_in_block will check if block-0
802 * is free and can now be re-used.
803 */
eea49cc9 804static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 805 struct packet_sock *po)
806{
807 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 808 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 809}
810
811#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
812
813/*
814 * If the next block is free then we will dispatch it
815 * and return a good offset.
816 * Else, we will freeze the queue.
817 * So, caller must check the return value.
818 */
bc59ba39 819static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 820 struct packet_sock *po)
821{
bc59ba39 822 struct tpacket_block_desc *pbd;
f6fb8f10 823
824 smp_rmb();
825
826 /* 1. Get current block num */
827 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
828
829 /* 2. If this block is currently in_use then freeze the queue */
830 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
831 prb_freeze_queue(pkc, po);
832 return NULL;
833 }
834
835 /*
836 * 3.
837 * open this block and return the offset where the first packet
838 * needs to get stored.
839 */
840 prb_open_block(pkc, pbd);
841 return (void *)pkc->nxt_offset;
842}
843
bc59ba39 844static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 845 struct packet_sock *po, unsigned int status)
846{
bc59ba39 847 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 848
849 /* retire/close the current block */
850 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
851 /*
852 * Plug the case where copy_bits() is in progress on
853 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
854 * have space to copy the pkt in the current block and
855 * called prb_retire_current_block()
856 *
857 * We don't need to worry about the TMO case because
858 * the timer-handler already handled this case.
859 */
860 if (!(status & TP_STATUS_BLK_TMO)) {
861 while (atomic_read(&pkc->blk_fill_in_prog)) {
862 /* Waiting for skb_copy_bits to finish... */
863 cpu_relax();
864 }
865 }
866 prb_close_block(pkc, pbd, po, status);
867 return;
868 }
f6fb8f10 869}
870
eea49cc9 871static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 872 struct tpacket_block_desc *pbd)
f6fb8f10 873{
874 return TP_STATUS_USER & BLOCK_STATUS(pbd);
875}
876
eea49cc9 877static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 878{
879 return pkc->reset_pending_on_curr_blk;
880}
881
eea49cc9 882static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 883{
bc59ba39 884 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 885 atomic_dec(&pkc->blk_fill_in_prog);
886}
887
eea49cc9 888static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 889 struct tpacket3_hdr *ppd)
890{
891 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
892}
893
eea49cc9 894static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 895 struct tpacket3_hdr *ppd)
896{
897 ppd->hv1.tp_rxhash = 0;
898}
899
eea49cc9 900static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 901 struct tpacket3_hdr *ppd)
902{
903 if (vlan_tx_tag_present(pkc->skb)) {
904 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
905 ppd->tp_status = TP_STATUS_VLAN_VALID;
906 } else {
9e67030a 907 ppd->hv1.tp_vlan_tci = 0;
908 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 909 }
910}
911
bc59ba39 912static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 913 struct tpacket3_hdr *ppd)
914{
915 prb_fill_vlan_info(pkc, ppd);
916
917 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
918 prb_fill_rxhash(pkc, ppd);
919 else
920 prb_clear_rxhash(pkc, ppd);
921}
922
eea49cc9 923static void prb_fill_curr_block(char *curr,
bc59ba39 924 struct tpacket_kbdq_core *pkc,
925 struct tpacket_block_desc *pbd,
f6fb8f10 926 unsigned int len)
927{
928 struct tpacket3_hdr *ppd;
929
930 ppd = (struct tpacket3_hdr *)curr;
931 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
932 pkc->prev = curr;
933 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
934 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
935 BLOCK_NUM_PKTS(pbd) += 1;
936 atomic_inc(&pkc->blk_fill_in_prog);
937 prb_run_all_ft_ops(pkc, ppd);
938}
939
940/* Assumes caller has the sk->rx_queue.lock */
941static void *__packet_lookup_frame_in_block(struct packet_sock *po,
942 struct sk_buff *skb,
943 int status,
944 unsigned int len
945 )
946{
bc59ba39 947 struct tpacket_kbdq_core *pkc;
948 struct tpacket_block_desc *pbd;
f6fb8f10 949 char *curr, *end;
950
e3192690 951 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 952 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
953
954 /* Queue is frozen when user space is lagging behind */
955 if (prb_queue_frozen(pkc)) {
956 /*
957 * Check if that last block which caused the queue to freeze,
958 * is still in_use by user-space.
959 */
960 if (prb_curr_blk_in_use(pkc, pbd)) {
961 /* Can't record this packet */
962 return NULL;
963 } else {
964 /*
965 * Ok, the block was released by user-space.
966 * Now let's open that block.
967 * opening a block also thaws the queue.
968 * Thawing is a side effect.
969 */
970 prb_open_block(pkc, pbd);
971 }
972 }
973
974 smp_mb();
975 curr = pkc->nxt_offset;
976 pkc->skb = skb;
e3192690 977 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 978
979 /* first try the current block */
980 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
981 prb_fill_curr_block(curr, pkc, pbd, len);
982 return (void *)curr;
983 }
984
985 /* Ok, close the current block */
986 prb_retire_current_block(pkc, po, 0);
987
988 /* Now, try to dispatch the next block */
989 curr = (char *)prb_dispatch_next_block(pkc, po);
990 if (curr) {
991 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
992 prb_fill_curr_block(curr, pkc, pbd, len);
993 return (void *)curr;
994 }
995
996 /*
997 * No free blocks are available.user_space hasn't caught up yet.
998 * Queue was just frozen and now this packet will get dropped.
999 */
1000 return NULL;
1001}
1002
eea49cc9 1003static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1004 struct sk_buff *skb,
1005 int status, unsigned int len)
1006{
1007 char *curr = NULL;
1008 switch (po->tp_version) {
1009 case TPACKET_V1:
1010 case TPACKET_V2:
1011 curr = packet_lookup_frame(po, &po->rx_ring,
1012 po->rx_ring.head, status);
1013 return curr;
1014 case TPACKET_V3:
1015 return __packet_lookup_frame_in_block(po, skb, status, len);
1016 default:
1017 WARN(1, "TPACKET version not supported\n");
1018 BUG();
99aa3473 1019 return NULL;
f6fb8f10 1020 }
1021}
1022
eea49cc9 1023static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1024 struct packet_ring_buffer *rb,
77f65ebd 1025 unsigned int idx,
f6fb8f10 1026 int status)
1027{
bc59ba39 1028 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1029 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1030
1031 if (status != BLOCK_STATUS(pbd))
1032 return NULL;
1033 return pbd;
1034}
1035
eea49cc9 1036static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1037{
1038 unsigned int prev;
1039 if (rb->prb_bdqc.kactive_blk_num)
1040 prev = rb->prb_bdqc.kactive_blk_num-1;
1041 else
1042 prev = rb->prb_bdqc.knum_blocks-1;
1043 return prev;
1044}
1045
1046/* Assumes caller has held the rx_queue.lock */
eea49cc9 1047static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1048 struct packet_ring_buffer *rb,
1049 int status)
1050{
1051 unsigned int previous = prb_previous_blk_num(rb);
1052 return prb_lookup_block(po, rb, previous, status);
1053}
1054
eea49cc9 1055static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1056 struct packet_ring_buffer *rb,
1057 int status)
1058{
1059 if (po->tp_version <= TPACKET_V2)
1060 return packet_previous_frame(po, rb, status);
1061
1062 return __prb_previous_block(po, rb, status);
1063}
1064
eea49cc9 1065static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1066 struct packet_ring_buffer *rb)
1067{
1068 switch (po->tp_version) {
1069 case TPACKET_V1:
1070 case TPACKET_V2:
1071 return packet_increment_head(rb);
1072 case TPACKET_V3:
1073 default:
1074 WARN(1, "TPACKET version not supported.\n");
1075 BUG();
1076 return;
1077 }
1078}
1079
eea49cc9 1080static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1081 struct packet_ring_buffer *rb,
1082 int status)
1083{
1084 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1085 return packet_lookup_frame(po, rb, previous, status);
1086}
1087
eea49cc9 1088static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1089{
1090 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1091}
1092
77f65ebd
WB		 1093static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1094{
1095 struct sock *sk = &po->sk;
1096 bool has_room;
1097
1098 if (po->prot_hook.func != tpacket_rcv)
1099 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1100 <= sk->sk_rcvbuf;
1101
1102 spin_lock(&sk->sk_receive_queue.lock);
1103 if (po->tp_version == TPACKET_V3)
1104 has_room = prb_lookup_block(po, &po->rx_ring,
1105 po->rx_ring.prb_bdqc.kactive_blk_num,
1106 TP_STATUS_KERNEL);
1107 else
1108 has_room = packet_lookup_frame(po, &po->rx_ring,
1109 po->rx_ring.head,
1110 TP_STATUS_KERNEL);
1111 spin_unlock(&sk->sk_receive_queue.lock);
1112
1113 return has_room;
1114}
1115
1da177e4
LT		 1116static void packet_sock_destruct(struct sock *sk)
1117{
ed85b565
RC		 1118 skb_queue_purge(&sk->sk_error_queue);
1119
547b792c
IJ		 1120 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1121 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1122
1123 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1124 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1125 return;
1126 }
1127
17ab56a2 1128 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1129}
1130
dc99f600
DM		 1131static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1132{
1133 int x = atomic_read(&f->rr_cur) + 1;
1134
1135 if (x >= num)
1136 x = 0;
1137
1138 return x;
1139}
1140
77f65ebd
WB		 1141static unsigned int fanout_demux_hash(struct packet_fanout *f,
1142 struct sk_buff *skb,
1143 unsigned int num)
dc99f600 1144{
f55d112e 1145 return reciprocal_divide(skb->rxhash, num);
dc99f600
DM		 1146}
1147
77f65ebd
WB		 1148static unsigned int fanout_demux_lb(struct packet_fanout *f,
1149 struct sk_buff *skb,
1150 unsigned int num)
dc99f600
DM		 1151{
1152 int cur, old;
1153
1154 cur = atomic_read(&f->rr_cur);
1155 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1156 fanout_rr_next(f, num))) != cur)
1157 cur = old;
77f65ebd
WB		 1158 return cur;
1159}
1160
1161static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1162 struct sk_buff *skb,
1163 unsigned int num)
1164{
1165 return smp_processor_id() % num;
dc99f600
DM		 1166}
1167
5df0ddfb
DB		 1168static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1169 struct sk_buff *skb,
1170 unsigned int num)
1171{
1172 return reciprocal_divide(prandom_u32(), num);
1173}
1174
77f65ebd
WB		 1175static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1176 struct sk_buff *skb,
1177 unsigned int idx, unsigned int skip,
1178 unsigned int num)
95ec3eb4 1179{
77f65ebd 1180 unsigned int i, j;
95ec3eb4 1181
77f65ebd
WB		 1182 i = j = min_t(int, f->next[idx], num - 1);
1183 do {
1184 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1185 if (i != j)
1186 f->next[idx] = i;
1187 return i;
1188 }
1189 if (++i == num)
1190 i = 0;
1191 } while (i != j);
1192
1193 return idx;
1194}
1195
1196static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1197{
1198 return f->flags & (flag >> 8);
95ec3eb4
DM		 1199}
1200
95ec3eb4
DM		 1201static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1202 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1203{
1204 struct packet_fanout *f = pt->af_packet_priv;
1205 unsigned int num = f->num_members;
1206 struct packet_sock *po;
77f65ebd 1207 unsigned int idx;
dc99f600
DM		 1208
1209 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1210 !num) {
1211 kfree_skb(skb);
1212 return 0;
1213 }
1214
95ec3eb4
DM		 1215 switch (f->type) {
1216 case PACKET_FANOUT_HASH:
1217 default:
77f65ebd 1218 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1219 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1220 if (!skb)
1221 return 0;
1222 }
1223 skb_get_rxhash(skb);
77f65ebd 1224 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1225 break;
1226 case PACKET_FANOUT_LB:
77f65ebd 1227 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1228 break;
1229 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1230 idx = fanout_demux_cpu(f, skb, num);
1231 break;
5df0ddfb
DB		 1232 case PACKET_FANOUT_RND:
1233 idx = fanout_demux_rnd(f, skb, num);
1234 break;
77f65ebd
WB		 1235 case PACKET_FANOUT_ROLLOVER:
1236 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1237 break;
dc99f600
DM		 1238 }
1239
77f65ebd
WB		 1240 po = pkt_sk(f->arr[idx]);
1241 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1242 unlikely(!packet_rcv_has_room(po, skb))) {
1243 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1244 po = pkt_sk(f->arr[idx]);
1245 }
dc99f600
DM		 1246
1247 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1248}
1249
fff3321d
PE		 1250DEFINE_MUTEX(fanout_mutex);
1251EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1252static LIST_HEAD(fanout_list);
1253
1254static void __fanout_link(struct sock *sk, struct packet_sock *po)
1255{
1256 struct packet_fanout *f = po->fanout;
1257
1258 spin_lock(&f->lock);
1259 f->arr[f->num_members] = sk;
1260 smp_wmb();
1261 f->num_members++;
1262 spin_unlock(&f->lock);
1263}
1264
1265static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1266{
1267 struct packet_fanout *f = po->fanout;
1268 int i;
1269
1270 spin_lock(&f->lock);
1271 for (i = 0; i < f->num_members; i++) {
1272 if (f->arr[i] == sk)
1273 break;
1274 }
1275 BUG_ON(i >= f->num_members);
1276 f->arr[i] = f->arr[f->num_members - 1];
1277 f->num_members--;
1278 spin_unlock(&f->lock);
1279}
1280
a0dfb263 1281static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
c0de08d0
EL		 1282{
1283 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1284 return true;
1285
1286 return false;
1287}
1288
7736d33f 1289static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1290{
1291 struct packet_sock *po = pkt_sk(sk);
1292 struct packet_fanout *f, *match;
7736d33f 1293 u8 type = type_flags & 0xff;
77f65ebd 1294 u8 flags = type_flags >> 8;
dc99f600
DM		 1295 int err;
1296
1297 switch (type) {
77f65ebd
WB		 1298 case PACKET_FANOUT_ROLLOVER:
1299 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1300 return -EINVAL;
dc99f600
DM		 1301 case PACKET_FANOUT_HASH:
1302 case PACKET_FANOUT_LB:
95ec3eb4 1303 case PACKET_FANOUT_CPU:
5df0ddfb 1304 case PACKET_FANOUT_RND:
dc99f600
DM		 1305 break;
1306 default:
1307 return -EINVAL;
1308 }
1309
1310 if (!po->running)
1311 return -EINVAL;
1312
1313 if (po->fanout)
1314 return -EALREADY;
1315
1316 mutex_lock(&fanout_mutex);
1317 match = NULL;
1318 list_for_each_entry(f, &fanout_list, list) {
1319 if (f->id == id &&
1320 read_pnet(&f->net) == sock_net(sk)) {
1321 match = f;
1322 break;
1323 }
1324 }
afe62c68 1325 err = -EINVAL;
77f65ebd 1326 if (match && match->flags != flags)
afe62c68 1327 goto out;
dc99f600 1328 if (!match) {
afe62c68 1329 err = -ENOMEM;
dc99f600 1330 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1331 if (!match)
1332 goto out;
1333 write_pnet(&match->net, sock_net(sk));
1334 match->id = id;
1335 match->type = type;
77f65ebd 1336 match->flags = flags;
afe62c68
ED		 1337 atomic_set(&match->rr_cur, 0);
1338 INIT_LIST_HEAD(&match->list);
1339 spin_lock_init(&match->lock);
1340 atomic_set(&match->sk_ref, 0);
1341 match->prot_hook.type = po->prot_hook.type;
1342 match->prot_hook.dev = po->prot_hook.dev;
1343 match->prot_hook.func = packet_rcv_fanout;
1344 match->prot_hook.af_packet_priv = match;
c0de08d0 1345 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1346 dev_add_pack(&match->prot_hook);
1347 list_add(&match->list, &fanout_list);
dc99f600 1348 }
afe62c68
ED		 1349 err = -EINVAL;
1350 if (match->type == type &&
1351 match->prot_hook.type == po->prot_hook.type &&
1352 match->prot_hook.dev == po->prot_hook.dev) {
1353 err = -ENOSPC;
1354 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1355 __dev_remove_pack(&po->prot_hook);
1356 po->fanout = match;
1357 atomic_inc(&match->sk_ref);
1358 __fanout_link(sk, po);
1359 err = 0;
dc99f600
DM		 1360 }
1361 }
afe62c68 1362out:
dc99f600
DM		 1363 mutex_unlock(&fanout_mutex);
1364 return err;
1365}
1366
1367static void fanout_release(struct sock *sk)
1368{
1369 struct packet_sock *po = pkt_sk(sk);
1370 struct packet_fanout *f;
1371
1372 f = po->fanout;
1373 if (!f)
1374 return;
1375
fff3321d 1376 mutex_lock(&fanout_mutex);
dc99f600
DM		 1377 po->fanout = NULL;
1378
dc99f600
DM		 1379 if (atomic_dec_and_test(&f->sk_ref)) {
1380 list_del(&f->list);
1381 dev_remove_pack(&f->prot_hook);
1382 kfree(f);
1383 }
1384 mutex_unlock(&fanout_mutex);
1385}
1da177e4 1386
90ddc4f0 1387static const struct proto_ops packet_ops;
1da177e4 1388
90ddc4f0 1389static const struct proto_ops packet_ops_spkt;
1da177e4 1390
40d4e3df
ED		 1391static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1392 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1393{
1394 struct sock *sk;
1395 struct sockaddr_pkt *spkt;
1396
1397 /*
1398 * When we registered the protocol we saved the socket in the data
1399 * field for just this event.
1400 */
1401
1402 sk = pt->af_packet_priv;
1ce4f28b 1403
1da177e4
LT		 1404 /*
1405 * Yank back the headers [hope the device set this
1406 * right or kerboom...]
1407 *
1408 * Incoming packets have ll header pulled,
1409 * push it back.
1410 *
98e399f8 1411 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1412 * so that this procedure is noop.
1413 */
1414
1415 if (skb->pkt_type == PACKET_LOOPBACK)
1416 goto out;
1417
09ad9bc7 1418 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1419 goto out;
1420
40d4e3df
ED		 1421 skb = skb_share_check(skb, GFP_ATOMIC);
1422 if (skb == NULL)
1da177e4
LT		 1423 goto oom;
1424
1425 /* drop any routing info */
adf30907 1426 skb_dst_drop(skb);
1da177e4 1427
84531c24
PO		 1428 /* drop conntrack reference */
1429 nf_reset(skb);
1430
ffbc6111 1431 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1432
98e399f8 1433 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1434
1435 /*
1436 * The SOCK_PACKET socket receives _all_ frames.
1437 */
1438
1439 spkt->spkt_family = dev->type;
1440 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1441 spkt->spkt_protocol = skb->protocol;
1442
1443 /*
1444 * Charge the memory to the socket. This is done specifically
1445 * to prevent sockets using all the memory up.
1446 */
1447
40d4e3df 1448 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1449 return 0;
1450
1451out:
1452 kfree_skb(skb);
1453oom:
1454 return 0;
1455}
1456
1457
1458/*
1459 * Output a raw packet to a device layer. This bypasses all the other
1460 * protocol layers and you must therefore supply it with a complete frame
1461 */
1ce4f28b 1462
1da177e4
LT		 1463static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1464 struct msghdr *msg, size_t len)
1465{
1466 struct sock *sk = sock->sk;
40d4e3df 1467 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1468 struct sk_buff *skb = NULL;
1da177e4 1469 struct net_device *dev;
40d4e3df 1470 __be16 proto = 0;
1da177e4 1471 int err;
3bdc0eba 1472 int extra_len = 0;
1ce4f28b 1473
1da177e4 1474 /*
1ce4f28b 1475 * Get and verify the address.
1da177e4
LT		 1476 */
1477
40d4e3df 1478 if (saddr) {
1da177e4 1479 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1480 return -EINVAL;
1481 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1482 proto = saddr->spkt_protocol;
1483 } else
1484 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1485
1486 /*
1ce4f28b 1487 * Find the device first to size check it
1da177e4
LT		 1488 */
1489
de74e92a 1490 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1491retry:
654d1f8a
ED		 1492 rcu_read_lock();
1493 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1494 err = -ENODEV;
1495 if (dev == NULL)
1496 goto out_unlock;
1ce4f28b 1497
d5e76b0a
DM		 1498 err = -ENETDOWN;
1499 if (!(dev->flags & IFF_UP))
1500 goto out_unlock;
1501
1da177e4 1502 /*
40d4e3df
ED		 1503 * You may not queue a frame bigger than the mtu. This is the lowest level
1504 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1505 */
1ce4f28b 1506
3bdc0eba
BG		 1507 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1508 if (!netif_supports_nofcs(dev)) {
1509 err = -EPROTONOSUPPORT;
1510 goto out_unlock;
1511 }
1512 extra_len = 4; /* We're doing our own CRC */
1513 }
1514
1da177e4 1515 err = -EMSGSIZE;
3bdc0eba 1516 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1517 goto out_unlock;
1518
1a35ca80
ED		 1519 if (!skb) {
1520 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1521 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1522 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1523
1524 rcu_read_unlock();
4ce40912 1525 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1526 if (skb == NULL)
1527 return -ENOBUFS;
1528 /* FIXME: Save some space for broken drivers that write a hard
1529 * header at transmission time by themselves. PPP is the notable
1530 * one here. This should really be fixed at the driver level.
1531 */
1532 skb_reserve(skb, reserved);
1533 skb_reset_network_header(skb);
1534
1535 /* Try to align data part correctly */
1536 if (hhlen) {
1537 skb->data -= hhlen;
1538 skb->tail -= hhlen;
1539 if (len < hhlen)
1540 skb_reset_network_header(skb);
1541 }
1542 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1543 if (err)
1544 goto out_free;
1545 goto retry;
1da177e4
LT		 1546 }
1547
3bdc0eba 1548 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1549 /* Earlier code assumed this would be a VLAN pkt,
1550 * double-check this now that we have the actual
1551 * packet in hand.
1552 */
1553 struct ethhdr *ehdr;
1554 skb_reset_mac_header(skb);
1555 ehdr = eth_hdr(skb);
1556 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1557 err = -EMSGSIZE;
1558 goto out_unlock;
1559 }
1560 }
1a35ca80 1561
1da177e4
LT		 1562 skb->protocol = proto;
1563 skb->dev = dev;
1564 skb->priority = sk->sk_priority;
2d37a186 1565 skb->mark = sk->sk_mark;
bf84a010
DB		 1566
1567 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1568
3bdc0eba
BG		 1569 if (unlikely(extra_len == 4))
1570 skb->no_fcs = 1;
1571
40893fd0 1572 skb_probe_transport_header(skb, 0);
c1aad275 1573
1da177e4 1574 dev_queue_xmit(skb);
654d1f8a 1575 rcu_read_unlock();
40d4e3df 1576 return len;
1da177e4 1577
1da177e4 1578out_unlock:
654d1f8a 1579 rcu_read_unlock();
1a35ca80
ED		 1580out_free:
1581 kfree_skb(skb);
1da177e4
LT		 1582 return err;
1583}
1da177e4 1584
eea49cc9 1585static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1586 const struct sock *sk,
dbcb5855 1587 unsigned int res)
1da177e4
LT		 1588{
1589 struct sk_filter *filter;
fda9ef5d 1590
80f8f102
ED		 1591 rcu_read_lock();
1592 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1593 if (filter != NULL)
0a14842f 1594 res = SK_RUN_FILTER(filter, skb);
80f8f102 1595 rcu_read_unlock();
1da177e4 1596
dbcb5855 1597 return res;
1da177e4
LT		 1598}
1599
1600/*
62ab0812
ED		 1601 * This function makes lazy skb cloning in hope that most of packets
1602 * are discarded by BPF.
1603 *
1604 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1605 * and skb->cb are mangled. It works because (and until) packets
1606 * falling here are owned by current CPU. Output packets are cloned
1607 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1608 * sequencially, so that if we return skb to original state on exit,
1609 * we will not harm anyone.
1da177e4
LT		 1610 */
1611
40d4e3df
ED		 1612static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1613 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1614{
1615 struct sock *sk;
1616 struct sockaddr_ll *sll;
1617 struct packet_sock *po;
40d4e3df 1618 u8 *skb_head = skb->data;
1da177e4 1619 int skb_len = skb->len;
dbcb5855 1620 unsigned int snaplen, res;
1da177e4
LT		 1621
1622 if (skb->pkt_type == PACKET_LOOPBACK)
1623 goto drop;
1624
1625 sk = pt->af_packet_priv;
1626 po = pkt_sk(sk);
1627
09ad9bc7 1628 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1629 goto drop;
1630
1da177e4
LT		 1631 skb->dev = dev;
1632
3b04ddde 1633 if (dev->header_ops) {
1da177e4 1634 /* The device has an explicit notion of ll header,
62ab0812
ED		 1635 * exported to higher levels.
1636 *
1637 * Otherwise, the device hides details of its frame
1638 * structure, so that corresponding packet head is
1639 * never delivered to user.
1da177e4
LT		 1640 */
1641 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1642 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1643 else if (skb->pkt_type == PACKET_OUTGOING) {
1644 /* Special case: outgoing packets have ll header at head */
bbe735e4 1645 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1646 }
1647 }
1648
1649 snaplen = skb->len;
1650
dbcb5855
DM		 1651 res = run_filter(skb, sk, snaplen);
1652 if (!res)
fda9ef5d 1653 goto drop_n_restore;
dbcb5855
DM		 1654 if (snaplen > res)
1655 snaplen = res;
1da177e4 1656
0fd7bac6 1657 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1658 goto drop_n_acct;
1659
1660 if (skb_shared(skb)) {
1661 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1662 if (nskb == NULL)
1663 goto drop_n_acct;
1664
1665 if (skb_head != skb->data) {
1666 skb->data = skb_head;
1667 skb->len = skb_len;
1668 }
abc4e4fa 1669 consume_skb(skb);
1da177e4
LT		 1670 skb = nskb;
1671 }
1672
ffbc6111
HX		 1673 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1674 sizeof(skb->cb));
1675
1676 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1677 sll->sll_family = AF_PACKET;
1678 sll->sll_hatype = dev->type;
1679 sll->sll_protocol = skb->protocol;
1680 sll->sll_pkttype = skb->pkt_type;
8032b464 1681 if (unlikely(po->origdev))
80feaacb
PWJ		 1682 sll->sll_ifindex = orig_dev->ifindex;
1683 else
1684 sll->sll_ifindex = dev->ifindex;
1da177e4 1685
b95cce35 1686 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1687
ffbc6111 1688 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1689
1da177e4
LT		 1690 if (pskb_trim(skb, snaplen))
1691 goto drop_n_acct;
1692
1693 skb_set_owner_r(skb, sk);
1694 skb->dev = NULL;
adf30907 1695 skb_dst_drop(skb);
1da177e4 1696
84531c24
PO		 1697 /* drop conntrack reference */
1698 nf_reset(skb);
1699
1da177e4 1700 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1701 po->stats.stats1.tp_packets++;
3b885787 1702 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1703 __skb_queue_tail(&sk->sk_receive_queue, skb);
1704 spin_unlock(&sk->sk_receive_queue.lock);
1705 sk->sk_data_ready(sk, skb->len);
1706 return 0;
1707
1708drop_n_acct:
7091fbd8 1709 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1710 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1711 atomic_inc(&sk->sk_drops);
1712 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1713
1714drop_n_restore:
1715 if (skb_head != skb->data && skb_shared(skb)) {
1716 skb->data = skb_head;
1717 skb->len = skb_len;
1718 }
1719drop:
ead2ceb0 1720 consume_skb(skb);
1da177e4
LT		 1721 return 0;
1722}
1723
40d4e3df
ED		 1724static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1725 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1726{
1727 struct sock *sk;
1728 struct packet_sock *po;
1729 struct sockaddr_ll *sll;
184f489e 1730 union tpacket_uhdr h;
40d4e3df 1731 u8 *skb_head = skb->data;
1da177e4 1732 int skb_len = skb->len;
dbcb5855 1733 unsigned int snaplen, res;
f6fb8f10 1734 unsigned long status = TP_STATUS_USER;
bbd6ef87 1735 unsigned short macoff, netoff, hdrlen;
1da177e4 1736 struct sk_buff *copy_skb = NULL;
bbd6ef87 1737 struct timespec ts;
b9c32fb2 1738 __u32 ts_status;
1da177e4
LT		 1739
1740 if (skb->pkt_type == PACKET_LOOPBACK)
1741 goto drop;
1742
1743 sk = pt->af_packet_priv;
1744 po = pkt_sk(sk);
1745
09ad9bc7 1746 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1747 goto drop;
1748
3b04ddde 1749 if (dev->header_ops) {
1da177e4 1750 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1751 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1752 else if (skb->pkt_type == PACKET_OUTGOING) {
1753 /* Special case: outgoing packets have ll header at head */
bbe735e4 1754 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1755 }
1756 }
1757
8dc41944
HX		 1758 if (skb->ip_summed == CHECKSUM_PARTIAL)
1759 status |= TP_STATUS_CSUMNOTREADY;
1760
1da177e4
LT		 1761 snaplen = skb->len;
1762
dbcb5855
DM		 1763 res = run_filter(skb, sk, snaplen);
1764 if (!res)
fda9ef5d 1765 goto drop_n_restore;
dbcb5855
DM		 1766 if (snaplen > res)
1767 snaplen = res;
1da177e4
LT		 1768
1769 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1770 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1771 po->tp_reserve;
1da177e4 1772 } else {
95c96174 1773 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1774 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1775 (maclen < 16 ? 16 : maclen)) +
1776 po->tp_reserve;
1da177e4
LT		 1777 macoff = netoff - maclen;
1778 }
f6fb8f10 1779 if (po->tp_version <= TPACKET_V2) {
1780 if (macoff + snaplen > po->rx_ring.frame_size) {
1781 if (po->copy_thresh &&
0fd7bac6 1782 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1783 if (skb_shared(skb)) {
1784 copy_skb = skb_clone(skb, GFP_ATOMIC);
1785 } else {
1786 copy_skb = skb_get(skb);
1787 skb_head = skb->data;
1788 }
1789 if (copy_skb)
1790 skb_set_owner_r(copy_skb, sk);
1da177e4 1791 }
f6fb8f10 1792 snaplen = po->rx_ring.frame_size - macoff;
1793 if ((int)snaplen < 0)
1794 snaplen = 0;
1da177e4 1795 }
1da177e4 1796 }
1da177e4 1797 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1798 h.raw = packet_current_rx_frame(po, skb,
1799 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1800 if (!h.raw)
1da177e4 1801 goto ring_is_full;
f6fb8f10 1802 if (po->tp_version <= TPACKET_V2) {
1803 packet_increment_rx_head(po, &po->rx_ring);
1804 /*
1805 * LOSING will be reported till you read the stats,
1806 * because it's COR - Clear On Read.
1807 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1808 * at packet level.
1809 */
ee80fbf3 1810 if (po->stats.stats1.tp_drops)
f6fb8f10 1811 status |= TP_STATUS_LOSING;
1812 }
ee80fbf3 1813 po->stats.stats1.tp_packets++;
1da177e4
LT		 1814 if (copy_skb) {
1815 status |= TP_STATUS_COPY;
1816 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1817 }
1da177e4
LT		 1818 spin_unlock(&sk->sk_receive_queue.lock);
1819
bbd6ef87 1820 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1821
1822 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1823 getnstimeofday(&ts);
1da177e4 1824
b9c32fb2
DB		 1825 status |= ts_status;
1826
bbd6ef87
PM		 1827 switch (po->tp_version) {
1828 case TPACKET_V1:
1829 h.h1->tp_len = skb->len;
1830 h.h1->tp_snaplen = snaplen;
1831 h.h1->tp_mac = macoff;
1832 h.h1->tp_net = netoff;
4b457bdf
DB		 1833 h.h1->tp_sec = ts.tv_sec;
1834 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1835 hdrlen = sizeof(*h.h1);
1836 break;
1837 case TPACKET_V2:
1838 h.h2->tp_len = skb->len;
1839 h.h2->tp_snaplen = snaplen;
1840 h.h2->tp_mac = macoff;
1841 h.h2->tp_net = netoff;
bbd6ef87
PM		 1842 h.h2->tp_sec = ts.tv_sec;
1843 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1844 if (vlan_tx_tag_present(skb)) {
1845 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1846 status |= TP_STATUS_VLAN_VALID;
1847 } else {
1848 h.h2->tp_vlan_tci = 0;
1849 }
13fcb7bd 1850 h.h2->tp_padding = 0;
bbd6ef87
PM		 1851 hdrlen = sizeof(*h.h2);
1852 break;
f6fb8f10 1853 case TPACKET_V3:
1854 /* tp_nxt_offset,vlan are already populated above.
1855 * So DONT clear those fields here
1856 */
1857 h.h3->tp_status |= status;
1858 h.h3->tp_len = skb->len;
1859 h.h3->tp_snaplen = snaplen;
1860 h.h3->tp_mac = macoff;
1861 h.h3->tp_net = netoff;
f6fb8f10 1862 h.h3->tp_sec = ts.tv_sec;
1863 h.h3->tp_nsec = ts.tv_nsec;
1864 hdrlen = sizeof(*h.h3);
1865 break;
bbd6ef87
PM		 1866 default:
1867 BUG();
1868 }
1da177e4 1869
bbd6ef87 1870 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1871 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1872 sll->sll_family = AF_PACKET;
1873 sll->sll_hatype = dev->type;
1874 sll->sll_protocol = skb->protocol;
1875 sll->sll_pkttype = skb->pkt_type;
8032b464 1876 if (unlikely(po->origdev))
80feaacb
PWJ		 1877 sll->sll_ifindex = orig_dev->ifindex;
1878 else
1879 sll->sll_ifindex = dev->ifindex;
1da177e4 1880
e16aa207 1881 smp_mb();
f6dafa95 1882#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1883 {
0af55bb5
CG		 1884 u8 *start, *end;
1885
f6fb8f10 1886 if (po->tp_version <= TPACKET_V2) {
1887 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1888 + macoff + snaplen);
1889 for (start = h.raw; start < end; start += PAGE_SIZE)
1890 flush_dcache_page(pgv_to_page(start));
1891 }
cc9f01b2 1892 smp_wmb();
1da177e4 1893 }
f6dafa95 1894#endif
f6fb8f10 1895 if (po->tp_version <= TPACKET_V2)
1896 __packet_set_status(po, h.raw, status);
1897 else
1898 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1899
1900 sk->sk_data_ready(sk, 0);
1901
1902drop_n_restore:
1903 if (skb_head != skb->data && skb_shared(skb)) {
1904 skb->data = skb_head;
1905 skb->len = skb_len;
1906 }
1907drop:
1ce4f28b 1908 kfree_skb(skb);
1da177e4
LT		 1909 return 0;
1910
1911ring_is_full:
ee80fbf3 1912 po->stats.stats1.tp_drops++;
1da177e4
LT		 1913 spin_unlock(&sk->sk_receive_queue.lock);
1914
1915 sk->sk_data_ready(sk, 0);
acb5d75b 1916 kfree_skb(copy_skb);
1da177e4
LT		 1917 goto drop_n_restore;
1918}
1919
69e3c75f
JB		 1920static void tpacket_destruct_skb(struct sk_buff *skb)
1921{
1922 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1923 void *ph;
1da177e4 1924
69e3c75f 1925 if (likely(po->tx_ring.pg_vec)) {
b9c32fb2
DB		 1926 __u32 ts;
1927
69e3c75f 1928 ph = skb_shinfo(skb)->destructor_arg;
69e3c75f
JB		 1929 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1930 atomic_dec(&po->tx_ring.pending);
b9c32fb2
DB		 1931
1932 ts = __packet_set_timestamp(po, ph, skb);
1933 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 1934 }
1935
1936 sock_wfree(skb);
1937}
1938
40d4e3df
ED		 1939static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1940 void *frame, struct net_device *dev, int size_max,
ae641949 1941 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 1942{
184f489e 1943 union tpacket_uhdr ph;
09effa67 1944 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 1945 struct socket *sock = po->sk.sk_socket;
1946 struct page *page;
1947 void *data;
1948 int err;
1949
1950 ph.raw = frame;
1951
1952 skb->protocol = proto;
1953 skb->dev = dev;
1954 skb->priority = po->sk.sk_priority;
2d37a186 1955 skb->mark = po->sk.sk_mark;
2e31396f 1956 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 1957 skb_shinfo(skb)->destructor_arg = ph.raw;
1958
1959 switch (po->tp_version) {
1960 case TPACKET_V2:
1961 tp_len = ph.h2->tp_len;
1962 break;
1963 default:
1964 tp_len = ph.h1->tp_len;
1965 break;
1966 }
09effa67
DM		 1967 if (unlikely(tp_len > size_max)) {
1968 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
1969 return -EMSGSIZE;
1970 }
69e3c75f 1971
ae641949 1972 skb_reserve(skb, hlen);
69e3c75f 1973 skb_reset_network_header(skb);
40893fd0 1974 skb_probe_transport_header(skb, 0);
c1aad275 1975
5920cd3a
PC		 1976 if (po->tp_tx_has_off) {
1977 int off_min, off_max, off;
1978 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
1979 off_max = po->tx_ring.frame_size - tp_len;
1980 if (sock->type == SOCK_DGRAM) {
1981 switch (po->tp_version) {
1982 case TPACKET_V2:
1983 off = ph.h2->tp_net;
1984 break;
1985 default:
1986 off = ph.h1->tp_net;
1987 break;
1988 }
1989 } else {
1990 switch (po->tp_version) {
1991 case TPACKET_V2:
1992 off = ph.h2->tp_mac;
1993 break;
1994 default:
1995 off = ph.h1->tp_mac;
1996 break;
1997 }
1998 }
1999 if (unlikely((off < off_min) || (off_max < off)))
2000 return -EINVAL;
2001 data = ph.raw + off;
2002 } else {
2003 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2004 }
69e3c75f
JB		 2005 to_write = tp_len;
2006
2007 if (sock->type == SOCK_DGRAM) {
2008 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2009 NULL, tp_len);
2010 if (unlikely(err < 0))
2011 return -EINVAL;
40d4e3df 2012 } else if (dev->hard_header_len) {
69e3c75f
JB		 2013 /* net device doesn't like empty head */
2014 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2015 pr_err("packet size is too short (%d < %d)\n",
2016 tp_len, dev->hard_header_len);
69e3c75f
JB		 2017 return -EINVAL;
2018 }
2019
2020 skb_push(skb, dev->hard_header_len);
2021 err = skb_store_bits(skb, 0, data,
2022 dev->hard_header_len);
2023 if (unlikely(err))
2024 return err;
2025
2026 data += dev->hard_header_len;
2027 to_write -= dev->hard_header_len;
2028 }
2029
69e3c75f
JB		 2030 offset = offset_in_page(data);
2031 len_max = PAGE_SIZE - offset;
2032 len = ((to_write > len_max) ? len_max : to_write);
2033
2034 skb->data_len = to_write;
2035 skb->len += to_write;
2036 skb->truesize += to_write;
2037 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2038
2039 while (likely(to_write)) {
2040 nr_frags = skb_shinfo(skb)->nr_frags;
2041
2042 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2043 pr_err("Packet exceed the number of skb frags(%lu)\n",
2044 MAX_SKB_FRAGS);
69e3c75f
JB		 2045 return -EFAULT;
2046 }
2047
0af55bb5
CG		 2048 page = pgv_to_page(data);
2049 data += len;
69e3c75f
JB		 2050 flush_dcache_page(page);
2051 get_page(page);
0af55bb5 2052 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2053 to_write -= len;
2054 offset = 0;
2055 len_max = PAGE_SIZE;
2056 len = ((to_write > len_max) ? len_max : to_write);
2057 }
2058
2059 return tp_len;
2060}
2061
e40526cb
DB		 2062static struct net_device *packet_cached_dev_get(struct packet_sock *po)
2063{
2064 struct net_device *dev;
2065
2066 rcu_read_lock();
2067 dev = rcu_dereference(po->cached_dev);
2068 if (dev)
2069 dev_hold(dev);
2070 rcu_read_unlock();
2071
2072 return dev;
2073}
2074
69e3c75f
JB		 2075static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2076{
69e3c75f
JB		 2077 struct sk_buff *skb;
2078 struct net_device *dev;
2079 __be16 proto;
09effa67 2080 int err, reserve = 0;
40d4e3df
ED		 2081 void *ph;
2082 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2083 int tp_len, size_max;
2084 unsigned char *addr;
2085 int len_sum = 0;
9e67030a 2086 int status = TP_STATUS_AVAILABLE;
ae641949 2087 int hlen, tlen;
69e3c75f 2088
69e3c75f
JB		 2089 mutex_lock(&po->pg_vec_lock);
2090
69e3c75f 2091 if (saddr == NULL) {
e40526cb 2092 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2093 proto = po->num;
2094 addr = NULL;
2095 } else {
2096 err = -EINVAL;
2097 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2098 goto out;
2099 if (msg->msg_namelen < (saddr->sll_halen
2100 + offsetof(struct sockaddr_ll,
2101 sll_addr)))
2102 goto out;
69e3c75f
JB		 2103 proto = saddr->sll_protocol;
2104 addr = saddr->sll_addr;
827d9780 2105 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2106 }
2107
69e3c75f
JB		 2108 err = -ENXIO;
2109 if (unlikely(dev == NULL))
2110 goto out;
69e3c75f
JB		 2111 err = -ENETDOWN;
2112 if (unlikely(!(dev->flags & IFF_UP)))
2113 goto out_put;
2114
e40526cb
DB		 2115 reserve = dev->hard_header_len;
2116
69e3c75f 2117 size_max = po->tx_ring.frame_size
b5dd884e 2118 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2119
09effa67
DM		 2120 if (size_max > dev->mtu + reserve)
2121 size_max = dev->mtu + reserve;
2122
69e3c75f
JB		 2123 do {
2124 ph = packet_current_frame(po, &po->tx_ring,
2125 TP_STATUS_SEND_REQUEST);
2126
2127 if (unlikely(ph == NULL)) {
2128 schedule();
2129 continue;
2130 }
2131
2132 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2133 hlen = LL_RESERVED_SPACE(dev);
2134 tlen = dev->needed_tailroom;
69e3c75f 2135 skb = sock_alloc_send_skb(&po->sk,
ae641949 2136 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2137 0, &err);
2138
2139 if (unlikely(skb == NULL))
2140 goto out_status;
2141
2142 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2143 addr, hlen);
69e3c75f
JB		 2144
2145 if (unlikely(tp_len < 0)) {
2146 if (po->tp_loss) {
2147 __packet_set_status(po, ph,
2148 TP_STATUS_AVAILABLE);
2149 packet_increment_head(&po->tx_ring);
2150 kfree_skb(skb);
2151 continue;
2152 } else {
2153 status = TP_STATUS_WRONG_FORMAT;
2154 err = tp_len;
2155 goto out_status;
2156 }
2157 }
2158
2159 skb->destructor = tpacket_destruct_skb;
2160 __packet_set_status(po, ph, TP_STATUS_SENDING);
2161 atomic_inc(&po->tx_ring.pending);
2162
2163 status = TP_STATUS_SEND_REQUEST;
2164 err = dev_queue_xmit(skb);
eb70df13
JP		 2165 if (unlikely(err > 0)) {
2166 err = net_xmit_errno(err);
2167 if (err && __packet_get_status(po, ph) ==
2168 TP_STATUS_AVAILABLE) {
2169 /* skb was destructed already */
2170 skb = NULL;
2171 goto out_status;
2172 }
2173 /*
2174 * skb was dropped but not destructed yet;
2175 * let's treat it like congestion or err < 0
2176 */
2177 err = 0;
2178 }
69e3c75f
JB		 2179 packet_increment_head(&po->tx_ring);
2180 len_sum += tp_len;
f64f9e71
JP		 2181 } while (likely((ph != NULL) ||
2182 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2183 (atomic_read(&po->tx_ring.pending))))
2184 );
69e3c75f
JB		 2185
2186 err = len_sum;
2187 goto out_put;
2188
69e3c75f
JB		 2189out_status:
2190 __packet_set_status(po, ph, status);
2191 kfree_skb(skb);
2192out_put:
e40526cb 2193 dev_put(dev);
69e3c75f
JB		 2194out:
2195 mutex_unlock(&po->pg_vec_lock);
2196 return err;
2197}
69e3c75f 2198
eea49cc9
OJ		 2199static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2200 size_t reserve, size_t len,
2201 size_t linear, int noblock,
2202 int *err)
bfd5f4a3
SS		 2203{
2204 struct sk_buff *skb;
2205
2206 /* Under a page? Don't bother with paged skb. */
2207 if (prepad + len < PAGE_SIZE || !linear)
2208 linear = len;
2209
2210 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2211 err, 0);
bfd5f4a3
SS		 2212 if (!skb)
2213 return NULL;
2214
2215 skb_reserve(skb, reserve);
2216 skb_put(skb, linear);
2217 skb->data_len = len - linear;
2218 skb->len += len - linear;
2219
2220 return skb;
2221}
2222
69e3c75f 2223static int packet_snd(struct socket *sock,
1da177e4
LT		 2224 struct msghdr *msg, size_t len)
2225{
2226 struct sock *sk = sock->sk;
40d4e3df 2227 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2228 struct sk_buff *skb;
2229 struct net_device *dev;
0e11c91e 2230 __be16 proto;
1da177e4 2231 unsigned char *addr;
827d9780 2232 int err, reserve = 0;
bfd5f4a3
SS		 2233 struct virtio_net_hdr vnet_hdr = { 0 };
2234 int offset = 0;
2235 int vnet_hdr_len;
2236 struct packet_sock *po = pkt_sk(sk);
2237 unsigned short gso_type = 0;
ae641949 2238 int hlen, tlen;
3bdc0eba 2239 int extra_len = 0;
1da177e4
LT		 2240
2241 /*
1ce4f28b 2242 * Get and verify the address.
1da177e4 2243 */
1ce4f28b 2244
1da177e4 2245 if (saddr == NULL) {
e40526cb 2246 dev = packet_cached_dev_get(po);
1da177e4
LT		 2247 proto = po->num;
2248 addr = NULL;
2249 } else {
2250 err = -EINVAL;
2251 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2252 goto out;
0fb375fb
EB		 2253 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2254 goto out;
1da177e4
LT		 2255 proto = saddr->sll_protocol;
2256 addr = saddr->sll_addr;
827d9780 2257 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2258 }
2259
1da177e4 2260 err = -ENXIO;
e40526cb 2261 if (unlikely(dev == NULL))
1da177e4 2262 goto out_unlock;
d5e76b0a 2263 err = -ENETDOWN;
e40526cb 2264 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2265 goto out_unlock;
2266
e40526cb
DB		 2267 if (sock->type == SOCK_RAW)
2268 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2269 if (po->has_vnet_hdr) {
2270 vnet_hdr_len = sizeof(vnet_hdr);
2271
2272 err = -EINVAL;
2273 if (len < vnet_hdr_len)
2274 goto out_unlock;
2275
2276 len -= vnet_hdr_len;
2277
2278 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2279 vnet_hdr_len);
2280 if (err < 0)
2281 goto out_unlock;
2282
2283 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2284 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2285 vnet_hdr.hdr_len))
2286 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2287 vnet_hdr.csum_offset + 2;
2288
2289 err = -EINVAL;
2290 if (vnet_hdr.hdr_len > len)
2291 goto out_unlock;
2292
2293 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2294 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2295 case VIRTIO_NET_HDR_GSO_TCPV4:
2296 gso_type = SKB_GSO_TCPV4;
2297 break;
2298 case VIRTIO_NET_HDR_GSO_TCPV6:
2299 gso_type = SKB_GSO_TCPV6;
2300 break;
2301 case VIRTIO_NET_HDR_GSO_UDP:
2302 gso_type = SKB_GSO_UDP;
2303 break;
2304 default:
2305 goto out_unlock;
2306 }
2307
2308 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2309 gso_type |= SKB_GSO_TCP_ECN;
2310
2311 if (vnet_hdr.gso_size == 0)
2312 goto out_unlock;
2313
2314 }
2315 }
2316
3bdc0eba
BG		 2317 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2318 if (!netif_supports_nofcs(dev)) {
2319 err = -EPROTONOSUPPORT;
2320 goto out_unlock;
2321 }
2322 extra_len = 4; /* We're doing our own CRC */
2323 }
2324
1da177e4 2325 err = -EMSGSIZE;
3bdc0eba 2326 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2327 goto out_unlock;
2328
bfd5f4a3 2329 err = -ENOBUFS;
ae641949
HX		 2330 hlen = LL_RESERVED_SPACE(dev);
2331 tlen = dev->needed_tailroom;
2332 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2333 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2334 if (skb == NULL)
1da177e4
LT		 2335 goto out_unlock;
2336
bfd5f4a3 2337 skb_set_network_header(skb, reserve);
1da177e4 2338
0c4e8581
SH		 2339 err = -EINVAL;
2340 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2341 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2342 goto out_free;
1da177e4
LT		 2343
2344 /* Returns -EFAULT on error */
bfd5f4a3 2345 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2346 if (err)
2347 goto out_free;
bf84a010
DB		 2348
2349 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2350
3bdc0eba 2351 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2352 /* Earlier code assumed this would be a VLAN pkt,
2353 * double-check this now that we have the actual
2354 * packet in hand.
2355 */
2356 struct ethhdr *ehdr;
2357 skb_reset_mac_header(skb);
2358 ehdr = eth_hdr(skb);
2359 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2360 err = -EMSGSIZE;
2361 goto out_free;
2362 }
57f89bfa
BG		 2363 }
2364
09effa67
DM		 2365 skb->protocol = proto;
2366 skb->dev = dev;
1da177e4 2367 skb->priority = sk->sk_priority;
2d37a186 2368 skb->mark = sk->sk_mark;
1da177e4 2369
bfd5f4a3
SS		 2370 if (po->has_vnet_hdr) {
2371 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2372 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2373 vnet_hdr.csum_offset)) {
2374 err = -EINVAL;
2375 goto out_free;
2376 }
2377 }
2378
2379 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2380 skb_shinfo(skb)->gso_type = gso_type;
2381
2382 /* Header must be checked, and gso_segs computed. */
2383 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2384 skb_shinfo(skb)->gso_segs = 0;
2385
2386 len += vnet_hdr_len;
2387 }
2388
40893fd0 2389 skb_probe_transport_header(skb, reserve);
c1aad275 2390
3bdc0eba
BG		 2391 if (unlikely(extra_len == 4))
2392 skb->no_fcs = 1;
2393
1da177e4
LT		 2394 /*
2395 * Now send it
2396 */
2397
2398 err = dev_queue_xmit(skb);
2399 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2400 goto out_unlock;
2401
e40526cb 2402 dev_put(dev);
1da177e4 2403
40d4e3df 2404 return len;
1da177e4
LT		 2405
2406out_free:
2407 kfree_skb(skb);
2408out_unlock:
e40526cb 2409 if (dev)
1da177e4
LT		 2410 dev_put(dev);
2411out:
2412 return err;
2413}
2414
69e3c75f
JB		 2415static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2416 struct msghdr *msg, size_t len)
2417{
69e3c75f
JB		 2418 struct sock *sk = sock->sk;
2419 struct packet_sock *po = pkt_sk(sk);
2420 if (po->tx_ring.pg_vec)
2421 return tpacket_snd(po, msg);
2422 else
69e3c75f
JB		 2423 return packet_snd(sock, msg, len);
2424}
2425
1da177e4
LT		 2426/*
2427 * Close a PACKET socket. This is fairly simple. We immediately go
2428 * to 'closed' state and remove our protocol entry in the device list.
2429 */
2430
2431static int packet_release(struct socket *sock)
2432{
2433 struct sock *sk = sock->sk;
2434 struct packet_sock *po;
d12d01d6 2435 struct net *net;
f6fb8f10 2436 union tpacket_req_u req_u;
1da177e4
LT		 2437
2438 if (!sk)
2439 return 0;
2440
3b1e0a65 2441 net = sock_net(sk);
1da177e4
LT		 2442 po = pkt_sk(sk);
2443
0fa7fa98 2444 mutex_lock(&net->packet.sklist_lock);
808f5114 2445 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2446 mutex_unlock(&net->packet.sklist_lock);
2447
2448 preempt_disable();
920de804 2449 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2450 preempt_enable();
1da177e4 2451
808f5114 2452 spin_lock(&po->bind_lock);
ce06b03e 2453 unregister_prot_hook(sk, false);
160ff18a
BG		 2454 if (po->prot_hook.dev) {
2455 dev_put(po->prot_hook.dev);
2456 po->prot_hook.dev = NULL;
2457 }
808f5114 2458 spin_unlock(&po->bind_lock);
1da177e4 2459
1da177e4 2460 packet_flush_mclist(sk);
1da177e4 2461
9665d5d6
PS		 2462 if (po->rx_ring.pg_vec) {
2463 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2464 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2465 }
69e3c75f 2466
9665d5d6
PS		 2467 if (po->tx_ring.pg_vec) {
2468 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2469 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2470 }
1da177e4 2471
dc99f600
DM		 2472 fanout_release(sk);
2473
808f5114 2474 synchronize_net();
1da177e4
LT		 2475 /*
2476 * Now the socket is dead. No more input will appear.
2477 */
1da177e4
LT		 2478 sock_orphan(sk);
2479 sock->sk = NULL;
2480
2481 /* Purge queues */
2482
2483 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2484 sk_refcnt_debug_release(sk);
1da177e4
LT		 2485
2486 sock_put(sk);
2487 return 0;
2488}
2489
2490/*
2491 * Attach a packet hook.
2492 */
2493
0e11c91e 2494static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2495{
2496 struct packet_sock *po = pkt_sk(sk);
dc99f600 2497
aef950b4
WY		 2498 if (po->fanout) {
2499 if (dev)
2500 dev_put(dev);
2501
dc99f600 2502 return -EINVAL;
aef950b4 2503 }
1da177e4
LT		 2504
2505 lock_sock(sk);
2506
2507 spin_lock(&po->bind_lock);
ce06b03e 2508 unregister_prot_hook(sk, true);
1da177e4
LT		 2509 po->num = protocol;
2510 po->prot_hook.type = protocol;
160ff18a
BG		 2511 if (po->prot_hook.dev)
2512 dev_put(po->prot_hook.dev);
1da177e4
LT		 2513 po->prot_hook.dev = dev;
2514
2515 po->ifindex = dev ? dev->ifindex : 0;
2516
2517 if (protocol == 0)
2518 goto out_unlock;
2519
be85d4ad 2520 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2521 register_prot_hook(sk);
be85d4ad
UT		 2522 } else {
2523 sk->sk_err = ENETDOWN;
2524 if (!sock_flag(sk, SOCK_DEAD))
2525 sk->sk_error_report(sk);
1da177e4
LT		 2526 }
2527
2528out_unlock:
2529 spin_unlock(&po->bind_lock);
2530 release_sock(sk);
2531 return 0;
2532}
2533
2534/*
2535 * Bind a packet socket to a device
2536 */
2537
40d4e3df
ED		 2538static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2539 int addr_len)
1da177e4 2540{
40d4e3df 2541 struct sock *sk = sock->sk;
1da177e4
LT		 2542 char name[15];
2543 struct net_device *dev;
2544 int err = -ENODEV;
1ce4f28b 2545
1da177e4
LT		 2546 /*
2547 * Check legality
2548 */
1ce4f28b 2549
8ae55f04 2550 if (addr_len != sizeof(struct sockaddr))
1da177e4 2551 return -EINVAL;
40d4e3df 2552 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2553
3b1e0a65 2554 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2555 if (dev)
1da177e4 2556 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2557 return err;
2558}
1da177e4
LT		 2559
2560static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2561{
40d4e3df
ED		 2562 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2563 struct sock *sk = sock->sk;
1da177e4
LT		 2564 struct net_device *dev = NULL;
2565 int err;
2566
2567
2568 /*
2569 * Check legality
2570 */
1ce4f28b 2571
1da177e4
LT		 2572 if (addr_len < sizeof(struct sockaddr_ll))
2573 return -EINVAL;
2574 if (sll->sll_family != AF_PACKET)
2575 return -EINVAL;
2576
2577 if (sll->sll_ifindex) {
2578 err = -ENODEV;
3b1e0a65 2579 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2580 if (dev == NULL)
2581 goto out;
2582 }
2583 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2584
2585out:
2586 return err;
2587}
2588
2589static struct proto packet_proto = {
2590 .name = "PACKET",
2591 .owner = THIS_MODULE,
2592 .obj_size = sizeof(struct packet_sock),
2593};
2594
2595/*
1ce4f28b 2596 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2597 */
2598
3f378b68
EP		 2599static int packet_create(struct net *net, struct socket *sock, int protocol,
2600 int kern)
1da177e4
LT		 2601{
2602 struct sock *sk;
2603 struct packet_sock *po;
0e11c91e 2604 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2605 int err;
2606
df008c91 2607 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2608 return -EPERM;
be02097c
DM		 2609 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2610 sock->type != SOCK_PACKET)
1da177e4
LT		 2611 return -ESOCKTNOSUPPORT;
2612
2613 sock->state = SS_UNCONNECTED;
2614
2615 err = -ENOBUFS;
6257ff21 2616 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2617 if (sk == NULL)
2618 goto out;
2619
2620 sock->ops = &packet_ops;
1da177e4
LT		 2621 if (sock->type == SOCK_PACKET)
2622 sock->ops = &packet_ops_spkt;
be02097c 2623
1da177e4
LT		 2624 sock_init_data(sock, sk);
2625
2626 po = pkt_sk(sk);
2627 sk->sk_family = PF_PACKET;
0e11c91e 2628 po->num = proto;
e40526cb 2629 RCU_INIT_POINTER(po->cached_dev, NULL);
1da177e4
LT		 2630
2631 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2632 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2633
2634 /*
2635 * Attach a protocol block
2636 */
2637
2638 spin_lock_init(&po->bind_lock);
905db440 2639 mutex_init(&po->pg_vec_lock);
1da177e4 2640 po->prot_hook.func = packet_rcv;
be02097c 2641
1da177e4
LT		 2642 if (sock->type == SOCK_PACKET)
2643 po->prot_hook.func = packet_rcv_spkt;
be02097c 2644
1da177e4
LT		 2645 po->prot_hook.af_packet_priv = sk;
2646
0e11c91e
AV		 2647 if (proto) {
2648 po->prot_hook.type = proto;
ce06b03e 2649 register_prot_hook(sk);
1da177e4
LT		 2650 }
2651
0fa7fa98 2652 mutex_lock(&net->packet.sklist_lock);
808f5114 2653 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2654 mutex_unlock(&net->packet.sklist_lock);
2655
2656 preempt_disable();
3680453c 2657 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2658 preempt_enable();
808f5114 2659
40d4e3df 2660 return 0;
1da177e4
LT		 2661out:
2662 return err;
2663}
2664
2665/*
2666 * Pull a packet from our receive queue and hand it to the user.
2667 * If necessary we block.
2668 */
2669
2670static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2671 struct msghdr *msg, size_t len, int flags)
2672{
2673 struct sock *sk = sock->sk;
2674 struct sk_buff *skb;
2675 int copied, err;
bfd5f4a3 2676 int vnet_hdr_len = 0;
1da177e4
LT		 2677
2678 err = -EINVAL;
ed85b565 2679 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2680 goto out;
2681
2682#if 0
2683 /* What error should we return now? EUNATTACH? */
2684 if (pkt_sk(sk)->ifindex < 0)
2685 return -ENODEV;
2686#endif
2687
ed85b565 2688 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2689 err = sock_recv_errqueue(sk, msg, len,
2690 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2691 goto out;
2692 }
2693
1da177e4
LT		 2694 /*
2695 * Call the generic datagram receiver. This handles all sorts
2696 * of horrible races and re-entrancy so we can forget about it
2697 * in the protocol layers.
2698 *
2699 * Now it will return ENETDOWN, if device have just gone down,
2700 * but then it will block.
2701 */
2702
40d4e3df 2703 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2704
2705 /*
1ce4f28b 2706 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2707 * handles the blocking we don't see and worry about blocking
2708 * retries.
2709 */
2710
8ae55f04 2711 if (skb == NULL)
1da177e4
LT		 2712 goto out;
2713
bfd5f4a3
SS		 2714 if (pkt_sk(sk)->has_vnet_hdr) {
2715 struct virtio_net_hdr vnet_hdr = { 0 };
2716
2717 err = -EINVAL;
2718 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2719 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2720 goto out_free;
2721
1f18b717
MK		 2722 len -= vnet_hdr_len;
2723
bfd5f4a3
SS		 2724 if (skb_is_gso(skb)) {
2725 struct skb_shared_info *sinfo = skb_shinfo(skb);
2726
2727 /* This is a hint as to how much should be linear. */
2728 vnet_hdr.hdr_len = skb_headlen(skb);
2729 vnet_hdr.gso_size = sinfo->gso_size;
2730 if (sinfo->gso_type & SKB_GSO_TCPV4)
2731 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2732 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2733 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2734 else if (sinfo->gso_type & SKB_GSO_UDP)
2735 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2736 else if (sinfo->gso_type & SKB_GSO_FCOE)
2737 goto out_free;
2738 else
2739 BUG();
2740 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2741 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2742 } else
2743 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2744
2745 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2746 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2747 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2748 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2749 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2750 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2751 } /* else everything is zero */
2752
2753 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2754 vnet_hdr_len);
2755 if (err < 0)
2756 goto out_free;
2757 }
2758
f3d33426
HFS		 2759 /* You lose any data beyond the buffer you gave. If it worries
2760 * a user program they can ask the device for its MTU
2761 * anyway.
1da177e4 2762 */
1da177e4 2763 copied = skb->len;
40d4e3df
ED		 2764 if (copied > len) {
2765 copied = len;
2766 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2767 }
2768
2769 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2770 if (err)
2771 goto out_free;
2772
3b885787 2773 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2774
f3d33426
HFS		 2775 if (msg->msg_name) {
2776 /* If the address length field is there to be filled
2777 * in, we fill it in now.
2778 */
2779 if (sock->type == SOCK_PACKET) {
2780 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2781 } else {
2782 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2783 msg->msg_namelen = sll->sll_halen +
2784 offsetof(struct sockaddr_ll, sll_addr);
2785 }
ffbc6111
HX		 2786 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2787 msg->msg_namelen);
f3d33426 2788 }
1da177e4 2789
8dc41944 2790 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2791 struct tpacket_auxdata aux;
2792
2793 aux.tp_status = TP_STATUS_USER;
2794 if (skb->ip_summed == CHECKSUM_PARTIAL)
2795 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2796 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2797 aux.tp_snaplen = skb->len;
2798 aux.tp_mac = 0;
bbe735e4 2799 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2800 if (vlan_tx_tag_present(skb)) {
2801 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2802 aux.tp_status |= TP_STATUS_VLAN_VALID;
2803 } else {
2804 aux.tp_vlan_tci = 0;
2805 }
13fcb7bd 2806 aux.tp_padding = 0;
ffbc6111 2807 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2808 }
2809
1da177e4
LT		 2810 /*
2811 * Free or return the buffer as appropriate. Again this
2812 * hides all the races and re-entrancy issues from us.
2813 */
bfd5f4a3 2814 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2815
2816out_free:
2817 skb_free_datagram(sk, skb);
2818out:
2819 return err;
2820}
2821
1da177e4
LT		 2822static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2823 int *uaddr_len, int peer)
2824{
2825 struct net_device *dev;
2826 struct sock *sk = sock->sk;
2827
2828 if (peer)
2829 return -EOPNOTSUPP;
2830
2831 uaddr->sa_family = AF_PACKET;
2dc85bf3 2832 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 2833 rcu_read_lock();
2834 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2835 if (dev)
2dc85bf3 2836 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 2837 rcu_read_unlock();
1da177e4
LT		 2838 *uaddr_len = sizeof(*uaddr);
2839
2840 return 0;
2841}
1da177e4
LT		 2842
2843static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2844 int *uaddr_len, int peer)
2845{
2846 struct net_device *dev;
2847 struct sock *sk = sock->sk;
2848 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2849 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2850
2851 if (peer)
2852 return -EOPNOTSUPP;
2853
2854 sll->sll_family = AF_PACKET;
2855 sll->sll_ifindex = po->ifindex;
2856 sll->sll_protocol = po->num;
67286640 2857 sll->sll_pkttype = 0;
654d1f8a
ED		 2858 rcu_read_lock();
2859 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2860 if (dev) {
2861 sll->sll_hatype = dev->type;
2862 sll->sll_halen = dev->addr_len;
2863 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2864 } else {
2865 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2866 sll->sll_halen = 0;
2867 }
654d1f8a 2868 rcu_read_unlock();
0fb375fb 2869 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2870
2871 return 0;
2872}
2873
2aeb0b88
WC		 2874static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2875 int what)
1da177e4
LT		 2876{
2877 switch (i->type) {
2878 case PACKET_MR_MULTICAST:
1162563f
JP		 2879 if (i->alen != dev->addr_len)
2880 return -EINVAL;
1da177e4 2881 if (what > 0)
22bedad3 2882 return dev_mc_add(dev, i->addr);
1da177e4 2883 else
22bedad3 2884 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2885 break;
2886 case PACKET_MR_PROMISC:
2aeb0b88 2887 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2888 break;
2889 case PACKET_MR_ALLMULTI:
2aeb0b88 2890 return dev_set_allmulti(dev, what);
1da177e4 2891 break;
d95ed927 2892 case PACKET_MR_UNICAST:
1162563f
JP		 2893 if (i->alen != dev->addr_len)
2894 return -EINVAL;
d95ed927 2895 if (what > 0)
a748ee24 2896 return dev_uc_add(dev, i->addr);
d95ed927 2897 else
a748ee24 2898 return dev_uc_del(dev, i->addr);
d95ed927 2899 break;
40d4e3df
ED		 2900 default:
2901 break;
1da177e4 2902 }
2aeb0b88 2903 return 0;
1da177e4
LT		 2904}
2905
2906static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2907{
40d4e3df 2908 for ( ; i; i = i->next) {
1da177e4
LT		 2909 if (i->ifindex == dev->ifindex)
2910 packet_dev_mc(dev, i, what);
2911 }
2912}
2913
0fb375fb 2914static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2915{
2916 struct packet_sock *po = pkt_sk(sk);
2917 struct packet_mclist *ml, *i;
2918 struct net_device *dev;
2919 int err;
2920
2921 rtnl_lock();
2922
2923 err = -ENODEV;
3b1e0a65 2924 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2925 if (!dev)
2926 goto done;
2927
2928 err = -EINVAL;
1162563f 2929 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2930 goto done;
2931
2932 err = -ENOBUFS;
8b3a7005 2933 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 2934 if (i == NULL)
2935 goto done;
2936
2937 err = 0;
2938 for (ml = po->mclist; ml; ml = ml->next) {
2939 if (ml->ifindex == mreq->mr_ifindex &&
2940 ml->type == mreq->mr_type &&
2941 ml->alen == mreq->mr_alen &&
2942 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2943 ml->count++;
2944 /* Free the new element ... */
2945 kfree(i);
2946 goto done;
2947 }
2948 }
2949
2950 i->type = mreq->mr_type;
2951 i->ifindex = mreq->mr_ifindex;
2952 i->alen = mreq->mr_alen;
2953 memcpy(i->addr, mreq->mr_address, i->alen);
2954 i->count = 1;
2955 i->next = po->mclist;
2956 po->mclist = i;
2aeb0b88
WC		 2957 err = packet_dev_mc(dev, i, 1);
2958 if (err) {
2959 po->mclist = i->next;
2960 kfree(i);
2961 }
1da177e4
LT		 2962
2963done:
2964 rtnl_unlock();
2965 return err;
2966}
2967
0fb375fb 2968static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2969{
2970 struct packet_mclist *ml, **mlp;
2971
2972 rtnl_lock();
2973
2974 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
2975 if (ml->ifindex == mreq->mr_ifindex &&
2976 ml->type == mreq->mr_type &&
2977 ml->alen == mreq->mr_alen &&
2978 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2979 if (--ml->count == 0) {
2980 struct net_device *dev;
2981 *mlp = ml->next;
ad959e76
ED		 2982 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2983 if (dev)
1da177e4 2984 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 2985 kfree(ml);
2986 }
2987 rtnl_unlock();
2988 return 0;
2989 }
2990 }
2991 rtnl_unlock();
2992 return -EADDRNOTAVAIL;
2993}
2994
2995static void packet_flush_mclist(struct sock *sk)
2996{
2997 struct packet_sock *po = pkt_sk(sk);
2998 struct packet_mclist *ml;
2999
3000 if (!po->mclist)
3001 return;
3002
3003 rtnl_lock();
3004 while ((ml = po->mclist) != NULL) {
3005 struct net_device *dev;
3006
3007 po->mclist = ml->next;
ad959e76
ED		 3008 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3009 if (dev != NULL)
1da177e4 3010 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3011 kfree(ml);
3012 }
3013 rtnl_unlock();
3014}
1da177e4
LT		 3015
3016static int
b7058842 3017packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3018{
3019 struct sock *sk = sock->sk;
8dc41944 3020 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3021 int ret;
3022
3023 if (level != SOL_PACKET)
3024 return -ENOPROTOOPT;
3025
69e3c75f 3026 switch (optname) {
1ce4f28b 3027 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3028 case PACKET_DROP_MEMBERSHIP:
3029 {
0fb375fb
EB		 3030 struct packet_mreq_max mreq;
3031 int len = optlen;
3032 memset(&mreq, 0, sizeof(mreq));
3033 if (len < sizeof(struct packet_mreq))
1da177e4 3034 return -EINVAL;
0fb375fb
EB		 3035 if (len > sizeof(mreq))
3036 len = sizeof(mreq);
40d4e3df 3037 if (copy_from_user(&mreq, optval, len))
1da177e4 3038 return -EFAULT;
0fb375fb
EB		 3039 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3040 return -EINVAL;
1da177e4
LT		 3041 if (optname == PACKET_ADD_MEMBERSHIP)
3042 ret = packet_mc_add(sk, &mreq);
3043 else
3044 ret = packet_mc_drop(sk, &mreq);
3045 return ret;
3046 }
a2efcfa0 3047
1da177e4 3048 case PACKET_RX_RING:
69e3c75f 3049 case PACKET_TX_RING:
1da177e4 3050 {
f6fb8f10 3051 union tpacket_req_u req_u;
3052 int len;
1da177e4 3053
f6fb8f10 3054 switch (po->tp_version) {
3055 case TPACKET_V1:
3056 case TPACKET_V2:
3057 len = sizeof(req_u.req);
3058 break;
3059 case TPACKET_V3:
3060 default:
3061 len = sizeof(req_u.req3);
3062 break;
3063 }
3064 if (optlen < len)
1da177e4 3065 return -EINVAL;
bfd5f4a3
SS		 3066 if (pkt_sk(sk)->has_vnet_hdr)
3067 return -EINVAL;
f6fb8f10 3068 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3069 return -EFAULT;
f6fb8f10 3070 return packet_set_ring(sk, &req_u, 0,
3071 optname == PACKET_TX_RING);
1da177e4
LT		 3072 }
3073 case PACKET_COPY_THRESH:
3074 {
3075 int val;
3076
40d4e3df 3077 if (optlen != sizeof(val))
1da177e4 3078 return -EINVAL;
40d4e3df 3079 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3080 return -EFAULT;
3081
3082 pkt_sk(sk)->copy_thresh = val;
3083 return 0;
3084 }
bbd6ef87
PM		 3085 case PACKET_VERSION:
3086 {
3087 int val;
3088
3089 if (optlen != sizeof(val))
3090 return -EINVAL;
69e3c75f 3091 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3092 return -EBUSY;
3093 if (copy_from_user(&val, optval, sizeof(val)))
3094 return -EFAULT;
3095 switch (val) {
3096 case TPACKET_V1:
3097 case TPACKET_V2:
f6fb8f10 3098 case TPACKET_V3:
bbd6ef87
PM		 3099 po->tp_version = val;
3100 return 0;
3101 default:
3102 return -EINVAL;
3103 }
3104 }
8913336a
PM		 3105 case PACKET_RESERVE:
3106 {
3107 unsigned int val;
3108
3109 if (optlen != sizeof(val))
3110 return -EINVAL;
69e3c75f 3111 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3112 return -EBUSY;
3113 if (copy_from_user(&val, optval, sizeof(val)))
3114 return -EFAULT;
3115 po->tp_reserve = val;
3116 return 0;
3117 }
69e3c75f
JB		 3118 case PACKET_LOSS:
3119 {
3120 unsigned int val;
3121
3122 if (optlen != sizeof(val))
3123 return -EINVAL;
3124 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3125 return -EBUSY;
3126 if (copy_from_user(&val, optval, sizeof(val)))
3127 return -EFAULT;
3128 po->tp_loss = !!val;
3129 return 0;
3130 }
8dc41944
HX		 3131 case PACKET_AUXDATA:
3132 {
3133 int val;
3134
3135 if (optlen < sizeof(val))
3136 return -EINVAL;
3137 if (copy_from_user(&val, optval, sizeof(val)))
3138 return -EFAULT;
3139
3140 po->auxdata = !!val;
3141 return 0;
3142 }
80feaacb
PWJ		 3143 case PACKET_ORIGDEV:
3144 {
3145 int val;
3146
3147 if (optlen < sizeof(val))
3148 return -EINVAL;
3149 if (copy_from_user(&val, optval, sizeof(val)))
3150 return -EFAULT;
3151
3152 po->origdev = !!val;
3153 return 0;
3154 }
bfd5f4a3
SS		 3155 case PACKET_VNET_HDR:
3156 {
3157 int val;
3158
3159 if (sock->type != SOCK_RAW)
3160 return -EINVAL;
3161 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3162 return -EBUSY;
3163 if (optlen < sizeof(val))
3164 return -EINVAL;
3165 if (copy_from_user(&val, optval, sizeof(val)))
3166 return -EFAULT;
3167
3168 po->has_vnet_hdr = !!val;
3169 return 0;
3170 }
614f60fa
SM		 3171 case PACKET_TIMESTAMP:
3172 {
3173 int val;
3174
3175 if (optlen != sizeof(val))
3176 return -EINVAL;
3177 if (copy_from_user(&val, optval, sizeof(val)))
3178 return -EFAULT;
3179
3180 po->tp_tstamp = val;
3181 return 0;
3182 }
dc99f600
DM		 3183 case PACKET_FANOUT:
3184 {
3185 int val;
3186
3187 if (optlen != sizeof(val))
3188 return -EINVAL;
3189 if (copy_from_user(&val, optval, sizeof(val)))
3190 return -EFAULT;
3191
3192 return fanout_add(sk, val & 0xffff, val >> 16);
3193 }
5920cd3a
PC		 3194 case PACKET_TX_HAS_OFF:
3195 {
3196 unsigned int val;
3197
3198 if (optlen != sizeof(val))
3199 return -EINVAL;
3200 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3201 return -EBUSY;
3202 if (copy_from_user(&val, optval, sizeof(val)))
3203 return -EFAULT;
3204 po->tp_tx_has_off = !!val;
3205 return 0;
3206 }
1da177e4
LT		 3207 default:
3208 return -ENOPROTOOPT;
3209 }
3210}
3211
3212static int packet_getsockopt(struct socket *sock, int level, int optname,
3213 char __user *optval, int __user *optlen)
3214{
3215 int len;
c06fff6e 3216 int val, lv = sizeof(val);
1da177e4
LT		 3217 struct sock *sk = sock->sk;
3218 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3219 void *data = &val;
ee80fbf3 3220 union tpacket_stats_u st;
1da177e4
LT		 3221
3222 if (level != SOL_PACKET)
3223 return -ENOPROTOOPT;
3224
8ae55f04
KK		 3225 if (get_user(len, optlen))
3226 return -EFAULT;
1da177e4
LT		 3227
3228 if (len < 0)
3229 return -EINVAL;
1ce4f28b 3230
69e3c75f 3231 switch (optname) {
1da177e4 3232 case PACKET_STATISTICS:
1da177e4 3233 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3234 memcpy(&st, &po->stats, sizeof(st));
3235 memset(&po->stats, 0, sizeof(po->stats));
3236 spin_unlock_bh(&sk->sk_receive_queue.lock);
3237
f6fb8f10 3238 if (po->tp_version == TPACKET_V3) {
c06fff6e 3239 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3240 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3241 data = &st.stats3;
f6fb8f10 3242 } else {
c06fff6e 3243 lv = sizeof(struct tpacket_stats);
8bcdeaff 3244 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3245 data = &st.stats1;
f6fb8f10 3246 }
ee80fbf3 3247
8dc41944
HX		 3248 break;
3249 case PACKET_AUXDATA:
8dc41944 3250 val = po->auxdata;
80feaacb
PWJ		 3251 break;
3252 case PACKET_ORIGDEV:
80feaacb 3253 val = po->origdev;
bfd5f4a3
SS		 3254 break;
3255 case PACKET_VNET_HDR:
bfd5f4a3 3256 val = po->has_vnet_hdr;
1da177e4 3257 break;
bbd6ef87 3258 case PACKET_VERSION:
bbd6ef87 3259 val = po->tp_version;
bbd6ef87
PM		 3260 break;
3261 case PACKET_HDRLEN:
3262 if (len > sizeof(int))
3263 len = sizeof(int);
3264 if (copy_from_user(&val, optval, len))
3265 return -EFAULT;
3266 switch (val) {
3267 case TPACKET_V1:
3268 val = sizeof(struct tpacket_hdr);
3269 break;
3270 case TPACKET_V2:
3271 val = sizeof(struct tpacket2_hdr);
3272 break;
f6fb8f10 3273 case TPACKET_V3:
3274 val = sizeof(struct tpacket3_hdr);
3275 break;
bbd6ef87
PM		 3276 default:
3277 return -EINVAL;
3278 }
bbd6ef87 3279 break;
8913336a 3280 case PACKET_RESERVE:
8913336a 3281 val = po->tp_reserve;
8913336a 3282 break;
69e3c75f 3283 case PACKET_LOSS:
69e3c75f 3284 val = po->tp_loss;
69e3c75f 3285 break;
614f60fa 3286 case PACKET_TIMESTAMP:
614f60fa 3287 val = po->tp_tstamp;
614f60fa 3288 break;
dc99f600 3289 case PACKET_FANOUT:
dc99f600
DM		 3290 val = (po->fanout ?
3291 ((u32)po->fanout->id |
77f65ebd
WB		 3292 ((u32)po->fanout->type << 16) |
3293 ((u32)po->fanout->flags << 24)) :
dc99f600 3294 0);
dc99f600 3295 break;
5920cd3a
PC		 3296 case PACKET_TX_HAS_OFF:
3297 val = po->tp_tx_has_off;
3298 break;
1da177e4
LT		 3299 default:
3300 return -ENOPROTOOPT;
3301 }
3302
c06fff6e
ED		 3303 if (len > lv)
3304 len = lv;
8ae55f04
KK		 3305 if (put_user(len, optlen))
3306 return -EFAULT;
8dc41944
HX		 3307 if (copy_to_user(optval, data, len))
3308 return -EFAULT;
8ae55f04 3309 return 0;
1da177e4
LT		 3310}
3311
3312
351638e7
JP		 3313static int packet_notifier(struct notifier_block *this,
3314 unsigned long msg, void *ptr)
1da177e4
LT		 3315{
3316 struct sock *sk;
351638e7 3317 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3318 struct net *net = dev_net(dev);
1da177e4 3319
808f5114 3320 rcu_read_lock();
b67bfe0d 3321 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3322 struct packet_sock *po = pkt_sk(sk);
3323
3324 switch (msg) {
3325 case NETDEV_UNREGISTER:
1da177e4
LT		 3326 if (po->mclist)
3327 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3328 /* fallthrough */
3329
1da177e4
LT		 3330 case NETDEV_DOWN:
3331 if (dev->ifindex == po->ifindex) {
3332 spin_lock(&po->bind_lock);
3333 if (po->running) {
ce06b03e 3334 __unregister_prot_hook(sk, false);
1da177e4
LT		 3335 sk->sk_err = ENETDOWN;
3336 if (!sock_flag(sk, SOCK_DEAD))
3337 sk->sk_error_report(sk);
3338 }
3339 if (msg == NETDEV_UNREGISTER) {
3340 po->ifindex = -1;
160ff18a
BG		 3341 if (po->prot_hook.dev)
3342 dev_put(po->prot_hook.dev);
1da177e4
LT		 3343 po->prot_hook.dev = NULL;
3344 }
3345 spin_unlock(&po->bind_lock);
3346 }
3347 break;
3348 case NETDEV_UP:
808f5114 3349 if (dev->ifindex == po->ifindex) {
3350 spin_lock(&po->bind_lock);
ce06b03e
DM		 3351 if (po->num)
3352 register_prot_hook(sk);
808f5114 3353 spin_unlock(&po->bind_lock);
1da177e4 3354 }
1da177e4
LT		 3355 break;
3356 }
3357 }
808f5114 3358 rcu_read_unlock();
1da177e4
LT		 3359 return NOTIFY_DONE;
3360}
3361
3362
3363static int packet_ioctl(struct socket *sock, unsigned int cmd,
3364 unsigned long arg)
3365{
3366 struct sock *sk = sock->sk;
3367
69e3c75f 3368 switch (cmd) {
40d4e3df
ED		 3369 case SIOCOUTQ:
3370 {
3371 int amount = sk_wmem_alloc_get(sk);
31e6d363 3372
40d4e3df
ED		 3373 return put_user(amount, (int __user *)arg);
3374 }
3375 case SIOCINQ:
3376 {
3377 struct sk_buff *skb;
3378 int amount = 0;
3379
3380 spin_lock_bh(&sk->sk_receive_queue.lock);
3381 skb = skb_peek(&sk->sk_receive_queue);
3382 if (skb)
3383 amount = skb->len;
3384 spin_unlock_bh(&sk->sk_receive_queue.lock);
3385 return put_user(amount, (int __user *)arg);
3386 }
3387 case SIOCGSTAMP:
3388 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3389 case SIOCGSTAMPNS:
3390 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3391
1da177e4 3392#ifdef CONFIG_INET
40d4e3df
ED		 3393 case SIOCADDRT:
3394 case SIOCDELRT:
3395 case SIOCDARP:
3396 case SIOCGARP:
3397 case SIOCSARP:
3398 case SIOCGIFADDR:
3399 case SIOCSIFADDR:
3400 case SIOCGIFBRDADDR:
3401 case SIOCSIFBRDADDR:
3402 case SIOCGIFNETMASK:
3403 case SIOCSIFNETMASK:
3404 case SIOCGIFDSTADDR:
3405 case SIOCSIFDSTADDR:
3406 case SIOCSIFFLAGS:
40d4e3df 3407 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3408#endif
3409
40d4e3df
ED		 3410 default:
3411 return -ENOIOCTLCMD;
1da177e4
LT		 3412 }
3413 return 0;
3414}
3415
40d4e3df 3416static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3417 poll_table *wait)
3418{
3419 struct sock *sk = sock->sk;
3420 struct packet_sock *po = pkt_sk(sk);
3421 unsigned int mask = datagram_poll(file, sock, wait);
3422
3423 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3424 if (po->rx_ring.pg_vec) {
f6fb8f10 3425 if (!packet_previous_rx_frame(po, &po->rx_ring,
3426 TP_STATUS_KERNEL))
1da177e4
LT		 3427 mask |= POLLIN | POLLRDNORM;
3428 }
3429 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3430 spin_lock_bh(&sk->sk_write_queue.lock);
3431 if (po->tx_ring.pg_vec) {
3432 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3433 mask |= POLLOUT | POLLWRNORM;
3434 }
3435 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3436 return mask;
3437}
3438
3439
3440/* Dirty? Well, I still did not learn better way to account
3441 * for user mmaps.
3442 */
3443
3444static void packet_mm_open(struct vm_area_struct *vma)
3445{
3446 struct file *file = vma->vm_file;
40d4e3df 3447 struct socket *sock = file->private_data;
1da177e4 3448 struct sock *sk = sock->sk;
1ce4f28b 3449
1da177e4
LT		 3450 if (sk)
3451 atomic_inc(&pkt_sk(sk)->mapped);
3452}
3453
3454static void packet_mm_close(struct vm_area_struct *vma)
3455{
3456 struct file *file = vma->vm_file;
40d4e3df 3457 struct socket *sock = file->private_data;
1da177e4 3458 struct sock *sk = sock->sk;
1ce4f28b 3459
1da177e4
LT		 3460 if (sk)
3461 atomic_dec(&pkt_sk(sk)->mapped);
3462}
3463
f0f37e2f 3464static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3465 .open = packet_mm_open,
3466 .close = packet_mm_close,
1da177e4
LT		 3467};
3468
0e3125c7
NH		 3469static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3470 unsigned int len)
1da177e4
LT		 3471{
3472 int i;
3473
4ebf0ae2 3474 for (i = 0; i < len; i++) {
0e3125c7 3475 if (likely(pg_vec[i].buffer)) {
c56b4d90 3476 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3477 vfree(pg_vec[i].buffer);
3478 else
3479 free_pages((unsigned long)pg_vec[i].buffer,
3480 order);
3481 pg_vec[i].buffer = NULL;
3482 }
1da177e4
LT		 3483 }
3484 kfree(pg_vec);
3485}
3486
eea49cc9 3487static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3488{
0e3125c7
NH		 3489 char *buffer = NULL;
3490 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3491 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3492
3493 buffer = (char *) __get_free_pages(gfp_flags, order);
3494
3495 if (buffer)
3496 return buffer;
3497
3498 /*
3499 * __get_free_pages failed, fall back to vmalloc
3500 */
bbce5a59 3501 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3502
0e3125c7
NH		 3503 if (buffer)
3504 return buffer;
3505
3506 /*
3507 * vmalloc failed, lets dig into swap here
3508 */
0e3125c7
NH		 3509 gfp_flags &= ~__GFP_NORETRY;
3510 buffer = (char *)__get_free_pages(gfp_flags, order);
3511 if (buffer)
3512 return buffer;
3513
3514 /*
3515 * complete and utter failure
3516 */
3517 return NULL;
4ebf0ae2
DM		 3518}
3519
0e3125c7 3520static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3521{
3522 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3523 struct pgv *pg_vec;
4ebf0ae2
DM		 3524 int i;
3525
0e3125c7 3526 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3527 if (unlikely(!pg_vec))
3528 goto out;
3529
3530 for (i = 0; i < block_nr; i++) {
c56b4d90 3531 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3532 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3533 goto out_free_pgvec;
3534 }
3535
3536out:
3537 return pg_vec;
3538
3539out_free_pgvec:
3540 free_pg_vec(pg_vec, order, block_nr);
3541 pg_vec = NULL;
3542 goto out;
3543}
1da177e4 3544
f6fb8f10 3545static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3546 int closing, int tx_ring)
1da177e4 3547{
0e3125c7 3548 struct pgv *pg_vec = NULL;
1da177e4 3549 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3550 int was_running, order = 0;
69e3c75f
JB		 3551 struct packet_ring_buffer *rb;
3552 struct sk_buff_head *rb_queue;
0e11c91e 3553 __be16 num;
f6fb8f10 3554 int err = -EINVAL;
3555 /* Added to avoid minimal code churn */
3556 struct tpacket_req *req = &req_u->req;
3557
3558 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3559 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3560 WARN(1, "Tx-ring is not supported.\n");
3561 goto out;
3562 }
1ce4f28b 3563
69e3c75f
JB		 3564 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3565 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3566
69e3c75f
JB		 3567 err = -EBUSY;
3568 if (!closing) {
3569 if (atomic_read(&po->mapped))
3570 goto out;
3571 if (atomic_read(&rb->pending))
3572 goto out;
3573 }
1da177e4 3574
69e3c75f
JB		 3575 if (req->tp_block_nr) {
3576 /* Sanity tests and some calculations */
3577 err = -EBUSY;
3578 if (unlikely(rb->pg_vec))
3579 goto out;
1da177e4 3580
bbd6ef87
PM		 3581 switch (po->tp_version) {
3582 case TPACKET_V1:
3583 po->tp_hdrlen = TPACKET_HDRLEN;
3584 break;
3585 case TPACKET_V2:
3586 po->tp_hdrlen = TPACKET2_HDRLEN;
3587 break;
f6fb8f10 3588 case TPACKET_V3:
3589 po->tp_hdrlen = TPACKET3_HDRLEN;
3590 break;
bbd6ef87
PM		 3591 }
3592
69e3c75f 3593 err = -EINVAL;
4ebf0ae2 3594 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3595 goto out;
4ebf0ae2 3596 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3597 goto out;
8913336a 3598 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3599 po->tp_reserve))
3600 goto out;
4ebf0ae2 3601 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3602 goto out;
1da177e4 3603
69e3c75f
JB		 3604 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3605 if (unlikely(rb->frames_per_block <= 0))
3606 goto out;
3607 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3608 req->tp_frame_nr))
3609 goto out;
1da177e4
LT		 3610
3611 err = -ENOMEM;
4ebf0ae2
DM		 3612 order = get_order(req->tp_block_size);
3613 pg_vec = alloc_pg_vec(req, order);
3614 if (unlikely(!pg_vec))
1da177e4 3615 goto out;
f6fb8f10 3616 switch (po->tp_version) {
3617 case TPACKET_V3:
3618 /* Transmit path is not supported. We checked
3619 * it above but just being paranoid
3620 */
3621 if (!tx_ring)
3622 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3623 break;
3624 default:
3625 break;
3626 }
69e3c75f
JB		 3627 }
3628 /* Done */
3629 else {
3630 err = -EINVAL;
4ebf0ae2 3631 if (unlikely(req->tp_frame_nr))
69e3c75f 3632 goto out;
1da177e4
LT		 3633 }
3634
3635 lock_sock(sk);
3636
3637 /* Detach socket from network */
3638 spin_lock(&po->bind_lock);
3639 was_running = po->running;
3640 num = po->num;
3641 if (was_running) {
1da177e4 3642 po->num = 0;
ce06b03e 3643 __unregister_prot_hook(sk, false);
1da177e4
LT		 3644 }
3645 spin_unlock(&po->bind_lock);
1ce4f28b 3646
1da177e4
LT		 3647 synchronize_net();
3648
3649 err = -EBUSY;
905db440 3650 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3651 if (closing || atomic_read(&po->mapped) == 0) {
3652 err = 0;
69e3c75f 3653 spin_lock_bh(&rb_queue->lock);
c053fd96 3654 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3655 rb->frame_max = (req->tp_frame_nr - 1);
3656 rb->head = 0;
3657 rb->frame_size = req->tp_frame_size;
3658 spin_unlock_bh(&rb_queue->lock);
3659
c053fd96
CG		 3660 swap(rb->pg_vec_order, order);
3661 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3662
3663 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3664 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3665 tpacket_rcv : packet_rcv;
3666 skb_queue_purge(rb_queue);
1da177e4 3667 if (atomic_read(&po->mapped))
40d4e3df
ED		 3668 pr_err("packet_mmap: vma is busy: %d\n",
3669 atomic_read(&po->mapped));
1da177e4 3670 }
905db440 3671 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3672
3673 spin_lock(&po->bind_lock);
ce06b03e 3674 if (was_running) {
1da177e4 3675 po->num = num;
ce06b03e 3676 register_prot_hook(sk);
1da177e4
LT		 3677 }
3678 spin_unlock(&po->bind_lock);
f6fb8f10 3679 if (closing && (po->tp_version > TPACKET_V2)) {
3680 /* Because we don't support block-based V3 on tx-ring */
3681 if (!tx_ring)
3682 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3683 }
1da177e4
LT		 3684 release_sock(sk);
3685
1da177e4
LT		 3686 if (pg_vec)
3687 free_pg_vec(pg_vec, order, req->tp_block_nr);
3688out:
3689 return err;
3690}
3691
69e3c75f
JB		 3692static int packet_mmap(struct file *file, struct socket *sock,
3693 struct vm_area_struct *vma)
1da177e4
LT		 3694{
3695 struct sock *sk = sock->sk;
3696 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3697 unsigned long size, expected_size;
3698 struct packet_ring_buffer *rb;
1da177e4
LT		 3699 unsigned long start;
3700 int err = -EINVAL;
3701 int i;
3702
3703 if (vma->vm_pgoff)
3704 return -EINVAL;
3705
905db440 3706 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3707
3708 expected_size = 0;
3709 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3710 if (rb->pg_vec) {
3711 expected_size += rb->pg_vec_len
3712 * rb->pg_vec_pages
3713 * PAGE_SIZE;
3714 }
3715 }
3716
3717 if (expected_size == 0)
1da177e4 3718 goto out;
69e3c75f
JB		 3719
3720 size = vma->vm_end - vma->vm_start;
3721 if (size != expected_size)
1da177e4
LT		 3722 goto out;
3723
1da177e4 3724 start = vma->vm_start;
69e3c75f
JB		 3725 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3726 if (rb->pg_vec == NULL)
3727 continue;
3728
3729 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3730 struct page *page;
3731 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3732 int pg_num;
3733
c56b4d90
CG		 3734 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3735 page = pgv_to_page(kaddr);
69e3c75f
JB		 3736 err = vm_insert_page(vma, start, page);
3737 if (unlikely(err))
3738 goto out;
3739 start += PAGE_SIZE;
0e3125c7 3740 kaddr += PAGE_SIZE;
69e3c75f 3741 }
4ebf0ae2 3742 }
1da177e4 3743 }
69e3c75f 3744
4ebf0ae2 3745 atomic_inc(&po->mapped);
1da177e4
LT		 3746 vma->vm_ops = &packet_mmap_ops;
3747 err = 0;
3748
3749out:
905db440 3750 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3751 return err;
3752}
1da177e4 3753
90ddc4f0 3754static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3755 .family = PF_PACKET,
3756 .owner = THIS_MODULE,
3757 .release = packet_release,
3758 .bind = packet_bind_spkt,
3759 .connect = sock_no_connect,
3760 .socketpair = sock_no_socketpair,
3761 .accept = sock_no_accept,
3762 .getname = packet_getname_spkt,
3763 .poll = datagram_poll,
3764 .ioctl = packet_ioctl,
3765 .listen = sock_no_listen,
3766 .shutdown = sock_no_shutdown,
3767 .setsockopt = sock_no_setsockopt,
3768 .getsockopt = sock_no_getsockopt,
3769 .sendmsg = packet_sendmsg_spkt,
3770 .recvmsg = packet_recvmsg,
3771 .mmap = sock_no_mmap,
3772 .sendpage = sock_no_sendpage,
3773};
1da177e4 3774
90ddc4f0 3775static const struct proto_ops packet_ops = {
1da177e4
LT		 3776 .family = PF_PACKET,
3777 .owner = THIS_MODULE,
3778 .release = packet_release,
3779 .bind = packet_bind,
3780 .connect = sock_no_connect,
3781 .socketpair = sock_no_socketpair,
3782 .accept = sock_no_accept,
1ce4f28b 3783 .getname = packet_getname,
1da177e4
LT		 3784 .poll = packet_poll,
3785 .ioctl = packet_ioctl,
3786 .listen = sock_no_listen,
3787 .shutdown = sock_no_shutdown,
3788 .setsockopt = packet_setsockopt,
3789 .getsockopt = packet_getsockopt,
3790 .sendmsg = packet_sendmsg,
3791 .recvmsg = packet_recvmsg,
3792 .mmap = packet_mmap,
3793 .sendpage = sock_no_sendpage,
3794};
3795
ec1b4cf7 3796static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3797 .family = PF_PACKET,
3798 .create = packet_create,
3799 .owner = THIS_MODULE,
3800};
3801
3802static struct notifier_block packet_netdev_notifier = {
40d4e3df 3803 .notifier_call = packet_notifier,
1da177e4
LT		 3804};
3805
3806#ifdef CONFIG_PROC_FS
1da177e4
LT		 3807
3808static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3809 __acquires(RCU)
1da177e4 3810{
e372c414 3811 struct net *net = seq_file_net(seq);
808f5114 3812
3813 rcu_read_lock();
3814 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3815}
3816
3817static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3818{
1bf40954 3819 struct net *net = seq_file_net(seq);
808f5114 3820 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3821}
3822
3823static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3824 __releases(RCU)
1da177e4 3825{
808f5114 3826 rcu_read_unlock();
1da177e4
LT		 3827}
3828
1ce4f28b 3829static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3830{
3831 if (v == SEQ_START_TOKEN)
3832 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3833 else {
b7ceabd9 3834 struct sock *s = sk_entry(v);
1da177e4
LT		 3835 const struct packet_sock *po = pkt_sk(s);
3836
3837 seq_printf(seq,
71338aa7 3838 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3839 s,
3840 atomic_read(&s->sk_refcnt),
3841 s->sk_type,
3842 ntohs(po->num),
3843 po->ifindex,
3844 po->running,
3845 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 3846 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 3847 sock_i_ino(s));
1da177e4
LT		 3848 }
3849
3850 return 0;
3851}
3852
56b3d975 3853static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3854 .start = packet_seq_start,
3855 .next = packet_seq_next,
3856 .stop = packet_seq_stop,
3857 .show = packet_seq_show,
3858};
3859
3860static int packet_seq_open(struct inode *inode, struct file *file)
3861{
e372c414
DL		 3862 return seq_open_net(inode, file, &packet_seq_ops,
3863 sizeof(struct seq_net_private));
1da177e4
LT		 3864}
3865
da7071d7 3866static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3867 .owner = THIS_MODULE,
3868 .open = packet_seq_open,
3869 .read = seq_read,
3870 .llseek = seq_lseek,
e372c414 3871 .release = seq_release_net,
1da177e4
LT		 3872};
3873
3874#endif
3875
2c8c1e72 3876static int __net_init packet_net_init(struct net *net)
d12d01d6 3877{
0fa7fa98 3878 mutex_init(&net->packet.sklist_lock);
2aaef4e4 3879 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 3880
d4beaa66 3881 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 3882 return -ENOMEM;
3883
3884 return 0;
3885}
3886
2c8c1e72 3887static void __net_exit packet_net_exit(struct net *net)
d12d01d6 3888{
ece31ffd 3889 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 3890}
3891
3892static struct pernet_operations packet_net_ops = {
3893 .init = packet_net_init,
3894 .exit = packet_net_exit,
3895};
3896
3897
1da177e4
LT		 3898static void __exit packet_exit(void)
3899{
1da177e4 3900 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3901 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3902 sock_unregister(PF_PACKET);
3903 proto_unregister(&packet_proto);
3904}
3905
3906static int __init packet_init(void)
3907{
3908 int rc = proto_register(&packet_proto, 0);
3909
3910 if (rc != 0)
3911 goto out;
3912
3913 sock_register(&packet_family_ops);
d12d01d6 3914 register_pernet_subsys(&packet_net_ops);
1da177e4 3915 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3916out:
3917 return rc;
3918}
3919
3920module_init(packet_init);
3921module_exit(packet_exit);
3922MODULE_LICENSE("GPL");
3923MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 1.162318 seconds and 5 git commands to generate.