projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
vmci: propagate msghdr all way down to __qp_memcpy_from_queue()
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
b0138408 91#include <linux/percpu.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
d346a3fa
DB		 240static int packet_direct_xmit(struct sk_buff *skb)
241{
242 struct net_device *dev = skb->dev;
d346a3fa
DB		 243 netdev_features_t features;
244 struct netdev_queue *txq;
43279500 245 int ret = NETDEV_TX_BUSY;
d346a3fa
DB		 246
247 if (unlikely(!netif_running(dev) ||
43279500
DB		 248 !netif_carrier_ok(dev)))
249 goto drop;
d346a3fa
DB		 250
251 features = netif_skb_features(skb);
252 if (skb_needs_linearize(skb, features) &&
43279500
DB		 253 __skb_linearize(skb))
254 goto drop;
d346a3fa 255
10c51b56 256 txq = skb_get_tx_queue(dev, skb);
d346a3fa 257
43279500
DB		 258 local_bh_disable();
259
260 HARD_TX_LOCK(dev, txq, smp_processor_id());
10b3ad8c 261 if (!netif_xmit_frozen_or_drv_stopped(txq))
fa2dbdc2 262 ret = netdev_start_xmit(skb, dev, txq, false);
43279500 263 HARD_TX_UNLOCK(dev, txq);
d346a3fa 264
43279500
DB		 265 local_bh_enable();
266
267 if (!dev_xmit_complete(ret))
d346a3fa 268 kfree_skb(skb);
43279500 269
d346a3fa 270 return ret;
43279500 271drop:
0f97ede4 272 atomic_long_inc(&dev->tx_dropped);
43279500
DB		 273 kfree_skb(skb);
274 return NET_XMIT_DROP;
d346a3fa
DB		 275}
276
66e56cd4
DB		 277static struct net_device *packet_cached_dev_get(struct packet_sock *po)
278{
279 struct net_device *dev;
280
281 rcu_read_lock();
282 dev = rcu_dereference(po->cached_dev);
283 if (likely(dev))
284 dev_hold(dev);
285 rcu_read_unlock();
286
287 return dev;
288}
289
290static void packet_cached_dev_assign(struct packet_sock *po,
291 struct net_device *dev)
292{
293 rcu_assign_pointer(po->cached_dev, dev);
294}
295
296static void packet_cached_dev_reset(struct packet_sock *po)
297{
298 RCU_INIT_POINTER(po->cached_dev, NULL);
299}
300
d346a3fa
DB		 301static bool packet_use_direct_xmit(const struct packet_sock *po)
302{
303 return po->xmit == packet_direct_xmit;
304}
305
0fd5d57b 306static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
d346a3fa 307{
1cbac010 308 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 309}
310
0fd5d57b
DB		 311static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
312{
313 const struct net_device_ops *ops = dev->netdev_ops;
314 u16 queue_index;
315
316 if (ops->ndo_select_queue) {
317 queue_index = ops->ndo_select_queue(dev, skb, NULL,
318 __packet_pick_tx_queue);
319 queue_index = netdev_cap_txqueue(dev, queue_index);
320 } else {
321 queue_index = __packet_pick_tx_queue(dev, skb);
322 }
323
324 skb_set_queue_mapping(skb, queue_index);
325}
326
ce06b03e
DM		 327/* register_prot_hook must be invoked with the po->bind_lock held,
328 * or from a context in which asynchronous accesses to the packet
329 * socket is not possible (packet_create()).
330 */
331static void register_prot_hook(struct sock *sk)
332{
333 struct packet_sock *po = pkt_sk(sk);
e40526cb 334
ce06b03e 335 if (!po->running) {
66e56cd4 336 if (po->fanout)
dc99f600 337 __fanout_link(sk, po);
66e56cd4 338 else
dc99f600 339 dev_add_pack(&po->prot_hook);
e40526cb 340
ce06b03e
DM		 341 sock_hold(sk);
342 po->running = 1;
343 }
344}
345
346/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
347 * held. If the sync parameter is true, we will temporarily drop
348 * the po->bind_lock and do a synchronize_net to make sure no
349 * asynchronous packet processing paths still refer to the elements
350 * of po->prot_hook. If the sync parameter is false, it is the
351 * callers responsibility to take care of this.
352 */
353static void __unregister_prot_hook(struct sock *sk, bool sync)
354{
355 struct packet_sock *po = pkt_sk(sk);
356
357 po->running = 0;
66e56cd4
DB		 358
359 if (po->fanout)
dc99f600 360 __fanout_unlink(sk, po);
66e56cd4 361 else
dc99f600 362 __dev_remove_pack(&po->prot_hook);
e40526cb 363
ce06b03e
DM		 364 __sock_put(sk);
365
366 if (sync) {
367 spin_unlock(&po->bind_lock);
368 synchronize_net();
369 spin_lock(&po->bind_lock);
370 }
371}
372
373static void unregister_prot_hook(struct sock *sk, bool sync)
374{
375 struct packet_sock *po = pkt_sk(sk);
376
377 if (po->running)
378 __unregister_prot_hook(sk, sync);
379}
380
6e58040b 381static inline struct page * __pure pgv_to_page(void *addr)
0af55bb5
CG		 382{
383 if (is_vmalloc_addr(addr))
384 return vmalloc_to_page(addr);
385 return virt_to_page(addr);
386}
387
69e3c75f 388static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 389{
184f489e 390 union tpacket_uhdr h;
1da177e4 391
69e3c75f 392 h.raw = frame;
bbd6ef87
PM		 393 switch (po->tp_version) {
394 case TPACKET_V1:
69e3c75f 395 h.h1->tp_status = status;
0af55bb5 396 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 397 break;
398 case TPACKET_V2:
69e3c75f 399 h.h2->tp_status = status;
0af55bb5 400 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 401 break;
f6fb8f10 402 case TPACKET_V3:
69e3c75f 403 default:
f6fb8f10 404 WARN(1, "TPACKET version not supported.\n");
69e3c75f 405 BUG();
bbd6ef87 406 }
69e3c75f
JB		 407
408 smp_wmb();
bbd6ef87
PM		 409}
410
69e3c75f 411static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 412{
184f489e 413 union tpacket_uhdr h;
bbd6ef87 414
69e3c75f
JB		 415 smp_rmb();
416
bbd6ef87
PM		 417 h.raw = frame;
418 switch (po->tp_version) {
419 case TPACKET_V1:
0af55bb5 420 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 421 return h.h1->tp_status;
bbd6ef87 422 case TPACKET_V2:
0af55bb5 423 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 424 return h.h2->tp_status;
f6fb8f10 425 case TPACKET_V3:
69e3c75f 426 default:
f6fb8f10 427 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 428 BUG();
429 return 0;
bbd6ef87 430 }
1da177e4 431}
69e3c75f 432
b9c32fb2
DB		 433static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
434 unsigned int flags)
7a51384c
DB		 435{
436 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
437
68a360e8
WB		 438 if (shhwtstamps &&
439 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
440 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
441 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 442
443 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 444 return TP_STATUS_TS_SOFTWARE;
7a51384c 445
b9c32fb2 446 return 0;
7a51384c
DB		 447}
448
b9c32fb2
DB		 449static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
450 struct sk_buff *skb)
2e31396f
WB		 451{
452 union tpacket_uhdr h;
453 struct timespec ts;
b9c32fb2 454 __u32 ts_status;
2e31396f 455
b9c32fb2
DB		 456 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
457 return 0;
2e31396f
WB		 458
459 h.raw = frame;
460 switch (po->tp_version) {
461 case TPACKET_V1:
462 h.h1->tp_sec = ts.tv_sec;
463 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
464 break;
465 case TPACKET_V2:
466 h.h2->tp_sec = ts.tv_sec;
467 h.h2->tp_nsec = ts.tv_nsec;
468 break;
469 case TPACKET_V3:
470 default:
471 WARN(1, "TPACKET version not supported.\n");
472 BUG();
473 }
474
475 /* one flush is safe, as both fields always lie on the same cacheline */
476 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
477 smp_wmb();
b9c32fb2
DB		 478
479 return ts_status;
2e31396f
WB		 480}
481
69e3c75f
JB		 482static void *packet_lookup_frame(struct packet_sock *po,
483 struct packet_ring_buffer *rb,
484 unsigned int position,
485 int status)
486{
487 unsigned int pg_vec_pos, frame_offset;
184f489e 488 union tpacket_uhdr h;
69e3c75f
JB		 489
490 pg_vec_pos = position / rb->frames_per_block;
491 frame_offset = position % rb->frames_per_block;
492
0e3125c7
NH		 493 h.raw = rb->pg_vec[pg_vec_pos].buffer +
494 (frame_offset * rb->frame_size);
69e3c75f
JB		 495
496 if (status != __packet_get_status(po, h.raw))
497 return NULL;
498
499 return h.raw;
500}
501
eea49cc9 502static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 503 struct packet_ring_buffer *rb,
504 int status)
505{
506 return packet_lookup_frame(po, rb, rb->head, status);
507}
508
bc59ba39 509static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 510{
511 del_timer_sync(&pkc->retire_blk_timer);
512}
513
514static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
515 int tx_ring,
516 struct sk_buff_head *rb_queue)
517{
bc59ba39 518 struct tpacket_kbdq_core *pkc;
f6fb8f10 519
22781a5b
DJ		 520 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
521 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 522
ec6f809f 523 spin_lock_bh(&rb_queue->lock);
f6fb8f10 524 pkc->delete_blk_timer = 1;
ec6f809f 525 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 526
527 prb_del_retire_blk_timer(pkc);
528}
529
530static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 531 struct tpacket_kbdq_core *pkc,
f6fb8f10 532 void (*func) (unsigned long))
533{
534 init_timer(&pkc->retire_blk_timer);
535 pkc->retire_blk_timer.data = (long)po;
536 pkc->retire_blk_timer.function = func;
537 pkc->retire_blk_timer.expires = jiffies;
538}
539
540static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
541{
bc59ba39 542 struct tpacket_kbdq_core *pkc;
f6fb8f10 543
544 if (tx_ring)
545 BUG();
546
22781a5b
DJ		 547 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
548 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 549 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
550}
551
552static int prb_calc_retire_blk_tmo(struct packet_sock *po,
553 int blk_size_in_bytes)
554{
555 struct net_device *dev;
556 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 557 struct ethtool_cmd ecmd;
558 int err;
e440cf2c 559 u32 speed;
f6fb8f10 560
4bc71cb9
JP		 561 rtnl_lock();
562 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
563 if (unlikely(!dev)) {
564 rtnl_unlock();
f6fb8f10 565 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 566 }
567 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 568 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 569 rtnl_unlock();
570 if (!err) {
4bc71cb9
JP		 571 /*
572 * If the link speed is so slow you don't really
573 * need to worry about perf anyways
574 */
e440cf2c 575 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 576 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 577 } else {
578 msec = 1;
579 div = speed / 1000;
f6fb8f10 580 }
581 }
582
583 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
584
585 if (div)
586 mbits /= div;
587
588 tmo = mbits * msec;
589
590 if (div)
591 return tmo+1;
592 return tmo;
593}
594
bc59ba39 595static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 596 union tpacket_req_u *req_u)
597{
598 p1->feature_req_word = req_u->req3.tp_feature_req_word;
599}
600
601static void init_prb_bdqc(struct packet_sock *po,
602 struct packet_ring_buffer *rb,
603 struct pgv *pg_vec,
604 union tpacket_req_u *req_u, int tx_ring)
605{
22781a5b 606 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 607 struct tpacket_block_desc *pbd;
f6fb8f10 608
609 memset(p1, 0x0, sizeof(*p1));
610
611 p1->knxt_seq_num = 1;
612 p1->pkbdq = pg_vec;
bc59ba39 613 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 614 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 615 p1->kblk_size = req_u->req3.tp_block_size;
616 p1->knum_blocks = req_u->req3.tp_block_nr;
617 p1->hdrlen = po->tp_hdrlen;
618 p1->version = po->tp_version;
619 p1->last_kactive_blk_num = 0;
ee80fbf3 620 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 621 if (req_u->req3.tp_retire_blk_tov)
622 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
623 else
624 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
625 req_u->req3.tp_block_size);
626 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
627 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
628
dc808110 629 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
f6fb8f10 630 prb_init_ft_ops(p1, req_u);
631 prb_setup_retire_blk_timer(po, tx_ring);
632 prb_open_block(p1, pbd);
633}
634
635/* Do NOT update the last_blk_num first.
636 * Assumes sk_buff_head lock is held.
637 */
bc59ba39 638static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 639{
640 mod_timer(&pkc->retire_blk_timer,
641 jiffies + pkc->tov_in_jiffies);
642 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
643}
644
645/*
646 * Timer logic:
647 * 1) We refresh the timer only when we open a block.
648 * By doing this we don't waste cycles refreshing the timer
649 * on packet-by-packet basis.
650 *
651 * With a 1MB block-size, on a 1Gbps line, it will take
652 * i) ~8 ms to fill a block + ii) memcpy etc.
653 * In this cut we are not accounting for the memcpy time.
654 *
655 * So, if the user sets the 'tmo' to 10ms then the timer
656 * will never fire while the block is still getting filled
657 * (which is what we want). However, the user could choose
658 * to close a block early and that's fine.
659 *
660 * But when the timer does fire, we check whether or not to refresh it.
661 * Since the tmo granularity is in msecs, it is not too expensive
662 * to refresh the timer, lets say every '8' msecs.
663 * Either the user can set the 'tmo' or we can derive it based on
664 * a) line-speed and b) block-size.
665 * prb_calc_retire_blk_tmo() calculates the tmo.
666 *
667 */
668static void prb_retire_rx_blk_timer_expired(unsigned long data)
669{
670 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 671 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 672 unsigned int frozen;
bc59ba39 673 struct tpacket_block_desc *pbd;
f6fb8f10 674
675 spin_lock(&po->sk.sk_receive_queue.lock);
676
677 frozen = prb_queue_frozen(pkc);
678 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
679
680 if (unlikely(pkc->delete_blk_timer))
681 goto out;
682
683 /* We only need to plug the race when the block is partially filled.
684 * tpacket_rcv:
685 * lock(); increment BLOCK_NUM_PKTS; unlock()
686 * copy_bits() is in progress ...
687 * timer fires on other cpu:
688 * we can't retire the current block because copy_bits
689 * is in progress.
690 *
691 */
692 if (BLOCK_NUM_PKTS(pbd)) {
693 while (atomic_read(&pkc->blk_fill_in_prog)) {
694 /* Waiting for skb_copy_bits to finish... */
695 cpu_relax();
696 }
697 }
698
699 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
700 if (!frozen) {
701 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
702 if (!prb_dispatch_next_block(pkc, po))
703 goto refresh_timer;
704 else
705 goto out;
706 } else {
707 /* Case 1. Queue was frozen because user-space was
708 * lagging behind.
709 */
710 if (prb_curr_blk_in_use(pkc, pbd)) {
711 /*
712 * Ok, user-space is still behind.
713 * So just refresh the timer.
714 */
715 goto refresh_timer;
716 } else {
717 /* Case 2. queue was frozen,user-space caught up,
718 * now the link went idle && the timer fired.
719 * We don't have a block to close.So we open this
720 * block and restart the timer.
721 * opening a block thaws the queue,restarts timer
722 * Thawing/timer-refresh is a side effect.
723 */
724 prb_open_block(pkc, pbd);
725 goto out;
726 }
727 }
728 }
729
730refresh_timer:
731 _prb_refresh_rx_retire_blk_timer(pkc);
732
733out:
734 spin_unlock(&po->sk.sk_receive_queue.lock);
735}
736
eea49cc9 737static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 738 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 739{
740 /* Flush everything minus the block header */
741
742#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
743 u8 *start, *end;
744
745 start = (u8 *)pbd1;
746
747 /* Skip the block header(we know header WILL fit in 4K) */
748 start += PAGE_SIZE;
749
750 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
751 for (; start < end; start += PAGE_SIZE)
752 flush_dcache_page(pgv_to_page(start));
753
754 smp_wmb();
755#endif
756
757 /* Now update the block status. */
758
759 BLOCK_STATUS(pbd1) = status;
760
761 /* Flush the block header */
762
763#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
764 start = (u8 *)pbd1;
765 flush_dcache_page(pgv_to_page(start));
766
767 smp_wmb();
768#endif
769}
770
771/*
772 * Side effect:
773 *
774 * 1) flush the block
775 * 2) Increment active_blk_num
776 *
777 * Note:We DONT refresh the timer on purpose.
778 * Because almost always the next block will be opened.
779 */
bc59ba39 780static void prb_close_block(struct tpacket_kbdq_core *pkc1,
781 struct tpacket_block_desc *pbd1,
f6fb8f10 782 struct packet_sock *po, unsigned int stat)
783{
784 __u32 status = TP_STATUS_USER | stat;
785
786 struct tpacket3_hdr *last_pkt;
bc59ba39 787 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 788
ee80fbf3 789 if (po->stats.stats3.tp_drops)
f6fb8f10 790 status |= TP_STATUS_LOSING;
791
792 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
793 last_pkt->tp_next_offset = 0;
794
795 /* Get the ts of the last pkt */
796 if (BLOCK_NUM_PKTS(pbd1)) {
797 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
798 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
799 } else {
800 /* Ok, we tmo'd - so get the current time */
801 struct timespec ts;
802 getnstimeofday(&ts);
803 h1->ts_last_pkt.ts_sec = ts.tv_sec;
804 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
805 }
806
807 smp_wmb();
808
809 /* Flush the block */
810 prb_flush_block(pkc1, pbd1, status);
811
812 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
813}
814
eea49cc9 815static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 816{
817 pkc->reset_pending_on_curr_blk = 0;
818}
819
820/*
821 * Side effect of opening a block:
822 *
823 * 1) prb_queue is thawed.
824 * 2) retire_blk_timer is refreshed.
825 *
826 */
bc59ba39 827static void prb_open_block(struct tpacket_kbdq_core *pkc1,
828 struct tpacket_block_desc *pbd1)
f6fb8f10 829{
830 struct timespec ts;
bc59ba39 831 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 832
833 smp_rmb();
834
8da3056c
DB		 835 /* We could have just memset this but we will lose the
836 * flexibility of making the priv area sticky
837 */
f6fb8f10 838
8da3056c
DB		 839 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
840 BLOCK_NUM_PKTS(pbd1) = 0;
841 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 842
8da3056c
DB		 843 getnstimeofday(&ts);
844
845 h1->ts_first_pkt.ts_sec = ts.tv_sec;
846 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 847
8da3056c
DB		 848 pkc1->pkblk_start = (char *)pbd1;
849 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
850
851 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
852 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
853
854 pbd1->version = pkc1->version;
855 pkc1->prev = pkc1->nxt_offset;
856 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
857
858 prb_thaw_queue(pkc1);
859 _prb_refresh_rx_retire_blk_timer(pkc1);
860
861 smp_wmb();
f6fb8f10 862}
863
864/*
865 * Queue freeze logic:
866 * 1) Assume tp_block_nr = 8 blocks.
867 * 2) At time 't0', user opens Rx ring.
868 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
869 * 4) user-space is either sleeping or processing block '0'.
870 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
871 * it will close block-7,loop around and try to fill block '0'.
872 * call-flow:
873 * __packet_lookup_frame_in_block
874 * prb_retire_current_block()
875 * prb_dispatch_next_block()
876 * |->(BLOCK_STATUS == USER) evaluates to true
877 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
878 * 6) Now there are two cases:
879 * 6.1) Link goes idle right after the queue is frozen.
880 * But remember, the last open_block() refreshed the timer.
881 * When this timer expires,it will refresh itself so that we can
882 * re-open block-0 in near future.
883 * 6.2) Link is busy and keeps on receiving packets. This is a simple
884 * case and __packet_lookup_frame_in_block will check if block-0
885 * is free and can now be re-used.
886 */
eea49cc9 887static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 888 struct packet_sock *po)
889{
890 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 891 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 892}
893
894#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
895
896/*
897 * If the next block is free then we will dispatch it
898 * and return a good offset.
899 * Else, we will freeze the queue.
900 * So, caller must check the return value.
901 */
bc59ba39 902static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 903 struct packet_sock *po)
904{
bc59ba39 905 struct tpacket_block_desc *pbd;
f6fb8f10 906
907 smp_rmb();
908
909 /* 1. Get current block num */
910 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
911
912 /* 2. If this block is currently in_use then freeze the queue */
913 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
914 prb_freeze_queue(pkc, po);
915 return NULL;
916 }
917
918 /*
919 * 3.
920 * open this block and return the offset where the first packet
921 * needs to get stored.
922 */
923 prb_open_block(pkc, pbd);
924 return (void *)pkc->nxt_offset;
925}
926
bc59ba39 927static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 928 struct packet_sock *po, unsigned int status)
929{
bc59ba39 930 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 931
932 /* retire/close the current block */
933 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
934 /*
935 * Plug the case where copy_bits() is in progress on
936 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
937 * have space to copy the pkt in the current block and
938 * called prb_retire_current_block()
939 *
940 * We don't need to worry about the TMO case because
941 * the timer-handler already handled this case.
942 */
943 if (!(status & TP_STATUS_BLK_TMO)) {
944 while (atomic_read(&pkc->blk_fill_in_prog)) {
945 /* Waiting for skb_copy_bits to finish... */
946 cpu_relax();
947 }
948 }
949 prb_close_block(pkc, pbd, po, status);
950 return;
951 }
f6fb8f10 952}
953
eea49cc9 954static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 955 struct tpacket_block_desc *pbd)
f6fb8f10 956{
957 return TP_STATUS_USER & BLOCK_STATUS(pbd);
958}
959
eea49cc9 960static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 961{
962 return pkc->reset_pending_on_curr_blk;
963}
964
eea49cc9 965static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 966{
bc59ba39 967 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 968 atomic_dec(&pkc->blk_fill_in_prog);
969}
970
eea49cc9 971static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 972 struct tpacket3_hdr *ppd)
973{
3958afa1 974 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 975}
976
eea49cc9 977static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 978 struct tpacket3_hdr *ppd)
979{
980 ppd->hv1.tp_rxhash = 0;
981}
982
eea49cc9 983static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 984 struct tpacket3_hdr *ppd)
985{
986 if (vlan_tx_tag_present(pkc->skb)) {
987 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
a0cdfcf3
AW		 988 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
989 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 990 } else {
9e67030a 991 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 992 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 993 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 994 }
995}
996
bc59ba39 997static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 998 struct tpacket3_hdr *ppd)
999{
a0cdfcf3 1000 ppd->hv1.tp_padding = 0;
f6fb8f10 1001 prb_fill_vlan_info(pkc, ppd);
1002
1003 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1004 prb_fill_rxhash(pkc, ppd);
1005 else
1006 prb_clear_rxhash(pkc, ppd);
1007}
1008
eea49cc9 1009static void prb_fill_curr_block(char *curr,
bc59ba39 1010 struct tpacket_kbdq_core *pkc,
1011 struct tpacket_block_desc *pbd,
f6fb8f10 1012 unsigned int len)
1013{
1014 struct tpacket3_hdr *ppd;
1015
1016 ppd = (struct tpacket3_hdr *)curr;
1017 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1018 pkc->prev = curr;
1019 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1020 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1021 BLOCK_NUM_PKTS(pbd) += 1;
1022 atomic_inc(&pkc->blk_fill_in_prog);
1023 prb_run_all_ft_ops(pkc, ppd);
1024}
1025
1026/* Assumes caller has the sk->rx_queue.lock */
1027static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1028 struct sk_buff *skb,
1029 int status,
1030 unsigned int len
1031 )
1032{
bc59ba39 1033 struct tpacket_kbdq_core *pkc;
1034 struct tpacket_block_desc *pbd;
f6fb8f10 1035 char *curr, *end;
1036
e3192690 1037 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1038 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1039
1040 /* Queue is frozen when user space is lagging behind */
1041 if (prb_queue_frozen(pkc)) {
1042 /*
1043 * Check if that last block which caused the queue to freeze,
1044 * is still in_use by user-space.
1045 */
1046 if (prb_curr_blk_in_use(pkc, pbd)) {
1047 /* Can't record this packet */
1048 return NULL;
1049 } else {
1050 /*
1051 * Ok, the block was released by user-space.
1052 * Now let's open that block.
1053 * opening a block also thaws the queue.
1054 * Thawing is a side effect.
1055 */
1056 prb_open_block(pkc, pbd);
1057 }
1058 }
1059
1060 smp_mb();
1061 curr = pkc->nxt_offset;
1062 pkc->skb = skb;
e3192690 1063 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1064
1065 /* first try the current block */
1066 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1067 prb_fill_curr_block(curr, pkc, pbd, len);
1068 return (void *)curr;
1069 }
1070
1071 /* Ok, close the current block */
1072 prb_retire_current_block(pkc, po, 0);
1073
1074 /* Now, try to dispatch the next block */
1075 curr = (char *)prb_dispatch_next_block(pkc, po);
1076 if (curr) {
1077 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1078 prb_fill_curr_block(curr, pkc, pbd, len);
1079 return (void *)curr;
1080 }
1081
1082 /*
1083 * No free blocks are available.user_space hasn't caught up yet.
1084 * Queue was just frozen and now this packet will get dropped.
1085 */
1086 return NULL;
1087}
1088
eea49cc9 1089static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1090 struct sk_buff *skb,
1091 int status, unsigned int len)
1092{
1093 char *curr = NULL;
1094 switch (po->tp_version) {
1095 case TPACKET_V1:
1096 case TPACKET_V2:
1097 curr = packet_lookup_frame(po, &po->rx_ring,
1098 po->rx_ring.head, status);
1099 return curr;
1100 case TPACKET_V3:
1101 return __packet_lookup_frame_in_block(po, skb, status, len);
1102 default:
1103 WARN(1, "TPACKET version not supported\n");
1104 BUG();
99aa3473 1105 return NULL;
f6fb8f10 1106 }
1107}
1108
eea49cc9 1109static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1110 struct packet_ring_buffer *rb,
77f65ebd 1111 unsigned int idx,
f6fb8f10 1112 int status)
1113{
bc59ba39 1114 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1115 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1116
1117 if (status != BLOCK_STATUS(pbd))
1118 return NULL;
1119 return pbd;
1120}
1121
eea49cc9 1122static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1123{
1124 unsigned int prev;
1125 if (rb->prb_bdqc.kactive_blk_num)
1126 prev = rb->prb_bdqc.kactive_blk_num-1;
1127 else
1128 prev = rb->prb_bdqc.knum_blocks-1;
1129 return prev;
1130}
1131
1132/* Assumes caller has held the rx_queue.lock */
eea49cc9 1133static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1134 struct packet_ring_buffer *rb,
1135 int status)
1136{
1137 unsigned int previous = prb_previous_blk_num(rb);
1138 return prb_lookup_block(po, rb, previous, status);
1139}
1140
eea49cc9 1141static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1142 struct packet_ring_buffer *rb,
1143 int status)
1144{
1145 if (po->tp_version <= TPACKET_V2)
1146 return packet_previous_frame(po, rb, status);
1147
1148 return __prb_previous_block(po, rb, status);
1149}
1150
eea49cc9 1151static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1152 struct packet_ring_buffer *rb)
1153{
1154 switch (po->tp_version) {
1155 case TPACKET_V1:
1156 case TPACKET_V2:
1157 return packet_increment_head(rb);
1158 case TPACKET_V3:
1159 default:
1160 WARN(1, "TPACKET version not supported.\n");
1161 BUG();
1162 return;
1163 }
1164}
1165
eea49cc9 1166static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1167 struct packet_ring_buffer *rb,
1168 int status)
1169{
1170 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1171 return packet_lookup_frame(po, rb, previous, status);
1172}
1173
eea49cc9 1174static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1175{
1176 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1177}
1178
b0138408
DB		 1179static void packet_inc_pending(struct packet_ring_buffer *rb)
1180{
1181 this_cpu_inc(*rb->pending_refcnt);
1182}
1183
1184static void packet_dec_pending(struct packet_ring_buffer *rb)
1185{
1186 this_cpu_dec(*rb->pending_refcnt);
1187}
1188
1189static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1190{
1191 unsigned int refcnt = 0;
1192 int cpu;
1193
1194 /* We don't use pending refcount in rx_ring. */
1195 if (rb->pending_refcnt == NULL)
1196 return 0;
1197
1198 for_each_possible_cpu(cpu)
1199 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1200
1201 return refcnt;
1202}
1203
1204static int packet_alloc_pending(struct packet_sock *po)
1205{
1206 po->rx_ring.pending_refcnt = NULL;
1207
1208 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1209 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1210 return -ENOBUFS;
1211
1212 return 0;
1213}
1214
1215static void packet_free_pending(struct packet_sock *po)
1216{
1217 free_percpu(po->tx_ring.pending_refcnt);
1218}
1219
77f65ebd
WB		 1220static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1221{
1222 struct sock *sk = &po->sk;
1223 bool has_room;
1224
1225 if (po->prot_hook.func != tpacket_rcv)
1226 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1227 <= sk->sk_rcvbuf;
1228
1229 spin_lock(&sk->sk_receive_queue.lock);
1230 if (po->tp_version == TPACKET_V3)
1231 has_room = prb_lookup_block(po, &po->rx_ring,
1232 po->rx_ring.prb_bdqc.kactive_blk_num,
1233 TP_STATUS_KERNEL);
1234 else
1235 has_room = packet_lookup_frame(po, &po->rx_ring,
1236 po->rx_ring.head,
1237 TP_STATUS_KERNEL);
1238 spin_unlock(&sk->sk_receive_queue.lock);
1239
1240 return has_room;
1241}
1242
1da177e4
LT		 1243static void packet_sock_destruct(struct sock *sk)
1244{
ed85b565
RC		 1245 skb_queue_purge(&sk->sk_error_queue);
1246
547b792c
IJ		 1247 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1248 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1249
1250 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1251 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1252 return;
1253 }
1254
17ab56a2 1255 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1256}
1257
dc99f600
DM		 1258static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1259{
1260 int x = atomic_read(&f->rr_cur) + 1;
1261
1262 if (x >= num)
1263 x = 0;
1264
1265 return x;
1266}
1267
77f65ebd
WB		 1268static unsigned int fanout_demux_hash(struct packet_fanout *f,
1269 struct sk_buff *skb,
1270 unsigned int num)
dc99f600 1271{
61b905da 1272 return reciprocal_scale(skb_get_hash(skb), num);
dc99f600
DM		 1273}
1274
77f65ebd
WB		 1275static unsigned int fanout_demux_lb(struct packet_fanout *f,
1276 struct sk_buff *skb,
1277 unsigned int num)
dc99f600
DM		 1278{
1279 int cur, old;
1280
1281 cur = atomic_read(&f->rr_cur);
1282 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1283 fanout_rr_next(f, num))) != cur)
1284 cur = old;
77f65ebd
WB		 1285 return cur;
1286}
1287
1288static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1289 struct sk_buff *skb,
1290 unsigned int num)
1291{
1292 return smp_processor_id() % num;
dc99f600
DM		 1293}
1294
5df0ddfb
DB		 1295static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1296 struct sk_buff *skb,
1297 unsigned int num)
1298{
f337db64 1299 return prandom_u32_max(num);
5df0ddfb
DB		 1300}
1301
77f65ebd
WB		 1302static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1303 struct sk_buff *skb,
1304 unsigned int idx, unsigned int skip,
1305 unsigned int num)
95ec3eb4 1306{
77f65ebd 1307 unsigned int i, j;
95ec3eb4 1308
77f65ebd
WB		 1309 i = j = min_t(int, f->next[idx], num - 1);
1310 do {
1311 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1312 if (i != j)
1313 f->next[idx] = i;
1314 return i;
1315 }
1316 if (++i == num)
1317 i = 0;
1318 } while (i != j);
1319
1320 return idx;
1321}
1322
2d36097d
NH		 1323static unsigned int fanout_demux_qm(struct packet_fanout *f,
1324 struct sk_buff *skb,
1325 unsigned int num)
1326{
1327 return skb_get_queue_mapping(skb) % num;
1328}
1329
77f65ebd
WB		 1330static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1331{
1332 return f->flags & (flag >> 8);
95ec3eb4
DM		 1333}
1334
95ec3eb4
DM		 1335static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1336 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1337{
1338 struct packet_fanout *f = pt->af_packet_priv;
1339 unsigned int num = f->num_members;
1340 struct packet_sock *po;
77f65ebd 1341 unsigned int idx;
dc99f600
DM		 1342
1343 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1344 !num) {
1345 kfree_skb(skb);
1346 return 0;
1347 }
1348
95ec3eb4
DM		 1349 switch (f->type) {
1350 case PACKET_FANOUT_HASH:
1351 default:
77f65ebd 1352 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1353 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1354 if (!skb)
1355 return 0;
1356 }
77f65ebd 1357 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1358 break;
1359 case PACKET_FANOUT_LB:
77f65ebd 1360 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1361 break;
1362 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1363 idx = fanout_demux_cpu(f, skb, num);
1364 break;
5df0ddfb
DB		 1365 case PACKET_FANOUT_RND:
1366 idx = fanout_demux_rnd(f, skb, num);
1367 break;
2d36097d
NH		 1368 case PACKET_FANOUT_QM:
1369 idx = fanout_demux_qm(f, skb, num);
1370 break;
77f65ebd
WB		 1371 case PACKET_FANOUT_ROLLOVER:
1372 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1373 break;
dc99f600
DM		 1374 }
1375
77f65ebd
WB		 1376 po = pkt_sk(f->arr[idx]);
1377 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1378 unlikely(!packet_rcv_has_room(po, skb))) {
1379 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1380 po = pkt_sk(f->arr[idx]);
1381 }
dc99f600
DM		 1382
1383 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1384}
1385
fff3321d
PE		 1386DEFINE_MUTEX(fanout_mutex);
1387EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1388static LIST_HEAD(fanout_list);
1389
1390static void __fanout_link(struct sock *sk, struct packet_sock *po)
1391{
1392 struct packet_fanout *f = po->fanout;
1393
1394 spin_lock(&f->lock);
1395 f->arr[f->num_members] = sk;
1396 smp_wmb();
1397 f->num_members++;
1398 spin_unlock(&f->lock);
1399}
1400
1401static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1402{
1403 struct packet_fanout *f = po->fanout;
1404 int i;
1405
1406 spin_lock(&f->lock);
1407 for (i = 0; i < f->num_members; i++) {
1408 if (f->arr[i] == sk)
1409 break;
1410 }
1411 BUG_ON(i >= f->num_members);
1412 f->arr[i] = f->arr[f->num_members - 1];
1413 f->num_members--;
1414 spin_unlock(&f->lock);
1415}
1416
d4dd8aee 1417static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1418{
d4dd8aee 1419 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
c0de08d0
EL		 1420 return true;
1421
1422 return false;
1423}
1424
7736d33f 1425static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1426{
1427 struct packet_sock *po = pkt_sk(sk);
1428 struct packet_fanout *f, *match;
7736d33f 1429 u8 type = type_flags & 0xff;
77f65ebd 1430 u8 flags = type_flags >> 8;
dc99f600
DM		 1431 int err;
1432
1433 switch (type) {
77f65ebd
WB		 1434 case PACKET_FANOUT_ROLLOVER:
1435 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1436 return -EINVAL;
dc99f600
DM		 1437 case PACKET_FANOUT_HASH:
1438 case PACKET_FANOUT_LB:
95ec3eb4 1439 case PACKET_FANOUT_CPU:
5df0ddfb 1440 case PACKET_FANOUT_RND:
2d36097d 1441 case PACKET_FANOUT_QM:
dc99f600
DM		 1442 break;
1443 default:
1444 return -EINVAL;
1445 }
1446
1447 if (!po->running)
1448 return -EINVAL;
1449
1450 if (po->fanout)
1451 return -EALREADY;
1452
1453 mutex_lock(&fanout_mutex);
1454 match = NULL;
1455 list_for_each_entry(f, &fanout_list, list) {
1456 if (f->id == id &&
1457 read_pnet(&f->net) == sock_net(sk)) {
1458 match = f;
1459 break;
1460 }
1461 }
afe62c68 1462 err = -EINVAL;
77f65ebd 1463 if (match && match->flags != flags)
afe62c68 1464 goto out;
dc99f600 1465 if (!match) {
afe62c68 1466 err = -ENOMEM;
dc99f600 1467 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1468 if (!match)
1469 goto out;
1470 write_pnet(&match->net, sock_net(sk));
1471 match->id = id;
1472 match->type = type;
77f65ebd 1473 match->flags = flags;
afe62c68
ED		 1474 atomic_set(&match->rr_cur, 0);
1475 INIT_LIST_HEAD(&match->list);
1476 spin_lock_init(&match->lock);
1477 atomic_set(&match->sk_ref, 0);
1478 match->prot_hook.type = po->prot_hook.type;
1479 match->prot_hook.dev = po->prot_hook.dev;
1480 match->prot_hook.func = packet_rcv_fanout;
1481 match->prot_hook.af_packet_priv = match;
c0de08d0 1482 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1483 dev_add_pack(&match->prot_hook);
1484 list_add(&match->list, &fanout_list);
dc99f600 1485 }
afe62c68
ED		 1486 err = -EINVAL;
1487 if (match->type == type &&
1488 match->prot_hook.type == po->prot_hook.type &&
1489 match->prot_hook.dev == po->prot_hook.dev) {
1490 err = -ENOSPC;
1491 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1492 __dev_remove_pack(&po->prot_hook);
1493 po->fanout = match;
1494 atomic_inc(&match->sk_ref);
1495 __fanout_link(sk, po);
1496 err = 0;
dc99f600
DM		 1497 }
1498 }
afe62c68 1499out:
dc99f600
DM		 1500 mutex_unlock(&fanout_mutex);
1501 return err;
1502}
1503
1504static void fanout_release(struct sock *sk)
1505{
1506 struct packet_sock *po = pkt_sk(sk);
1507 struct packet_fanout *f;
1508
1509 f = po->fanout;
1510 if (!f)
1511 return;
1512
fff3321d 1513 mutex_lock(&fanout_mutex);
dc99f600
DM		 1514 po->fanout = NULL;
1515
dc99f600
DM		 1516 if (atomic_dec_and_test(&f->sk_ref)) {
1517 list_del(&f->list);
1518 dev_remove_pack(&f->prot_hook);
1519 kfree(f);
1520 }
1521 mutex_unlock(&fanout_mutex);
1522}
1da177e4 1523
90ddc4f0 1524static const struct proto_ops packet_ops;
1da177e4 1525
90ddc4f0 1526static const struct proto_ops packet_ops_spkt;
1da177e4 1527
40d4e3df
ED		 1528static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1529 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1530{
1531 struct sock *sk;
1532 struct sockaddr_pkt *spkt;
1533
1534 /*
1535 * When we registered the protocol we saved the socket in the data
1536 * field for just this event.
1537 */
1538
1539 sk = pt->af_packet_priv;
1ce4f28b 1540
1da177e4
LT		 1541 /*
1542 * Yank back the headers [hope the device set this
1543 * right or kerboom...]
1544 *
1545 * Incoming packets have ll header pulled,
1546 * push it back.
1547 *
98e399f8 1548 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1549 * so that this procedure is noop.
1550 */
1551
1552 if (skb->pkt_type == PACKET_LOOPBACK)
1553 goto out;
1554
09ad9bc7 1555 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1556 goto out;
1557
40d4e3df
ED		 1558 skb = skb_share_check(skb, GFP_ATOMIC);
1559 if (skb == NULL)
1da177e4
LT		 1560 goto oom;
1561
1562 /* drop any routing info */
adf30907 1563 skb_dst_drop(skb);
1da177e4 1564
84531c24
PO		 1565 /* drop conntrack reference */
1566 nf_reset(skb);
1567
ffbc6111 1568 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1569
98e399f8 1570 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1571
1572 /*
1573 * The SOCK_PACKET socket receives _all_ frames.
1574 */
1575
1576 spkt->spkt_family = dev->type;
1577 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1578 spkt->spkt_protocol = skb->protocol;
1579
1580 /*
1581 * Charge the memory to the socket. This is done specifically
1582 * to prevent sockets using all the memory up.
1583 */
1584
40d4e3df 1585 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1586 return 0;
1587
1588out:
1589 kfree_skb(skb);
1590oom:
1591 return 0;
1592}
1593
1594
1595/*
1596 * Output a raw packet to a device layer. This bypasses all the other
1597 * protocol layers and you must therefore supply it with a complete frame
1598 */
1ce4f28b 1599
1da177e4
LT		 1600static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1601 struct msghdr *msg, size_t len)
1602{
1603 struct sock *sk = sock->sk;
342dfc30 1604 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1605 struct sk_buff *skb = NULL;
1da177e4 1606 struct net_device *dev;
40d4e3df 1607 __be16 proto = 0;
1da177e4 1608 int err;
3bdc0eba 1609 int extra_len = 0;
1ce4f28b 1610
1da177e4 1611 /*
1ce4f28b 1612 * Get and verify the address.
1da177e4
LT		 1613 */
1614
40d4e3df 1615 if (saddr) {
1da177e4 1616 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1617 return -EINVAL;
1618 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1619 proto = saddr->spkt_protocol;
1620 } else
1621 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1622
1623 /*
1ce4f28b 1624 * Find the device first to size check it
1da177e4
LT		 1625 */
1626
de74e92a 1627 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1628retry:
654d1f8a
ED		 1629 rcu_read_lock();
1630 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1631 err = -ENODEV;
1632 if (dev == NULL)
1633 goto out_unlock;
1ce4f28b 1634
d5e76b0a
DM		 1635 err = -ENETDOWN;
1636 if (!(dev->flags & IFF_UP))
1637 goto out_unlock;
1638
1da177e4 1639 /*
40d4e3df
ED		 1640 * You may not queue a frame bigger than the mtu. This is the lowest level
1641 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1642 */
1ce4f28b 1643
3bdc0eba
BG		 1644 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1645 if (!netif_supports_nofcs(dev)) {
1646 err = -EPROTONOSUPPORT;
1647 goto out_unlock;
1648 }
1649 extra_len = 4; /* We're doing our own CRC */
1650 }
1651
1da177e4 1652 err = -EMSGSIZE;
3bdc0eba 1653 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1654 goto out_unlock;
1655
1a35ca80
ED		 1656 if (!skb) {
1657 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1658 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1659 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1660
1661 rcu_read_unlock();
4ce40912 1662 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1663 if (skb == NULL)
1664 return -ENOBUFS;
1665 /* FIXME: Save some space for broken drivers that write a hard
1666 * header at transmission time by themselves. PPP is the notable
1667 * one here. This should really be fixed at the driver level.
1668 */
1669 skb_reserve(skb, reserved);
1670 skb_reset_network_header(skb);
1671
1672 /* Try to align data part correctly */
1673 if (hhlen) {
1674 skb->data -= hhlen;
1675 skb->tail -= hhlen;
1676 if (len < hhlen)
1677 skb_reset_network_header(skb);
1678 }
6ce8e9ce 1679 err = memcpy_from_msg(skb_put(skb, len), msg, len);
1a35ca80
ED		 1680 if (err)
1681 goto out_free;
1682 goto retry;
1da177e4
LT		 1683 }
1684
3bdc0eba 1685 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1686 /* Earlier code assumed this would be a VLAN pkt,
1687 * double-check this now that we have the actual
1688 * packet in hand.
1689 */
1690 struct ethhdr *ehdr;
1691 skb_reset_mac_header(skb);
1692 ehdr = eth_hdr(skb);
1693 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1694 err = -EMSGSIZE;
1695 goto out_unlock;
1696 }
1697 }
1a35ca80 1698
1da177e4
LT		 1699 skb->protocol = proto;
1700 skb->dev = dev;
1701 skb->priority = sk->sk_priority;
2d37a186 1702 skb->mark = sk->sk_mark;
bf84a010
DB		 1703
1704 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1705
3bdc0eba
BG		 1706 if (unlikely(extra_len == 4))
1707 skb->no_fcs = 1;
1708
40893fd0 1709 skb_probe_transport_header(skb, 0);
c1aad275 1710
1da177e4 1711 dev_queue_xmit(skb);
654d1f8a 1712 rcu_read_unlock();
40d4e3df 1713 return len;
1da177e4 1714
1da177e4 1715out_unlock:
654d1f8a 1716 rcu_read_unlock();
1a35ca80
ED		 1717out_free:
1718 kfree_skb(skb);
1da177e4
LT		 1719 return err;
1720}
1da177e4 1721
eea49cc9 1722static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1723 const struct sock *sk,
dbcb5855 1724 unsigned int res)
1da177e4
LT		 1725{
1726 struct sk_filter *filter;
fda9ef5d 1727
80f8f102
ED		 1728 rcu_read_lock();
1729 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1730 if (filter != NULL)
0a14842f 1731 res = SK_RUN_FILTER(filter, skb);
80f8f102 1732 rcu_read_unlock();
1da177e4 1733
dbcb5855 1734 return res;
1da177e4
LT		 1735}
1736
1737/*
62ab0812
ED		 1738 * This function makes lazy skb cloning in hope that most of packets
1739 * are discarded by BPF.
1740 *
1741 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1742 * and skb->cb are mangled. It works because (and until) packets
1743 * falling here are owned by current CPU. Output packets are cloned
1744 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1745 * sequencially, so that if we return skb to original state on exit,
1746 * we will not harm anyone.
1da177e4
LT		 1747 */
1748
40d4e3df
ED		 1749static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1750 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1751{
1752 struct sock *sk;
1753 struct sockaddr_ll *sll;
1754 struct packet_sock *po;
40d4e3df 1755 u8 *skb_head = skb->data;
1da177e4 1756 int skb_len = skb->len;
dbcb5855 1757 unsigned int snaplen, res;
1da177e4
LT		 1758
1759 if (skb->pkt_type == PACKET_LOOPBACK)
1760 goto drop;
1761
1762 sk = pt->af_packet_priv;
1763 po = pkt_sk(sk);
1764
09ad9bc7 1765 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1766 goto drop;
1767
1da177e4
LT		 1768 skb->dev = dev;
1769
3b04ddde 1770 if (dev->header_ops) {
1da177e4 1771 /* The device has an explicit notion of ll header,
62ab0812
ED		 1772 * exported to higher levels.
1773 *
1774 * Otherwise, the device hides details of its frame
1775 * structure, so that corresponding packet head is
1776 * never delivered to user.
1da177e4
LT		 1777 */
1778 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1779 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1780 else if (skb->pkt_type == PACKET_OUTGOING) {
1781 /* Special case: outgoing packets have ll header at head */
bbe735e4 1782 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1783 }
1784 }
1785
1786 snaplen = skb->len;
1787
dbcb5855
DM		 1788 res = run_filter(skb, sk, snaplen);
1789 if (!res)
fda9ef5d 1790 goto drop_n_restore;
dbcb5855
DM		 1791 if (snaplen > res)
1792 snaplen = res;
1da177e4 1793
0fd7bac6 1794 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1795 goto drop_n_acct;
1796
1797 if (skb_shared(skb)) {
1798 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1799 if (nskb == NULL)
1800 goto drop_n_acct;
1801
1802 if (skb_head != skb->data) {
1803 skb->data = skb_head;
1804 skb->len = skb_len;
1805 }
abc4e4fa 1806 consume_skb(skb);
1da177e4
LT		 1807 skb = nskb;
1808 }
1809
ffbc6111
HX		 1810 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1811 sizeof(skb->cb));
1812
1813 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1814 sll->sll_family = AF_PACKET;
1815 sll->sll_hatype = dev->type;
1816 sll->sll_protocol = skb->protocol;
1817 sll->sll_pkttype = skb->pkt_type;
8032b464 1818 if (unlikely(po->origdev))
80feaacb
PWJ		 1819 sll->sll_ifindex = orig_dev->ifindex;
1820 else
1821 sll->sll_ifindex = dev->ifindex;
1da177e4 1822
b95cce35 1823 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1824
ffbc6111 1825 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1826
1da177e4
LT		 1827 if (pskb_trim(skb, snaplen))
1828 goto drop_n_acct;
1829
1830 skb_set_owner_r(skb, sk);
1831 skb->dev = NULL;
adf30907 1832 skb_dst_drop(skb);
1da177e4 1833
84531c24
PO		 1834 /* drop conntrack reference */
1835 nf_reset(skb);
1836
1da177e4 1837 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1838 po->stats.stats1.tp_packets++;
3b885787 1839 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1840 __skb_queue_tail(&sk->sk_receive_queue, skb);
1841 spin_unlock(&sk->sk_receive_queue.lock);
676d2369 1842 sk->sk_data_ready(sk);
1da177e4
LT		 1843 return 0;
1844
1845drop_n_acct:
7091fbd8 1846 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1847 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1848 atomic_inc(&sk->sk_drops);
1849 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1850
1851drop_n_restore:
1852 if (skb_head != skb->data && skb_shared(skb)) {
1853 skb->data = skb_head;
1854 skb->len = skb_len;
1855 }
1856drop:
ead2ceb0 1857 consume_skb(skb);
1da177e4
LT		 1858 return 0;
1859}
1860
40d4e3df
ED		 1861static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1862 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1863{
1864 struct sock *sk;
1865 struct packet_sock *po;
1866 struct sockaddr_ll *sll;
184f489e 1867 union tpacket_uhdr h;
40d4e3df 1868 u8 *skb_head = skb->data;
1da177e4 1869 int skb_len = skb->len;
dbcb5855 1870 unsigned int snaplen, res;
f6fb8f10 1871 unsigned long status = TP_STATUS_USER;
bbd6ef87 1872 unsigned short macoff, netoff, hdrlen;
1da177e4 1873 struct sk_buff *copy_skb = NULL;
bbd6ef87 1874 struct timespec ts;
b9c32fb2 1875 __u32 ts_status;
1da177e4 1876
51846355
AW		 1877 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1878 * We may add members to them until current aligned size without forcing
1879 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1880 */
1881 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1882 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1883
1da177e4
LT		 1884 if (skb->pkt_type == PACKET_LOOPBACK)
1885 goto drop;
1886
1887 sk = pt->af_packet_priv;
1888 po = pkt_sk(sk);
1889
09ad9bc7 1890 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1891 goto drop;
1892
3b04ddde 1893 if (dev->header_ops) {
1da177e4 1894 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1895 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1896 else if (skb->pkt_type == PACKET_OUTGOING) {
1897 /* Special case: outgoing packets have ll header at head */
bbe735e4 1898 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1899 }
1900 }
1901
8dc41944
HX		 1902 if (skb->ip_summed == CHECKSUM_PARTIAL)
1903 status |= TP_STATUS_CSUMNOTREADY;
1904
1da177e4
LT		 1905 snaplen = skb->len;
1906
dbcb5855
DM		 1907 res = run_filter(skb, sk, snaplen);
1908 if (!res)
fda9ef5d 1909 goto drop_n_restore;
dbcb5855
DM		 1910 if (snaplen > res)
1911 snaplen = res;
1da177e4
LT		 1912
1913 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1914 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1915 po->tp_reserve;
1da177e4 1916 } else {
95c96174 1917 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1918 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1919 (maclen < 16 ? 16 : maclen)) +
1920 po->tp_reserve;
1da177e4
LT		 1921 macoff = netoff - maclen;
1922 }
f6fb8f10 1923 if (po->tp_version <= TPACKET_V2) {
1924 if (macoff + snaplen > po->rx_ring.frame_size) {
1925 if (po->copy_thresh &&
0fd7bac6 1926 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1927 if (skb_shared(skb)) {
1928 copy_skb = skb_clone(skb, GFP_ATOMIC);
1929 } else {
1930 copy_skb = skb_get(skb);
1931 skb_head = skb->data;
1932 }
1933 if (copy_skb)
1934 skb_set_owner_r(copy_skb, sk);
1da177e4 1935 }
f6fb8f10 1936 snaplen = po->rx_ring.frame_size - macoff;
1937 if ((int)snaplen < 0)
1938 snaplen = 0;
1da177e4 1939 }
dc808110
ED		 1940 } else if (unlikely(macoff + snaplen >
1941 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
1942 u32 nval;
1943
1944 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
1945 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
1946 snaplen, nval, macoff);
1947 snaplen = nval;
1948 if (unlikely((int)snaplen < 0)) {
1949 snaplen = 0;
1950 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
1951 }
1da177e4 1952 }
1da177e4 1953 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1954 h.raw = packet_current_rx_frame(po, skb,
1955 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1956 if (!h.raw)
1da177e4 1957 goto ring_is_full;
f6fb8f10 1958 if (po->tp_version <= TPACKET_V2) {
1959 packet_increment_rx_head(po, &po->rx_ring);
1960 /*
1961 * LOSING will be reported till you read the stats,
1962 * because it's COR - Clear On Read.
1963 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1964 * at packet level.
1965 */
ee80fbf3 1966 if (po->stats.stats1.tp_drops)
f6fb8f10 1967 status |= TP_STATUS_LOSING;
1968 }
ee80fbf3 1969 po->stats.stats1.tp_packets++;
1da177e4
LT		 1970 if (copy_skb) {
1971 status |= TP_STATUS_COPY;
1972 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1973 }
1da177e4
LT		 1974 spin_unlock(&sk->sk_receive_queue.lock);
1975
bbd6ef87 1976 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1977
1978 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1979 getnstimeofday(&ts);
1da177e4 1980
b9c32fb2
DB		 1981 status |= ts_status;
1982
bbd6ef87
PM		 1983 switch (po->tp_version) {
1984 case TPACKET_V1:
1985 h.h1->tp_len = skb->len;
1986 h.h1->tp_snaplen = snaplen;
1987 h.h1->tp_mac = macoff;
1988 h.h1->tp_net = netoff;
4b457bdf
DB		 1989 h.h1->tp_sec = ts.tv_sec;
1990 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1991 hdrlen = sizeof(*h.h1);
1992 break;
1993 case TPACKET_V2:
1994 h.h2->tp_len = skb->len;
1995 h.h2->tp_snaplen = snaplen;
1996 h.h2->tp_mac = macoff;
1997 h.h2->tp_net = netoff;
bbd6ef87
PM		 1998 h.h2->tp_sec = ts.tv_sec;
1999 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 2000 if (vlan_tx_tag_present(skb)) {
2001 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 2002 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2003 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2004 } else {
2005 h.h2->tp_vlan_tci = 0;
a0cdfcf3 2006 h.h2->tp_vlan_tpid = 0;
a3bcc23e 2007 }
e4d26f4b 2008 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 2009 hdrlen = sizeof(*h.h2);
2010 break;
f6fb8f10 2011 case TPACKET_V3:
2012 /* tp_nxt_offset,vlan are already populated above.
2013 * So DONT clear those fields here
2014 */
2015 h.h3->tp_status |= status;
2016 h.h3->tp_len = skb->len;
2017 h.h3->tp_snaplen = snaplen;
2018 h.h3->tp_mac = macoff;
2019 h.h3->tp_net = netoff;
f6fb8f10 2020 h.h3->tp_sec = ts.tv_sec;
2021 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2022 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2023 hdrlen = sizeof(*h.h3);
2024 break;
bbd6ef87
PM		 2025 default:
2026 BUG();
2027 }
1da177e4 2028
bbd6ef87 2029 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2030 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2031 sll->sll_family = AF_PACKET;
2032 sll->sll_hatype = dev->type;
2033 sll->sll_protocol = skb->protocol;
2034 sll->sll_pkttype = skb->pkt_type;
8032b464 2035 if (unlikely(po->origdev))
80feaacb
PWJ		 2036 sll->sll_ifindex = orig_dev->ifindex;
2037 else
2038 sll->sll_ifindex = dev->ifindex;
1da177e4 2039
e16aa207 2040 smp_mb();
f0d4eb29 2041
f6dafa95 2042#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2043 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2044 u8 *start, *end;
2045
f0d4eb29
DB		 2046 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2047 macoff + snaplen);
2048
2049 for (start = h.raw; start < end; start += PAGE_SIZE)
2050 flush_dcache_page(pgv_to_page(start));
1da177e4 2051 }
f0d4eb29 2052 smp_wmb();
f6dafa95 2053#endif
f0d4eb29 2054
f6fb8f10 2055 if (po->tp_version <= TPACKET_V2)
2056 __packet_set_status(po, h.raw, status);
2057 else
2058 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4 2059
676d2369 2060 sk->sk_data_ready(sk);
1da177e4
LT		 2061
2062drop_n_restore:
2063 if (skb_head != skb->data && skb_shared(skb)) {
2064 skb->data = skb_head;
2065 skb->len = skb_len;
2066 }
2067drop:
1ce4f28b 2068 kfree_skb(skb);
1da177e4
LT		 2069 return 0;
2070
2071ring_is_full:
ee80fbf3 2072 po->stats.stats1.tp_drops++;
1da177e4
LT		 2073 spin_unlock(&sk->sk_receive_queue.lock);
2074
676d2369 2075 sk->sk_data_ready(sk);
acb5d75b 2076 kfree_skb(copy_skb);
1da177e4
LT		 2077 goto drop_n_restore;
2078}
2079
69e3c75f
JB		 2080static void tpacket_destruct_skb(struct sk_buff *skb)
2081{
2082 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2083
69e3c75f 2084 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2085 void *ph;
b9c32fb2
DB		 2086 __u32 ts;
2087
69e3c75f 2088 ph = skb_shinfo(skb)->destructor_arg;
b0138408 2089 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2090
2091 ts = __packet_set_timestamp(po, ph, skb);
2092 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2093 }
2094
2095 sock_wfree(skb);
2096}
2097
9c707762
WB		 2098static bool ll_header_truncated(const struct net_device *dev, int len)
2099{
2100 /* net device doesn't like empty head */
2101 if (unlikely(len <= dev->hard_header_len)) {
2102 net_warn_ratelimited("%s: packet size is too short (%d < %d)\n",
2103 current->comm, len, dev->hard_header_len);
2104 return true;
2105 }
2106
2107 return false;
2108}
2109
40d4e3df
ED		 2110static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2111 void *frame, struct net_device *dev, int size_max,
ae641949 2112 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 2113{
184f489e 2114 union tpacket_uhdr ph;
09effa67 2115 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 2116 struct socket *sock = po->sk.sk_socket;
2117 struct page *page;
2118 void *data;
2119 int err;
2120
2121 ph.raw = frame;
2122
2123 skb->protocol = proto;
2124 skb->dev = dev;
2125 skb->priority = po->sk.sk_priority;
2d37a186 2126 skb->mark = po->sk.sk_mark;
2e31396f 2127 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2128 skb_shinfo(skb)->destructor_arg = ph.raw;
2129
2130 switch (po->tp_version) {
2131 case TPACKET_V2:
2132 tp_len = ph.h2->tp_len;
2133 break;
2134 default:
2135 tp_len = ph.h1->tp_len;
2136 break;
2137 }
09effa67
DM		 2138 if (unlikely(tp_len > size_max)) {
2139 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2140 return -EMSGSIZE;
2141 }
69e3c75f 2142
ae641949 2143 skb_reserve(skb, hlen);
69e3c75f 2144 skb_reset_network_header(skb);
c1aad275 2145
d346a3fa
DB		 2146 if (!packet_use_direct_xmit(po))
2147 skb_probe_transport_header(skb, 0);
2148 if (unlikely(po->tp_tx_has_off)) {
5920cd3a
PC		 2149 int off_min, off_max, off;
2150 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2151 off_max = po->tx_ring.frame_size - tp_len;
2152 if (sock->type == SOCK_DGRAM) {
2153 switch (po->tp_version) {
2154 case TPACKET_V2:
2155 off = ph.h2->tp_net;
2156 break;
2157 default:
2158 off = ph.h1->tp_net;
2159 break;
2160 }
2161 } else {
2162 switch (po->tp_version) {
2163 case TPACKET_V2:
2164 off = ph.h2->tp_mac;
2165 break;
2166 default:
2167 off = ph.h1->tp_mac;
2168 break;
2169 }
2170 }
2171 if (unlikely((off < off_min) || (off_max < off)))
2172 return -EINVAL;
2173 data = ph.raw + off;
2174 } else {
2175 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2176 }
69e3c75f
JB		 2177 to_write = tp_len;
2178
2179 if (sock->type == SOCK_DGRAM) {
2180 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2181 NULL, tp_len);
2182 if (unlikely(err < 0))
2183 return -EINVAL;
40d4e3df 2184 } else if (dev->hard_header_len) {
9c707762 2185 if (ll_header_truncated(dev, tp_len))
69e3c75f 2186 return -EINVAL;
69e3c75f
JB		 2187
2188 skb_push(skb, dev->hard_header_len);
2189 err = skb_store_bits(skb, 0, data,
2190 dev->hard_header_len);
2191 if (unlikely(err))
2192 return err;
2193
2194 data += dev->hard_header_len;
2195 to_write -= dev->hard_header_len;
2196 }
2197
69e3c75f
JB		 2198 offset = offset_in_page(data);
2199 len_max = PAGE_SIZE - offset;
2200 len = ((to_write > len_max) ? len_max : to_write);
2201
2202 skb->data_len = to_write;
2203 skb->len += to_write;
2204 skb->truesize += to_write;
2205 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2206
2207 while (likely(to_write)) {
2208 nr_frags = skb_shinfo(skb)->nr_frags;
2209
2210 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2211 pr_err("Packet exceed the number of skb frags(%lu)\n",
2212 MAX_SKB_FRAGS);
69e3c75f
JB		 2213 return -EFAULT;
2214 }
2215
0af55bb5
CG		 2216 page = pgv_to_page(data);
2217 data += len;
69e3c75f
JB		 2218 flush_dcache_page(page);
2219 get_page(page);
0af55bb5 2220 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2221 to_write -= len;
2222 offset = 0;
2223 len_max = PAGE_SIZE;
2224 len = ((to_write > len_max) ? len_max : to_write);
2225 }
2226
2227 return tp_len;
2228}
2229
2230static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2231{
69e3c75f
JB		 2232 struct sk_buff *skb;
2233 struct net_device *dev;
2234 __be16 proto;
09effa67 2235 int err, reserve = 0;
40d4e3df 2236 void *ph;
342dfc30 2237 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2238 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
69e3c75f
JB		 2239 int tp_len, size_max;
2240 unsigned char *addr;
2241 int len_sum = 0;
9e67030a 2242 int status = TP_STATUS_AVAILABLE;
ae641949 2243 int hlen, tlen;
69e3c75f 2244
69e3c75f
JB		 2245 mutex_lock(&po->pg_vec_lock);
2246
66e56cd4 2247 if (likely(saddr == NULL)) {
e40526cb 2248 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2249 proto = po->num;
2250 addr = NULL;
2251 } else {
2252 err = -EINVAL;
2253 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2254 goto out;
2255 if (msg->msg_namelen < (saddr->sll_halen
2256 + offsetof(struct sockaddr_ll,
2257 sll_addr)))
2258 goto out;
69e3c75f
JB		 2259 proto = saddr->sll_protocol;
2260 addr = saddr->sll_addr;
827d9780 2261 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2262 }
2263
69e3c75f
JB		 2264 err = -ENXIO;
2265 if (unlikely(dev == NULL))
2266 goto out;
69e3c75f
JB		 2267 err = -ENETDOWN;
2268 if (unlikely(!(dev->flags & IFF_UP)))
2269 goto out_put;
2270
52f1454f 2271 reserve = dev->hard_header_len + VLAN_HLEN;
69e3c75f 2272 size_max = po->tx_ring.frame_size
b5dd884e 2273 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2274
09effa67
DM		 2275 if (size_max > dev->mtu + reserve)
2276 size_max = dev->mtu + reserve;
2277
69e3c75f
JB		 2278 do {
2279 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2280 TP_STATUS_SEND_REQUEST);
69e3c75f 2281 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2282 if (need_wait && need_resched())
2283 schedule();
69e3c75f
JB		 2284 continue;
2285 }
2286
2287 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2288 hlen = LL_RESERVED_SPACE(dev);
2289 tlen = dev->needed_tailroom;
69e3c75f 2290 skb = sock_alloc_send_skb(&po->sk,
ae641949 2291 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2292 0, &err);
2293
2294 if (unlikely(skb == NULL))
2295 goto out_status;
2296
2297 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
52f1454f
DB		 2298 addr, hlen);
2299 if (tp_len > dev->mtu + dev->hard_header_len) {
2300 struct ethhdr *ehdr;
2301 /* Earlier code assumed this would be a VLAN pkt,
2302 * double-check this now that we have the actual
2303 * packet in hand.
2304 */
69e3c75f 2305
52f1454f
DB		 2306 skb_reset_mac_header(skb);
2307 ehdr = eth_hdr(skb);
2308 if (ehdr->h_proto != htons(ETH_P_8021Q))
2309 tp_len = -EMSGSIZE;
2310 }
69e3c75f
JB		 2311 if (unlikely(tp_len < 0)) {
2312 if (po->tp_loss) {
2313 __packet_set_status(po, ph,
2314 TP_STATUS_AVAILABLE);
2315 packet_increment_head(&po->tx_ring);
2316 kfree_skb(skb);
2317 continue;
2318 } else {
2319 status = TP_STATUS_WRONG_FORMAT;
2320 err = tp_len;
2321 goto out_status;
2322 }
2323 }
2324
0fd5d57b
DB		 2325 packet_pick_tx_queue(dev, skb);
2326
69e3c75f
JB		 2327 skb->destructor = tpacket_destruct_skb;
2328 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2329 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2330
2331 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2332 err = po->xmit(skb);
eb70df13
JP		 2333 if (unlikely(err > 0)) {
2334 err = net_xmit_errno(err);
2335 if (err && __packet_get_status(po, ph) ==
2336 TP_STATUS_AVAILABLE) {
2337 /* skb was destructed already */
2338 skb = NULL;
2339 goto out_status;
2340 }
2341 /*
2342 * skb was dropped but not destructed yet;
2343 * let's treat it like congestion or err < 0
2344 */
2345 err = 0;
2346 }
69e3c75f
JB		 2347 packet_increment_head(&po->tx_ring);
2348 len_sum += tp_len;
b0138408
DB		 2349 } while (likely((ph != NULL) ||
2350 /* Note: packet_read_pending() might be slow if we have
2351 * to call it as it's per_cpu variable, but in fast-path
2352 * we already short-circuit the loop with the first
2353 * condition, and luckily don't have to go that path
2354 * anyway.
2355 */
2356 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2357
2358 err = len_sum;
2359 goto out_put;
2360
69e3c75f
JB		 2361out_status:
2362 __packet_set_status(po, ph, status);
2363 kfree_skb(skb);
2364out_put:
e40526cb 2365 dev_put(dev);
69e3c75f
JB		 2366out:
2367 mutex_unlock(&po->pg_vec_lock);
2368 return err;
2369}
69e3c75f 2370
eea49cc9
OJ		 2371static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2372 size_t reserve, size_t len,
2373 size_t linear, int noblock,
2374 int *err)
bfd5f4a3
SS		 2375{
2376 struct sk_buff *skb;
2377
2378 /* Under a page? Don't bother with paged skb. */
2379 if (prepad + len < PAGE_SIZE || !linear)
2380 linear = len;
2381
2382 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2383 err, 0);
bfd5f4a3
SS		 2384 if (!skb)
2385 return NULL;
2386
2387 skb_reserve(skb, reserve);
2388 skb_put(skb, linear);
2389 skb->data_len = len - linear;
2390 skb->len += len - linear;
2391
2392 return skb;
2393}
2394
d346a3fa 2395static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2396{
2397 struct sock *sk = sock->sk;
342dfc30 2398 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2399 struct sk_buff *skb;
2400 struct net_device *dev;
0e11c91e 2401 __be16 proto;
1da177e4 2402 unsigned char *addr;
827d9780 2403 int err, reserve = 0;
bfd5f4a3
SS		 2404 struct virtio_net_hdr vnet_hdr = { 0 };
2405 int offset = 0;
2406 int vnet_hdr_len;
2407 struct packet_sock *po = pkt_sk(sk);
2408 unsigned short gso_type = 0;
ae641949 2409 int hlen, tlen;
3bdc0eba 2410 int extra_len = 0;
8feb2fb2
AV		 2411 struct iov_iter from;
2412 ssize_t n;
2413
2414 iov_iter_init(&from, WRITE, msg->msg_iov, msg->msg_iovlen, len);
1da177e4
LT		 2415
2416 /*
1ce4f28b 2417 * Get and verify the address.
1da177e4 2418 */
1ce4f28b 2419
66e56cd4 2420 if (likely(saddr == NULL)) {
e40526cb 2421 dev = packet_cached_dev_get(po);
1da177e4
LT		 2422 proto = po->num;
2423 addr = NULL;
2424 } else {
2425 err = -EINVAL;
2426 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2427 goto out;
0fb375fb
EB		 2428 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2429 goto out;
1da177e4
LT		 2430 proto = saddr->sll_protocol;
2431 addr = saddr->sll_addr;
827d9780 2432 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2433 }
2434
1da177e4 2435 err = -ENXIO;
e40526cb 2436 if (unlikely(dev == NULL))
1da177e4 2437 goto out_unlock;
d5e76b0a 2438 err = -ENETDOWN;
e40526cb 2439 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2440 goto out_unlock;
2441
e40526cb
DB		 2442 if (sock->type == SOCK_RAW)
2443 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2444 if (po->has_vnet_hdr) {
2445 vnet_hdr_len = sizeof(vnet_hdr);
2446
2447 err = -EINVAL;
2448 if (len < vnet_hdr_len)
2449 goto out_unlock;
2450
2451 len -= vnet_hdr_len;
2452
8feb2fb2
AV		 2453 err = -EFAULT;
2454 n = copy_from_iter(&vnet_hdr, vnet_hdr_len, &from);
2455 if (n != vnet_hdr_len)
bfd5f4a3
SS		 2456 goto out_unlock;
2457
2458 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2459 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2460 vnet_hdr.hdr_len))
2461 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2462 vnet_hdr.csum_offset + 2;
2463
2464 err = -EINVAL;
2465 if (vnet_hdr.hdr_len > len)
2466 goto out_unlock;
2467
2468 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2469 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2470 case VIRTIO_NET_HDR_GSO_TCPV4:
2471 gso_type = SKB_GSO_TCPV4;
2472 break;
2473 case VIRTIO_NET_HDR_GSO_TCPV6:
2474 gso_type = SKB_GSO_TCPV6;
2475 break;
2476 case VIRTIO_NET_HDR_GSO_UDP:
2477 gso_type = SKB_GSO_UDP;
2478 break;
2479 default:
2480 goto out_unlock;
2481 }
2482
2483 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2484 gso_type |= SKB_GSO_TCP_ECN;
2485
2486 if (vnet_hdr.gso_size == 0)
2487 goto out_unlock;
2488
2489 }
2490 }
2491
3bdc0eba
BG		 2492 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2493 if (!netif_supports_nofcs(dev)) {
2494 err = -EPROTONOSUPPORT;
2495 goto out_unlock;
2496 }
2497 extra_len = 4; /* We're doing our own CRC */
2498 }
2499
1da177e4 2500 err = -EMSGSIZE;
3bdc0eba 2501 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2502 goto out_unlock;
2503
bfd5f4a3 2504 err = -ENOBUFS;
ae641949
HX		 2505 hlen = LL_RESERVED_SPACE(dev);
2506 tlen = dev->needed_tailroom;
2507 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2508 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2509 if (skb == NULL)
1da177e4
LT		 2510 goto out_unlock;
2511
bfd5f4a3 2512 skb_set_network_header(skb, reserve);
1da177e4 2513
0c4e8581 2514 err = -EINVAL;
9c707762
WB		 2515 if (sock->type == SOCK_DGRAM) {
2516 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
2517 if (unlikely(offset) < 0)
2518 goto out_free;
2519 } else {
2520 if (ll_header_truncated(dev, len))
2521 goto out_free;
2522 }
1da177e4
LT		 2523
2524 /* Returns -EFAULT on error */
8feb2fb2 2525 err = skb_copy_datagram_from_iter(skb, offset, &from, len);
1da177e4
LT		 2526 if (err)
2527 goto out_free;
bf84a010
DB		 2528
2529 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2530
3bdc0eba 2531 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2532 /* Earlier code assumed this would be a VLAN pkt,
2533 * double-check this now that we have the actual
2534 * packet in hand.
2535 */
2536 struct ethhdr *ehdr;
2537 skb_reset_mac_header(skb);
2538 ehdr = eth_hdr(skb);
2539 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2540 err = -EMSGSIZE;
2541 goto out_free;
2542 }
57f89bfa
BG		 2543 }
2544
09effa67
DM		 2545 skb->protocol = proto;
2546 skb->dev = dev;
1da177e4 2547 skb->priority = sk->sk_priority;
2d37a186 2548 skb->mark = sk->sk_mark;
0fd5d57b
DB		 2549
2550 packet_pick_tx_queue(dev, skb);
1da177e4 2551
bfd5f4a3
SS		 2552 if (po->has_vnet_hdr) {
2553 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2554 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2555 vnet_hdr.csum_offset)) {
2556 err = -EINVAL;
2557 goto out_free;
2558 }
2559 }
2560
2561 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2562 skb_shinfo(skb)->gso_type = gso_type;
2563
2564 /* Header must be checked, and gso_segs computed. */
2565 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2566 skb_shinfo(skb)->gso_segs = 0;
2567
2568 len += vnet_hdr_len;
2569 }
2570
d346a3fa
DB		 2571 if (!packet_use_direct_xmit(po))
2572 skb_probe_transport_header(skb, reserve);
3bdc0eba
BG		 2573 if (unlikely(extra_len == 4))
2574 skb->no_fcs = 1;
2575
d346a3fa 2576 err = po->xmit(skb);
1da177e4
LT		 2577 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2578 goto out_unlock;
2579
e40526cb 2580 dev_put(dev);
1da177e4 2581
40d4e3df 2582 return len;
1da177e4
LT		 2583
2584out_free:
2585 kfree_skb(skb);
2586out_unlock:
e40526cb 2587 if (dev)
1da177e4
LT		 2588 dev_put(dev);
2589out:
2590 return err;
2591}
2592
69e3c75f
JB		 2593static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2594 struct msghdr *msg, size_t len)
2595{
69e3c75f
JB		 2596 struct sock *sk = sock->sk;
2597 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2598
69e3c75f
JB		 2599 if (po->tx_ring.pg_vec)
2600 return tpacket_snd(po, msg);
2601 else
69e3c75f
JB		 2602 return packet_snd(sock, msg, len);
2603}
2604
1da177e4
LT		 2605/*
2606 * Close a PACKET socket. This is fairly simple. We immediately go
2607 * to 'closed' state and remove our protocol entry in the device list.
2608 */
2609
2610static int packet_release(struct socket *sock)
2611{
2612 struct sock *sk = sock->sk;
2613 struct packet_sock *po;
d12d01d6 2614 struct net *net;
f6fb8f10 2615 union tpacket_req_u req_u;
1da177e4
LT		 2616
2617 if (!sk)
2618 return 0;
2619
3b1e0a65 2620 net = sock_net(sk);
1da177e4
LT		 2621 po = pkt_sk(sk);
2622
0fa7fa98 2623 mutex_lock(&net->packet.sklist_lock);
808f5114 2624 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2625 mutex_unlock(&net->packet.sklist_lock);
2626
2627 preempt_disable();
920de804 2628 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2629 preempt_enable();
1da177e4 2630
808f5114 2631 spin_lock(&po->bind_lock);
ce06b03e 2632 unregister_prot_hook(sk, false);
66e56cd4
DB		 2633 packet_cached_dev_reset(po);
2634
160ff18a
BG		 2635 if (po->prot_hook.dev) {
2636 dev_put(po->prot_hook.dev);
2637 po->prot_hook.dev = NULL;
2638 }
808f5114 2639 spin_unlock(&po->bind_lock);
1da177e4 2640
1da177e4 2641 packet_flush_mclist(sk);
1da177e4 2642
9665d5d6
PS		 2643 if (po->rx_ring.pg_vec) {
2644 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2645 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2646 }
69e3c75f 2647
9665d5d6
PS		 2648 if (po->tx_ring.pg_vec) {
2649 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2650 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2651 }
1da177e4 2652
dc99f600
DM		 2653 fanout_release(sk);
2654
808f5114 2655 synchronize_net();
1da177e4
LT		 2656 /*
2657 * Now the socket is dead. No more input will appear.
2658 */
1da177e4
LT		 2659 sock_orphan(sk);
2660 sock->sk = NULL;
2661
2662 /* Purge queues */
2663
2664 skb_queue_purge(&sk->sk_receive_queue);
b0138408 2665 packet_free_pending(po);
17ab56a2 2666 sk_refcnt_debug_release(sk);
1da177e4
LT		 2667
2668 sock_put(sk);
2669 return 0;
2670}
2671
2672/*
2673 * Attach a packet hook.
2674 */
2675
902fefb8 2676static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
1da177e4
LT		 2677{
2678 struct packet_sock *po = pkt_sk(sk);
902fefb8
DB		 2679 const struct net_device *dev_curr;
2680 __be16 proto_curr;
2681 bool need_rehook;
dc99f600 2682
aef950b4
WY		 2683 if (po->fanout) {
2684 if (dev)
2685 dev_put(dev);
2686
dc99f600 2687 return -EINVAL;
aef950b4 2688 }
1da177e4
LT		 2689
2690 lock_sock(sk);
1da177e4 2691 spin_lock(&po->bind_lock);
66e56cd4 2692
902fefb8
DB		 2693 proto_curr = po->prot_hook.type;
2694 dev_curr = po->prot_hook.dev;
2695
2696 need_rehook = proto_curr != proto || dev_curr != dev;
2697
2698 if (need_rehook) {
2699 unregister_prot_hook(sk, true);
1da177e4 2700
902fefb8
DB		 2701 po->num = proto;
2702 po->prot_hook.type = proto;
1da177e4 2703
902fefb8
DB		 2704 if (po->prot_hook.dev)
2705 dev_put(po->prot_hook.dev);
2706
2707 po->prot_hook.dev = dev;
2708
2709 po->ifindex = dev ? dev->ifindex : 0;
2710 packet_cached_dev_assign(po, dev);
2711 }
66e56cd4 2712
902fefb8 2713 if (proto == 0 || !need_rehook)
1da177e4
LT		 2714 goto out_unlock;
2715
be85d4ad 2716 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2717 register_prot_hook(sk);
be85d4ad
UT		 2718 } else {
2719 sk->sk_err = ENETDOWN;
2720 if (!sock_flag(sk, SOCK_DEAD))
2721 sk->sk_error_report(sk);
1da177e4
LT		 2722 }
2723
2724out_unlock:
2725 spin_unlock(&po->bind_lock);
2726 release_sock(sk);
2727 return 0;
2728}
2729
2730/*
2731 * Bind a packet socket to a device
2732 */
2733
40d4e3df
ED		 2734static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2735 int addr_len)
1da177e4 2736{
40d4e3df 2737 struct sock *sk = sock->sk;
1da177e4
LT		 2738 char name[15];
2739 struct net_device *dev;
2740 int err = -ENODEV;
1ce4f28b 2741
1da177e4
LT		 2742 /*
2743 * Check legality
2744 */
1ce4f28b 2745
8ae55f04 2746 if (addr_len != sizeof(struct sockaddr))
1da177e4 2747 return -EINVAL;
40d4e3df 2748 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2749
3b1e0a65 2750 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2751 if (dev)
1da177e4 2752 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2753 return err;
2754}
1da177e4
LT		 2755
2756static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2757{
40d4e3df
ED		 2758 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2759 struct sock *sk = sock->sk;
1da177e4
LT		 2760 struct net_device *dev = NULL;
2761 int err;
2762
2763
2764 /*
2765 * Check legality
2766 */
1ce4f28b 2767
1da177e4
LT		 2768 if (addr_len < sizeof(struct sockaddr_ll))
2769 return -EINVAL;
2770 if (sll->sll_family != AF_PACKET)
2771 return -EINVAL;
2772
2773 if (sll->sll_ifindex) {
2774 err = -ENODEV;
3b1e0a65 2775 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2776 if (dev == NULL)
2777 goto out;
2778 }
2779 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2780
2781out:
2782 return err;
2783}
2784
2785static struct proto packet_proto = {
2786 .name = "PACKET",
2787 .owner = THIS_MODULE,
2788 .obj_size = sizeof(struct packet_sock),
2789};
2790
2791/*
1ce4f28b 2792 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2793 */
2794
3f378b68
EP		 2795static int packet_create(struct net *net, struct socket *sock, int protocol,
2796 int kern)
1da177e4
LT		 2797{
2798 struct sock *sk;
2799 struct packet_sock *po;
0e11c91e 2800 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2801 int err;
2802
df008c91 2803 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2804 return -EPERM;
be02097c
DM		 2805 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2806 sock->type != SOCK_PACKET)
1da177e4
LT		 2807 return -ESOCKTNOSUPPORT;
2808
2809 sock->state = SS_UNCONNECTED;
2810
2811 err = -ENOBUFS;
6257ff21 2812 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2813 if (sk == NULL)
2814 goto out;
2815
2816 sock->ops = &packet_ops;
1da177e4
LT		 2817 if (sock->type == SOCK_PACKET)
2818 sock->ops = &packet_ops_spkt;
be02097c 2819
1da177e4
LT		 2820 sock_init_data(sock, sk);
2821
2822 po = pkt_sk(sk);
2823 sk->sk_family = PF_PACKET;
0e11c91e 2824 po->num = proto;
d346a3fa 2825 po->xmit = dev_queue_xmit;
66e56cd4 2826
b0138408
DB		 2827 err = packet_alloc_pending(po);
2828 if (err)
2829 goto out2;
2830
66e56cd4 2831 packet_cached_dev_reset(po);
1da177e4
LT		 2832
2833 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2834 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2835
2836 /*
2837 * Attach a protocol block
2838 */
2839
2840 spin_lock_init(&po->bind_lock);
905db440 2841 mutex_init(&po->pg_vec_lock);
1da177e4 2842 po->prot_hook.func = packet_rcv;
be02097c 2843
1da177e4
LT		 2844 if (sock->type == SOCK_PACKET)
2845 po->prot_hook.func = packet_rcv_spkt;
be02097c 2846
1da177e4
LT		 2847 po->prot_hook.af_packet_priv = sk;
2848
0e11c91e
AV		 2849 if (proto) {
2850 po->prot_hook.type = proto;
ce06b03e 2851 register_prot_hook(sk);
1da177e4
LT		 2852 }
2853
0fa7fa98 2854 mutex_lock(&net->packet.sklist_lock);
808f5114 2855 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2856 mutex_unlock(&net->packet.sklist_lock);
2857
2858 preempt_disable();
3680453c 2859 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2860 preempt_enable();
808f5114 2861
40d4e3df 2862 return 0;
b0138408
DB		 2863out2:
2864 sk_free(sk);
1da177e4
LT		 2865out:
2866 return err;
2867}
2868
2869/*
2870 * Pull a packet from our receive queue and hand it to the user.
2871 * If necessary we block.
2872 */
2873
2874static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2875 struct msghdr *msg, size_t len, int flags)
2876{
2877 struct sock *sk = sock->sk;
2878 struct sk_buff *skb;
2879 int copied, err;
bfd5f4a3 2880 int vnet_hdr_len = 0;
1da177e4
LT		 2881
2882 err = -EINVAL;
ed85b565 2883 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2884 goto out;
2885
2886#if 0
2887 /* What error should we return now? EUNATTACH? */
2888 if (pkt_sk(sk)->ifindex < 0)
2889 return -ENODEV;
2890#endif
2891
ed85b565 2892 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2893 err = sock_recv_errqueue(sk, msg, len,
2894 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2895 goto out;
2896 }
2897
1da177e4
LT		 2898 /*
2899 * Call the generic datagram receiver. This handles all sorts
2900 * of horrible races and re-entrancy so we can forget about it
2901 * in the protocol layers.
2902 *
2903 * Now it will return ENETDOWN, if device have just gone down,
2904 * but then it will block.
2905 */
2906
40d4e3df 2907 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2908
2909 /*
1ce4f28b 2910 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2911 * handles the blocking we don't see and worry about blocking
2912 * retries.
2913 */
2914
8ae55f04 2915 if (skb == NULL)
1da177e4
LT		 2916 goto out;
2917
bfd5f4a3
SS		 2918 if (pkt_sk(sk)->has_vnet_hdr) {
2919 struct virtio_net_hdr vnet_hdr = { 0 };
2920
2921 err = -EINVAL;
2922 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2923 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2924 goto out_free;
2925
1f18b717
MK		 2926 len -= vnet_hdr_len;
2927
bfd5f4a3
SS		 2928 if (skb_is_gso(skb)) {
2929 struct skb_shared_info *sinfo = skb_shinfo(skb);
2930
2931 /* This is a hint as to how much should be linear. */
2932 vnet_hdr.hdr_len = skb_headlen(skb);
2933 vnet_hdr.gso_size = sinfo->gso_size;
2934 if (sinfo->gso_type & SKB_GSO_TCPV4)
2935 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2936 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2937 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2938 else if (sinfo->gso_type & SKB_GSO_UDP)
2939 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2940 else if (sinfo->gso_type & SKB_GSO_FCOE)
2941 goto out_free;
2942 else
2943 BUG();
2944 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2945 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2946 } else
2947 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2948
2949 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2950 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2951 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2952 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2953 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2954 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2955 } /* else everything is zero */
2956
7eab8d9e 2957 err = memcpy_to_msg(msg, (void *)&vnet_hdr, vnet_hdr_len);
bfd5f4a3
SS		 2958 if (err < 0)
2959 goto out_free;
2960 }
2961
f3d33426
HFS		 2962 /* You lose any data beyond the buffer you gave. If it worries
2963 * a user program they can ask the device for its MTU
2964 * anyway.
1da177e4 2965 */
1da177e4 2966 copied = skb->len;
40d4e3df
ED		 2967 if (copied > len) {
2968 copied = len;
2969 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2970 }
2971
51f3d02b 2972 err = skb_copy_datagram_msg(skb, 0, msg, copied);
1da177e4
LT		 2973 if (err)
2974 goto out_free;
2975
3b885787 2976 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2977
f3d33426
HFS		 2978 if (msg->msg_name) {
2979 /* If the address length field is there to be filled
2980 * in, we fill it in now.
2981 */
2982 if (sock->type == SOCK_PACKET) {
342dfc30 2983 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426
HFS		 2984 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2985 } else {
2986 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2987 msg->msg_namelen = sll->sll_halen +
2988 offsetof(struct sockaddr_ll, sll_addr);
2989 }
ffbc6111
HX		 2990 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2991 msg->msg_namelen);
f3d33426 2992 }
1da177e4 2993
8dc41944 2994 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2995 struct tpacket_auxdata aux;
2996
2997 aux.tp_status = TP_STATUS_USER;
2998 if (skb->ip_summed == CHECKSUM_PARTIAL)
2999 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
3000 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
3001 aux.tp_snaplen = skb->len;
3002 aux.tp_mac = 0;
bbe735e4 3003 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 3004 if (vlan_tx_tag_present(skb)) {
3005 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 3006 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3007 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 3008 } else {
3009 aux.tp_vlan_tci = 0;
a0cdfcf3 3010 aux.tp_vlan_tpid = 0;
a3bcc23e 3011 }
ffbc6111 3012 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 3013 }
3014
1da177e4
LT		 3015 /*
3016 * Free or return the buffer as appropriate. Again this
3017 * hides all the races and re-entrancy issues from us.
3018 */
bfd5f4a3 3019 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 3020
3021out_free:
3022 skb_free_datagram(sk, skb);
3023out:
3024 return err;
3025}
3026
1da177e4
LT		 3027static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3028 int *uaddr_len, int peer)
3029{
3030 struct net_device *dev;
3031 struct sock *sk = sock->sk;
3032
3033 if (peer)
3034 return -EOPNOTSUPP;
3035
3036 uaddr->sa_family = AF_PACKET;
2dc85bf3 3037 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 3038 rcu_read_lock();
3039 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3040 if (dev)
2dc85bf3 3041 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 3042 rcu_read_unlock();
1da177e4
LT		 3043 *uaddr_len = sizeof(*uaddr);
3044
3045 return 0;
3046}
1da177e4
LT		 3047
3048static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3049 int *uaddr_len, int peer)
3050{
3051 struct net_device *dev;
3052 struct sock *sk = sock->sk;
3053 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3054 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3055
3056 if (peer)
3057 return -EOPNOTSUPP;
3058
3059 sll->sll_family = AF_PACKET;
3060 sll->sll_ifindex = po->ifindex;
3061 sll->sll_protocol = po->num;
67286640 3062 sll->sll_pkttype = 0;
654d1f8a
ED		 3063 rcu_read_lock();
3064 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3065 if (dev) {
3066 sll->sll_hatype = dev->type;
3067 sll->sll_halen = dev->addr_len;
3068 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3069 } else {
3070 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3071 sll->sll_halen = 0;
3072 }
654d1f8a 3073 rcu_read_unlock();
0fb375fb 3074 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3075
3076 return 0;
3077}
3078
2aeb0b88
WC		 3079static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3080 int what)
1da177e4
LT		 3081{
3082 switch (i->type) {
3083 case PACKET_MR_MULTICAST:
1162563f
JP		 3084 if (i->alen != dev->addr_len)
3085 return -EINVAL;
1da177e4 3086 if (what > 0)
22bedad3 3087 return dev_mc_add(dev, i->addr);
1da177e4 3088 else
22bedad3 3089 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3090 break;
3091 case PACKET_MR_PROMISC:
2aeb0b88 3092 return dev_set_promiscuity(dev, what);
1da177e4 3093 case PACKET_MR_ALLMULTI:
2aeb0b88 3094 return dev_set_allmulti(dev, what);
d95ed927 3095 case PACKET_MR_UNICAST:
1162563f
JP		 3096 if (i->alen != dev->addr_len)
3097 return -EINVAL;
d95ed927 3098 if (what > 0)
a748ee24 3099 return dev_uc_add(dev, i->addr);
d95ed927 3100 else
a748ee24 3101 return dev_uc_del(dev, i->addr);
d95ed927 3102 break;
40d4e3df
ED		 3103 default:
3104 break;
1da177e4 3105 }
2aeb0b88 3106 return 0;
1da177e4
LT		 3107}
3108
3109static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
3110{
40d4e3df 3111 for ( ; i; i = i->next) {
1da177e4
LT		 3112 if (i->ifindex == dev->ifindex)
3113 packet_dev_mc(dev, i, what);
3114 }
3115}
3116
0fb375fb 3117static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3118{
3119 struct packet_sock *po = pkt_sk(sk);
3120 struct packet_mclist *ml, *i;
3121 struct net_device *dev;
3122 int err;
3123
3124 rtnl_lock();
3125
3126 err = -ENODEV;
3b1e0a65 3127 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3128 if (!dev)
3129 goto done;
3130
3131 err = -EINVAL;
1162563f 3132 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3133 goto done;
3134
3135 err = -ENOBUFS;
8b3a7005 3136 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3137 if (i == NULL)
3138 goto done;
3139
3140 err = 0;
3141 for (ml = po->mclist; ml; ml = ml->next) {
3142 if (ml->ifindex == mreq->mr_ifindex &&
3143 ml->type == mreq->mr_type &&
3144 ml->alen == mreq->mr_alen &&
3145 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3146 ml->count++;
3147 /* Free the new element ... */
3148 kfree(i);
3149 goto done;
3150 }
3151 }
3152
3153 i->type = mreq->mr_type;
3154 i->ifindex = mreq->mr_ifindex;
3155 i->alen = mreq->mr_alen;
3156 memcpy(i->addr, mreq->mr_address, i->alen);
3157 i->count = 1;
3158 i->next = po->mclist;
3159 po->mclist = i;
2aeb0b88
WC		 3160 err = packet_dev_mc(dev, i, 1);
3161 if (err) {
3162 po->mclist = i->next;
3163 kfree(i);
3164 }
1da177e4
LT		 3165
3166done:
3167 rtnl_unlock();
3168 return err;
3169}
3170
0fb375fb 3171static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3172{
3173 struct packet_mclist *ml, **mlp;
3174
3175 rtnl_lock();
3176
3177 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3178 if (ml->ifindex == mreq->mr_ifindex &&
3179 ml->type == mreq->mr_type &&
3180 ml->alen == mreq->mr_alen &&
3181 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3182 if (--ml->count == 0) {
3183 struct net_device *dev;
3184 *mlp = ml->next;
ad959e76
ED		 3185 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3186 if (dev)
1da177e4 3187 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3188 kfree(ml);
3189 }
3190 rtnl_unlock();
3191 return 0;
3192 }
3193 }
3194 rtnl_unlock();
3195 return -EADDRNOTAVAIL;
3196}
3197
3198static void packet_flush_mclist(struct sock *sk)
3199{
3200 struct packet_sock *po = pkt_sk(sk);
3201 struct packet_mclist *ml;
3202
3203 if (!po->mclist)
3204 return;
3205
3206 rtnl_lock();
3207 while ((ml = po->mclist) != NULL) {
3208 struct net_device *dev;
3209
3210 po->mclist = ml->next;
ad959e76
ED		 3211 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3212 if (dev != NULL)
1da177e4 3213 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3214 kfree(ml);
3215 }
3216 rtnl_unlock();
3217}
1da177e4
LT		 3218
3219static int
b7058842 3220packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3221{
3222 struct sock *sk = sock->sk;
8dc41944 3223 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3224 int ret;
3225
3226 if (level != SOL_PACKET)
3227 return -ENOPROTOOPT;
3228
69e3c75f 3229 switch (optname) {
1ce4f28b 3230 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3231 case PACKET_DROP_MEMBERSHIP:
3232 {
0fb375fb
EB		 3233 struct packet_mreq_max mreq;
3234 int len = optlen;
3235 memset(&mreq, 0, sizeof(mreq));
3236 if (len < sizeof(struct packet_mreq))
1da177e4 3237 return -EINVAL;
0fb375fb
EB		 3238 if (len > sizeof(mreq))
3239 len = sizeof(mreq);
40d4e3df 3240 if (copy_from_user(&mreq, optval, len))
1da177e4 3241 return -EFAULT;
0fb375fb
EB		 3242 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3243 return -EINVAL;
1da177e4
LT		 3244 if (optname == PACKET_ADD_MEMBERSHIP)
3245 ret = packet_mc_add(sk, &mreq);
3246 else
3247 ret = packet_mc_drop(sk, &mreq);
3248 return ret;
3249 }
a2efcfa0 3250
1da177e4 3251 case PACKET_RX_RING:
69e3c75f 3252 case PACKET_TX_RING:
1da177e4 3253 {
f6fb8f10 3254 union tpacket_req_u req_u;
3255 int len;
1da177e4 3256
f6fb8f10 3257 switch (po->tp_version) {
3258 case TPACKET_V1:
3259 case TPACKET_V2:
3260 len = sizeof(req_u.req);
3261 break;
3262 case TPACKET_V3:
3263 default:
3264 len = sizeof(req_u.req3);
3265 break;
3266 }
3267 if (optlen < len)
1da177e4 3268 return -EINVAL;
bfd5f4a3
SS		 3269 if (pkt_sk(sk)->has_vnet_hdr)
3270 return -EINVAL;
f6fb8f10 3271 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3272 return -EFAULT;
f6fb8f10 3273 return packet_set_ring(sk, &req_u, 0,
3274 optname == PACKET_TX_RING);
1da177e4
LT		 3275 }
3276 case PACKET_COPY_THRESH:
3277 {
3278 int val;
3279
40d4e3df 3280 if (optlen != sizeof(val))
1da177e4 3281 return -EINVAL;
40d4e3df 3282 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3283 return -EFAULT;
3284
3285 pkt_sk(sk)->copy_thresh = val;
3286 return 0;
3287 }
bbd6ef87
PM		 3288 case PACKET_VERSION:
3289 {
3290 int val;
3291
3292 if (optlen != sizeof(val))
3293 return -EINVAL;
69e3c75f 3294 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3295 return -EBUSY;
3296 if (copy_from_user(&val, optval, sizeof(val)))
3297 return -EFAULT;
3298 switch (val) {
3299 case TPACKET_V1:
3300 case TPACKET_V2:
f6fb8f10 3301 case TPACKET_V3:
bbd6ef87
PM		 3302 po->tp_version = val;
3303 return 0;
3304 default:
3305 return -EINVAL;
3306 }
3307 }
8913336a
PM		 3308 case PACKET_RESERVE:
3309 {
3310 unsigned int val;
3311
3312 if (optlen != sizeof(val))
3313 return -EINVAL;
69e3c75f 3314 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3315 return -EBUSY;
3316 if (copy_from_user(&val, optval, sizeof(val)))
3317 return -EFAULT;
3318 po->tp_reserve = val;
3319 return 0;
3320 }
69e3c75f
JB		 3321 case PACKET_LOSS:
3322 {
3323 unsigned int val;
3324
3325 if (optlen != sizeof(val))
3326 return -EINVAL;
3327 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3328 return -EBUSY;
3329 if (copy_from_user(&val, optval, sizeof(val)))
3330 return -EFAULT;
3331 po->tp_loss = !!val;
3332 return 0;
3333 }
8dc41944
HX		 3334 case PACKET_AUXDATA:
3335 {
3336 int val;
3337
3338 if (optlen < sizeof(val))
3339 return -EINVAL;
3340 if (copy_from_user(&val, optval, sizeof(val)))
3341 return -EFAULT;
3342
3343 po->auxdata = !!val;
3344 return 0;
3345 }
80feaacb
PWJ		 3346 case PACKET_ORIGDEV:
3347 {
3348 int val;
3349
3350 if (optlen < sizeof(val))
3351 return -EINVAL;
3352 if (copy_from_user(&val, optval, sizeof(val)))
3353 return -EFAULT;
3354
3355 po->origdev = !!val;
3356 return 0;
3357 }
bfd5f4a3
SS		 3358 case PACKET_VNET_HDR:
3359 {
3360 int val;
3361
3362 if (sock->type != SOCK_RAW)
3363 return -EINVAL;
3364 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3365 return -EBUSY;
3366 if (optlen < sizeof(val))
3367 return -EINVAL;
3368 if (copy_from_user(&val, optval, sizeof(val)))
3369 return -EFAULT;
3370
3371 po->has_vnet_hdr = !!val;
3372 return 0;
3373 }
614f60fa
SM		 3374 case PACKET_TIMESTAMP:
3375 {
3376 int val;
3377
3378 if (optlen != sizeof(val))
3379 return -EINVAL;
3380 if (copy_from_user(&val, optval, sizeof(val)))
3381 return -EFAULT;
3382
3383 po->tp_tstamp = val;
3384 return 0;
3385 }
dc99f600
DM		 3386 case PACKET_FANOUT:
3387 {
3388 int val;
3389
3390 if (optlen != sizeof(val))
3391 return -EINVAL;
3392 if (copy_from_user(&val, optval, sizeof(val)))
3393 return -EFAULT;
3394
3395 return fanout_add(sk, val & 0xffff, val >> 16);
3396 }
5920cd3a
PC		 3397 case PACKET_TX_HAS_OFF:
3398 {
3399 unsigned int val;
3400
3401 if (optlen != sizeof(val))
3402 return -EINVAL;
3403 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3404 return -EBUSY;
3405 if (copy_from_user(&val, optval, sizeof(val)))
3406 return -EFAULT;
3407 po->tp_tx_has_off = !!val;
3408 return 0;
3409 }
d346a3fa
DB		 3410 case PACKET_QDISC_BYPASS:
3411 {
3412 int val;
3413
3414 if (optlen != sizeof(val))
3415 return -EINVAL;
3416 if (copy_from_user(&val, optval, sizeof(val)))
3417 return -EFAULT;
3418
3419 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3420 return 0;
3421 }
1da177e4
LT		 3422 default:
3423 return -ENOPROTOOPT;
3424 }
3425}
3426
3427static int packet_getsockopt(struct socket *sock, int level, int optname,
3428 char __user *optval, int __user *optlen)
3429{
3430 int len;
c06fff6e 3431 int val, lv = sizeof(val);
1da177e4
LT		 3432 struct sock *sk = sock->sk;
3433 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3434 void *data = &val;
ee80fbf3 3435 union tpacket_stats_u st;
1da177e4
LT		 3436
3437 if (level != SOL_PACKET)
3438 return -ENOPROTOOPT;
3439
8ae55f04
KK		 3440 if (get_user(len, optlen))
3441 return -EFAULT;
1da177e4
LT		 3442
3443 if (len < 0)
3444 return -EINVAL;
1ce4f28b 3445
69e3c75f 3446 switch (optname) {
1da177e4 3447 case PACKET_STATISTICS:
1da177e4 3448 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3449 memcpy(&st, &po->stats, sizeof(st));
3450 memset(&po->stats, 0, sizeof(po->stats));
3451 spin_unlock_bh(&sk->sk_receive_queue.lock);
3452
f6fb8f10 3453 if (po->tp_version == TPACKET_V3) {
c06fff6e 3454 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3455 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3456 data = &st.stats3;
f6fb8f10 3457 } else {
c06fff6e 3458 lv = sizeof(struct tpacket_stats);
8bcdeaff 3459 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3460 data = &st.stats1;
f6fb8f10 3461 }
ee80fbf3 3462
8dc41944
HX		 3463 break;
3464 case PACKET_AUXDATA:
8dc41944 3465 val = po->auxdata;
80feaacb
PWJ		 3466 break;
3467 case PACKET_ORIGDEV:
80feaacb 3468 val = po->origdev;
bfd5f4a3
SS		 3469 break;
3470 case PACKET_VNET_HDR:
bfd5f4a3 3471 val = po->has_vnet_hdr;
1da177e4 3472 break;
bbd6ef87 3473 case PACKET_VERSION:
bbd6ef87 3474 val = po->tp_version;
bbd6ef87
PM		 3475 break;
3476 case PACKET_HDRLEN:
3477 if (len > sizeof(int))
3478 len = sizeof(int);
3479 if (copy_from_user(&val, optval, len))
3480 return -EFAULT;
3481 switch (val) {
3482 case TPACKET_V1:
3483 val = sizeof(struct tpacket_hdr);
3484 break;
3485 case TPACKET_V2:
3486 val = sizeof(struct tpacket2_hdr);
3487 break;
f6fb8f10 3488 case TPACKET_V3:
3489 val = sizeof(struct tpacket3_hdr);
3490 break;
bbd6ef87
PM		 3491 default:
3492 return -EINVAL;
3493 }
bbd6ef87 3494 break;
8913336a 3495 case PACKET_RESERVE:
8913336a 3496 val = po->tp_reserve;
8913336a 3497 break;
69e3c75f 3498 case PACKET_LOSS:
69e3c75f 3499 val = po->tp_loss;
69e3c75f 3500 break;
614f60fa 3501 case PACKET_TIMESTAMP:
614f60fa 3502 val = po->tp_tstamp;
614f60fa 3503 break;
dc99f600 3504 case PACKET_FANOUT:
dc99f600
DM		 3505 val = (po->fanout ?
3506 ((u32)po->fanout->id |
77f65ebd
WB		 3507 ((u32)po->fanout->type << 16) |
3508 ((u32)po->fanout->flags << 24)) :
dc99f600 3509 0);
dc99f600 3510 break;
5920cd3a
PC		 3511 case PACKET_TX_HAS_OFF:
3512 val = po->tp_tx_has_off;
3513 break;
d346a3fa
DB		 3514 case PACKET_QDISC_BYPASS:
3515 val = packet_use_direct_xmit(po);
3516 break;
1da177e4
LT		 3517 default:
3518 return -ENOPROTOOPT;
3519 }
3520
c06fff6e
ED		 3521 if (len > lv)
3522 len = lv;
8ae55f04
KK		 3523 if (put_user(len, optlen))
3524 return -EFAULT;
8dc41944
HX		 3525 if (copy_to_user(optval, data, len))
3526 return -EFAULT;
8ae55f04 3527 return 0;
1da177e4
LT		 3528}
3529
3530
351638e7
JP		 3531static int packet_notifier(struct notifier_block *this,
3532 unsigned long msg, void *ptr)
1da177e4
LT		 3533{
3534 struct sock *sk;
351638e7 3535 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3536 struct net *net = dev_net(dev);
1da177e4 3537
808f5114 3538 rcu_read_lock();
b67bfe0d 3539 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3540 struct packet_sock *po = pkt_sk(sk);
3541
3542 switch (msg) {
3543 case NETDEV_UNREGISTER:
1da177e4
LT		 3544 if (po->mclist)
3545 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3546 /* fallthrough */
3547
1da177e4
LT		 3548 case NETDEV_DOWN:
3549 if (dev->ifindex == po->ifindex) {
3550 spin_lock(&po->bind_lock);
3551 if (po->running) {
ce06b03e 3552 __unregister_prot_hook(sk, false);
1da177e4
LT		 3553 sk->sk_err = ENETDOWN;
3554 if (!sock_flag(sk, SOCK_DEAD))
3555 sk->sk_error_report(sk);
3556 }
3557 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3558 packet_cached_dev_reset(po);
1da177e4 3559 po->ifindex = -1;
160ff18a
BG		 3560 if (po->prot_hook.dev)
3561 dev_put(po->prot_hook.dev);
1da177e4
LT		 3562 po->prot_hook.dev = NULL;
3563 }
3564 spin_unlock(&po->bind_lock);
3565 }
3566 break;
3567 case NETDEV_UP:
808f5114 3568 if (dev->ifindex == po->ifindex) {
3569 spin_lock(&po->bind_lock);
ce06b03e
DM		 3570 if (po->num)
3571 register_prot_hook(sk);
808f5114 3572 spin_unlock(&po->bind_lock);
1da177e4 3573 }
1da177e4
LT		 3574 break;
3575 }
3576 }
808f5114 3577 rcu_read_unlock();
1da177e4
LT		 3578 return NOTIFY_DONE;
3579}
3580
3581
3582static int packet_ioctl(struct socket *sock, unsigned int cmd,
3583 unsigned long arg)
3584{
3585 struct sock *sk = sock->sk;
3586
69e3c75f 3587 switch (cmd) {
40d4e3df
ED		 3588 case SIOCOUTQ:
3589 {
3590 int amount = sk_wmem_alloc_get(sk);
31e6d363 3591
40d4e3df
ED		 3592 return put_user(amount, (int __user *)arg);
3593 }
3594 case SIOCINQ:
3595 {
3596 struct sk_buff *skb;
3597 int amount = 0;
3598
3599 spin_lock_bh(&sk->sk_receive_queue.lock);
3600 skb = skb_peek(&sk->sk_receive_queue);
3601 if (skb)
3602 amount = skb->len;
3603 spin_unlock_bh(&sk->sk_receive_queue.lock);
3604 return put_user(amount, (int __user *)arg);
3605 }
3606 case SIOCGSTAMP:
3607 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3608 case SIOCGSTAMPNS:
3609 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3610
1da177e4 3611#ifdef CONFIG_INET
40d4e3df
ED		 3612 case SIOCADDRT:
3613 case SIOCDELRT:
3614 case SIOCDARP:
3615 case SIOCGARP:
3616 case SIOCSARP:
3617 case SIOCGIFADDR:
3618 case SIOCSIFADDR:
3619 case SIOCGIFBRDADDR:
3620 case SIOCSIFBRDADDR:
3621 case SIOCGIFNETMASK:
3622 case SIOCSIFNETMASK:
3623 case SIOCGIFDSTADDR:
3624 case SIOCSIFDSTADDR:
3625 case SIOCSIFFLAGS:
40d4e3df 3626 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3627#endif
3628
40d4e3df
ED		 3629 default:
3630 return -ENOIOCTLCMD;
1da177e4
LT		 3631 }
3632 return 0;
3633}
3634
40d4e3df 3635static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3636 poll_table *wait)
3637{
3638 struct sock *sk = sock->sk;
3639 struct packet_sock *po = pkt_sk(sk);
3640 unsigned int mask = datagram_poll(file, sock, wait);
3641
3642 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3643 if (po->rx_ring.pg_vec) {
f6fb8f10 3644 if (!packet_previous_rx_frame(po, &po->rx_ring,
3645 TP_STATUS_KERNEL))
1da177e4
LT		 3646 mask |= POLLIN | POLLRDNORM;
3647 }
3648 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3649 spin_lock_bh(&sk->sk_write_queue.lock);
3650 if (po->tx_ring.pg_vec) {
3651 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3652 mask |= POLLOUT | POLLWRNORM;
3653 }
3654 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3655 return mask;
3656}
3657
3658
3659/* Dirty? Well, I still did not learn better way to account
3660 * for user mmaps.
3661 */
3662
3663static void packet_mm_open(struct vm_area_struct *vma)
3664{
3665 struct file *file = vma->vm_file;
40d4e3df 3666 struct socket *sock = file->private_data;
1da177e4 3667 struct sock *sk = sock->sk;
1ce4f28b 3668
1da177e4
LT		 3669 if (sk)
3670 atomic_inc(&pkt_sk(sk)->mapped);
3671}
3672
3673static void packet_mm_close(struct vm_area_struct *vma)
3674{
3675 struct file *file = vma->vm_file;
40d4e3df 3676 struct socket *sock = file->private_data;
1da177e4 3677 struct sock *sk = sock->sk;
1ce4f28b 3678
1da177e4
LT		 3679 if (sk)
3680 atomic_dec(&pkt_sk(sk)->mapped);
3681}
3682
f0f37e2f 3683static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3684 .open = packet_mm_open,
3685 .close = packet_mm_close,
1da177e4
LT		 3686};
3687
0e3125c7
NH		 3688static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3689 unsigned int len)
1da177e4
LT		 3690{
3691 int i;
3692
4ebf0ae2 3693 for (i = 0; i < len; i++) {
0e3125c7 3694 if (likely(pg_vec[i].buffer)) {
c56b4d90 3695 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3696 vfree(pg_vec[i].buffer);
3697 else
3698 free_pages((unsigned long)pg_vec[i].buffer,
3699 order);
3700 pg_vec[i].buffer = NULL;
3701 }
1da177e4
LT		 3702 }
3703 kfree(pg_vec);
3704}
3705
eea49cc9 3706static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3707{
f0d4eb29 3708 char *buffer;
0e3125c7
NH		 3709 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3710 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3711
3712 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3713 if (buffer)
3714 return buffer;
3715
f0d4eb29 3716 /* __get_free_pages failed, fall back to vmalloc */
bbce5a59 3717 buffer = vzalloc((1 << order) * PAGE_SIZE);
0e3125c7
NH		 3718 if (buffer)
3719 return buffer;
3720
f0d4eb29 3721 /* vmalloc failed, lets dig into swap here */
0e3125c7 3722 gfp_flags &= ~__GFP_NORETRY;
f0d4eb29 3723 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3724 if (buffer)
3725 return buffer;
3726
f0d4eb29 3727 /* complete and utter failure */
0e3125c7 3728 return NULL;
4ebf0ae2
DM		 3729}
3730
0e3125c7 3731static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3732{
3733 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3734 struct pgv *pg_vec;
4ebf0ae2
DM		 3735 int i;
3736
0e3125c7 3737 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3738 if (unlikely(!pg_vec))
3739 goto out;
3740
3741 for (i = 0; i < block_nr; i++) {
c56b4d90 3742 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3743 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3744 goto out_free_pgvec;
3745 }
3746
3747out:
3748 return pg_vec;
3749
3750out_free_pgvec:
3751 free_pg_vec(pg_vec, order, block_nr);
3752 pg_vec = NULL;
3753 goto out;
3754}
1da177e4 3755
f6fb8f10 3756static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3757 int closing, int tx_ring)
1da177e4 3758{
0e3125c7 3759 struct pgv *pg_vec = NULL;
1da177e4 3760 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3761 int was_running, order = 0;
69e3c75f
JB		 3762 struct packet_ring_buffer *rb;
3763 struct sk_buff_head *rb_queue;
0e11c91e 3764 __be16 num;
f6fb8f10 3765 int err = -EINVAL;
3766 /* Added to avoid minimal code churn */
3767 struct tpacket_req *req = &req_u->req;
3768
3769 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3770 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3771 WARN(1, "Tx-ring is not supported.\n");
3772 goto out;
3773 }
1ce4f28b 3774
69e3c75f
JB		 3775 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3776 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3777
69e3c75f
JB		 3778 err = -EBUSY;
3779 if (!closing) {
3780 if (atomic_read(&po->mapped))
3781 goto out;
b0138408 3782 if (packet_read_pending(rb))
69e3c75f
JB		 3783 goto out;
3784 }
1da177e4 3785
69e3c75f
JB		 3786 if (req->tp_block_nr) {
3787 /* Sanity tests and some calculations */
3788 err = -EBUSY;
3789 if (unlikely(rb->pg_vec))
3790 goto out;
1da177e4 3791
bbd6ef87
PM		 3792 switch (po->tp_version) {
3793 case TPACKET_V1:
3794 po->tp_hdrlen = TPACKET_HDRLEN;
3795 break;
3796 case TPACKET_V2:
3797 po->tp_hdrlen = TPACKET2_HDRLEN;
3798 break;
f6fb8f10 3799 case TPACKET_V3:
3800 po->tp_hdrlen = TPACKET3_HDRLEN;
3801 break;
bbd6ef87
PM		 3802 }
3803
69e3c75f 3804 err = -EINVAL;
4ebf0ae2 3805 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3806 goto out;
4ebf0ae2 3807 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3808 goto out;
dc808110
ED		 3809 if (po->tp_version >= TPACKET_V3 &&
3810 (int)(req->tp_block_size -
3811 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
3812 goto out;
8913336a 3813 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3814 po->tp_reserve))
3815 goto out;
4ebf0ae2 3816 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3817 goto out;
1da177e4 3818
69e3c75f
JB		 3819 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3820 if (unlikely(rb->frames_per_block <= 0))
3821 goto out;
3822 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3823 req->tp_frame_nr))
3824 goto out;
1da177e4
LT		 3825
3826 err = -ENOMEM;
4ebf0ae2
DM		 3827 order = get_order(req->tp_block_size);
3828 pg_vec = alloc_pg_vec(req, order);
3829 if (unlikely(!pg_vec))
1da177e4 3830 goto out;
f6fb8f10 3831 switch (po->tp_version) {
3832 case TPACKET_V3:
3833 /* Transmit path is not supported. We checked
3834 * it above but just being paranoid
3835 */
3836 if (!tx_ring)
3837 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
d7cf0c34 3838 break;
f6fb8f10 3839 default:
3840 break;
3841 }
69e3c75f
JB		 3842 }
3843 /* Done */
3844 else {
3845 err = -EINVAL;
4ebf0ae2 3846 if (unlikely(req->tp_frame_nr))
69e3c75f 3847 goto out;
1da177e4
LT		 3848 }
3849
3850 lock_sock(sk);
3851
3852 /* Detach socket from network */
3853 spin_lock(&po->bind_lock);
3854 was_running = po->running;
3855 num = po->num;
3856 if (was_running) {
1da177e4 3857 po->num = 0;
ce06b03e 3858 __unregister_prot_hook(sk, false);
1da177e4
LT		 3859 }
3860 spin_unlock(&po->bind_lock);
1ce4f28b 3861
1da177e4
LT		 3862 synchronize_net();
3863
3864 err = -EBUSY;
905db440 3865 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3866 if (closing || atomic_read(&po->mapped) == 0) {
3867 err = 0;
69e3c75f 3868 spin_lock_bh(&rb_queue->lock);
c053fd96 3869 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3870 rb->frame_max = (req->tp_frame_nr - 1);
3871 rb->head = 0;
3872 rb->frame_size = req->tp_frame_size;
3873 spin_unlock_bh(&rb_queue->lock);
3874
c053fd96
CG		 3875 swap(rb->pg_vec_order, order);
3876 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3877
3878 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3879 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3880 tpacket_rcv : packet_rcv;
3881 skb_queue_purge(rb_queue);
1da177e4 3882 if (atomic_read(&po->mapped))
40d4e3df
ED		 3883 pr_err("packet_mmap: vma is busy: %d\n",
3884 atomic_read(&po->mapped));
1da177e4 3885 }
905db440 3886 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3887
3888 spin_lock(&po->bind_lock);
ce06b03e 3889 if (was_running) {
1da177e4 3890 po->num = num;
ce06b03e 3891 register_prot_hook(sk);
1da177e4
LT		 3892 }
3893 spin_unlock(&po->bind_lock);
f6fb8f10 3894 if (closing && (po->tp_version > TPACKET_V2)) {
3895 /* Because we don't support block-based V3 on tx-ring */
3896 if (!tx_ring)
3897 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3898 }
1da177e4
LT		 3899 release_sock(sk);
3900
1da177e4
LT		 3901 if (pg_vec)
3902 free_pg_vec(pg_vec, order, req->tp_block_nr);
3903out:
3904 return err;
3905}
3906
69e3c75f
JB		 3907static int packet_mmap(struct file *file, struct socket *sock,
3908 struct vm_area_struct *vma)
1da177e4
LT		 3909{
3910 struct sock *sk = sock->sk;
3911 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3912 unsigned long size, expected_size;
3913 struct packet_ring_buffer *rb;
1da177e4
LT		 3914 unsigned long start;
3915 int err = -EINVAL;
3916 int i;
3917
3918 if (vma->vm_pgoff)
3919 return -EINVAL;
3920
905db440 3921 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3922
3923 expected_size = 0;
3924 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3925 if (rb->pg_vec) {
3926 expected_size += rb->pg_vec_len
3927 * rb->pg_vec_pages
3928 * PAGE_SIZE;
3929 }
3930 }
3931
3932 if (expected_size == 0)
1da177e4 3933 goto out;
69e3c75f
JB		 3934
3935 size = vma->vm_end - vma->vm_start;
3936 if (size != expected_size)
1da177e4
LT		 3937 goto out;
3938
1da177e4 3939 start = vma->vm_start;
69e3c75f
JB		 3940 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3941 if (rb->pg_vec == NULL)
3942 continue;
3943
3944 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3945 struct page *page;
3946 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3947 int pg_num;
3948
c56b4d90
CG		 3949 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3950 page = pgv_to_page(kaddr);
69e3c75f
JB		 3951 err = vm_insert_page(vma, start, page);
3952 if (unlikely(err))
3953 goto out;
3954 start += PAGE_SIZE;
0e3125c7 3955 kaddr += PAGE_SIZE;
69e3c75f 3956 }
4ebf0ae2 3957 }
1da177e4 3958 }
69e3c75f 3959
4ebf0ae2 3960 atomic_inc(&po->mapped);
1da177e4
LT		 3961 vma->vm_ops = &packet_mmap_ops;
3962 err = 0;
3963
3964out:
905db440 3965 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3966 return err;
3967}
1da177e4 3968
90ddc4f0 3969static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3970 .family = PF_PACKET,
3971 .owner = THIS_MODULE,
3972 .release = packet_release,
3973 .bind = packet_bind_spkt,
3974 .connect = sock_no_connect,
3975 .socketpair = sock_no_socketpair,
3976 .accept = sock_no_accept,
3977 .getname = packet_getname_spkt,
3978 .poll = datagram_poll,
3979 .ioctl = packet_ioctl,
3980 .listen = sock_no_listen,
3981 .shutdown = sock_no_shutdown,
3982 .setsockopt = sock_no_setsockopt,
3983 .getsockopt = sock_no_getsockopt,
3984 .sendmsg = packet_sendmsg_spkt,
3985 .recvmsg = packet_recvmsg,
3986 .mmap = sock_no_mmap,
3987 .sendpage = sock_no_sendpage,
3988};
1da177e4 3989
90ddc4f0 3990static const struct proto_ops packet_ops = {
1da177e4
LT		 3991 .family = PF_PACKET,
3992 .owner = THIS_MODULE,
3993 .release = packet_release,
3994 .bind = packet_bind,
3995 .connect = sock_no_connect,
3996 .socketpair = sock_no_socketpair,
3997 .accept = sock_no_accept,
1ce4f28b 3998 .getname = packet_getname,
1da177e4
LT		 3999 .poll = packet_poll,
4000 .ioctl = packet_ioctl,
4001 .listen = sock_no_listen,
4002 .shutdown = sock_no_shutdown,
4003 .setsockopt = packet_setsockopt,
4004 .getsockopt = packet_getsockopt,
4005 .sendmsg = packet_sendmsg,
4006 .recvmsg = packet_recvmsg,
4007 .mmap = packet_mmap,
4008 .sendpage = sock_no_sendpage,
4009};
4010
ec1b4cf7 4011static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 4012 .family = PF_PACKET,
4013 .create = packet_create,
4014 .owner = THIS_MODULE,
4015};
4016
4017static struct notifier_block packet_netdev_notifier = {
40d4e3df 4018 .notifier_call = packet_notifier,
1da177e4
LT		 4019};
4020
4021#ifdef CONFIG_PROC_FS
1da177e4
LT		 4022
4023static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 4024 __acquires(RCU)
1da177e4 4025{
e372c414 4026 struct net *net = seq_file_net(seq);
808f5114 4027
4028 rcu_read_lock();
4029 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 4030}
4031
4032static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4033{
1bf40954 4034 struct net *net = seq_file_net(seq);
808f5114 4035 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 4036}
4037
4038static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 4039 __releases(RCU)
1da177e4 4040{
808f5114 4041 rcu_read_unlock();
1da177e4
LT		 4042}
4043
1ce4f28b 4044static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 4045{
4046 if (v == SEQ_START_TOKEN)
4047 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4048 else {
b7ceabd9 4049 struct sock *s = sk_entry(v);
1da177e4
LT		 4050 const struct packet_sock *po = pkt_sk(s);
4051
4052 seq_printf(seq,
71338aa7 4053 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 4054 s,
4055 atomic_read(&s->sk_refcnt),
4056 s->sk_type,
4057 ntohs(po->num),
4058 po->ifindex,
4059 po->running,
4060 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4061 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4062 sock_i_ino(s));
1da177e4
LT		 4063 }
4064
4065 return 0;
4066}
4067
56b3d975 4068static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4069 .start = packet_seq_start,
4070 .next = packet_seq_next,
4071 .stop = packet_seq_stop,
4072 .show = packet_seq_show,
4073};
4074
4075static int packet_seq_open(struct inode *inode, struct file *file)
4076{
e372c414
DL		 4077 return seq_open_net(inode, file, &packet_seq_ops,
4078 sizeof(struct seq_net_private));
1da177e4
LT		 4079}
4080
da7071d7 4081static const struct file_operations packet_seq_fops = {
1da177e4
LT		 4082 .owner = THIS_MODULE,
4083 .open = packet_seq_open,
4084 .read = seq_read,
4085 .llseek = seq_lseek,
e372c414 4086 .release = seq_release_net,
1da177e4
LT		 4087};
4088
4089#endif
4090
2c8c1e72 4091static int __net_init packet_net_init(struct net *net)
d12d01d6 4092{
0fa7fa98 4093 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4094 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4095
d4beaa66 4096 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 4097 return -ENOMEM;
4098
4099 return 0;
4100}
4101
2c8c1e72 4102static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4103{
ece31ffd 4104 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 4105}
4106
4107static struct pernet_operations packet_net_ops = {
4108 .init = packet_net_init,
4109 .exit = packet_net_exit,
4110};
4111
4112
1da177e4
LT		 4113static void __exit packet_exit(void)
4114{
1da177e4 4115 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4116 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4117 sock_unregister(PF_PACKET);
4118 proto_unregister(&packet_proto);
4119}
4120
4121static int __init packet_init(void)
4122{
4123 int rc = proto_register(&packet_proto, 0);
4124
4125 if (rc != 0)
4126 goto out;
4127
4128 sock_register(&packet_family_ops);
d12d01d6 4129 register_pernet_subsys(&packet_net_ops);
1da177e4 4130 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4131out:
4132 return rc;
4133}
4134
4135module_init(packet_init);
4136module_exit(packet_exit);
4137MODULE_LICENSE("GPL");
4138MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 1.676762 seconds and 5 git commands to generate.