projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: stmmac: fix warning from Sparse for socfpga
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
b0138408 91#include <linux/percpu.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
d346a3fa
DB		 240static int packet_direct_xmit(struct sk_buff *skb)
241{
242 struct net_device *dev = skb->dev;
d346a3fa
DB		 243 netdev_features_t features;
244 struct netdev_queue *txq;
43279500 245 int ret = NETDEV_TX_BUSY;
d346a3fa
DB		 246
247 if (unlikely(!netif_running(dev) ||
43279500
DB		 248 !netif_carrier_ok(dev)))
249 goto drop;
d346a3fa
DB		 250
251 features = netif_skb_features(skb);
252 if (skb_needs_linearize(skb, features) &&
43279500
DB		 253 __skb_linearize(skb))
254 goto drop;
d346a3fa 255
10c51b56 256 txq = skb_get_tx_queue(dev, skb);
d346a3fa 257
43279500
DB		 258 local_bh_disable();
259
260 HARD_TX_LOCK(dev, txq, smp_processor_id());
8e2f1a63 261 if (!netif_xmit_frozen_or_drv_stopped(txq)) {
4798248e 262 ret = netdev_start_xmit(skb, dev);
43279500
DB		 263 if (ret == NETDEV_TX_OK)
264 txq_trans_update(txq);
d346a3fa 265 }
43279500 266 HARD_TX_UNLOCK(dev, txq);
d346a3fa 267
43279500
DB		 268 local_bh_enable();
269
270 if (!dev_xmit_complete(ret))
d346a3fa 271 kfree_skb(skb);
43279500 272
d346a3fa 273 return ret;
43279500 274drop:
0f97ede4 275 atomic_long_inc(&dev->tx_dropped);
43279500
DB		 276 kfree_skb(skb);
277 return NET_XMIT_DROP;
d346a3fa
DB		 278}
279
66e56cd4
DB		 280static struct net_device *packet_cached_dev_get(struct packet_sock *po)
281{
282 struct net_device *dev;
283
284 rcu_read_lock();
285 dev = rcu_dereference(po->cached_dev);
286 if (likely(dev))
287 dev_hold(dev);
288 rcu_read_unlock();
289
290 return dev;
291}
292
293static void packet_cached_dev_assign(struct packet_sock *po,
294 struct net_device *dev)
295{
296 rcu_assign_pointer(po->cached_dev, dev);
297}
298
299static void packet_cached_dev_reset(struct packet_sock *po)
300{
301 RCU_INIT_POINTER(po->cached_dev, NULL);
302}
303
d346a3fa
DB		 304static bool packet_use_direct_xmit(const struct packet_sock *po)
305{
306 return po->xmit == packet_direct_xmit;
307}
308
0fd5d57b 309static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
d346a3fa 310{
1cbac010 311 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 312}
313
0fd5d57b
DB		 314static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
315{
316 const struct net_device_ops *ops = dev->netdev_ops;
317 u16 queue_index;
318
319 if (ops->ndo_select_queue) {
320 queue_index = ops->ndo_select_queue(dev, skb, NULL,
321 __packet_pick_tx_queue);
322 queue_index = netdev_cap_txqueue(dev, queue_index);
323 } else {
324 queue_index = __packet_pick_tx_queue(dev, skb);
325 }
326
327 skb_set_queue_mapping(skb, queue_index);
328}
329
ce06b03e
DM		 330/* register_prot_hook must be invoked with the po->bind_lock held,
331 * or from a context in which asynchronous accesses to the packet
332 * socket is not possible (packet_create()).
333 */
334static void register_prot_hook(struct sock *sk)
335{
336 struct packet_sock *po = pkt_sk(sk);
e40526cb 337
ce06b03e 338 if (!po->running) {
66e56cd4 339 if (po->fanout)
dc99f600 340 __fanout_link(sk, po);
66e56cd4 341 else
dc99f600 342 dev_add_pack(&po->prot_hook);
e40526cb 343
ce06b03e
DM		 344 sock_hold(sk);
345 po->running = 1;
346 }
347}
348
349/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
350 * held. If the sync parameter is true, we will temporarily drop
351 * the po->bind_lock and do a synchronize_net to make sure no
352 * asynchronous packet processing paths still refer to the elements
353 * of po->prot_hook. If the sync parameter is false, it is the
354 * callers responsibility to take care of this.
355 */
356static void __unregister_prot_hook(struct sock *sk, bool sync)
357{
358 struct packet_sock *po = pkt_sk(sk);
359
360 po->running = 0;
66e56cd4
DB		 361
362 if (po->fanout)
dc99f600 363 __fanout_unlink(sk, po);
66e56cd4 364 else
dc99f600 365 __dev_remove_pack(&po->prot_hook);
e40526cb 366
ce06b03e
DM		 367 __sock_put(sk);
368
369 if (sync) {
370 spin_unlock(&po->bind_lock);
371 synchronize_net();
372 spin_lock(&po->bind_lock);
373 }
374}
375
376static void unregister_prot_hook(struct sock *sk, bool sync)
377{
378 struct packet_sock *po = pkt_sk(sk);
379
380 if (po->running)
381 __unregister_prot_hook(sk, sync);
382}
383
f6dafa95 384static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 385{
386 if (is_vmalloc_addr(addr))
387 return vmalloc_to_page(addr);
388 return virt_to_page(addr);
389}
390
69e3c75f 391static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 392{
184f489e 393 union tpacket_uhdr h;
1da177e4 394
69e3c75f 395 h.raw = frame;
bbd6ef87
PM		 396 switch (po->tp_version) {
397 case TPACKET_V1:
69e3c75f 398 h.h1->tp_status = status;
0af55bb5 399 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 400 break;
401 case TPACKET_V2:
69e3c75f 402 h.h2->tp_status = status;
0af55bb5 403 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 404 break;
f6fb8f10 405 case TPACKET_V3:
69e3c75f 406 default:
f6fb8f10 407 WARN(1, "TPACKET version not supported.\n");
69e3c75f 408 BUG();
bbd6ef87 409 }
69e3c75f
JB		 410
411 smp_wmb();
bbd6ef87
PM		 412}
413
69e3c75f 414static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 415{
184f489e 416 union tpacket_uhdr h;
bbd6ef87 417
69e3c75f
JB		 418 smp_rmb();
419
bbd6ef87
PM		 420 h.raw = frame;
421 switch (po->tp_version) {
422 case TPACKET_V1:
0af55bb5 423 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 424 return h.h1->tp_status;
bbd6ef87 425 case TPACKET_V2:
0af55bb5 426 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 427 return h.h2->tp_status;
f6fb8f10 428 case TPACKET_V3:
69e3c75f 429 default:
f6fb8f10 430 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 431 BUG();
432 return 0;
bbd6ef87 433 }
1da177e4 434}
69e3c75f 435
b9c32fb2
DB		 436static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
437 unsigned int flags)
7a51384c
DB		 438{
439 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
440
68a360e8
WB		 441 if (shhwtstamps &&
442 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
443 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
444 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 445
446 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 447 return TP_STATUS_TS_SOFTWARE;
7a51384c 448
b9c32fb2 449 return 0;
7a51384c
DB		 450}
451
b9c32fb2
DB		 452static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
453 struct sk_buff *skb)
2e31396f
WB		 454{
455 union tpacket_uhdr h;
456 struct timespec ts;
b9c32fb2 457 __u32 ts_status;
2e31396f 458
b9c32fb2
DB		 459 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
460 return 0;
2e31396f
WB		 461
462 h.raw = frame;
463 switch (po->tp_version) {
464 case TPACKET_V1:
465 h.h1->tp_sec = ts.tv_sec;
466 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
467 break;
468 case TPACKET_V2:
469 h.h2->tp_sec = ts.tv_sec;
470 h.h2->tp_nsec = ts.tv_nsec;
471 break;
472 case TPACKET_V3:
473 default:
474 WARN(1, "TPACKET version not supported.\n");
475 BUG();
476 }
477
478 /* one flush is safe, as both fields always lie on the same cacheline */
479 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
480 smp_wmb();
b9c32fb2
DB		 481
482 return ts_status;
2e31396f
WB		 483}
484
69e3c75f
JB		 485static void *packet_lookup_frame(struct packet_sock *po,
486 struct packet_ring_buffer *rb,
487 unsigned int position,
488 int status)
489{
490 unsigned int pg_vec_pos, frame_offset;
184f489e 491 union tpacket_uhdr h;
69e3c75f
JB		 492
493 pg_vec_pos = position / rb->frames_per_block;
494 frame_offset = position % rb->frames_per_block;
495
0e3125c7
NH		 496 h.raw = rb->pg_vec[pg_vec_pos].buffer +
497 (frame_offset * rb->frame_size);
69e3c75f
JB		 498
499 if (status != __packet_get_status(po, h.raw))
500 return NULL;
501
502 return h.raw;
503}
504
eea49cc9 505static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 506 struct packet_ring_buffer *rb,
507 int status)
508{
509 return packet_lookup_frame(po, rb, rb->head, status);
510}
511
bc59ba39 512static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 513{
514 del_timer_sync(&pkc->retire_blk_timer);
515}
516
517static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
518 int tx_ring,
519 struct sk_buff_head *rb_queue)
520{
bc59ba39 521 struct tpacket_kbdq_core *pkc;
f6fb8f10 522
22781a5b
DJ		 523 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
524 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 525
ec6f809f 526 spin_lock_bh(&rb_queue->lock);
f6fb8f10 527 pkc->delete_blk_timer = 1;
ec6f809f 528 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 529
530 prb_del_retire_blk_timer(pkc);
531}
532
533static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 534 struct tpacket_kbdq_core *pkc,
f6fb8f10 535 void (*func) (unsigned long))
536{
537 init_timer(&pkc->retire_blk_timer);
538 pkc->retire_blk_timer.data = (long)po;
539 pkc->retire_blk_timer.function = func;
540 pkc->retire_blk_timer.expires = jiffies;
541}
542
543static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
544{
bc59ba39 545 struct tpacket_kbdq_core *pkc;
f6fb8f10 546
547 if (tx_ring)
548 BUG();
549
22781a5b
DJ		 550 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
551 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 552 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
553}
554
555static int prb_calc_retire_blk_tmo(struct packet_sock *po,
556 int blk_size_in_bytes)
557{
558 struct net_device *dev;
559 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 560 struct ethtool_cmd ecmd;
561 int err;
e440cf2c 562 u32 speed;
f6fb8f10 563
4bc71cb9
JP		 564 rtnl_lock();
565 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
566 if (unlikely(!dev)) {
567 rtnl_unlock();
f6fb8f10 568 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 569 }
570 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 571 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 572 rtnl_unlock();
573 if (!err) {
4bc71cb9
JP		 574 /*
575 * If the link speed is so slow you don't really
576 * need to worry about perf anyways
577 */
e440cf2c 578 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 579 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 580 } else {
581 msec = 1;
582 div = speed / 1000;
f6fb8f10 583 }
584 }
585
586 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
587
588 if (div)
589 mbits /= div;
590
591 tmo = mbits * msec;
592
593 if (div)
594 return tmo+1;
595 return tmo;
596}
597
bc59ba39 598static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 599 union tpacket_req_u *req_u)
600{
601 p1->feature_req_word = req_u->req3.tp_feature_req_word;
602}
603
604static void init_prb_bdqc(struct packet_sock *po,
605 struct packet_ring_buffer *rb,
606 struct pgv *pg_vec,
607 union tpacket_req_u *req_u, int tx_ring)
608{
22781a5b 609 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 610 struct tpacket_block_desc *pbd;
f6fb8f10 611
612 memset(p1, 0x0, sizeof(*p1));
613
614 p1->knxt_seq_num = 1;
615 p1->pkbdq = pg_vec;
bc59ba39 616 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 617 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 618 p1->kblk_size = req_u->req3.tp_block_size;
619 p1->knum_blocks = req_u->req3.tp_block_nr;
620 p1->hdrlen = po->tp_hdrlen;
621 p1->version = po->tp_version;
622 p1->last_kactive_blk_num = 0;
ee80fbf3 623 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 624 if (req_u->req3.tp_retire_blk_tov)
625 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
626 else
627 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
628 req_u->req3.tp_block_size);
629 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
630 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
631
dc808110 632 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
f6fb8f10 633 prb_init_ft_ops(p1, req_u);
634 prb_setup_retire_blk_timer(po, tx_ring);
635 prb_open_block(p1, pbd);
636}
637
638/* Do NOT update the last_blk_num first.
639 * Assumes sk_buff_head lock is held.
640 */
bc59ba39 641static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 642{
643 mod_timer(&pkc->retire_blk_timer,
644 jiffies + pkc->tov_in_jiffies);
645 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
646}
647
648/*
649 * Timer logic:
650 * 1) We refresh the timer only when we open a block.
651 * By doing this we don't waste cycles refreshing the timer
652 * on packet-by-packet basis.
653 *
654 * With a 1MB block-size, on a 1Gbps line, it will take
655 * i) ~8 ms to fill a block + ii) memcpy etc.
656 * In this cut we are not accounting for the memcpy time.
657 *
658 * So, if the user sets the 'tmo' to 10ms then the timer
659 * will never fire while the block is still getting filled
660 * (which is what we want). However, the user could choose
661 * to close a block early and that's fine.
662 *
663 * But when the timer does fire, we check whether or not to refresh it.
664 * Since the tmo granularity is in msecs, it is not too expensive
665 * to refresh the timer, lets say every '8' msecs.
666 * Either the user can set the 'tmo' or we can derive it based on
667 * a) line-speed and b) block-size.
668 * prb_calc_retire_blk_tmo() calculates the tmo.
669 *
670 */
671static void prb_retire_rx_blk_timer_expired(unsigned long data)
672{
673 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 674 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 675 unsigned int frozen;
bc59ba39 676 struct tpacket_block_desc *pbd;
f6fb8f10 677
678 spin_lock(&po->sk.sk_receive_queue.lock);
679
680 frozen = prb_queue_frozen(pkc);
681 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
682
683 if (unlikely(pkc->delete_blk_timer))
684 goto out;
685
686 /* We only need to plug the race when the block is partially filled.
687 * tpacket_rcv:
688 * lock(); increment BLOCK_NUM_PKTS; unlock()
689 * copy_bits() is in progress ...
690 * timer fires on other cpu:
691 * we can't retire the current block because copy_bits
692 * is in progress.
693 *
694 */
695 if (BLOCK_NUM_PKTS(pbd)) {
696 while (atomic_read(&pkc->blk_fill_in_prog)) {
697 /* Waiting for skb_copy_bits to finish... */
698 cpu_relax();
699 }
700 }
701
702 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
703 if (!frozen) {
704 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
705 if (!prb_dispatch_next_block(pkc, po))
706 goto refresh_timer;
707 else
708 goto out;
709 } else {
710 /* Case 1. Queue was frozen because user-space was
711 * lagging behind.
712 */
713 if (prb_curr_blk_in_use(pkc, pbd)) {
714 /*
715 * Ok, user-space is still behind.
716 * So just refresh the timer.
717 */
718 goto refresh_timer;
719 } else {
720 /* Case 2. queue was frozen,user-space caught up,
721 * now the link went idle && the timer fired.
722 * We don't have a block to close.So we open this
723 * block and restart the timer.
724 * opening a block thaws the queue,restarts timer
725 * Thawing/timer-refresh is a side effect.
726 */
727 prb_open_block(pkc, pbd);
728 goto out;
729 }
730 }
731 }
732
733refresh_timer:
734 _prb_refresh_rx_retire_blk_timer(pkc);
735
736out:
737 spin_unlock(&po->sk.sk_receive_queue.lock);
738}
739
eea49cc9 740static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 741 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 742{
743 /* Flush everything minus the block header */
744
745#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
746 u8 *start, *end;
747
748 start = (u8 *)pbd1;
749
750 /* Skip the block header(we know header WILL fit in 4K) */
751 start += PAGE_SIZE;
752
753 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
754 for (; start < end; start += PAGE_SIZE)
755 flush_dcache_page(pgv_to_page(start));
756
757 smp_wmb();
758#endif
759
760 /* Now update the block status. */
761
762 BLOCK_STATUS(pbd1) = status;
763
764 /* Flush the block header */
765
766#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
767 start = (u8 *)pbd1;
768 flush_dcache_page(pgv_to_page(start));
769
770 smp_wmb();
771#endif
772}
773
774/*
775 * Side effect:
776 *
777 * 1) flush the block
778 * 2) Increment active_blk_num
779 *
780 * Note:We DONT refresh the timer on purpose.
781 * Because almost always the next block will be opened.
782 */
bc59ba39 783static void prb_close_block(struct tpacket_kbdq_core *pkc1,
784 struct tpacket_block_desc *pbd1,
f6fb8f10 785 struct packet_sock *po, unsigned int stat)
786{
787 __u32 status = TP_STATUS_USER | stat;
788
789 struct tpacket3_hdr *last_pkt;
bc59ba39 790 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 791
ee80fbf3 792 if (po->stats.stats3.tp_drops)
f6fb8f10 793 status |= TP_STATUS_LOSING;
794
795 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
796 last_pkt->tp_next_offset = 0;
797
798 /* Get the ts of the last pkt */
799 if (BLOCK_NUM_PKTS(pbd1)) {
800 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
801 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
802 } else {
803 /* Ok, we tmo'd - so get the current time */
804 struct timespec ts;
805 getnstimeofday(&ts);
806 h1->ts_last_pkt.ts_sec = ts.tv_sec;
807 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
808 }
809
810 smp_wmb();
811
812 /* Flush the block */
813 prb_flush_block(pkc1, pbd1, status);
814
815 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
816}
817
eea49cc9 818static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 819{
820 pkc->reset_pending_on_curr_blk = 0;
821}
822
823/*
824 * Side effect of opening a block:
825 *
826 * 1) prb_queue is thawed.
827 * 2) retire_blk_timer is refreshed.
828 *
829 */
bc59ba39 830static void prb_open_block(struct tpacket_kbdq_core *pkc1,
831 struct tpacket_block_desc *pbd1)
f6fb8f10 832{
833 struct timespec ts;
bc59ba39 834 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 835
836 smp_rmb();
837
8da3056c
DB		 838 /* We could have just memset this but we will lose the
839 * flexibility of making the priv area sticky
840 */
f6fb8f10 841
8da3056c
DB		 842 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
843 BLOCK_NUM_PKTS(pbd1) = 0;
844 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 845
8da3056c
DB		 846 getnstimeofday(&ts);
847
848 h1->ts_first_pkt.ts_sec = ts.tv_sec;
849 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 850
8da3056c
DB		 851 pkc1->pkblk_start = (char *)pbd1;
852 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
853
854 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
855 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
856
857 pbd1->version = pkc1->version;
858 pkc1->prev = pkc1->nxt_offset;
859 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
860
861 prb_thaw_queue(pkc1);
862 _prb_refresh_rx_retire_blk_timer(pkc1);
863
864 smp_wmb();
f6fb8f10 865}
866
867/*
868 * Queue freeze logic:
869 * 1) Assume tp_block_nr = 8 blocks.
870 * 2) At time 't0', user opens Rx ring.
871 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
872 * 4) user-space is either sleeping or processing block '0'.
873 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
874 * it will close block-7,loop around and try to fill block '0'.
875 * call-flow:
876 * __packet_lookup_frame_in_block
877 * prb_retire_current_block()
878 * prb_dispatch_next_block()
879 * |->(BLOCK_STATUS == USER) evaluates to true
880 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
881 * 6) Now there are two cases:
882 * 6.1) Link goes idle right after the queue is frozen.
883 * But remember, the last open_block() refreshed the timer.
884 * When this timer expires,it will refresh itself so that we can
885 * re-open block-0 in near future.
886 * 6.2) Link is busy and keeps on receiving packets. This is a simple
887 * case and __packet_lookup_frame_in_block will check if block-0
888 * is free and can now be re-used.
889 */
eea49cc9 890static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 891 struct packet_sock *po)
892{
893 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 894 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 895}
896
897#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
898
899/*
900 * If the next block is free then we will dispatch it
901 * and return a good offset.
902 * Else, we will freeze the queue.
903 * So, caller must check the return value.
904 */
bc59ba39 905static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 906 struct packet_sock *po)
907{
bc59ba39 908 struct tpacket_block_desc *pbd;
f6fb8f10 909
910 smp_rmb();
911
912 /* 1. Get current block num */
913 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
914
915 /* 2. If this block is currently in_use then freeze the queue */
916 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
917 prb_freeze_queue(pkc, po);
918 return NULL;
919 }
920
921 /*
922 * 3.
923 * open this block and return the offset where the first packet
924 * needs to get stored.
925 */
926 prb_open_block(pkc, pbd);
927 return (void *)pkc->nxt_offset;
928}
929
bc59ba39 930static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 931 struct packet_sock *po, unsigned int status)
932{
bc59ba39 933 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 934
935 /* retire/close the current block */
936 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
937 /*
938 * Plug the case where copy_bits() is in progress on
939 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
940 * have space to copy the pkt in the current block and
941 * called prb_retire_current_block()
942 *
943 * We don't need to worry about the TMO case because
944 * the timer-handler already handled this case.
945 */
946 if (!(status & TP_STATUS_BLK_TMO)) {
947 while (atomic_read(&pkc->blk_fill_in_prog)) {
948 /* Waiting for skb_copy_bits to finish... */
949 cpu_relax();
950 }
951 }
952 prb_close_block(pkc, pbd, po, status);
953 return;
954 }
f6fb8f10 955}
956
eea49cc9 957static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 958 struct tpacket_block_desc *pbd)
f6fb8f10 959{
960 return TP_STATUS_USER & BLOCK_STATUS(pbd);
961}
962
eea49cc9 963static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 964{
965 return pkc->reset_pending_on_curr_blk;
966}
967
eea49cc9 968static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 969{
bc59ba39 970 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 971 atomic_dec(&pkc->blk_fill_in_prog);
972}
973
eea49cc9 974static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 975 struct tpacket3_hdr *ppd)
976{
3958afa1 977 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 978}
979
eea49cc9 980static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 981 struct tpacket3_hdr *ppd)
982{
983 ppd->hv1.tp_rxhash = 0;
984}
985
eea49cc9 986static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 987 struct tpacket3_hdr *ppd)
988{
989 if (vlan_tx_tag_present(pkc->skb)) {
990 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
a0cdfcf3
AW		 991 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
992 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 993 } else {
9e67030a 994 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 995 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 996 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 997 }
998}
999
bc59ba39 1000static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 1001 struct tpacket3_hdr *ppd)
1002{
a0cdfcf3 1003 ppd->hv1.tp_padding = 0;
f6fb8f10 1004 prb_fill_vlan_info(pkc, ppd);
1005
1006 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1007 prb_fill_rxhash(pkc, ppd);
1008 else
1009 prb_clear_rxhash(pkc, ppd);
1010}
1011
eea49cc9 1012static void prb_fill_curr_block(char *curr,
bc59ba39 1013 struct tpacket_kbdq_core *pkc,
1014 struct tpacket_block_desc *pbd,
f6fb8f10 1015 unsigned int len)
1016{
1017 struct tpacket3_hdr *ppd;
1018
1019 ppd = (struct tpacket3_hdr *)curr;
1020 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1021 pkc->prev = curr;
1022 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1023 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1024 BLOCK_NUM_PKTS(pbd) += 1;
1025 atomic_inc(&pkc->blk_fill_in_prog);
1026 prb_run_all_ft_ops(pkc, ppd);
1027}
1028
1029/* Assumes caller has the sk->rx_queue.lock */
1030static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1031 struct sk_buff *skb,
1032 int status,
1033 unsigned int len
1034 )
1035{
bc59ba39 1036 struct tpacket_kbdq_core *pkc;
1037 struct tpacket_block_desc *pbd;
f6fb8f10 1038 char *curr, *end;
1039
e3192690 1040 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1041 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1042
1043 /* Queue is frozen when user space is lagging behind */
1044 if (prb_queue_frozen(pkc)) {
1045 /*
1046 * Check if that last block which caused the queue to freeze,
1047 * is still in_use by user-space.
1048 */
1049 if (prb_curr_blk_in_use(pkc, pbd)) {
1050 /* Can't record this packet */
1051 return NULL;
1052 } else {
1053 /*
1054 * Ok, the block was released by user-space.
1055 * Now let's open that block.
1056 * opening a block also thaws the queue.
1057 * Thawing is a side effect.
1058 */
1059 prb_open_block(pkc, pbd);
1060 }
1061 }
1062
1063 smp_mb();
1064 curr = pkc->nxt_offset;
1065 pkc->skb = skb;
e3192690 1066 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1067
1068 /* first try the current block */
1069 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1070 prb_fill_curr_block(curr, pkc, pbd, len);
1071 return (void *)curr;
1072 }
1073
1074 /* Ok, close the current block */
1075 prb_retire_current_block(pkc, po, 0);
1076
1077 /* Now, try to dispatch the next block */
1078 curr = (char *)prb_dispatch_next_block(pkc, po);
1079 if (curr) {
1080 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1081 prb_fill_curr_block(curr, pkc, pbd, len);
1082 return (void *)curr;
1083 }
1084
1085 /*
1086 * No free blocks are available.user_space hasn't caught up yet.
1087 * Queue was just frozen and now this packet will get dropped.
1088 */
1089 return NULL;
1090}
1091
eea49cc9 1092static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1093 struct sk_buff *skb,
1094 int status, unsigned int len)
1095{
1096 char *curr = NULL;
1097 switch (po->tp_version) {
1098 case TPACKET_V1:
1099 case TPACKET_V2:
1100 curr = packet_lookup_frame(po, &po->rx_ring,
1101 po->rx_ring.head, status);
1102 return curr;
1103 case TPACKET_V3:
1104 return __packet_lookup_frame_in_block(po, skb, status, len);
1105 default:
1106 WARN(1, "TPACKET version not supported\n");
1107 BUG();
99aa3473 1108 return NULL;
f6fb8f10 1109 }
1110}
1111
eea49cc9 1112static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1113 struct packet_ring_buffer *rb,
77f65ebd 1114 unsigned int idx,
f6fb8f10 1115 int status)
1116{
bc59ba39 1117 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1118 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1119
1120 if (status != BLOCK_STATUS(pbd))
1121 return NULL;
1122 return pbd;
1123}
1124
eea49cc9 1125static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1126{
1127 unsigned int prev;
1128 if (rb->prb_bdqc.kactive_blk_num)
1129 prev = rb->prb_bdqc.kactive_blk_num-1;
1130 else
1131 prev = rb->prb_bdqc.knum_blocks-1;
1132 return prev;
1133}
1134
1135/* Assumes caller has held the rx_queue.lock */
eea49cc9 1136static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1137 struct packet_ring_buffer *rb,
1138 int status)
1139{
1140 unsigned int previous = prb_previous_blk_num(rb);
1141 return prb_lookup_block(po, rb, previous, status);
1142}
1143
eea49cc9 1144static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1145 struct packet_ring_buffer *rb,
1146 int status)
1147{
1148 if (po->tp_version <= TPACKET_V2)
1149 return packet_previous_frame(po, rb, status);
1150
1151 return __prb_previous_block(po, rb, status);
1152}
1153
eea49cc9 1154static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1155 struct packet_ring_buffer *rb)
1156{
1157 switch (po->tp_version) {
1158 case TPACKET_V1:
1159 case TPACKET_V2:
1160 return packet_increment_head(rb);
1161 case TPACKET_V3:
1162 default:
1163 WARN(1, "TPACKET version not supported.\n");
1164 BUG();
1165 return;
1166 }
1167}
1168
eea49cc9 1169static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1170 struct packet_ring_buffer *rb,
1171 int status)
1172{
1173 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1174 return packet_lookup_frame(po, rb, previous, status);
1175}
1176
eea49cc9 1177static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1178{
1179 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1180}
1181
b0138408
DB		 1182static void packet_inc_pending(struct packet_ring_buffer *rb)
1183{
1184 this_cpu_inc(*rb->pending_refcnt);
1185}
1186
1187static void packet_dec_pending(struct packet_ring_buffer *rb)
1188{
1189 this_cpu_dec(*rb->pending_refcnt);
1190}
1191
1192static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1193{
1194 unsigned int refcnt = 0;
1195 int cpu;
1196
1197 /* We don't use pending refcount in rx_ring. */
1198 if (rb->pending_refcnt == NULL)
1199 return 0;
1200
1201 for_each_possible_cpu(cpu)
1202 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1203
1204 return refcnt;
1205}
1206
1207static int packet_alloc_pending(struct packet_sock *po)
1208{
1209 po->rx_ring.pending_refcnt = NULL;
1210
1211 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1212 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1213 return -ENOBUFS;
1214
1215 return 0;
1216}
1217
1218static void packet_free_pending(struct packet_sock *po)
1219{
1220 free_percpu(po->tx_ring.pending_refcnt);
1221}
1222
77f65ebd
WB		 1223static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1224{
1225 struct sock *sk = &po->sk;
1226 bool has_room;
1227
1228 if (po->prot_hook.func != tpacket_rcv)
1229 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1230 <= sk->sk_rcvbuf;
1231
1232 spin_lock(&sk->sk_receive_queue.lock);
1233 if (po->tp_version == TPACKET_V3)
1234 has_room = prb_lookup_block(po, &po->rx_ring,
1235 po->rx_ring.prb_bdqc.kactive_blk_num,
1236 TP_STATUS_KERNEL);
1237 else
1238 has_room = packet_lookup_frame(po, &po->rx_ring,
1239 po->rx_ring.head,
1240 TP_STATUS_KERNEL);
1241 spin_unlock(&sk->sk_receive_queue.lock);
1242
1243 return has_room;
1244}
1245
1da177e4
LT		 1246static void packet_sock_destruct(struct sock *sk)
1247{
ed85b565
RC		 1248 skb_queue_purge(&sk->sk_error_queue);
1249
547b792c
IJ		 1250 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1251 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1252
1253 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1254 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1255 return;
1256 }
1257
17ab56a2 1258 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1259}
1260
dc99f600
DM		 1261static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1262{
1263 int x = atomic_read(&f->rr_cur) + 1;
1264
1265 if (x >= num)
1266 x = 0;
1267
1268 return x;
1269}
1270
77f65ebd
WB		 1271static unsigned int fanout_demux_hash(struct packet_fanout *f,
1272 struct sk_buff *skb,
1273 unsigned int num)
dc99f600 1274{
61b905da 1275 return reciprocal_scale(skb_get_hash(skb), num);
dc99f600
DM		 1276}
1277
77f65ebd
WB		 1278static unsigned int fanout_demux_lb(struct packet_fanout *f,
1279 struct sk_buff *skb,
1280 unsigned int num)
dc99f600
DM		 1281{
1282 int cur, old;
1283
1284 cur = atomic_read(&f->rr_cur);
1285 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1286 fanout_rr_next(f, num))) != cur)
1287 cur = old;
77f65ebd
WB		 1288 return cur;
1289}
1290
1291static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1292 struct sk_buff *skb,
1293 unsigned int num)
1294{
1295 return smp_processor_id() % num;
dc99f600
DM		 1296}
1297
5df0ddfb
DB		 1298static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1299 struct sk_buff *skb,
1300 unsigned int num)
1301{
f337db64 1302 return prandom_u32_max(num);
5df0ddfb
DB		 1303}
1304
77f65ebd
WB		 1305static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1306 struct sk_buff *skb,
1307 unsigned int idx, unsigned int skip,
1308 unsigned int num)
95ec3eb4 1309{
77f65ebd 1310 unsigned int i, j;
95ec3eb4 1311
77f65ebd
WB		 1312 i = j = min_t(int, f->next[idx], num - 1);
1313 do {
1314 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1315 if (i != j)
1316 f->next[idx] = i;
1317 return i;
1318 }
1319 if (++i == num)
1320 i = 0;
1321 } while (i != j);
1322
1323 return idx;
1324}
1325
2d36097d
NH		 1326static unsigned int fanout_demux_qm(struct packet_fanout *f,
1327 struct sk_buff *skb,
1328 unsigned int num)
1329{
1330 return skb_get_queue_mapping(skb) % num;
1331}
1332
77f65ebd
WB		 1333static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1334{
1335 return f->flags & (flag >> 8);
95ec3eb4
DM		 1336}
1337
95ec3eb4
DM		 1338static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1339 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1340{
1341 struct packet_fanout *f = pt->af_packet_priv;
1342 unsigned int num = f->num_members;
1343 struct packet_sock *po;
77f65ebd 1344 unsigned int idx;
dc99f600
DM		 1345
1346 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1347 !num) {
1348 kfree_skb(skb);
1349 return 0;
1350 }
1351
95ec3eb4
DM		 1352 switch (f->type) {
1353 case PACKET_FANOUT_HASH:
1354 default:
77f65ebd 1355 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1356 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1357 if (!skb)
1358 return 0;
1359 }
77f65ebd 1360 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1361 break;
1362 case PACKET_FANOUT_LB:
77f65ebd 1363 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1364 break;
1365 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1366 idx = fanout_demux_cpu(f, skb, num);
1367 break;
5df0ddfb
DB		 1368 case PACKET_FANOUT_RND:
1369 idx = fanout_demux_rnd(f, skb, num);
1370 break;
2d36097d
NH		 1371 case PACKET_FANOUT_QM:
1372 idx = fanout_demux_qm(f, skb, num);
1373 break;
77f65ebd
WB		 1374 case PACKET_FANOUT_ROLLOVER:
1375 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1376 break;
dc99f600
DM		 1377 }
1378
77f65ebd
WB		 1379 po = pkt_sk(f->arr[idx]);
1380 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1381 unlikely(!packet_rcv_has_room(po, skb))) {
1382 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1383 po = pkt_sk(f->arr[idx]);
1384 }
dc99f600
DM		 1385
1386 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1387}
1388
fff3321d
PE		 1389DEFINE_MUTEX(fanout_mutex);
1390EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1391static LIST_HEAD(fanout_list);
1392
1393static void __fanout_link(struct sock *sk, struct packet_sock *po)
1394{
1395 struct packet_fanout *f = po->fanout;
1396
1397 spin_lock(&f->lock);
1398 f->arr[f->num_members] = sk;
1399 smp_wmb();
1400 f->num_members++;
1401 spin_unlock(&f->lock);
1402}
1403
1404static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1405{
1406 struct packet_fanout *f = po->fanout;
1407 int i;
1408
1409 spin_lock(&f->lock);
1410 for (i = 0; i < f->num_members; i++) {
1411 if (f->arr[i] == sk)
1412 break;
1413 }
1414 BUG_ON(i >= f->num_members);
1415 f->arr[i] = f->arr[f->num_members - 1];
1416 f->num_members--;
1417 spin_unlock(&f->lock);
1418}
1419
d4dd8aee 1420static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1421{
d4dd8aee 1422 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
c0de08d0
EL		 1423 return true;
1424
1425 return false;
1426}
1427
7736d33f 1428static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1429{
1430 struct packet_sock *po = pkt_sk(sk);
1431 struct packet_fanout *f, *match;
7736d33f 1432 u8 type = type_flags & 0xff;
77f65ebd 1433 u8 flags = type_flags >> 8;
dc99f600
DM		 1434 int err;
1435
1436 switch (type) {
77f65ebd
WB		 1437 case PACKET_FANOUT_ROLLOVER:
1438 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1439 return -EINVAL;
dc99f600
DM		 1440 case PACKET_FANOUT_HASH:
1441 case PACKET_FANOUT_LB:
95ec3eb4 1442 case PACKET_FANOUT_CPU:
5df0ddfb 1443 case PACKET_FANOUT_RND:
2d36097d 1444 case PACKET_FANOUT_QM:
dc99f600
DM		 1445 break;
1446 default:
1447 return -EINVAL;
1448 }
1449
1450 if (!po->running)
1451 return -EINVAL;
1452
1453 if (po->fanout)
1454 return -EALREADY;
1455
1456 mutex_lock(&fanout_mutex);
1457 match = NULL;
1458 list_for_each_entry(f, &fanout_list, list) {
1459 if (f->id == id &&
1460 read_pnet(&f->net) == sock_net(sk)) {
1461 match = f;
1462 break;
1463 }
1464 }
afe62c68 1465 err = -EINVAL;
77f65ebd 1466 if (match && match->flags != flags)
afe62c68 1467 goto out;
dc99f600 1468 if (!match) {
afe62c68 1469 err = -ENOMEM;
dc99f600 1470 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1471 if (!match)
1472 goto out;
1473 write_pnet(&match->net, sock_net(sk));
1474 match->id = id;
1475 match->type = type;
77f65ebd 1476 match->flags = flags;
afe62c68
ED		 1477 atomic_set(&match->rr_cur, 0);
1478 INIT_LIST_HEAD(&match->list);
1479 spin_lock_init(&match->lock);
1480 atomic_set(&match->sk_ref, 0);
1481 match->prot_hook.type = po->prot_hook.type;
1482 match->prot_hook.dev = po->prot_hook.dev;
1483 match->prot_hook.func = packet_rcv_fanout;
1484 match->prot_hook.af_packet_priv = match;
c0de08d0 1485 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1486 dev_add_pack(&match->prot_hook);
1487 list_add(&match->list, &fanout_list);
dc99f600 1488 }
afe62c68
ED		 1489 err = -EINVAL;
1490 if (match->type == type &&
1491 match->prot_hook.type == po->prot_hook.type &&
1492 match->prot_hook.dev == po->prot_hook.dev) {
1493 err = -ENOSPC;
1494 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1495 __dev_remove_pack(&po->prot_hook);
1496 po->fanout = match;
1497 atomic_inc(&match->sk_ref);
1498 __fanout_link(sk, po);
1499 err = 0;
dc99f600
DM		 1500 }
1501 }
afe62c68 1502out:
dc99f600
DM		 1503 mutex_unlock(&fanout_mutex);
1504 return err;
1505}
1506
1507static void fanout_release(struct sock *sk)
1508{
1509 struct packet_sock *po = pkt_sk(sk);
1510 struct packet_fanout *f;
1511
1512 f = po->fanout;
1513 if (!f)
1514 return;
1515
fff3321d 1516 mutex_lock(&fanout_mutex);
dc99f600
DM		 1517 po->fanout = NULL;
1518
dc99f600
DM		 1519 if (atomic_dec_and_test(&f->sk_ref)) {
1520 list_del(&f->list);
1521 dev_remove_pack(&f->prot_hook);
1522 kfree(f);
1523 }
1524 mutex_unlock(&fanout_mutex);
1525}
1da177e4 1526
90ddc4f0 1527static const struct proto_ops packet_ops;
1da177e4 1528
90ddc4f0 1529static const struct proto_ops packet_ops_spkt;
1da177e4 1530
40d4e3df
ED		 1531static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1532 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1533{
1534 struct sock *sk;
1535 struct sockaddr_pkt *spkt;
1536
1537 /*
1538 * When we registered the protocol we saved the socket in the data
1539 * field for just this event.
1540 */
1541
1542 sk = pt->af_packet_priv;
1ce4f28b 1543
1da177e4
LT		 1544 /*
1545 * Yank back the headers [hope the device set this
1546 * right or kerboom...]
1547 *
1548 * Incoming packets have ll header pulled,
1549 * push it back.
1550 *
98e399f8 1551 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1552 * so that this procedure is noop.
1553 */
1554
1555 if (skb->pkt_type == PACKET_LOOPBACK)
1556 goto out;
1557
09ad9bc7 1558 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1559 goto out;
1560
40d4e3df
ED		 1561 skb = skb_share_check(skb, GFP_ATOMIC);
1562 if (skb == NULL)
1da177e4
LT		 1563 goto oom;
1564
1565 /* drop any routing info */
adf30907 1566 skb_dst_drop(skb);
1da177e4 1567
84531c24
PO		 1568 /* drop conntrack reference */
1569 nf_reset(skb);
1570
ffbc6111 1571 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1572
98e399f8 1573 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1574
1575 /*
1576 * The SOCK_PACKET socket receives _all_ frames.
1577 */
1578
1579 spkt->spkt_family = dev->type;
1580 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1581 spkt->spkt_protocol = skb->protocol;
1582
1583 /*
1584 * Charge the memory to the socket. This is done specifically
1585 * to prevent sockets using all the memory up.
1586 */
1587
40d4e3df 1588 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1589 return 0;
1590
1591out:
1592 kfree_skb(skb);
1593oom:
1594 return 0;
1595}
1596
1597
1598/*
1599 * Output a raw packet to a device layer. This bypasses all the other
1600 * protocol layers and you must therefore supply it with a complete frame
1601 */
1ce4f28b 1602
1da177e4
LT		 1603static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1604 struct msghdr *msg, size_t len)
1605{
1606 struct sock *sk = sock->sk;
342dfc30 1607 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1608 struct sk_buff *skb = NULL;
1da177e4 1609 struct net_device *dev;
40d4e3df 1610 __be16 proto = 0;
1da177e4 1611 int err;
3bdc0eba 1612 int extra_len = 0;
1ce4f28b 1613
1da177e4 1614 /*
1ce4f28b 1615 * Get and verify the address.
1da177e4
LT		 1616 */
1617
40d4e3df 1618 if (saddr) {
1da177e4 1619 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1620 return -EINVAL;
1621 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1622 proto = saddr->spkt_protocol;
1623 } else
1624 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1625
1626 /*
1ce4f28b 1627 * Find the device first to size check it
1da177e4
LT		 1628 */
1629
de74e92a 1630 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1631retry:
654d1f8a
ED		 1632 rcu_read_lock();
1633 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1634 err = -ENODEV;
1635 if (dev == NULL)
1636 goto out_unlock;
1ce4f28b 1637
d5e76b0a
DM		 1638 err = -ENETDOWN;
1639 if (!(dev->flags & IFF_UP))
1640 goto out_unlock;
1641
1da177e4 1642 /*
40d4e3df
ED		 1643 * You may not queue a frame bigger than the mtu. This is the lowest level
1644 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1645 */
1ce4f28b 1646
3bdc0eba
BG		 1647 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1648 if (!netif_supports_nofcs(dev)) {
1649 err = -EPROTONOSUPPORT;
1650 goto out_unlock;
1651 }
1652 extra_len = 4; /* We're doing our own CRC */
1653 }
1654
1da177e4 1655 err = -EMSGSIZE;
3bdc0eba 1656 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1657 goto out_unlock;
1658
1a35ca80
ED		 1659 if (!skb) {
1660 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1661 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1662 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1663
1664 rcu_read_unlock();
4ce40912 1665 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1666 if (skb == NULL)
1667 return -ENOBUFS;
1668 /* FIXME: Save some space for broken drivers that write a hard
1669 * header at transmission time by themselves. PPP is the notable
1670 * one here. This should really be fixed at the driver level.
1671 */
1672 skb_reserve(skb, reserved);
1673 skb_reset_network_header(skb);
1674
1675 /* Try to align data part correctly */
1676 if (hhlen) {
1677 skb->data -= hhlen;
1678 skb->tail -= hhlen;
1679 if (len < hhlen)
1680 skb_reset_network_header(skb);
1681 }
1682 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1683 if (err)
1684 goto out_free;
1685 goto retry;
1da177e4
LT		 1686 }
1687
3bdc0eba 1688 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1689 /* Earlier code assumed this would be a VLAN pkt,
1690 * double-check this now that we have the actual
1691 * packet in hand.
1692 */
1693 struct ethhdr *ehdr;
1694 skb_reset_mac_header(skb);
1695 ehdr = eth_hdr(skb);
1696 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1697 err = -EMSGSIZE;
1698 goto out_unlock;
1699 }
1700 }
1a35ca80 1701
1da177e4
LT		 1702 skb->protocol = proto;
1703 skb->dev = dev;
1704 skb->priority = sk->sk_priority;
2d37a186 1705 skb->mark = sk->sk_mark;
bf84a010
DB		 1706
1707 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1708
3bdc0eba
BG		 1709 if (unlikely(extra_len == 4))
1710 skb->no_fcs = 1;
1711
40893fd0 1712 skb_probe_transport_header(skb, 0);
c1aad275 1713
1da177e4 1714 dev_queue_xmit(skb);
654d1f8a 1715 rcu_read_unlock();
40d4e3df 1716 return len;
1da177e4 1717
1da177e4 1718out_unlock:
654d1f8a 1719 rcu_read_unlock();
1a35ca80
ED		 1720out_free:
1721 kfree_skb(skb);
1da177e4
LT		 1722 return err;
1723}
1da177e4 1724
eea49cc9 1725static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1726 const struct sock *sk,
dbcb5855 1727 unsigned int res)
1da177e4
LT		 1728{
1729 struct sk_filter *filter;
fda9ef5d 1730
80f8f102
ED		 1731 rcu_read_lock();
1732 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1733 if (filter != NULL)
0a14842f 1734 res = SK_RUN_FILTER(filter, skb);
80f8f102 1735 rcu_read_unlock();
1da177e4 1736
dbcb5855 1737 return res;
1da177e4
LT		 1738}
1739
1740/*
62ab0812
ED		 1741 * This function makes lazy skb cloning in hope that most of packets
1742 * are discarded by BPF.
1743 *
1744 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1745 * and skb->cb are mangled. It works because (and until) packets
1746 * falling here are owned by current CPU. Output packets are cloned
1747 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1748 * sequencially, so that if we return skb to original state on exit,
1749 * we will not harm anyone.
1da177e4
LT		 1750 */
1751
40d4e3df
ED		 1752static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1753 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1754{
1755 struct sock *sk;
1756 struct sockaddr_ll *sll;
1757 struct packet_sock *po;
40d4e3df 1758 u8 *skb_head = skb->data;
1da177e4 1759 int skb_len = skb->len;
dbcb5855 1760 unsigned int snaplen, res;
1da177e4
LT		 1761
1762 if (skb->pkt_type == PACKET_LOOPBACK)
1763 goto drop;
1764
1765 sk = pt->af_packet_priv;
1766 po = pkt_sk(sk);
1767
09ad9bc7 1768 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1769 goto drop;
1770
1da177e4
LT		 1771 skb->dev = dev;
1772
3b04ddde 1773 if (dev->header_ops) {
1da177e4 1774 /* The device has an explicit notion of ll header,
62ab0812
ED		 1775 * exported to higher levels.
1776 *
1777 * Otherwise, the device hides details of its frame
1778 * structure, so that corresponding packet head is
1779 * never delivered to user.
1da177e4
LT		 1780 */
1781 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1782 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1783 else if (skb->pkt_type == PACKET_OUTGOING) {
1784 /* Special case: outgoing packets have ll header at head */
bbe735e4 1785 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1786 }
1787 }
1788
1789 snaplen = skb->len;
1790
dbcb5855
DM		 1791 res = run_filter(skb, sk, snaplen);
1792 if (!res)
fda9ef5d 1793 goto drop_n_restore;
dbcb5855
DM		 1794 if (snaplen > res)
1795 snaplen = res;
1da177e4 1796
0fd7bac6 1797 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1798 goto drop_n_acct;
1799
1800 if (skb_shared(skb)) {
1801 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1802 if (nskb == NULL)
1803 goto drop_n_acct;
1804
1805 if (skb_head != skb->data) {
1806 skb->data = skb_head;
1807 skb->len = skb_len;
1808 }
abc4e4fa 1809 consume_skb(skb);
1da177e4
LT		 1810 skb = nskb;
1811 }
1812
ffbc6111
HX		 1813 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1814 sizeof(skb->cb));
1815
1816 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1817 sll->sll_family = AF_PACKET;
1818 sll->sll_hatype = dev->type;
1819 sll->sll_protocol = skb->protocol;
1820 sll->sll_pkttype = skb->pkt_type;
8032b464 1821 if (unlikely(po->origdev))
80feaacb
PWJ		 1822 sll->sll_ifindex = orig_dev->ifindex;
1823 else
1824 sll->sll_ifindex = dev->ifindex;
1da177e4 1825
b95cce35 1826 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1827
ffbc6111 1828 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1829
1da177e4
LT		 1830 if (pskb_trim(skb, snaplen))
1831 goto drop_n_acct;
1832
1833 skb_set_owner_r(skb, sk);
1834 skb->dev = NULL;
adf30907 1835 skb_dst_drop(skb);
1da177e4 1836
84531c24
PO		 1837 /* drop conntrack reference */
1838 nf_reset(skb);
1839
1da177e4 1840 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1841 po->stats.stats1.tp_packets++;
3b885787 1842 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1843 __skb_queue_tail(&sk->sk_receive_queue, skb);
1844 spin_unlock(&sk->sk_receive_queue.lock);
676d2369 1845 sk->sk_data_ready(sk);
1da177e4
LT		 1846 return 0;
1847
1848drop_n_acct:
7091fbd8 1849 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1850 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1851 atomic_inc(&sk->sk_drops);
1852 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1853
1854drop_n_restore:
1855 if (skb_head != skb->data && skb_shared(skb)) {
1856 skb->data = skb_head;
1857 skb->len = skb_len;
1858 }
1859drop:
ead2ceb0 1860 consume_skb(skb);
1da177e4
LT		 1861 return 0;
1862}
1863
40d4e3df
ED		 1864static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1865 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1866{
1867 struct sock *sk;
1868 struct packet_sock *po;
1869 struct sockaddr_ll *sll;
184f489e 1870 union tpacket_uhdr h;
40d4e3df 1871 u8 *skb_head = skb->data;
1da177e4 1872 int skb_len = skb->len;
dbcb5855 1873 unsigned int snaplen, res;
f6fb8f10 1874 unsigned long status = TP_STATUS_USER;
bbd6ef87 1875 unsigned short macoff, netoff, hdrlen;
1da177e4 1876 struct sk_buff *copy_skb = NULL;
bbd6ef87 1877 struct timespec ts;
b9c32fb2 1878 __u32 ts_status;
1da177e4 1879
51846355
AW		 1880 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1881 * We may add members to them until current aligned size without forcing
1882 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1883 */
1884 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1885 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1886
1da177e4
LT		 1887 if (skb->pkt_type == PACKET_LOOPBACK)
1888 goto drop;
1889
1890 sk = pt->af_packet_priv;
1891 po = pkt_sk(sk);
1892
09ad9bc7 1893 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1894 goto drop;
1895
3b04ddde 1896 if (dev->header_ops) {
1da177e4 1897 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1898 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1899 else if (skb->pkt_type == PACKET_OUTGOING) {
1900 /* Special case: outgoing packets have ll header at head */
bbe735e4 1901 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1902 }
1903 }
1904
8dc41944
HX		 1905 if (skb->ip_summed == CHECKSUM_PARTIAL)
1906 status |= TP_STATUS_CSUMNOTREADY;
1907
1da177e4
LT		 1908 snaplen = skb->len;
1909
dbcb5855
DM		 1910 res = run_filter(skb, sk, snaplen);
1911 if (!res)
fda9ef5d 1912 goto drop_n_restore;
dbcb5855
DM		 1913 if (snaplen > res)
1914 snaplen = res;
1da177e4
LT		 1915
1916 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1917 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1918 po->tp_reserve;
1da177e4 1919 } else {
95c96174 1920 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1921 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1922 (maclen < 16 ? 16 : maclen)) +
1923 po->tp_reserve;
1da177e4
LT		 1924 macoff = netoff - maclen;
1925 }
f6fb8f10 1926 if (po->tp_version <= TPACKET_V2) {
1927 if (macoff + snaplen > po->rx_ring.frame_size) {
1928 if (po->copy_thresh &&
0fd7bac6 1929 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1930 if (skb_shared(skb)) {
1931 copy_skb = skb_clone(skb, GFP_ATOMIC);
1932 } else {
1933 copy_skb = skb_get(skb);
1934 skb_head = skb->data;
1935 }
1936 if (copy_skb)
1937 skb_set_owner_r(copy_skb, sk);
1da177e4 1938 }
f6fb8f10 1939 snaplen = po->rx_ring.frame_size - macoff;
1940 if ((int)snaplen < 0)
1941 snaplen = 0;
1da177e4 1942 }
dc808110
ED		 1943 } else if (unlikely(macoff + snaplen >
1944 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
1945 u32 nval;
1946
1947 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
1948 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
1949 snaplen, nval, macoff);
1950 snaplen = nval;
1951 if (unlikely((int)snaplen < 0)) {
1952 snaplen = 0;
1953 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
1954 }
1da177e4 1955 }
1da177e4 1956 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1957 h.raw = packet_current_rx_frame(po, skb,
1958 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1959 if (!h.raw)
1da177e4 1960 goto ring_is_full;
f6fb8f10 1961 if (po->tp_version <= TPACKET_V2) {
1962 packet_increment_rx_head(po, &po->rx_ring);
1963 /*
1964 * LOSING will be reported till you read the stats,
1965 * because it's COR - Clear On Read.
1966 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1967 * at packet level.
1968 */
ee80fbf3 1969 if (po->stats.stats1.tp_drops)
f6fb8f10 1970 status |= TP_STATUS_LOSING;
1971 }
ee80fbf3 1972 po->stats.stats1.tp_packets++;
1da177e4
LT		 1973 if (copy_skb) {
1974 status |= TP_STATUS_COPY;
1975 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1976 }
1da177e4
LT		 1977 spin_unlock(&sk->sk_receive_queue.lock);
1978
bbd6ef87 1979 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1980
1981 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1982 getnstimeofday(&ts);
1da177e4 1983
b9c32fb2
DB		 1984 status |= ts_status;
1985
bbd6ef87
PM		 1986 switch (po->tp_version) {
1987 case TPACKET_V1:
1988 h.h1->tp_len = skb->len;
1989 h.h1->tp_snaplen = snaplen;
1990 h.h1->tp_mac = macoff;
1991 h.h1->tp_net = netoff;
4b457bdf
DB		 1992 h.h1->tp_sec = ts.tv_sec;
1993 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1994 hdrlen = sizeof(*h.h1);
1995 break;
1996 case TPACKET_V2:
1997 h.h2->tp_len = skb->len;
1998 h.h2->tp_snaplen = snaplen;
1999 h.h2->tp_mac = macoff;
2000 h.h2->tp_net = netoff;
bbd6ef87
PM		 2001 h.h2->tp_sec = ts.tv_sec;
2002 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 2003 if (vlan_tx_tag_present(skb)) {
2004 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 2005 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2006 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2007 } else {
2008 h.h2->tp_vlan_tci = 0;
a0cdfcf3 2009 h.h2->tp_vlan_tpid = 0;
a3bcc23e 2010 }
e4d26f4b 2011 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 2012 hdrlen = sizeof(*h.h2);
2013 break;
f6fb8f10 2014 case TPACKET_V3:
2015 /* tp_nxt_offset,vlan are already populated above.
2016 * So DONT clear those fields here
2017 */
2018 h.h3->tp_status |= status;
2019 h.h3->tp_len = skb->len;
2020 h.h3->tp_snaplen = snaplen;
2021 h.h3->tp_mac = macoff;
2022 h.h3->tp_net = netoff;
f6fb8f10 2023 h.h3->tp_sec = ts.tv_sec;
2024 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2025 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2026 hdrlen = sizeof(*h.h3);
2027 break;
bbd6ef87
PM		 2028 default:
2029 BUG();
2030 }
1da177e4 2031
bbd6ef87 2032 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2033 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2034 sll->sll_family = AF_PACKET;
2035 sll->sll_hatype = dev->type;
2036 sll->sll_protocol = skb->protocol;
2037 sll->sll_pkttype = skb->pkt_type;
8032b464 2038 if (unlikely(po->origdev))
80feaacb
PWJ		 2039 sll->sll_ifindex = orig_dev->ifindex;
2040 else
2041 sll->sll_ifindex = dev->ifindex;
1da177e4 2042
e16aa207 2043 smp_mb();
f0d4eb29 2044
f6dafa95 2045#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2046 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2047 u8 *start, *end;
2048
f0d4eb29
DB		 2049 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2050 macoff + snaplen);
2051
2052 for (start = h.raw; start < end; start += PAGE_SIZE)
2053 flush_dcache_page(pgv_to_page(start));
1da177e4 2054 }
f0d4eb29 2055 smp_wmb();
f6dafa95 2056#endif
f0d4eb29 2057
f6fb8f10 2058 if (po->tp_version <= TPACKET_V2)
2059 __packet_set_status(po, h.raw, status);
2060 else
2061 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4 2062
676d2369 2063 sk->sk_data_ready(sk);
1da177e4
LT		 2064
2065drop_n_restore:
2066 if (skb_head != skb->data && skb_shared(skb)) {
2067 skb->data = skb_head;
2068 skb->len = skb_len;
2069 }
2070drop:
1ce4f28b 2071 kfree_skb(skb);
1da177e4
LT		 2072 return 0;
2073
2074ring_is_full:
ee80fbf3 2075 po->stats.stats1.tp_drops++;
1da177e4
LT		 2076 spin_unlock(&sk->sk_receive_queue.lock);
2077
676d2369 2078 sk->sk_data_ready(sk);
acb5d75b 2079 kfree_skb(copy_skb);
1da177e4
LT		 2080 goto drop_n_restore;
2081}
2082
69e3c75f
JB		 2083static void tpacket_destruct_skb(struct sk_buff *skb)
2084{
2085 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2086
69e3c75f 2087 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2088 void *ph;
b9c32fb2
DB		 2089 __u32 ts;
2090
69e3c75f 2091 ph = skb_shinfo(skb)->destructor_arg;
b0138408 2092 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2093
2094 ts = __packet_set_timestamp(po, ph, skb);
2095 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2096 }
2097
2098 sock_wfree(skb);
2099}
2100
40d4e3df
ED		 2101static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2102 void *frame, struct net_device *dev, int size_max,
ae641949 2103 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 2104{
184f489e 2105 union tpacket_uhdr ph;
09effa67 2106 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 2107 struct socket *sock = po->sk.sk_socket;
2108 struct page *page;
2109 void *data;
2110 int err;
2111
2112 ph.raw = frame;
2113
2114 skb->protocol = proto;
2115 skb->dev = dev;
2116 skb->priority = po->sk.sk_priority;
2d37a186 2117 skb->mark = po->sk.sk_mark;
2e31396f 2118 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2119 skb_shinfo(skb)->destructor_arg = ph.raw;
2120
2121 switch (po->tp_version) {
2122 case TPACKET_V2:
2123 tp_len = ph.h2->tp_len;
2124 break;
2125 default:
2126 tp_len = ph.h1->tp_len;
2127 break;
2128 }
09effa67
DM		 2129 if (unlikely(tp_len > size_max)) {
2130 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2131 return -EMSGSIZE;
2132 }
69e3c75f 2133
ae641949 2134 skb_reserve(skb, hlen);
69e3c75f 2135 skb_reset_network_header(skb);
c1aad275 2136
d346a3fa
DB		 2137 if (!packet_use_direct_xmit(po))
2138 skb_probe_transport_header(skb, 0);
2139 if (unlikely(po->tp_tx_has_off)) {
5920cd3a
PC		 2140 int off_min, off_max, off;
2141 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2142 off_max = po->tx_ring.frame_size - tp_len;
2143 if (sock->type == SOCK_DGRAM) {
2144 switch (po->tp_version) {
2145 case TPACKET_V2:
2146 off = ph.h2->tp_net;
2147 break;
2148 default:
2149 off = ph.h1->tp_net;
2150 break;
2151 }
2152 } else {
2153 switch (po->tp_version) {
2154 case TPACKET_V2:
2155 off = ph.h2->tp_mac;
2156 break;
2157 default:
2158 off = ph.h1->tp_mac;
2159 break;
2160 }
2161 }
2162 if (unlikely((off < off_min) || (off_max < off)))
2163 return -EINVAL;
2164 data = ph.raw + off;
2165 } else {
2166 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2167 }
69e3c75f
JB		 2168 to_write = tp_len;
2169
2170 if (sock->type == SOCK_DGRAM) {
2171 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2172 NULL, tp_len);
2173 if (unlikely(err < 0))
2174 return -EINVAL;
40d4e3df 2175 } else if (dev->hard_header_len) {
69e3c75f
JB		 2176 /* net device doesn't like empty head */
2177 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2178 pr_err("packet size is too short (%d < %d)\n",
2179 tp_len, dev->hard_header_len);
69e3c75f
JB		 2180 return -EINVAL;
2181 }
2182
2183 skb_push(skb, dev->hard_header_len);
2184 err = skb_store_bits(skb, 0, data,
2185 dev->hard_header_len);
2186 if (unlikely(err))
2187 return err;
2188
2189 data += dev->hard_header_len;
2190 to_write -= dev->hard_header_len;
2191 }
2192
69e3c75f
JB		 2193 offset = offset_in_page(data);
2194 len_max = PAGE_SIZE - offset;
2195 len = ((to_write > len_max) ? len_max : to_write);
2196
2197 skb->data_len = to_write;
2198 skb->len += to_write;
2199 skb->truesize += to_write;
2200 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2201
2202 while (likely(to_write)) {
2203 nr_frags = skb_shinfo(skb)->nr_frags;
2204
2205 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2206 pr_err("Packet exceed the number of skb frags(%lu)\n",
2207 MAX_SKB_FRAGS);
69e3c75f
JB		 2208 return -EFAULT;
2209 }
2210
0af55bb5
CG		 2211 page = pgv_to_page(data);
2212 data += len;
69e3c75f
JB		 2213 flush_dcache_page(page);
2214 get_page(page);
0af55bb5 2215 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2216 to_write -= len;
2217 offset = 0;
2218 len_max = PAGE_SIZE;
2219 len = ((to_write > len_max) ? len_max : to_write);
2220 }
2221
2222 return tp_len;
2223}
2224
2225static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2226{
69e3c75f
JB		 2227 struct sk_buff *skb;
2228 struct net_device *dev;
2229 __be16 proto;
09effa67 2230 int err, reserve = 0;
40d4e3df 2231 void *ph;
342dfc30 2232 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2233 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
69e3c75f
JB		 2234 int tp_len, size_max;
2235 unsigned char *addr;
2236 int len_sum = 0;
9e67030a 2237 int status = TP_STATUS_AVAILABLE;
ae641949 2238 int hlen, tlen;
69e3c75f 2239
69e3c75f
JB		 2240 mutex_lock(&po->pg_vec_lock);
2241
66e56cd4 2242 if (likely(saddr == NULL)) {
e40526cb 2243 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2244 proto = po->num;
2245 addr = NULL;
2246 } else {
2247 err = -EINVAL;
2248 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2249 goto out;
2250 if (msg->msg_namelen < (saddr->sll_halen
2251 + offsetof(struct sockaddr_ll,
2252 sll_addr)))
2253 goto out;
69e3c75f
JB		 2254 proto = saddr->sll_protocol;
2255 addr = saddr->sll_addr;
827d9780 2256 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2257 }
2258
69e3c75f
JB		 2259 err = -ENXIO;
2260 if (unlikely(dev == NULL))
2261 goto out;
69e3c75f
JB		 2262 err = -ENETDOWN;
2263 if (unlikely(!(dev->flags & IFF_UP)))
2264 goto out_put;
2265
52f1454f 2266 reserve = dev->hard_header_len + VLAN_HLEN;
69e3c75f 2267 size_max = po->tx_ring.frame_size
b5dd884e 2268 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2269
09effa67
DM		 2270 if (size_max > dev->mtu + reserve)
2271 size_max = dev->mtu + reserve;
2272
69e3c75f
JB		 2273 do {
2274 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2275 TP_STATUS_SEND_REQUEST);
69e3c75f 2276 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2277 if (need_wait && need_resched())
2278 schedule();
69e3c75f
JB		 2279 continue;
2280 }
2281
2282 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2283 hlen = LL_RESERVED_SPACE(dev);
2284 tlen = dev->needed_tailroom;
69e3c75f 2285 skb = sock_alloc_send_skb(&po->sk,
ae641949 2286 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2287 0, &err);
2288
2289 if (unlikely(skb == NULL))
2290 goto out_status;
2291
2292 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
52f1454f
DB		 2293 addr, hlen);
2294 if (tp_len > dev->mtu + dev->hard_header_len) {
2295 struct ethhdr *ehdr;
2296 /* Earlier code assumed this would be a VLAN pkt,
2297 * double-check this now that we have the actual
2298 * packet in hand.
2299 */
69e3c75f 2300
52f1454f
DB		 2301 skb_reset_mac_header(skb);
2302 ehdr = eth_hdr(skb);
2303 if (ehdr->h_proto != htons(ETH_P_8021Q))
2304 tp_len = -EMSGSIZE;
2305 }
69e3c75f
JB		 2306 if (unlikely(tp_len < 0)) {
2307 if (po->tp_loss) {
2308 __packet_set_status(po, ph,
2309 TP_STATUS_AVAILABLE);
2310 packet_increment_head(&po->tx_ring);
2311 kfree_skb(skb);
2312 continue;
2313 } else {
2314 status = TP_STATUS_WRONG_FORMAT;
2315 err = tp_len;
2316 goto out_status;
2317 }
2318 }
2319
0fd5d57b
DB		 2320 packet_pick_tx_queue(dev, skb);
2321
69e3c75f
JB		 2322 skb->destructor = tpacket_destruct_skb;
2323 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2324 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2325
2326 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2327 err = po->xmit(skb);
eb70df13
JP		 2328 if (unlikely(err > 0)) {
2329 err = net_xmit_errno(err);
2330 if (err && __packet_get_status(po, ph) ==
2331 TP_STATUS_AVAILABLE) {
2332 /* skb was destructed already */
2333 skb = NULL;
2334 goto out_status;
2335 }
2336 /*
2337 * skb was dropped but not destructed yet;
2338 * let's treat it like congestion or err < 0
2339 */
2340 err = 0;
2341 }
69e3c75f
JB		 2342 packet_increment_head(&po->tx_ring);
2343 len_sum += tp_len;
b0138408
DB		 2344 } while (likely((ph != NULL) ||
2345 /* Note: packet_read_pending() might be slow if we have
2346 * to call it as it's per_cpu variable, but in fast-path
2347 * we already short-circuit the loop with the first
2348 * condition, and luckily don't have to go that path
2349 * anyway.
2350 */
2351 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2352
2353 err = len_sum;
2354 goto out_put;
2355
69e3c75f
JB		 2356out_status:
2357 __packet_set_status(po, ph, status);
2358 kfree_skb(skb);
2359out_put:
e40526cb 2360 dev_put(dev);
69e3c75f
JB		 2361out:
2362 mutex_unlock(&po->pg_vec_lock);
2363 return err;
2364}
69e3c75f 2365
eea49cc9
OJ		 2366static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2367 size_t reserve, size_t len,
2368 size_t linear, int noblock,
2369 int *err)
bfd5f4a3
SS		 2370{
2371 struct sk_buff *skb;
2372
2373 /* Under a page? Don't bother with paged skb. */
2374 if (prepad + len < PAGE_SIZE || !linear)
2375 linear = len;
2376
2377 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2378 err, 0);
bfd5f4a3
SS		 2379 if (!skb)
2380 return NULL;
2381
2382 skb_reserve(skb, reserve);
2383 skb_put(skb, linear);
2384 skb->data_len = len - linear;
2385 skb->len += len - linear;
2386
2387 return skb;
2388}
2389
d346a3fa 2390static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2391{
2392 struct sock *sk = sock->sk;
342dfc30 2393 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2394 struct sk_buff *skb;
2395 struct net_device *dev;
0e11c91e 2396 __be16 proto;
1da177e4 2397 unsigned char *addr;
827d9780 2398 int err, reserve = 0;
bfd5f4a3
SS		 2399 struct virtio_net_hdr vnet_hdr = { 0 };
2400 int offset = 0;
2401 int vnet_hdr_len;
2402 struct packet_sock *po = pkt_sk(sk);
2403 unsigned short gso_type = 0;
ae641949 2404 int hlen, tlen;
3bdc0eba 2405 int extra_len = 0;
1da177e4
LT		 2406
2407 /*
1ce4f28b 2408 * Get and verify the address.
1da177e4 2409 */
1ce4f28b 2410
66e56cd4 2411 if (likely(saddr == NULL)) {
e40526cb 2412 dev = packet_cached_dev_get(po);
1da177e4
LT		 2413 proto = po->num;
2414 addr = NULL;
2415 } else {
2416 err = -EINVAL;
2417 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2418 goto out;
0fb375fb
EB		 2419 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2420 goto out;
1da177e4
LT		 2421 proto = saddr->sll_protocol;
2422 addr = saddr->sll_addr;
827d9780 2423 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2424 }
2425
1da177e4 2426 err = -ENXIO;
e40526cb 2427 if (unlikely(dev == NULL))
1da177e4 2428 goto out_unlock;
d5e76b0a 2429 err = -ENETDOWN;
e40526cb 2430 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2431 goto out_unlock;
2432
e40526cb
DB		 2433 if (sock->type == SOCK_RAW)
2434 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2435 if (po->has_vnet_hdr) {
2436 vnet_hdr_len = sizeof(vnet_hdr);
2437
2438 err = -EINVAL;
2439 if (len < vnet_hdr_len)
2440 goto out_unlock;
2441
2442 len -= vnet_hdr_len;
2443
2444 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2445 vnet_hdr_len);
2446 if (err < 0)
2447 goto out_unlock;
2448
2449 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2450 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2451 vnet_hdr.hdr_len))
2452 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2453 vnet_hdr.csum_offset + 2;
2454
2455 err = -EINVAL;
2456 if (vnet_hdr.hdr_len > len)
2457 goto out_unlock;
2458
2459 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2460 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2461 case VIRTIO_NET_HDR_GSO_TCPV4:
2462 gso_type = SKB_GSO_TCPV4;
2463 break;
2464 case VIRTIO_NET_HDR_GSO_TCPV6:
2465 gso_type = SKB_GSO_TCPV6;
2466 break;
2467 case VIRTIO_NET_HDR_GSO_UDP:
2468 gso_type = SKB_GSO_UDP;
2469 break;
2470 default:
2471 goto out_unlock;
2472 }
2473
2474 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2475 gso_type |= SKB_GSO_TCP_ECN;
2476
2477 if (vnet_hdr.gso_size == 0)
2478 goto out_unlock;
2479
2480 }
2481 }
2482
3bdc0eba
BG		 2483 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2484 if (!netif_supports_nofcs(dev)) {
2485 err = -EPROTONOSUPPORT;
2486 goto out_unlock;
2487 }
2488 extra_len = 4; /* We're doing our own CRC */
2489 }
2490
1da177e4 2491 err = -EMSGSIZE;
3bdc0eba 2492 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2493 goto out_unlock;
2494
bfd5f4a3 2495 err = -ENOBUFS;
ae641949
HX		 2496 hlen = LL_RESERVED_SPACE(dev);
2497 tlen = dev->needed_tailroom;
2498 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2499 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2500 if (skb == NULL)
1da177e4
LT		 2501 goto out_unlock;
2502
bfd5f4a3 2503 skb_set_network_header(skb, reserve);
1da177e4 2504
0c4e8581
SH		 2505 err = -EINVAL;
2506 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2507 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2508 goto out_free;
1da177e4
LT		 2509
2510 /* Returns -EFAULT on error */
bfd5f4a3 2511 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2512 if (err)
2513 goto out_free;
bf84a010
DB		 2514
2515 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2516
3bdc0eba 2517 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2518 /* Earlier code assumed this would be a VLAN pkt,
2519 * double-check this now that we have the actual
2520 * packet in hand.
2521 */
2522 struct ethhdr *ehdr;
2523 skb_reset_mac_header(skb);
2524 ehdr = eth_hdr(skb);
2525 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2526 err = -EMSGSIZE;
2527 goto out_free;
2528 }
57f89bfa
BG		 2529 }
2530
09effa67
DM		 2531 skb->protocol = proto;
2532 skb->dev = dev;
1da177e4 2533 skb->priority = sk->sk_priority;
2d37a186 2534 skb->mark = sk->sk_mark;
0fd5d57b
DB		 2535
2536 packet_pick_tx_queue(dev, skb);
1da177e4 2537
bfd5f4a3
SS		 2538 if (po->has_vnet_hdr) {
2539 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2540 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2541 vnet_hdr.csum_offset)) {
2542 err = -EINVAL;
2543 goto out_free;
2544 }
2545 }
2546
2547 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2548 skb_shinfo(skb)->gso_type = gso_type;
2549
2550 /* Header must be checked, and gso_segs computed. */
2551 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2552 skb_shinfo(skb)->gso_segs = 0;
2553
2554 len += vnet_hdr_len;
2555 }
2556
d346a3fa
DB		 2557 if (!packet_use_direct_xmit(po))
2558 skb_probe_transport_header(skb, reserve);
3bdc0eba
BG		 2559 if (unlikely(extra_len == 4))
2560 skb->no_fcs = 1;
2561
d346a3fa 2562 err = po->xmit(skb);
1da177e4
LT		 2563 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2564 goto out_unlock;
2565
e40526cb 2566 dev_put(dev);
1da177e4 2567
40d4e3df 2568 return len;
1da177e4
LT		 2569
2570out_free:
2571 kfree_skb(skb);
2572out_unlock:
e40526cb 2573 if (dev)
1da177e4
LT		 2574 dev_put(dev);
2575out:
2576 return err;
2577}
2578
69e3c75f
JB		 2579static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2580 struct msghdr *msg, size_t len)
2581{
69e3c75f
JB		 2582 struct sock *sk = sock->sk;
2583 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2584
69e3c75f
JB		 2585 if (po->tx_ring.pg_vec)
2586 return tpacket_snd(po, msg);
2587 else
69e3c75f
JB		 2588 return packet_snd(sock, msg, len);
2589}
2590
1da177e4
LT		 2591/*
2592 * Close a PACKET socket. This is fairly simple. We immediately go
2593 * to 'closed' state and remove our protocol entry in the device list.
2594 */
2595
2596static int packet_release(struct socket *sock)
2597{
2598 struct sock *sk = sock->sk;
2599 struct packet_sock *po;
d12d01d6 2600 struct net *net;
f6fb8f10 2601 union tpacket_req_u req_u;
1da177e4
LT		 2602
2603 if (!sk)
2604 return 0;
2605
3b1e0a65 2606 net = sock_net(sk);
1da177e4
LT		 2607 po = pkt_sk(sk);
2608
0fa7fa98 2609 mutex_lock(&net->packet.sklist_lock);
808f5114 2610 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2611 mutex_unlock(&net->packet.sklist_lock);
2612
2613 preempt_disable();
920de804 2614 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2615 preempt_enable();
1da177e4 2616
808f5114 2617 spin_lock(&po->bind_lock);
ce06b03e 2618 unregister_prot_hook(sk, false);
66e56cd4
DB		 2619 packet_cached_dev_reset(po);
2620
160ff18a
BG		 2621 if (po->prot_hook.dev) {
2622 dev_put(po->prot_hook.dev);
2623 po->prot_hook.dev = NULL;
2624 }
808f5114 2625 spin_unlock(&po->bind_lock);
1da177e4 2626
1da177e4 2627 packet_flush_mclist(sk);
1da177e4 2628
9665d5d6
PS		 2629 if (po->rx_ring.pg_vec) {
2630 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2631 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2632 }
69e3c75f 2633
9665d5d6
PS		 2634 if (po->tx_ring.pg_vec) {
2635 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2636 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2637 }
1da177e4 2638
dc99f600
DM		 2639 fanout_release(sk);
2640
808f5114 2641 synchronize_net();
1da177e4
LT		 2642 /*
2643 * Now the socket is dead. No more input will appear.
2644 */
1da177e4
LT		 2645 sock_orphan(sk);
2646 sock->sk = NULL;
2647
2648 /* Purge queues */
2649
2650 skb_queue_purge(&sk->sk_receive_queue);
b0138408 2651 packet_free_pending(po);
17ab56a2 2652 sk_refcnt_debug_release(sk);
1da177e4
LT		 2653
2654 sock_put(sk);
2655 return 0;
2656}
2657
2658/*
2659 * Attach a packet hook.
2660 */
2661
902fefb8 2662static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
1da177e4
LT		 2663{
2664 struct packet_sock *po = pkt_sk(sk);
902fefb8
DB		 2665 const struct net_device *dev_curr;
2666 __be16 proto_curr;
2667 bool need_rehook;
dc99f600 2668
aef950b4
WY		 2669 if (po->fanout) {
2670 if (dev)
2671 dev_put(dev);
2672
dc99f600 2673 return -EINVAL;
aef950b4 2674 }
1da177e4
LT		 2675
2676 lock_sock(sk);
1da177e4 2677 spin_lock(&po->bind_lock);
66e56cd4 2678
902fefb8
DB		 2679 proto_curr = po->prot_hook.type;
2680 dev_curr = po->prot_hook.dev;
2681
2682 need_rehook = proto_curr != proto || dev_curr != dev;
2683
2684 if (need_rehook) {
2685 unregister_prot_hook(sk, true);
1da177e4 2686
902fefb8
DB		 2687 po->num = proto;
2688 po->prot_hook.type = proto;
1da177e4 2689
902fefb8
DB		 2690 if (po->prot_hook.dev)
2691 dev_put(po->prot_hook.dev);
2692
2693 po->prot_hook.dev = dev;
2694
2695 po->ifindex = dev ? dev->ifindex : 0;
2696 packet_cached_dev_assign(po, dev);
2697 }
66e56cd4 2698
902fefb8 2699 if (proto == 0 || !need_rehook)
1da177e4
LT		 2700 goto out_unlock;
2701
be85d4ad 2702 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2703 register_prot_hook(sk);
be85d4ad
UT		 2704 } else {
2705 sk->sk_err = ENETDOWN;
2706 if (!sock_flag(sk, SOCK_DEAD))
2707 sk->sk_error_report(sk);
1da177e4
LT		 2708 }
2709
2710out_unlock:
2711 spin_unlock(&po->bind_lock);
2712 release_sock(sk);
2713 return 0;
2714}
2715
2716/*
2717 * Bind a packet socket to a device
2718 */
2719
40d4e3df
ED		 2720static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2721 int addr_len)
1da177e4 2722{
40d4e3df 2723 struct sock *sk = sock->sk;
1da177e4
LT		 2724 char name[15];
2725 struct net_device *dev;
2726 int err = -ENODEV;
1ce4f28b 2727
1da177e4
LT		 2728 /*
2729 * Check legality
2730 */
1ce4f28b 2731
8ae55f04 2732 if (addr_len != sizeof(struct sockaddr))
1da177e4 2733 return -EINVAL;
40d4e3df 2734 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2735
3b1e0a65 2736 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2737 if (dev)
1da177e4 2738 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2739 return err;
2740}
1da177e4
LT		 2741
2742static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2743{
40d4e3df
ED		 2744 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2745 struct sock *sk = sock->sk;
1da177e4
LT		 2746 struct net_device *dev = NULL;
2747 int err;
2748
2749
2750 /*
2751 * Check legality
2752 */
1ce4f28b 2753
1da177e4
LT		 2754 if (addr_len < sizeof(struct sockaddr_ll))
2755 return -EINVAL;
2756 if (sll->sll_family != AF_PACKET)
2757 return -EINVAL;
2758
2759 if (sll->sll_ifindex) {
2760 err = -ENODEV;
3b1e0a65 2761 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2762 if (dev == NULL)
2763 goto out;
2764 }
2765 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2766
2767out:
2768 return err;
2769}
2770
2771static struct proto packet_proto = {
2772 .name = "PACKET",
2773 .owner = THIS_MODULE,
2774 .obj_size = sizeof(struct packet_sock),
2775};
2776
2777/*
1ce4f28b 2778 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2779 */
2780
3f378b68
EP		 2781static int packet_create(struct net *net, struct socket *sock, int protocol,
2782 int kern)
1da177e4
LT		 2783{
2784 struct sock *sk;
2785 struct packet_sock *po;
0e11c91e 2786 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2787 int err;
2788
df008c91 2789 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2790 return -EPERM;
be02097c
DM		 2791 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2792 sock->type != SOCK_PACKET)
1da177e4
LT		 2793 return -ESOCKTNOSUPPORT;
2794
2795 sock->state = SS_UNCONNECTED;
2796
2797 err = -ENOBUFS;
6257ff21 2798 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2799 if (sk == NULL)
2800 goto out;
2801
2802 sock->ops = &packet_ops;
1da177e4
LT		 2803 if (sock->type == SOCK_PACKET)
2804 sock->ops = &packet_ops_spkt;
be02097c 2805
1da177e4
LT		 2806 sock_init_data(sock, sk);
2807
2808 po = pkt_sk(sk);
2809 sk->sk_family = PF_PACKET;
0e11c91e 2810 po->num = proto;
d346a3fa 2811 po->xmit = dev_queue_xmit;
66e56cd4 2812
b0138408
DB		 2813 err = packet_alloc_pending(po);
2814 if (err)
2815 goto out2;
2816
66e56cd4 2817 packet_cached_dev_reset(po);
1da177e4
LT		 2818
2819 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2820 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2821
2822 /*
2823 * Attach a protocol block
2824 */
2825
2826 spin_lock_init(&po->bind_lock);
905db440 2827 mutex_init(&po->pg_vec_lock);
1da177e4 2828 po->prot_hook.func = packet_rcv;
be02097c 2829
1da177e4
LT		 2830 if (sock->type == SOCK_PACKET)
2831 po->prot_hook.func = packet_rcv_spkt;
be02097c 2832
1da177e4
LT		 2833 po->prot_hook.af_packet_priv = sk;
2834
0e11c91e
AV		 2835 if (proto) {
2836 po->prot_hook.type = proto;
ce06b03e 2837 register_prot_hook(sk);
1da177e4
LT		 2838 }
2839
0fa7fa98 2840 mutex_lock(&net->packet.sklist_lock);
808f5114 2841 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2842 mutex_unlock(&net->packet.sklist_lock);
2843
2844 preempt_disable();
3680453c 2845 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2846 preempt_enable();
808f5114 2847
40d4e3df 2848 return 0;
b0138408
DB		 2849out2:
2850 sk_free(sk);
1da177e4
LT		 2851out:
2852 return err;
2853}
2854
2855/*
2856 * Pull a packet from our receive queue and hand it to the user.
2857 * If necessary we block.
2858 */
2859
2860static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2861 struct msghdr *msg, size_t len, int flags)
2862{
2863 struct sock *sk = sock->sk;
2864 struct sk_buff *skb;
2865 int copied, err;
bfd5f4a3 2866 int vnet_hdr_len = 0;
1da177e4
LT		 2867
2868 err = -EINVAL;
ed85b565 2869 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2870 goto out;
2871
2872#if 0
2873 /* What error should we return now? EUNATTACH? */
2874 if (pkt_sk(sk)->ifindex < 0)
2875 return -ENODEV;
2876#endif
2877
ed85b565 2878 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2879 err = sock_recv_errqueue(sk, msg, len,
2880 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2881 goto out;
2882 }
2883
1da177e4
LT		 2884 /*
2885 * Call the generic datagram receiver. This handles all sorts
2886 * of horrible races and re-entrancy so we can forget about it
2887 * in the protocol layers.
2888 *
2889 * Now it will return ENETDOWN, if device have just gone down,
2890 * but then it will block.
2891 */
2892
40d4e3df 2893 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2894
2895 /*
1ce4f28b 2896 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2897 * handles the blocking we don't see and worry about blocking
2898 * retries.
2899 */
2900
8ae55f04 2901 if (skb == NULL)
1da177e4
LT		 2902 goto out;
2903
bfd5f4a3
SS		 2904 if (pkt_sk(sk)->has_vnet_hdr) {
2905 struct virtio_net_hdr vnet_hdr = { 0 };
2906
2907 err = -EINVAL;
2908 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2909 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2910 goto out_free;
2911
1f18b717
MK		 2912 len -= vnet_hdr_len;
2913
bfd5f4a3
SS		 2914 if (skb_is_gso(skb)) {
2915 struct skb_shared_info *sinfo = skb_shinfo(skb);
2916
2917 /* This is a hint as to how much should be linear. */
2918 vnet_hdr.hdr_len = skb_headlen(skb);
2919 vnet_hdr.gso_size = sinfo->gso_size;
2920 if (sinfo->gso_type & SKB_GSO_TCPV4)
2921 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2922 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2923 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2924 else if (sinfo->gso_type & SKB_GSO_UDP)
2925 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2926 else if (sinfo->gso_type & SKB_GSO_FCOE)
2927 goto out_free;
2928 else
2929 BUG();
2930 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2931 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2932 } else
2933 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2934
2935 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2936 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2937 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2938 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2939 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2940 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2941 } /* else everything is zero */
2942
2943 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2944 vnet_hdr_len);
2945 if (err < 0)
2946 goto out_free;
2947 }
2948
f3d33426
HFS		 2949 /* You lose any data beyond the buffer you gave. If it worries
2950 * a user program they can ask the device for its MTU
2951 * anyway.
1da177e4 2952 */
1da177e4 2953 copied = skb->len;
40d4e3df
ED		 2954 if (copied > len) {
2955 copied = len;
2956 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2957 }
2958
2959 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2960 if (err)
2961 goto out_free;
2962
3b885787 2963 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2964
f3d33426
HFS		 2965 if (msg->msg_name) {
2966 /* If the address length field is there to be filled
2967 * in, we fill it in now.
2968 */
2969 if (sock->type == SOCK_PACKET) {
342dfc30 2970 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426
HFS		 2971 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2972 } else {
2973 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2974 msg->msg_namelen = sll->sll_halen +
2975 offsetof(struct sockaddr_ll, sll_addr);
2976 }
ffbc6111
HX		 2977 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2978 msg->msg_namelen);
f3d33426 2979 }
1da177e4 2980
8dc41944 2981 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2982 struct tpacket_auxdata aux;
2983
2984 aux.tp_status = TP_STATUS_USER;
2985 if (skb->ip_summed == CHECKSUM_PARTIAL)
2986 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2987 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2988 aux.tp_snaplen = skb->len;
2989 aux.tp_mac = 0;
bbe735e4 2990 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2991 if (vlan_tx_tag_present(skb)) {
2992 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 2993 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
2994 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2995 } else {
2996 aux.tp_vlan_tci = 0;
a0cdfcf3 2997 aux.tp_vlan_tpid = 0;
a3bcc23e 2998 }
ffbc6111 2999 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 3000 }
3001
1da177e4
LT		 3002 /*
3003 * Free or return the buffer as appropriate. Again this
3004 * hides all the races and re-entrancy issues from us.
3005 */
bfd5f4a3 3006 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 3007
3008out_free:
3009 skb_free_datagram(sk, skb);
3010out:
3011 return err;
3012}
3013
1da177e4
LT		 3014static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3015 int *uaddr_len, int peer)
3016{
3017 struct net_device *dev;
3018 struct sock *sk = sock->sk;
3019
3020 if (peer)
3021 return -EOPNOTSUPP;
3022
3023 uaddr->sa_family = AF_PACKET;
2dc85bf3 3024 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 3025 rcu_read_lock();
3026 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3027 if (dev)
2dc85bf3 3028 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 3029 rcu_read_unlock();
1da177e4
LT		 3030 *uaddr_len = sizeof(*uaddr);
3031
3032 return 0;
3033}
1da177e4
LT		 3034
3035static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3036 int *uaddr_len, int peer)
3037{
3038 struct net_device *dev;
3039 struct sock *sk = sock->sk;
3040 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3041 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3042
3043 if (peer)
3044 return -EOPNOTSUPP;
3045
3046 sll->sll_family = AF_PACKET;
3047 sll->sll_ifindex = po->ifindex;
3048 sll->sll_protocol = po->num;
67286640 3049 sll->sll_pkttype = 0;
654d1f8a
ED		 3050 rcu_read_lock();
3051 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3052 if (dev) {
3053 sll->sll_hatype = dev->type;
3054 sll->sll_halen = dev->addr_len;
3055 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3056 } else {
3057 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3058 sll->sll_halen = 0;
3059 }
654d1f8a 3060 rcu_read_unlock();
0fb375fb 3061 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3062
3063 return 0;
3064}
3065
2aeb0b88
WC		 3066static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3067 int what)
1da177e4
LT		 3068{
3069 switch (i->type) {
3070 case PACKET_MR_MULTICAST:
1162563f
JP		 3071 if (i->alen != dev->addr_len)
3072 return -EINVAL;
1da177e4 3073 if (what > 0)
22bedad3 3074 return dev_mc_add(dev, i->addr);
1da177e4 3075 else
22bedad3 3076 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3077 break;
3078 case PACKET_MR_PROMISC:
2aeb0b88 3079 return dev_set_promiscuity(dev, what);
1da177e4 3080 case PACKET_MR_ALLMULTI:
2aeb0b88 3081 return dev_set_allmulti(dev, what);
d95ed927 3082 case PACKET_MR_UNICAST:
1162563f
JP		 3083 if (i->alen != dev->addr_len)
3084 return -EINVAL;
d95ed927 3085 if (what > 0)
a748ee24 3086 return dev_uc_add(dev, i->addr);
d95ed927 3087 else
a748ee24 3088 return dev_uc_del(dev, i->addr);
d95ed927 3089 break;
40d4e3df
ED		 3090 default:
3091 break;
1da177e4 3092 }
2aeb0b88 3093 return 0;
1da177e4
LT		 3094}
3095
3096static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
3097{
40d4e3df 3098 for ( ; i; i = i->next) {
1da177e4
LT		 3099 if (i->ifindex == dev->ifindex)
3100 packet_dev_mc(dev, i, what);
3101 }
3102}
3103
0fb375fb 3104static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3105{
3106 struct packet_sock *po = pkt_sk(sk);
3107 struct packet_mclist *ml, *i;
3108 struct net_device *dev;
3109 int err;
3110
3111 rtnl_lock();
3112
3113 err = -ENODEV;
3b1e0a65 3114 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3115 if (!dev)
3116 goto done;
3117
3118 err = -EINVAL;
1162563f 3119 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3120 goto done;
3121
3122 err = -ENOBUFS;
8b3a7005 3123 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3124 if (i == NULL)
3125 goto done;
3126
3127 err = 0;
3128 for (ml = po->mclist; ml; ml = ml->next) {
3129 if (ml->ifindex == mreq->mr_ifindex &&
3130 ml->type == mreq->mr_type &&
3131 ml->alen == mreq->mr_alen &&
3132 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3133 ml->count++;
3134 /* Free the new element ... */
3135 kfree(i);
3136 goto done;
3137 }
3138 }
3139
3140 i->type = mreq->mr_type;
3141 i->ifindex = mreq->mr_ifindex;
3142 i->alen = mreq->mr_alen;
3143 memcpy(i->addr, mreq->mr_address, i->alen);
3144 i->count = 1;
3145 i->next = po->mclist;
3146 po->mclist = i;
2aeb0b88
WC		 3147 err = packet_dev_mc(dev, i, 1);
3148 if (err) {
3149 po->mclist = i->next;
3150 kfree(i);
3151 }
1da177e4
LT		 3152
3153done:
3154 rtnl_unlock();
3155 return err;
3156}
3157
0fb375fb 3158static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3159{
3160 struct packet_mclist *ml, **mlp;
3161
3162 rtnl_lock();
3163
3164 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3165 if (ml->ifindex == mreq->mr_ifindex &&
3166 ml->type == mreq->mr_type &&
3167 ml->alen == mreq->mr_alen &&
3168 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3169 if (--ml->count == 0) {
3170 struct net_device *dev;
3171 *mlp = ml->next;
ad959e76
ED		 3172 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3173 if (dev)
1da177e4 3174 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3175 kfree(ml);
3176 }
3177 rtnl_unlock();
3178 return 0;
3179 }
3180 }
3181 rtnl_unlock();
3182 return -EADDRNOTAVAIL;
3183}
3184
3185static void packet_flush_mclist(struct sock *sk)
3186{
3187 struct packet_sock *po = pkt_sk(sk);
3188 struct packet_mclist *ml;
3189
3190 if (!po->mclist)
3191 return;
3192
3193 rtnl_lock();
3194 while ((ml = po->mclist) != NULL) {
3195 struct net_device *dev;
3196
3197 po->mclist = ml->next;
ad959e76
ED		 3198 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3199 if (dev != NULL)
1da177e4 3200 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3201 kfree(ml);
3202 }
3203 rtnl_unlock();
3204}
1da177e4
LT		 3205
3206static int
b7058842 3207packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3208{
3209 struct sock *sk = sock->sk;
8dc41944 3210 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3211 int ret;
3212
3213 if (level != SOL_PACKET)
3214 return -ENOPROTOOPT;
3215
69e3c75f 3216 switch (optname) {
1ce4f28b 3217 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3218 case PACKET_DROP_MEMBERSHIP:
3219 {
0fb375fb
EB		 3220 struct packet_mreq_max mreq;
3221 int len = optlen;
3222 memset(&mreq, 0, sizeof(mreq));
3223 if (len < sizeof(struct packet_mreq))
1da177e4 3224 return -EINVAL;
0fb375fb
EB		 3225 if (len > sizeof(mreq))
3226 len = sizeof(mreq);
40d4e3df 3227 if (copy_from_user(&mreq, optval, len))
1da177e4 3228 return -EFAULT;
0fb375fb
EB		 3229 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3230 return -EINVAL;
1da177e4
LT		 3231 if (optname == PACKET_ADD_MEMBERSHIP)
3232 ret = packet_mc_add(sk, &mreq);
3233 else
3234 ret = packet_mc_drop(sk, &mreq);
3235 return ret;
3236 }
a2efcfa0 3237
1da177e4 3238 case PACKET_RX_RING:
69e3c75f 3239 case PACKET_TX_RING:
1da177e4 3240 {
f6fb8f10 3241 union tpacket_req_u req_u;
3242 int len;
1da177e4 3243
f6fb8f10 3244 switch (po->tp_version) {
3245 case TPACKET_V1:
3246 case TPACKET_V2:
3247 len = sizeof(req_u.req);
3248 break;
3249 case TPACKET_V3:
3250 default:
3251 len = sizeof(req_u.req3);
3252 break;
3253 }
3254 if (optlen < len)
1da177e4 3255 return -EINVAL;
bfd5f4a3
SS		 3256 if (pkt_sk(sk)->has_vnet_hdr)
3257 return -EINVAL;
f6fb8f10 3258 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3259 return -EFAULT;
f6fb8f10 3260 return packet_set_ring(sk, &req_u, 0,
3261 optname == PACKET_TX_RING);
1da177e4
LT		 3262 }
3263 case PACKET_COPY_THRESH:
3264 {
3265 int val;
3266
40d4e3df 3267 if (optlen != sizeof(val))
1da177e4 3268 return -EINVAL;
40d4e3df 3269 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3270 return -EFAULT;
3271
3272 pkt_sk(sk)->copy_thresh = val;
3273 return 0;
3274 }
bbd6ef87
PM		 3275 case PACKET_VERSION:
3276 {
3277 int val;
3278
3279 if (optlen != sizeof(val))
3280 return -EINVAL;
69e3c75f 3281 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3282 return -EBUSY;
3283 if (copy_from_user(&val, optval, sizeof(val)))
3284 return -EFAULT;
3285 switch (val) {
3286 case TPACKET_V1:
3287 case TPACKET_V2:
f6fb8f10 3288 case TPACKET_V3:
bbd6ef87
PM		 3289 po->tp_version = val;
3290 return 0;
3291 default:
3292 return -EINVAL;
3293 }
3294 }
8913336a
PM		 3295 case PACKET_RESERVE:
3296 {
3297 unsigned int val;
3298
3299 if (optlen != sizeof(val))
3300 return -EINVAL;
69e3c75f 3301 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3302 return -EBUSY;
3303 if (copy_from_user(&val, optval, sizeof(val)))
3304 return -EFAULT;
3305 po->tp_reserve = val;
3306 return 0;
3307 }
69e3c75f
JB		 3308 case PACKET_LOSS:
3309 {
3310 unsigned int val;
3311
3312 if (optlen != sizeof(val))
3313 return -EINVAL;
3314 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3315 return -EBUSY;
3316 if (copy_from_user(&val, optval, sizeof(val)))
3317 return -EFAULT;
3318 po->tp_loss = !!val;
3319 return 0;
3320 }
8dc41944
HX		 3321 case PACKET_AUXDATA:
3322 {
3323 int val;
3324
3325 if (optlen < sizeof(val))
3326 return -EINVAL;
3327 if (copy_from_user(&val, optval, sizeof(val)))
3328 return -EFAULT;
3329
3330 po->auxdata = !!val;
3331 return 0;
3332 }
80feaacb
PWJ		 3333 case PACKET_ORIGDEV:
3334 {
3335 int val;
3336
3337 if (optlen < sizeof(val))
3338 return -EINVAL;
3339 if (copy_from_user(&val, optval, sizeof(val)))
3340 return -EFAULT;
3341
3342 po->origdev = !!val;
3343 return 0;
3344 }
bfd5f4a3
SS		 3345 case PACKET_VNET_HDR:
3346 {
3347 int val;
3348
3349 if (sock->type != SOCK_RAW)
3350 return -EINVAL;
3351 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3352 return -EBUSY;
3353 if (optlen < sizeof(val))
3354 return -EINVAL;
3355 if (copy_from_user(&val, optval, sizeof(val)))
3356 return -EFAULT;
3357
3358 po->has_vnet_hdr = !!val;
3359 return 0;
3360 }
614f60fa
SM		 3361 case PACKET_TIMESTAMP:
3362 {
3363 int val;
3364
3365 if (optlen != sizeof(val))
3366 return -EINVAL;
3367 if (copy_from_user(&val, optval, sizeof(val)))
3368 return -EFAULT;
3369
3370 po->tp_tstamp = val;
3371 return 0;
3372 }
dc99f600
DM		 3373 case PACKET_FANOUT:
3374 {
3375 int val;
3376
3377 if (optlen != sizeof(val))
3378 return -EINVAL;
3379 if (copy_from_user(&val, optval, sizeof(val)))
3380 return -EFAULT;
3381
3382 return fanout_add(sk, val & 0xffff, val >> 16);
3383 }
5920cd3a
PC		 3384 case PACKET_TX_HAS_OFF:
3385 {
3386 unsigned int val;
3387
3388 if (optlen != sizeof(val))
3389 return -EINVAL;
3390 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3391 return -EBUSY;
3392 if (copy_from_user(&val, optval, sizeof(val)))
3393 return -EFAULT;
3394 po->tp_tx_has_off = !!val;
3395 return 0;
3396 }
d346a3fa
DB		 3397 case PACKET_QDISC_BYPASS:
3398 {
3399 int val;
3400
3401 if (optlen != sizeof(val))
3402 return -EINVAL;
3403 if (copy_from_user(&val, optval, sizeof(val)))
3404 return -EFAULT;
3405
3406 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3407 return 0;
3408 }
1da177e4
LT		 3409 default:
3410 return -ENOPROTOOPT;
3411 }
3412}
3413
3414static int packet_getsockopt(struct socket *sock, int level, int optname,
3415 char __user *optval, int __user *optlen)
3416{
3417 int len;
c06fff6e 3418 int val, lv = sizeof(val);
1da177e4
LT		 3419 struct sock *sk = sock->sk;
3420 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3421 void *data = &val;
ee80fbf3 3422 union tpacket_stats_u st;
1da177e4
LT		 3423
3424 if (level != SOL_PACKET)
3425 return -ENOPROTOOPT;
3426
8ae55f04
KK		 3427 if (get_user(len, optlen))
3428 return -EFAULT;
1da177e4
LT		 3429
3430 if (len < 0)
3431 return -EINVAL;
1ce4f28b 3432
69e3c75f 3433 switch (optname) {
1da177e4 3434 case PACKET_STATISTICS:
1da177e4 3435 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3436 memcpy(&st, &po->stats, sizeof(st));
3437 memset(&po->stats, 0, sizeof(po->stats));
3438 spin_unlock_bh(&sk->sk_receive_queue.lock);
3439
f6fb8f10 3440 if (po->tp_version == TPACKET_V3) {
c06fff6e 3441 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3442 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3443 data = &st.stats3;
f6fb8f10 3444 } else {
c06fff6e 3445 lv = sizeof(struct tpacket_stats);
8bcdeaff 3446 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3447 data = &st.stats1;
f6fb8f10 3448 }
ee80fbf3 3449
8dc41944
HX		 3450 break;
3451 case PACKET_AUXDATA:
8dc41944 3452 val = po->auxdata;
80feaacb
PWJ		 3453 break;
3454 case PACKET_ORIGDEV:
80feaacb 3455 val = po->origdev;
bfd5f4a3
SS		 3456 break;
3457 case PACKET_VNET_HDR:
bfd5f4a3 3458 val = po->has_vnet_hdr;
1da177e4 3459 break;
bbd6ef87 3460 case PACKET_VERSION:
bbd6ef87 3461 val = po->tp_version;
bbd6ef87
PM		 3462 break;
3463 case PACKET_HDRLEN:
3464 if (len > sizeof(int))
3465 len = sizeof(int);
3466 if (copy_from_user(&val, optval, len))
3467 return -EFAULT;
3468 switch (val) {
3469 case TPACKET_V1:
3470 val = sizeof(struct tpacket_hdr);
3471 break;
3472 case TPACKET_V2:
3473 val = sizeof(struct tpacket2_hdr);
3474 break;
f6fb8f10 3475 case TPACKET_V3:
3476 val = sizeof(struct tpacket3_hdr);
3477 break;
bbd6ef87
PM		 3478 default:
3479 return -EINVAL;
3480 }
bbd6ef87 3481 break;
8913336a 3482 case PACKET_RESERVE:
8913336a 3483 val = po->tp_reserve;
8913336a 3484 break;
69e3c75f 3485 case PACKET_LOSS:
69e3c75f 3486 val = po->tp_loss;
69e3c75f 3487 break;
614f60fa 3488 case PACKET_TIMESTAMP:
614f60fa 3489 val = po->tp_tstamp;
614f60fa 3490 break;
dc99f600 3491 case PACKET_FANOUT:
dc99f600
DM		 3492 val = (po->fanout ?
3493 ((u32)po->fanout->id |
77f65ebd
WB		 3494 ((u32)po->fanout->type << 16) |
3495 ((u32)po->fanout->flags << 24)) :
dc99f600 3496 0);
dc99f600 3497 break;
5920cd3a
PC		 3498 case PACKET_TX_HAS_OFF:
3499 val = po->tp_tx_has_off;
3500 break;
d346a3fa
DB		 3501 case PACKET_QDISC_BYPASS:
3502 val = packet_use_direct_xmit(po);
3503 break;
1da177e4
LT		 3504 default:
3505 return -ENOPROTOOPT;
3506 }
3507
c06fff6e
ED		 3508 if (len > lv)
3509 len = lv;
8ae55f04
KK		 3510 if (put_user(len, optlen))
3511 return -EFAULT;
8dc41944
HX		 3512 if (copy_to_user(optval, data, len))
3513 return -EFAULT;
8ae55f04 3514 return 0;
1da177e4
LT		 3515}
3516
3517
351638e7
JP		 3518static int packet_notifier(struct notifier_block *this,
3519 unsigned long msg, void *ptr)
1da177e4
LT		 3520{
3521 struct sock *sk;
351638e7 3522 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3523 struct net *net = dev_net(dev);
1da177e4 3524
808f5114 3525 rcu_read_lock();
b67bfe0d 3526 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3527 struct packet_sock *po = pkt_sk(sk);
3528
3529 switch (msg) {
3530 case NETDEV_UNREGISTER:
1da177e4
LT		 3531 if (po->mclist)
3532 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3533 /* fallthrough */
3534
1da177e4
LT		 3535 case NETDEV_DOWN:
3536 if (dev->ifindex == po->ifindex) {
3537 spin_lock(&po->bind_lock);
3538 if (po->running) {
ce06b03e 3539 __unregister_prot_hook(sk, false);
1da177e4
LT		 3540 sk->sk_err = ENETDOWN;
3541 if (!sock_flag(sk, SOCK_DEAD))
3542 sk->sk_error_report(sk);
3543 }
3544 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3545 packet_cached_dev_reset(po);
1da177e4 3546 po->ifindex = -1;
160ff18a
BG		 3547 if (po->prot_hook.dev)
3548 dev_put(po->prot_hook.dev);
1da177e4
LT		 3549 po->prot_hook.dev = NULL;
3550 }
3551 spin_unlock(&po->bind_lock);
3552 }
3553 break;
3554 case NETDEV_UP:
808f5114 3555 if (dev->ifindex == po->ifindex) {
3556 spin_lock(&po->bind_lock);
ce06b03e
DM		 3557 if (po->num)
3558 register_prot_hook(sk);
808f5114 3559 spin_unlock(&po->bind_lock);
1da177e4 3560 }
1da177e4
LT		 3561 break;
3562 }
3563 }
808f5114 3564 rcu_read_unlock();
1da177e4
LT		 3565 return NOTIFY_DONE;
3566}
3567
3568
3569static int packet_ioctl(struct socket *sock, unsigned int cmd,
3570 unsigned long arg)
3571{
3572 struct sock *sk = sock->sk;
3573
69e3c75f 3574 switch (cmd) {
40d4e3df
ED		 3575 case SIOCOUTQ:
3576 {
3577 int amount = sk_wmem_alloc_get(sk);
31e6d363 3578
40d4e3df
ED		 3579 return put_user(amount, (int __user *)arg);
3580 }
3581 case SIOCINQ:
3582 {
3583 struct sk_buff *skb;
3584 int amount = 0;
3585
3586 spin_lock_bh(&sk->sk_receive_queue.lock);
3587 skb = skb_peek(&sk->sk_receive_queue);
3588 if (skb)
3589 amount = skb->len;
3590 spin_unlock_bh(&sk->sk_receive_queue.lock);
3591 return put_user(amount, (int __user *)arg);
3592 }
3593 case SIOCGSTAMP:
3594 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3595 case SIOCGSTAMPNS:
3596 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3597
1da177e4 3598#ifdef CONFIG_INET
40d4e3df
ED		 3599 case SIOCADDRT:
3600 case SIOCDELRT:
3601 case SIOCDARP:
3602 case SIOCGARP:
3603 case SIOCSARP:
3604 case SIOCGIFADDR:
3605 case SIOCSIFADDR:
3606 case SIOCGIFBRDADDR:
3607 case SIOCSIFBRDADDR:
3608 case SIOCGIFNETMASK:
3609 case SIOCSIFNETMASK:
3610 case SIOCGIFDSTADDR:
3611 case SIOCSIFDSTADDR:
3612 case SIOCSIFFLAGS:
40d4e3df 3613 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3614#endif
3615
40d4e3df
ED		 3616 default:
3617 return -ENOIOCTLCMD;
1da177e4
LT		 3618 }
3619 return 0;
3620}
3621
40d4e3df 3622static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3623 poll_table *wait)
3624{
3625 struct sock *sk = sock->sk;
3626 struct packet_sock *po = pkt_sk(sk);
3627 unsigned int mask = datagram_poll(file, sock, wait);
3628
3629 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3630 if (po->rx_ring.pg_vec) {
f6fb8f10 3631 if (!packet_previous_rx_frame(po, &po->rx_ring,
3632 TP_STATUS_KERNEL))
1da177e4
LT		 3633 mask |= POLLIN | POLLRDNORM;
3634 }
3635 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3636 spin_lock_bh(&sk->sk_write_queue.lock);
3637 if (po->tx_ring.pg_vec) {
3638 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3639 mask |= POLLOUT | POLLWRNORM;
3640 }
3641 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3642 return mask;
3643}
3644
3645
3646/* Dirty? Well, I still did not learn better way to account
3647 * for user mmaps.
3648 */
3649
3650static void packet_mm_open(struct vm_area_struct *vma)
3651{
3652 struct file *file = vma->vm_file;
40d4e3df 3653 struct socket *sock = file->private_data;
1da177e4 3654 struct sock *sk = sock->sk;
1ce4f28b 3655
1da177e4
LT		 3656 if (sk)
3657 atomic_inc(&pkt_sk(sk)->mapped);
3658}
3659
3660static void packet_mm_close(struct vm_area_struct *vma)
3661{
3662 struct file *file = vma->vm_file;
40d4e3df 3663 struct socket *sock = file->private_data;
1da177e4 3664 struct sock *sk = sock->sk;
1ce4f28b 3665
1da177e4
LT		 3666 if (sk)
3667 atomic_dec(&pkt_sk(sk)->mapped);
3668}
3669
f0f37e2f 3670static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3671 .open = packet_mm_open,
3672 .close = packet_mm_close,
1da177e4
LT		 3673};
3674
0e3125c7
NH		 3675static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3676 unsigned int len)
1da177e4
LT		 3677{
3678 int i;
3679
4ebf0ae2 3680 for (i = 0; i < len; i++) {
0e3125c7 3681 if (likely(pg_vec[i].buffer)) {
c56b4d90 3682 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3683 vfree(pg_vec[i].buffer);
3684 else
3685 free_pages((unsigned long)pg_vec[i].buffer,
3686 order);
3687 pg_vec[i].buffer = NULL;
3688 }
1da177e4
LT		 3689 }
3690 kfree(pg_vec);
3691}
3692
eea49cc9 3693static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3694{
f0d4eb29 3695 char *buffer;
0e3125c7
NH		 3696 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3697 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3698
3699 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3700 if (buffer)
3701 return buffer;
3702
f0d4eb29 3703 /* __get_free_pages failed, fall back to vmalloc */
bbce5a59 3704 buffer = vzalloc((1 << order) * PAGE_SIZE);
0e3125c7
NH		 3705 if (buffer)
3706 return buffer;
3707
f0d4eb29 3708 /* vmalloc failed, lets dig into swap here */
0e3125c7 3709 gfp_flags &= ~__GFP_NORETRY;
f0d4eb29 3710 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3711 if (buffer)
3712 return buffer;
3713
f0d4eb29 3714 /* complete and utter failure */
0e3125c7 3715 return NULL;
4ebf0ae2
DM		 3716}
3717
0e3125c7 3718static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3719{
3720 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3721 struct pgv *pg_vec;
4ebf0ae2
DM		 3722 int i;
3723
0e3125c7 3724 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3725 if (unlikely(!pg_vec))
3726 goto out;
3727
3728 for (i = 0; i < block_nr; i++) {
c56b4d90 3729 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3730 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3731 goto out_free_pgvec;
3732 }
3733
3734out:
3735 return pg_vec;
3736
3737out_free_pgvec:
3738 free_pg_vec(pg_vec, order, block_nr);
3739 pg_vec = NULL;
3740 goto out;
3741}
1da177e4 3742
f6fb8f10 3743static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3744 int closing, int tx_ring)
1da177e4 3745{
0e3125c7 3746 struct pgv *pg_vec = NULL;
1da177e4 3747 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3748 int was_running, order = 0;
69e3c75f
JB		 3749 struct packet_ring_buffer *rb;
3750 struct sk_buff_head *rb_queue;
0e11c91e 3751 __be16 num;
f6fb8f10 3752 int err = -EINVAL;
3753 /* Added to avoid minimal code churn */
3754 struct tpacket_req *req = &req_u->req;
3755
3756 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3757 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3758 WARN(1, "Tx-ring is not supported.\n");
3759 goto out;
3760 }
1ce4f28b 3761
69e3c75f
JB		 3762 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3763 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3764
69e3c75f
JB		 3765 err = -EBUSY;
3766 if (!closing) {
3767 if (atomic_read(&po->mapped))
3768 goto out;
b0138408 3769 if (packet_read_pending(rb))
69e3c75f
JB		 3770 goto out;
3771 }
1da177e4 3772
69e3c75f
JB		 3773 if (req->tp_block_nr) {
3774 /* Sanity tests and some calculations */
3775 err = -EBUSY;
3776 if (unlikely(rb->pg_vec))
3777 goto out;
1da177e4 3778
bbd6ef87
PM		 3779 switch (po->tp_version) {
3780 case TPACKET_V1:
3781 po->tp_hdrlen = TPACKET_HDRLEN;
3782 break;
3783 case TPACKET_V2:
3784 po->tp_hdrlen = TPACKET2_HDRLEN;
3785 break;
f6fb8f10 3786 case TPACKET_V3:
3787 po->tp_hdrlen = TPACKET3_HDRLEN;
3788 break;
bbd6ef87
PM		 3789 }
3790
69e3c75f 3791 err = -EINVAL;
4ebf0ae2 3792 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3793 goto out;
4ebf0ae2 3794 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3795 goto out;
dc808110
ED		 3796 if (po->tp_version >= TPACKET_V3 &&
3797 (int)(req->tp_block_size -
3798 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
3799 goto out;
8913336a 3800 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3801 po->tp_reserve))
3802 goto out;
4ebf0ae2 3803 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3804 goto out;
1da177e4 3805
69e3c75f
JB		 3806 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3807 if (unlikely(rb->frames_per_block <= 0))
3808 goto out;
3809 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3810 req->tp_frame_nr))
3811 goto out;
1da177e4
LT		 3812
3813 err = -ENOMEM;
4ebf0ae2
DM		 3814 order = get_order(req->tp_block_size);
3815 pg_vec = alloc_pg_vec(req, order);
3816 if (unlikely(!pg_vec))
1da177e4 3817 goto out;
f6fb8f10 3818 switch (po->tp_version) {
3819 case TPACKET_V3:
3820 /* Transmit path is not supported. We checked
3821 * it above but just being paranoid
3822 */
3823 if (!tx_ring)
3824 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
d7cf0c34 3825 break;
f6fb8f10 3826 default:
3827 break;
3828 }
69e3c75f
JB		 3829 }
3830 /* Done */
3831 else {
3832 err = -EINVAL;
4ebf0ae2 3833 if (unlikely(req->tp_frame_nr))
69e3c75f 3834 goto out;
1da177e4
LT		 3835 }
3836
3837 lock_sock(sk);
3838
3839 /* Detach socket from network */
3840 spin_lock(&po->bind_lock);
3841 was_running = po->running;
3842 num = po->num;
3843 if (was_running) {
1da177e4 3844 po->num = 0;
ce06b03e 3845 __unregister_prot_hook(sk, false);
1da177e4
LT		 3846 }
3847 spin_unlock(&po->bind_lock);
1ce4f28b 3848
1da177e4
LT		 3849 synchronize_net();
3850
3851 err = -EBUSY;
905db440 3852 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3853 if (closing || atomic_read(&po->mapped) == 0) {
3854 err = 0;
69e3c75f 3855 spin_lock_bh(&rb_queue->lock);
c053fd96 3856 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3857 rb->frame_max = (req->tp_frame_nr - 1);
3858 rb->head = 0;
3859 rb->frame_size = req->tp_frame_size;
3860 spin_unlock_bh(&rb_queue->lock);
3861
c053fd96
CG		 3862 swap(rb->pg_vec_order, order);
3863 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3864
3865 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3866 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3867 tpacket_rcv : packet_rcv;
3868 skb_queue_purge(rb_queue);
1da177e4 3869 if (atomic_read(&po->mapped))
40d4e3df
ED		 3870 pr_err("packet_mmap: vma is busy: %d\n",
3871 atomic_read(&po->mapped));
1da177e4 3872 }
905db440 3873 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3874
3875 spin_lock(&po->bind_lock);
ce06b03e 3876 if (was_running) {
1da177e4 3877 po->num = num;
ce06b03e 3878 register_prot_hook(sk);
1da177e4
LT		 3879 }
3880 spin_unlock(&po->bind_lock);
f6fb8f10 3881 if (closing && (po->tp_version > TPACKET_V2)) {
3882 /* Because we don't support block-based V3 on tx-ring */
3883 if (!tx_ring)
3884 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3885 }
1da177e4
LT		 3886 release_sock(sk);
3887
1da177e4
LT		 3888 if (pg_vec)
3889 free_pg_vec(pg_vec, order, req->tp_block_nr);
3890out:
3891 return err;
3892}
3893
69e3c75f
JB		 3894static int packet_mmap(struct file *file, struct socket *sock,
3895 struct vm_area_struct *vma)
1da177e4
LT		 3896{
3897 struct sock *sk = sock->sk;
3898 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3899 unsigned long size, expected_size;
3900 struct packet_ring_buffer *rb;
1da177e4
LT		 3901 unsigned long start;
3902 int err = -EINVAL;
3903 int i;
3904
3905 if (vma->vm_pgoff)
3906 return -EINVAL;
3907
905db440 3908 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3909
3910 expected_size = 0;
3911 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3912 if (rb->pg_vec) {
3913 expected_size += rb->pg_vec_len
3914 * rb->pg_vec_pages
3915 * PAGE_SIZE;
3916 }
3917 }
3918
3919 if (expected_size == 0)
1da177e4 3920 goto out;
69e3c75f
JB		 3921
3922 size = vma->vm_end - vma->vm_start;
3923 if (size != expected_size)
1da177e4
LT		 3924 goto out;
3925
1da177e4 3926 start = vma->vm_start;
69e3c75f
JB		 3927 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3928 if (rb->pg_vec == NULL)
3929 continue;
3930
3931 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3932 struct page *page;
3933 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3934 int pg_num;
3935
c56b4d90
CG		 3936 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3937 page = pgv_to_page(kaddr);
69e3c75f
JB		 3938 err = vm_insert_page(vma, start, page);
3939 if (unlikely(err))
3940 goto out;
3941 start += PAGE_SIZE;
0e3125c7 3942 kaddr += PAGE_SIZE;
69e3c75f 3943 }
4ebf0ae2 3944 }
1da177e4 3945 }
69e3c75f 3946
4ebf0ae2 3947 atomic_inc(&po->mapped);
1da177e4
LT		 3948 vma->vm_ops = &packet_mmap_ops;
3949 err = 0;
3950
3951out:
905db440 3952 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3953 return err;
3954}
1da177e4 3955
90ddc4f0 3956static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3957 .family = PF_PACKET,
3958 .owner = THIS_MODULE,
3959 .release = packet_release,
3960 .bind = packet_bind_spkt,
3961 .connect = sock_no_connect,
3962 .socketpair = sock_no_socketpair,
3963 .accept = sock_no_accept,
3964 .getname = packet_getname_spkt,
3965 .poll = datagram_poll,
3966 .ioctl = packet_ioctl,
3967 .listen = sock_no_listen,
3968 .shutdown = sock_no_shutdown,
3969 .setsockopt = sock_no_setsockopt,
3970 .getsockopt = sock_no_getsockopt,
3971 .sendmsg = packet_sendmsg_spkt,
3972 .recvmsg = packet_recvmsg,
3973 .mmap = sock_no_mmap,
3974 .sendpage = sock_no_sendpage,
3975};
1da177e4 3976
90ddc4f0 3977static const struct proto_ops packet_ops = {
1da177e4
LT		 3978 .family = PF_PACKET,
3979 .owner = THIS_MODULE,
3980 .release = packet_release,
3981 .bind = packet_bind,
3982 .connect = sock_no_connect,
3983 .socketpair = sock_no_socketpair,
3984 .accept = sock_no_accept,
1ce4f28b 3985 .getname = packet_getname,
1da177e4
LT		 3986 .poll = packet_poll,
3987 .ioctl = packet_ioctl,
3988 .listen = sock_no_listen,
3989 .shutdown = sock_no_shutdown,
3990 .setsockopt = packet_setsockopt,
3991 .getsockopt = packet_getsockopt,
3992 .sendmsg = packet_sendmsg,
3993 .recvmsg = packet_recvmsg,
3994 .mmap = packet_mmap,
3995 .sendpage = sock_no_sendpage,
3996};
3997
ec1b4cf7 3998static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3999 .family = PF_PACKET,
4000 .create = packet_create,
4001 .owner = THIS_MODULE,
4002};
4003
4004static struct notifier_block packet_netdev_notifier = {
40d4e3df 4005 .notifier_call = packet_notifier,
1da177e4
LT		 4006};
4007
4008#ifdef CONFIG_PROC_FS
1da177e4
LT		 4009
4010static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 4011 __acquires(RCU)
1da177e4 4012{
e372c414 4013 struct net *net = seq_file_net(seq);
808f5114 4014
4015 rcu_read_lock();
4016 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 4017}
4018
4019static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4020{
1bf40954 4021 struct net *net = seq_file_net(seq);
808f5114 4022 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 4023}
4024
4025static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 4026 __releases(RCU)
1da177e4 4027{
808f5114 4028 rcu_read_unlock();
1da177e4
LT		 4029}
4030
1ce4f28b 4031static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 4032{
4033 if (v == SEQ_START_TOKEN)
4034 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4035 else {
b7ceabd9 4036 struct sock *s = sk_entry(v);
1da177e4
LT		 4037 const struct packet_sock *po = pkt_sk(s);
4038
4039 seq_printf(seq,
71338aa7 4040 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 4041 s,
4042 atomic_read(&s->sk_refcnt),
4043 s->sk_type,
4044 ntohs(po->num),
4045 po->ifindex,
4046 po->running,
4047 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4048 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4049 sock_i_ino(s));
1da177e4
LT		 4050 }
4051
4052 return 0;
4053}
4054
56b3d975 4055static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4056 .start = packet_seq_start,
4057 .next = packet_seq_next,
4058 .stop = packet_seq_stop,
4059 .show = packet_seq_show,
4060};
4061
4062static int packet_seq_open(struct inode *inode, struct file *file)
4063{
e372c414
DL		 4064 return seq_open_net(inode, file, &packet_seq_ops,
4065 sizeof(struct seq_net_private));
1da177e4
LT		 4066}
4067
da7071d7 4068static const struct file_operations packet_seq_fops = {
1da177e4
LT		 4069 .owner = THIS_MODULE,
4070 .open = packet_seq_open,
4071 .read = seq_read,
4072 .llseek = seq_lseek,
e372c414 4073 .release = seq_release_net,
1da177e4
LT		 4074};
4075
4076#endif
4077
2c8c1e72 4078static int __net_init packet_net_init(struct net *net)
d12d01d6 4079{
0fa7fa98 4080 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4081 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4082
d4beaa66 4083 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 4084 return -ENOMEM;
4085
4086 return 0;
4087}
4088
2c8c1e72 4089static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4090{
ece31ffd 4091 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 4092}
4093
4094static struct pernet_operations packet_net_ops = {
4095 .init = packet_net_init,
4096 .exit = packet_net_exit,
4097};
4098
4099
1da177e4
LT		 4100static void __exit packet_exit(void)
4101{
1da177e4 4102 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4103 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4104 sock_unregister(PF_PACKET);
4105 proto_unregister(&packet_proto);
4106}
4107
4108static int __init packet_init(void)
4109{
4110 int rc = proto_register(&packet_proto, 0);
4111
4112 if (rc != 0)
4113 goto out;
4114
4115 sock_register(&packet_family_ops);
d12d01d6 4116 register_pernet_subsys(&packet_net_ops);
1da177e4 4117 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4118out:
4119 return rc;
4120}
4121
4122module_init(packet_init);
4123module_exit(packet_exit);
4124MODULE_LICENSE("GPL");
4125MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 1.108831 seconds and 5 git commands to generate.