projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: nlmon: flag nlmon devs with LLTX/SG
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
b0138408 91#include <linux/percpu.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
d346a3fa
DB		 240static int packet_direct_xmit(struct sk_buff *skb)
241{
242 struct net_device *dev = skb->dev;
243 const struct net_device_ops *ops = dev->netdev_ops;
244 netdev_features_t features;
245 struct netdev_queue *txq;
246 u16 queue_map;
247 int ret;
248
249 if (unlikely(!netif_running(dev) ||
250 !netif_carrier_ok(dev))) {
251 kfree_skb(skb);
252 return NET_XMIT_DROP;
253 }
254
255 features = netif_skb_features(skb);
256 if (skb_needs_linearize(skb, features) &&
257 __skb_linearize(skb)) {
258 kfree_skb(skb);
259 return NET_XMIT_DROP;
260 }
261
262 queue_map = skb_get_queue_mapping(skb);
263 txq = netdev_get_tx_queue(dev, queue_map);
264
265 __netif_tx_lock_bh(txq);
266 if (unlikely(netif_xmit_frozen_or_stopped(txq))) {
267 ret = NETDEV_TX_BUSY;
268 kfree_skb(skb);
269 goto out;
270 }
271
272 ret = ops->ndo_start_xmit(skb, dev);
273 if (likely(dev_xmit_complete(ret)))
274 txq_trans_update(txq);
275 else
276 kfree_skb(skb);
277out:
278 __netif_tx_unlock_bh(txq);
279 return ret;
280}
281
66e56cd4
DB		 282static struct net_device *packet_cached_dev_get(struct packet_sock *po)
283{
284 struct net_device *dev;
285
286 rcu_read_lock();
287 dev = rcu_dereference(po->cached_dev);
288 if (likely(dev))
289 dev_hold(dev);
290 rcu_read_unlock();
291
292 return dev;
293}
294
295static void packet_cached_dev_assign(struct packet_sock *po,
296 struct net_device *dev)
297{
298 rcu_assign_pointer(po->cached_dev, dev);
299}
300
301static void packet_cached_dev_reset(struct packet_sock *po)
302{
303 RCU_INIT_POINTER(po->cached_dev, NULL);
304}
305
d346a3fa
DB		 306static bool packet_use_direct_xmit(const struct packet_sock *po)
307{
308 return po->xmit == packet_direct_xmit;
309}
310
0fd5d57b 311static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
d346a3fa 312{
1cbac010 313 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 314}
315
0fd5d57b
DB		 316static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
317{
318 const struct net_device_ops *ops = dev->netdev_ops;
319 u16 queue_index;
320
321 if (ops->ndo_select_queue) {
322 queue_index = ops->ndo_select_queue(dev, skb, NULL,
323 __packet_pick_tx_queue);
324 queue_index = netdev_cap_txqueue(dev, queue_index);
325 } else {
326 queue_index = __packet_pick_tx_queue(dev, skb);
327 }
328
329 skb_set_queue_mapping(skb, queue_index);
330}
331
ce06b03e
DM		 332/* register_prot_hook must be invoked with the po->bind_lock held,
333 * or from a context in which asynchronous accesses to the packet
334 * socket is not possible (packet_create()).
335 */
336static void register_prot_hook(struct sock *sk)
337{
338 struct packet_sock *po = pkt_sk(sk);
e40526cb 339
ce06b03e 340 if (!po->running) {
66e56cd4 341 if (po->fanout)
dc99f600 342 __fanout_link(sk, po);
66e56cd4 343 else
dc99f600 344 dev_add_pack(&po->prot_hook);
e40526cb 345
ce06b03e
DM		 346 sock_hold(sk);
347 po->running = 1;
348 }
349}
350
351/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
352 * held. If the sync parameter is true, we will temporarily drop
353 * the po->bind_lock and do a synchronize_net to make sure no
354 * asynchronous packet processing paths still refer to the elements
355 * of po->prot_hook. If the sync parameter is false, it is the
356 * callers responsibility to take care of this.
357 */
358static void __unregister_prot_hook(struct sock *sk, bool sync)
359{
360 struct packet_sock *po = pkt_sk(sk);
361
362 po->running = 0;
66e56cd4
DB		 363
364 if (po->fanout)
dc99f600 365 __fanout_unlink(sk, po);
66e56cd4 366 else
dc99f600 367 __dev_remove_pack(&po->prot_hook);
e40526cb 368
ce06b03e
DM		 369 __sock_put(sk);
370
371 if (sync) {
372 spin_unlock(&po->bind_lock);
373 synchronize_net();
374 spin_lock(&po->bind_lock);
375 }
376}
377
378static void unregister_prot_hook(struct sock *sk, bool sync)
379{
380 struct packet_sock *po = pkt_sk(sk);
381
382 if (po->running)
383 __unregister_prot_hook(sk, sync);
384}
385
f6dafa95 386static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 387{
388 if (is_vmalloc_addr(addr))
389 return vmalloc_to_page(addr);
390 return virt_to_page(addr);
391}
392
69e3c75f 393static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 394{
184f489e 395 union tpacket_uhdr h;
1da177e4 396
69e3c75f 397 h.raw = frame;
bbd6ef87
PM		 398 switch (po->tp_version) {
399 case TPACKET_V1:
69e3c75f 400 h.h1->tp_status = status;
0af55bb5 401 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 402 break;
403 case TPACKET_V2:
69e3c75f 404 h.h2->tp_status = status;
0af55bb5 405 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 406 break;
f6fb8f10 407 case TPACKET_V3:
69e3c75f 408 default:
f6fb8f10 409 WARN(1, "TPACKET version not supported.\n");
69e3c75f 410 BUG();
bbd6ef87 411 }
69e3c75f
JB		 412
413 smp_wmb();
bbd6ef87
PM		 414}
415
69e3c75f 416static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 417{
184f489e 418 union tpacket_uhdr h;
bbd6ef87 419
69e3c75f
JB		 420 smp_rmb();
421
bbd6ef87
PM		 422 h.raw = frame;
423 switch (po->tp_version) {
424 case TPACKET_V1:
0af55bb5 425 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 426 return h.h1->tp_status;
bbd6ef87 427 case TPACKET_V2:
0af55bb5 428 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 429 return h.h2->tp_status;
f6fb8f10 430 case TPACKET_V3:
69e3c75f 431 default:
f6fb8f10 432 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 433 BUG();
434 return 0;
bbd6ef87 435 }
1da177e4 436}
69e3c75f 437
b9c32fb2
DB		 438static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
439 unsigned int flags)
7a51384c
DB		 440{
441 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
442
443 if (shhwtstamps) {
444 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
445 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
b9c32fb2 446 return TP_STATUS_TS_SYS_HARDWARE;
7a51384c
DB		 447 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
448 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
b9c32fb2 449 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 450 }
451
452 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 453 return TP_STATUS_TS_SOFTWARE;
7a51384c 454
b9c32fb2 455 return 0;
7a51384c
DB		 456}
457
b9c32fb2
DB		 458static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
459 struct sk_buff *skb)
2e31396f
WB		 460{
461 union tpacket_uhdr h;
462 struct timespec ts;
b9c32fb2 463 __u32 ts_status;
2e31396f 464
b9c32fb2
DB		 465 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
466 return 0;
2e31396f
WB		 467
468 h.raw = frame;
469 switch (po->tp_version) {
470 case TPACKET_V1:
471 h.h1->tp_sec = ts.tv_sec;
472 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
473 break;
474 case TPACKET_V2:
475 h.h2->tp_sec = ts.tv_sec;
476 h.h2->tp_nsec = ts.tv_nsec;
477 break;
478 case TPACKET_V3:
479 default:
480 WARN(1, "TPACKET version not supported.\n");
481 BUG();
482 }
483
484 /* one flush is safe, as both fields always lie on the same cacheline */
485 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
486 smp_wmb();
b9c32fb2
DB		 487
488 return ts_status;
2e31396f
WB		 489}
490
69e3c75f
JB		 491static void *packet_lookup_frame(struct packet_sock *po,
492 struct packet_ring_buffer *rb,
493 unsigned int position,
494 int status)
495{
496 unsigned int pg_vec_pos, frame_offset;
184f489e 497 union tpacket_uhdr h;
69e3c75f
JB		 498
499 pg_vec_pos = position / rb->frames_per_block;
500 frame_offset = position % rb->frames_per_block;
501
0e3125c7
NH		 502 h.raw = rb->pg_vec[pg_vec_pos].buffer +
503 (frame_offset * rb->frame_size);
69e3c75f
JB		 504
505 if (status != __packet_get_status(po, h.raw))
506 return NULL;
507
508 return h.raw;
509}
510
eea49cc9 511static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 512 struct packet_ring_buffer *rb,
513 int status)
514{
515 return packet_lookup_frame(po, rb, rb->head, status);
516}
517
bc59ba39 518static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 519{
520 del_timer_sync(&pkc->retire_blk_timer);
521}
522
523static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
524 int tx_ring,
525 struct sk_buff_head *rb_queue)
526{
bc59ba39 527 struct tpacket_kbdq_core *pkc;
f6fb8f10 528
22781a5b
DJ		 529 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
530 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 531
ec6f809f 532 spin_lock_bh(&rb_queue->lock);
f6fb8f10 533 pkc->delete_blk_timer = 1;
ec6f809f 534 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 535
536 prb_del_retire_blk_timer(pkc);
537}
538
539static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 540 struct tpacket_kbdq_core *pkc,
f6fb8f10 541 void (*func) (unsigned long))
542{
543 init_timer(&pkc->retire_blk_timer);
544 pkc->retire_blk_timer.data = (long)po;
545 pkc->retire_blk_timer.function = func;
546 pkc->retire_blk_timer.expires = jiffies;
547}
548
549static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
550{
bc59ba39 551 struct tpacket_kbdq_core *pkc;
f6fb8f10 552
553 if (tx_ring)
554 BUG();
555
22781a5b
DJ		 556 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
557 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 558 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
559}
560
561static int prb_calc_retire_blk_tmo(struct packet_sock *po,
562 int blk_size_in_bytes)
563{
564 struct net_device *dev;
565 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 566 struct ethtool_cmd ecmd;
567 int err;
e440cf2c 568 u32 speed;
f6fb8f10 569
4bc71cb9
JP		 570 rtnl_lock();
571 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
572 if (unlikely(!dev)) {
573 rtnl_unlock();
f6fb8f10 574 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 575 }
576 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 577 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 578 rtnl_unlock();
579 if (!err) {
4bc71cb9
JP		 580 /*
581 * If the link speed is so slow you don't really
582 * need to worry about perf anyways
583 */
e440cf2c 584 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 585 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 586 } else {
587 msec = 1;
588 div = speed / 1000;
f6fb8f10 589 }
590 }
591
592 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
593
594 if (div)
595 mbits /= div;
596
597 tmo = mbits * msec;
598
599 if (div)
600 return tmo+1;
601 return tmo;
602}
603
bc59ba39 604static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 605 union tpacket_req_u *req_u)
606{
607 p1->feature_req_word = req_u->req3.tp_feature_req_word;
608}
609
610static void init_prb_bdqc(struct packet_sock *po,
611 struct packet_ring_buffer *rb,
612 struct pgv *pg_vec,
613 union tpacket_req_u *req_u, int tx_ring)
614{
22781a5b 615 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 616 struct tpacket_block_desc *pbd;
f6fb8f10 617
618 memset(p1, 0x0, sizeof(*p1));
619
620 p1->knxt_seq_num = 1;
621 p1->pkbdq = pg_vec;
bc59ba39 622 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 623 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 624 p1->kblk_size = req_u->req3.tp_block_size;
625 p1->knum_blocks = req_u->req3.tp_block_nr;
626 p1->hdrlen = po->tp_hdrlen;
627 p1->version = po->tp_version;
628 p1->last_kactive_blk_num = 0;
ee80fbf3 629 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 630 if (req_u->req3.tp_retire_blk_tov)
631 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
632 else
633 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
634 req_u->req3.tp_block_size);
635 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
636 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
637
638 prb_init_ft_ops(p1, req_u);
639 prb_setup_retire_blk_timer(po, tx_ring);
640 prb_open_block(p1, pbd);
641}
642
643/* Do NOT update the last_blk_num first.
644 * Assumes sk_buff_head lock is held.
645 */
bc59ba39 646static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 647{
648 mod_timer(&pkc->retire_blk_timer,
649 jiffies + pkc->tov_in_jiffies);
650 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
651}
652
653/*
654 * Timer logic:
655 * 1) We refresh the timer only when we open a block.
656 * By doing this we don't waste cycles refreshing the timer
657 * on packet-by-packet basis.
658 *
659 * With a 1MB block-size, on a 1Gbps line, it will take
660 * i) ~8 ms to fill a block + ii) memcpy etc.
661 * In this cut we are not accounting for the memcpy time.
662 *
663 * So, if the user sets the 'tmo' to 10ms then the timer
664 * will never fire while the block is still getting filled
665 * (which is what we want). However, the user could choose
666 * to close a block early and that's fine.
667 *
668 * But when the timer does fire, we check whether or not to refresh it.
669 * Since the tmo granularity is in msecs, it is not too expensive
670 * to refresh the timer, lets say every '8' msecs.
671 * Either the user can set the 'tmo' or we can derive it based on
672 * a) line-speed and b) block-size.
673 * prb_calc_retire_blk_tmo() calculates the tmo.
674 *
675 */
676static void prb_retire_rx_blk_timer_expired(unsigned long data)
677{
678 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 679 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 680 unsigned int frozen;
bc59ba39 681 struct tpacket_block_desc *pbd;
f6fb8f10 682
683 spin_lock(&po->sk.sk_receive_queue.lock);
684
685 frozen = prb_queue_frozen(pkc);
686 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
687
688 if (unlikely(pkc->delete_blk_timer))
689 goto out;
690
691 /* We only need to plug the race when the block is partially filled.
692 * tpacket_rcv:
693 * lock(); increment BLOCK_NUM_PKTS; unlock()
694 * copy_bits() is in progress ...
695 * timer fires on other cpu:
696 * we can't retire the current block because copy_bits
697 * is in progress.
698 *
699 */
700 if (BLOCK_NUM_PKTS(pbd)) {
701 while (atomic_read(&pkc->blk_fill_in_prog)) {
702 /* Waiting for skb_copy_bits to finish... */
703 cpu_relax();
704 }
705 }
706
707 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
708 if (!frozen) {
709 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
710 if (!prb_dispatch_next_block(pkc, po))
711 goto refresh_timer;
712 else
713 goto out;
714 } else {
715 /* Case 1. Queue was frozen because user-space was
716 * lagging behind.
717 */
718 if (prb_curr_blk_in_use(pkc, pbd)) {
719 /*
720 * Ok, user-space is still behind.
721 * So just refresh the timer.
722 */
723 goto refresh_timer;
724 } else {
725 /* Case 2. queue was frozen,user-space caught up,
726 * now the link went idle && the timer fired.
727 * We don't have a block to close.So we open this
728 * block and restart the timer.
729 * opening a block thaws the queue,restarts timer
730 * Thawing/timer-refresh is a side effect.
731 */
732 prb_open_block(pkc, pbd);
733 goto out;
734 }
735 }
736 }
737
738refresh_timer:
739 _prb_refresh_rx_retire_blk_timer(pkc);
740
741out:
742 spin_unlock(&po->sk.sk_receive_queue.lock);
743}
744
eea49cc9 745static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 746 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 747{
748 /* Flush everything minus the block header */
749
750#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
751 u8 *start, *end;
752
753 start = (u8 *)pbd1;
754
755 /* Skip the block header(we know header WILL fit in 4K) */
756 start += PAGE_SIZE;
757
758 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
759 for (; start < end; start += PAGE_SIZE)
760 flush_dcache_page(pgv_to_page(start));
761
762 smp_wmb();
763#endif
764
765 /* Now update the block status. */
766
767 BLOCK_STATUS(pbd1) = status;
768
769 /* Flush the block header */
770
771#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
772 start = (u8 *)pbd1;
773 flush_dcache_page(pgv_to_page(start));
774
775 smp_wmb();
776#endif
777}
778
779/*
780 * Side effect:
781 *
782 * 1) flush the block
783 * 2) Increment active_blk_num
784 *
785 * Note:We DONT refresh the timer on purpose.
786 * Because almost always the next block will be opened.
787 */
bc59ba39 788static void prb_close_block(struct tpacket_kbdq_core *pkc1,
789 struct tpacket_block_desc *pbd1,
f6fb8f10 790 struct packet_sock *po, unsigned int stat)
791{
792 __u32 status = TP_STATUS_USER | stat;
793
794 struct tpacket3_hdr *last_pkt;
bc59ba39 795 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 796
ee80fbf3 797 if (po->stats.stats3.tp_drops)
f6fb8f10 798 status |= TP_STATUS_LOSING;
799
800 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
801 last_pkt->tp_next_offset = 0;
802
803 /* Get the ts of the last pkt */
804 if (BLOCK_NUM_PKTS(pbd1)) {
805 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
806 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
807 } else {
808 /* Ok, we tmo'd - so get the current time */
809 struct timespec ts;
810 getnstimeofday(&ts);
811 h1->ts_last_pkt.ts_sec = ts.tv_sec;
812 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
813 }
814
815 smp_wmb();
816
817 /* Flush the block */
818 prb_flush_block(pkc1, pbd1, status);
819
820 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
821}
822
eea49cc9 823static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 824{
825 pkc->reset_pending_on_curr_blk = 0;
826}
827
828/*
829 * Side effect of opening a block:
830 *
831 * 1) prb_queue is thawed.
832 * 2) retire_blk_timer is refreshed.
833 *
834 */
bc59ba39 835static void prb_open_block(struct tpacket_kbdq_core *pkc1,
836 struct tpacket_block_desc *pbd1)
f6fb8f10 837{
838 struct timespec ts;
bc59ba39 839 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 840
841 smp_rmb();
842
8da3056c
DB		 843 /* We could have just memset this but we will lose the
844 * flexibility of making the priv area sticky
845 */
f6fb8f10 846
8da3056c
DB		 847 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
848 BLOCK_NUM_PKTS(pbd1) = 0;
849 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 850
8da3056c
DB		 851 getnstimeofday(&ts);
852
853 h1->ts_first_pkt.ts_sec = ts.tv_sec;
854 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 855
8da3056c
DB		 856 pkc1->pkblk_start = (char *)pbd1;
857 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
858
859 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
860 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
861
862 pbd1->version = pkc1->version;
863 pkc1->prev = pkc1->nxt_offset;
864 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
865
866 prb_thaw_queue(pkc1);
867 _prb_refresh_rx_retire_blk_timer(pkc1);
868
869 smp_wmb();
f6fb8f10 870}
871
872/*
873 * Queue freeze logic:
874 * 1) Assume tp_block_nr = 8 blocks.
875 * 2) At time 't0', user opens Rx ring.
876 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
877 * 4) user-space is either sleeping or processing block '0'.
878 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
879 * it will close block-7,loop around and try to fill block '0'.
880 * call-flow:
881 * __packet_lookup_frame_in_block
882 * prb_retire_current_block()
883 * prb_dispatch_next_block()
884 * |->(BLOCK_STATUS == USER) evaluates to true
885 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
886 * 6) Now there are two cases:
887 * 6.1) Link goes idle right after the queue is frozen.
888 * But remember, the last open_block() refreshed the timer.
889 * When this timer expires,it will refresh itself so that we can
890 * re-open block-0 in near future.
891 * 6.2) Link is busy and keeps on receiving packets. This is a simple
892 * case and __packet_lookup_frame_in_block will check if block-0
893 * is free and can now be re-used.
894 */
eea49cc9 895static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 896 struct packet_sock *po)
897{
898 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 899 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 900}
901
902#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
903
904/*
905 * If the next block is free then we will dispatch it
906 * and return a good offset.
907 * Else, we will freeze the queue.
908 * So, caller must check the return value.
909 */
bc59ba39 910static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 911 struct packet_sock *po)
912{
bc59ba39 913 struct tpacket_block_desc *pbd;
f6fb8f10 914
915 smp_rmb();
916
917 /* 1. Get current block num */
918 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
919
920 /* 2. If this block is currently in_use then freeze the queue */
921 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
922 prb_freeze_queue(pkc, po);
923 return NULL;
924 }
925
926 /*
927 * 3.
928 * open this block and return the offset where the first packet
929 * needs to get stored.
930 */
931 prb_open_block(pkc, pbd);
932 return (void *)pkc->nxt_offset;
933}
934
bc59ba39 935static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 936 struct packet_sock *po, unsigned int status)
937{
bc59ba39 938 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 939
940 /* retire/close the current block */
941 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
942 /*
943 * Plug the case where copy_bits() is in progress on
944 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
945 * have space to copy the pkt in the current block and
946 * called prb_retire_current_block()
947 *
948 * We don't need to worry about the TMO case because
949 * the timer-handler already handled this case.
950 */
951 if (!(status & TP_STATUS_BLK_TMO)) {
952 while (atomic_read(&pkc->blk_fill_in_prog)) {
953 /* Waiting for skb_copy_bits to finish... */
954 cpu_relax();
955 }
956 }
957 prb_close_block(pkc, pbd, po, status);
958 return;
959 }
f6fb8f10 960}
961
eea49cc9 962static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 963 struct tpacket_block_desc *pbd)
f6fb8f10 964{
965 return TP_STATUS_USER & BLOCK_STATUS(pbd);
966}
967
eea49cc9 968static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 969{
970 return pkc->reset_pending_on_curr_blk;
971}
972
eea49cc9 973static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 974{
bc59ba39 975 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 976 atomic_dec(&pkc->blk_fill_in_prog);
977}
978
eea49cc9 979static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 980 struct tpacket3_hdr *ppd)
981{
3958afa1 982 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 983}
984
eea49cc9 985static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 986 struct tpacket3_hdr *ppd)
987{
988 ppd->hv1.tp_rxhash = 0;
989}
990
eea49cc9 991static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 992 struct tpacket3_hdr *ppd)
993{
994 if (vlan_tx_tag_present(pkc->skb)) {
995 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
a0cdfcf3
AW		 996 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
997 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 998 } else {
9e67030a 999 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 1000 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 1001 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 1002 }
1003}
1004
bc59ba39 1005static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 1006 struct tpacket3_hdr *ppd)
1007{
a0cdfcf3 1008 ppd->hv1.tp_padding = 0;
f6fb8f10 1009 prb_fill_vlan_info(pkc, ppd);
1010
1011 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1012 prb_fill_rxhash(pkc, ppd);
1013 else
1014 prb_clear_rxhash(pkc, ppd);
1015}
1016
eea49cc9 1017static void prb_fill_curr_block(char *curr,
bc59ba39 1018 struct tpacket_kbdq_core *pkc,
1019 struct tpacket_block_desc *pbd,
f6fb8f10 1020 unsigned int len)
1021{
1022 struct tpacket3_hdr *ppd;
1023
1024 ppd = (struct tpacket3_hdr *)curr;
1025 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1026 pkc->prev = curr;
1027 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1028 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1029 BLOCK_NUM_PKTS(pbd) += 1;
1030 atomic_inc(&pkc->blk_fill_in_prog);
1031 prb_run_all_ft_ops(pkc, ppd);
1032}
1033
1034/* Assumes caller has the sk->rx_queue.lock */
1035static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1036 struct sk_buff *skb,
1037 int status,
1038 unsigned int len
1039 )
1040{
bc59ba39 1041 struct tpacket_kbdq_core *pkc;
1042 struct tpacket_block_desc *pbd;
f6fb8f10 1043 char *curr, *end;
1044
e3192690 1045 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1046 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1047
1048 /* Queue is frozen when user space is lagging behind */
1049 if (prb_queue_frozen(pkc)) {
1050 /*
1051 * Check if that last block which caused the queue to freeze,
1052 * is still in_use by user-space.
1053 */
1054 if (prb_curr_blk_in_use(pkc, pbd)) {
1055 /* Can't record this packet */
1056 return NULL;
1057 } else {
1058 /*
1059 * Ok, the block was released by user-space.
1060 * Now let's open that block.
1061 * opening a block also thaws the queue.
1062 * Thawing is a side effect.
1063 */
1064 prb_open_block(pkc, pbd);
1065 }
1066 }
1067
1068 smp_mb();
1069 curr = pkc->nxt_offset;
1070 pkc->skb = skb;
e3192690 1071 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1072
1073 /* first try the current block */
1074 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1075 prb_fill_curr_block(curr, pkc, pbd, len);
1076 return (void *)curr;
1077 }
1078
1079 /* Ok, close the current block */
1080 prb_retire_current_block(pkc, po, 0);
1081
1082 /* Now, try to dispatch the next block */
1083 curr = (char *)prb_dispatch_next_block(pkc, po);
1084 if (curr) {
1085 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1086 prb_fill_curr_block(curr, pkc, pbd, len);
1087 return (void *)curr;
1088 }
1089
1090 /*
1091 * No free blocks are available.user_space hasn't caught up yet.
1092 * Queue was just frozen and now this packet will get dropped.
1093 */
1094 return NULL;
1095}
1096
eea49cc9 1097static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1098 struct sk_buff *skb,
1099 int status, unsigned int len)
1100{
1101 char *curr = NULL;
1102 switch (po->tp_version) {
1103 case TPACKET_V1:
1104 case TPACKET_V2:
1105 curr = packet_lookup_frame(po, &po->rx_ring,
1106 po->rx_ring.head, status);
1107 return curr;
1108 case TPACKET_V3:
1109 return __packet_lookup_frame_in_block(po, skb, status, len);
1110 default:
1111 WARN(1, "TPACKET version not supported\n");
1112 BUG();
99aa3473 1113 return NULL;
f6fb8f10 1114 }
1115}
1116
eea49cc9 1117static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1118 struct packet_ring_buffer *rb,
77f65ebd 1119 unsigned int idx,
f6fb8f10 1120 int status)
1121{
bc59ba39 1122 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1123 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1124
1125 if (status != BLOCK_STATUS(pbd))
1126 return NULL;
1127 return pbd;
1128}
1129
eea49cc9 1130static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1131{
1132 unsigned int prev;
1133 if (rb->prb_bdqc.kactive_blk_num)
1134 prev = rb->prb_bdqc.kactive_blk_num-1;
1135 else
1136 prev = rb->prb_bdqc.knum_blocks-1;
1137 return prev;
1138}
1139
1140/* Assumes caller has held the rx_queue.lock */
eea49cc9 1141static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1142 struct packet_ring_buffer *rb,
1143 int status)
1144{
1145 unsigned int previous = prb_previous_blk_num(rb);
1146 return prb_lookup_block(po, rb, previous, status);
1147}
1148
eea49cc9 1149static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1150 struct packet_ring_buffer *rb,
1151 int status)
1152{
1153 if (po->tp_version <= TPACKET_V2)
1154 return packet_previous_frame(po, rb, status);
1155
1156 return __prb_previous_block(po, rb, status);
1157}
1158
eea49cc9 1159static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1160 struct packet_ring_buffer *rb)
1161{
1162 switch (po->tp_version) {
1163 case TPACKET_V1:
1164 case TPACKET_V2:
1165 return packet_increment_head(rb);
1166 case TPACKET_V3:
1167 default:
1168 WARN(1, "TPACKET version not supported.\n");
1169 BUG();
1170 return;
1171 }
1172}
1173
eea49cc9 1174static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1175 struct packet_ring_buffer *rb,
1176 int status)
1177{
1178 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1179 return packet_lookup_frame(po, rb, previous, status);
1180}
1181
eea49cc9 1182static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1183{
1184 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1185}
1186
b0138408
DB		 1187static void packet_inc_pending(struct packet_ring_buffer *rb)
1188{
1189 this_cpu_inc(*rb->pending_refcnt);
1190}
1191
1192static void packet_dec_pending(struct packet_ring_buffer *rb)
1193{
1194 this_cpu_dec(*rb->pending_refcnt);
1195}
1196
1197static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1198{
1199 unsigned int refcnt = 0;
1200 int cpu;
1201
1202 /* We don't use pending refcount in rx_ring. */
1203 if (rb->pending_refcnt == NULL)
1204 return 0;
1205
1206 for_each_possible_cpu(cpu)
1207 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1208
1209 return refcnt;
1210}
1211
1212static int packet_alloc_pending(struct packet_sock *po)
1213{
1214 po->rx_ring.pending_refcnt = NULL;
1215
1216 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1217 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1218 return -ENOBUFS;
1219
1220 return 0;
1221}
1222
1223static void packet_free_pending(struct packet_sock *po)
1224{
1225 free_percpu(po->tx_ring.pending_refcnt);
1226}
1227
77f65ebd
WB		 1228static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1229{
1230 struct sock *sk = &po->sk;
1231 bool has_room;
1232
1233 if (po->prot_hook.func != tpacket_rcv)
1234 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1235 <= sk->sk_rcvbuf;
1236
1237 spin_lock(&sk->sk_receive_queue.lock);
1238 if (po->tp_version == TPACKET_V3)
1239 has_room = prb_lookup_block(po, &po->rx_ring,
1240 po->rx_ring.prb_bdqc.kactive_blk_num,
1241 TP_STATUS_KERNEL);
1242 else
1243 has_room = packet_lookup_frame(po, &po->rx_ring,
1244 po->rx_ring.head,
1245 TP_STATUS_KERNEL);
1246 spin_unlock(&sk->sk_receive_queue.lock);
1247
1248 return has_room;
1249}
1250
1da177e4
LT		 1251static void packet_sock_destruct(struct sock *sk)
1252{
ed85b565
RC		 1253 skb_queue_purge(&sk->sk_error_queue);
1254
547b792c
IJ		 1255 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1256 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1257
1258 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1259 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1260 return;
1261 }
1262
17ab56a2 1263 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1264}
1265
dc99f600
DM		 1266static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1267{
1268 int x = atomic_read(&f->rr_cur) + 1;
1269
1270 if (x >= num)
1271 x = 0;
1272
1273 return x;
1274}
1275
77f65ebd
WB		 1276static unsigned int fanout_demux_hash(struct packet_fanout *f,
1277 struct sk_buff *skb,
1278 unsigned int num)
dc99f600 1279{
61b905da 1280 return reciprocal_scale(skb_get_hash(skb), num);
dc99f600
DM		 1281}
1282
77f65ebd
WB		 1283static unsigned int fanout_demux_lb(struct packet_fanout *f,
1284 struct sk_buff *skb,
1285 unsigned int num)
dc99f600
DM		 1286{
1287 int cur, old;
1288
1289 cur = atomic_read(&f->rr_cur);
1290 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1291 fanout_rr_next(f, num))) != cur)
1292 cur = old;
77f65ebd
WB		 1293 return cur;
1294}
1295
1296static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1297 struct sk_buff *skb,
1298 unsigned int num)
1299{
1300 return smp_processor_id() % num;
dc99f600
DM		 1301}
1302
5df0ddfb
DB		 1303static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1304 struct sk_buff *skb,
1305 unsigned int num)
1306{
f337db64 1307 return prandom_u32_max(num);
5df0ddfb
DB		 1308}
1309
77f65ebd
WB		 1310static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1311 struct sk_buff *skb,
1312 unsigned int idx, unsigned int skip,
1313 unsigned int num)
95ec3eb4 1314{
77f65ebd 1315 unsigned int i, j;
95ec3eb4 1316
77f65ebd
WB		 1317 i = j = min_t(int, f->next[idx], num - 1);
1318 do {
1319 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1320 if (i != j)
1321 f->next[idx] = i;
1322 return i;
1323 }
1324 if (++i == num)
1325 i = 0;
1326 } while (i != j);
1327
1328 return idx;
1329}
1330
2d36097d
NH		 1331static unsigned int fanout_demux_qm(struct packet_fanout *f,
1332 struct sk_buff *skb,
1333 unsigned int num)
1334{
1335 return skb_get_queue_mapping(skb) % num;
1336}
1337
77f65ebd
WB		 1338static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1339{
1340 return f->flags & (flag >> 8);
95ec3eb4
DM		 1341}
1342
95ec3eb4
DM		 1343static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1344 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1345{
1346 struct packet_fanout *f = pt->af_packet_priv;
1347 unsigned int num = f->num_members;
1348 struct packet_sock *po;
77f65ebd 1349 unsigned int idx;
dc99f600
DM		 1350
1351 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1352 !num) {
1353 kfree_skb(skb);
1354 return 0;
1355 }
1356
95ec3eb4
DM		 1357 switch (f->type) {
1358 case PACKET_FANOUT_HASH:
1359 default:
77f65ebd 1360 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1361 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1362 if (!skb)
1363 return 0;
1364 }
77f65ebd 1365 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1366 break;
1367 case PACKET_FANOUT_LB:
77f65ebd 1368 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1369 break;
1370 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1371 idx = fanout_demux_cpu(f, skb, num);
1372 break;
5df0ddfb
DB		 1373 case PACKET_FANOUT_RND:
1374 idx = fanout_demux_rnd(f, skb, num);
1375 break;
2d36097d
NH		 1376 case PACKET_FANOUT_QM:
1377 idx = fanout_demux_qm(f, skb, num);
1378 break;
77f65ebd
WB		 1379 case PACKET_FANOUT_ROLLOVER:
1380 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1381 break;
dc99f600
DM		 1382 }
1383
77f65ebd
WB		 1384 po = pkt_sk(f->arr[idx]);
1385 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1386 unlikely(!packet_rcv_has_room(po, skb))) {
1387 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1388 po = pkt_sk(f->arr[idx]);
1389 }
dc99f600
DM		 1390
1391 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1392}
1393
fff3321d
PE		 1394DEFINE_MUTEX(fanout_mutex);
1395EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1396static LIST_HEAD(fanout_list);
1397
1398static void __fanout_link(struct sock *sk, struct packet_sock *po)
1399{
1400 struct packet_fanout *f = po->fanout;
1401
1402 spin_lock(&f->lock);
1403 f->arr[f->num_members] = sk;
1404 smp_wmb();
1405 f->num_members++;
1406 spin_unlock(&f->lock);
1407}
1408
1409static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1410{
1411 struct packet_fanout *f = po->fanout;
1412 int i;
1413
1414 spin_lock(&f->lock);
1415 for (i = 0; i < f->num_members; i++) {
1416 if (f->arr[i] == sk)
1417 break;
1418 }
1419 BUG_ON(i >= f->num_members);
1420 f->arr[i] = f->arr[f->num_members - 1];
1421 f->num_members--;
1422 spin_unlock(&f->lock);
1423}
1424
d4dd8aee 1425static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1426{
d4dd8aee 1427 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
c0de08d0
EL		 1428 return true;
1429
1430 return false;
1431}
1432
7736d33f 1433static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1434{
1435 struct packet_sock *po = pkt_sk(sk);
1436 struct packet_fanout *f, *match;
7736d33f 1437 u8 type = type_flags & 0xff;
77f65ebd 1438 u8 flags = type_flags >> 8;
dc99f600
DM		 1439 int err;
1440
1441 switch (type) {
77f65ebd
WB		 1442 case PACKET_FANOUT_ROLLOVER:
1443 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1444 return -EINVAL;
dc99f600
DM		 1445 case PACKET_FANOUT_HASH:
1446 case PACKET_FANOUT_LB:
95ec3eb4 1447 case PACKET_FANOUT_CPU:
5df0ddfb 1448 case PACKET_FANOUT_RND:
2d36097d 1449 case PACKET_FANOUT_QM:
dc99f600
DM		 1450 break;
1451 default:
1452 return -EINVAL;
1453 }
1454
1455 if (!po->running)
1456 return -EINVAL;
1457
1458 if (po->fanout)
1459 return -EALREADY;
1460
1461 mutex_lock(&fanout_mutex);
1462 match = NULL;
1463 list_for_each_entry(f, &fanout_list, list) {
1464 if (f->id == id &&
1465 read_pnet(&f->net) == sock_net(sk)) {
1466 match = f;
1467 break;
1468 }
1469 }
afe62c68 1470 err = -EINVAL;
77f65ebd 1471 if (match && match->flags != flags)
afe62c68 1472 goto out;
dc99f600 1473 if (!match) {
afe62c68 1474 err = -ENOMEM;
dc99f600 1475 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1476 if (!match)
1477 goto out;
1478 write_pnet(&match->net, sock_net(sk));
1479 match->id = id;
1480 match->type = type;
77f65ebd 1481 match->flags = flags;
afe62c68
ED		 1482 atomic_set(&match->rr_cur, 0);
1483 INIT_LIST_HEAD(&match->list);
1484 spin_lock_init(&match->lock);
1485 atomic_set(&match->sk_ref, 0);
1486 match->prot_hook.type = po->prot_hook.type;
1487 match->prot_hook.dev = po->prot_hook.dev;
1488 match->prot_hook.func = packet_rcv_fanout;
1489 match->prot_hook.af_packet_priv = match;
c0de08d0 1490 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1491 dev_add_pack(&match->prot_hook);
1492 list_add(&match->list, &fanout_list);
dc99f600 1493 }
afe62c68
ED		 1494 err = -EINVAL;
1495 if (match->type == type &&
1496 match->prot_hook.type == po->prot_hook.type &&
1497 match->prot_hook.dev == po->prot_hook.dev) {
1498 err = -ENOSPC;
1499 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1500 __dev_remove_pack(&po->prot_hook);
1501 po->fanout = match;
1502 atomic_inc(&match->sk_ref);
1503 __fanout_link(sk, po);
1504 err = 0;
dc99f600
DM		 1505 }
1506 }
afe62c68 1507out:
dc99f600
DM		 1508 mutex_unlock(&fanout_mutex);
1509 return err;
1510}
1511
1512static void fanout_release(struct sock *sk)
1513{
1514 struct packet_sock *po = pkt_sk(sk);
1515 struct packet_fanout *f;
1516
1517 f = po->fanout;
1518 if (!f)
1519 return;
1520
fff3321d 1521 mutex_lock(&fanout_mutex);
dc99f600
DM		 1522 po->fanout = NULL;
1523
dc99f600
DM		 1524 if (atomic_dec_and_test(&f->sk_ref)) {
1525 list_del(&f->list);
1526 dev_remove_pack(&f->prot_hook);
1527 kfree(f);
1528 }
1529 mutex_unlock(&fanout_mutex);
1530}
1da177e4 1531
90ddc4f0 1532static const struct proto_ops packet_ops;
1da177e4 1533
90ddc4f0 1534static const struct proto_ops packet_ops_spkt;
1da177e4 1535
40d4e3df
ED		 1536static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1537 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1538{
1539 struct sock *sk;
1540 struct sockaddr_pkt *spkt;
1541
1542 /*
1543 * When we registered the protocol we saved the socket in the data
1544 * field for just this event.
1545 */
1546
1547 sk = pt->af_packet_priv;
1ce4f28b 1548
1da177e4
LT		 1549 /*
1550 * Yank back the headers [hope the device set this
1551 * right or kerboom...]
1552 *
1553 * Incoming packets have ll header pulled,
1554 * push it back.
1555 *
98e399f8 1556 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1557 * so that this procedure is noop.
1558 */
1559
1560 if (skb->pkt_type == PACKET_LOOPBACK)
1561 goto out;
1562
09ad9bc7 1563 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1564 goto out;
1565
40d4e3df
ED		 1566 skb = skb_share_check(skb, GFP_ATOMIC);
1567 if (skb == NULL)
1da177e4
LT		 1568 goto oom;
1569
1570 /* drop any routing info */
adf30907 1571 skb_dst_drop(skb);
1da177e4 1572
84531c24
PO		 1573 /* drop conntrack reference */
1574 nf_reset(skb);
1575
ffbc6111 1576 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1577
98e399f8 1578 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1579
1580 /*
1581 * The SOCK_PACKET socket receives _all_ frames.
1582 */
1583
1584 spkt->spkt_family = dev->type;
1585 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1586 spkt->spkt_protocol = skb->protocol;
1587
1588 /*
1589 * Charge the memory to the socket. This is done specifically
1590 * to prevent sockets using all the memory up.
1591 */
1592
40d4e3df 1593 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1594 return 0;
1595
1596out:
1597 kfree_skb(skb);
1598oom:
1599 return 0;
1600}
1601
1602
1603/*
1604 * Output a raw packet to a device layer. This bypasses all the other
1605 * protocol layers and you must therefore supply it with a complete frame
1606 */
1ce4f28b 1607
1da177e4
LT		 1608static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1609 struct msghdr *msg, size_t len)
1610{
1611 struct sock *sk = sock->sk;
342dfc30 1612 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1613 struct sk_buff *skb = NULL;
1da177e4 1614 struct net_device *dev;
40d4e3df 1615 __be16 proto = 0;
1da177e4 1616 int err;
3bdc0eba 1617 int extra_len = 0;
1ce4f28b 1618
1da177e4 1619 /*
1ce4f28b 1620 * Get and verify the address.
1da177e4
LT		 1621 */
1622
40d4e3df 1623 if (saddr) {
1da177e4 1624 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1625 return -EINVAL;
1626 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1627 proto = saddr->spkt_protocol;
1628 } else
1629 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1630
1631 /*
1ce4f28b 1632 * Find the device first to size check it
1da177e4
LT		 1633 */
1634
de74e92a 1635 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1636retry:
654d1f8a
ED		 1637 rcu_read_lock();
1638 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1639 err = -ENODEV;
1640 if (dev == NULL)
1641 goto out_unlock;
1ce4f28b 1642
d5e76b0a
DM		 1643 err = -ENETDOWN;
1644 if (!(dev->flags & IFF_UP))
1645 goto out_unlock;
1646
1da177e4 1647 /*
40d4e3df
ED		 1648 * You may not queue a frame bigger than the mtu. This is the lowest level
1649 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1650 */
1ce4f28b 1651
3bdc0eba
BG		 1652 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1653 if (!netif_supports_nofcs(dev)) {
1654 err = -EPROTONOSUPPORT;
1655 goto out_unlock;
1656 }
1657 extra_len = 4; /* We're doing our own CRC */
1658 }
1659
1da177e4 1660 err = -EMSGSIZE;
3bdc0eba 1661 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1662 goto out_unlock;
1663
1a35ca80
ED		 1664 if (!skb) {
1665 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1666 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1667 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1668
1669 rcu_read_unlock();
4ce40912 1670 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1671 if (skb == NULL)
1672 return -ENOBUFS;
1673 /* FIXME: Save some space for broken drivers that write a hard
1674 * header at transmission time by themselves. PPP is the notable
1675 * one here. This should really be fixed at the driver level.
1676 */
1677 skb_reserve(skb, reserved);
1678 skb_reset_network_header(skb);
1679
1680 /* Try to align data part correctly */
1681 if (hhlen) {
1682 skb->data -= hhlen;
1683 skb->tail -= hhlen;
1684 if (len < hhlen)
1685 skb_reset_network_header(skb);
1686 }
1687 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1688 if (err)
1689 goto out_free;
1690 goto retry;
1da177e4
LT		 1691 }
1692
3bdc0eba 1693 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1694 /* Earlier code assumed this would be a VLAN pkt,
1695 * double-check this now that we have the actual
1696 * packet in hand.
1697 */
1698 struct ethhdr *ehdr;
1699 skb_reset_mac_header(skb);
1700 ehdr = eth_hdr(skb);
1701 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1702 err = -EMSGSIZE;
1703 goto out_unlock;
1704 }
1705 }
1a35ca80 1706
1da177e4
LT		 1707 skb->protocol = proto;
1708 skb->dev = dev;
1709 skb->priority = sk->sk_priority;
2d37a186 1710 skb->mark = sk->sk_mark;
bf84a010
DB		 1711
1712 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1713
3bdc0eba
BG		 1714 if (unlikely(extra_len == 4))
1715 skb->no_fcs = 1;
1716
40893fd0 1717 skb_probe_transport_header(skb, 0);
c1aad275 1718
1da177e4 1719 dev_queue_xmit(skb);
654d1f8a 1720 rcu_read_unlock();
40d4e3df 1721 return len;
1da177e4 1722
1da177e4 1723out_unlock:
654d1f8a 1724 rcu_read_unlock();
1a35ca80
ED		 1725out_free:
1726 kfree_skb(skb);
1da177e4
LT		 1727 return err;
1728}
1da177e4 1729
eea49cc9 1730static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1731 const struct sock *sk,
dbcb5855 1732 unsigned int res)
1da177e4
LT		 1733{
1734 struct sk_filter *filter;
fda9ef5d 1735
80f8f102
ED		 1736 rcu_read_lock();
1737 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1738 if (filter != NULL)
0a14842f 1739 res = SK_RUN_FILTER(filter, skb);
80f8f102 1740 rcu_read_unlock();
1da177e4 1741
dbcb5855 1742 return res;
1da177e4
LT		 1743}
1744
1745/*
62ab0812
ED		 1746 * This function makes lazy skb cloning in hope that most of packets
1747 * are discarded by BPF.
1748 *
1749 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1750 * and skb->cb are mangled. It works because (and until) packets
1751 * falling here are owned by current CPU. Output packets are cloned
1752 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1753 * sequencially, so that if we return skb to original state on exit,
1754 * we will not harm anyone.
1da177e4
LT		 1755 */
1756
40d4e3df
ED		 1757static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1758 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1759{
1760 struct sock *sk;
1761 struct sockaddr_ll *sll;
1762 struct packet_sock *po;
40d4e3df 1763 u8 *skb_head = skb->data;
1da177e4 1764 int skb_len = skb->len;
dbcb5855 1765 unsigned int snaplen, res;
1da177e4
LT		 1766
1767 if (skb->pkt_type == PACKET_LOOPBACK)
1768 goto drop;
1769
1770 sk = pt->af_packet_priv;
1771 po = pkt_sk(sk);
1772
09ad9bc7 1773 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1774 goto drop;
1775
1da177e4
LT		 1776 skb->dev = dev;
1777
3b04ddde 1778 if (dev->header_ops) {
1da177e4 1779 /* The device has an explicit notion of ll header,
62ab0812
ED		 1780 * exported to higher levels.
1781 *
1782 * Otherwise, the device hides details of its frame
1783 * structure, so that corresponding packet head is
1784 * never delivered to user.
1da177e4
LT		 1785 */
1786 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1787 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1788 else if (skb->pkt_type == PACKET_OUTGOING) {
1789 /* Special case: outgoing packets have ll header at head */
bbe735e4 1790 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1791 }
1792 }
1793
1794 snaplen = skb->len;
1795
dbcb5855
DM		 1796 res = run_filter(skb, sk, snaplen);
1797 if (!res)
fda9ef5d 1798 goto drop_n_restore;
dbcb5855
DM		 1799 if (snaplen > res)
1800 snaplen = res;
1da177e4 1801
0fd7bac6 1802 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1803 goto drop_n_acct;
1804
1805 if (skb_shared(skb)) {
1806 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1807 if (nskb == NULL)
1808 goto drop_n_acct;
1809
1810 if (skb_head != skb->data) {
1811 skb->data = skb_head;
1812 skb->len = skb_len;
1813 }
abc4e4fa 1814 consume_skb(skb);
1da177e4
LT		 1815 skb = nskb;
1816 }
1817
ffbc6111
HX		 1818 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1819 sizeof(skb->cb));
1820
1821 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1822 sll->sll_family = AF_PACKET;
1823 sll->sll_hatype = dev->type;
1824 sll->sll_protocol = skb->protocol;
1825 sll->sll_pkttype = skb->pkt_type;
8032b464 1826 if (unlikely(po->origdev))
80feaacb
PWJ		 1827 sll->sll_ifindex = orig_dev->ifindex;
1828 else
1829 sll->sll_ifindex = dev->ifindex;
1da177e4 1830
b95cce35 1831 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1832
ffbc6111 1833 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1834
1da177e4
LT		 1835 if (pskb_trim(skb, snaplen))
1836 goto drop_n_acct;
1837
1838 skb_set_owner_r(skb, sk);
1839 skb->dev = NULL;
adf30907 1840 skb_dst_drop(skb);
1da177e4 1841
84531c24
PO		 1842 /* drop conntrack reference */
1843 nf_reset(skb);
1844
1da177e4 1845 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1846 po->stats.stats1.tp_packets++;
3b885787 1847 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1848 __skb_queue_tail(&sk->sk_receive_queue, skb);
1849 spin_unlock(&sk->sk_receive_queue.lock);
1850 sk->sk_data_ready(sk, skb->len);
1851 return 0;
1852
1853drop_n_acct:
7091fbd8 1854 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1855 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1856 atomic_inc(&sk->sk_drops);
1857 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1858
1859drop_n_restore:
1860 if (skb_head != skb->data && skb_shared(skb)) {
1861 skb->data = skb_head;
1862 skb->len = skb_len;
1863 }
1864drop:
ead2ceb0 1865 consume_skb(skb);
1da177e4
LT		 1866 return 0;
1867}
1868
40d4e3df
ED		 1869static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1870 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1871{
1872 struct sock *sk;
1873 struct packet_sock *po;
1874 struct sockaddr_ll *sll;
184f489e 1875 union tpacket_uhdr h;
40d4e3df 1876 u8 *skb_head = skb->data;
1da177e4 1877 int skb_len = skb->len;
dbcb5855 1878 unsigned int snaplen, res;
f6fb8f10 1879 unsigned long status = TP_STATUS_USER;
bbd6ef87 1880 unsigned short macoff, netoff, hdrlen;
1da177e4 1881 struct sk_buff *copy_skb = NULL;
bbd6ef87 1882 struct timespec ts;
b9c32fb2 1883 __u32 ts_status;
1da177e4 1884
51846355
AW		 1885 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1886 * We may add members to them until current aligned size without forcing
1887 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1888 */
1889 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1890 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1891
1da177e4
LT		 1892 if (skb->pkt_type == PACKET_LOOPBACK)
1893 goto drop;
1894
1895 sk = pt->af_packet_priv;
1896 po = pkt_sk(sk);
1897
09ad9bc7 1898 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1899 goto drop;
1900
3b04ddde 1901 if (dev->header_ops) {
1da177e4 1902 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1903 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1904 else if (skb->pkt_type == PACKET_OUTGOING) {
1905 /* Special case: outgoing packets have ll header at head */
bbe735e4 1906 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1907 }
1908 }
1909
8dc41944
HX		 1910 if (skb->ip_summed == CHECKSUM_PARTIAL)
1911 status |= TP_STATUS_CSUMNOTREADY;
1912
1da177e4
LT		 1913 snaplen = skb->len;
1914
dbcb5855
DM		 1915 res = run_filter(skb, sk, snaplen);
1916 if (!res)
fda9ef5d 1917 goto drop_n_restore;
dbcb5855
DM		 1918 if (snaplen > res)
1919 snaplen = res;
1da177e4
LT		 1920
1921 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1922 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1923 po->tp_reserve;
1da177e4 1924 } else {
95c96174 1925 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1926 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1927 (maclen < 16 ? 16 : maclen)) +
1928 po->tp_reserve;
1da177e4
LT		 1929 macoff = netoff - maclen;
1930 }
f6fb8f10 1931 if (po->tp_version <= TPACKET_V2) {
1932 if (macoff + snaplen > po->rx_ring.frame_size) {
1933 if (po->copy_thresh &&
0fd7bac6 1934 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1935 if (skb_shared(skb)) {
1936 copy_skb = skb_clone(skb, GFP_ATOMIC);
1937 } else {
1938 copy_skb = skb_get(skb);
1939 skb_head = skb->data;
1940 }
1941 if (copy_skb)
1942 skb_set_owner_r(copy_skb, sk);
1da177e4 1943 }
f6fb8f10 1944 snaplen = po->rx_ring.frame_size - macoff;
1945 if ((int)snaplen < 0)
1946 snaplen = 0;
1da177e4 1947 }
1da177e4 1948 }
1da177e4 1949 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1950 h.raw = packet_current_rx_frame(po, skb,
1951 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1952 if (!h.raw)
1da177e4 1953 goto ring_is_full;
f6fb8f10 1954 if (po->tp_version <= TPACKET_V2) {
1955 packet_increment_rx_head(po, &po->rx_ring);
1956 /*
1957 * LOSING will be reported till you read the stats,
1958 * because it's COR - Clear On Read.
1959 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1960 * at packet level.
1961 */
ee80fbf3 1962 if (po->stats.stats1.tp_drops)
f6fb8f10 1963 status |= TP_STATUS_LOSING;
1964 }
ee80fbf3 1965 po->stats.stats1.tp_packets++;
1da177e4
LT		 1966 if (copy_skb) {
1967 status |= TP_STATUS_COPY;
1968 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1969 }
1da177e4
LT		 1970 spin_unlock(&sk->sk_receive_queue.lock);
1971
bbd6ef87 1972 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1973
1974 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1975 getnstimeofday(&ts);
1da177e4 1976
b9c32fb2
DB		 1977 status |= ts_status;
1978
bbd6ef87
PM		 1979 switch (po->tp_version) {
1980 case TPACKET_V1:
1981 h.h1->tp_len = skb->len;
1982 h.h1->tp_snaplen = snaplen;
1983 h.h1->tp_mac = macoff;
1984 h.h1->tp_net = netoff;
4b457bdf
DB		 1985 h.h1->tp_sec = ts.tv_sec;
1986 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1987 hdrlen = sizeof(*h.h1);
1988 break;
1989 case TPACKET_V2:
1990 h.h2->tp_len = skb->len;
1991 h.h2->tp_snaplen = snaplen;
1992 h.h2->tp_mac = macoff;
1993 h.h2->tp_net = netoff;
bbd6ef87
PM		 1994 h.h2->tp_sec = ts.tv_sec;
1995 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1996 if (vlan_tx_tag_present(skb)) {
1997 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 1998 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
1999 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2000 } else {
2001 h.h2->tp_vlan_tci = 0;
a0cdfcf3 2002 h.h2->tp_vlan_tpid = 0;
a3bcc23e 2003 }
e4d26f4b 2004 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 2005 hdrlen = sizeof(*h.h2);
2006 break;
f6fb8f10 2007 case TPACKET_V3:
2008 /* tp_nxt_offset,vlan are already populated above.
2009 * So DONT clear those fields here
2010 */
2011 h.h3->tp_status |= status;
2012 h.h3->tp_len = skb->len;
2013 h.h3->tp_snaplen = snaplen;
2014 h.h3->tp_mac = macoff;
2015 h.h3->tp_net = netoff;
f6fb8f10 2016 h.h3->tp_sec = ts.tv_sec;
2017 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2018 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2019 hdrlen = sizeof(*h.h3);
2020 break;
bbd6ef87
PM		 2021 default:
2022 BUG();
2023 }
1da177e4 2024
bbd6ef87 2025 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2026 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2027 sll->sll_family = AF_PACKET;
2028 sll->sll_hatype = dev->type;
2029 sll->sll_protocol = skb->protocol;
2030 sll->sll_pkttype = skb->pkt_type;
8032b464 2031 if (unlikely(po->origdev))
80feaacb
PWJ		 2032 sll->sll_ifindex = orig_dev->ifindex;
2033 else
2034 sll->sll_ifindex = dev->ifindex;
1da177e4 2035
e16aa207 2036 smp_mb();
f0d4eb29 2037
f6dafa95 2038#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2039 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2040 u8 *start, *end;
2041
f0d4eb29
DB		 2042 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2043 macoff + snaplen);
2044
2045 for (start = h.raw; start < end; start += PAGE_SIZE)
2046 flush_dcache_page(pgv_to_page(start));
1da177e4 2047 }
f0d4eb29 2048 smp_wmb();
f6dafa95 2049#endif
f0d4eb29 2050
f6fb8f10 2051 if (po->tp_version <= TPACKET_V2)
2052 __packet_set_status(po, h.raw, status);
2053 else
2054 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 2055
2056 sk->sk_data_ready(sk, 0);
2057
2058drop_n_restore:
2059 if (skb_head != skb->data && skb_shared(skb)) {
2060 skb->data = skb_head;
2061 skb->len = skb_len;
2062 }
2063drop:
1ce4f28b 2064 kfree_skb(skb);
1da177e4
LT		 2065 return 0;
2066
2067ring_is_full:
ee80fbf3 2068 po->stats.stats1.tp_drops++;
1da177e4
LT		 2069 spin_unlock(&sk->sk_receive_queue.lock);
2070
2071 sk->sk_data_ready(sk, 0);
acb5d75b 2072 kfree_skb(copy_skb);
1da177e4
LT		 2073 goto drop_n_restore;
2074}
2075
69e3c75f
JB		 2076static void tpacket_destruct_skb(struct sk_buff *skb)
2077{
2078 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2079
69e3c75f 2080 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2081 void *ph;
b9c32fb2
DB		 2082 __u32 ts;
2083
69e3c75f 2084 ph = skb_shinfo(skb)->destructor_arg;
b0138408 2085 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2086
2087 ts = __packet_set_timestamp(po, ph, skb);
2088 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2089 }
2090
2091 sock_wfree(skb);
2092}
2093
40d4e3df
ED		 2094static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2095 void *frame, struct net_device *dev, int size_max,
ae641949 2096 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 2097{
184f489e 2098 union tpacket_uhdr ph;
09effa67 2099 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 2100 struct socket *sock = po->sk.sk_socket;
2101 struct page *page;
2102 void *data;
2103 int err;
2104
2105 ph.raw = frame;
2106
2107 skb->protocol = proto;
2108 skb->dev = dev;
2109 skb->priority = po->sk.sk_priority;
2d37a186 2110 skb->mark = po->sk.sk_mark;
2e31396f 2111 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2112 skb_shinfo(skb)->destructor_arg = ph.raw;
2113
2114 switch (po->tp_version) {
2115 case TPACKET_V2:
2116 tp_len = ph.h2->tp_len;
2117 break;
2118 default:
2119 tp_len = ph.h1->tp_len;
2120 break;
2121 }
09effa67
DM		 2122 if (unlikely(tp_len > size_max)) {
2123 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2124 return -EMSGSIZE;
2125 }
69e3c75f 2126
ae641949 2127 skb_reserve(skb, hlen);
69e3c75f 2128 skb_reset_network_header(skb);
c1aad275 2129
d346a3fa
DB		 2130 if (!packet_use_direct_xmit(po))
2131 skb_probe_transport_header(skb, 0);
2132 if (unlikely(po->tp_tx_has_off)) {
5920cd3a
PC		 2133 int off_min, off_max, off;
2134 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2135 off_max = po->tx_ring.frame_size - tp_len;
2136 if (sock->type == SOCK_DGRAM) {
2137 switch (po->tp_version) {
2138 case TPACKET_V2:
2139 off = ph.h2->tp_net;
2140 break;
2141 default:
2142 off = ph.h1->tp_net;
2143 break;
2144 }
2145 } else {
2146 switch (po->tp_version) {
2147 case TPACKET_V2:
2148 off = ph.h2->tp_mac;
2149 break;
2150 default:
2151 off = ph.h1->tp_mac;
2152 break;
2153 }
2154 }
2155 if (unlikely((off < off_min) || (off_max < off)))
2156 return -EINVAL;
2157 data = ph.raw + off;
2158 } else {
2159 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2160 }
69e3c75f
JB		 2161 to_write = tp_len;
2162
2163 if (sock->type == SOCK_DGRAM) {
2164 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2165 NULL, tp_len);
2166 if (unlikely(err < 0))
2167 return -EINVAL;
40d4e3df 2168 } else if (dev->hard_header_len) {
69e3c75f
JB		 2169 /* net device doesn't like empty head */
2170 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2171 pr_err("packet size is too short (%d < %d)\n",
2172 tp_len, dev->hard_header_len);
69e3c75f
JB		 2173 return -EINVAL;
2174 }
2175
2176 skb_push(skb, dev->hard_header_len);
2177 err = skb_store_bits(skb, 0, data,
2178 dev->hard_header_len);
2179 if (unlikely(err))
2180 return err;
2181
2182 data += dev->hard_header_len;
2183 to_write -= dev->hard_header_len;
2184 }
2185
69e3c75f
JB		 2186 offset = offset_in_page(data);
2187 len_max = PAGE_SIZE - offset;
2188 len = ((to_write > len_max) ? len_max : to_write);
2189
2190 skb->data_len = to_write;
2191 skb->len += to_write;
2192 skb->truesize += to_write;
2193 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2194
2195 while (likely(to_write)) {
2196 nr_frags = skb_shinfo(skb)->nr_frags;
2197
2198 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2199 pr_err("Packet exceed the number of skb frags(%lu)\n",
2200 MAX_SKB_FRAGS);
69e3c75f
JB		 2201 return -EFAULT;
2202 }
2203
0af55bb5
CG		 2204 page = pgv_to_page(data);
2205 data += len;
69e3c75f
JB		 2206 flush_dcache_page(page);
2207 get_page(page);
0af55bb5 2208 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2209 to_write -= len;
2210 offset = 0;
2211 len_max = PAGE_SIZE;
2212 len = ((to_write > len_max) ? len_max : to_write);
2213 }
2214
2215 return tp_len;
2216}
2217
2218static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2219{
69e3c75f
JB		 2220 struct sk_buff *skb;
2221 struct net_device *dev;
2222 __be16 proto;
09effa67 2223 int err, reserve = 0;
40d4e3df 2224 void *ph;
342dfc30 2225 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2226 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
69e3c75f
JB		 2227 int tp_len, size_max;
2228 unsigned char *addr;
2229 int len_sum = 0;
9e67030a 2230 int status = TP_STATUS_AVAILABLE;
ae641949 2231 int hlen, tlen;
69e3c75f 2232
69e3c75f
JB		 2233 mutex_lock(&po->pg_vec_lock);
2234
66e56cd4 2235 if (likely(saddr == NULL)) {
e40526cb 2236 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2237 proto = po->num;
2238 addr = NULL;
2239 } else {
2240 err = -EINVAL;
2241 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2242 goto out;
2243 if (msg->msg_namelen < (saddr->sll_halen
2244 + offsetof(struct sockaddr_ll,
2245 sll_addr)))
2246 goto out;
69e3c75f
JB		 2247 proto = saddr->sll_protocol;
2248 addr = saddr->sll_addr;
827d9780 2249 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2250 }
2251
69e3c75f
JB		 2252 err = -ENXIO;
2253 if (unlikely(dev == NULL))
2254 goto out;
69e3c75f
JB		 2255 err = -ENETDOWN;
2256 if (unlikely(!(dev->flags & IFF_UP)))
2257 goto out_put;
2258
52f1454f 2259 reserve = dev->hard_header_len + VLAN_HLEN;
69e3c75f 2260 size_max = po->tx_ring.frame_size
b5dd884e 2261 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2262
09effa67
DM		 2263 if (size_max > dev->mtu + reserve)
2264 size_max = dev->mtu + reserve;
2265
69e3c75f
JB		 2266 do {
2267 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2268 TP_STATUS_SEND_REQUEST);
69e3c75f 2269 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2270 if (need_wait && need_resched())
2271 schedule();
69e3c75f
JB		 2272 continue;
2273 }
2274
2275 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2276 hlen = LL_RESERVED_SPACE(dev);
2277 tlen = dev->needed_tailroom;
69e3c75f 2278 skb = sock_alloc_send_skb(&po->sk,
ae641949 2279 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2280 0, &err);
2281
2282 if (unlikely(skb == NULL))
2283 goto out_status;
2284
2285 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
52f1454f
DB		 2286 addr, hlen);
2287 if (tp_len > dev->mtu + dev->hard_header_len) {
2288 struct ethhdr *ehdr;
2289 /* Earlier code assumed this would be a VLAN pkt,
2290 * double-check this now that we have the actual
2291 * packet in hand.
2292 */
69e3c75f 2293
52f1454f
DB		 2294 skb_reset_mac_header(skb);
2295 ehdr = eth_hdr(skb);
2296 if (ehdr->h_proto != htons(ETH_P_8021Q))
2297 tp_len = -EMSGSIZE;
2298 }
69e3c75f
JB		 2299 if (unlikely(tp_len < 0)) {
2300 if (po->tp_loss) {
2301 __packet_set_status(po, ph,
2302 TP_STATUS_AVAILABLE);
2303 packet_increment_head(&po->tx_ring);
2304 kfree_skb(skb);
2305 continue;
2306 } else {
2307 status = TP_STATUS_WRONG_FORMAT;
2308 err = tp_len;
2309 goto out_status;
2310 }
2311 }
2312
0fd5d57b
DB		 2313 packet_pick_tx_queue(dev, skb);
2314
69e3c75f
JB		 2315 skb->destructor = tpacket_destruct_skb;
2316 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2317 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2318
2319 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2320 err = po->xmit(skb);
eb70df13
JP		 2321 if (unlikely(err > 0)) {
2322 err = net_xmit_errno(err);
2323 if (err && __packet_get_status(po, ph) ==
2324 TP_STATUS_AVAILABLE) {
2325 /* skb was destructed already */
2326 skb = NULL;
2327 goto out_status;
2328 }
2329 /*
2330 * skb was dropped but not destructed yet;
2331 * let's treat it like congestion or err < 0
2332 */
2333 err = 0;
2334 }
69e3c75f
JB		 2335 packet_increment_head(&po->tx_ring);
2336 len_sum += tp_len;
b0138408
DB		 2337 } while (likely((ph != NULL) ||
2338 /* Note: packet_read_pending() might be slow if we have
2339 * to call it as it's per_cpu variable, but in fast-path
2340 * we already short-circuit the loop with the first
2341 * condition, and luckily don't have to go that path
2342 * anyway.
2343 */
2344 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2345
2346 err = len_sum;
2347 goto out_put;
2348
69e3c75f
JB		 2349out_status:
2350 __packet_set_status(po, ph, status);
2351 kfree_skb(skb);
2352out_put:
e40526cb 2353 dev_put(dev);
69e3c75f
JB		 2354out:
2355 mutex_unlock(&po->pg_vec_lock);
2356 return err;
2357}
69e3c75f 2358
eea49cc9
OJ		 2359static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2360 size_t reserve, size_t len,
2361 size_t linear, int noblock,
2362 int *err)
bfd5f4a3
SS		 2363{
2364 struct sk_buff *skb;
2365
2366 /* Under a page? Don't bother with paged skb. */
2367 if (prepad + len < PAGE_SIZE || !linear)
2368 linear = len;
2369
2370 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2371 err, 0);
bfd5f4a3
SS		 2372 if (!skb)
2373 return NULL;
2374
2375 skb_reserve(skb, reserve);
2376 skb_put(skb, linear);
2377 skb->data_len = len - linear;
2378 skb->len += len - linear;
2379
2380 return skb;
2381}
2382
d346a3fa 2383static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2384{
2385 struct sock *sk = sock->sk;
342dfc30 2386 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2387 struct sk_buff *skb;
2388 struct net_device *dev;
0e11c91e 2389 __be16 proto;
1da177e4 2390 unsigned char *addr;
827d9780 2391 int err, reserve = 0;
bfd5f4a3
SS		 2392 struct virtio_net_hdr vnet_hdr = { 0 };
2393 int offset = 0;
2394 int vnet_hdr_len;
2395 struct packet_sock *po = pkt_sk(sk);
2396 unsigned short gso_type = 0;
ae641949 2397 int hlen, tlen;
3bdc0eba 2398 int extra_len = 0;
1da177e4
LT		 2399
2400 /*
1ce4f28b 2401 * Get and verify the address.
1da177e4 2402 */
1ce4f28b 2403
66e56cd4 2404 if (likely(saddr == NULL)) {
e40526cb 2405 dev = packet_cached_dev_get(po);
1da177e4
LT		 2406 proto = po->num;
2407 addr = NULL;
2408 } else {
2409 err = -EINVAL;
2410 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2411 goto out;
0fb375fb
EB		 2412 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2413 goto out;
1da177e4
LT		 2414 proto = saddr->sll_protocol;
2415 addr = saddr->sll_addr;
827d9780 2416 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2417 }
2418
1da177e4 2419 err = -ENXIO;
e40526cb 2420 if (unlikely(dev == NULL))
1da177e4 2421 goto out_unlock;
d5e76b0a 2422 err = -ENETDOWN;
e40526cb 2423 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2424 goto out_unlock;
2425
e40526cb
DB		 2426 if (sock->type == SOCK_RAW)
2427 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2428 if (po->has_vnet_hdr) {
2429 vnet_hdr_len = sizeof(vnet_hdr);
2430
2431 err = -EINVAL;
2432 if (len < vnet_hdr_len)
2433 goto out_unlock;
2434
2435 len -= vnet_hdr_len;
2436
2437 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2438 vnet_hdr_len);
2439 if (err < 0)
2440 goto out_unlock;
2441
2442 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2443 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2444 vnet_hdr.hdr_len))
2445 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2446 vnet_hdr.csum_offset + 2;
2447
2448 err = -EINVAL;
2449 if (vnet_hdr.hdr_len > len)
2450 goto out_unlock;
2451
2452 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2453 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2454 case VIRTIO_NET_HDR_GSO_TCPV4:
2455 gso_type = SKB_GSO_TCPV4;
2456 break;
2457 case VIRTIO_NET_HDR_GSO_TCPV6:
2458 gso_type = SKB_GSO_TCPV6;
2459 break;
2460 case VIRTIO_NET_HDR_GSO_UDP:
2461 gso_type = SKB_GSO_UDP;
2462 break;
2463 default:
2464 goto out_unlock;
2465 }
2466
2467 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2468 gso_type |= SKB_GSO_TCP_ECN;
2469
2470 if (vnet_hdr.gso_size == 0)
2471 goto out_unlock;
2472
2473 }
2474 }
2475
3bdc0eba
BG		 2476 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2477 if (!netif_supports_nofcs(dev)) {
2478 err = -EPROTONOSUPPORT;
2479 goto out_unlock;
2480 }
2481 extra_len = 4; /* We're doing our own CRC */
2482 }
2483
1da177e4 2484 err = -EMSGSIZE;
3bdc0eba 2485 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2486 goto out_unlock;
2487
bfd5f4a3 2488 err = -ENOBUFS;
ae641949
HX		 2489 hlen = LL_RESERVED_SPACE(dev);
2490 tlen = dev->needed_tailroom;
2491 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2492 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2493 if (skb == NULL)
1da177e4
LT		 2494 goto out_unlock;
2495
bfd5f4a3 2496 skb_set_network_header(skb, reserve);
1da177e4 2497
0c4e8581
SH		 2498 err = -EINVAL;
2499 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2500 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2501 goto out_free;
1da177e4
LT		 2502
2503 /* Returns -EFAULT on error */
bfd5f4a3 2504 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2505 if (err)
2506 goto out_free;
bf84a010
DB		 2507
2508 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2509
3bdc0eba 2510 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2511 /* Earlier code assumed this would be a VLAN pkt,
2512 * double-check this now that we have the actual
2513 * packet in hand.
2514 */
2515 struct ethhdr *ehdr;
2516 skb_reset_mac_header(skb);
2517 ehdr = eth_hdr(skb);
2518 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2519 err = -EMSGSIZE;
2520 goto out_free;
2521 }
57f89bfa
BG		 2522 }
2523
09effa67
DM		 2524 skb->protocol = proto;
2525 skb->dev = dev;
1da177e4 2526 skb->priority = sk->sk_priority;
2d37a186 2527 skb->mark = sk->sk_mark;
0fd5d57b
DB		 2528
2529 packet_pick_tx_queue(dev, skb);
1da177e4 2530
bfd5f4a3
SS		 2531 if (po->has_vnet_hdr) {
2532 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2533 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2534 vnet_hdr.csum_offset)) {
2535 err = -EINVAL;
2536 goto out_free;
2537 }
2538 }
2539
2540 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2541 skb_shinfo(skb)->gso_type = gso_type;
2542
2543 /* Header must be checked, and gso_segs computed. */
2544 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2545 skb_shinfo(skb)->gso_segs = 0;
2546
2547 len += vnet_hdr_len;
2548 }
2549
d346a3fa
DB		 2550 if (!packet_use_direct_xmit(po))
2551 skb_probe_transport_header(skb, reserve);
3bdc0eba
BG		 2552 if (unlikely(extra_len == 4))
2553 skb->no_fcs = 1;
2554
d346a3fa 2555 err = po->xmit(skb);
1da177e4
LT		 2556 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2557 goto out_unlock;
2558
e40526cb 2559 dev_put(dev);
1da177e4 2560
40d4e3df 2561 return len;
1da177e4
LT		 2562
2563out_free:
2564 kfree_skb(skb);
2565out_unlock:
e40526cb 2566 if (dev)
1da177e4
LT		 2567 dev_put(dev);
2568out:
2569 return err;
2570}
2571
69e3c75f
JB		 2572static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2573 struct msghdr *msg, size_t len)
2574{
69e3c75f
JB		 2575 struct sock *sk = sock->sk;
2576 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2577
69e3c75f
JB		 2578 if (po->tx_ring.pg_vec)
2579 return tpacket_snd(po, msg);
2580 else
69e3c75f
JB		 2581 return packet_snd(sock, msg, len);
2582}
2583
1da177e4
LT		 2584/*
2585 * Close a PACKET socket. This is fairly simple. We immediately go
2586 * to 'closed' state and remove our protocol entry in the device list.
2587 */
2588
2589static int packet_release(struct socket *sock)
2590{
2591 struct sock *sk = sock->sk;
2592 struct packet_sock *po;
d12d01d6 2593 struct net *net;
f6fb8f10 2594 union tpacket_req_u req_u;
1da177e4
LT		 2595
2596 if (!sk)
2597 return 0;
2598
3b1e0a65 2599 net = sock_net(sk);
1da177e4
LT		 2600 po = pkt_sk(sk);
2601
0fa7fa98 2602 mutex_lock(&net->packet.sklist_lock);
808f5114 2603 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2604 mutex_unlock(&net->packet.sklist_lock);
2605
2606 preempt_disable();
920de804 2607 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2608 preempt_enable();
1da177e4 2609
808f5114 2610 spin_lock(&po->bind_lock);
ce06b03e 2611 unregister_prot_hook(sk, false);
66e56cd4
DB		 2612 packet_cached_dev_reset(po);
2613
160ff18a
BG		 2614 if (po->prot_hook.dev) {
2615 dev_put(po->prot_hook.dev);
2616 po->prot_hook.dev = NULL;
2617 }
808f5114 2618 spin_unlock(&po->bind_lock);
1da177e4 2619
1da177e4 2620 packet_flush_mclist(sk);
1da177e4 2621
9665d5d6
PS		 2622 if (po->rx_ring.pg_vec) {
2623 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2624 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2625 }
69e3c75f 2626
9665d5d6
PS		 2627 if (po->tx_ring.pg_vec) {
2628 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2629 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2630 }
1da177e4 2631
dc99f600
DM		 2632 fanout_release(sk);
2633
808f5114 2634 synchronize_net();
1da177e4
LT		 2635 /*
2636 * Now the socket is dead. No more input will appear.
2637 */
1da177e4
LT		 2638 sock_orphan(sk);
2639 sock->sk = NULL;
2640
2641 /* Purge queues */
2642
2643 skb_queue_purge(&sk->sk_receive_queue);
b0138408 2644 packet_free_pending(po);
17ab56a2 2645 sk_refcnt_debug_release(sk);
1da177e4
LT		 2646
2647 sock_put(sk);
2648 return 0;
2649}
2650
2651/*
2652 * Attach a packet hook.
2653 */
2654
902fefb8 2655static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
1da177e4
LT		 2656{
2657 struct packet_sock *po = pkt_sk(sk);
902fefb8
DB		 2658 const struct net_device *dev_curr;
2659 __be16 proto_curr;
2660 bool need_rehook;
dc99f600 2661
aef950b4
WY		 2662 if (po->fanout) {
2663 if (dev)
2664 dev_put(dev);
2665
dc99f600 2666 return -EINVAL;
aef950b4 2667 }
1da177e4
LT		 2668
2669 lock_sock(sk);
1da177e4 2670 spin_lock(&po->bind_lock);
66e56cd4 2671
902fefb8
DB		 2672 proto_curr = po->prot_hook.type;
2673 dev_curr = po->prot_hook.dev;
2674
2675 need_rehook = proto_curr != proto || dev_curr != dev;
2676
2677 if (need_rehook) {
2678 unregister_prot_hook(sk, true);
1da177e4 2679
902fefb8
DB		 2680 po->num = proto;
2681 po->prot_hook.type = proto;
1da177e4 2682
902fefb8
DB		 2683 if (po->prot_hook.dev)
2684 dev_put(po->prot_hook.dev);
2685
2686 po->prot_hook.dev = dev;
2687
2688 po->ifindex = dev ? dev->ifindex : 0;
2689 packet_cached_dev_assign(po, dev);
2690 }
66e56cd4 2691
902fefb8 2692 if (proto == 0 || !need_rehook)
1da177e4
LT		 2693 goto out_unlock;
2694
be85d4ad 2695 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2696 register_prot_hook(sk);
be85d4ad
UT		 2697 } else {
2698 sk->sk_err = ENETDOWN;
2699 if (!sock_flag(sk, SOCK_DEAD))
2700 sk->sk_error_report(sk);
1da177e4
LT		 2701 }
2702
2703out_unlock:
2704 spin_unlock(&po->bind_lock);
2705 release_sock(sk);
2706 return 0;
2707}
2708
2709/*
2710 * Bind a packet socket to a device
2711 */
2712
40d4e3df
ED		 2713static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2714 int addr_len)
1da177e4 2715{
40d4e3df 2716 struct sock *sk = sock->sk;
1da177e4
LT		 2717 char name[15];
2718 struct net_device *dev;
2719 int err = -ENODEV;
1ce4f28b 2720
1da177e4
LT		 2721 /*
2722 * Check legality
2723 */
1ce4f28b 2724
8ae55f04 2725 if (addr_len != sizeof(struct sockaddr))
1da177e4 2726 return -EINVAL;
40d4e3df 2727 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2728
3b1e0a65 2729 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2730 if (dev)
1da177e4 2731 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2732 return err;
2733}
1da177e4
LT		 2734
2735static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2736{
40d4e3df
ED		 2737 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2738 struct sock *sk = sock->sk;
1da177e4
LT		 2739 struct net_device *dev = NULL;
2740 int err;
2741
2742
2743 /*
2744 * Check legality
2745 */
1ce4f28b 2746
1da177e4
LT		 2747 if (addr_len < sizeof(struct sockaddr_ll))
2748 return -EINVAL;
2749 if (sll->sll_family != AF_PACKET)
2750 return -EINVAL;
2751
2752 if (sll->sll_ifindex) {
2753 err = -ENODEV;
3b1e0a65 2754 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2755 if (dev == NULL)
2756 goto out;
2757 }
2758 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2759
2760out:
2761 return err;
2762}
2763
2764static struct proto packet_proto = {
2765 .name = "PACKET",
2766 .owner = THIS_MODULE,
2767 .obj_size = sizeof(struct packet_sock),
2768};
2769
2770/*
1ce4f28b 2771 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2772 */
2773
3f378b68
EP		 2774static int packet_create(struct net *net, struct socket *sock, int protocol,
2775 int kern)
1da177e4
LT		 2776{
2777 struct sock *sk;
2778 struct packet_sock *po;
0e11c91e 2779 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2780 int err;
2781
df008c91 2782 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2783 return -EPERM;
be02097c
DM		 2784 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2785 sock->type != SOCK_PACKET)
1da177e4
LT		 2786 return -ESOCKTNOSUPPORT;
2787
2788 sock->state = SS_UNCONNECTED;
2789
2790 err = -ENOBUFS;
6257ff21 2791 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2792 if (sk == NULL)
2793 goto out;
2794
2795 sock->ops = &packet_ops;
1da177e4
LT		 2796 if (sock->type == SOCK_PACKET)
2797 sock->ops = &packet_ops_spkt;
be02097c 2798
1da177e4
LT		 2799 sock_init_data(sock, sk);
2800
2801 po = pkt_sk(sk);
2802 sk->sk_family = PF_PACKET;
0e11c91e 2803 po->num = proto;
d346a3fa 2804 po->xmit = dev_queue_xmit;
66e56cd4 2805
b0138408
DB		 2806 err = packet_alloc_pending(po);
2807 if (err)
2808 goto out2;
2809
66e56cd4 2810 packet_cached_dev_reset(po);
1da177e4
LT		 2811
2812 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2813 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2814
2815 /*
2816 * Attach a protocol block
2817 */
2818
2819 spin_lock_init(&po->bind_lock);
905db440 2820 mutex_init(&po->pg_vec_lock);
1da177e4 2821 po->prot_hook.func = packet_rcv;
be02097c 2822
1da177e4
LT		 2823 if (sock->type == SOCK_PACKET)
2824 po->prot_hook.func = packet_rcv_spkt;
be02097c 2825
1da177e4
LT		 2826 po->prot_hook.af_packet_priv = sk;
2827
0e11c91e
AV		 2828 if (proto) {
2829 po->prot_hook.type = proto;
ce06b03e 2830 register_prot_hook(sk);
1da177e4
LT		 2831 }
2832
0fa7fa98 2833 mutex_lock(&net->packet.sklist_lock);
808f5114 2834 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2835 mutex_unlock(&net->packet.sklist_lock);
2836
2837 preempt_disable();
3680453c 2838 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2839 preempt_enable();
808f5114 2840
40d4e3df 2841 return 0;
b0138408
DB		 2842out2:
2843 sk_free(sk);
1da177e4
LT		 2844out:
2845 return err;
2846}
2847
2848/*
2849 * Pull a packet from our receive queue and hand it to the user.
2850 * If necessary we block.
2851 */
2852
2853static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2854 struct msghdr *msg, size_t len, int flags)
2855{
2856 struct sock *sk = sock->sk;
2857 struct sk_buff *skb;
2858 int copied, err;
bfd5f4a3 2859 int vnet_hdr_len = 0;
1da177e4
LT		 2860
2861 err = -EINVAL;
ed85b565 2862 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2863 goto out;
2864
2865#if 0
2866 /* What error should we return now? EUNATTACH? */
2867 if (pkt_sk(sk)->ifindex < 0)
2868 return -ENODEV;
2869#endif
2870
ed85b565 2871 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2872 err = sock_recv_errqueue(sk, msg, len,
2873 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2874 goto out;
2875 }
2876
1da177e4
LT		 2877 /*
2878 * Call the generic datagram receiver. This handles all sorts
2879 * of horrible races and re-entrancy so we can forget about it
2880 * in the protocol layers.
2881 *
2882 * Now it will return ENETDOWN, if device have just gone down,
2883 * but then it will block.
2884 */
2885
40d4e3df 2886 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2887
2888 /*
1ce4f28b 2889 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2890 * handles the blocking we don't see and worry about blocking
2891 * retries.
2892 */
2893
8ae55f04 2894 if (skb == NULL)
1da177e4
LT		 2895 goto out;
2896
bfd5f4a3
SS		 2897 if (pkt_sk(sk)->has_vnet_hdr) {
2898 struct virtio_net_hdr vnet_hdr = { 0 };
2899
2900 err = -EINVAL;
2901 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2902 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2903 goto out_free;
2904
1f18b717
MK		 2905 len -= vnet_hdr_len;
2906
bfd5f4a3
SS		 2907 if (skb_is_gso(skb)) {
2908 struct skb_shared_info *sinfo = skb_shinfo(skb);
2909
2910 /* This is a hint as to how much should be linear. */
2911 vnet_hdr.hdr_len = skb_headlen(skb);
2912 vnet_hdr.gso_size = sinfo->gso_size;
2913 if (sinfo->gso_type & SKB_GSO_TCPV4)
2914 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2915 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2916 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2917 else if (sinfo->gso_type & SKB_GSO_UDP)
2918 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2919 else if (sinfo->gso_type & SKB_GSO_FCOE)
2920 goto out_free;
2921 else
2922 BUG();
2923 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2924 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2925 } else
2926 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2927
2928 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2929 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2930 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2931 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2932 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2933 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2934 } /* else everything is zero */
2935
2936 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2937 vnet_hdr_len);
2938 if (err < 0)
2939 goto out_free;
2940 }
2941
f3d33426
HFS		 2942 /* You lose any data beyond the buffer you gave. If it worries
2943 * a user program they can ask the device for its MTU
2944 * anyway.
1da177e4 2945 */
1da177e4 2946 copied = skb->len;
40d4e3df
ED		 2947 if (copied > len) {
2948 copied = len;
2949 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2950 }
2951
2952 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2953 if (err)
2954 goto out_free;
2955
3b885787 2956 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2957
f3d33426
HFS		 2958 if (msg->msg_name) {
2959 /* If the address length field is there to be filled
2960 * in, we fill it in now.
2961 */
2962 if (sock->type == SOCK_PACKET) {
342dfc30 2963 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426
HFS		 2964 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2965 } else {
2966 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2967 msg->msg_namelen = sll->sll_halen +
2968 offsetof(struct sockaddr_ll, sll_addr);
2969 }
ffbc6111
HX		 2970 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2971 msg->msg_namelen);
f3d33426 2972 }
1da177e4 2973
8dc41944 2974 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2975 struct tpacket_auxdata aux;
2976
2977 aux.tp_status = TP_STATUS_USER;
2978 if (skb->ip_summed == CHECKSUM_PARTIAL)
2979 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2980 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2981 aux.tp_snaplen = skb->len;
2982 aux.tp_mac = 0;
bbe735e4 2983 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2984 if (vlan_tx_tag_present(skb)) {
2985 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 2986 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
2987 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2988 } else {
2989 aux.tp_vlan_tci = 0;
a0cdfcf3 2990 aux.tp_vlan_tpid = 0;
a3bcc23e 2991 }
ffbc6111 2992 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2993 }
2994
1da177e4
LT		 2995 /*
2996 * Free or return the buffer as appropriate. Again this
2997 * hides all the races and re-entrancy issues from us.
2998 */
bfd5f4a3 2999 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 3000
3001out_free:
3002 skb_free_datagram(sk, skb);
3003out:
3004 return err;
3005}
3006
1da177e4
LT		 3007static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3008 int *uaddr_len, int peer)
3009{
3010 struct net_device *dev;
3011 struct sock *sk = sock->sk;
3012
3013 if (peer)
3014 return -EOPNOTSUPP;
3015
3016 uaddr->sa_family = AF_PACKET;
2dc85bf3 3017 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 3018 rcu_read_lock();
3019 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3020 if (dev)
2dc85bf3 3021 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 3022 rcu_read_unlock();
1da177e4
LT		 3023 *uaddr_len = sizeof(*uaddr);
3024
3025 return 0;
3026}
1da177e4
LT		 3027
3028static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3029 int *uaddr_len, int peer)
3030{
3031 struct net_device *dev;
3032 struct sock *sk = sock->sk;
3033 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3034 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3035
3036 if (peer)
3037 return -EOPNOTSUPP;
3038
3039 sll->sll_family = AF_PACKET;
3040 sll->sll_ifindex = po->ifindex;
3041 sll->sll_protocol = po->num;
67286640 3042 sll->sll_pkttype = 0;
654d1f8a
ED		 3043 rcu_read_lock();
3044 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3045 if (dev) {
3046 sll->sll_hatype = dev->type;
3047 sll->sll_halen = dev->addr_len;
3048 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3049 } else {
3050 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3051 sll->sll_halen = 0;
3052 }
654d1f8a 3053 rcu_read_unlock();
0fb375fb 3054 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3055
3056 return 0;
3057}
3058
2aeb0b88
WC		 3059static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3060 int what)
1da177e4
LT		 3061{
3062 switch (i->type) {
3063 case PACKET_MR_MULTICAST:
1162563f
JP		 3064 if (i->alen != dev->addr_len)
3065 return -EINVAL;
1da177e4 3066 if (what > 0)
22bedad3 3067 return dev_mc_add(dev, i->addr);
1da177e4 3068 else
22bedad3 3069 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3070 break;
3071 case PACKET_MR_PROMISC:
2aeb0b88 3072 return dev_set_promiscuity(dev, what);
1da177e4
LT		 3073 break;
3074 case PACKET_MR_ALLMULTI:
2aeb0b88 3075 return dev_set_allmulti(dev, what);
1da177e4 3076 break;
d95ed927 3077 case PACKET_MR_UNICAST:
1162563f
JP		 3078 if (i->alen != dev->addr_len)
3079 return -EINVAL;
d95ed927 3080 if (what > 0)
a748ee24 3081 return dev_uc_add(dev, i->addr);
d95ed927 3082 else
a748ee24 3083 return dev_uc_del(dev, i->addr);
d95ed927 3084 break;
40d4e3df
ED		 3085 default:
3086 break;
1da177e4 3087 }
2aeb0b88 3088 return 0;
1da177e4
LT		 3089}
3090
3091static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
3092{
40d4e3df 3093 for ( ; i; i = i->next) {
1da177e4
LT		 3094 if (i->ifindex == dev->ifindex)
3095 packet_dev_mc(dev, i, what);
3096 }
3097}
3098
0fb375fb 3099static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3100{
3101 struct packet_sock *po = pkt_sk(sk);
3102 struct packet_mclist *ml, *i;
3103 struct net_device *dev;
3104 int err;
3105
3106 rtnl_lock();
3107
3108 err = -ENODEV;
3b1e0a65 3109 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3110 if (!dev)
3111 goto done;
3112
3113 err = -EINVAL;
1162563f 3114 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3115 goto done;
3116
3117 err = -ENOBUFS;
8b3a7005 3118 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3119 if (i == NULL)
3120 goto done;
3121
3122 err = 0;
3123 for (ml = po->mclist; ml; ml = ml->next) {
3124 if (ml->ifindex == mreq->mr_ifindex &&
3125 ml->type == mreq->mr_type &&
3126 ml->alen == mreq->mr_alen &&
3127 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3128 ml->count++;
3129 /* Free the new element ... */
3130 kfree(i);
3131 goto done;
3132 }
3133 }
3134
3135 i->type = mreq->mr_type;
3136 i->ifindex = mreq->mr_ifindex;
3137 i->alen = mreq->mr_alen;
3138 memcpy(i->addr, mreq->mr_address, i->alen);
3139 i->count = 1;
3140 i->next = po->mclist;
3141 po->mclist = i;
2aeb0b88
WC		 3142 err = packet_dev_mc(dev, i, 1);
3143 if (err) {
3144 po->mclist = i->next;
3145 kfree(i);
3146 }
1da177e4
LT		 3147
3148done:
3149 rtnl_unlock();
3150 return err;
3151}
3152
0fb375fb 3153static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3154{
3155 struct packet_mclist *ml, **mlp;
3156
3157 rtnl_lock();
3158
3159 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3160 if (ml->ifindex == mreq->mr_ifindex &&
3161 ml->type == mreq->mr_type &&
3162 ml->alen == mreq->mr_alen &&
3163 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3164 if (--ml->count == 0) {
3165 struct net_device *dev;
3166 *mlp = ml->next;
ad959e76
ED		 3167 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3168 if (dev)
1da177e4 3169 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3170 kfree(ml);
3171 }
3172 rtnl_unlock();
3173 return 0;
3174 }
3175 }
3176 rtnl_unlock();
3177 return -EADDRNOTAVAIL;
3178}
3179
3180static void packet_flush_mclist(struct sock *sk)
3181{
3182 struct packet_sock *po = pkt_sk(sk);
3183 struct packet_mclist *ml;
3184
3185 if (!po->mclist)
3186 return;
3187
3188 rtnl_lock();
3189 while ((ml = po->mclist) != NULL) {
3190 struct net_device *dev;
3191
3192 po->mclist = ml->next;
ad959e76
ED		 3193 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3194 if (dev != NULL)
1da177e4 3195 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3196 kfree(ml);
3197 }
3198 rtnl_unlock();
3199}
1da177e4
LT		 3200
3201static int
b7058842 3202packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3203{
3204 struct sock *sk = sock->sk;
8dc41944 3205 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3206 int ret;
3207
3208 if (level != SOL_PACKET)
3209 return -ENOPROTOOPT;
3210
69e3c75f 3211 switch (optname) {
1ce4f28b 3212 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3213 case PACKET_DROP_MEMBERSHIP:
3214 {
0fb375fb
EB		 3215 struct packet_mreq_max mreq;
3216 int len = optlen;
3217 memset(&mreq, 0, sizeof(mreq));
3218 if (len < sizeof(struct packet_mreq))
1da177e4 3219 return -EINVAL;
0fb375fb
EB		 3220 if (len > sizeof(mreq))
3221 len = sizeof(mreq);
40d4e3df 3222 if (copy_from_user(&mreq, optval, len))
1da177e4 3223 return -EFAULT;
0fb375fb
EB		 3224 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3225 return -EINVAL;
1da177e4
LT		 3226 if (optname == PACKET_ADD_MEMBERSHIP)
3227 ret = packet_mc_add(sk, &mreq);
3228 else
3229 ret = packet_mc_drop(sk, &mreq);
3230 return ret;
3231 }
a2efcfa0 3232
1da177e4 3233 case PACKET_RX_RING:
69e3c75f 3234 case PACKET_TX_RING:
1da177e4 3235 {
f6fb8f10 3236 union tpacket_req_u req_u;
3237 int len;
1da177e4 3238
f6fb8f10 3239 switch (po->tp_version) {
3240 case TPACKET_V1:
3241 case TPACKET_V2:
3242 len = sizeof(req_u.req);
3243 break;
3244 case TPACKET_V3:
3245 default:
3246 len = sizeof(req_u.req3);
3247 break;
3248 }
3249 if (optlen < len)
1da177e4 3250 return -EINVAL;
bfd5f4a3
SS		 3251 if (pkt_sk(sk)->has_vnet_hdr)
3252 return -EINVAL;
f6fb8f10 3253 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3254 return -EFAULT;
f6fb8f10 3255 return packet_set_ring(sk, &req_u, 0,
3256 optname == PACKET_TX_RING);
1da177e4
LT		 3257 }
3258 case PACKET_COPY_THRESH:
3259 {
3260 int val;
3261
40d4e3df 3262 if (optlen != sizeof(val))
1da177e4 3263 return -EINVAL;
40d4e3df 3264 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3265 return -EFAULT;
3266
3267 pkt_sk(sk)->copy_thresh = val;
3268 return 0;
3269 }
bbd6ef87
PM		 3270 case PACKET_VERSION:
3271 {
3272 int val;
3273
3274 if (optlen != sizeof(val))
3275 return -EINVAL;
69e3c75f 3276 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3277 return -EBUSY;
3278 if (copy_from_user(&val, optval, sizeof(val)))
3279 return -EFAULT;
3280 switch (val) {
3281 case TPACKET_V1:
3282 case TPACKET_V2:
f6fb8f10 3283 case TPACKET_V3:
bbd6ef87
PM		 3284 po->tp_version = val;
3285 return 0;
3286 default:
3287 return -EINVAL;
3288 }
3289 }
8913336a
PM		 3290 case PACKET_RESERVE:
3291 {
3292 unsigned int val;
3293
3294 if (optlen != sizeof(val))
3295 return -EINVAL;
69e3c75f 3296 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3297 return -EBUSY;
3298 if (copy_from_user(&val, optval, sizeof(val)))
3299 return -EFAULT;
3300 po->tp_reserve = val;
3301 return 0;
3302 }
69e3c75f
JB		 3303 case PACKET_LOSS:
3304 {
3305 unsigned int val;
3306
3307 if (optlen != sizeof(val))
3308 return -EINVAL;
3309 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3310 return -EBUSY;
3311 if (copy_from_user(&val, optval, sizeof(val)))
3312 return -EFAULT;
3313 po->tp_loss = !!val;
3314 return 0;
3315 }
8dc41944
HX		 3316 case PACKET_AUXDATA:
3317 {
3318 int val;
3319
3320 if (optlen < sizeof(val))
3321 return -EINVAL;
3322 if (copy_from_user(&val, optval, sizeof(val)))
3323 return -EFAULT;
3324
3325 po->auxdata = !!val;
3326 return 0;
3327 }
80feaacb
PWJ		 3328 case PACKET_ORIGDEV:
3329 {
3330 int val;
3331
3332 if (optlen < sizeof(val))
3333 return -EINVAL;
3334 if (copy_from_user(&val, optval, sizeof(val)))
3335 return -EFAULT;
3336
3337 po->origdev = !!val;
3338 return 0;
3339 }
bfd5f4a3
SS		 3340 case PACKET_VNET_HDR:
3341 {
3342 int val;
3343
3344 if (sock->type != SOCK_RAW)
3345 return -EINVAL;
3346 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3347 return -EBUSY;
3348 if (optlen < sizeof(val))
3349 return -EINVAL;
3350 if (copy_from_user(&val, optval, sizeof(val)))
3351 return -EFAULT;
3352
3353 po->has_vnet_hdr = !!val;
3354 return 0;
3355 }
614f60fa
SM		 3356 case PACKET_TIMESTAMP:
3357 {
3358 int val;
3359
3360 if (optlen != sizeof(val))
3361 return -EINVAL;
3362 if (copy_from_user(&val, optval, sizeof(val)))
3363 return -EFAULT;
3364
3365 po->tp_tstamp = val;
3366 return 0;
3367 }
dc99f600
DM		 3368 case PACKET_FANOUT:
3369 {
3370 int val;
3371
3372 if (optlen != sizeof(val))
3373 return -EINVAL;
3374 if (copy_from_user(&val, optval, sizeof(val)))
3375 return -EFAULT;
3376
3377 return fanout_add(sk, val & 0xffff, val >> 16);
3378 }
5920cd3a
PC		 3379 case PACKET_TX_HAS_OFF:
3380 {
3381 unsigned int val;
3382
3383 if (optlen != sizeof(val))
3384 return -EINVAL;
3385 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3386 return -EBUSY;
3387 if (copy_from_user(&val, optval, sizeof(val)))
3388 return -EFAULT;
3389 po->tp_tx_has_off = !!val;
3390 return 0;
3391 }
d346a3fa
DB		 3392 case PACKET_QDISC_BYPASS:
3393 {
3394 int val;
3395
3396 if (optlen != sizeof(val))
3397 return -EINVAL;
3398 if (copy_from_user(&val, optval, sizeof(val)))
3399 return -EFAULT;
3400
3401 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3402 return 0;
3403 }
1da177e4
LT		 3404 default:
3405 return -ENOPROTOOPT;
3406 }
3407}
3408
3409static int packet_getsockopt(struct socket *sock, int level, int optname,
3410 char __user *optval, int __user *optlen)
3411{
3412 int len;
c06fff6e 3413 int val, lv = sizeof(val);
1da177e4
LT		 3414 struct sock *sk = sock->sk;
3415 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3416 void *data = &val;
ee80fbf3 3417 union tpacket_stats_u st;
1da177e4
LT		 3418
3419 if (level != SOL_PACKET)
3420 return -ENOPROTOOPT;
3421
8ae55f04
KK		 3422 if (get_user(len, optlen))
3423 return -EFAULT;
1da177e4
LT		 3424
3425 if (len < 0)
3426 return -EINVAL;
1ce4f28b 3427
69e3c75f 3428 switch (optname) {
1da177e4 3429 case PACKET_STATISTICS:
1da177e4 3430 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3431 memcpy(&st, &po->stats, sizeof(st));
3432 memset(&po->stats, 0, sizeof(po->stats));
3433 spin_unlock_bh(&sk->sk_receive_queue.lock);
3434
f6fb8f10 3435 if (po->tp_version == TPACKET_V3) {
c06fff6e 3436 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3437 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3438 data = &st.stats3;
f6fb8f10 3439 } else {
c06fff6e 3440 lv = sizeof(struct tpacket_stats);
8bcdeaff 3441 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3442 data = &st.stats1;
f6fb8f10 3443 }
ee80fbf3 3444
8dc41944
HX		 3445 break;
3446 case PACKET_AUXDATA:
8dc41944 3447 val = po->auxdata;
80feaacb
PWJ		 3448 break;
3449 case PACKET_ORIGDEV:
80feaacb 3450 val = po->origdev;
bfd5f4a3
SS		 3451 break;
3452 case PACKET_VNET_HDR:
bfd5f4a3 3453 val = po->has_vnet_hdr;
1da177e4 3454 break;
bbd6ef87 3455 case PACKET_VERSION:
bbd6ef87 3456 val = po->tp_version;
bbd6ef87
PM		 3457 break;
3458 case PACKET_HDRLEN:
3459 if (len > sizeof(int))
3460 len = sizeof(int);
3461 if (copy_from_user(&val, optval, len))
3462 return -EFAULT;
3463 switch (val) {
3464 case TPACKET_V1:
3465 val = sizeof(struct tpacket_hdr);
3466 break;
3467 case TPACKET_V2:
3468 val = sizeof(struct tpacket2_hdr);
3469 break;
f6fb8f10 3470 case TPACKET_V3:
3471 val = sizeof(struct tpacket3_hdr);
3472 break;
bbd6ef87
PM		 3473 default:
3474 return -EINVAL;
3475 }
bbd6ef87 3476 break;
8913336a 3477 case PACKET_RESERVE:
8913336a 3478 val = po->tp_reserve;
8913336a 3479 break;
69e3c75f 3480 case PACKET_LOSS:
69e3c75f 3481 val = po->tp_loss;
69e3c75f 3482 break;
614f60fa 3483 case PACKET_TIMESTAMP:
614f60fa 3484 val = po->tp_tstamp;
614f60fa 3485 break;
dc99f600 3486 case PACKET_FANOUT:
dc99f600
DM		 3487 val = (po->fanout ?
3488 ((u32)po->fanout->id |
77f65ebd
WB		 3489 ((u32)po->fanout->type << 16) |
3490 ((u32)po->fanout->flags << 24)) :
dc99f600 3491 0);
dc99f600 3492 break;
5920cd3a
PC		 3493 case PACKET_TX_HAS_OFF:
3494 val = po->tp_tx_has_off;
3495 break;
d346a3fa
DB		 3496 case PACKET_QDISC_BYPASS:
3497 val = packet_use_direct_xmit(po);
3498 break;
1da177e4
LT		 3499 default:
3500 return -ENOPROTOOPT;
3501 }
3502
c06fff6e
ED		 3503 if (len > lv)
3504 len = lv;
8ae55f04
KK		 3505 if (put_user(len, optlen))
3506 return -EFAULT;
8dc41944
HX		 3507 if (copy_to_user(optval, data, len))
3508 return -EFAULT;
8ae55f04 3509 return 0;
1da177e4
LT		 3510}
3511
3512
351638e7
JP		 3513static int packet_notifier(struct notifier_block *this,
3514 unsigned long msg, void *ptr)
1da177e4
LT		 3515{
3516 struct sock *sk;
351638e7 3517 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3518 struct net *net = dev_net(dev);
1da177e4 3519
808f5114 3520 rcu_read_lock();
b67bfe0d 3521 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3522 struct packet_sock *po = pkt_sk(sk);
3523
3524 switch (msg) {
3525 case NETDEV_UNREGISTER:
1da177e4
LT		 3526 if (po->mclist)
3527 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3528 /* fallthrough */
3529
1da177e4
LT		 3530 case NETDEV_DOWN:
3531 if (dev->ifindex == po->ifindex) {
3532 spin_lock(&po->bind_lock);
3533 if (po->running) {
ce06b03e 3534 __unregister_prot_hook(sk, false);
1da177e4
LT		 3535 sk->sk_err = ENETDOWN;
3536 if (!sock_flag(sk, SOCK_DEAD))
3537 sk->sk_error_report(sk);
3538 }
3539 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3540 packet_cached_dev_reset(po);
1da177e4 3541 po->ifindex = -1;
160ff18a
BG		 3542 if (po->prot_hook.dev)
3543 dev_put(po->prot_hook.dev);
1da177e4
LT		 3544 po->prot_hook.dev = NULL;
3545 }
3546 spin_unlock(&po->bind_lock);
3547 }
3548 break;
3549 case NETDEV_UP:
808f5114 3550 if (dev->ifindex == po->ifindex) {
3551 spin_lock(&po->bind_lock);
ce06b03e
DM		 3552 if (po->num)
3553 register_prot_hook(sk);
808f5114 3554 spin_unlock(&po->bind_lock);
1da177e4 3555 }
1da177e4
LT		 3556 break;
3557 }
3558 }
808f5114 3559 rcu_read_unlock();
1da177e4
LT		 3560 return NOTIFY_DONE;
3561}
3562
3563
3564static int packet_ioctl(struct socket *sock, unsigned int cmd,
3565 unsigned long arg)
3566{
3567 struct sock *sk = sock->sk;
3568
69e3c75f 3569 switch (cmd) {
40d4e3df
ED		 3570 case SIOCOUTQ:
3571 {
3572 int amount = sk_wmem_alloc_get(sk);
31e6d363 3573
40d4e3df
ED		 3574 return put_user(amount, (int __user *)arg);
3575 }
3576 case SIOCINQ:
3577 {
3578 struct sk_buff *skb;
3579 int amount = 0;
3580
3581 spin_lock_bh(&sk->sk_receive_queue.lock);
3582 skb = skb_peek(&sk->sk_receive_queue);
3583 if (skb)
3584 amount = skb->len;
3585 spin_unlock_bh(&sk->sk_receive_queue.lock);
3586 return put_user(amount, (int __user *)arg);
3587 }
3588 case SIOCGSTAMP:
3589 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3590 case SIOCGSTAMPNS:
3591 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3592
1da177e4 3593#ifdef CONFIG_INET
40d4e3df
ED		 3594 case SIOCADDRT:
3595 case SIOCDELRT:
3596 case SIOCDARP:
3597 case SIOCGARP:
3598 case SIOCSARP:
3599 case SIOCGIFADDR:
3600 case SIOCSIFADDR:
3601 case SIOCGIFBRDADDR:
3602 case SIOCSIFBRDADDR:
3603 case SIOCGIFNETMASK:
3604 case SIOCSIFNETMASK:
3605 case SIOCGIFDSTADDR:
3606 case SIOCSIFDSTADDR:
3607 case SIOCSIFFLAGS:
40d4e3df 3608 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3609#endif
3610
40d4e3df
ED		 3611 default:
3612 return -ENOIOCTLCMD;
1da177e4
LT		 3613 }
3614 return 0;
3615}
3616
40d4e3df 3617static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3618 poll_table *wait)
3619{
3620 struct sock *sk = sock->sk;
3621 struct packet_sock *po = pkt_sk(sk);
3622 unsigned int mask = datagram_poll(file, sock, wait);
3623
3624 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3625 if (po->rx_ring.pg_vec) {
f6fb8f10 3626 if (!packet_previous_rx_frame(po, &po->rx_ring,
3627 TP_STATUS_KERNEL))
1da177e4
LT		 3628 mask |= POLLIN | POLLRDNORM;
3629 }
3630 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3631 spin_lock_bh(&sk->sk_write_queue.lock);
3632 if (po->tx_ring.pg_vec) {
3633 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3634 mask |= POLLOUT | POLLWRNORM;
3635 }
3636 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3637 return mask;
3638}
3639
3640
3641/* Dirty? Well, I still did not learn better way to account
3642 * for user mmaps.
3643 */
3644
3645static void packet_mm_open(struct vm_area_struct *vma)
3646{
3647 struct file *file = vma->vm_file;
40d4e3df 3648 struct socket *sock = file->private_data;
1da177e4 3649 struct sock *sk = sock->sk;
1ce4f28b 3650
1da177e4
LT		 3651 if (sk)
3652 atomic_inc(&pkt_sk(sk)->mapped);
3653}
3654
3655static void packet_mm_close(struct vm_area_struct *vma)
3656{
3657 struct file *file = vma->vm_file;
40d4e3df 3658 struct socket *sock = file->private_data;
1da177e4 3659 struct sock *sk = sock->sk;
1ce4f28b 3660
1da177e4
LT		 3661 if (sk)
3662 atomic_dec(&pkt_sk(sk)->mapped);
3663}
3664
f0f37e2f 3665static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3666 .open = packet_mm_open,
3667 .close = packet_mm_close,
1da177e4
LT		 3668};
3669
0e3125c7
NH		 3670static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3671 unsigned int len)
1da177e4
LT		 3672{
3673 int i;
3674
4ebf0ae2 3675 for (i = 0; i < len; i++) {
0e3125c7 3676 if (likely(pg_vec[i].buffer)) {
c56b4d90 3677 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3678 vfree(pg_vec[i].buffer);
3679 else
3680 free_pages((unsigned long)pg_vec[i].buffer,
3681 order);
3682 pg_vec[i].buffer = NULL;
3683 }
1da177e4
LT		 3684 }
3685 kfree(pg_vec);
3686}
3687
eea49cc9 3688static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3689{
f0d4eb29 3690 char *buffer;
0e3125c7
NH		 3691 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3692 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3693
3694 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3695 if (buffer)
3696 return buffer;
3697
f0d4eb29 3698 /* __get_free_pages failed, fall back to vmalloc */
bbce5a59 3699 buffer = vzalloc((1 << order) * PAGE_SIZE);
0e3125c7
NH		 3700 if (buffer)
3701 return buffer;
3702
f0d4eb29 3703 /* vmalloc failed, lets dig into swap here */
0e3125c7 3704 gfp_flags &= ~__GFP_NORETRY;
f0d4eb29 3705 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3706 if (buffer)
3707 return buffer;
3708
f0d4eb29 3709 /* complete and utter failure */
0e3125c7 3710 return NULL;
4ebf0ae2
DM		 3711}
3712
0e3125c7 3713static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3714{
3715 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3716 struct pgv *pg_vec;
4ebf0ae2
DM		 3717 int i;
3718
0e3125c7 3719 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3720 if (unlikely(!pg_vec))
3721 goto out;
3722
3723 for (i = 0; i < block_nr; i++) {
c56b4d90 3724 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3725 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3726 goto out_free_pgvec;
3727 }
3728
3729out:
3730 return pg_vec;
3731
3732out_free_pgvec:
3733 free_pg_vec(pg_vec, order, block_nr);
3734 pg_vec = NULL;
3735 goto out;
3736}
1da177e4 3737
f6fb8f10 3738static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3739 int closing, int tx_ring)
1da177e4 3740{
0e3125c7 3741 struct pgv *pg_vec = NULL;
1da177e4 3742 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3743 int was_running, order = 0;
69e3c75f
JB		 3744 struct packet_ring_buffer *rb;
3745 struct sk_buff_head *rb_queue;
0e11c91e 3746 __be16 num;
f6fb8f10 3747 int err = -EINVAL;
3748 /* Added to avoid minimal code churn */
3749 struct tpacket_req *req = &req_u->req;
3750
3751 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3752 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3753 WARN(1, "Tx-ring is not supported.\n");
3754 goto out;
3755 }
1ce4f28b 3756
69e3c75f
JB		 3757 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3758 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3759
69e3c75f
JB		 3760 err = -EBUSY;
3761 if (!closing) {
3762 if (atomic_read(&po->mapped))
3763 goto out;
b0138408 3764 if (packet_read_pending(rb))
69e3c75f
JB		 3765 goto out;
3766 }
1da177e4 3767
69e3c75f
JB		 3768 if (req->tp_block_nr) {
3769 /* Sanity tests and some calculations */
3770 err = -EBUSY;
3771 if (unlikely(rb->pg_vec))
3772 goto out;
1da177e4 3773
bbd6ef87
PM		 3774 switch (po->tp_version) {
3775 case TPACKET_V1:
3776 po->tp_hdrlen = TPACKET_HDRLEN;
3777 break;
3778 case TPACKET_V2:
3779 po->tp_hdrlen = TPACKET2_HDRLEN;
3780 break;
f6fb8f10 3781 case TPACKET_V3:
3782 po->tp_hdrlen = TPACKET3_HDRLEN;
3783 break;
bbd6ef87
PM		 3784 }
3785
69e3c75f 3786 err = -EINVAL;
4ebf0ae2 3787 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3788 goto out;
4ebf0ae2 3789 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3790 goto out;
8913336a 3791 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3792 po->tp_reserve))
3793 goto out;
4ebf0ae2 3794 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3795 goto out;
1da177e4 3796
69e3c75f
JB		 3797 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3798 if (unlikely(rb->frames_per_block <= 0))
3799 goto out;
3800 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3801 req->tp_frame_nr))
3802 goto out;
1da177e4
LT		 3803
3804 err = -ENOMEM;
4ebf0ae2
DM		 3805 order = get_order(req->tp_block_size);
3806 pg_vec = alloc_pg_vec(req, order);
3807 if (unlikely(!pg_vec))
1da177e4 3808 goto out;
f6fb8f10 3809 switch (po->tp_version) {
3810 case TPACKET_V3:
3811 /* Transmit path is not supported. We checked
3812 * it above but just being paranoid
3813 */
3814 if (!tx_ring)
3815 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
d7cf0c34 3816 break;
f6fb8f10 3817 default:
3818 break;
3819 }
69e3c75f
JB		 3820 }
3821 /* Done */
3822 else {
3823 err = -EINVAL;
4ebf0ae2 3824 if (unlikely(req->tp_frame_nr))
69e3c75f 3825 goto out;
1da177e4
LT		 3826 }
3827
3828 lock_sock(sk);
3829
3830 /* Detach socket from network */
3831 spin_lock(&po->bind_lock);
3832 was_running = po->running;
3833 num = po->num;
3834 if (was_running) {
1da177e4 3835 po->num = 0;
ce06b03e 3836 __unregister_prot_hook(sk, false);
1da177e4
LT		 3837 }
3838 spin_unlock(&po->bind_lock);
1ce4f28b 3839
1da177e4
LT		 3840 synchronize_net();
3841
3842 err = -EBUSY;
905db440 3843 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3844 if (closing || atomic_read(&po->mapped) == 0) {
3845 err = 0;
69e3c75f 3846 spin_lock_bh(&rb_queue->lock);
c053fd96 3847 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3848 rb->frame_max = (req->tp_frame_nr - 1);
3849 rb->head = 0;
3850 rb->frame_size = req->tp_frame_size;
3851 spin_unlock_bh(&rb_queue->lock);
3852
c053fd96
CG		 3853 swap(rb->pg_vec_order, order);
3854 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3855
3856 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3857 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3858 tpacket_rcv : packet_rcv;
3859 skb_queue_purge(rb_queue);
1da177e4 3860 if (atomic_read(&po->mapped))
40d4e3df
ED		 3861 pr_err("packet_mmap: vma is busy: %d\n",
3862 atomic_read(&po->mapped));
1da177e4 3863 }
905db440 3864 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3865
3866 spin_lock(&po->bind_lock);
ce06b03e 3867 if (was_running) {
1da177e4 3868 po->num = num;
ce06b03e 3869 register_prot_hook(sk);
1da177e4
LT		 3870 }
3871 spin_unlock(&po->bind_lock);
f6fb8f10 3872 if (closing && (po->tp_version > TPACKET_V2)) {
3873 /* Because we don't support block-based V3 on tx-ring */
3874 if (!tx_ring)
3875 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3876 }
1da177e4
LT		 3877 release_sock(sk);
3878
1da177e4
LT		 3879 if (pg_vec)
3880 free_pg_vec(pg_vec, order, req->tp_block_nr);
3881out:
3882 return err;
3883}
3884
69e3c75f
JB		 3885static int packet_mmap(struct file *file, struct socket *sock,
3886 struct vm_area_struct *vma)
1da177e4
LT		 3887{
3888 struct sock *sk = sock->sk;
3889 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3890 unsigned long size, expected_size;
3891 struct packet_ring_buffer *rb;
1da177e4
LT		 3892 unsigned long start;
3893 int err = -EINVAL;
3894 int i;
3895
3896 if (vma->vm_pgoff)
3897 return -EINVAL;
3898
905db440 3899 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3900
3901 expected_size = 0;
3902 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3903 if (rb->pg_vec) {
3904 expected_size += rb->pg_vec_len
3905 * rb->pg_vec_pages
3906 * PAGE_SIZE;
3907 }
3908 }
3909
3910 if (expected_size == 0)
1da177e4 3911 goto out;
69e3c75f
JB		 3912
3913 size = vma->vm_end - vma->vm_start;
3914 if (size != expected_size)
1da177e4
LT		 3915 goto out;
3916
1da177e4 3917 start = vma->vm_start;
69e3c75f
JB		 3918 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3919 if (rb->pg_vec == NULL)
3920 continue;
3921
3922 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3923 struct page *page;
3924 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3925 int pg_num;
3926
c56b4d90
CG		 3927 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3928 page = pgv_to_page(kaddr);
69e3c75f
JB		 3929 err = vm_insert_page(vma, start, page);
3930 if (unlikely(err))
3931 goto out;
3932 start += PAGE_SIZE;
0e3125c7 3933 kaddr += PAGE_SIZE;
69e3c75f 3934 }
4ebf0ae2 3935 }
1da177e4 3936 }
69e3c75f 3937
4ebf0ae2 3938 atomic_inc(&po->mapped);
1da177e4
LT		 3939 vma->vm_ops = &packet_mmap_ops;
3940 err = 0;
3941
3942out:
905db440 3943 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3944 return err;
3945}
1da177e4 3946
90ddc4f0 3947static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3948 .family = PF_PACKET,
3949 .owner = THIS_MODULE,
3950 .release = packet_release,
3951 .bind = packet_bind_spkt,
3952 .connect = sock_no_connect,
3953 .socketpair = sock_no_socketpair,
3954 .accept = sock_no_accept,
3955 .getname = packet_getname_spkt,
3956 .poll = datagram_poll,
3957 .ioctl = packet_ioctl,
3958 .listen = sock_no_listen,
3959 .shutdown = sock_no_shutdown,
3960 .setsockopt = sock_no_setsockopt,
3961 .getsockopt = sock_no_getsockopt,
3962 .sendmsg = packet_sendmsg_spkt,
3963 .recvmsg = packet_recvmsg,
3964 .mmap = sock_no_mmap,
3965 .sendpage = sock_no_sendpage,
3966};
1da177e4 3967
90ddc4f0 3968static const struct proto_ops packet_ops = {
1da177e4
LT		 3969 .family = PF_PACKET,
3970 .owner = THIS_MODULE,
3971 .release = packet_release,
3972 .bind = packet_bind,
3973 .connect = sock_no_connect,
3974 .socketpair = sock_no_socketpair,
3975 .accept = sock_no_accept,
1ce4f28b 3976 .getname = packet_getname,
1da177e4
LT		 3977 .poll = packet_poll,
3978 .ioctl = packet_ioctl,
3979 .listen = sock_no_listen,
3980 .shutdown = sock_no_shutdown,
3981 .setsockopt = packet_setsockopt,
3982 .getsockopt = packet_getsockopt,
3983 .sendmsg = packet_sendmsg,
3984 .recvmsg = packet_recvmsg,
3985 .mmap = packet_mmap,
3986 .sendpage = sock_no_sendpage,
3987};
3988
ec1b4cf7 3989static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3990 .family = PF_PACKET,
3991 .create = packet_create,
3992 .owner = THIS_MODULE,
3993};
3994
3995static struct notifier_block packet_netdev_notifier = {
40d4e3df 3996 .notifier_call = packet_notifier,
1da177e4
LT		 3997};
3998
3999#ifdef CONFIG_PROC_FS
1da177e4
LT		 4000
4001static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 4002 __acquires(RCU)
1da177e4 4003{
e372c414 4004 struct net *net = seq_file_net(seq);
808f5114 4005
4006 rcu_read_lock();
4007 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 4008}
4009
4010static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4011{
1bf40954 4012 struct net *net = seq_file_net(seq);
808f5114 4013 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 4014}
4015
4016static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 4017 __releases(RCU)
1da177e4 4018{
808f5114 4019 rcu_read_unlock();
1da177e4
LT		 4020}
4021
1ce4f28b 4022static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 4023{
4024 if (v == SEQ_START_TOKEN)
4025 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4026 else {
b7ceabd9 4027 struct sock *s = sk_entry(v);
1da177e4
LT		 4028 const struct packet_sock *po = pkt_sk(s);
4029
4030 seq_printf(seq,
71338aa7 4031 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 4032 s,
4033 atomic_read(&s->sk_refcnt),
4034 s->sk_type,
4035 ntohs(po->num),
4036 po->ifindex,
4037 po->running,
4038 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4039 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4040 sock_i_ino(s));
1da177e4
LT		 4041 }
4042
4043 return 0;
4044}
4045
56b3d975 4046static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4047 .start = packet_seq_start,
4048 .next = packet_seq_next,
4049 .stop = packet_seq_stop,
4050 .show = packet_seq_show,
4051};
4052
4053static int packet_seq_open(struct inode *inode, struct file *file)
4054{
e372c414
DL		 4055 return seq_open_net(inode, file, &packet_seq_ops,
4056 sizeof(struct seq_net_private));
1da177e4
LT		 4057}
4058
da7071d7 4059static const struct file_operations packet_seq_fops = {
1da177e4
LT		 4060 .owner = THIS_MODULE,
4061 .open = packet_seq_open,
4062 .read = seq_read,
4063 .llseek = seq_lseek,
e372c414 4064 .release = seq_release_net,
1da177e4
LT		 4065};
4066
4067#endif
4068
2c8c1e72 4069static int __net_init packet_net_init(struct net *net)
d12d01d6 4070{
0fa7fa98 4071 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4072 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4073
d4beaa66 4074 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 4075 return -ENOMEM;
4076
4077 return 0;
4078}
4079
2c8c1e72 4080static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4081{
ece31ffd 4082 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 4083}
4084
4085static struct pernet_operations packet_net_ops = {
4086 .init = packet_net_init,
4087 .exit = packet_net_exit,
4088};
4089
4090
1da177e4
LT		 4091static void __exit packet_exit(void)
4092{
1da177e4 4093 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4094 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4095 sock_unregister(PF_PACKET);
4096 proto_unregister(&packet_proto);
4097}
4098
4099static int __init packet_init(void)
4100{
4101 int rc = proto_register(&packet_proto, 0);
4102
4103 if (rc != 0)
4104 goto out;
4105
4106 sock_register(&packet_family_ops);
d12d01d6 4107 register_pernet_subsys(&packet_net_ops);
1da177e4 4108 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4109out:
4110 return rc;
4111}
4112
4113module_init(packet_init);
4114module_exit(packet_exit);
4115MODULE_LICENSE("GPL");
4116MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 1.125927 seconds and 5 git commands to generate.