projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'hyperv'
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
b0138408 91#include <linux/percpu.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
d346a3fa
DB		 240static int packet_direct_xmit(struct sk_buff *skb)
241{
242 struct net_device *dev = skb->dev;
243 const struct net_device_ops *ops = dev->netdev_ops;
244 netdev_features_t features;
245 struct netdev_queue *txq;
43279500 246 int ret = NETDEV_TX_BUSY;
d346a3fa 247 u16 queue_map;
d346a3fa
DB		 248
249 if (unlikely(!netif_running(dev) ||
43279500
DB		 250 !netif_carrier_ok(dev)))
251 goto drop;
d346a3fa
DB		 252
253 features = netif_skb_features(skb);
254 if (skb_needs_linearize(skb, features) &&
43279500
DB		 255 __skb_linearize(skb))
256 goto drop;
d346a3fa
DB		 257
258 queue_map = skb_get_queue_mapping(skb);
259 txq = netdev_get_tx_queue(dev, queue_map);
260
43279500
DB		 261 local_bh_disable();
262
263 HARD_TX_LOCK(dev, txq, smp_processor_id());
8e2f1a63 264 if (!netif_xmit_frozen_or_drv_stopped(txq)) {
43279500
DB		 265 ret = ops->ndo_start_xmit(skb, dev);
266 if (ret == NETDEV_TX_OK)
267 txq_trans_update(txq);
d346a3fa 268 }
43279500 269 HARD_TX_UNLOCK(dev, txq);
d346a3fa 270
43279500
DB		 271 local_bh_enable();
272
273 if (!dev_xmit_complete(ret))
d346a3fa 274 kfree_skb(skb);
43279500 275
d346a3fa 276 return ret;
43279500 277drop:
0f97ede4 278 atomic_long_inc(&dev->tx_dropped);
43279500
DB		 279 kfree_skb(skb);
280 return NET_XMIT_DROP;
d346a3fa
DB		 281}
282
66e56cd4
DB		 283static struct net_device *packet_cached_dev_get(struct packet_sock *po)
284{
285 struct net_device *dev;
286
287 rcu_read_lock();
288 dev = rcu_dereference(po->cached_dev);
289 if (likely(dev))
290 dev_hold(dev);
291 rcu_read_unlock();
292
293 return dev;
294}
295
296static void packet_cached_dev_assign(struct packet_sock *po,
297 struct net_device *dev)
298{
299 rcu_assign_pointer(po->cached_dev, dev);
300}
301
302static void packet_cached_dev_reset(struct packet_sock *po)
303{
304 RCU_INIT_POINTER(po->cached_dev, NULL);
305}
306
d346a3fa
DB		 307static bool packet_use_direct_xmit(const struct packet_sock *po)
308{
309 return po->xmit == packet_direct_xmit;
310}
311
0fd5d57b 312static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
d346a3fa 313{
1cbac010 314 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 315}
316
0fd5d57b
DB		 317static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
318{
319 const struct net_device_ops *ops = dev->netdev_ops;
320 u16 queue_index;
321
322 if (ops->ndo_select_queue) {
323 queue_index = ops->ndo_select_queue(dev, skb, NULL,
324 __packet_pick_tx_queue);
325 queue_index = netdev_cap_txqueue(dev, queue_index);
326 } else {
327 queue_index = __packet_pick_tx_queue(dev, skb);
328 }
329
330 skb_set_queue_mapping(skb, queue_index);
331}
332
ce06b03e
DM		 333/* register_prot_hook must be invoked with the po->bind_lock held,
334 * or from a context in which asynchronous accesses to the packet
335 * socket is not possible (packet_create()).
336 */
337static void register_prot_hook(struct sock *sk)
338{
339 struct packet_sock *po = pkt_sk(sk);
e40526cb 340
ce06b03e 341 if (!po->running) {
66e56cd4 342 if (po->fanout)
dc99f600 343 __fanout_link(sk, po);
66e56cd4 344 else
dc99f600 345 dev_add_pack(&po->prot_hook);
e40526cb 346
ce06b03e
DM		 347 sock_hold(sk);
348 po->running = 1;
349 }
350}
351
352/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
353 * held. If the sync parameter is true, we will temporarily drop
354 * the po->bind_lock and do a synchronize_net to make sure no
355 * asynchronous packet processing paths still refer to the elements
356 * of po->prot_hook. If the sync parameter is false, it is the
357 * callers responsibility to take care of this.
358 */
359static void __unregister_prot_hook(struct sock *sk, bool sync)
360{
361 struct packet_sock *po = pkt_sk(sk);
362
363 po->running = 0;
66e56cd4
DB		 364
365 if (po->fanout)
dc99f600 366 __fanout_unlink(sk, po);
66e56cd4 367 else
dc99f600 368 __dev_remove_pack(&po->prot_hook);
e40526cb 369
ce06b03e
DM		 370 __sock_put(sk);
371
372 if (sync) {
373 spin_unlock(&po->bind_lock);
374 synchronize_net();
375 spin_lock(&po->bind_lock);
376 }
377}
378
379static void unregister_prot_hook(struct sock *sk, bool sync)
380{
381 struct packet_sock *po = pkt_sk(sk);
382
383 if (po->running)
384 __unregister_prot_hook(sk, sync);
385}
386
f6dafa95 387static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 388{
389 if (is_vmalloc_addr(addr))
390 return vmalloc_to_page(addr);
391 return virt_to_page(addr);
392}
393
69e3c75f 394static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 395{
184f489e 396 union tpacket_uhdr h;
1da177e4 397
69e3c75f 398 h.raw = frame;
bbd6ef87
PM		 399 switch (po->tp_version) {
400 case TPACKET_V1:
69e3c75f 401 h.h1->tp_status = status;
0af55bb5 402 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 403 break;
404 case TPACKET_V2:
69e3c75f 405 h.h2->tp_status = status;
0af55bb5 406 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 407 break;
f6fb8f10 408 case TPACKET_V3:
69e3c75f 409 default:
f6fb8f10 410 WARN(1, "TPACKET version not supported.\n");
69e3c75f 411 BUG();
bbd6ef87 412 }
69e3c75f
JB		 413
414 smp_wmb();
bbd6ef87
PM		 415}
416
69e3c75f 417static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 418{
184f489e 419 union tpacket_uhdr h;
bbd6ef87 420
69e3c75f
JB		 421 smp_rmb();
422
bbd6ef87
PM		 423 h.raw = frame;
424 switch (po->tp_version) {
425 case TPACKET_V1:
0af55bb5 426 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 427 return h.h1->tp_status;
bbd6ef87 428 case TPACKET_V2:
0af55bb5 429 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 430 return h.h2->tp_status;
f6fb8f10 431 case TPACKET_V3:
69e3c75f 432 default:
f6fb8f10 433 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 434 BUG();
435 return 0;
bbd6ef87 436 }
1da177e4 437}
69e3c75f 438
b9c32fb2
DB		 439static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
440 unsigned int flags)
7a51384c
DB		 441{
442 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
443
444 if (shhwtstamps) {
445 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
446 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
b9c32fb2 447 return TP_STATUS_TS_SYS_HARDWARE;
7a51384c
DB		 448 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
449 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
b9c32fb2 450 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 451 }
452
453 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 454 return TP_STATUS_TS_SOFTWARE;
7a51384c 455
b9c32fb2 456 return 0;
7a51384c
DB		 457}
458
b9c32fb2
DB		 459static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
460 struct sk_buff *skb)
2e31396f
WB		 461{
462 union tpacket_uhdr h;
463 struct timespec ts;
b9c32fb2 464 __u32 ts_status;
2e31396f 465
b9c32fb2
DB		 466 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
467 return 0;
2e31396f
WB		 468
469 h.raw = frame;
470 switch (po->tp_version) {
471 case TPACKET_V1:
472 h.h1->tp_sec = ts.tv_sec;
473 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
474 break;
475 case TPACKET_V2:
476 h.h2->tp_sec = ts.tv_sec;
477 h.h2->tp_nsec = ts.tv_nsec;
478 break;
479 case TPACKET_V3:
480 default:
481 WARN(1, "TPACKET version not supported.\n");
482 BUG();
483 }
484
485 /* one flush is safe, as both fields always lie on the same cacheline */
486 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
487 smp_wmb();
b9c32fb2
DB		 488
489 return ts_status;
2e31396f
WB		 490}
491
69e3c75f
JB		 492static void *packet_lookup_frame(struct packet_sock *po,
493 struct packet_ring_buffer *rb,
494 unsigned int position,
495 int status)
496{
497 unsigned int pg_vec_pos, frame_offset;
184f489e 498 union tpacket_uhdr h;
69e3c75f
JB		 499
500 pg_vec_pos = position / rb->frames_per_block;
501 frame_offset = position % rb->frames_per_block;
502
0e3125c7
NH		 503 h.raw = rb->pg_vec[pg_vec_pos].buffer +
504 (frame_offset * rb->frame_size);
69e3c75f
JB		 505
506 if (status != __packet_get_status(po, h.raw))
507 return NULL;
508
509 return h.raw;
510}
511
eea49cc9 512static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 513 struct packet_ring_buffer *rb,
514 int status)
515{
516 return packet_lookup_frame(po, rb, rb->head, status);
517}
518
bc59ba39 519static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 520{
521 del_timer_sync(&pkc->retire_blk_timer);
522}
523
524static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
525 int tx_ring,
526 struct sk_buff_head *rb_queue)
527{
bc59ba39 528 struct tpacket_kbdq_core *pkc;
f6fb8f10 529
22781a5b
DJ		 530 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
531 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 532
ec6f809f 533 spin_lock_bh(&rb_queue->lock);
f6fb8f10 534 pkc->delete_blk_timer = 1;
ec6f809f 535 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 536
537 prb_del_retire_blk_timer(pkc);
538}
539
540static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 541 struct tpacket_kbdq_core *pkc,
f6fb8f10 542 void (*func) (unsigned long))
543{
544 init_timer(&pkc->retire_blk_timer);
545 pkc->retire_blk_timer.data = (long)po;
546 pkc->retire_blk_timer.function = func;
547 pkc->retire_blk_timer.expires = jiffies;
548}
549
550static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
551{
bc59ba39 552 struct tpacket_kbdq_core *pkc;
f6fb8f10 553
554 if (tx_ring)
555 BUG();
556
22781a5b
DJ		 557 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
558 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 559 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
560}
561
562static int prb_calc_retire_blk_tmo(struct packet_sock *po,
563 int blk_size_in_bytes)
564{
565 struct net_device *dev;
566 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 567 struct ethtool_cmd ecmd;
568 int err;
e440cf2c 569 u32 speed;
f6fb8f10 570
4bc71cb9
JP		 571 rtnl_lock();
572 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
573 if (unlikely(!dev)) {
574 rtnl_unlock();
f6fb8f10 575 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 576 }
577 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 578 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 579 rtnl_unlock();
580 if (!err) {
4bc71cb9
JP		 581 /*
582 * If the link speed is so slow you don't really
583 * need to worry about perf anyways
584 */
e440cf2c 585 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 586 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 587 } else {
588 msec = 1;
589 div = speed / 1000;
f6fb8f10 590 }
591 }
592
593 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
594
595 if (div)
596 mbits /= div;
597
598 tmo = mbits * msec;
599
600 if (div)
601 return tmo+1;
602 return tmo;
603}
604
bc59ba39 605static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 606 union tpacket_req_u *req_u)
607{
608 p1->feature_req_word = req_u->req3.tp_feature_req_word;
609}
610
611static void init_prb_bdqc(struct packet_sock *po,
612 struct packet_ring_buffer *rb,
613 struct pgv *pg_vec,
614 union tpacket_req_u *req_u, int tx_ring)
615{
22781a5b 616 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 617 struct tpacket_block_desc *pbd;
f6fb8f10 618
619 memset(p1, 0x0, sizeof(*p1));
620
621 p1->knxt_seq_num = 1;
622 p1->pkbdq = pg_vec;
bc59ba39 623 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 624 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 625 p1->kblk_size = req_u->req3.tp_block_size;
626 p1->knum_blocks = req_u->req3.tp_block_nr;
627 p1->hdrlen = po->tp_hdrlen;
628 p1->version = po->tp_version;
629 p1->last_kactive_blk_num = 0;
ee80fbf3 630 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 631 if (req_u->req3.tp_retire_blk_tov)
632 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
633 else
634 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
635 req_u->req3.tp_block_size);
636 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
637 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
638
639 prb_init_ft_ops(p1, req_u);
640 prb_setup_retire_blk_timer(po, tx_ring);
641 prb_open_block(p1, pbd);
642}
643
644/* Do NOT update the last_blk_num first.
645 * Assumes sk_buff_head lock is held.
646 */
bc59ba39 647static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 648{
649 mod_timer(&pkc->retire_blk_timer,
650 jiffies + pkc->tov_in_jiffies);
651 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
652}
653
654/*
655 * Timer logic:
656 * 1) We refresh the timer only when we open a block.
657 * By doing this we don't waste cycles refreshing the timer
658 * on packet-by-packet basis.
659 *
660 * With a 1MB block-size, on a 1Gbps line, it will take
661 * i) ~8 ms to fill a block + ii) memcpy etc.
662 * In this cut we are not accounting for the memcpy time.
663 *
664 * So, if the user sets the 'tmo' to 10ms then the timer
665 * will never fire while the block is still getting filled
666 * (which is what we want). However, the user could choose
667 * to close a block early and that's fine.
668 *
669 * But when the timer does fire, we check whether or not to refresh it.
670 * Since the tmo granularity is in msecs, it is not too expensive
671 * to refresh the timer, lets say every '8' msecs.
672 * Either the user can set the 'tmo' or we can derive it based on
673 * a) line-speed and b) block-size.
674 * prb_calc_retire_blk_tmo() calculates the tmo.
675 *
676 */
677static void prb_retire_rx_blk_timer_expired(unsigned long data)
678{
679 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 680 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 681 unsigned int frozen;
bc59ba39 682 struct tpacket_block_desc *pbd;
f6fb8f10 683
684 spin_lock(&po->sk.sk_receive_queue.lock);
685
686 frozen = prb_queue_frozen(pkc);
687 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
688
689 if (unlikely(pkc->delete_blk_timer))
690 goto out;
691
692 /* We only need to plug the race when the block is partially filled.
693 * tpacket_rcv:
694 * lock(); increment BLOCK_NUM_PKTS; unlock()
695 * copy_bits() is in progress ...
696 * timer fires on other cpu:
697 * we can't retire the current block because copy_bits
698 * is in progress.
699 *
700 */
701 if (BLOCK_NUM_PKTS(pbd)) {
702 while (atomic_read(&pkc->blk_fill_in_prog)) {
703 /* Waiting for skb_copy_bits to finish... */
704 cpu_relax();
705 }
706 }
707
708 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
709 if (!frozen) {
710 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
711 if (!prb_dispatch_next_block(pkc, po))
712 goto refresh_timer;
713 else
714 goto out;
715 } else {
716 /* Case 1. Queue was frozen because user-space was
717 * lagging behind.
718 */
719 if (prb_curr_blk_in_use(pkc, pbd)) {
720 /*
721 * Ok, user-space is still behind.
722 * So just refresh the timer.
723 */
724 goto refresh_timer;
725 } else {
726 /* Case 2. queue was frozen,user-space caught up,
727 * now the link went idle && the timer fired.
728 * We don't have a block to close.So we open this
729 * block and restart the timer.
730 * opening a block thaws the queue,restarts timer
731 * Thawing/timer-refresh is a side effect.
732 */
733 prb_open_block(pkc, pbd);
734 goto out;
735 }
736 }
737 }
738
739refresh_timer:
740 _prb_refresh_rx_retire_blk_timer(pkc);
741
742out:
743 spin_unlock(&po->sk.sk_receive_queue.lock);
744}
745
eea49cc9 746static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 747 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 748{
749 /* Flush everything minus the block header */
750
751#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
752 u8 *start, *end;
753
754 start = (u8 *)pbd1;
755
756 /* Skip the block header(we know header WILL fit in 4K) */
757 start += PAGE_SIZE;
758
759 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
760 for (; start < end; start += PAGE_SIZE)
761 flush_dcache_page(pgv_to_page(start));
762
763 smp_wmb();
764#endif
765
766 /* Now update the block status. */
767
768 BLOCK_STATUS(pbd1) = status;
769
770 /* Flush the block header */
771
772#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
773 start = (u8 *)pbd1;
774 flush_dcache_page(pgv_to_page(start));
775
776 smp_wmb();
777#endif
778}
779
780/*
781 * Side effect:
782 *
783 * 1) flush the block
784 * 2) Increment active_blk_num
785 *
786 * Note:We DONT refresh the timer on purpose.
787 * Because almost always the next block will be opened.
788 */
bc59ba39 789static void prb_close_block(struct tpacket_kbdq_core *pkc1,
790 struct tpacket_block_desc *pbd1,
f6fb8f10 791 struct packet_sock *po, unsigned int stat)
792{
793 __u32 status = TP_STATUS_USER | stat;
794
795 struct tpacket3_hdr *last_pkt;
bc59ba39 796 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 797
ee80fbf3 798 if (po->stats.stats3.tp_drops)
f6fb8f10 799 status |= TP_STATUS_LOSING;
800
801 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
802 last_pkt->tp_next_offset = 0;
803
804 /* Get the ts of the last pkt */
805 if (BLOCK_NUM_PKTS(pbd1)) {
806 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
807 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
808 } else {
809 /* Ok, we tmo'd - so get the current time */
810 struct timespec ts;
811 getnstimeofday(&ts);
812 h1->ts_last_pkt.ts_sec = ts.tv_sec;
813 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
814 }
815
816 smp_wmb();
817
818 /* Flush the block */
819 prb_flush_block(pkc1, pbd1, status);
820
821 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
822}
823
eea49cc9 824static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 825{
826 pkc->reset_pending_on_curr_blk = 0;
827}
828
829/*
830 * Side effect of opening a block:
831 *
832 * 1) prb_queue is thawed.
833 * 2) retire_blk_timer is refreshed.
834 *
835 */
bc59ba39 836static void prb_open_block(struct tpacket_kbdq_core *pkc1,
837 struct tpacket_block_desc *pbd1)
f6fb8f10 838{
839 struct timespec ts;
bc59ba39 840 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 841
842 smp_rmb();
843
8da3056c
DB		 844 /* We could have just memset this but we will lose the
845 * flexibility of making the priv area sticky
846 */
f6fb8f10 847
8da3056c
DB		 848 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
849 BLOCK_NUM_PKTS(pbd1) = 0;
850 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 851
8da3056c
DB		 852 getnstimeofday(&ts);
853
854 h1->ts_first_pkt.ts_sec = ts.tv_sec;
855 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 856
8da3056c
DB		 857 pkc1->pkblk_start = (char *)pbd1;
858 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
859
860 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
861 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
862
863 pbd1->version = pkc1->version;
864 pkc1->prev = pkc1->nxt_offset;
865 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
866
867 prb_thaw_queue(pkc1);
868 _prb_refresh_rx_retire_blk_timer(pkc1);
869
870 smp_wmb();
f6fb8f10 871}
872
873/*
874 * Queue freeze logic:
875 * 1) Assume tp_block_nr = 8 blocks.
876 * 2) At time 't0', user opens Rx ring.
877 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
878 * 4) user-space is either sleeping or processing block '0'.
879 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
880 * it will close block-7,loop around and try to fill block '0'.
881 * call-flow:
882 * __packet_lookup_frame_in_block
883 * prb_retire_current_block()
884 * prb_dispatch_next_block()
885 * |->(BLOCK_STATUS == USER) evaluates to true
886 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
887 * 6) Now there are two cases:
888 * 6.1) Link goes idle right after the queue is frozen.
889 * But remember, the last open_block() refreshed the timer.
890 * When this timer expires,it will refresh itself so that we can
891 * re-open block-0 in near future.
892 * 6.2) Link is busy and keeps on receiving packets. This is a simple
893 * case and __packet_lookup_frame_in_block will check if block-0
894 * is free and can now be re-used.
895 */
eea49cc9 896static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 897 struct packet_sock *po)
898{
899 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 900 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 901}
902
903#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
904
905/*
906 * If the next block is free then we will dispatch it
907 * and return a good offset.
908 * Else, we will freeze the queue.
909 * So, caller must check the return value.
910 */
bc59ba39 911static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 912 struct packet_sock *po)
913{
bc59ba39 914 struct tpacket_block_desc *pbd;
f6fb8f10 915
916 smp_rmb();
917
918 /* 1. Get current block num */
919 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
920
921 /* 2. If this block is currently in_use then freeze the queue */
922 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
923 prb_freeze_queue(pkc, po);
924 return NULL;
925 }
926
927 /*
928 * 3.
929 * open this block and return the offset where the first packet
930 * needs to get stored.
931 */
932 prb_open_block(pkc, pbd);
933 return (void *)pkc->nxt_offset;
934}
935
bc59ba39 936static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 937 struct packet_sock *po, unsigned int status)
938{
bc59ba39 939 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 940
941 /* retire/close the current block */
942 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
943 /*
944 * Plug the case where copy_bits() is in progress on
945 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
946 * have space to copy the pkt in the current block and
947 * called prb_retire_current_block()
948 *
949 * We don't need to worry about the TMO case because
950 * the timer-handler already handled this case.
951 */
952 if (!(status & TP_STATUS_BLK_TMO)) {
953 while (atomic_read(&pkc->blk_fill_in_prog)) {
954 /* Waiting for skb_copy_bits to finish... */
955 cpu_relax();
956 }
957 }
958 prb_close_block(pkc, pbd, po, status);
959 return;
960 }
f6fb8f10 961}
962
eea49cc9 963static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 964 struct tpacket_block_desc *pbd)
f6fb8f10 965{
966 return TP_STATUS_USER & BLOCK_STATUS(pbd);
967}
968
eea49cc9 969static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 970{
971 return pkc->reset_pending_on_curr_blk;
972}
973
eea49cc9 974static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 975{
bc59ba39 976 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 977 atomic_dec(&pkc->blk_fill_in_prog);
978}
979
eea49cc9 980static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 981 struct tpacket3_hdr *ppd)
982{
3958afa1 983 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 984}
985
eea49cc9 986static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 987 struct tpacket3_hdr *ppd)
988{
989 ppd->hv1.tp_rxhash = 0;
990}
991
eea49cc9 992static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 993 struct tpacket3_hdr *ppd)
994{
995 if (vlan_tx_tag_present(pkc->skb)) {
996 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
a0cdfcf3
AW		 997 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
998 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 999 } else {
9e67030a 1000 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 1001 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 1002 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 1003 }
1004}
1005
bc59ba39 1006static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 1007 struct tpacket3_hdr *ppd)
1008{
a0cdfcf3 1009 ppd->hv1.tp_padding = 0;
f6fb8f10 1010 prb_fill_vlan_info(pkc, ppd);
1011
1012 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1013 prb_fill_rxhash(pkc, ppd);
1014 else
1015 prb_clear_rxhash(pkc, ppd);
1016}
1017
eea49cc9 1018static void prb_fill_curr_block(char *curr,
bc59ba39 1019 struct tpacket_kbdq_core *pkc,
1020 struct tpacket_block_desc *pbd,
f6fb8f10 1021 unsigned int len)
1022{
1023 struct tpacket3_hdr *ppd;
1024
1025 ppd = (struct tpacket3_hdr *)curr;
1026 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1027 pkc->prev = curr;
1028 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1029 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1030 BLOCK_NUM_PKTS(pbd) += 1;
1031 atomic_inc(&pkc->blk_fill_in_prog);
1032 prb_run_all_ft_ops(pkc, ppd);
1033}
1034
1035/* Assumes caller has the sk->rx_queue.lock */
1036static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1037 struct sk_buff *skb,
1038 int status,
1039 unsigned int len
1040 )
1041{
bc59ba39 1042 struct tpacket_kbdq_core *pkc;
1043 struct tpacket_block_desc *pbd;
f6fb8f10 1044 char *curr, *end;
1045
e3192690 1046 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1047 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1048
1049 /* Queue is frozen when user space is lagging behind */
1050 if (prb_queue_frozen(pkc)) {
1051 /*
1052 * Check if that last block which caused the queue to freeze,
1053 * is still in_use by user-space.
1054 */
1055 if (prb_curr_blk_in_use(pkc, pbd)) {
1056 /* Can't record this packet */
1057 return NULL;
1058 } else {
1059 /*
1060 * Ok, the block was released by user-space.
1061 * Now let's open that block.
1062 * opening a block also thaws the queue.
1063 * Thawing is a side effect.
1064 */
1065 prb_open_block(pkc, pbd);
1066 }
1067 }
1068
1069 smp_mb();
1070 curr = pkc->nxt_offset;
1071 pkc->skb = skb;
e3192690 1072 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1073
1074 /* first try the current block */
1075 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1076 prb_fill_curr_block(curr, pkc, pbd, len);
1077 return (void *)curr;
1078 }
1079
1080 /* Ok, close the current block */
1081 prb_retire_current_block(pkc, po, 0);
1082
1083 /* Now, try to dispatch the next block */
1084 curr = (char *)prb_dispatch_next_block(pkc, po);
1085 if (curr) {
1086 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1087 prb_fill_curr_block(curr, pkc, pbd, len);
1088 return (void *)curr;
1089 }
1090
1091 /*
1092 * No free blocks are available.user_space hasn't caught up yet.
1093 * Queue was just frozen and now this packet will get dropped.
1094 */
1095 return NULL;
1096}
1097
eea49cc9 1098static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1099 struct sk_buff *skb,
1100 int status, unsigned int len)
1101{
1102 char *curr = NULL;
1103 switch (po->tp_version) {
1104 case TPACKET_V1:
1105 case TPACKET_V2:
1106 curr = packet_lookup_frame(po, &po->rx_ring,
1107 po->rx_ring.head, status);
1108 return curr;
1109 case TPACKET_V3:
1110 return __packet_lookup_frame_in_block(po, skb, status, len);
1111 default:
1112 WARN(1, "TPACKET version not supported\n");
1113 BUG();
99aa3473 1114 return NULL;
f6fb8f10 1115 }
1116}
1117
eea49cc9 1118static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1119 struct packet_ring_buffer *rb,
77f65ebd 1120 unsigned int idx,
f6fb8f10 1121 int status)
1122{
bc59ba39 1123 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1124 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1125
1126 if (status != BLOCK_STATUS(pbd))
1127 return NULL;
1128 return pbd;
1129}
1130
eea49cc9 1131static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1132{
1133 unsigned int prev;
1134 if (rb->prb_bdqc.kactive_blk_num)
1135 prev = rb->prb_bdqc.kactive_blk_num-1;
1136 else
1137 prev = rb->prb_bdqc.knum_blocks-1;
1138 return prev;
1139}
1140
1141/* Assumes caller has held the rx_queue.lock */
eea49cc9 1142static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1143 struct packet_ring_buffer *rb,
1144 int status)
1145{
1146 unsigned int previous = prb_previous_blk_num(rb);
1147 return prb_lookup_block(po, rb, previous, status);
1148}
1149
eea49cc9 1150static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1151 struct packet_ring_buffer *rb,
1152 int status)
1153{
1154 if (po->tp_version <= TPACKET_V2)
1155 return packet_previous_frame(po, rb, status);
1156
1157 return __prb_previous_block(po, rb, status);
1158}
1159
eea49cc9 1160static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1161 struct packet_ring_buffer *rb)
1162{
1163 switch (po->tp_version) {
1164 case TPACKET_V1:
1165 case TPACKET_V2:
1166 return packet_increment_head(rb);
1167 case TPACKET_V3:
1168 default:
1169 WARN(1, "TPACKET version not supported.\n");
1170 BUG();
1171 return;
1172 }
1173}
1174
eea49cc9 1175static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1176 struct packet_ring_buffer *rb,
1177 int status)
1178{
1179 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1180 return packet_lookup_frame(po, rb, previous, status);
1181}
1182
eea49cc9 1183static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1184{
1185 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1186}
1187
b0138408
DB		 1188static void packet_inc_pending(struct packet_ring_buffer *rb)
1189{
1190 this_cpu_inc(*rb->pending_refcnt);
1191}
1192
1193static void packet_dec_pending(struct packet_ring_buffer *rb)
1194{
1195 this_cpu_dec(*rb->pending_refcnt);
1196}
1197
1198static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1199{
1200 unsigned int refcnt = 0;
1201 int cpu;
1202
1203 /* We don't use pending refcount in rx_ring. */
1204 if (rb->pending_refcnt == NULL)
1205 return 0;
1206
1207 for_each_possible_cpu(cpu)
1208 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1209
1210 return refcnt;
1211}
1212
1213static int packet_alloc_pending(struct packet_sock *po)
1214{
1215 po->rx_ring.pending_refcnt = NULL;
1216
1217 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1218 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1219 return -ENOBUFS;
1220
1221 return 0;
1222}
1223
1224static void packet_free_pending(struct packet_sock *po)
1225{
1226 free_percpu(po->tx_ring.pending_refcnt);
1227}
1228
77f65ebd
WB		 1229static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1230{
1231 struct sock *sk = &po->sk;
1232 bool has_room;
1233
1234 if (po->prot_hook.func != tpacket_rcv)
1235 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1236 <= sk->sk_rcvbuf;
1237
1238 spin_lock(&sk->sk_receive_queue.lock);
1239 if (po->tp_version == TPACKET_V3)
1240 has_room = prb_lookup_block(po, &po->rx_ring,
1241 po->rx_ring.prb_bdqc.kactive_blk_num,
1242 TP_STATUS_KERNEL);
1243 else
1244 has_room = packet_lookup_frame(po, &po->rx_ring,
1245 po->rx_ring.head,
1246 TP_STATUS_KERNEL);
1247 spin_unlock(&sk->sk_receive_queue.lock);
1248
1249 return has_room;
1250}
1251
1da177e4
LT		 1252static void packet_sock_destruct(struct sock *sk)
1253{
ed85b565
RC		 1254 skb_queue_purge(&sk->sk_error_queue);
1255
547b792c
IJ		 1256 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1257 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1258
1259 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1260 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1261 return;
1262 }
1263
17ab56a2 1264 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1265}
1266
dc99f600
DM		 1267static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1268{
1269 int x = atomic_read(&f->rr_cur) + 1;
1270
1271 if (x >= num)
1272 x = 0;
1273
1274 return x;
1275}
1276
77f65ebd
WB		 1277static unsigned int fanout_demux_hash(struct packet_fanout *f,
1278 struct sk_buff *skb,
1279 unsigned int num)
dc99f600 1280{
61b905da 1281 return reciprocal_scale(skb_get_hash(skb), num);
dc99f600
DM		 1282}
1283
77f65ebd
WB		 1284static unsigned int fanout_demux_lb(struct packet_fanout *f,
1285 struct sk_buff *skb,
1286 unsigned int num)
dc99f600
DM		 1287{
1288 int cur, old;
1289
1290 cur = atomic_read(&f->rr_cur);
1291 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1292 fanout_rr_next(f, num))) != cur)
1293 cur = old;
77f65ebd
WB		 1294 return cur;
1295}
1296
1297static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1298 struct sk_buff *skb,
1299 unsigned int num)
1300{
1301 return smp_processor_id() % num;
dc99f600
DM		 1302}
1303
5df0ddfb
DB		 1304static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1305 struct sk_buff *skb,
1306 unsigned int num)
1307{
f337db64 1308 return prandom_u32_max(num);
5df0ddfb
DB		 1309}
1310
77f65ebd
WB		 1311static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1312 struct sk_buff *skb,
1313 unsigned int idx, unsigned int skip,
1314 unsigned int num)
95ec3eb4 1315{
77f65ebd 1316 unsigned int i, j;
95ec3eb4 1317
77f65ebd
WB		 1318 i = j = min_t(int, f->next[idx], num - 1);
1319 do {
1320 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1321 if (i != j)
1322 f->next[idx] = i;
1323 return i;
1324 }
1325 if (++i == num)
1326 i = 0;
1327 } while (i != j);
1328
1329 return idx;
1330}
1331
2d36097d
NH		 1332static unsigned int fanout_demux_qm(struct packet_fanout *f,
1333 struct sk_buff *skb,
1334 unsigned int num)
1335{
1336 return skb_get_queue_mapping(skb) % num;
1337}
1338
77f65ebd
WB		 1339static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1340{
1341 return f->flags & (flag >> 8);
95ec3eb4
DM		 1342}
1343
95ec3eb4
DM		 1344static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1345 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1346{
1347 struct packet_fanout *f = pt->af_packet_priv;
1348 unsigned int num = f->num_members;
1349 struct packet_sock *po;
77f65ebd 1350 unsigned int idx;
dc99f600
DM		 1351
1352 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1353 !num) {
1354 kfree_skb(skb);
1355 return 0;
1356 }
1357
95ec3eb4
DM		 1358 switch (f->type) {
1359 case PACKET_FANOUT_HASH:
1360 default:
77f65ebd 1361 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1362 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1363 if (!skb)
1364 return 0;
1365 }
77f65ebd 1366 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1367 break;
1368 case PACKET_FANOUT_LB:
77f65ebd 1369 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1370 break;
1371 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1372 idx = fanout_demux_cpu(f, skb, num);
1373 break;
5df0ddfb
DB		 1374 case PACKET_FANOUT_RND:
1375 idx = fanout_demux_rnd(f, skb, num);
1376 break;
2d36097d
NH		 1377 case PACKET_FANOUT_QM:
1378 idx = fanout_demux_qm(f, skb, num);
1379 break;
77f65ebd
WB		 1380 case PACKET_FANOUT_ROLLOVER:
1381 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1382 break;
dc99f600
DM		 1383 }
1384
77f65ebd
WB		 1385 po = pkt_sk(f->arr[idx]);
1386 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1387 unlikely(!packet_rcv_has_room(po, skb))) {
1388 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1389 po = pkt_sk(f->arr[idx]);
1390 }
dc99f600
DM		 1391
1392 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1393}
1394
fff3321d
PE		 1395DEFINE_MUTEX(fanout_mutex);
1396EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1397static LIST_HEAD(fanout_list);
1398
1399static void __fanout_link(struct sock *sk, struct packet_sock *po)
1400{
1401 struct packet_fanout *f = po->fanout;
1402
1403 spin_lock(&f->lock);
1404 f->arr[f->num_members] = sk;
1405 smp_wmb();
1406 f->num_members++;
1407 spin_unlock(&f->lock);
1408}
1409
1410static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1411{
1412 struct packet_fanout *f = po->fanout;
1413 int i;
1414
1415 spin_lock(&f->lock);
1416 for (i = 0; i < f->num_members; i++) {
1417 if (f->arr[i] == sk)
1418 break;
1419 }
1420 BUG_ON(i >= f->num_members);
1421 f->arr[i] = f->arr[f->num_members - 1];
1422 f->num_members--;
1423 spin_unlock(&f->lock);
1424}
1425
d4dd8aee 1426static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1427{
d4dd8aee 1428 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
c0de08d0
EL		 1429 return true;
1430
1431 return false;
1432}
1433
7736d33f 1434static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1435{
1436 struct packet_sock *po = pkt_sk(sk);
1437 struct packet_fanout *f, *match;
7736d33f 1438 u8 type = type_flags & 0xff;
77f65ebd 1439 u8 flags = type_flags >> 8;
dc99f600
DM		 1440 int err;
1441
1442 switch (type) {
77f65ebd
WB		 1443 case PACKET_FANOUT_ROLLOVER:
1444 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1445 return -EINVAL;
dc99f600
DM		 1446 case PACKET_FANOUT_HASH:
1447 case PACKET_FANOUT_LB:
95ec3eb4 1448 case PACKET_FANOUT_CPU:
5df0ddfb 1449 case PACKET_FANOUT_RND:
2d36097d 1450 case PACKET_FANOUT_QM:
dc99f600
DM		 1451 break;
1452 default:
1453 return -EINVAL;
1454 }
1455
1456 if (!po->running)
1457 return -EINVAL;
1458
1459 if (po->fanout)
1460 return -EALREADY;
1461
1462 mutex_lock(&fanout_mutex);
1463 match = NULL;
1464 list_for_each_entry(f, &fanout_list, list) {
1465 if (f->id == id &&
1466 read_pnet(&f->net) == sock_net(sk)) {
1467 match = f;
1468 break;
1469 }
1470 }
afe62c68 1471 err = -EINVAL;
77f65ebd 1472 if (match && match->flags != flags)
afe62c68 1473 goto out;
dc99f600 1474 if (!match) {
afe62c68 1475 err = -ENOMEM;
dc99f600 1476 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1477 if (!match)
1478 goto out;
1479 write_pnet(&match->net, sock_net(sk));
1480 match->id = id;
1481 match->type = type;
77f65ebd 1482 match->flags = flags;
afe62c68
ED		 1483 atomic_set(&match->rr_cur, 0);
1484 INIT_LIST_HEAD(&match->list);
1485 spin_lock_init(&match->lock);
1486 atomic_set(&match->sk_ref, 0);
1487 match->prot_hook.type = po->prot_hook.type;
1488 match->prot_hook.dev = po->prot_hook.dev;
1489 match->prot_hook.func = packet_rcv_fanout;
1490 match->prot_hook.af_packet_priv = match;
c0de08d0 1491 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1492 dev_add_pack(&match->prot_hook);
1493 list_add(&match->list, &fanout_list);
dc99f600 1494 }
afe62c68
ED		 1495 err = -EINVAL;
1496 if (match->type == type &&
1497 match->prot_hook.type == po->prot_hook.type &&
1498 match->prot_hook.dev == po->prot_hook.dev) {
1499 err = -ENOSPC;
1500 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1501 __dev_remove_pack(&po->prot_hook);
1502 po->fanout = match;
1503 atomic_inc(&match->sk_ref);
1504 __fanout_link(sk, po);
1505 err = 0;
dc99f600
DM		 1506 }
1507 }
afe62c68 1508out:
dc99f600
DM		 1509 mutex_unlock(&fanout_mutex);
1510 return err;
1511}
1512
1513static void fanout_release(struct sock *sk)
1514{
1515 struct packet_sock *po = pkt_sk(sk);
1516 struct packet_fanout *f;
1517
1518 f = po->fanout;
1519 if (!f)
1520 return;
1521
fff3321d 1522 mutex_lock(&fanout_mutex);
dc99f600
DM		 1523 po->fanout = NULL;
1524
dc99f600
DM		 1525 if (atomic_dec_and_test(&f->sk_ref)) {
1526 list_del(&f->list);
1527 dev_remove_pack(&f->prot_hook);
1528 kfree(f);
1529 }
1530 mutex_unlock(&fanout_mutex);
1531}
1da177e4 1532
90ddc4f0 1533static const struct proto_ops packet_ops;
1da177e4 1534
90ddc4f0 1535static const struct proto_ops packet_ops_spkt;
1da177e4 1536
40d4e3df
ED		 1537static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1538 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1539{
1540 struct sock *sk;
1541 struct sockaddr_pkt *spkt;
1542
1543 /*
1544 * When we registered the protocol we saved the socket in the data
1545 * field for just this event.
1546 */
1547
1548 sk = pt->af_packet_priv;
1ce4f28b 1549
1da177e4
LT		 1550 /*
1551 * Yank back the headers [hope the device set this
1552 * right or kerboom...]
1553 *
1554 * Incoming packets have ll header pulled,
1555 * push it back.
1556 *
98e399f8 1557 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1558 * so that this procedure is noop.
1559 */
1560
1561 if (skb->pkt_type == PACKET_LOOPBACK)
1562 goto out;
1563
09ad9bc7 1564 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1565 goto out;
1566
40d4e3df
ED		 1567 skb = skb_share_check(skb, GFP_ATOMIC);
1568 if (skb == NULL)
1da177e4
LT		 1569 goto oom;
1570
1571 /* drop any routing info */
adf30907 1572 skb_dst_drop(skb);
1da177e4 1573
84531c24
PO		 1574 /* drop conntrack reference */
1575 nf_reset(skb);
1576
ffbc6111 1577 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1578
98e399f8 1579 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1580
1581 /*
1582 * The SOCK_PACKET socket receives _all_ frames.
1583 */
1584
1585 spkt->spkt_family = dev->type;
1586 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1587 spkt->spkt_protocol = skb->protocol;
1588
1589 /*
1590 * Charge the memory to the socket. This is done specifically
1591 * to prevent sockets using all the memory up.
1592 */
1593
40d4e3df 1594 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1595 return 0;
1596
1597out:
1598 kfree_skb(skb);
1599oom:
1600 return 0;
1601}
1602
1603
1604/*
1605 * Output a raw packet to a device layer. This bypasses all the other
1606 * protocol layers and you must therefore supply it with a complete frame
1607 */
1ce4f28b 1608
1da177e4
LT		 1609static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1610 struct msghdr *msg, size_t len)
1611{
1612 struct sock *sk = sock->sk;
342dfc30 1613 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1614 struct sk_buff *skb = NULL;
1da177e4 1615 struct net_device *dev;
40d4e3df 1616 __be16 proto = 0;
1da177e4 1617 int err;
3bdc0eba 1618 int extra_len = 0;
1ce4f28b 1619
1da177e4 1620 /*
1ce4f28b 1621 * Get and verify the address.
1da177e4
LT		 1622 */
1623
40d4e3df 1624 if (saddr) {
1da177e4 1625 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1626 return -EINVAL;
1627 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1628 proto = saddr->spkt_protocol;
1629 } else
1630 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1631
1632 /*
1ce4f28b 1633 * Find the device first to size check it
1da177e4
LT		 1634 */
1635
de74e92a 1636 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1637retry:
654d1f8a
ED		 1638 rcu_read_lock();
1639 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1640 err = -ENODEV;
1641 if (dev == NULL)
1642 goto out_unlock;
1ce4f28b 1643
d5e76b0a
DM		 1644 err = -ENETDOWN;
1645 if (!(dev->flags & IFF_UP))
1646 goto out_unlock;
1647
1da177e4 1648 /*
40d4e3df
ED		 1649 * You may not queue a frame bigger than the mtu. This is the lowest level
1650 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1651 */
1ce4f28b 1652
3bdc0eba
BG		 1653 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1654 if (!netif_supports_nofcs(dev)) {
1655 err = -EPROTONOSUPPORT;
1656 goto out_unlock;
1657 }
1658 extra_len = 4; /* We're doing our own CRC */
1659 }
1660
1da177e4 1661 err = -EMSGSIZE;
3bdc0eba 1662 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1663 goto out_unlock;
1664
1a35ca80
ED		 1665 if (!skb) {
1666 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1667 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1668 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1669
1670 rcu_read_unlock();
4ce40912 1671 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1672 if (skb == NULL)
1673 return -ENOBUFS;
1674 /* FIXME: Save some space for broken drivers that write a hard
1675 * header at transmission time by themselves. PPP is the notable
1676 * one here. This should really be fixed at the driver level.
1677 */
1678 skb_reserve(skb, reserved);
1679 skb_reset_network_header(skb);
1680
1681 /* Try to align data part correctly */
1682 if (hhlen) {
1683 skb->data -= hhlen;
1684 skb->tail -= hhlen;
1685 if (len < hhlen)
1686 skb_reset_network_header(skb);
1687 }
1688 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1689 if (err)
1690 goto out_free;
1691 goto retry;
1da177e4
LT		 1692 }
1693
3bdc0eba 1694 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1695 /* Earlier code assumed this would be a VLAN pkt,
1696 * double-check this now that we have the actual
1697 * packet in hand.
1698 */
1699 struct ethhdr *ehdr;
1700 skb_reset_mac_header(skb);
1701 ehdr = eth_hdr(skb);
1702 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1703 err = -EMSGSIZE;
1704 goto out_unlock;
1705 }
1706 }
1a35ca80 1707
1da177e4
LT		 1708 skb->protocol = proto;
1709 skb->dev = dev;
1710 skb->priority = sk->sk_priority;
2d37a186 1711 skb->mark = sk->sk_mark;
bf84a010
DB		 1712
1713 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1714
3bdc0eba
BG		 1715 if (unlikely(extra_len == 4))
1716 skb->no_fcs = 1;
1717
40893fd0 1718 skb_probe_transport_header(skb, 0);
c1aad275 1719
1da177e4 1720 dev_queue_xmit(skb);
654d1f8a 1721 rcu_read_unlock();
40d4e3df 1722 return len;
1da177e4 1723
1da177e4 1724out_unlock:
654d1f8a 1725 rcu_read_unlock();
1a35ca80
ED		 1726out_free:
1727 kfree_skb(skb);
1da177e4
LT		 1728 return err;
1729}
1da177e4 1730
eea49cc9 1731static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1732 const struct sock *sk,
dbcb5855 1733 unsigned int res)
1da177e4
LT		 1734{
1735 struct sk_filter *filter;
fda9ef5d 1736
80f8f102
ED		 1737 rcu_read_lock();
1738 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1739 if (filter != NULL)
0a14842f 1740 res = SK_RUN_FILTER(filter, skb);
80f8f102 1741 rcu_read_unlock();
1da177e4 1742
dbcb5855 1743 return res;
1da177e4
LT		 1744}
1745
1746/*
62ab0812
ED		 1747 * This function makes lazy skb cloning in hope that most of packets
1748 * are discarded by BPF.
1749 *
1750 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1751 * and skb->cb are mangled. It works because (and until) packets
1752 * falling here are owned by current CPU. Output packets are cloned
1753 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1754 * sequencially, so that if we return skb to original state on exit,
1755 * we will not harm anyone.
1da177e4
LT		 1756 */
1757
40d4e3df
ED		 1758static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1759 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1760{
1761 struct sock *sk;
1762 struct sockaddr_ll *sll;
1763 struct packet_sock *po;
40d4e3df 1764 u8 *skb_head = skb->data;
1da177e4 1765 int skb_len = skb->len;
dbcb5855 1766 unsigned int snaplen, res;
1da177e4
LT		 1767
1768 if (skb->pkt_type == PACKET_LOOPBACK)
1769 goto drop;
1770
1771 sk = pt->af_packet_priv;
1772 po = pkt_sk(sk);
1773
09ad9bc7 1774 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1775 goto drop;
1776
1da177e4
LT		 1777 skb->dev = dev;
1778
3b04ddde 1779 if (dev->header_ops) {
1da177e4 1780 /* The device has an explicit notion of ll header,
62ab0812
ED		 1781 * exported to higher levels.
1782 *
1783 * Otherwise, the device hides details of its frame
1784 * structure, so that corresponding packet head is
1785 * never delivered to user.
1da177e4
LT		 1786 */
1787 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1788 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1789 else if (skb->pkt_type == PACKET_OUTGOING) {
1790 /* Special case: outgoing packets have ll header at head */
bbe735e4 1791 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1792 }
1793 }
1794
1795 snaplen = skb->len;
1796
dbcb5855
DM		 1797 res = run_filter(skb, sk, snaplen);
1798 if (!res)
fda9ef5d 1799 goto drop_n_restore;
dbcb5855
DM		 1800 if (snaplen > res)
1801 snaplen = res;
1da177e4 1802
0fd7bac6 1803 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1804 goto drop_n_acct;
1805
1806 if (skb_shared(skb)) {
1807 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1808 if (nskb == NULL)
1809 goto drop_n_acct;
1810
1811 if (skb_head != skb->data) {
1812 skb->data = skb_head;
1813 skb->len = skb_len;
1814 }
abc4e4fa 1815 consume_skb(skb);
1da177e4
LT		 1816 skb = nskb;
1817 }
1818
ffbc6111
HX		 1819 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1820 sizeof(skb->cb));
1821
1822 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1823 sll->sll_family = AF_PACKET;
1824 sll->sll_hatype = dev->type;
1825 sll->sll_protocol = skb->protocol;
1826 sll->sll_pkttype = skb->pkt_type;
8032b464 1827 if (unlikely(po->origdev))
80feaacb
PWJ		 1828 sll->sll_ifindex = orig_dev->ifindex;
1829 else
1830 sll->sll_ifindex = dev->ifindex;
1da177e4 1831
b95cce35 1832 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1833
ffbc6111 1834 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1835
1da177e4
LT		 1836 if (pskb_trim(skb, snaplen))
1837 goto drop_n_acct;
1838
1839 skb_set_owner_r(skb, sk);
1840 skb->dev = NULL;
adf30907 1841 skb_dst_drop(skb);
1da177e4 1842
84531c24
PO		 1843 /* drop conntrack reference */
1844 nf_reset(skb);
1845
1da177e4 1846 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1847 po->stats.stats1.tp_packets++;
3b885787 1848 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1849 __skb_queue_tail(&sk->sk_receive_queue, skb);
1850 spin_unlock(&sk->sk_receive_queue.lock);
1851 sk->sk_data_ready(sk, skb->len);
1852 return 0;
1853
1854drop_n_acct:
7091fbd8 1855 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1856 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1857 atomic_inc(&sk->sk_drops);
1858 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1859
1860drop_n_restore:
1861 if (skb_head != skb->data && skb_shared(skb)) {
1862 skb->data = skb_head;
1863 skb->len = skb_len;
1864 }
1865drop:
ead2ceb0 1866 consume_skb(skb);
1da177e4
LT		 1867 return 0;
1868}
1869
40d4e3df
ED		 1870static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1871 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1872{
1873 struct sock *sk;
1874 struct packet_sock *po;
1875 struct sockaddr_ll *sll;
184f489e 1876 union tpacket_uhdr h;
40d4e3df 1877 u8 *skb_head = skb->data;
1da177e4 1878 int skb_len = skb->len;
dbcb5855 1879 unsigned int snaplen, res;
f6fb8f10 1880 unsigned long status = TP_STATUS_USER;
bbd6ef87 1881 unsigned short macoff, netoff, hdrlen;
1da177e4 1882 struct sk_buff *copy_skb = NULL;
bbd6ef87 1883 struct timespec ts;
b9c32fb2 1884 __u32 ts_status;
1da177e4 1885
51846355
AW		 1886 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1887 * We may add members to them until current aligned size without forcing
1888 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1889 */
1890 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1891 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1892
1da177e4
LT		 1893 if (skb->pkt_type == PACKET_LOOPBACK)
1894 goto drop;
1895
1896 sk = pt->af_packet_priv;
1897 po = pkt_sk(sk);
1898
09ad9bc7 1899 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1900 goto drop;
1901
3b04ddde 1902 if (dev->header_ops) {
1da177e4 1903 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1904 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1905 else if (skb->pkt_type == PACKET_OUTGOING) {
1906 /* Special case: outgoing packets have ll header at head */
bbe735e4 1907 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1908 }
1909 }
1910
8dc41944
HX		 1911 if (skb->ip_summed == CHECKSUM_PARTIAL)
1912 status |= TP_STATUS_CSUMNOTREADY;
1913
1da177e4
LT		 1914 snaplen = skb->len;
1915
dbcb5855
DM		 1916 res = run_filter(skb, sk, snaplen);
1917 if (!res)
fda9ef5d 1918 goto drop_n_restore;
dbcb5855
DM		 1919 if (snaplen > res)
1920 snaplen = res;
1da177e4
LT		 1921
1922 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1923 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1924 po->tp_reserve;
1da177e4 1925 } else {
95c96174 1926 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1927 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1928 (maclen < 16 ? 16 : maclen)) +
1929 po->tp_reserve;
1da177e4
LT		 1930 macoff = netoff - maclen;
1931 }
f6fb8f10 1932 if (po->tp_version <= TPACKET_V2) {
1933 if (macoff + snaplen > po->rx_ring.frame_size) {
1934 if (po->copy_thresh &&
0fd7bac6 1935 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1936 if (skb_shared(skb)) {
1937 copy_skb = skb_clone(skb, GFP_ATOMIC);
1938 } else {
1939 copy_skb = skb_get(skb);
1940 skb_head = skb->data;
1941 }
1942 if (copy_skb)
1943 skb_set_owner_r(copy_skb, sk);
1da177e4 1944 }
f6fb8f10 1945 snaplen = po->rx_ring.frame_size - macoff;
1946 if ((int)snaplen < 0)
1947 snaplen = 0;
1da177e4 1948 }
1da177e4 1949 }
1da177e4 1950 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1951 h.raw = packet_current_rx_frame(po, skb,
1952 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1953 if (!h.raw)
1da177e4 1954 goto ring_is_full;
f6fb8f10 1955 if (po->tp_version <= TPACKET_V2) {
1956 packet_increment_rx_head(po, &po->rx_ring);
1957 /*
1958 * LOSING will be reported till you read the stats,
1959 * because it's COR - Clear On Read.
1960 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1961 * at packet level.
1962 */
ee80fbf3 1963 if (po->stats.stats1.tp_drops)
f6fb8f10 1964 status |= TP_STATUS_LOSING;
1965 }
ee80fbf3 1966 po->stats.stats1.tp_packets++;
1da177e4
LT		 1967 if (copy_skb) {
1968 status |= TP_STATUS_COPY;
1969 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1970 }
1da177e4
LT		 1971 spin_unlock(&sk->sk_receive_queue.lock);
1972
bbd6ef87 1973 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1974
1975 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1976 getnstimeofday(&ts);
1da177e4 1977
b9c32fb2
DB		 1978 status |= ts_status;
1979
bbd6ef87
PM		 1980 switch (po->tp_version) {
1981 case TPACKET_V1:
1982 h.h1->tp_len = skb->len;
1983 h.h1->tp_snaplen = snaplen;
1984 h.h1->tp_mac = macoff;
1985 h.h1->tp_net = netoff;
4b457bdf
DB		 1986 h.h1->tp_sec = ts.tv_sec;
1987 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1988 hdrlen = sizeof(*h.h1);
1989 break;
1990 case TPACKET_V2:
1991 h.h2->tp_len = skb->len;
1992 h.h2->tp_snaplen = snaplen;
1993 h.h2->tp_mac = macoff;
1994 h.h2->tp_net = netoff;
bbd6ef87
PM		 1995 h.h2->tp_sec = ts.tv_sec;
1996 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1997 if (vlan_tx_tag_present(skb)) {
1998 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 1999 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2000 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2001 } else {
2002 h.h2->tp_vlan_tci = 0;
a0cdfcf3 2003 h.h2->tp_vlan_tpid = 0;
a3bcc23e 2004 }
e4d26f4b 2005 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 2006 hdrlen = sizeof(*h.h2);
2007 break;
f6fb8f10 2008 case TPACKET_V3:
2009 /* tp_nxt_offset,vlan are already populated above.
2010 * So DONT clear those fields here
2011 */
2012 h.h3->tp_status |= status;
2013 h.h3->tp_len = skb->len;
2014 h.h3->tp_snaplen = snaplen;
2015 h.h3->tp_mac = macoff;
2016 h.h3->tp_net = netoff;
f6fb8f10 2017 h.h3->tp_sec = ts.tv_sec;
2018 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2019 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2020 hdrlen = sizeof(*h.h3);
2021 break;
bbd6ef87
PM		 2022 default:
2023 BUG();
2024 }
1da177e4 2025
bbd6ef87 2026 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2027 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2028 sll->sll_family = AF_PACKET;
2029 sll->sll_hatype = dev->type;
2030 sll->sll_protocol = skb->protocol;
2031 sll->sll_pkttype = skb->pkt_type;
8032b464 2032 if (unlikely(po->origdev))
80feaacb
PWJ		 2033 sll->sll_ifindex = orig_dev->ifindex;
2034 else
2035 sll->sll_ifindex = dev->ifindex;
1da177e4 2036
e16aa207 2037 smp_mb();
f0d4eb29 2038
f6dafa95 2039#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2040 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2041 u8 *start, *end;
2042
f0d4eb29
DB		 2043 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2044 macoff + snaplen);
2045
2046 for (start = h.raw; start < end; start += PAGE_SIZE)
2047 flush_dcache_page(pgv_to_page(start));
1da177e4 2048 }
f0d4eb29 2049 smp_wmb();
f6dafa95 2050#endif
f0d4eb29 2051
f6fb8f10 2052 if (po->tp_version <= TPACKET_V2)
2053 __packet_set_status(po, h.raw, status);
2054 else
2055 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 2056
2057 sk->sk_data_ready(sk, 0);
2058
2059drop_n_restore:
2060 if (skb_head != skb->data && skb_shared(skb)) {
2061 skb->data = skb_head;
2062 skb->len = skb_len;
2063 }
2064drop:
1ce4f28b 2065 kfree_skb(skb);
1da177e4
LT		 2066 return 0;
2067
2068ring_is_full:
ee80fbf3 2069 po->stats.stats1.tp_drops++;
1da177e4
LT		 2070 spin_unlock(&sk->sk_receive_queue.lock);
2071
2072 sk->sk_data_ready(sk, 0);
acb5d75b 2073 kfree_skb(copy_skb);
1da177e4
LT		 2074 goto drop_n_restore;
2075}
2076
69e3c75f
JB		 2077static void tpacket_destruct_skb(struct sk_buff *skb)
2078{
2079 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2080
69e3c75f 2081 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2082 void *ph;
b9c32fb2
DB		 2083 __u32 ts;
2084
69e3c75f 2085 ph = skb_shinfo(skb)->destructor_arg;
b0138408 2086 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2087
2088 ts = __packet_set_timestamp(po, ph, skb);
2089 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2090 }
2091
2092 sock_wfree(skb);
2093}
2094
40d4e3df
ED		 2095static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2096 void *frame, struct net_device *dev, int size_max,
ae641949 2097 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 2098{
184f489e 2099 union tpacket_uhdr ph;
09effa67 2100 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 2101 struct socket *sock = po->sk.sk_socket;
2102 struct page *page;
2103 void *data;
2104 int err;
2105
2106 ph.raw = frame;
2107
2108 skb->protocol = proto;
2109 skb->dev = dev;
2110 skb->priority = po->sk.sk_priority;
2d37a186 2111 skb->mark = po->sk.sk_mark;
2e31396f 2112 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2113 skb_shinfo(skb)->destructor_arg = ph.raw;
2114
2115 switch (po->tp_version) {
2116 case TPACKET_V2:
2117 tp_len = ph.h2->tp_len;
2118 break;
2119 default:
2120 tp_len = ph.h1->tp_len;
2121 break;
2122 }
09effa67
DM		 2123 if (unlikely(tp_len > size_max)) {
2124 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2125 return -EMSGSIZE;
2126 }
69e3c75f 2127
ae641949 2128 skb_reserve(skb, hlen);
69e3c75f 2129 skb_reset_network_header(skb);
c1aad275 2130
d346a3fa
DB		 2131 if (!packet_use_direct_xmit(po))
2132 skb_probe_transport_header(skb, 0);
2133 if (unlikely(po->tp_tx_has_off)) {
5920cd3a
PC		 2134 int off_min, off_max, off;
2135 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2136 off_max = po->tx_ring.frame_size - tp_len;
2137 if (sock->type == SOCK_DGRAM) {
2138 switch (po->tp_version) {
2139 case TPACKET_V2:
2140 off = ph.h2->tp_net;
2141 break;
2142 default:
2143 off = ph.h1->tp_net;
2144 break;
2145 }
2146 } else {
2147 switch (po->tp_version) {
2148 case TPACKET_V2:
2149 off = ph.h2->tp_mac;
2150 break;
2151 default:
2152 off = ph.h1->tp_mac;
2153 break;
2154 }
2155 }
2156 if (unlikely((off < off_min) || (off_max < off)))
2157 return -EINVAL;
2158 data = ph.raw + off;
2159 } else {
2160 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2161 }
69e3c75f
JB		 2162 to_write = tp_len;
2163
2164 if (sock->type == SOCK_DGRAM) {
2165 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2166 NULL, tp_len);
2167 if (unlikely(err < 0))
2168 return -EINVAL;
40d4e3df 2169 } else if (dev->hard_header_len) {
69e3c75f
JB		 2170 /* net device doesn't like empty head */
2171 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2172 pr_err("packet size is too short (%d < %d)\n",
2173 tp_len, dev->hard_header_len);
69e3c75f
JB		 2174 return -EINVAL;
2175 }
2176
2177 skb_push(skb, dev->hard_header_len);
2178 err = skb_store_bits(skb, 0, data,
2179 dev->hard_header_len);
2180 if (unlikely(err))
2181 return err;
2182
2183 data += dev->hard_header_len;
2184 to_write -= dev->hard_header_len;
2185 }
2186
69e3c75f
JB		 2187 offset = offset_in_page(data);
2188 len_max = PAGE_SIZE - offset;
2189 len = ((to_write > len_max) ? len_max : to_write);
2190
2191 skb->data_len = to_write;
2192 skb->len += to_write;
2193 skb->truesize += to_write;
2194 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2195
2196 while (likely(to_write)) {
2197 nr_frags = skb_shinfo(skb)->nr_frags;
2198
2199 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2200 pr_err("Packet exceed the number of skb frags(%lu)\n",
2201 MAX_SKB_FRAGS);
69e3c75f
JB		 2202 return -EFAULT;
2203 }
2204
0af55bb5
CG		 2205 page = pgv_to_page(data);
2206 data += len;
69e3c75f
JB		 2207 flush_dcache_page(page);
2208 get_page(page);
0af55bb5 2209 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2210 to_write -= len;
2211 offset = 0;
2212 len_max = PAGE_SIZE;
2213 len = ((to_write > len_max) ? len_max : to_write);
2214 }
2215
2216 return tp_len;
2217}
2218
2219static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2220{
69e3c75f
JB		 2221 struct sk_buff *skb;
2222 struct net_device *dev;
2223 __be16 proto;
09effa67 2224 int err, reserve = 0;
40d4e3df 2225 void *ph;
342dfc30 2226 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2227 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
69e3c75f
JB		 2228 int tp_len, size_max;
2229 unsigned char *addr;
2230 int len_sum = 0;
9e67030a 2231 int status = TP_STATUS_AVAILABLE;
ae641949 2232 int hlen, tlen;
69e3c75f 2233
69e3c75f
JB		 2234 mutex_lock(&po->pg_vec_lock);
2235
66e56cd4 2236 if (likely(saddr == NULL)) {
e40526cb 2237 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2238 proto = po->num;
2239 addr = NULL;
2240 } else {
2241 err = -EINVAL;
2242 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2243 goto out;
2244 if (msg->msg_namelen < (saddr->sll_halen
2245 + offsetof(struct sockaddr_ll,
2246 sll_addr)))
2247 goto out;
69e3c75f
JB		 2248 proto = saddr->sll_protocol;
2249 addr = saddr->sll_addr;
827d9780 2250 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2251 }
2252
69e3c75f
JB		 2253 err = -ENXIO;
2254 if (unlikely(dev == NULL))
2255 goto out;
69e3c75f
JB		 2256 err = -ENETDOWN;
2257 if (unlikely(!(dev->flags & IFF_UP)))
2258 goto out_put;
2259
52f1454f 2260 reserve = dev->hard_header_len + VLAN_HLEN;
69e3c75f 2261 size_max = po->tx_ring.frame_size
b5dd884e 2262 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2263
09effa67
DM		 2264 if (size_max > dev->mtu + reserve)
2265 size_max = dev->mtu + reserve;
2266
69e3c75f
JB		 2267 do {
2268 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2269 TP_STATUS_SEND_REQUEST);
69e3c75f 2270 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2271 if (need_wait && need_resched())
2272 schedule();
69e3c75f
JB		 2273 continue;
2274 }
2275
2276 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2277 hlen = LL_RESERVED_SPACE(dev);
2278 tlen = dev->needed_tailroom;
69e3c75f 2279 skb = sock_alloc_send_skb(&po->sk,
ae641949 2280 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2281 0, &err);
2282
2283 if (unlikely(skb == NULL))
2284 goto out_status;
2285
2286 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
52f1454f
DB		 2287 addr, hlen);
2288 if (tp_len > dev->mtu + dev->hard_header_len) {
2289 struct ethhdr *ehdr;
2290 /* Earlier code assumed this would be a VLAN pkt,
2291 * double-check this now that we have the actual
2292 * packet in hand.
2293 */
69e3c75f 2294
52f1454f
DB		 2295 skb_reset_mac_header(skb);
2296 ehdr = eth_hdr(skb);
2297 if (ehdr->h_proto != htons(ETH_P_8021Q))
2298 tp_len = -EMSGSIZE;
2299 }
69e3c75f
JB		 2300 if (unlikely(tp_len < 0)) {
2301 if (po->tp_loss) {
2302 __packet_set_status(po, ph,
2303 TP_STATUS_AVAILABLE);
2304 packet_increment_head(&po->tx_ring);
2305 kfree_skb(skb);
2306 continue;
2307 } else {
2308 status = TP_STATUS_WRONG_FORMAT;
2309 err = tp_len;
2310 goto out_status;
2311 }
2312 }
2313
0fd5d57b
DB		 2314 packet_pick_tx_queue(dev, skb);
2315
69e3c75f
JB		 2316 skb->destructor = tpacket_destruct_skb;
2317 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2318 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2319
2320 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2321 err = po->xmit(skb);
eb70df13
JP		 2322 if (unlikely(err > 0)) {
2323 err = net_xmit_errno(err);
2324 if (err && __packet_get_status(po, ph) ==
2325 TP_STATUS_AVAILABLE) {
2326 /* skb was destructed already */
2327 skb = NULL;
2328 goto out_status;
2329 }
2330 /*
2331 * skb was dropped but not destructed yet;
2332 * let's treat it like congestion or err < 0
2333 */
2334 err = 0;
2335 }
69e3c75f
JB		 2336 packet_increment_head(&po->tx_ring);
2337 len_sum += tp_len;
b0138408
DB		 2338 } while (likely((ph != NULL) ||
2339 /* Note: packet_read_pending() might be slow if we have
2340 * to call it as it's per_cpu variable, but in fast-path
2341 * we already short-circuit the loop with the first
2342 * condition, and luckily don't have to go that path
2343 * anyway.
2344 */
2345 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2346
2347 err = len_sum;
2348 goto out_put;
2349
69e3c75f
JB		 2350out_status:
2351 __packet_set_status(po, ph, status);
2352 kfree_skb(skb);
2353out_put:
e40526cb 2354 dev_put(dev);
69e3c75f
JB		 2355out:
2356 mutex_unlock(&po->pg_vec_lock);
2357 return err;
2358}
69e3c75f 2359
eea49cc9
OJ		 2360static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2361 size_t reserve, size_t len,
2362 size_t linear, int noblock,
2363 int *err)
bfd5f4a3
SS		 2364{
2365 struct sk_buff *skb;
2366
2367 /* Under a page? Don't bother with paged skb. */
2368 if (prepad + len < PAGE_SIZE || !linear)
2369 linear = len;
2370
2371 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2372 err, 0);
bfd5f4a3
SS		 2373 if (!skb)
2374 return NULL;
2375
2376 skb_reserve(skb, reserve);
2377 skb_put(skb, linear);
2378 skb->data_len = len - linear;
2379 skb->len += len - linear;
2380
2381 return skb;
2382}
2383
d346a3fa 2384static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2385{
2386 struct sock *sk = sock->sk;
342dfc30 2387 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2388 struct sk_buff *skb;
2389 struct net_device *dev;
0e11c91e 2390 __be16 proto;
1da177e4 2391 unsigned char *addr;
827d9780 2392 int err, reserve = 0;
bfd5f4a3
SS		 2393 struct virtio_net_hdr vnet_hdr = { 0 };
2394 int offset = 0;
2395 int vnet_hdr_len;
2396 struct packet_sock *po = pkt_sk(sk);
2397 unsigned short gso_type = 0;
ae641949 2398 int hlen, tlen;
3bdc0eba 2399 int extra_len = 0;
1da177e4
LT		 2400
2401 /*
1ce4f28b 2402 * Get and verify the address.
1da177e4 2403 */
1ce4f28b 2404
66e56cd4 2405 if (likely(saddr == NULL)) {
e40526cb 2406 dev = packet_cached_dev_get(po);
1da177e4
LT		 2407 proto = po->num;
2408 addr = NULL;
2409 } else {
2410 err = -EINVAL;
2411 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2412 goto out;
0fb375fb
EB		 2413 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2414 goto out;
1da177e4
LT		 2415 proto = saddr->sll_protocol;
2416 addr = saddr->sll_addr;
827d9780 2417 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2418 }
2419
1da177e4 2420 err = -ENXIO;
e40526cb 2421 if (unlikely(dev == NULL))
1da177e4 2422 goto out_unlock;
d5e76b0a 2423 err = -ENETDOWN;
e40526cb 2424 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2425 goto out_unlock;
2426
e40526cb
DB		 2427 if (sock->type == SOCK_RAW)
2428 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2429 if (po->has_vnet_hdr) {
2430 vnet_hdr_len = sizeof(vnet_hdr);
2431
2432 err = -EINVAL;
2433 if (len < vnet_hdr_len)
2434 goto out_unlock;
2435
2436 len -= vnet_hdr_len;
2437
2438 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2439 vnet_hdr_len);
2440 if (err < 0)
2441 goto out_unlock;
2442
2443 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2444 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2445 vnet_hdr.hdr_len))
2446 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2447 vnet_hdr.csum_offset + 2;
2448
2449 err = -EINVAL;
2450 if (vnet_hdr.hdr_len > len)
2451 goto out_unlock;
2452
2453 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2454 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2455 case VIRTIO_NET_HDR_GSO_TCPV4:
2456 gso_type = SKB_GSO_TCPV4;
2457 break;
2458 case VIRTIO_NET_HDR_GSO_TCPV6:
2459 gso_type = SKB_GSO_TCPV6;
2460 break;
2461 case VIRTIO_NET_HDR_GSO_UDP:
2462 gso_type = SKB_GSO_UDP;
2463 break;
2464 default:
2465 goto out_unlock;
2466 }
2467
2468 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2469 gso_type |= SKB_GSO_TCP_ECN;
2470
2471 if (vnet_hdr.gso_size == 0)
2472 goto out_unlock;
2473
2474 }
2475 }
2476
3bdc0eba
BG		 2477 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2478 if (!netif_supports_nofcs(dev)) {
2479 err = -EPROTONOSUPPORT;
2480 goto out_unlock;
2481 }
2482 extra_len = 4; /* We're doing our own CRC */
2483 }
2484
1da177e4 2485 err = -EMSGSIZE;
3bdc0eba 2486 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2487 goto out_unlock;
2488
bfd5f4a3 2489 err = -ENOBUFS;
ae641949
HX		 2490 hlen = LL_RESERVED_SPACE(dev);
2491 tlen = dev->needed_tailroom;
2492 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2493 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2494 if (skb == NULL)
1da177e4
LT		 2495 goto out_unlock;
2496
bfd5f4a3 2497 skb_set_network_header(skb, reserve);
1da177e4 2498
0c4e8581
SH		 2499 err = -EINVAL;
2500 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2501 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2502 goto out_free;
1da177e4
LT		 2503
2504 /* Returns -EFAULT on error */
bfd5f4a3 2505 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2506 if (err)
2507 goto out_free;
bf84a010
DB		 2508
2509 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2510
3bdc0eba 2511 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2512 /* Earlier code assumed this would be a VLAN pkt,
2513 * double-check this now that we have the actual
2514 * packet in hand.
2515 */
2516 struct ethhdr *ehdr;
2517 skb_reset_mac_header(skb);
2518 ehdr = eth_hdr(skb);
2519 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2520 err = -EMSGSIZE;
2521 goto out_free;
2522 }
57f89bfa
BG		 2523 }
2524
09effa67
DM		 2525 skb->protocol = proto;
2526 skb->dev = dev;
1da177e4 2527 skb->priority = sk->sk_priority;
2d37a186 2528 skb->mark = sk->sk_mark;
0fd5d57b
DB		 2529
2530 packet_pick_tx_queue(dev, skb);
1da177e4 2531
bfd5f4a3
SS		 2532 if (po->has_vnet_hdr) {
2533 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2534 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2535 vnet_hdr.csum_offset)) {
2536 err = -EINVAL;
2537 goto out_free;
2538 }
2539 }
2540
2541 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2542 skb_shinfo(skb)->gso_type = gso_type;
2543
2544 /* Header must be checked, and gso_segs computed. */
2545 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2546 skb_shinfo(skb)->gso_segs = 0;
2547
2548 len += vnet_hdr_len;
2549 }
2550
d346a3fa
DB		 2551 if (!packet_use_direct_xmit(po))
2552 skb_probe_transport_header(skb, reserve);
3bdc0eba
BG		 2553 if (unlikely(extra_len == 4))
2554 skb->no_fcs = 1;
2555
d346a3fa 2556 err = po->xmit(skb);
1da177e4
LT		 2557 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2558 goto out_unlock;
2559
e40526cb 2560 dev_put(dev);
1da177e4 2561
40d4e3df 2562 return len;
1da177e4
LT		 2563
2564out_free:
2565 kfree_skb(skb);
2566out_unlock:
e40526cb 2567 if (dev)
1da177e4
LT		 2568 dev_put(dev);
2569out:
2570 return err;
2571}
2572
69e3c75f
JB		 2573static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2574 struct msghdr *msg, size_t len)
2575{
69e3c75f
JB		 2576 struct sock *sk = sock->sk;
2577 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2578
69e3c75f
JB		 2579 if (po->tx_ring.pg_vec)
2580 return tpacket_snd(po, msg);
2581 else
69e3c75f
JB		 2582 return packet_snd(sock, msg, len);
2583}
2584
1da177e4
LT		 2585/*
2586 * Close a PACKET socket. This is fairly simple. We immediately go
2587 * to 'closed' state and remove our protocol entry in the device list.
2588 */
2589
2590static int packet_release(struct socket *sock)
2591{
2592 struct sock *sk = sock->sk;
2593 struct packet_sock *po;
d12d01d6 2594 struct net *net;
f6fb8f10 2595 union tpacket_req_u req_u;
1da177e4
LT		 2596
2597 if (!sk)
2598 return 0;
2599
3b1e0a65 2600 net = sock_net(sk);
1da177e4
LT		 2601 po = pkt_sk(sk);
2602
0fa7fa98 2603 mutex_lock(&net->packet.sklist_lock);
808f5114 2604 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2605 mutex_unlock(&net->packet.sklist_lock);
2606
2607 preempt_disable();
920de804 2608 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2609 preempt_enable();
1da177e4 2610
808f5114 2611 spin_lock(&po->bind_lock);
ce06b03e 2612 unregister_prot_hook(sk, false);
66e56cd4
DB		 2613 packet_cached_dev_reset(po);
2614
160ff18a
BG		 2615 if (po->prot_hook.dev) {
2616 dev_put(po->prot_hook.dev);
2617 po->prot_hook.dev = NULL;
2618 }
808f5114 2619 spin_unlock(&po->bind_lock);
1da177e4 2620
1da177e4 2621 packet_flush_mclist(sk);
1da177e4 2622
9665d5d6
PS		 2623 if (po->rx_ring.pg_vec) {
2624 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2625 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2626 }
69e3c75f 2627
9665d5d6
PS		 2628 if (po->tx_ring.pg_vec) {
2629 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2630 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2631 }
1da177e4 2632
dc99f600
DM		 2633 fanout_release(sk);
2634
808f5114 2635 synchronize_net();
1da177e4
LT		 2636 /*
2637 * Now the socket is dead. No more input will appear.
2638 */
1da177e4
LT		 2639 sock_orphan(sk);
2640 sock->sk = NULL;
2641
2642 /* Purge queues */
2643
2644 skb_queue_purge(&sk->sk_receive_queue);
b0138408 2645 packet_free_pending(po);
17ab56a2 2646 sk_refcnt_debug_release(sk);
1da177e4
LT		 2647
2648 sock_put(sk);
2649 return 0;
2650}
2651
2652/*
2653 * Attach a packet hook.
2654 */
2655
902fefb8 2656static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
1da177e4
LT		 2657{
2658 struct packet_sock *po = pkt_sk(sk);
902fefb8
DB		 2659 const struct net_device *dev_curr;
2660 __be16 proto_curr;
2661 bool need_rehook;
dc99f600 2662
aef950b4
WY		 2663 if (po->fanout) {
2664 if (dev)
2665 dev_put(dev);
2666
dc99f600 2667 return -EINVAL;
aef950b4 2668 }
1da177e4
LT		 2669
2670 lock_sock(sk);
1da177e4 2671 spin_lock(&po->bind_lock);
66e56cd4 2672
902fefb8
DB		 2673 proto_curr = po->prot_hook.type;
2674 dev_curr = po->prot_hook.dev;
2675
2676 need_rehook = proto_curr != proto || dev_curr != dev;
2677
2678 if (need_rehook) {
2679 unregister_prot_hook(sk, true);
1da177e4 2680
902fefb8
DB		 2681 po->num = proto;
2682 po->prot_hook.type = proto;
1da177e4 2683
902fefb8
DB		 2684 if (po->prot_hook.dev)
2685 dev_put(po->prot_hook.dev);
2686
2687 po->prot_hook.dev = dev;
2688
2689 po->ifindex = dev ? dev->ifindex : 0;
2690 packet_cached_dev_assign(po, dev);
2691 }
66e56cd4 2692
902fefb8 2693 if (proto == 0 || !need_rehook)
1da177e4
LT		 2694 goto out_unlock;
2695
be85d4ad 2696 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2697 register_prot_hook(sk);
be85d4ad
UT		 2698 } else {
2699 sk->sk_err = ENETDOWN;
2700 if (!sock_flag(sk, SOCK_DEAD))
2701 sk->sk_error_report(sk);
1da177e4
LT		 2702 }
2703
2704out_unlock:
2705 spin_unlock(&po->bind_lock);
2706 release_sock(sk);
2707 return 0;
2708}
2709
2710/*
2711 * Bind a packet socket to a device
2712 */
2713
40d4e3df
ED		 2714static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2715 int addr_len)
1da177e4 2716{
40d4e3df 2717 struct sock *sk = sock->sk;
1da177e4
LT		 2718 char name[15];
2719 struct net_device *dev;
2720 int err = -ENODEV;
1ce4f28b 2721
1da177e4
LT		 2722 /*
2723 * Check legality
2724 */
1ce4f28b 2725
8ae55f04 2726 if (addr_len != sizeof(struct sockaddr))
1da177e4 2727 return -EINVAL;
40d4e3df 2728 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2729
3b1e0a65 2730 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2731 if (dev)
1da177e4 2732 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2733 return err;
2734}
1da177e4
LT		 2735
2736static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2737{
40d4e3df
ED		 2738 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2739 struct sock *sk = sock->sk;
1da177e4
LT		 2740 struct net_device *dev = NULL;
2741 int err;
2742
2743
2744 /*
2745 * Check legality
2746 */
1ce4f28b 2747
1da177e4
LT		 2748 if (addr_len < sizeof(struct sockaddr_ll))
2749 return -EINVAL;
2750 if (sll->sll_family != AF_PACKET)
2751 return -EINVAL;
2752
2753 if (sll->sll_ifindex) {
2754 err = -ENODEV;
3b1e0a65 2755 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2756 if (dev == NULL)
2757 goto out;
2758 }
2759 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2760
2761out:
2762 return err;
2763}
2764
2765static struct proto packet_proto = {
2766 .name = "PACKET",
2767 .owner = THIS_MODULE,
2768 .obj_size = sizeof(struct packet_sock),
2769};
2770
2771/*
1ce4f28b 2772 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2773 */
2774
3f378b68
EP		 2775static int packet_create(struct net *net, struct socket *sock, int protocol,
2776 int kern)
1da177e4
LT		 2777{
2778 struct sock *sk;
2779 struct packet_sock *po;
0e11c91e 2780 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2781 int err;
2782
df008c91 2783 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2784 return -EPERM;
be02097c
DM		 2785 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2786 sock->type != SOCK_PACKET)
1da177e4
LT		 2787 return -ESOCKTNOSUPPORT;
2788
2789 sock->state = SS_UNCONNECTED;
2790
2791 err = -ENOBUFS;
6257ff21 2792 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2793 if (sk == NULL)
2794 goto out;
2795
2796 sock->ops = &packet_ops;
1da177e4
LT		 2797 if (sock->type == SOCK_PACKET)
2798 sock->ops = &packet_ops_spkt;
be02097c 2799
1da177e4
LT		 2800 sock_init_data(sock, sk);
2801
2802 po = pkt_sk(sk);
2803 sk->sk_family = PF_PACKET;
0e11c91e 2804 po->num = proto;
d346a3fa 2805 po->xmit = dev_queue_xmit;
66e56cd4 2806
b0138408
DB		 2807 err = packet_alloc_pending(po);
2808 if (err)
2809 goto out2;
2810
66e56cd4 2811 packet_cached_dev_reset(po);
1da177e4
LT		 2812
2813 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2814 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2815
2816 /*
2817 * Attach a protocol block
2818 */
2819
2820 spin_lock_init(&po->bind_lock);
905db440 2821 mutex_init(&po->pg_vec_lock);
1da177e4 2822 po->prot_hook.func = packet_rcv;
be02097c 2823
1da177e4
LT		 2824 if (sock->type == SOCK_PACKET)
2825 po->prot_hook.func = packet_rcv_spkt;
be02097c 2826
1da177e4
LT		 2827 po->prot_hook.af_packet_priv = sk;
2828
0e11c91e
AV		 2829 if (proto) {
2830 po->prot_hook.type = proto;
ce06b03e 2831 register_prot_hook(sk);
1da177e4
LT		 2832 }
2833
0fa7fa98 2834 mutex_lock(&net->packet.sklist_lock);
808f5114 2835 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2836 mutex_unlock(&net->packet.sklist_lock);
2837
2838 preempt_disable();
3680453c 2839 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2840 preempt_enable();
808f5114 2841
40d4e3df 2842 return 0;
b0138408
DB		 2843out2:
2844 sk_free(sk);
1da177e4
LT		 2845out:
2846 return err;
2847}
2848
2849/*
2850 * Pull a packet from our receive queue and hand it to the user.
2851 * If necessary we block.
2852 */
2853
2854static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2855 struct msghdr *msg, size_t len, int flags)
2856{
2857 struct sock *sk = sock->sk;
2858 struct sk_buff *skb;
2859 int copied, err;
bfd5f4a3 2860 int vnet_hdr_len = 0;
1da177e4
LT		 2861
2862 err = -EINVAL;
ed85b565 2863 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2864 goto out;
2865
2866#if 0
2867 /* What error should we return now? EUNATTACH? */
2868 if (pkt_sk(sk)->ifindex < 0)
2869 return -ENODEV;
2870#endif
2871
ed85b565 2872 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2873 err = sock_recv_errqueue(sk, msg, len,
2874 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2875 goto out;
2876 }
2877
1da177e4
LT		 2878 /*
2879 * Call the generic datagram receiver. This handles all sorts
2880 * of horrible races and re-entrancy so we can forget about it
2881 * in the protocol layers.
2882 *
2883 * Now it will return ENETDOWN, if device have just gone down,
2884 * but then it will block.
2885 */
2886
40d4e3df 2887 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2888
2889 /*
1ce4f28b 2890 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2891 * handles the blocking we don't see and worry about blocking
2892 * retries.
2893 */
2894
8ae55f04 2895 if (skb == NULL)
1da177e4
LT		 2896 goto out;
2897
bfd5f4a3
SS		 2898 if (pkt_sk(sk)->has_vnet_hdr) {
2899 struct virtio_net_hdr vnet_hdr = { 0 };
2900
2901 err = -EINVAL;
2902 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2903 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2904 goto out_free;
2905
1f18b717
MK		 2906 len -= vnet_hdr_len;
2907
bfd5f4a3
SS		 2908 if (skb_is_gso(skb)) {
2909 struct skb_shared_info *sinfo = skb_shinfo(skb);
2910
2911 /* This is a hint as to how much should be linear. */
2912 vnet_hdr.hdr_len = skb_headlen(skb);
2913 vnet_hdr.gso_size = sinfo->gso_size;
2914 if (sinfo->gso_type & SKB_GSO_TCPV4)
2915 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2916 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2917 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2918 else if (sinfo->gso_type & SKB_GSO_UDP)
2919 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2920 else if (sinfo->gso_type & SKB_GSO_FCOE)
2921 goto out_free;
2922 else
2923 BUG();
2924 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2925 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2926 } else
2927 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2928
2929 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2930 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2931 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2932 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2933 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2934 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2935 } /* else everything is zero */
2936
2937 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2938 vnet_hdr_len);
2939 if (err < 0)
2940 goto out_free;
2941 }
2942
f3d33426
HFS		 2943 /* You lose any data beyond the buffer you gave. If it worries
2944 * a user program they can ask the device for its MTU
2945 * anyway.
1da177e4 2946 */
1da177e4 2947 copied = skb->len;
40d4e3df
ED		 2948 if (copied > len) {
2949 copied = len;
2950 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2951 }
2952
2953 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2954 if (err)
2955 goto out_free;
2956
3b885787 2957 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2958
f3d33426
HFS		 2959 if (msg->msg_name) {
2960 /* If the address length field is there to be filled
2961 * in, we fill it in now.
2962 */
2963 if (sock->type == SOCK_PACKET) {
342dfc30 2964 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426
HFS		 2965 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2966 } else {
2967 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2968 msg->msg_namelen = sll->sll_halen +
2969 offsetof(struct sockaddr_ll, sll_addr);
2970 }
ffbc6111
HX		 2971 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2972 msg->msg_namelen);
f3d33426 2973 }
1da177e4 2974
8dc41944 2975 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2976 struct tpacket_auxdata aux;
2977
2978 aux.tp_status = TP_STATUS_USER;
2979 if (skb->ip_summed == CHECKSUM_PARTIAL)
2980 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2981 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2982 aux.tp_snaplen = skb->len;
2983 aux.tp_mac = 0;
bbe735e4 2984 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2985 if (vlan_tx_tag_present(skb)) {
2986 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 2987 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
2988 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2989 } else {
2990 aux.tp_vlan_tci = 0;
a0cdfcf3 2991 aux.tp_vlan_tpid = 0;
a3bcc23e 2992 }
ffbc6111 2993 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2994 }
2995
1da177e4
LT		 2996 /*
2997 * Free or return the buffer as appropriate. Again this
2998 * hides all the races and re-entrancy issues from us.
2999 */
bfd5f4a3 3000 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 3001
3002out_free:
3003 skb_free_datagram(sk, skb);
3004out:
3005 return err;
3006}
3007
1da177e4
LT		 3008static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3009 int *uaddr_len, int peer)
3010{
3011 struct net_device *dev;
3012 struct sock *sk = sock->sk;
3013
3014 if (peer)
3015 return -EOPNOTSUPP;
3016
3017 uaddr->sa_family = AF_PACKET;
2dc85bf3 3018 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 3019 rcu_read_lock();
3020 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3021 if (dev)
2dc85bf3 3022 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 3023 rcu_read_unlock();
1da177e4
LT		 3024 *uaddr_len = sizeof(*uaddr);
3025
3026 return 0;
3027}
1da177e4
LT		 3028
3029static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3030 int *uaddr_len, int peer)
3031{
3032 struct net_device *dev;
3033 struct sock *sk = sock->sk;
3034 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3035 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3036
3037 if (peer)
3038 return -EOPNOTSUPP;
3039
3040 sll->sll_family = AF_PACKET;
3041 sll->sll_ifindex = po->ifindex;
3042 sll->sll_protocol = po->num;
67286640 3043 sll->sll_pkttype = 0;
654d1f8a
ED		 3044 rcu_read_lock();
3045 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3046 if (dev) {
3047 sll->sll_hatype = dev->type;
3048 sll->sll_halen = dev->addr_len;
3049 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3050 } else {
3051 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3052 sll->sll_halen = 0;
3053 }
654d1f8a 3054 rcu_read_unlock();
0fb375fb 3055 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3056
3057 return 0;
3058}
3059
2aeb0b88
WC		 3060static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3061 int what)
1da177e4
LT		 3062{
3063 switch (i->type) {
3064 case PACKET_MR_MULTICAST:
1162563f
JP		 3065 if (i->alen != dev->addr_len)
3066 return -EINVAL;
1da177e4 3067 if (what > 0)
22bedad3 3068 return dev_mc_add(dev, i->addr);
1da177e4 3069 else
22bedad3 3070 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3071 break;
3072 case PACKET_MR_PROMISC:
2aeb0b88 3073 return dev_set_promiscuity(dev, what);
1da177e4
LT		 3074 break;
3075 case PACKET_MR_ALLMULTI:
2aeb0b88 3076 return dev_set_allmulti(dev, what);
1da177e4 3077 break;
d95ed927 3078 case PACKET_MR_UNICAST:
1162563f
JP		 3079 if (i->alen != dev->addr_len)
3080 return -EINVAL;
d95ed927 3081 if (what > 0)
a748ee24 3082 return dev_uc_add(dev, i->addr);
d95ed927 3083 else
a748ee24 3084 return dev_uc_del(dev, i->addr);
d95ed927 3085 break;
40d4e3df
ED		 3086 default:
3087 break;
1da177e4 3088 }
2aeb0b88 3089 return 0;
1da177e4
LT		 3090}
3091
3092static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
3093{
40d4e3df 3094 for ( ; i; i = i->next) {
1da177e4
LT		 3095 if (i->ifindex == dev->ifindex)
3096 packet_dev_mc(dev, i, what);
3097 }
3098}
3099
0fb375fb 3100static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3101{
3102 struct packet_sock *po = pkt_sk(sk);
3103 struct packet_mclist *ml, *i;
3104 struct net_device *dev;
3105 int err;
3106
3107 rtnl_lock();
3108
3109 err = -ENODEV;
3b1e0a65 3110 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3111 if (!dev)
3112 goto done;
3113
3114 err = -EINVAL;
1162563f 3115 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3116 goto done;
3117
3118 err = -ENOBUFS;
8b3a7005 3119 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3120 if (i == NULL)
3121 goto done;
3122
3123 err = 0;
3124 for (ml = po->mclist; ml; ml = ml->next) {
3125 if (ml->ifindex == mreq->mr_ifindex &&
3126 ml->type == mreq->mr_type &&
3127 ml->alen == mreq->mr_alen &&
3128 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3129 ml->count++;
3130 /* Free the new element ... */
3131 kfree(i);
3132 goto done;
3133 }
3134 }
3135
3136 i->type = mreq->mr_type;
3137 i->ifindex = mreq->mr_ifindex;
3138 i->alen = mreq->mr_alen;
3139 memcpy(i->addr, mreq->mr_address, i->alen);
3140 i->count = 1;
3141 i->next = po->mclist;
3142 po->mclist = i;
2aeb0b88
WC		 3143 err = packet_dev_mc(dev, i, 1);
3144 if (err) {
3145 po->mclist = i->next;
3146 kfree(i);
3147 }
1da177e4
LT		 3148
3149done:
3150 rtnl_unlock();
3151 return err;
3152}
3153
0fb375fb 3154static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3155{
3156 struct packet_mclist *ml, **mlp;
3157
3158 rtnl_lock();
3159
3160 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3161 if (ml->ifindex == mreq->mr_ifindex &&
3162 ml->type == mreq->mr_type &&
3163 ml->alen == mreq->mr_alen &&
3164 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3165 if (--ml->count == 0) {
3166 struct net_device *dev;
3167 *mlp = ml->next;
ad959e76
ED		 3168 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3169 if (dev)
1da177e4 3170 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3171 kfree(ml);
3172 }
3173 rtnl_unlock();
3174 return 0;
3175 }
3176 }
3177 rtnl_unlock();
3178 return -EADDRNOTAVAIL;
3179}
3180
3181static void packet_flush_mclist(struct sock *sk)
3182{
3183 struct packet_sock *po = pkt_sk(sk);
3184 struct packet_mclist *ml;
3185
3186 if (!po->mclist)
3187 return;
3188
3189 rtnl_lock();
3190 while ((ml = po->mclist) != NULL) {
3191 struct net_device *dev;
3192
3193 po->mclist = ml->next;
ad959e76
ED		 3194 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3195 if (dev != NULL)
1da177e4 3196 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3197 kfree(ml);
3198 }
3199 rtnl_unlock();
3200}
1da177e4
LT		 3201
3202static int
b7058842 3203packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3204{
3205 struct sock *sk = sock->sk;
8dc41944 3206 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3207 int ret;
3208
3209 if (level != SOL_PACKET)
3210 return -ENOPROTOOPT;
3211
69e3c75f 3212 switch (optname) {
1ce4f28b 3213 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3214 case PACKET_DROP_MEMBERSHIP:
3215 {
0fb375fb
EB		 3216 struct packet_mreq_max mreq;
3217 int len = optlen;
3218 memset(&mreq, 0, sizeof(mreq));
3219 if (len < sizeof(struct packet_mreq))
1da177e4 3220 return -EINVAL;
0fb375fb
EB		 3221 if (len > sizeof(mreq))
3222 len = sizeof(mreq);
40d4e3df 3223 if (copy_from_user(&mreq, optval, len))
1da177e4 3224 return -EFAULT;
0fb375fb
EB		 3225 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3226 return -EINVAL;
1da177e4
LT		 3227 if (optname == PACKET_ADD_MEMBERSHIP)
3228 ret = packet_mc_add(sk, &mreq);
3229 else
3230 ret = packet_mc_drop(sk, &mreq);
3231 return ret;
3232 }
a2efcfa0 3233
1da177e4 3234 case PACKET_RX_RING:
69e3c75f 3235 case PACKET_TX_RING:
1da177e4 3236 {
f6fb8f10 3237 union tpacket_req_u req_u;
3238 int len;
1da177e4 3239
f6fb8f10 3240 switch (po->tp_version) {
3241 case TPACKET_V1:
3242 case TPACKET_V2:
3243 len = sizeof(req_u.req);
3244 break;
3245 case TPACKET_V3:
3246 default:
3247 len = sizeof(req_u.req3);
3248 break;
3249 }
3250 if (optlen < len)
1da177e4 3251 return -EINVAL;
bfd5f4a3
SS		 3252 if (pkt_sk(sk)->has_vnet_hdr)
3253 return -EINVAL;
f6fb8f10 3254 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3255 return -EFAULT;
f6fb8f10 3256 return packet_set_ring(sk, &req_u, 0,
3257 optname == PACKET_TX_RING);
1da177e4
LT		 3258 }
3259 case PACKET_COPY_THRESH:
3260 {
3261 int val;
3262
40d4e3df 3263 if (optlen != sizeof(val))
1da177e4 3264 return -EINVAL;
40d4e3df 3265 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3266 return -EFAULT;
3267
3268 pkt_sk(sk)->copy_thresh = val;
3269 return 0;
3270 }
bbd6ef87
PM		 3271 case PACKET_VERSION:
3272 {
3273 int val;
3274
3275 if (optlen != sizeof(val))
3276 return -EINVAL;
69e3c75f 3277 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3278 return -EBUSY;
3279 if (copy_from_user(&val, optval, sizeof(val)))
3280 return -EFAULT;
3281 switch (val) {
3282 case TPACKET_V1:
3283 case TPACKET_V2:
f6fb8f10 3284 case TPACKET_V3:
bbd6ef87
PM		 3285 po->tp_version = val;
3286 return 0;
3287 default:
3288 return -EINVAL;
3289 }
3290 }
8913336a
PM		 3291 case PACKET_RESERVE:
3292 {
3293 unsigned int val;
3294
3295 if (optlen != sizeof(val))
3296 return -EINVAL;
69e3c75f 3297 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3298 return -EBUSY;
3299 if (copy_from_user(&val, optval, sizeof(val)))
3300 return -EFAULT;
3301 po->tp_reserve = val;
3302 return 0;
3303 }
69e3c75f
JB		 3304 case PACKET_LOSS:
3305 {
3306 unsigned int val;
3307
3308 if (optlen != sizeof(val))
3309 return -EINVAL;
3310 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3311 return -EBUSY;
3312 if (copy_from_user(&val, optval, sizeof(val)))
3313 return -EFAULT;
3314 po->tp_loss = !!val;
3315 return 0;
3316 }
8dc41944
HX		 3317 case PACKET_AUXDATA:
3318 {
3319 int val;
3320
3321 if (optlen < sizeof(val))
3322 return -EINVAL;
3323 if (copy_from_user(&val, optval, sizeof(val)))
3324 return -EFAULT;
3325
3326 po->auxdata = !!val;
3327 return 0;
3328 }
80feaacb
PWJ		 3329 case PACKET_ORIGDEV:
3330 {
3331 int val;
3332
3333 if (optlen < sizeof(val))
3334 return -EINVAL;
3335 if (copy_from_user(&val, optval, sizeof(val)))
3336 return -EFAULT;
3337
3338 po->origdev = !!val;
3339 return 0;
3340 }
bfd5f4a3
SS		 3341 case PACKET_VNET_HDR:
3342 {
3343 int val;
3344
3345 if (sock->type != SOCK_RAW)
3346 return -EINVAL;
3347 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3348 return -EBUSY;
3349 if (optlen < sizeof(val))
3350 return -EINVAL;
3351 if (copy_from_user(&val, optval, sizeof(val)))
3352 return -EFAULT;
3353
3354 po->has_vnet_hdr = !!val;
3355 return 0;
3356 }
614f60fa
SM		 3357 case PACKET_TIMESTAMP:
3358 {
3359 int val;
3360
3361 if (optlen != sizeof(val))
3362 return -EINVAL;
3363 if (copy_from_user(&val, optval, sizeof(val)))
3364 return -EFAULT;
3365
3366 po->tp_tstamp = val;
3367 return 0;
3368 }
dc99f600
DM		 3369 case PACKET_FANOUT:
3370 {
3371 int val;
3372
3373 if (optlen != sizeof(val))
3374 return -EINVAL;
3375 if (copy_from_user(&val, optval, sizeof(val)))
3376 return -EFAULT;
3377
3378 return fanout_add(sk, val & 0xffff, val >> 16);
3379 }
5920cd3a
PC		 3380 case PACKET_TX_HAS_OFF:
3381 {
3382 unsigned int val;
3383
3384 if (optlen != sizeof(val))
3385 return -EINVAL;
3386 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3387 return -EBUSY;
3388 if (copy_from_user(&val, optval, sizeof(val)))
3389 return -EFAULT;
3390 po->tp_tx_has_off = !!val;
3391 return 0;
3392 }
d346a3fa
DB		 3393 case PACKET_QDISC_BYPASS:
3394 {
3395 int val;
3396
3397 if (optlen != sizeof(val))
3398 return -EINVAL;
3399 if (copy_from_user(&val, optval, sizeof(val)))
3400 return -EFAULT;
3401
3402 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3403 return 0;
3404 }
1da177e4
LT		 3405 default:
3406 return -ENOPROTOOPT;
3407 }
3408}
3409
3410static int packet_getsockopt(struct socket *sock, int level, int optname,
3411 char __user *optval, int __user *optlen)
3412{
3413 int len;
c06fff6e 3414 int val, lv = sizeof(val);
1da177e4
LT		 3415 struct sock *sk = sock->sk;
3416 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3417 void *data = &val;
ee80fbf3 3418 union tpacket_stats_u st;
1da177e4
LT		 3419
3420 if (level != SOL_PACKET)
3421 return -ENOPROTOOPT;
3422
8ae55f04
KK		 3423 if (get_user(len, optlen))
3424 return -EFAULT;
1da177e4
LT		 3425
3426 if (len < 0)
3427 return -EINVAL;
1ce4f28b 3428
69e3c75f 3429 switch (optname) {
1da177e4 3430 case PACKET_STATISTICS:
1da177e4 3431 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3432 memcpy(&st, &po->stats, sizeof(st));
3433 memset(&po->stats, 0, sizeof(po->stats));
3434 spin_unlock_bh(&sk->sk_receive_queue.lock);
3435
f6fb8f10 3436 if (po->tp_version == TPACKET_V3) {
c06fff6e 3437 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3438 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3439 data = &st.stats3;
f6fb8f10 3440 } else {
c06fff6e 3441 lv = sizeof(struct tpacket_stats);
8bcdeaff 3442 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3443 data = &st.stats1;
f6fb8f10 3444 }
ee80fbf3 3445
8dc41944
HX		 3446 break;
3447 case PACKET_AUXDATA:
8dc41944 3448 val = po->auxdata;
80feaacb
PWJ		 3449 break;
3450 case PACKET_ORIGDEV:
80feaacb 3451 val = po->origdev;
bfd5f4a3
SS		 3452 break;
3453 case PACKET_VNET_HDR:
bfd5f4a3 3454 val = po->has_vnet_hdr;
1da177e4 3455 break;
bbd6ef87 3456 case PACKET_VERSION:
bbd6ef87 3457 val = po->tp_version;
bbd6ef87
PM		 3458 break;
3459 case PACKET_HDRLEN:
3460 if (len > sizeof(int))
3461 len = sizeof(int);
3462 if (copy_from_user(&val, optval, len))
3463 return -EFAULT;
3464 switch (val) {
3465 case TPACKET_V1:
3466 val = sizeof(struct tpacket_hdr);
3467 break;
3468 case TPACKET_V2:
3469 val = sizeof(struct tpacket2_hdr);
3470 break;
f6fb8f10 3471 case TPACKET_V3:
3472 val = sizeof(struct tpacket3_hdr);
3473 break;
bbd6ef87
PM		 3474 default:
3475 return -EINVAL;
3476 }
bbd6ef87 3477 break;
8913336a 3478 case PACKET_RESERVE:
8913336a 3479 val = po->tp_reserve;
8913336a 3480 break;
69e3c75f 3481 case PACKET_LOSS:
69e3c75f 3482 val = po->tp_loss;
69e3c75f 3483 break;
614f60fa 3484 case PACKET_TIMESTAMP:
614f60fa 3485 val = po->tp_tstamp;
614f60fa 3486 break;
dc99f600 3487 case PACKET_FANOUT:
dc99f600
DM		 3488 val = (po->fanout ?
3489 ((u32)po->fanout->id |
77f65ebd
WB		 3490 ((u32)po->fanout->type << 16) |
3491 ((u32)po->fanout->flags << 24)) :
dc99f600 3492 0);
dc99f600 3493 break;
5920cd3a
PC		 3494 case PACKET_TX_HAS_OFF:
3495 val = po->tp_tx_has_off;
3496 break;
d346a3fa
DB		 3497 case PACKET_QDISC_BYPASS:
3498 val = packet_use_direct_xmit(po);
3499 break;
1da177e4
LT		 3500 default:
3501 return -ENOPROTOOPT;
3502 }
3503
c06fff6e
ED		 3504 if (len > lv)
3505 len = lv;
8ae55f04
KK		 3506 if (put_user(len, optlen))
3507 return -EFAULT;
8dc41944
HX		 3508 if (copy_to_user(optval, data, len))
3509 return -EFAULT;
8ae55f04 3510 return 0;
1da177e4
LT		 3511}
3512
3513
351638e7
JP		 3514static int packet_notifier(struct notifier_block *this,
3515 unsigned long msg, void *ptr)
1da177e4
LT		 3516{
3517 struct sock *sk;
351638e7 3518 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3519 struct net *net = dev_net(dev);
1da177e4 3520
808f5114 3521 rcu_read_lock();
b67bfe0d 3522 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3523 struct packet_sock *po = pkt_sk(sk);
3524
3525 switch (msg) {
3526 case NETDEV_UNREGISTER:
1da177e4
LT		 3527 if (po->mclist)
3528 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3529 /* fallthrough */
3530
1da177e4
LT		 3531 case NETDEV_DOWN:
3532 if (dev->ifindex == po->ifindex) {
3533 spin_lock(&po->bind_lock);
3534 if (po->running) {
ce06b03e 3535 __unregister_prot_hook(sk, false);
1da177e4
LT		 3536 sk->sk_err = ENETDOWN;
3537 if (!sock_flag(sk, SOCK_DEAD))
3538 sk->sk_error_report(sk);
3539 }
3540 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3541 packet_cached_dev_reset(po);
1da177e4 3542 po->ifindex = -1;
160ff18a
BG		 3543 if (po->prot_hook.dev)
3544 dev_put(po->prot_hook.dev);
1da177e4
LT		 3545 po->prot_hook.dev = NULL;
3546 }
3547 spin_unlock(&po->bind_lock);
3548 }
3549 break;
3550 case NETDEV_UP:
808f5114 3551 if (dev->ifindex == po->ifindex) {
3552 spin_lock(&po->bind_lock);
ce06b03e
DM		 3553 if (po->num)
3554 register_prot_hook(sk);
808f5114 3555 spin_unlock(&po->bind_lock);
1da177e4 3556 }
1da177e4
LT		 3557 break;
3558 }
3559 }
808f5114 3560 rcu_read_unlock();
1da177e4
LT		 3561 return NOTIFY_DONE;
3562}
3563
3564
3565static int packet_ioctl(struct socket *sock, unsigned int cmd,
3566 unsigned long arg)
3567{
3568 struct sock *sk = sock->sk;
3569
69e3c75f 3570 switch (cmd) {
40d4e3df
ED		 3571 case SIOCOUTQ:
3572 {
3573 int amount = sk_wmem_alloc_get(sk);
31e6d363 3574
40d4e3df
ED		 3575 return put_user(amount, (int __user *)arg);
3576 }
3577 case SIOCINQ:
3578 {
3579 struct sk_buff *skb;
3580 int amount = 0;
3581
3582 spin_lock_bh(&sk->sk_receive_queue.lock);
3583 skb = skb_peek(&sk->sk_receive_queue);
3584 if (skb)
3585 amount = skb->len;
3586 spin_unlock_bh(&sk->sk_receive_queue.lock);
3587 return put_user(amount, (int __user *)arg);
3588 }
3589 case SIOCGSTAMP:
3590 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3591 case SIOCGSTAMPNS:
3592 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3593
1da177e4 3594#ifdef CONFIG_INET
40d4e3df
ED		 3595 case SIOCADDRT:
3596 case SIOCDELRT:
3597 case SIOCDARP:
3598 case SIOCGARP:
3599 case SIOCSARP:
3600 case SIOCGIFADDR:
3601 case SIOCSIFADDR:
3602 case SIOCGIFBRDADDR:
3603 case SIOCSIFBRDADDR:
3604 case SIOCGIFNETMASK:
3605 case SIOCSIFNETMASK:
3606 case SIOCGIFDSTADDR:
3607 case SIOCSIFDSTADDR:
3608 case SIOCSIFFLAGS:
40d4e3df 3609 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3610#endif
3611
40d4e3df
ED		 3612 default:
3613 return -ENOIOCTLCMD;
1da177e4
LT		 3614 }
3615 return 0;
3616}
3617
40d4e3df 3618static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3619 poll_table *wait)
3620{
3621 struct sock *sk = sock->sk;
3622 struct packet_sock *po = pkt_sk(sk);
3623 unsigned int mask = datagram_poll(file, sock, wait);
3624
3625 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3626 if (po->rx_ring.pg_vec) {
f6fb8f10 3627 if (!packet_previous_rx_frame(po, &po->rx_ring,
3628 TP_STATUS_KERNEL))
1da177e4
LT		 3629 mask |= POLLIN | POLLRDNORM;
3630 }
3631 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3632 spin_lock_bh(&sk->sk_write_queue.lock);
3633 if (po->tx_ring.pg_vec) {
3634 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3635 mask |= POLLOUT | POLLWRNORM;
3636 }
3637 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3638 return mask;
3639}
3640
3641
3642/* Dirty? Well, I still did not learn better way to account
3643 * for user mmaps.
3644 */
3645
3646static void packet_mm_open(struct vm_area_struct *vma)
3647{
3648 struct file *file = vma->vm_file;
40d4e3df 3649 struct socket *sock = file->private_data;
1da177e4 3650 struct sock *sk = sock->sk;
1ce4f28b 3651
1da177e4
LT		 3652 if (sk)
3653 atomic_inc(&pkt_sk(sk)->mapped);
3654}
3655
3656static void packet_mm_close(struct vm_area_struct *vma)
3657{
3658 struct file *file = vma->vm_file;
40d4e3df 3659 struct socket *sock = file->private_data;
1da177e4 3660 struct sock *sk = sock->sk;
1ce4f28b 3661
1da177e4
LT		 3662 if (sk)
3663 atomic_dec(&pkt_sk(sk)->mapped);
3664}
3665
f0f37e2f 3666static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3667 .open = packet_mm_open,
3668 .close = packet_mm_close,
1da177e4
LT		 3669};
3670
0e3125c7
NH		 3671static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3672 unsigned int len)
1da177e4
LT		 3673{
3674 int i;
3675
4ebf0ae2 3676 for (i = 0; i < len; i++) {
0e3125c7 3677 if (likely(pg_vec[i].buffer)) {
c56b4d90 3678 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3679 vfree(pg_vec[i].buffer);
3680 else
3681 free_pages((unsigned long)pg_vec[i].buffer,
3682 order);
3683 pg_vec[i].buffer = NULL;
3684 }
1da177e4
LT		 3685 }
3686 kfree(pg_vec);
3687}
3688
eea49cc9 3689static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3690{
f0d4eb29 3691 char *buffer;
0e3125c7
NH		 3692 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3693 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3694
3695 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3696 if (buffer)
3697 return buffer;
3698
f0d4eb29 3699 /* __get_free_pages failed, fall back to vmalloc */
bbce5a59 3700 buffer = vzalloc((1 << order) * PAGE_SIZE);
0e3125c7
NH		 3701 if (buffer)
3702 return buffer;
3703
f0d4eb29 3704 /* vmalloc failed, lets dig into swap here */
0e3125c7 3705 gfp_flags &= ~__GFP_NORETRY;
f0d4eb29 3706 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3707 if (buffer)
3708 return buffer;
3709
f0d4eb29 3710 /* complete and utter failure */
0e3125c7 3711 return NULL;
4ebf0ae2
DM		 3712}
3713
0e3125c7 3714static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3715{
3716 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3717 struct pgv *pg_vec;
4ebf0ae2
DM		 3718 int i;
3719
0e3125c7 3720 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3721 if (unlikely(!pg_vec))
3722 goto out;
3723
3724 for (i = 0; i < block_nr; i++) {
c56b4d90 3725 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3726 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3727 goto out_free_pgvec;
3728 }
3729
3730out:
3731 return pg_vec;
3732
3733out_free_pgvec:
3734 free_pg_vec(pg_vec, order, block_nr);
3735 pg_vec = NULL;
3736 goto out;
3737}
1da177e4 3738
f6fb8f10 3739static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3740 int closing, int tx_ring)
1da177e4 3741{
0e3125c7 3742 struct pgv *pg_vec = NULL;
1da177e4 3743 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3744 int was_running, order = 0;
69e3c75f
JB		 3745 struct packet_ring_buffer *rb;
3746 struct sk_buff_head *rb_queue;
0e11c91e 3747 __be16 num;
f6fb8f10 3748 int err = -EINVAL;
3749 /* Added to avoid minimal code churn */
3750 struct tpacket_req *req = &req_u->req;
3751
3752 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3753 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3754 WARN(1, "Tx-ring is not supported.\n");
3755 goto out;
3756 }
1ce4f28b 3757
69e3c75f
JB		 3758 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3759 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3760
69e3c75f
JB		 3761 err = -EBUSY;
3762 if (!closing) {
3763 if (atomic_read(&po->mapped))
3764 goto out;
b0138408 3765 if (packet_read_pending(rb))
69e3c75f
JB		 3766 goto out;
3767 }
1da177e4 3768
69e3c75f
JB		 3769 if (req->tp_block_nr) {
3770 /* Sanity tests and some calculations */
3771 err = -EBUSY;
3772 if (unlikely(rb->pg_vec))
3773 goto out;
1da177e4 3774
bbd6ef87
PM		 3775 switch (po->tp_version) {
3776 case TPACKET_V1:
3777 po->tp_hdrlen = TPACKET_HDRLEN;
3778 break;
3779 case TPACKET_V2:
3780 po->tp_hdrlen = TPACKET2_HDRLEN;
3781 break;
f6fb8f10 3782 case TPACKET_V3:
3783 po->tp_hdrlen = TPACKET3_HDRLEN;
3784 break;
bbd6ef87
PM		 3785 }
3786
69e3c75f 3787 err = -EINVAL;
4ebf0ae2 3788 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3789 goto out;
4ebf0ae2 3790 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3791 goto out;
8913336a 3792 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3793 po->tp_reserve))
3794 goto out;
4ebf0ae2 3795 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3796 goto out;
1da177e4 3797
69e3c75f
JB		 3798 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3799 if (unlikely(rb->frames_per_block <= 0))
3800 goto out;
3801 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3802 req->tp_frame_nr))
3803 goto out;
1da177e4
LT		 3804
3805 err = -ENOMEM;
4ebf0ae2
DM		 3806 order = get_order(req->tp_block_size);
3807 pg_vec = alloc_pg_vec(req, order);
3808 if (unlikely(!pg_vec))
1da177e4 3809 goto out;
f6fb8f10 3810 switch (po->tp_version) {
3811 case TPACKET_V3:
3812 /* Transmit path is not supported. We checked
3813 * it above but just being paranoid
3814 */
3815 if (!tx_ring)
3816 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
d7cf0c34 3817 break;
f6fb8f10 3818 default:
3819 break;
3820 }
69e3c75f
JB		 3821 }
3822 /* Done */
3823 else {
3824 err = -EINVAL;
4ebf0ae2 3825 if (unlikely(req->tp_frame_nr))
69e3c75f 3826 goto out;
1da177e4
LT		 3827 }
3828
3829 lock_sock(sk);
3830
3831 /* Detach socket from network */
3832 spin_lock(&po->bind_lock);
3833 was_running = po->running;
3834 num = po->num;
3835 if (was_running) {
1da177e4 3836 po->num = 0;
ce06b03e 3837 __unregister_prot_hook(sk, false);
1da177e4
LT		 3838 }
3839 spin_unlock(&po->bind_lock);
1ce4f28b 3840
1da177e4
LT		 3841 synchronize_net();
3842
3843 err = -EBUSY;
905db440 3844 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3845 if (closing || atomic_read(&po->mapped) == 0) {
3846 err = 0;
69e3c75f 3847 spin_lock_bh(&rb_queue->lock);
c053fd96 3848 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3849 rb->frame_max = (req->tp_frame_nr - 1);
3850 rb->head = 0;
3851 rb->frame_size = req->tp_frame_size;
3852 spin_unlock_bh(&rb_queue->lock);
3853
c053fd96
CG		 3854 swap(rb->pg_vec_order, order);
3855 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3856
3857 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3858 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3859 tpacket_rcv : packet_rcv;
3860 skb_queue_purge(rb_queue);
1da177e4 3861 if (atomic_read(&po->mapped))
40d4e3df
ED		 3862 pr_err("packet_mmap: vma is busy: %d\n",
3863 atomic_read(&po->mapped));
1da177e4 3864 }
905db440 3865 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3866
3867 spin_lock(&po->bind_lock);
ce06b03e 3868 if (was_running) {
1da177e4 3869 po->num = num;
ce06b03e 3870 register_prot_hook(sk);
1da177e4
LT		 3871 }
3872 spin_unlock(&po->bind_lock);
f6fb8f10 3873 if (closing && (po->tp_version > TPACKET_V2)) {
3874 /* Because we don't support block-based V3 on tx-ring */
3875 if (!tx_ring)
3876 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3877 }
1da177e4
LT		 3878 release_sock(sk);
3879
1da177e4
LT		 3880 if (pg_vec)
3881 free_pg_vec(pg_vec, order, req->tp_block_nr);
3882out:
3883 return err;
3884}
3885
69e3c75f
JB		 3886static int packet_mmap(struct file *file, struct socket *sock,
3887 struct vm_area_struct *vma)
1da177e4
LT		 3888{
3889 struct sock *sk = sock->sk;
3890 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3891 unsigned long size, expected_size;
3892 struct packet_ring_buffer *rb;
1da177e4
LT		 3893 unsigned long start;
3894 int err = -EINVAL;
3895 int i;
3896
3897 if (vma->vm_pgoff)
3898 return -EINVAL;
3899
905db440 3900 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3901
3902 expected_size = 0;
3903 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3904 if (rb->pg_vec) {
3905 expected_size += rb->pg_vec_len
3906 * rb->pg_vec_pages
3907 * PAGE_SIZE;
3908 }
3909 }
3910
3911 if (expected_size == 0)
1da177e4 3912 goto out;
69e3c75f
JB		 3913
3914 size = vma->vm_end - vma->vm_start;
3915 if (size != expected_size)
1da177e4
LT		 3916 goto out;
3917
1da177e4 3918 start = vma->vm_start;
69e3c75f
JB		 3919 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3920 if (rb->pg_vec == NULL)
3921 continue;
3922
3923 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3924 struct page *page;
3925 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3926 int pg_num;
3927
c56b4d90
CG		 3928 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3929 page = pgv_to_page(kaddr);
69e3c75f
JB		 3930 err = vm_insert_page(vma, start, page);
3931 if (unlikely(err))
3932 goto out;
3933 start += PAGE_SIZE;
0e3125c7 3934 kaddr += PAGE_SIZE;
69e3c75f 3935 }
4ebf0ae2 3936 }
1da177e4 3937 }
69e3c75f 3938
4ebf0ae2 3939 atomic_inc(&po->mapped);
1da177e4
LT		 3940 vma->vm_ops = &packet_mmap_ops;
3941 err = 0;
3942
3943out:
905db440 3944 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3945 return err;
3946}
1da177e4 3947
90ddc4f0 3948static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3949 .family = PF_PACKET,
3950 .owner = THIS_MODULE,
3951 .release = packet_release,
3952 .bind = packet_bind_spkt,
3953 .connect = sock_no_connect,
3954 .socketpair = sock_no_socketpair,
3955 .accept = sock_no_accept,
3956 .getname = packet_getname_spkt,
3957 .poll = datagram_poll,
3958 .ioctl = packet_ioctl,
3959 .listen = sock_no_listen,
3960 .shutdown = sock_no_shutdown,
3961 .setsockopt = sock_no_setsockopt,
3962 .getsockopt = sock_no_getsockopt,
3963 .sendmsg = packet_sendmsg_spkt,
3964 .recvmsg = packet_recvmsg,
3965 .mmap = sock_no_mmap,
3966 .sendpage = sock_no_sendpage,
3967};
1da177e4 3968
90ddc4f0 3969static const struct proto_ops packet_ops = {
1da177e4
LT		 3970 .family = PF_PACKET,
3971 .owner = THIS_MODULE,
3972 .release = packet_release,
3973 .bind = packet_bind,
3974 .connect = sock_no_connect,
3975 .socketpair = sock_no_socketpair,
3976 .accept = sock_no_accept,
1ce4f28b 3977 .getname = packet_getname,
1da177e4
LT		 3978 .poll = packet_poll,
3979 .ioctl = packet_ioctl,
3980 .listen = sock_no_listen,
3981 .shutdown = sock_no_shutdown,
3982 .setsockopt = packet_setsockopt,
3983 .getsockopt = packet_getsockopt,
3984 .sendmsg = packet_sendmsg,
3985 .recvmsg = packet_recvmsg,
3986 .mmap = packet_mmap,
3987 .sendpage = sock_no_sendpage,
3988};
3989
ec1b4cf7 3990static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3991 .family = PF_PACKET,
3992 .create = packet_create,
3993 .owner = THIS_MODULE,
3994};
3995
3996static struct notifier_block packet_netdev_notifier = {
40d4e3df 3997 .notifier_call = packet_notifier,
1da177e4
LT		 3998};
3999
4000#ifdef CONFIG_PROC_FS
1da177e4
LT		 4001
4002static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 4003 __acquires(RCU)
1da177e4 4004{
e372c414 4005 struct net *net = seq_file_net(seq);
808f5114 4006
4007 rcu_read_lock();
4008 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 4009}
4010
4011static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4012{
1bf40954 4013 struct net *net = seq_file_net(seq);
808f5114 4014 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 4015}
4016
4017static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 4018 __releases(RCU)
1da177e4 4019{
808f5114 4020 rcu_read_unlock();
1da177e4
LT		 4021}
4022
1ce4f28b 4023static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 4024{
4025 if (v == SEQ_START_TOKEN)
4026 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4027 else {
b7ceabd9 4028 struct sock *s = sk_entry(v);
1da177e4
LT		 4029 const struct packet_sock *po = pkt_sk(s);
4030
4031 seq_printf(seq,
71338aa7 4032 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 4033 s,
4034 atomic_read(&s->sk_refcnt),
4035 s->sk_type,
4036 ntohs(po->num),
4037 po->ifindex,
4038 po->running,
4039 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4040 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4041 sock_i_ino(s));
1da177e4
LT		 4042 }
4043
4044 return 0;
4045}
4046
56b3d975 4047static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4048 .start = packet_seq_start,
4049 .next = packet_seq_next,
4050 .stop = packet_seq_stop,
4051 .show = packet_seq_show,
4052};
4053
4054static int packet_seq_open(struct inode *inode, struct file *file)
4055{
e372c414
DL		 4056 return seq_open_net(inode, file, &packet_seq_ops,
4057 sizeof(struct seq_net_private));
1da177e4
LT		 4058}
4059
da7071d7 4060static const struct file_operations packet_seq_fops = {
1da177e4
LT		 4061 .owner = THIS_MODULE,
4062 .open = packet_seq_open,
4063 .read = seq_read,
4064 .llseek = seq_lseek,
e372c414 4065 .release = seq_release_net,
1da177e4
LT		 4066};
4067
4068#endif
4069
2c8c1e72 4070static int __net_init packet_net_init(struct net *net)
d12d01d6 4071{
0fa7fa98 4072 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4073 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4074
d4beaa66 4075 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 4076 return -ENOMEM;
4077
4078 return 0;
4079}
4080
2c8c1e72 4081static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4082{
ece31ffd 4083 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 4084}
4085
4086static struct pernet_operations packet_net_ops = {
4087 .init = packet_net_init,
4088 .exit = packet_net_exit,
4089};
4090
4091
1da177e4
LT		 4092static void __exit packet_exit(void)
4093{
1da177e4 4094 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4095 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4096 sock_unregister(PF_PACKET);
4097 proto_unregister(&packet_proto);
4098}
4099
4100static int __init packet_init(void)
4101{
4102 int rc = proto_register(&packet_proto, 0);
4103
4104 if (rc != 0)
4105 goto out;
4106
4107 sock_register(&packet_family_ops);
d12d01d6 4108 register_pernet_subsys(&packet_net_ops);
1da177e4 4109 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4110out:
4111 return rc;
4112}
4113
4114module_init(packet_init);
4115module_exit(packet_exit);
4116MODULE_LICENSE("GPL");
4117MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 1.285064 seconds and 5 git commands to generate.