projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge git://git.kernel.org/pub/scm/linux/kernel/git/pablo/nf-next
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
b0138408 91#include <linux/percpu.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111 218struct packet_skb_cb {
ffbc6111
HX		 219 union {
220 struct sockaddr_pkt pkt;
2472d761
EB		 221 union {
222 /* Trick: alias skb original length with
223 * ll.sll_family and ll.protocol in order
224 * to save room.
225 */
226 unsigned int origlen;
227 struct sockaddr_ll ll;
228 };
ffbc6111
HX		 229 } sa;
230};
231
232#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 233
bc59ba39 234#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 235#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 236 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 237#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 238 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 239#define GET_NEXT_PRB_BLK_NUM(x) \
240 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
241 ((x)->kactive_blk_num+1) : 0)
242
dc99f600
DM		 243static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
244static void __fanout_link(struct sock *sk, struct packet_sock *po);
245
d346a3fa
DB		 246static int packet_direct_xmit(struct sk_buff *skb)
247{
248 struct net_device *dev = skb->dev;
d346a3fa
DB		 249 netdev_features_t features;
250 struct netdev_queue *txq;
43279500 251 int ret = NETDEV_TX_BUSY;
d346a3fa
DB		 252
253 if (unlikely(!netif_running(dev) ||
43279500
DB		 254 !netif_carrier_ok(dev)))
255 goto drop;
d346a3fa
DB		 256
257 features = netif_skb_features(skb);
258 if (skb_needs_linearize(skb, features) &&
43279500
DB		 259 __skb_linearize(skb))
260 goto drop;
d346a3fa 261
10c51b56 262 txq = skb_get_tx_queue(dev, skb);
d346a3fa 263
43279500
DB		 264 local_bh_disable();
265
266 HARD_TX_LOCK(dev, txq, smp_processor_id());
10b3ad8c 267 if (!netif_xmit_frozen_or_drv_stopped(txq))
fa2dbdc2 268 ret = netdev_start_xmit(skb, dev, txq, false);
43279500 269 HARD_TX_UNLOCK(dev, txq);
d346a3fa 270
43279500
DB		 271 local_bh_enable();
272
273 if (!dev_xmit_complete(ret))
d346a3fa 274 kfree_skb(skb);
43279500 275
d346a3fa 276 return ret;
43279500 277drop:
0f97ede4 278 atomic_long_inc(&dev->tx_dropped);
43279500
DB		 279 kfree_skb(skb);
280 return NET_XMIT_DROP;
d346a3fa
DB		 281}
282
66e56cd4
DB		 283static struct net_device *packet_cached_dev_get(struct packet_sock *po)
284{
285 struct net_device *dev;
286
287 rcu_read_lock();
288 dev = rcu_dereference(po->cached_dev);
289 if (likely(dev))
290 dev_hold(dev);
291 rcu_read_unlock();
292
293 return dev;
294}
295
296static void packet_cached_dev_assign(struct packet_sock *po,
297 struct net_device *dev)
298{
299 rcu_assign_pointer(po->cached_dev, dev);
300}
301
302static void packet_cached_dev_reset(struct packet_sock *po)
303{
304 RCU_INIT_POINTER(po->cached_dev, NULL);
305}
306
d346a3fa
DB		 307static bool packet_use_direct_xmit(const struct packet_sock *po)
308{
309 return po->xmit == packet_direct_xmit;
310}
311
0fd5d57b 312static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
d346a3fa 313{
1cbac010 314 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 315}
316
0fd5d57b
DB		 317static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
318{
319 const struct net_device_ops *ops = dev->netdev_ops;
320 u16 queue_index;
321
322 if (ops->ndo_select_queue) {
323 queue_index = ops->ndo_select_queue(dev, skb, NULL,
324 __packet_pick_tx_queue);
325 queue_index = netdev_cap_txqueue(dev, queue_index);
326 } else {
327 queue_index = __packet_pick_tx_queue(dev, skb);
328 }
329
330 skb_set_queue_mapping(skb, queue_index);
331}
332
ce06b03e
DM		 333/* register_prot_hook must be invoked with the po->bind_lock held,
334 * or from a context in which asynchronous accesses to the packet
335 * socket is not possible (packet_create()).
336 */
337static void register_prot_hook(struct sock *sk)
338{
339 struct packet_sock *po = pkt_sk(sk);
e40526cb 340
ce06b03e 341 if (!po->running) {
66e56cd4 342 if (po->fanout)
dc99f600 343 __fanout_link(sk, po);
66e56cd4 344 else
dc99f600 345 dev_add_pack(&po->prot_hook);
e40526cb 346
ce06b03e
DM		 347 sock_hold(sk);
348 po->running = 1;
349 }
350}
351
352/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
353 * held. If the sync parameter is true, we will temporarily drop
354 * the po->bind_lock and do a synchronize_net to make sure no
355 * asynchronous packet processing paths still refer to the elements
356 * of po->prot_hook. If the sync parameter is false, it is the
357 * callers responsibility to take care of this.
358 */
359static void __unregister_prot_hook(struct sock *sk, bool sync)
360{
361 struct packet_sock *po = pkt_sk(sk);
362
363 po->running = 0;
66e56cd4
DB		 364
365 if (po->fanout)
dc99f600 366 __fanout_unlink(sk, po);
66e56cd4 367 else
dc99f600 368 __dev_remove_pack(&po->prot_hook);
e40526cb 369
ce06b03e
DM		 370 __sock_put(sk);
371
372 if (sync) {
373 spin_unlock(&po->bind_lock);
374 synchronize_net();
375 spin_lock(&po->bind_lock);
376 }
377}
378
379static void unregister_prot_hook(struct sock *sk, bool sync)
380{
381 struct packet_sock *po = pkt_sk(sk);
382
383 if (po->running)
384 __unregister_prot_hook(sk, sync);
385}
386
6e58040b 387static inline struct page * __pure pgv_to_page(void *addr)
0af55bb5
CG		 388{
389 if (is_vmalloc_addr(addr))
390 return vmalloc_to_page(addr);
391 return virt_to_page(addr);
392}
393
69e3c75f 394static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 395{
184f489e 396 union tpacket_uhdr h;
1da177e4 397
69e3c75f 398 h.raw = frame;
bbd6ef87
PM		 399 switch (po->tp_version) {
400 case TPACKET_V1:
69e3c75f 401 h.h1->tp_status = status;
0af55bb5 402 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 403 break;
404 case TPACKET_V2:
69e3c75f 405 h.h2->tp_status = status;
0af55bb5 406 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 407 break;
f6fb8f10 408 case TPACKET_V3:
69e3c75f 409 default:
f6fb8f10 410 WARN(1, "TPACKET version not supported.\n");
69e3c75f 411 BUG();
bbd6ef87 412 }
69e3c75f
JB		 413
414 smp_wmb();
bbd6ef87
PM		 415}
416
69e3c75f 417static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 418{
184f489e 419 union tpacket_uhdr h;
bbd6ef87 420
69e3c75f
JB		 421 smp_rmb();
422
bbd6ef87
PM		 423 h.raw = frame;
424 switch (po->tp_version) {
425 case TPACKET_V1:
0af55bb5 426 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 427 return h.h1->tp_status;
bbd6ef87 428 case TPACKET_V2:
0af55bb5 429 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 430 return h.h2->tp_status;
f6fb8f10 431 case TPACKET_V3:
69e3c75f 432 default:
f6fb8f10 433 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 434 BUG();
435 return 0;
bbd6ef87 436 }
1da177e4 437}
69e3c75f 438
b9c32fb2
DB		 439static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
440 unsigned int flags)
7a51384c
DB		 441{
442 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
443
68a360e8
WB		 444 if (shhwtstamps &&
445 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
446 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
447 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 448
449 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 450 return TP_STATUS_TS_SOFTWARE;
7a51384c 451
b9c32fb2 452 return 0;
7a51384c
DB		 453}
454
b9c32fb2
DB		 455static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
456 struct sk_buff *skb)
2e31396f
WB		 457{
458 union tpacket_uhdr h;
459 struct timespec ts;
b9c32fb2 460 __u32 ts_status;
2e31396f 461
b9c32fb2
DB		 462 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
463 return 0;
2e31396f
WB		 464
465 h.raw = frame;
466 switch (po->tp_version) {
467 case TPACKET_V1:
468 h.h1->tp_sec = ts.tv_sec;
469 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
470 break;
471 case TPACKET_V2:
472 h.h2->tp_sec = ts.tv_sec;
473 h.h2->tp_nsec = ts.tv_nsec;
474 break;
475 case TPACKET_V3:
476 default:
477 WARN(1, "TPACKET version not supported.\n");
478 BUG();
479 }
480
481 /* one flush is safe, as both fields always lie on the same cacheline */
482 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
483 smp_wmb();
b9c32fb2
DB		 484
485 return ts_status;
2e31396f
WB		 486}
487
69e3c75f
JB		 488static void *packet_lookup_frame(struct packet_sock *po,
489 struct packet_ring_buffer *rb,
490 unsigned int position,
491 int status)
492{
493 unsigned int pg_vec_pos, frame_offset;
184f489e 494 union tpacket_uhdr h;
69e3c75f
JB		 495
496 pg_vec_pos = position / rb->frames_per_block;
497 frame_offset = position % rb->frames_per_block;
498
0e3125c7
NH		 499 h.raw = rb->pg_vec[pg_vec_pos].buffer +
500 (frame_offset * rb->frame_size);
69e3c75f
JB		 501
502 if (status != __packet_get_status(po, h.raw))
503 return NULL;
504
505 return h.raw;
506}
507
eea49cc9 508static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 509 struct packet_ring_buffer *rb,
510 int status)
511{
512 return packet_lookup_frame(po, rb, rb->head, status);
513}
514
bc59ba39 515static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 516{
517 del_timer_sync(&pkc->retire_blk_timer);
518}
519
520static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
521 int tx_ring,
522 struct sk_buff_head *rb_queue)
523{
bc59ba39 524 struct tpacket_kbdq_core *pkc;
f6fb8f10 525
22781a5b
DJ		 526 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
527 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 528
ec6f809f 529 spin_lock_bh(&rb_queue->lock);
f6fb8f10 530 pkc->delete_blk_timer = 1;
ec6f809f 531 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 532
533 prb_del_retire_blk_timer(pkc);
534}
535
536static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 537 struct tpacket_kbdq_core *pkc,
f6fb8f10 538 void (*func) (unsigned long))
539{
540 init_timer(&pkc->retire_blk_timer);
541 pkc->retire_blk_timer.data = (long)po;
542 pkc->retire_blk_timer.function = func;
543 pkc->retire_blk_timer.expires = jiffies;
544}
545
546static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
547{
bc59ba39 548 struct tpacket_kbdq_core *pkc;
f6fb8f10 549
550 if (tx_ring)
551 BUG();
552
22781a5b
DJ		 553 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
554 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 555 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
556}
557
558static int prb_calc_retire_blk_tmo(struct packet_sock *po,
559 int blk_size_in_bytes)
560{
561 struct net_device *dev;
562 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 563 struct ethtool_cmd ecmd;
564 int err;
e440cf2c 565 u32 speed;
f6fb8f10 566
4bc71cb9
JP		 567 rtnl_lock();
568 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
569 if (unlikely(!dev)) {
570 rtnl_unlock();
f6fb8f10 571 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 572 }
573 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 574 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 575 rtnl_unlock();
576 if (!err) {
4bc71cb9
JP		 577 /*
578 * If the link speed is so slow you don't really
579 * need to worry about perf anyways
580 */
e440cf2c 581 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 582 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 583 } else {
584 msec = 1;
585 div = speed / 1000;
f6fb8f10 586 }
587 }
588
589 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
590
591 if (div)
592 mbits /= div;
593
594 tmo = mbits * msec;
595
596 if (div)
597 return tmo+1;
598 return tmo;
599}
600
bc59ba39 601static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 602 union tpacket_req_u *req_u)
603{
604 p1->feature_req_word = req_u->req3.tp_feature_req_word;
605}
606
607static void init_prb_bdqc(struct packet_sock *po,
608 struct packet_ring_buffer *rb,
609 struct pgv *pg_vec,
610 union tpacket_req_u *req_u, int tx_ring)
611{
22781a5b 612 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 613 struct tpacket_block_desc *pbd;
f6fb8f10 614
615 memset(p1, 0x0, sizeof(*p1));
616
617 p1->knxt_seq_num = 1;
618 p1->pkbdq = pg_vec;
bc59ba39 619 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 620 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 621 p1->kblk_size = req_u->req3.tp_block_size;
622 p1->knum_blocks = req_u->req3.tp_block_nr;
623 p1->hdrlen = po->tp_hdrlen;
624 p1->version = po->tp_version;
625 p1->last_kactive_blk_num = 0;
ee80fbf3 626 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 627 if (req_u->req3.tp_retire_blk_tov)
628 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
629 else
630 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
631 req_u->req3.tp_block_size);
632 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
633 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
634
dc808110 635 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
f6fb8f10 636 prb_init_ft_ops(p1, req_u);
637 prb_setup_retire_blk_timer(po, tx_ring);
638 prb_open_block(p1, pbd);
639}
640
641/* Do NOT update the last_blk_num first.
642 * Assumes sk_buff_head lock is held.
643 */
bc59ba39 644static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 645{
646 mod_timer(&pkc->retire_blk_timer,
647 jiffies + pkc->tov_in_jiffies);
648 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
649}
650
651/*
652 * Timer logic:
653 * 1) We refresh the timer only when we open a block.
654 * By doing this we don't waste cycles refreshing the timer
655 * on packet-by-packet basis.
656 *
657 * With a 1MB block-size, on a 1Gbps line, it will take
658 * i) ~8 ms to fill a block + ii) memcpy etc.
659 * In this cut we are not accounting for the memcpy time.
660 *
661 * So, if the user sets the 'tmo' to 10ms then the timer
662 * will never fire while the block is still getting filled
663 * (which is what we want). However, the user could choose
664 * to close a block early and that's fine.
665 *
666 * But when the timer does fire, we check whether or not to refresh it.
667 * Since the tmo granularity is in msecs, it is not too expensive
668 * to refresh the timer, lets say every '8' msecs.
669 * Either the user can set the 'tmo' or we can derive it based on
670 * a) line-speed and b) block-size.
671 * prb_calc_retire_blk_tmo() calculates the tmo.
672 *
673 */
674static void prb_retire_rx_blk_timer_expired(unsigned long data)
675{
676 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 677 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 678 unsigned int frozen;
bc59ba39 679 struct tpacket_block_desc *pbd;
f6fb8f10 680
681 spin_lock(&po->sk.sk_receive_queue.lock);
682
683 frozen = prb_queue_frozen(pkc);
684 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
685
686 if (unlikely(pkc->delete_blk_timer))
687 goto out;
688
689 /* We only need to plug the race when the block is partially filled.
690 * tpacket_rcv:
691 * lock(); increment BLOCK_NUM_PKTS; unlock()
692 * copy_bits() is in progress ...
693 * timer fires on other cpu:
694 * we can't retire the current block because copy_bits
695 * is in progress.
696 *
697 */
698 if (BLOCK_NUM_PKTS(pbd)) {
699 while (atomic_read(&pkc->blk_fill_in_prog)) {
700 /* Waiting for skb_copy_bits to finish... */
701 cpu_relax();
702 }
703 }
704
705 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
706 if (!frozen) {
41a50d62
AD		 707 if (!BLOCK_NUM_PKTS(pbd)) {
708 /* An empty block. Just refresh the timer. */
709 goto refresh_timer;
710 }
f6fb8f10 711 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
712 if (!prb_dispatch_next_block(pkc, po))
713 goto refresh_timer;
714 else
715 goto out;
716 } else {
717 /* Case 1. Queue was frozen because user-space was
718 * lagging behind.
719 */
720 if (prb_curr_blk_in_use(pkc, pbd)) {
721 /*
722 * Ok, user-space is still behind.
723 * So just refresh the timer.
724 */
725 goto refresh_timer;
726 } else {
727 /* Case 2. queue was frozen,user-space caught up,
728 * now the link went idle && the timer fired.
729 * We don't have a block to close.So we open this
730 * block and restart the timer.
731 * opening a block thaws the queue,restarts timer
732 * Thawing/timer-refresh is a side effect.
733 */
734 prb_open_block(pkc, pbd);
735 goto out;
736 }
737 }
738 }
739
740refresh_timer:
741 _prb_refresh_rx_retire_blk_timer(pkc);
742
743out:
744 spin_unlock(&po->sk.sk_receive_queue.lock);
745}
746
eea49cc9 747static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 748 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 749{
750 /* Flush everything minus the block header */
751
752#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
753 u8 *start, *end;
754
755 start = (u8 *)pbd1;
756
757 /* Skip the block header(we know header WILL fit in 4K) */
758 start += PAGE_SIZE;
759
760 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
761 for (; start < end; start += PAGE_SIZE)
762 flush_dcache_page(pgv_to_page(start));
763
764 smp_wmb();
765#endif
766
767 /* Now update the block status. */
768
769 BLOCK_STATUS(pbd1) = status;
770
771 /* Flush the block header */
772
773#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
774 start = (u8 *)pbd1;
775 flush_dcache_page(pgv_to_page(start));
776
777 smp_wmb();
778#endif
779}
780
781/*
782 * Side effect:
783 *
784 * 1) flush the block
785 * 2) Increment active_blk_num
786 *
787 * Note:We DONT refresh the timer on purpose.
788 * Because almost always the next block will be opened.
789 */
bc59ba39 790static void prb_close_block(struct tpacket_kbdq_core *pkc1,
791 struct tpacket_block_desc *pbd1,
f6fb8f10 792 struct packet_sock *po, unsigned int stat)
793{
794 __u32 status = TP_STATUS_USER | stat;
795
796 struct tpacket3_hdr *last_pkt;
bc59ba39 797 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
da413eec 798 struct sock *sk = &po->sk;
f6fb8f10 799
ee80fbf3 800 if (po->stats.stats3.tp_drops)
f6fb8f10 801 status |= TP_STATUS_LOSING;
802
803 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
804 last_pkt->tp_next_offset = 0;
805
806 /* Get the ts of the last pkt */
807 if (BLOCK_NUM_PKTS(pbd1)) {
808 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
809 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
810 } else {
41a50d62
AD		 811 /* Ok, we tmo'd - so get the current time.
812 *
813 * It shouldn't really happen as we don't close empty
814 * blocks. See prb_retire_rx_blk_timer_expired().
815 */
f6fb8f10 816 struct timespec ts;
817 getnstimeofday(&ts);
818 h1->ts_last_pkt.ts_sec = ts.tv_sec;
819 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
820 }
821
822 smp_wmb();
823
824 /* Flush the block */
825 prb_flush_block(pkc1, pbd1, status);
826
da413eec
DC		 827 sk->sk_data_ready(sk);
828
f6fb8f10 829 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
830}
831
eea49cc9 832static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 833{
834 pkc->reset_pending_on_curr_blk = 0;
835}
836
837/*
838 * Side effect of opening a block:
839 *
840 * 1) prb_queue is thawed.
841 * 2) retire_blk_timer is refreshed.
842 *
843 */
bc59ba39 844static void prb_open_block(struct tpacket_kbdq_core *pkc1,
845 struct tpacket_block_desc *pbd1)
f6fb8f10 846{
847 struct timespec ts;
bc59ba39 848 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 849
850 smp_rmb();
851
8da3056c
DB		 852 /* We could have just memset this but we will lose the
853 * flexibility of making the priv area sticky
854 */
f6fb8f10 855
8da3056c
DB		 856 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
857 BLOCK_NUM_PKTS(pbd1) = 0;
858 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 859
8da3056c
DB		 860 getnstimeofday(&ts);
861
862 h1->ts_first_pkt.ts_sec = ts.tv_sec;
863 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 864
8da3056c
DB		 865 pkc1->pkblk_start = (char *)pbd1;
866 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
867
868 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
869 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
870
871 pbd1->version = pkc1->version;
872 pkc1->prev = pkc1->nxt_offset;
873 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
874
875 prb_thaw_queue(pkc1);
876 _prb_refresh_rx_retire_blk_timer(pkc1);
877
878 smp_wmb();
f6fb8f10 879}
880
881/*
882 * Queue freeze logic:
883 * 1) Assume tp_block_nr = 8 blocks.
884 * 2) At time 't0', user opens Rx ring.
885 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
886 * 4) user-space is either sleeping or processing block '0'.
887 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
888 * it will close block-7,loop around and try to fill block '0'.
889 * call-flow:
890 * __packet_lookup_frame_in_block
891 * prb_retire_current_block()
892 * prb_dispatch_next_block()
893 * |->(BLOCK_STATUS == USER) evaluates to true
894 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
895 * 6) Now there are two cases:
896 * 6.1) Link goes idle right after the queue is frozen.
897 * But remember, the last open_block() refreshed the timer.
898 * When this timer expires,it will refresh itself so that we can
899 * re-open block-0 in near future.
900 * 6.2) Link is busy and keeps on receiving packets. This is a simple
901 * case and __packet_lookup_frame_in_block will check if block-0
902 * is free and can now be re-used.
903 */
eea49cc9 904static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 905 struct packet_sock *po)
906{
907 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 908 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 909}
910
911#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
912
913/*
914 * If the next block is free then we will dispatch it
915 * and return a good offset.
916 * Else, we will freeze the queue.
917 * So, caller must check the return value.
918 */
bc59ba39 919static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 920 struct packet_sock *po)
921{
bc59ba39 922 struct tpacket_block_desc *pbd;
f6fb8f10 923
924 smp_rmb();
925
926 /* 1. Get current block num */
927 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
928
929 /* 2. If this block is currently in_use then freeze the queue */
930 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
931 prb_freeze_queue(pkc, po);
932 return NULL;
933 }
934
935 /*
936 * 3.
937 * open this block and return the offset where the first packet
938 * needs to get stored.
939 */
940 prb_open_block(pkc, pbd);
941 return (void *)pkc->nxt_offset;
942}
943
bc59ba39 944static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 945 struct packet_sock *po, unsigned int status)
946{
bc59ba39 947 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 948
949 /* retire/close the current block */
950 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
951 /*
952 * Plug the case where copy_bits() is in progress on
953 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
954 * have space to copy the pkt in the current block and
955 * called prb_retire_current_block()
956 *
957 * We don't need to worry about the TMO case because
958 * the timer-handler already handled this case.
959 */
960 if (!(status & TP_STATUS_BLK_TMO)) {
961 while (atomic_read(&pkc->blk_fill_in_prog)) {
962 /* Waiting for skb_copy_bits to finish... */
963 cpu_relax();
964 }
965 }
966 prb_close_block(pkc, pbd, po, status);
967 return;
968 }
f6fb8f10 969}
970
eea49cc9 971static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 972 struct tpacket_block_desc *pbd)
f6fb8f10 973{
974 return TP_STATUS_USER & BLOCK_STATUS(pbd);
975}
976
eea49cc9 977static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 978{
979 return pkc->reset_pending_on_curr_blk;
980}
981
eea49cc9 982static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 983{
bc59ba39 984 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 985 atomic_dec(&pkc->blk_fill_in_prog);
986}
987
eea49cc9 988static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 989 struct tpacket3_hdr *ppd)
990{
3958afa1 991 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 992}
993
eea49cc9 994static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 995 struct tpacket3_hdr *ppd)
996{
997 ppd->hv1.tp_rxhash = 0;
998}
999
eea49cc9 1000static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 1001 struct tpacket3_hdr *ppd)
1002{
df8a39de
JP		 1003 if (skb_vlan_tag_present(pkc->skb)) {
1004 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
a0cdfcf3
AW		 1005 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1006 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 1007 } else {
9e67030a 1008 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 1009 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 1010 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 1011 }
1012}
1013
bc59ba39 1014static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 1015 struct tpacket3_hdr *ppd)
1016{
a0cdfcf3 1017 ppd->hv1.tp_padding = 0;
f6fb8f10 1018 prb_fill_vlan_info(pkc, ppd);
1019
1020 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1021 prb_fill_rxhash(pkc, ppd);
1022 else
1023 prb_clear_rxhash(pkc, ppd);
1024}
1025
eea49cc9 1026static void prb_fill_curr_block(char *curr,
bc59ba39 1027 struct tpacket_kbdq_core *pkc,
1028 struct tpacket_block_desc *pbd,
f6fb8f10 1029 unsigned int len)
1030{
1031 struct tpacket3_hdr *ppd;
1032
1033 ppd = (struct tpacket3_hdr *)curr;
1034 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1035 pkc->prev = curr;
1036 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1037 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1038 BLOCK_NUM_PKTS(pbd) += 1;
1039 atomic_inc(&pkc->blk_fill_in_prog);
1040 prb_run_all_ft_ops(pkc, ppd);
1041}
1042
1043/* Assumes caller has the sk->rx_queue.lock */
1044static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1045 struct sk_buff *skb,
1046 int status,
1047 unsigned int len
1048 )
1049{
bc59ba39 1050 struct tpacket_kbdq_core *pkc;
1051 struct tpacket_block_desc *pbd;
f6fb8f10 1052 char *curr, *end;
1053
e3192690 1054 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1055 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1056
1057 /* Queue is frozen when user space is lagging behind */
1058 if (prb_queue_frozen(pkc)) {
1059 /*
1060 * Check if that last block which caused the queue to freeze,
1061 * is still in_use by user-space.
1062 */
1063 if (prb_curr_blk_in_use(pkc, pbd)) {
1064 /* Can't record this packet */
1065 return NULL;
1066 } else {
1067 /*
1068 * Ok, the block was released by user-space.
1069 * Now let's open that block.
1070 * opening a block also thaws the queue.
1071 * Thawing is a side effect.
1072 */
1073 prb_open_block(pkc, pbd);
1074 }
1075 }
1076
1077 smp_mb();
1078 curr = pkc->nxt_offset;
1079 pkc->skb = skb;
e3192690 1080 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1081
1082 /* first try the current block */
1083 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1084 prb_fill_curr_block(curr, pkc, pbd, len);
1085 return (void *)curr;
1086 }
1087
1088 /* Ok, close the current block */
1089 prb_retire_current_block(pkc, po, 0);
1090
1091 /* Now, try to dispatch the next block */
1092 curr = (char *)prb_dispatch_next_block(pkc, po);
1093 if (curr) {
1094 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1095 prb_fill_curr_block(curr, pkc, pbd, len);
1096 return (void *)curr;
1097 }
1098
1099 /*
1100 * No free blocks are available.user_space hasn't caught up yet.
1101 * Queue was just frozen and now this packet will get dropped.
1102 */
1103 return NULL;
1104}
1105
eea49cc9 1106static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1107 struct sk_buff *skb,
1108 int status, unsigned int len)
1109{
1110 char *curr = NULL;
1111 switch (po->tp_version) {
1112 case TPACKET_V1:
1113 case TPACKET_V2:
1114 curr = packet_lookup_frame(po, &po->rx_ring,
1115 po->rx_ring.head, status);
1116 return curr;
1117 case TPACKET_V3:
1118 return __packet_lookup_frame_in_block(po, skb, status, len);
1119 default:
1120 WARN(1, "TPACKET version not supported\n");
1121 BUG();
99aa3473 1122 return NULL;
f6fb8f10 1123 }
1124}
1125
eea49cc9 1126static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1127 struct packet_ring_buffer *rb,
77f65ebd 1128 unsigned int idx,
f6fb8f10 1129 int status)
1130{
bc59ba39 1131 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1132 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1133
1134 if (status != BLOCK_STATUS(pbd))
1135 return NULL;
1136 return pbd;
1137}
1138
eea49cc9 1139static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1140{
1141 unsigned int prev;
1142 if (rb->prb_bdqc.kactive_blk_num)
1143 prev = rb->prb_bdqc.kactive_blk_num-1;
1144 else
1145 prev = rb->prb_bdqc.knum_blocks-1;
1146 return prev;
1147}
1148
1149/* Assumes caller has held the rx_queue.lock */
eea49cc9 1150static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1151 struct packet_ring_buffer *rb,
1152 int status)
1153{
1154 unsigned int previous = prb_previous_blk_num(rb);
1155 return prb_lookup_block(po, rb, previous, status);
1156}
1157
eea49cc9 1158static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1159 struct packet_ring_buffer *rb,
1160 int status)
1161{
1162 if (po->tp_version <= TPACKET_V2)
1163 return packet_previous_frame(po, rb, status);
1164
1165 return __prb_previous_block(po, rb, status);
1166}
1167
eea49cc9 1168static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1169 struct packet_ring_buffer *rb)
1170{
1171 switch (po->tp_version) {
1172 case TPACKET_V1:
1173 case TPACKET_V2:
1174 return packet_increment_head(rb);
1175 case TPACKET_V3:
1176 default:
1177 WARN(1, "TPACKET version not supported.\n");
1178 BUG();
1179 return;
1180 }
1181}
1182
eea49cc9 1183static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1184 struct packet_ring_buffer *rb,
1185 int status)
1186{
1187 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1188 return packet_lookup_frame(po, rb, previous, status);
1189}
1190
eea49cc9 1191static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1192{
1193 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1194}
1195
b0138408
DB		 1196static void packet_inc_pending(struct packet_ring_buffer *rb)
1197{
1198 this_cpu_inc(*rb->pending_refcnt);
1199}
1200
1201static void packet_dec_pending(struct packet_ring_buffer *rb)
1202{
1203 this_cpu_dec(*rb->pending_refcnt);
1204}
1205
1206static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1207{
1208 unsigned int refcnt = 0;
1209 int cpu;
1210
1211 /* We don't use pending refcount in rx_ring. */
1212 if (rb->pending_refcnt == NULL)
1213 return 0;
1214
1215 for_each_possible_cpu(cpu)
1216 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1217
1218 return refcnt;
1219}
1220
1221static int packet_alloc_pending(struct packet_sock *po)
1222{
1223 po->rx_ring.pending_refcnt = NULL;
1224
1225 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1226 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1227 return -ENOBUFS;
1228
1229 return 0;
1230}
1231
1232static void packet_free_pending(struct packet_sock *po)
1233{
1234 free_percpu(po->tx_ring.pending_refcnt);
1235}
1236
9954729b
WB		 1237#define ROOM_POW_OFF 2
1238#define ROOM_NONE 0x0
1239#define ROOM_LOW 0x1
1240#define ROOM_NORMAL 0x2
1241
1242static bool __tpacket_has_room(struct packet_sock *po, int pow_off)
77f65ebd 1243{
9954729b
WB		 1244 int idx, len;
1245
1246 len = po->rx_ring.frame_max + 1;
1247 idx = po->rx_ring.head;
1248 if (pow_off)
1249 idx += len >> pow_off;
1250 if (idx >= len)
1251 idx -= len;
1252 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1253}
1254
1255static bool __tpacket_v3_has_room(struct packet_sock *po, int pow_off)
1256{
1257 int idx, len;
1258
1259 len = po->rx_ring.prb_bdqc.knum_blocks;
1260 idx = po->rx_ring.prb_bdqc.kactive_blk_num;
1261 if (pow_off)
1262 idx += len >> pow_off;
1263 if (idx >= len)
1264 idx -= len;
1265 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1266}
77f65ebd 1267
2ccdbaa6 1268static int __packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
9954729b
WB		 1269{
1270 struct sock *sk = &po->sk;
1271 int ret = ROOM_NONE;
1272
1273 if (po->prot_hook.func != tpacket_rcv) {
1274 int avail = sk->sk_rcvbuf - atomic_read(&sk->sk_rmem_alloc)
2ccdbaa6 1275 - (skb ? skb->truesize : 0);
9954729b
WB		 1276 if (avail > (sk->sk_rcvbuf >> ROOM_POW_OFF))
1277 return ROOM_NORMAL;
1278 else if (avail > 0)
1279 return ROOM_LOW;
1280 else
1281 return ROOM_NONE;
1282 }
77f65ebd 1283
9954729b
WB		 1284 if (po->tp_version == TPACKET_V3) {
1285 if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1286 ret = ROOM_NORMAL;
1287 else if (__tpacket_v3_has_room(po, 0))
1288 ret = ROOM_LOW;
1289 } else {
1290 if (__tpacket_has_room(po, ROOM_POW_OFF))
1291 ret = ROOM_NORMAL;
1292 else if (__tpacket_has_room(po, 0))
1293 ret = ROOM_LOW;
1294 }
2ccdbaa6
WB		 1295
1296 return ret;
1297}
1298
1299static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1300{
1301 int ret;
1302 bool has_room;
1303
54d7c01d
WB		 1304 spin_lock_bh(&po->sk.sk_receive_queue.lock);
1305 ret = __packet_rcv_has_room(po, skb);
2ccdbaa6
WB		 1306 has_room = ret == ROOM_NORMAL;
1307 if (po->pressure == has_room)
54d7c01d
WB		 1308 po->pressure = !has_room;
1309 spin_unlock_bh(&po->sk.sk_receive_queue.lock);
77f65ebd 1310
9954729b 1311 return ret;
77f65ebd
WB		 1312}
1313
1da177e4
LT		 1314static void packet_sock_destruct(struct sock *sk)
1315{
ed85b565
RC		 1316 skb_queue_purge(&sk->sk_error_queue);
1317
547b792c
IJ		 1318 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1319 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1320
1321 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1322 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1323 return;
1324 }
1325
17ab56a2 1326 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1327}
1328
dc99f600
DM		 1329static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1330{
1331 int x = atomic_read(&f->rr_cur) + 1;
1332
1333 if (x >= num)
1334 x = 0;
1335
1336 return x;
1337}
1338
3b3a5b0a
WB		 1339static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1340{
1341 u32 rxhash;
1342 int i, count = 0;
1343
1344 rxhash = skb_get_hash(skb);
1345 for (i = 0; i < ROLLOVER_HLEN; i++)
1346 if (po->rollover->history[i] == rxhash)
1347 count++;
1348
1349 po->rollover->history[prandom_u32() % ROLLOVER_HLEN] = rxhash;
1350 return count > (ROLLOVER_HLEN >> 1);
1351}
1352
77f65ebd
WB		 1353static unsigned int fanout_demux_hash(struct packet_fanout *f,
1354 struct sk_buff *skb,
1355 unsigned int num)
dc99f600 1356{
61b905da 1357 return reciprocal_scale(skb_get_hash(skb), num);
dc99f600
DM		 1358}
1359
77f65ebd
WB		 1360static unsigned int fanout_demux_lb(struct packet_fanout *f,
1361 struct sk_buff *skb,
1362 unsigned int num)
dc99f600
DM		 1363{
1364 int cur, old;
1365
1366 cur = atomic_read(&f->rr_cur);
1367 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1368 fanout_rr_next(f, num))) != cur)
1369 cur = old;
77f65ebd
WB		 1370 return cur;
1371}
1372
1373static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1374 struct sk_buff *skb,
1375 unsigned int num)
1376{
1377 return smp_processor_id() % num;
dc99f600
DM		 1378}
1379
5df0ddfb
DB		 1380static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1381 struct sk_buff *skb,
1382 unsigned int num)
1383{
f337db64 1384 return prandom_u32_max(num);
5df0ddfb
DB		 1385}
1386
77f65ebd
WB		 1387static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1388 struct sk_buff *skb,
ad377cab 1389 unsigned int idx, bool try_self,
77f65ebd 1390 unsigned int num)
95ec3eb4 1391{
4633c9e0 1392 struct packet_sock *po, *po_next, *po_skip = NULL;
a9b63918 1393 unsigned int i, j, room = ROOM_NONE;
95ec3eb4 1394
0648ab70 1395 po = pkt_sk(f->arr[idx]);
3b3a5b0a
WB		 1396
1397 if (try_self) {
1398 room = packet_rcv_has_room(po, skb);
1399 if (room == ROOM_NORMAL ||
1400 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1401 return idx;
4633c9e0 1402 po_skip = po;
3b3a5b0a 1403 }
ad377cab 1404
0648ab70 1405 i = j = min_t(int, po->rollover->sock, num - 1);
77f65ebd 1406 do {
2ccdbaa6 1407 po_next = pkt_sk(f->arr[i]);
4633c9e0 1408 if (po_next != po_skip && !po_next->pressure &&
2ccdbaa6 1409 packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
77f65ebd 1410 if (i != j)
0648ab70 1411 po->rollover->sock = i;
a9b63918
WB		 1412 atomic_long_inc(&po->rollover->num);
1413 if (room == ROOM_LOW)
1414 atomic_long_inc(&po->rollover->num_huge);
77f65ebd
WB		 1415 return i;
1416 }
ad377cab 1417
77f65ebd
WB		 1418 if (++i == num)
1419 i = 0;
1420 } while (i != j);
1421
a9b63918 1422 atomic_long_inc(&po->rollover->num_failed);
77f65ebd
WB		 1423 return idx;
1424}
1425
2d36097d
NH		 1426static unsigned int fanout_demux_qm(struct packet_fanout *f,
1427 struct sk_buff *skb,
1428 unsigned int num)
1429{
1430 return skb_get_queue_mapping(skb) % num;
1431}
1432
77f65ebd
WB		 1433static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1434{
1435 return f->flags & (flag >> 8);
95ec3eb4
DM		 1436}
1437
95ec3eb4
DM		 1438static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1439 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1440{
1441 struct packet_fanout *f = pt->af_packet_priv;
1442 unsigned int num = f->num_members;
1443 struct packet_sock *po;
77f65ebd 1444 unsigned int idx;
dc99f600
DM		 1445
1446 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1447 !num) {
1448 kfree_skb(skb);
1449 return 0;
1450 }
1451
3f34b24a
AD		 1452 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1453 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
1454 if (!skb)
1455 return 0;
1456 }
95ec3eb4
DM		 1457 switch (f->type) {
1458 case PACKET_FANOUT_HASH:
1459 default:
77f65ebd 1460 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1461 break;
1462 case PACKET_FANOUT_LB:
77f65ebd 1463 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1464 break;
1465 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1466 idx = fanout_demux_cpu(f, skb, num);
1467 break;
5df0ddfb
DB		 1468 case PACKET_FANOUT_RND:
1469 idx = fanout_demux_rnd(f, skb, num);
1470 break;
2d36097d
NH		 1471 case PACKET_FANOUT_QM:
1472 idx = fanout_demux_qm(f, skb, num);
1473 break;
77f65ebd 1474 case PACKET_FANOUT_ROLLOVER:
ad377cab 1475 idx = fanout_demux_rollover(f, skb, 0, false, num);
95ec3eb4 1476 break;
dc99f600
DM		 1477 }
1478
ad377cab
WB		 1479 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1480 idx = fanout_demux_rollover(f, skb, idx, true, num);
dc99f600 1481
ad377cab 1482 po = pkt_sk(f->arr[idx]);
dc99f600
DM		 1483 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1484}
1485
fff3321d
PE		 1486DEFINE_MUTEX(fanout_mutex);
1487EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1488static LIST_HEAD(fanout_list);
1489
1490static void __fanout_link(struct sock *sk, struct packet_sock *po)
1491{
1492 struct packet_fanout *f = po->fanout;
1493
1494 spin_lock(&f->lock);
1495 f->arr[f->num_members] = sk;
1496 smp_wmb();
1497 f->num_members++;
1498 spin_unlock(&f->lock);
1499}
1500
1501static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1502{
1503 struct packet_fanout *f = po->fanout;
1504 int i;
1505
1506 spin_lock(&f->lock);
1507 for (i = 0; i < f->num_members; i++) {
1508 if (f->arr[i] == sk)
1509 break;
1510 }
1511 BUG_ON(i >= f->num_members);
1512 f->arr[i] = f->arr[f->num_members - 1];
1513 f->num_members--;
1514 spin_unlock(&f->lock);
1515}
1516
d4dd8aee 1517static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1518{
d4dd8aee 1519 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
c0de08d0
EL		 1520 return true;
1521
1522 return false;
1523}
1524
7736d33f 1525static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1526{
1527 struct packet_sock *po = pkt_sk(sk);
1528 struct packet_fanout *f, *match;
7736d33f 1529 u8 type = type_flags & 0xff;
77f65ebd 1530 u8 flags = type_flags >> 8;
dc99f600
DM		 1531 int err;
1532
1533 switch (type) {
77f65ebd
WB		 1534 case PACKET_FANOUT_ROLLOVER:
1535 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1536 return -EINVAL;
dc99f600
DM		 1537 case PACKET_FANOUT_HASH:
1538 case PACKET_FANOUT_LB:
95ec3eb4 1539 case PACKET_FANOUT_CPU:
5df0ddfb 1540 case PACKET_FANOUT_RND:
2d36097d 1541 case PACKET_FANOUT_QM:
dc99f600
DM		 1542 break;
1543 default:
1544 return -EINVAL;
1545 }
1546
1547 if (!po->running)
1548 return -EINVAL;
1549
1550 if (po->fanout)
1551 return -EALREADY;
1552
4633c9e0
WB		 1553 if (type == PACKET_FANOUT_ROLLOVER ||
1554 (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
0648ab70
WB		 1555 po->rollover = kzalloc(sizeof(*po->rollover), GFP_KERNEL);
1556 if (!po->rollover)
1557 return -ENOMEM;
a9b63918
WB		 1558 atomic_long_set(&po->rollover->num, 0);
1559 atomic_long_set(&po->rollover->num_huge, 0);
1560 atomic_long_set(&po->rollover->num_failed, 0);
0648ab70
WB		 1561 }
1562
dc99f600
DM		 1563 mutex_lock(&fanout_mutex);
1564 match = NULL;
1565 list_for_each_entry(f, &fanout_list, list) {
1566 if (f->id == id &&
1567 read_pnet(&f->net) == sock_net(sk)) {
1568 match = f;
1569 break;
1570 }
1571 }
afe62c68 1572 err = -EINVAL;
77f65ebd 1573 if (match && match->flags != flags)
afe62c68 1574 goto out;
dc99f600 1575 if (!match) {
afe62c68 1576 err = -ENOMEM;
dc99f600 1577 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1578 if (!match)
1579 goto out;
1580 write_pnet(&match->net, sock_net(sk));
1581 match->id = id;
1582 match->type = type;
77f65ebd 1583 match->flags = flags;
afe62c68
ED		 1584 atomic_set(&match->rr_cur, 0);
1585 INIT_LIST_HEAD(&match->list);
1586 spin_lock_init(&match->lock);
1587 atomic_set(&match->sk_ref, 0);
1588 match->prot_hook.type = po->prot_hook.type;
1589 match->prot_hook.dev = po->prot_hook.dev;
1590 match->prot_hook.func = packet_rcv_fanout;
1591 match->prot_hook.af_packet_priv = match;
c0de08d0 1592 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1593 dev_add_pack(&match->prot_hook);
1594 list_add(&match->list, &fanout_list);
dc99f600 1595 }
afe62c68
ED		 1596 err = -EINVAL;
1597 if (match->type == type &&
1598 match->prot_hook.type == po->prot_hook.type &&
1599 match->prot_hook.dev == po->prot_hook.dev) {
1600 err = -ENOSPC;
1601 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1602 __dev_remove_pack(&po->prot_hook);
1603 po->fanout = match;
1604 atomic_inc(&match->sk_ref);
1605 __fanout_link(sk, po);
1606 err = 0;
dc99f600
DM		 1607 }
1608 }
afe62c68 1609out:
dc99f600 1610 mutex_unlock(&fanout_mutex);
0648ab70
WB		 1611 if (err) {
1612 kfree(po->rollover);
1613 po->rollover = NULL;
1614 }
dc99f600
DM		 1615 return err;
1616}
1617
1618static void fanout_release(struct sock *sk)
1619{
1620 struct packet_sock *po = pkt_sk(sk);
1621 struct packet_fanout *f;
1622
1623 f = po->fanout;
1624 if (!f)
1625 return;
1626
fff3321d 1627 mutex_lock(&fanout_mutex);
dc99f600
DM		 1628 po->fanout = NULL;
1629
dc99f600
DM		 1630 if (atomic_dec_and_test(&f->sk_ref)) {
1631 list_del(&f->list);
1632 dev_remove_pack(&f->prot_hook);
1633 kfree(f);
1634 }
1635 mutex_unlock(&fanout_mutex);
0648ab70
WB		 1636
1637 kfree(po->rollover);
dc99f600 1638}
1da177e4 1639
90ddc4f0 1640static const struct proto_ops packet_ops;
1da177e4 1641
90ddc4f0 1642static const struct proto_ops packet_ops_spkt;
1da177e4 1643
40d4e3df
ED		 1644static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1645 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1646{
1647 struct sock *sk;
1648 struct sockaddr_pkt *spkt;
1649
1650 /*
1651 * When we registered the protocol we saved the socket in the data
1652 * field for just this event.
1653 */
1654
1655 sk = pt->af_packet_priv;
1ce4f28b 1656
1da177e4
LT		 1657 /*
1658 * Yank back the headers [hope the device set this
1659 * right or kerboom...]
1660 *
1661 * Incoming packets have ll header pulled,
1662 * push it back.
1663 *
98e399f8 1664 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1665 * so that this procedure is noop.
1666 */
1667
1668 if (skb->pkt_type == PACKET_LOOPBACK)
1669 goto out;
1670
09ad9bc7 1671 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1672 goto out;
1673
40d4e3df
ED		 1674 skb = skb_share_check(skb, GFP_ATOMIC);
1675 if (skb == NULL)
1da177e4
LT		 1676 goto oom;
1677
1678 /* drop any routing info */
adf30907 1679 skb_dst_drop(skb);
1da177e4 1680
84531c24
PO		 1681 /* drop conntrack reference */
1682 nf_reset(skb);
1683
ffbc6111 1684 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1685
98e399f8 1686 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1687
1688 /*
1689 * The SOCK_PACKET socket receives _all_ frames.
1690 */
1691
1692 spkt->spkt_family = dev->type;
1693 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1694 spkt->spkt_protocol = skb->protocol;
1695
1696 /*
1697 * Charge the memory to the socket. This is done specifically
1698 * to prevent sockets using all the memory up.
1699 */
1700
40d4e3df 1701 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1702 return 0;
1703
1704out:
1705 kfree_skb(skb);
1706oom:
1707 return 0;
1708}
1709
1710
1711/*
1712 * Output a raw packet to a device layer. This bypasses all the other
1713 * protocol layers and you must therefore supply it with a complete frame
1714 */
1ce4f28b 1715
1b784140
YX		 1716static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1717 size_t len)
1da177e4
LT		 1718{
1719 struct sock *sk = sock->sk;
342dfc30 1720 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1721 struct sk_buff *skb = NULL;
1da177e4 1722 struct net_device *dev;
40d4e3df 1723 __be16 proto = 0;
1da177e4 1724 int err;
3bdc0eba 1725 int extra_len = 0;
1ce4f28b 1726
1da177e4 1727 /*
1ce4f28b 1728 * Get and verify the address.
1da177e4
LT		 1729 */
1730
40d4e3df 1731 if (saddr) {
1da177e4 1732 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1733 return -EINVAL;
1734 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1735 proto = saddr->spkt_protocol;
1736 } else
1737 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1738
1739 /*
1ce4f28b 1740 * Find the device first to size check it
1da177e4
LT		 1741 */
1742
de74e92a 1743 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1744retry:
654d1f8a
ED		 1745 rcu_read_lock();
1746 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1747 err = -ENODEV;
1748 if (dev == NULL)
1749 goto out_unlock;
1ce4f28b 1750
d5e76b0a
DM		 1751 err = -ENETDOWN;
1752 if (!(dev->flags & IFF_UP))
1753 goto out_unlock;
1754
1da177e4 1755 /*
40d4e3df
ED		 1756 * You may not queue a frame bigger than the mtu. This is the lowest level
1757 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1758 */
1ce4f28b 1759
3bdc0eba
BG		 1760 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1761 if (!netif_supports_nofcs(dev)) {
1762 err = -EPROTONOSUPPORT;
1763 goto out_unlock;
1764 }
1765 extra_len = 4; /* We're doing our own CRC */
1766 }
1767
1da177e4 1768 err = -EMSGSIZE;
3bdc0eba 1769 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1770 goto out_unlock;
1771
1a35ca80
ED		 1772 if (!skb) {
1773 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1774 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1775 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1776
1777 rcu_read_unlock();
4ce40912 1778 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1779 if (skb == NULL)
1780 return -ENOBUFS;
1781 /* FIXME: Save some space for broken drivers that write a hard
1782 * header at transmission time by themselves. PPP is the notable
1783 * one here. This should really be fixed at the driver level.
1784 */
1785 skb_reserve(skb, reserved);
1786 skb_reset_network_header(skb);
1787
1788 /* Try to align data part correctly */
1789 if (hhlen) {
1790 skb->data -= hhlen;
1791 skb->tail -= hhlen;
1792 if (len < hhlen)
1793 skb_reset_network_header(skb);
1794 }
6ce8e9ce 1795 err = memcpy_from_msg(skb_put(skb, len), msg, len);
1a35ca80
ED		 1796 if (err)
1797 goto out_free;
1798 goto retry;
1da177e4
LT		 1799 }
1800
3bdc0eba 1801 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1802 /* Earlier code assumed this would be a VLAN pkt,
1803 * double-check this now that we have the actual
1804 * packet in hand.
1805 */
1806 struct ethhdr *ehdr;
1807 skb_reset_mac_header(skb);
1808 ehdr = eth_hdr(skb);
1809 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1810 err = -EMSGSIZE;
1811 goto out_unlock;
1812 }
1813 }
1a35ca80 1814
1da177e4
LT		 1815 skb->protocol = proto;
1816 skb->dev = dev;
1817 skb->priority = sk->sk_priority;
2d37a186 1818 skb->mark = sk->sk_mark;
bf84a010
DB		 1819
1820 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1821
3bdc0eba
BG		 1822 if (unlikely(extra_len == 4))
1823 skb->no_fcs = 1;
1824
40893fd0 1825 skb_probe_transport_header(skb, 0);
c1aad275 1826
1da177e4 1827 dev_queue_xmit(skb);
654d1f8a 1828 rcu_read_unlock();
40d4e3df 1829 return len;
1da177e4 1830
1da177e4 1831out_unlock:
654d1f8a 1832 rcu_read_unlock();
1a35ca80
ED		 1833out_free:
1834 kfree_skb(skb);
1da177e4
LT		 1835 return err;
1836}
1da177e4 1837
eea49cc9 1838static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1839 const struct sock *sk,
dbcb5855 1840 unsigned int res)
1da177e4
LT		 1841{
1842 struct sk_filter *filter;
fda9ef5d 1843
80f8f102
ED		 1844 rcu_read_lock();
1845 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1846 if (filter != NULL)
0a14842f 1847 res = SK_RUN_FILTER(filter, skb);
80f8f102 1848 rcu_read_unlock();
1da177e4 1849
dbcb5855 1850 return res;
1da177e4
LT		 1851}
1852
1853/*
62ab0812
ED		 1854 * This function makes lazy skb cloning in hope that most of packets
1855 * are discarded by BPF.
1856 *
1857 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1858 * and skb->cb are mangled. It works because (and until) packets
1859 * falling here are owned by current CPU. Output packets are cloned
1860 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1861 * sequencially, so that if we return skb to original state on exit,
1862 * we will not harm anyone.
1da177e4
LT		 1863 */
1864
40d4e3df
ED		 1865static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1866 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1867{
1868 struct sock *sk;
1869 struct sockaddr_ll *sll;
1870 struct packet_sock *po;
40d4e3df 1871 u8 *skb_head = skb->data;
1da177e4 1872 int skb_len = skb->len;
dbcb5855 1873 unsigned int snaplen, res;
1da177e4
LT		 1874
1875 if (skb->pkt_type == PACKET_LOOPBACK)
1876 goto drop;
1877
1878 sk = pt->af_packet_priv;
1879 po = pkt_sk(sk);
1880
09ad9bc7 1881 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1882 goto drop;
1883
1da177e4
LT		 1884 skb->dev = dev;
1885
3b04ddde 1886 if (dev->header_ops) {
1da177e4 1887 /* The device has an explicit notion of ll header,
62ab0812
ED		 1888 * exported to higher levels.
1889 *
1890 * Otherwise, the device hides details of its frame
1891 * structure, so that corresponding packet head is
1892 * never delivered to user.
1da177e4
LT		 1893 */
1894 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1895 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1896 else if (skb->pkt_type == PACKET_OUTGOING) {
1897 /* Special case: outgoing packets have ll header at head */
bbe735e4 1898 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1899 }
1900 }
1901
1902 snaplen = skb->len;
1903
dbcb5855
DM		 1904 res = run_filter(skb, sk, snaplen);
1905 if (!res)
fda9ef5d 1906 goto drop_n_restore;
dbcb5855
DM		 1907 if (snaplen > res)
1908 snaplen = res;
1da177e4 1909
0fd7bac6 1910 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1911 goto drop_n_acct;
1912
1913 if (skb_shared(skb)) {
1914 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1915 if (nskb == NULL)
1916 goto drop_n_acct;
1917
1918 if (skb_head != skb->data) {
1919 skb->data = skb_head;
1920 skb->len = skb_len;
1921 }
abc4e4fa 1922 consume_skb(skb);
1da177e4
LT		 1923 skb = nskb;
1924 }
1925
b4772ef8 1926 sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
ffbc6111
HX		 1927
1928 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4 1929 sll->sll_hatype = dev->type;
1da177e4 1930 sll->sll_pkttype = skb->pkt_type;
8032b464 1931 if (unlikely(po->origdev))
80feaacb
PWJ		 1932 sll->sll_ifindex = orig_dev->ifindex;
1933 else
1934 sll->sll_ifindex = dev->ifindex;
1da177e4 1935
b95cce35 1936 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1937
2472d761
EB		 1938 /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
1939 * Use their space for storing the original skb length.
1940 */
1941 PACKET_SKB_CB(skb)->sa.origlen = skb->len;
8dc41944 1942
1da177e4
LT		 1943 if (pskb_trim(skb, snaplen))
1944 goto drop_n_acct;
1945
1946 skb_set_owner_r(skb, sk);
1947 skb->dev = NULL;
adf30907 1948 skb_dst_drop(skb);
1da177e4 1949
84531c24
PO		 1950 /* drop conntrack reference */
1951 nf_reset(skb);
1952
1da177e4 1953 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1954 po->stats.stats1.tp_packets++;
3bc3b96f 1955 sock_skb_set_dropcount(sk, skb);
1da177e4
LT		 1956 __skb_queue_tail(&sk->sk_receive_queue, skb);
1957 spin_unlock(&sk->sk_receive_queue.lock);
676d2369 1958 sk->sk_data_ready(sk);
1da177e4
LT		 1959 return 0;
1960
1961drop_n_acct:
7091fbd8 1962 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1963 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1964 atomic_inc(&sk->sk_drops);
1965 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1966
1967drop_n_restore:
1968 if (skb_head != skb->data && skb_shared(skb)) {
1969 skb->data = skb_head;
1970 skb->len = skb_len;
1971 }
1972drop:
ead2ceb0 1973 consume_skb(skb);
1da177e4
LT		 1974 return 0;
1975}
1976
40d4e3df
ED		 1977static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1978 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1979{
1980 struct sock *sk;
1981 struct packet_sock *po;
1982 struct sockaddr_ll *sll;
184f489e 1983 union tpacket_uhdr h;
40d4e3df 1984 u8 *skb_head = skb->data;
1da177e4 1985 int skb_len = skb->len;
dbcb5855 1986 unsigned int snaplen, res;
f6fb8f10 1987 unsigned long status = TP_STATUS_USER;
bbd6ef87 1988 unsigned short macoff, netoff, hdrlen;
1da177e4 1989 struct sk_buff *copy_skb = NULL;
bbd6ef87 1990 struct timespec ts;
b9c32fb2 1991 __u32 ts_status;
1da177e4 1992
51846355
AW		 1993 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1994 * We may add members to them until current aligned size without forcing
1995 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1996 */
1997 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1998 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1999
1da177e4
LT		 2000 if (skb->pkt_type == PACKET_LOOPBACK)
2001 goto drop;
2002
2003 sk = pt->af_packet_priv;
2004 po = pkt_sk(sk);
2005
09ad9bc7 2006 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 2007 goto drop;
2008
3b04ddde 2009 if (dev->header_ops) {
1da177e4 2010 if (sk->sk_type != SOCK_DGRAM)
98e399f8 2011 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 2012 else if (skb->pkt_type == PACKET_OUTGOING) {
2013 /* Special case: outgoing packets have ll header at head */
bbe735e4 2014 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 2015 }
2016 }
2017
2018 snaplen = skb->len;
2019
dbcb5855
DM		 2020 res = run_filter(skb, sk, snaplen);
2021 if (!res)
fda9ef5d 2022 goto drop_n_restore;
68c2e5de
AD		 2023
2024 if (skb->ip_summed == CHECKSUM_PARTIAL)
2025 status |= TP_STATUS_CSUMNOTREADY;
682f048b
AD		 2026 else if (skb->pkt_type != PACKET_OUTGOING &&
2027 (skb->ip_summed == CHECKSUM_COMPLETE ||
2028 skb_csum_unnecessary(skb)))
2029 status |= TP_STATUS_CSUM_VALID;
68c2e5de 2030
dbcb5855
DM		 2031 if (snaplen > res)
2032 snaplen = res;
1da177e4
LT		 2033
2034 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 2035 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2036 po->tp_reserve;
1da177e4 2037 } else {
95c96174 2038 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 2039 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 2040 (maclen < 16 ? 16 : maclen)) +
2041 po->tp_reserve;
1da177e4
LT		 2042 macoff = netoff - maclen;
2043 }
f6fb8f10 2044 if (po->tp_version <= TPACKET_V2) {
2045 if (macoff + snaplen > po->rx_ring.frame_size) {
2046 if (po->copy_thresh &&
0fd7bac6 2047 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 2048 if (skb_shared(skb)) {
2049 copy_skb = skb_clone(skb, GFP_ATOMIC);
2050 } else {
2051 copy_skb = skb_get(skb);
2052 skb_head = skb->data;
2053 }
2054 if (copy_skb)
2055 skb_set_owner_r(copy_skb, sk);
1da177e4 2056 }
f6fb8f10 2057 snaplen = po->rx_ring.frame_size - macoff;
2058 if ((int)snaplen < 0)
2059 snaplen = 0;
1da177e4 2060 }
dc808110
ED		 2061 } else if (unlikely(macoff + snaplen >
2062 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2063 u32 nval;
2064
2065 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2066 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2067 snaplen, nval, macoff);
2068 snaplen = nval;
2069 if (unlikely((int)snaplen < 0)) {
2070 snaplen = 0;
2071 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
2072 }
1da177e4 2073 }
1da177e4 2074 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 2075 h.raw = packet_current_rx_frame(po, skb,
2076 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 2077 if (!h.raw)
1da177e4 2078 goto ring_is_full;
f6fb8f10 2079 if (po->tp_version <= TPACKET_V2) {
2080 packet_increment_rx_head(po, &po->rx_ring);
2081 /*
2082 * LOSING will be reported till you read the stats,
2083 * because it's COR - Clear On Read.
2084 * Anyways, moving it for V1/V2 only as V3 doesn't need this
2085 * at packet level.
2086 */
ee80fbf3 2087 if (po->stats.stats1.tp_drops)
f6fb8f10 2088 status |= TP_STATUS_LOSING;
2089 }
ee80fbf3 2090 po->stats.stats1.tp_packets++;
1da177e4
LT		 2091 if (copy_skb) {
2092 status |= TP_STATUS_COPY;
2093 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2094 }
1da177e4
LT		 2095 spin_unlock(&sk->sk_receive_queue.lock);
2096
bbd6ef87 2097 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 2098
2099 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 2100 getnstimeofday(&ts);
1da177e4 2101
b9c32fb2
DB		 2102 status |= ts_status;
2103
bbd6ef87
PM		 2104 switch (po->tp_version) {
2105 case TPACKET_V1:
2106 h.h1->tp_len = skb->len;
2107 h.h1->tp_snaplen = snaplen;
2108 h.h1->tp_mac = macoff;
2109 h.h1->tp_net = netoff;
4b457bdf
DB		 2110 h.h1->tp_sec = ts.tv_sec;
2111 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 2112 hdrlen = sizeof(*h.h1);
2113 break;
2114 case TPACKET_V2:
2115 h.h2->tp_len = skb->len;
2116 h.h2->tp_snaplen = snaplen;
2117 h.h2->tp_mac = macoff;
2118 h.h2->tp_net = netoff;
bbd6ef87
PM		 2119 h.h2->tp_sec = ts.tv_sec;
2120 h.h2->tp_nsec = ts.tv_nsec;
df8a39de
JP		 2121 if (skb_vlan_tag_present(skb)) {
2122 h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
a0cdfcf3
AW		 2123 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2124 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2125 } else {
2126 h.h2->tp_vlan_tci = 0;
a0cdfcf3 2127 h.h2->tp_vlan_tpid = 0;
a3bcc23e 2128 }
e4d26f4b 2129 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 2130 hdrlen = sizeof(*h.h2);
2131 break;
f6fb8f10 2132 case TPACKET_V3:
2133 /* tp_nxt_offset,vlan are already populated above.
2134 * So DONT clear those fields here
2135 */
2136 h.h3->tp_status |= status;
2137 h.h3->tp_len = skb->len;
2138 h.h3->tp_snaplen = snaplen;
2139 h.h3->tp_mac = macoff;
2140 h.h3->tp_net = netoff;
f6fb8f10 2141 h.h3->tp_sec = ts.tv_sec;
2142 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2143 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2144 hdrlen = sizeof(*h.h3);
2145 break;
bbd6ef87
PM		 2146 default:
2147 BUG();
2148 }
1da177e4 2149
bbd6ef87 2150 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2151 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2152 sll->sll_family = AF_PACKET;
2153 sll->sll_hatype = dev->type;
2154 sll->sll_protocol = skb->protocol;
2155 sll->sll_pkttype = skb->pkt_type;
8032b464 2156 if (unlikely(po->origdev))
80feaacb
PWJ		 2157 sll->sll_ifindex = orig_dev->ifindex;
2158 else
2159 sll->sll_ifindex = dev->ifindex;
1da177e4 2160
e16aa207 2161 smp_mb();
f0d4eb29 2162
f6dafa95 2163#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2164 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2165 u8 *start, *end;
2166
f0d4eb29
DB		 2167 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2168 macoff + snaplen);
2169
2170 for (start = h.raw; start < end; start += PAGE_SIZE)
2171 flush_dcache_page(pgv_to_page(start));
1da177e4 2172 }
f0d4eb29 2173 smp_wmb();
f6dafa95 2174#endif
f0d4eb29 2175
da413eec 2176 if (po->tp_version <= TPACKET_V2) {
f6fb8f10 2177 __packet_set_status(po, h.raw, status);
da413eec
DC		 2178 sk->sk_data_ready(sk);
2179 } else {
f6fb8f10 2180 prb_clear_blk_fill_status(&po->rx_ring);
da413eec 2181 }
1da177e4
LT		 2182
2183drop_n_restore:
2184 if (skb_head != skb->data && skb_shared(skb)) {
2185 skb->data = skb_head;
2186 skb->len = skb_len;
2187 }
2188drop:
1ce4f28b 2189 kfree_skb(skb);
1da177e4
LT		 2190 return 0;
2191
2192ring_is_full:
ee80fbf3 2193 po->stats.stats1.tp_drops++;
1da177e4
LT		 2194 spin_unlock(&sk->sk_receive_queue.lock);
2195
676d2369 2196 sk->sk_data_ready(sk);
acb5d75b 2197 kfree_skb(copy_skb);
1da177e4
LT		 2198 goto drop_n_restore;
2199}
2200
69e3c75f
JB		 2201static void tpacket_destruct_skb(struct sk_buff *skb)
2202{
2203 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2204
69e3c75f 2205 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2206 void *ph;
b9c32fb2
DB		 2207 __u32 ts;
2208
69e3c75f 2209 ph = skb_shinfo(skb)->destructor_arg;
b0138408 2210 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2211
2212 ts = __packet_set_timestamp(po, ph, skb);
2213 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2214 }
2215
2216 sock_wfree(skb);
2217}
2218
9c707762
WB		 2219static bool ll_header_truncated(const struct net_device *dev, int len)
2220{
2221 /* net device doesn't like empty head */
2222 if (unlikely(len <= dev->hard_header_len)) {
eee2f04b 2223 net_warn_ratelimited("%s: packet size is too short (%d <= %d)\n",
9c707762
WB		 2224 current->comm, len, dev->hard_header_len);
2225 return true;
2226 }
2227
2228 return false;
2229}
2230
40d4e3df
ED		 2231static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2232 void *frame, struct net_device *dev, int size_max,
ae641949 2233 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 2234{
184f489e 2235 union tpacket_uhdr ph;
09effa67 2236 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 2237 struct socket *sock = po->sk.sk_socket;
2238 struct page *page;
2239 void *data;
2240 int err;
2241
2242 ph.raw = frame;
2243
2244 skb->protocol = proto;
2245 skb->dev = dev;
2246 skb->priority = po->sk.sk_priority;
2d37a186 2247 skb->mark = po->sk.sk_mark;
2e31396f 2248 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2249 skb_shinfo(skb)->destructor_arg = ph.raw;
2250
2251 switch (po->tp_version) {
2252 case TPACKET_V2:
2253 tp_len = ph.h2->tp_len;
2254 break;
2255 default:
2256 tp_len = ph.h1->tp_len;
2257 break;
2258 }
09effa67
DM		 2259 if (unlikely(tp_len > size_max)) {
2260 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2261 return -EMSGSIZE;
2262 }
69e3c75f 2263
ae641949 2264 skb_reserve(skb, hlen);
69e3c75f 2265 skb_reset_network_header(skb);
c1aad275 2266
d346a3fa
DB		 2267 if (!packet_use_direct_xmit(po))
2268 skb_probe_transport_header(skb, 0);
2269 if (unlikely(po->tp_tx_has_off)) {
5920cd3a
PC		 2270 int off_min, off_max, off;
2271 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2272 off_max = po->tx_ring.frame_size - tp_len;
2273 if (sock->type == SOCK_DGRAM) {
2274 switch (po->tp_version) {
2275 case TPACKET_V2:
2276 off = ph.h2->tp_net;
2277 break;
2278 default:
2279 off = ph.h1->tp_net;
2280 break;
2281 }
2282 } else {
2283 switch (po->tp_version) {
2284 case TPACKET_V2:
2285 off = ph.h2->tp_mac;
2286 break;
2287 default:
2288 off = ph.h1->tp_mac;
2289 break;
2290 }
2291 }
2292 if (unlikely((off < off_min) || (off_max < off)))
2293 return -EINVAL;
2294 data = ph.raw + off;
2295 } else {
2296 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2297 }
69e3c75f
JB		 2298 to_write = tp_len;
2299
2300 if (sock->type == SOCK_DGRAM) {
2301 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2302 NULL, tp_len);
2303 if (unlikely(err < 0))
2304 return -EINVAL;
40d4e3df 2305 } else if (dev->hard_header_len) {
9c707762 2306 if (ll_header_truncated(dev, tp_len))
69e3c75f 2307 return -EINVAL;
69e3c75f
JB		 2308
2309 skb_push(skb, dev->hard_header_len);
2310 err = skb_store_bits(skb, 0, data,
2311 dev->hard_header_len);
2312 if (unlikely(err))
2313 return err;
2314
2315 data += dev->hard_header_len;
2316 to_write -= dev->hard_header_len;
2317 }
2318
69e3c75f
JB		 2319 offset = offset_in_page(data);
2320 len_max = PAGE_SIZE - offset;
2321 len = ((to_write > len_max) ? len_max : to_write);
2322
2323 skb->data_len = to_write;
2324 skb->len += to_write;
2325 skb->truesize += to_write;
2326 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2327
2328 while (likely(to_write)) {
2329 nr_frags = skb_shinfo(skb)->nr_frags;
2330
2331 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2332 pr_err("Packet exceed the number of skb frags(%lu)\n",
2333 MAX_SKB_FRAGS);
69e3c75f
JB		 2334 return -EFAULT;
2335 }
2336
0af55bb5
CG		 2337 page = pgv_to_page(data);
2338 data += len;
69e3c75f
JB		 2339 flush_dcache_page(page);
2340 get_page(page);
0af55bb5 2341 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2342 to_write -= len;
2343 offset = 0;
2344 len_max = PAGE_SIZE;
2345 len = ((to_write > len_max) ? len_max : to_write);
2346 }
2347
2348 return tp_len;
2349}
2350
2351static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2352{
69e3c75f
JB		 2353 struct sk_buff *skb;
2354 struct net_device *dev;
2355 __be16 proto;
09effa67 2356 int err, reserve = 0;
40d4e3df 2357 void *ph;
342dfc30 2358 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2359 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
69e3c75f
JB		 2360 int tp_len, size_max;
2361 unsigned char *addr;
2362 int len_sum = 0;
9e67030a 2363 int status = TP_STATUS_AVAILABLE;
ae641949 2364 int hlen, tlen;
69e3c75f 2365
69e3c75f
JB		 2366 mutex_lock(&po->pg_vec_lock);
2367
66e56cd4 2368 if (likely(saddr == NULL)) {
e40526cb 2369 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2370 proto = po->num;
2371 addr = NULL;
2372 } else {
2373 err = -EINVAL;
2374 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2375 goto out;
2376 if (msg->msg_namelen < (saddr->sll_halen
2377 + offsetof(struct sockaddr_ll,
2378 sll_addr)))
2379 goto out;
69e3c75f
JB		 2380 proto = saddr->sll_protocol;
2381 addr = saddr->sll_addr;
827d9780 2382 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2383 }
2384
69e3c75f
JB		 2385 err = -ENXIO;
2386 if (unlikely(dev == NULL))
2387 goto out;
69e3c75f
JB		 2388 err = -ENETDOWN;
2389 if (unlikely(!(dev->flags & IFF_UP)))
2390 goto out_put;
2391
52f1454f 2392 reserve = dev->hard_header_len + VLAN_HLEN;
69e3c75f 2393 size_max = po->tx_ring.frame_size
b5dd884e 2394 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2395
09effa67
DM		 2396 if (size_max > dev->mtu + reserve)
2397 size_max = dev->mtu + reserve;
2398
69e3c75f
JB		 2399 do {
2400 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2401 TP_STATUS_SEND_REQUEST);
69e3c75f 2402 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2403 if (need_wait && need_resched())
2404 schedule();
69e3c75f
JB		 2405 continue;
2406 }
2407
2408 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2409 hlen = LL_RESERVED_SPACE(dev);
2410 tlen = dev->needed_tailroom;
69e3c75f 2411 skb = sock_alloc_send_skb(&po->sk,
ae641949 2412 hlen + tlen + sizeof(struct sockaddr_ll),
fbf33a28 2413 !need_wait, &err);
69e3c75f 2414
fbf33a28
KM		 2415 if (unlikely(skb == NULL)) {
2416 /* we assume the socket was initially writeable ... */
2417 if (likely(len_sum > 0))
2418 err = len_sum;
69e3c75f 2419 goto out_status;
fbf33a28 2420 }
69e3c75f 2421 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
52f1454f
DB		 2422 addr, hlen);
2423 if (tp_len > dev->mtu + dev->hard_header_len) {
2424 struct ethhdr *ehdr;
2425 /* Earlier code assumed this would be a VLAN pkt,
2426 * double-check this now that we have the actual
2427 * packet in hand.
2428 */
69e3c75f 2429
52f1454f
DB		 2430 skb_reset_mac_header(skb);
2431 ehdr = eth_hdr(skb);
2432 if (ehdr->h_proto != htons(ETH_P_8021Q))
2433 tp_len = -EMSGSIZE;
2434 }
69e3c75f
JB		 2435 if (unlikely(tp_len < 0)) {
2436 if (po->tp_loss) {
2437 __packet_set_status(po, ph,
2438 TP_STATUS_AVAILABLE);
2439 packet_increment_head(&po->tx_ring);
2440 kfree_skb(skb);
2441 continue;
2442 } else {
2443 status = TP_STATUS_WRONG_FORMAT;
2444 err = tp_len;
2445 goto out_status;
2446 }
2447 }
2448
0fd5d57b
DB		 2449 packet_pick_tx_queue(dev, skb);
2450
69e3c75f
JB		 2451 skb->destructor = tpacket_destruct_skb;
2452 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2453 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2454
2455 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2456 err = po->xmit(skb);
eb70df13
JP		 2457 if (unlikely(err > 0)) {
2458 err = net_xmit_errno(err);
2459 if (err && __packet_get_status(po, ph) ==
2460 TP_STATUS_AVAILABLE) {
2461 /* skb was destructed already */
2462 skb = NULL;
2463 goto out_status;
2464 }
2465 /*
2466 * skb was dropped but not destructed yet;
2467 * let's treat it like congestion or err < 0
2468 */
2469 err = 0;
2470 }
69e3c75f
JB		 2471 packet_increment_head(&po->tx_ring);
2472 len_sum += tp_len;
b0138408
DB		 2473 } while (likely((ph != NULL) ||
2474 /* Note: packet_read_pending() might be slow if we have
2475 * to call it as it's per_cpu variable, but in fast-path
2476 * we already short-circuit the loop with the first
2477 * condition, and luckily don't have to go that path
2478 * anyway.
2479 */
2480 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2481
2482 err = len_sum;
2483 goto out_put;
2484
69e3c75f
JB		 2485out_status:
2486 __packet_set_status(po, ph, status);
2487 kfree_skb(skb);
2488out_put:
e40526cb 2489 dev_put(dev);
69e3c75f
JB		 2490out:
2491 mutex_unlock(&po->pg_vec_lock);
2492 return err;
2493}
69e3c75f 2494
eea49cc9
OJ		 2495static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2496 size_t reserve, size_t len,
2497 size_t linear, int noblock,
2498 int *err)
bfd5f4a3
SS		 2499{
2500 struct sk_buff *skb;
2501
2502 /* Under a page? Don't bother with paged skb. */
2503 if (prepad + len < PAGE_SIZE || !linear)
2504 linear = len;
2505
2506 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2507 err, 0);
bfd5f4a3
SS		 2508 if (!skb)
2509 return NULL;
2510
2511 skb_reserve(skb, reserve);
2512 skb_put(skb, linear);
2513 skb->data_len = len - linear;
2514 skb->len += len - linear;
2515
2516 return skb;
2517}
2518
d346a3fa 2519static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2520{
2521 struct sock *sk = sock->sk;
342dfc30 2522 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2523 struct sk_buff *skb;
2524 struct net_device *dev;
0e11c91e 2525 __be16 proto;
1da177e4 2526 unsigned char *addr;
827d9780 2527 int err, reserve = 0;
bfd5f4a3
SS		 2528 struct virtio_net_hdr vnet_hdr = { 0 };
2529 int offset = 0;
2530 int vnet_hdr_len;
2531 struct packet_sock *po = pkt_sk(sk);
2532 unsigned short gso_type = 0;
ae641949 2533 int hlen, tlen;
3bdc0eba 2534 int extra_len = 0;
8feb2fb2 2535 ssize_t n;
1da177e4
LT		 2536
2537 /*
1ce4f28b 2538 * Get and verify the address.
1da177e4 2539 */
1ce4f28b 2540
66e56cd4 2541 if (likely(saddr == NULL)) {
e40526cb 2542 dev = packet_cached_dev_get(po);
1da177e4
LT		 2543 proto = po->num;
2544 addr = NULL;
2545 } else {
2546 err = -EINVAL;
2547 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2548 goto out;
0fb375fb
EB		 2549 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2550 goto out;
1da177e4
LT		 2551 proto = saddr->sll_protocol;
2552 addr = saddr->sll_addr;
827d9780 2553 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2554 }
2555
1da177e4 2556 err = -ENXIO;
e40526cb 2557 if (unlikely(dev == NULL))
1da177e4 2558 goto out_unlock;
d5e76b0a 2559 err = -ENETDOWN;
e40526cb 2560 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2561 goto out_unlock;
2562
e40526cb
DB		 2563 if (sock->type == SOCK_RAW)
2564 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2565 if (po->has_vnet_hdr) {
2566 vnet_hdr_len = sizeof(vnet_hdr);
2567
2568 err = -EINVAL;
2569 if (len < vnet_hdr_len)
2570 goto out_unlock;
2571
2572 len -= vnet_hdr_len;
2573
8feb2fb2 2574 err = -EFAULT;
c0371da6 2575 n = copy_from_iter(&vnet_hdr, vnet_hdr_len, &msg->msg_iter);
8feb2fb2 2576 if (n != vnet_hdr_len)
bfd5f4a3
SS		 2577 goto out_unlock;
2578
2579 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
dc9e5153
MT		 2580 (__virtio16_to_cpu(false, vnet_hdr.csum_start) +
2581 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2 >
2582 __virtio16_to_cpu(false, vnet_hdr.hdr_len)))
2583 vnet_hdr.hdr_len = __cpu_to_virtio16(false,
2584 __virtio16_to_cpu(false, vnet_hdr.csum_start) +
2585 __virtio16_to_cpu(false, vnet_hdr.csum_offset) + 2);
bfd5f4a3
SS		 2586
2587 err = -EINVAL;
dc9e5153 2588 if (__virtio16_to_cpu(false, vnet_hdr.hdr_len) > len)
bfd5f4a3
SS		 2589 goto out_unlock;
2590
2591 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2592 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2593 case VIRTIO_NET_HDR_GSO_TCPV4:
2594 gso_type = SKB_GSO_TCPV4;
2595 break;
2596 case VIRTIO_NET_HDR_GSO_TCPV6:
2597 gso_type = SKB_GSO_TCPV6;
2598 break;
2599 case VIRTIO_NET_HDR_GSO_UDP:
2600 gso_type = SKB_GSO_UDP;
2601 break;
2602 default:
2603 goto out_unlock;
2604 }
2605
2606 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2607 gso_type |= SKB_GSO_TCP_ECN;
2608
2609 if (vnet_hdr.gso_size == 0)
2610 goto out_unlock;
2611
2612 }
2613 }
2614
3bdc0eba
BG		 2615 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2616 if (!netif_supports_nofcs(dev)) {
2617 err = -EPROTONOSUPPORT;
2618 goto out_unlock;
2619 }
2620 extra_len = 4; /* We're doing our own CRC */
2621 }
2622
1da177e4 2623 err = -EMSGSIZE;
3bdc0eba 2624 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2625 goto out_unlock;
2626
bfd5f4a3 2627 err = -ENOBUFS;
ae641949
HX		 2628 hlen = LL_RESERVED_SPACE(dev);
2629 tlen = dev->needed_tailroom;
dc9e5153
MT		 2630 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len,
2631 __virtio16_to_cpu(false, vnet_hdr.hdr_len),
bfd5f4a3 2632 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2633 if (skb == NULL)
1da177e4
LT		 2634 goto out_unlock;
2635
bfd5f4a3 2636 skb_set_network_header(skb, reserve);
1da177e4 2637
0c4e8581 2638 err = -EINVAL;
9c707762
WB		 2639 if (sock->type == SOCK_DGRAM) {
2640 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
46d2cfb1 2641 if (unlikely(offset < 0))
9c707762
WB		 2642 goto out_free;
2643 } else {
2644 if (ll_header_truncated(dev, len))
2645 goto out_free;
2646 }
1da177e4
LT		 2647
2648 /* Returns -EFAULT on error */
c0371da6 2649 err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
1da177e4
LT		 2650 if (err)
2651 goto out_free;
bf84a010
DB		 2652
2653 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2654
3bdc0eba 2655 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2656 /* Earlier code assumed this would be a VLAN pkt,
2657 * double-check this now that we have the actual
2658 * packet in hand.
2659 */
2660 struct ethhdr *ehdr;
2661 skb_reset_mac_header(skb);
2662 ehdr = eth_hdr(skb);
2663 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2664 err = -EMSGSIZE;
2665 goto out_free;
2666 }
57f89bfa
BG		 2667 }
2668
09effa67
DM		 2669 skb->protocol = proto;
2670 skb->dev = dev;
1da177e4 2671 skb->priority = sk->sk_priority;
2d37a186 2672 skb->mark = sk->sk_mark;
0fd5d57b
DB		 2673
2674 packet_pick_tx_queue(dev, skb);
1da177e4 2675
bfd5f4a3
SS		 2676 if (po->has_vnet_hdr) {
2677 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
dc9e5153
MT		 2678 u16 s = __virtio16_to_cpu(false, vnet_hdr.csum_start);
2679 u16 o = __virtio16_to_cpu(false, vnet_hdr.csum_offset);
2680 if (!skb_partial_csum_set(skb, s, o)) {
bfd5f4a3
SS		 2681 err = -EINVAL;
2682 goto out_free;
2683 }
2684 }
2685
dc9e5153
MT		 2686 skb_shinfo(skb)->gso_size =
2687 __virtio16_to_cpu(false, vnet_hdr.gso_size);
bfd5f4a3
SS		 2688 skb_shinfo(skb)->gso_type = gso_type;
2689
2690 /* Header must be checked, and gso_segs computed. */
2691 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2692 skb_shinfo(skb)->gso_segs = 0;
2693
2694 len += vnet_hdr_len;
2695 }
2696
d346a3fa
DB		 2697 if (!packet_use_direct_xmit(po))
2698 skb_probe_transport_header(skb, reserve);
3bdc0eba
BG		 2699 if (unlikely(extra_len == 4))
2700 skb->no_fcs = 1;
2701
d346a3fa 2702 err = po->xmit(skb);
1da177e4
LT		 2703 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2704 goto out_unlock;
2705
e40526cb 2706 dev_put(dev);
1da177e4 2707
40d4e3df 2708 return len;
1da177e4
LT		 2709
2710out_free:
2711 kfree_skb(skb);
2712out_unlock:
e40526cb 2713 if (dev)
1da177e4
LT		 2714 dev_put(dev);
2715out:
2716 return err;
2717}
2718
1b784140 2719static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
69e3c75f 2720{
69e3c75f
JB		 2721 struct sock *sk = sock->sk;
2722 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2723
69e3c75f
JB		 2724 if (po->tx_ring.pg_vec)
2725 return tpacket_snd(po, msg);
2726 else
69e3c75f
JB		 2727 return packet_snd(sock, msg, len);
2728}
2729
1da177e4
LT		 2730/*
2731 * Close a PACKET socket. This is fairly simple. We immediately go
2732 * to 'closed' state and remove our protocol entry in the device list.
2733 */
2734
2735static int packet_release(struct socket *sock)
2736{
2737 struct sock *sk = sock->sk;
2738 struct packet_sock *po;
d12d01d6 2739 struct net *net;
f6fb8f10 2740 union tpacket_req_u req_u;
1da177e4
LT		 2741
2742 if (!sk)
2743 return 0;
2744
3b1e0a65 2745 net = sock_net(sk);
1da177e4
LT		 2746 po = pkt_sk(sk);
2747
0fa7fa98 2748 mutex_lock(&net->packet.sklist_lock);
808f5114 2749 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2750 mutex_unlock(&net->packet.sklist_lock);
2751
2752 preempt_disable();
920de804 2753 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2754 preempt_enable();
1da177e4 2755
808f5114 2756 spin_lock(&po->bind_lock);
ce06b03e 2757 unregister_prot_hook(sk, false);
66e56cd4
DB		 2758 packet_cached_dev_reset(po);
2759
160ff18a
BG		 2760 if (po->prot_hook.dev) {
2761 dev_put(po->prot_hook.dev);
2762 po->prot_hook.dev = NULL;
2763 }
808f5114 2764 spin_unlock(&po->bind_lock);
1da177e4 2765
1da177e4 2766 packet_flush_mclist(sk);
1da177e4 2767
9665d5d6
PS		 2768 if (po->rx_ring.pg_vec) {
2769 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2770 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2771 }
69e3c75f 2772
9665d5d6
PS		 2773 if (po->tx_ring.pg_vec) {
2774 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2775 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2776 }
1da177e4 2777
dc99f600
DM		 2778 fanout_release(sk);
2779
808f5114 2780 synchronize_net();
1da177e4
LT		 2781 /*
2782 * Now the socket is dead. No more input will appear.
2783 */
1da177e4
LT		 2784 sock_orphan(sk);
2785 sock->sk = NULL;
2786
2787 /* Purge queues */
2788
2789 skb_queue_purge(&sk->sk_receive_queue);
b0138408 2790 packet_free_pending(po);
17ab56a2 2791 sk_refcnt_debug_release(sk);
1da177e4
LT		 2792
2793 sock_put(sk);
2794 return 0;
2795}
2796
2797/*
2798 * Attach a packet hook.
2799 */
2800
902fefb8 2801static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
1da177e4
LT		 2802{
2803 struct packet_sock *po = pkt_sk(sk);
902fefb8
DB		 2804 const struct net_device *dev_curr;
2805 __be16 proto_curr;
2806 bool need_rehook;
dc99f600 2807
aef950b4
WY		 2808 if (po->fanout) {
2809 if (dev)
2810 dev_put(dev);
2811
dc99f600 2812 return -EINVAL;
aef950b4 2813 }
1da177e4
LT		 2814
2815 lock_sock(sk);
1da177e4 2816 spin_lock(&po->bind_lock);
66e56cd4 2817
902fefb8
DB		 2818 proto_curr = po->prot_hook.type;
2819 dev_curr = po->prot_hook.dev;
2820
2821 need_rehook = proto_curr != proto || dev_curr != dev;
2822
2823 if (need_rehook) {
2824 unregister_prot_hook(sk, true);
1da177e4 2825
902fefb8
DB		 2826 po->num = proto;
2827 po->prot_hook.type = proto;
1da177e4 2828
902fefb8
DB		 2829 if (po->prot_hook.dev)
2830 dev_put(po->prot_hook.dev);
2831
2832 po->prot_hook.dev = dev;
2833
2834 po->ifindex = dev ? dev->ifindex : 0;
2835 packet_cached_dev_assign(po, dev);
2836 }
66e56cd4 2837
902fefb8 2838 if (proto == 0 || !need_rehook)
1da177e4
LT		 2839 goto out_unlock;
2840
be85d4ad 2841 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2842 register_prot_hook(sk);
be85d4ad
UT		 2843 } else {
2844 sk->sk_err = ENETDOWN;
2845 if (!sock_flag(sk, SOCK_DEAD))
2846 sk->sk_error_report(sk);
1da177e4
LT		 2847 }
2848
2849out_unlock:
2850 spin_unlock(&po->bind_lock);
2851 release_sock(sk);
2852 return 0;
2853}
2854
2855/*
2856 * Bind a packet socket to a device
2857 */
2858
40d4e3df
ED		 2859static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2860 int addr_len)
1da177e4 2861{
40d4e3df 2862 struct sock *sk = sock->sk;
1da177e4
LT		 2863 char name[15];
2864 struct net_device *dev;
2865 int err = -ENODEV;
1ce4f28b 2866
1da177e4
LT		 2867 /*
2868 * Check legality
2869 */
1ce4f28b 2870
8ae55f04 2871 if (addr_len != sizeof(struct sockaddr))
1da177e4 2872 return -EINVAL;
40d4e3df 2873 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2874
3b1e0a65 2875 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2876 if (dev)
1da177e4 2877 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2878 return err;
2879}
1da177e4
LT		 2880
2881static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2882{
40d4e3df
ED		 2883 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2884 struct sock *sk = sock->sk;
1da177e4
LT		 2885 struct net_device *dev = NULL;
2886 int err;
2887
2888
2889 /*
2890 * Check legality
2891 */
1ce4f28b 2892
1da177e4
LT		 2893 if (addr_len < sizeof(struct sockaddr_ll))
2894 return -EINVAL;
2895 if (sll->sll_family != AF_PACKET)
2896 return -EINVAL;
2897
2898 if (sll->sll_ifindex) {
2899 err = -ENODEV;
3b1e0a65 2900 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2901 if (dev == NULL)
2902 goto out;
2903 }
2904 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2905
2906out:
2907 return err;
2908}
2909
2910static struct proto packet_proto = {
2911 .name = "PACKET",
2912 .owner = THIS_MODULE,
2913 .obj_size = sizeof(struct packet_sock),
2914};
2915
2916/*
1ce4f28b 2917 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2918 */
2919
3f378b68
EP		 2920static int packet_create(struct net *net, struct socket *sock, int protocol,
2921 int kern)
1da177e4
LT		 2922{
2923 struct sock *sk;
2924 struct packet_sock *po;
0e11c91e 2925 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2926 int err;
2927
df008c91 2928 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2929 return -EPERM;
be02097c
DM		 2930 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2931 sock->type != SOCK_PACKET)
1da177e4
LT		 2932 return -ESOCKTNOSUPPORT;
2933
2934 sock->state = SS_UNCONNECTED;
2935
2936 err = -ENOBUFS;
11aa9c28 2937 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
1da177e4
LT		 2938 if (sk == NULL)
2939 goto out;
2940
2941 sock->ops = &packet_ops;
1da177e4
LT		 2942 if (sock->type == SOCK_PACKET)
2943 sock->ops = &packet_ops_spkt;
be02097c 2944
1da177e4
LT		 2945 sock_init_data(sock, sk);
2946
2947 po = pkt_sk(sk);
2948 sk->sk_family = PF_PACKET;
0e11c91e 2949 po->num = proto;
d346a3fa 2950 po->xmit = dev_queue_xmit;
66e56cd4 2951
b0138408
DB		 2952 err = packet_alloc_pending(po);
2953 if (err)
2954 goto out2;
2955
66e56cd4 2956 packet_cached_dev_reset(po);
1da177e4
LT		 2957
2958 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2959 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2960
2961 /*
2962 * Attach a protocol block
2963 */
2964
2965 spin_lock_init(&po->bind_lock);
905db440 2966 mutex_init(&po->pg_vec_lock);
0648ab70 2967 po->rollover = NULL;
1da177e4 2968 po->prot_hook.func = packet_rcv;
be02097c 2969
1da177e4
LT		 2970 if (sock->type == SOCK_PACKET)
2971 po->prot_hook.func = packet_rcv_spkt;
be02097c 2972
1da177e4
LT		 2973 po->prot_hook.af_packet_priv = sk;
2974
0e11c91e
AV		 2975 if (proto) {
2976 po->prot_hook.type = proto;
ce06b03e 2977 register_prot_hook(sk);
1da177e4
LT		 2978 }
2979
0fa7fa98 2980 mutex_lock(&net->packet.sklist_lock);
808f5114 2981 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2982 mutex_unlock(&net->packet.sklist_lock);
2983
2984 preempt_disable();
3680453c 2985 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2986 preempt_enable();
808f5114 2987
40d4e3df 2988 return 0;
b0138408
DB		 2989out2:
2990 sk_free(sk);
1da177e4
LT		 2991out:
2992 return err;
2993}
2994
2995/*
2996 * Pull a packet from our receive queue and hand it to the user.
2997 * If necessary we block.
2998 */
2999
1b784140
YX		 3000static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
3001 int flags)
1da177e4
LT		 3002{
3003 struct sock *sk = sock->sk;
3004 struct sk_buff *skb;
3005 int copied, err;
bfd5f4a3 3006 int vnet_hdr_len = 0;
2472d761 3007 unsigned int origlen = 0;
1da177e4
LT		 3008
3009 err = -EINVAL;
ed85b565 3010 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 3011 goto out;
3012
3013#if 0
3014 /* What error should we return now? EUNATTACH? */
3015 if (pkt_sk(sk)->ifindex < 0)
3016 return -ENODEV;
3017#endif
3018
ed85b565 3019 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 3020 err = sock_recv_errqueue(sk, msg, len,
3021 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 3022 goto out;
3023 }
3024
1da177e4
LT		 3025 /*
3026 * Call the generic datagram receiver. This handles all sorts
3027 * of horrible races and re-entrancy so we can forget about it
3028 * in the protocol layers.
3029 *
3030 * Now it will return ENETDOWN, if device have just gone down,
3031 * but then it will block.
3032 */
3033
40d4e3df 3034 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 3035
3036 /*
1ce4f28b 3037 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 3038 * handles the blocking we don't see and worry about blocking
3039 * retries.
3040 */
3041
8ae55f04 3042 if (skb == NULL)
1da177e4
LT		 3043 goto out;
3044
2ccdbaa6
WB		 3045 if (pkt_sk(sk)->pressure)
3046 packet_rcv_has_room(pkt_sk(sk), NULL);
3047
bfd5f4a3
SS		 3048 if (pkt_sk(sk)->has_vnet_hdr) {
3049 struct virtio_net_hdr vnet_hdr = { 0 };
3050
3051 err = -EINVAL;
3052 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 3053 if (len < vnet_hdr_len)
bfd5f4a3
SS		 3054 goto out_free;
3055
1f18b717
MK		 3056 len -= vnet_hdr_len;
3057
bfd5f4a3
SS		 3058 if (skb_is_gso(skb)) {
3059 struct skb_shared_info *sinfo = skb_shinfo(skb);
3060
3061 /* This is a hint as to how much should be linear. */
dc9e5153
MT		 3062 vnet_hdr.hdr_len =
3063 __cpu_to_virtio16(false, skb_headlen(skb));
3064 vnet_hdr.gso_size =
3065 __cpu_to_virtio16(false, sinfo->gso_size);
bfd5f4a3
SS		 3066 if (sinfo->gso_type & SKB_GSO_TCPV4)
3067 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
3068 else if (sinfo->gso_type & SKB_GSO_TCPV6)
3069 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
3070 else if (sinfo->gso_type & SKB_GSO_UDP)
3071 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
3072 else if (sinfo->gso_type & SKB_GSO_FCOE)
3073 goto out_free;
3074 else
3075 BUG();
3076 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
3077 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
3078 } else
3079 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
3080
3081 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3082 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
dc9e5153
MT		 3083 vnet_hdr.csum_start = __cpu_to_virtio16(false,
3084 skb_checksum_start_offset(skb));
3085 vnet_hdr.csum_offset = __cpu_to_virtio16(false,
3086 skb->csum_offset);
10a8d94a
JW		 3087 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
3088 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 3089 } /* else everything is zero */
3090
7eab8d9e 3091 err = memcpy_to_msg(msg, (void *)&vnet_hdr, vnet_hdr_len);
bfd5f4a3
SS		 3092 if (err < 0)
3093 goto out_free;
3094 }
3095
f3d33426
HFS		 3096 /* You lose any data beyond the buffer you gave. If it worries
3097 * a user program they can ask the device for its MTU
3098 * anyway.
1da177e4 3099 */
1da177e4 3100 copied = skb->len;
40d4e3df
ED		 3101 if (copied > len) {
3102 copied = len;
3103 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 3104 }
3105
51f3d02b 3106 err = skb_copy_datagram_msg(skb, 0, msg, copied);
1da177e4
LT		 3107 if (err)
3108 goto out_free;
3109
2472d761
EB		 3110 if (sock->type != SOCK_PACKET) {
3111 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3112
3113 /* Original length was stored in sockaddr_ll fields */
3114 origlen = PACKET_SKB_CB(skb)->sa.origlen;
3115 sll->sll_family = AF_PACKET;
3116 sll->sll_protocol = skb->protocol;
3117 }
3118
3b885787 3119 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 3120
f3d33426
HFS		 3121 if (msg->msg_name) {
3122 /* If the address length field is there to be filled
3123 * in, we fill it in now.
3124 */
3125 if (sock->type == SOCK_PACKET) {
342dfc30 3126 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426
HFS		 3127 msg->msg_namelen = sizeof(struct sockaddr_pkt);
3128 } else {
3129 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2472d761 3130
f3d33426
HFS		 3131 msg->msg_namelen = sll->sll_halen +
3132 offsetof(struct sockaddr_ll, sll_addr);
3133 }
ffbc6111
HX		 3134 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
3135 msg->msg_namelen);
f3d33426 3136 }
1da177e4 3137
8dc41944 3138 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 3139 struct tpacket_auxdata aux;
3140
3141 aux.tp_status = TP_STATUS_USER;
3142 if (skb->ip_summed == CHECKSUM_PARTIAL)
3143 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
682f048b
AD		 3144 else if (skb->pkt_type != PACKET_OUTGOING &&
3145 (skb->ip_summed == CHECKSUM_COMPLETE ||
3146 skb_csum_unnecessary(skb)))
3147 aux.tp_status |= TP_STATUS_CSUM_VALID;
3148
2472d761 3149 aux.tp_len = origlen;
ffbc6111
HX		 3150 aux.tp_snaplen = skb->len;
3151 aux.tp_mac = 0;
bbe735e4 3152 aux.tp_net = skb_network_offset(skb);
df8a39de
JP		 3153 if (skb_vlan_tag_present(skb)) {
3154 aux.tp_vlan_tci = skb_vlan_tag_get(skb);
a0cdfcf3
AW		 3155 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3156 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 3157 } else {
3158 aux.tp_vlan_tci = 0;
a0cdfcf3 3159 aux.tp_vlan_tpid = 0;
a3bcc23e 3160 }
ffbc6111 3161 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 3162 }
3163
1da177e4
LT		 3164 /*
3165 * Free or return the buffer as appropriate. Again this
3166 * hides all the races and re-entrancy issues from us.
3167 */
bfd5f4a3 3168 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 3169
3170out_free:
3171 skb_free_datagram(sk, skb);
3172out:
3173 return err;
3174}
3175
1da177e4
LT		 3176static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3177 int *uaddr_len, int peer)
3178{
3179 struct net_device *dev;
3180 struct sock *sk = sock->sk;
3181
3182 if (peer)
3183 return -EOPNOTSUPP;
3184
3185 uaddr->sa_family = AF_PACKET;
2dc85bf3 3186 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 3187 rcu_read_lock();
3188 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3189 if (dev)
2dc85bf3 3190 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 3191 rcu_read_unlock();
1da177e4
LT		 3192 *uaddr_len = sizeof(*uaddr);
3193
3194 return 0;
3195}
1da177e4
LT		 3196
3197static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3198 int *uaddr_len, int peer)
3199{
3200 struct net_device *dev;
3201 struct sock *sk = sock->sk;
3202 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3203 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3204
3205 if (peer)
3206 return -EOPNOTSUPP;
3207
3208 sll->sll_family = AF_PACKET;
3209 sll->sll_ifindex = po->ifindex;
3210 sll->sll_protocol = po->num;
67286640 3211 sll->sll_pkttype = 0;
654d1f8a
ED		 3212 rcu_read_lock();
3213 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3214 if (dev) {
3215 sll->sll_hatype = dev->type;
3216 sll->sll_halen = dev->addr_len;
3217 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3218 } else {
3219 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3220 sll->sll_halen = 0;
3221 }
654d1f8a 3222 rcu_read_unlock();
0fb375fb 3223 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3224
3225 return 0;
3226}
3227
2aeb0b88
WC		 3228static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3229 int what)
1da177e4
LT		 3230{
3231 switch (i->type) {
3232 case PACKET_MR_MULTICAST:
1162563f
JP		 3233 if (i->alen != dev->addr_len)
3234 return -EINVAL;
1da177e4 3235 if (what > 0)
22bedad3 3236 return dev_mc_add(dev, i->addr);
1da177e4 3237 else
22bedad3 3238 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3239 break;
3240 case PACKET_MR_PROMISC:
2aeb0b88 3241 return dev_set_promiscuity(dev, what);
1da177e4 3242 case PACKET_MR_ALLMULTI:
2aeb0b88 3243 return dev_set_allmulti(dev, what);
d95ed927 3244 case PACKET_MR_UNICAST:
1162563f
JP		 3245 if (i->alen != dev->addr_len)
3246 return -EINVAL;
d95ed927 3247 if (what > 0)
a748ee24 3248 return dev_uc_add(dev, i->addr);
d95ed927 3249 else
a748ee24 3250 return dev_uc_del(dev, i->addr);
d95ed927 3251 break;
40d4e3df
ED		 3252 default:
3253 break;
1da177e4 3254 }
2aeb0b88 3255 return 0;
1da177e4
LT		 3256}
3257
82f17091
FR		 3258static void packet_dev_mclist_delete(struct net_device *dev,
3259 struct packet_mclist **mlp)
1da177e4 3260{
82f17091
FR		 3261 struct packet_mclist *ml;
3262
3263 while ((ml = *mlp) != NULL) {
3264 if (ml->ifindex == dev->ifindex) {
3265 packet_dev_mc(dev, ml, -1);
3266 *mlp = ml->next;
3267 kfree(ml);
3268 } else
3269 mlp = &ml->next;
1da177e4
LT		 3270 }
3271}
3272
0fb375fb 3273static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3274{
3275 struct packet_sock *po = pkt_sk(sk);
3276 struct packet_mclist *ml, *i;
3277 struct net_device *dev;
3278 int err;
3279
3280 rtnl_lock();
3281
3282 err = -ENODEV;
3b1e0a65 3283 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3284 if (!dev)
3285 goto done;
3286
3287 err = -EINVAL;
1162563f 3288 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3289 goto done;
3290
3291 err = -ENOBUFS;
8b3a7005 3292 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3293 if (i == NULL)
3294 goto done;
3295
3296 err = 0;
3297 for (ml = po->mclist; ml; ml = ml->next) {
3298 if (ml->ifindex == mreq->mr_ifindex &&
3299 ml->type == mreq->mr_type &&
3300 ml->alen == mreq->mr_alen &&
3301 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3302 ml->count++;
3303 /* Free the new element ... */
3304 kfree(i);
3305 goto done;
3306 }
3307 }
3308
3309 i->type = mreq->mr_type;
3310 i->ifindex = mreq->mr_ifindex;
3311 i->alen = mreq->mr_alen;
3312 memcpy(i->addr, mreq->mr_address, i->alen);
3313 i->count = 1;
3314 i->next = po->mclist;
3315 po->mclist = i;
2aeb0b88
WC		 3316 err = packet_dev_mc(dev, i, 1);
3317 if (err) {
3318 po->mclist = i->next;
3319 kfree(i);
3320 }
1da177e4
LT		 3321
3322done:
3323 rtnl_unlock();
3324 return err;
3325}
3326
0fb375fb 3327static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3328{
3329 struct packet_mclist *ml, **mlp;
3330
3331 rtnl_lock();
3332
3333 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3334 if (ml->ifindex == mreq->mr_ifindex &&
3335 ml->type == mreq->mr_type &&
3336 ml->alen == mreq->mr_alen &&
3337 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3338 if (--ml->count == 0) {
3339 struct net_device *dev;
3340 *mlp = ml->next;
ad959e76
ED		 3341 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3342 if (dev)
1da177e4 3343 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3344 kfree(ml);
3345 }
82f17091 3346 break;
1da177e4
LT		 3347 }
3348 }
3349 rtnl_unlock();
82f17091 3350 return 0;
1da177e4
LT		 3351}
3352
3353static void packet_flush_mclist(struct sock *sk)
3354{
3355 struct packet_sock *po = pkt_sk(sk);
3356 struct packet_mclist *ml;
3357
3358 if (!po->mclist)
3359 return;
3360
3361 rtnl_lock();
3362 while ((ml = po->mclist) != NULL) {
3363 struct net_device *dev;
3364
3365 po->mclist = ml->next;
ad959e76
ED		 3366 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3367 if (dev != NULL)
1da177e4 3368 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3369 kfree(ml);
3370 }
3371 rtnl_unlock();
3372}
1da177e4
LT		 3373
3374static int
b7058842 3375packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3376{
3377 struct sock *sk = sock->sk;
8dc41944 3378 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3379 int ret;
3380
3381 if (level != SOL_PACKET)
3382 return -ENOPROTOOPT;
3383
69e3c75f 3384 switch (optname) {
1ce4f28b 3385 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3386 case PACKET_DROP_MEMBERSHIP:
3387 {
0fb375fb
EB		 3388 struct packet_mreq_max mreq;
3389 int len = optlen;
3390 memset(&mreq, 0, sizeof(mreq));
3391 if (len < sizeof(struct packet_mreq))
1da177e4 3392 return -EINVAL;
0fb375fb
EB		 3393 if (len > sizeof(mreq))
3394 len = sizeof(mreq);
40d4e3df 3395 if (copy_from_user(&mreq, optval, len))
1da177e4 3396 return -EFAULT;
0fb375fb
EB		 3397 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3398 return -EINVAL;
1da177e4
LT		 3399 if (optname == PACKET_ADD_MEMBERSHIP)
3400 ret = packet_mc_add(sk, &mreq);
3401 else
3402 ret = packet_mc_drop(sk, &mreq);
3403 return ret;
3404 }
a2efcfa0 3405
1da177e4 3406 case PACKET_RX_RING:
69e3c75f 3407 case PACKET_TX_RING:
1da177e4 3408 {
f6fb8f10 3409 union tpacket_req_u req_u;
3410 int len;
1da177e4 3411
f6fb8f10 3412 switch (po->tp_version) {
3413 case TPACKET_V1:
3414 case TPACKET_V2:
3415 len = sizeof(req_u.req);
3416 break;
3417 case TPACKET_V3:
3418 default:
3419 len = sizeof(req_u.req3);
3420 break;
3421 }
3422 if (optlen < len)
1da177e4 3423 return -EINVAL;
bfd5f4a3
SS		 3424 if (pkt_sk(sk)->has_vnet_hdr)
3425 return -EINVAL;
f6fb8f10 3426 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3427 return -EFAULT;
f6fb8f10 3428 return packet_set_ring(sk, &req_u, 0,
3429 optname == PACKET_TX_RING);
1da177e4
LT		 3430 }
3431 case PACKET_COPY_THRESH:
3432 {
3433 int val;
3434
40d4e3df 3435 if (optlen != sizeof(val))
1da177e4 3436 return -EINVAL;
40d4e3df 3437 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3438 return -EFAULT;
3439
3440 pkt_sk(sk)->copy_thresh = val;
3441 return 0;
3442 }
bbd6ef87
PM		 3443 case PACKET_VERSION:
3444 {
3445 int val;
3446
3447 if (optlen != sizeof(val))
3448 return -EINVAL;
69e3c75f 3449 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3450 return -EBUSY;
3451 if (copy_from_user(&val, optval, sizeof(val)))
3452 return -EFAULT;
3453 switch (val) {
3454 case TPACKET_V1:
3455 case TPACKET_V2:
f6fb8f10 3456 case TPACKET_V3:
bbd6ef87
PM		 3457 po->tp_version = val;
3458 return 0;
3459 default:
3460 return -EINVAL;
3461 }
3462 }
8913336a
PM		 3463 case PACKET_RESERVE:
3464 {
3465 unsigned int val;
3466
3467 if (optlen != sizeof(val))
3468 return -EINVAL;
69e3c75f 3469 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3470 return -EBUSY;
3471 if (copy_from_user(&val, optval, sizeof(val)))
3472 return -EFAULT;
3473 po->tp_reserve = val;
3474 return 0;
3475 }
69e3c75f
JB		 3476 case PACKET_LOSS:
3477 {
3478 unsigned int val;
3479
3480 if (optlen != sizeof(val))
3481 return -EINVAL;
3482 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3483 return -EBUSY;
3484 if (copy_from_user(&val, optval, sizeof(val)))
3485 return -EFAULT;
3486 po->tp_loss = !!val;
3487 return 0;
3488 }
8dc41944
HX		 3489 case PACKET_AUXDATA:
3490 {
3491 int val;
3492
3493 if (optlen < sizeof(val))
3494 return -EINVAL;
3495 if (copy_from_user(&val, optval, sizeof(val)))
3496 return -EFAULT;
3497
3498 po->auxdata = !!val;
3499 return 0;
3500 }
80feaacb
PWJ		 3501 case PACKET_ORIGDEV:
3502 {
3503 int val;
3504
3505 if (optlen < sizeof(val))
3506 return -EINVAL;
3507 if (copy_from_user(&val, optval, sizeof(val)))
3508 return -EFAULT;
3509
3510 po->origdev = !!val;
3511 return 0;
3512 }
bfd5f4a3
SS		 3513 case PACKET_VNET_HDR:
3514 {
3515 int val;
3516
3517 if (sock->type != SOCK_RAW)
3518 return -EINVAL;
3519 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3520 return -EBUSY;
3521 if (optlen < sizeof(val))
3522 return -EINVAL;
3523 if (copy_from_user(&val, optval, sizeof(val)))
3524 return -EFAULT;
3525
3526 po->has_vnet_hdr = !!val;
3527 return 0;
3528 }
614f60fa
SM		 3529 case PACKET_TIMESTAMP:
3530 {
3531 int val;
3532
3533 if (optlen != sizeof(val))
3534 return -EINVAL;
3535 if (copy_from_user(&val, optval, sizeof(val)))
3536 return -EFAULT;
3537
3538 po->tp_tstamp = val;
3539 return 0;
3540 }
dc99f600
DM		 3541 case PACKET_FANOUT:
3542 {
3543 int val;
3544
3545 if (optlen != sizeof(val))
3546 return -EINVAL;
3547 if (copy_from_user(&val, optval, sizeof(val)))
3548 return -EFAULT;
3549
3550 return fanout_add(sk, val & 0xffff, val >> 16);
3551 }
5920cd3a
PC		 3552 case PACKET_TX_HAS_OFF:
3553 {
3554 unsigned int val;
3555
3556 if (optlen != sizeof(val))
3557 return -EINVAL;
3558 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3559 return -EBUSY;
3560 if (copy_from_user(&val, optval, sizeof(val)))
3561 return -EFAULT;
3562 po->tp_tx_has_off = !!val;
3563 return 0;
3564 }
d346a3fa
DB		 3565 case PACKET_QDISC_BYPASS:
3566 {
3567 int val;
3568
3569 if (optlen != sizeof(val))
3570 return -EINVAL;
3571 if (copy_from_user(&val, optval, sizeof(val)))
3572 return -EFAULT;
3573
3574 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3575 return 0;
3576 }
1da177e4
LT		 3577 default:
3578 return -ENOPROTOOPT;
3579 }
3580}
3581
3582static int packet_getsockopt(struct socket *sock, int level, int optname,
3583 char __user *optval, int __user *optlen)
3584{
3585 int len;
c06fff6e 3586 int val, lv = sizeof(val);
1da177e4
LT		 3587 struct sock *sk = sock->sk;
3588 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3589 void *data = &val;
ee80fbf3 3590 union tpacket_stats_u st;
a9b63918 3591 struct tpacket_rollover_stats rstats;
1da177e4
LT		 3592
3593 if (level != SOL_PACKET)
3594 return -ENOPROTOOPT;
3595
8ae55f04
KK		 3596 if (get_user(len, optlen))
3597 return -EFAULT;
1da177e4
LT		 3598
3599 if (len < 0)
3600 return -EINVAL;
1ce4f28b 3601
69e3c75f 3602 switch (optname) {
1da177e4 3603 case PACKET_STATISTICS:
1da177e4 3604 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3605 memcpy(&st, &po->stats, sizeof(st));
3606 memset(&po->stats, 0, sizeof(po->stats));
3607 spin_unlock_bh(&sk->sk_receive_queue.lock);
3608
f6fb8f10 3609 if (po->tp_version == TPACKET_V3) {
c06fff6e 3610 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3611 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3612 data = &st.stats3;
f6fb8f10 3613 } else {
c06fff6e 3614 lv = sizeof(struct tpacket_stats);
8bcdeaff 3615 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3616 data = &st.stats1;
f6fb8f10 3617 }
ee80fbf3 3618
8dc41944
HX		 3619 break;
3620 case PACKET_AUXDATA:
8dc41944 3621 val = po->auxdata;
80feaacb
PWJ		 3622 break;
3623 case PACKET_ORIGDEV:
80feaacb 3624 val = po->origdev;
bfd5f4a3
SS		 3625 break;
3626 case PACKET_VNET_HDR:
bfd5f4a3 3627 val = po->has_vnet_hdr;
1da177e4 3628 break;
bbd6ef87 3629 case PACKET_VERSION:
bbd6ef87 3630 val = po->tp_version;
bbd6ef87
PM		 3631 break;
3632 case PACKET_HDRLEN:
3633 if (len > sizeof(int))
3634 len = sizeof(int);
3635 if (copy_from_user(&val, optval, len))
3636 return -EFAULT;
3637 switch (val) {
3638 case TPACKET_V1:
3639 val = sizeof(struct tpacket_hdr);
3640 break;
3641 case TPACKET_V2:
3642 val = sizeof(struct tpacket2_hdr);
3643 break;
f6fb8f10 3644 case TPACKET_V3:
3645 val = sizeof(struct tpacket3_hdr);
3646 break;
bbd6ef87
PM		 3647 default:
3648 return -EINVAL;
3649 }
bbd6ef87 3650 break;
8913336a 3651 case PACKET_RESERVE:
8913336a 3652 val = po->tp_reserve;
8913336a 3653 break;
69e3c75f 3654 case PACKET_LOSS:
69e3c75f 3655 val = po->tp_loss;
69e3c75f 3656 break;
614f60fa 3657 case PACKET_TIMESTAMP:
614f60fa 3658 val = po->tp_tstamp;
614f60fa 3659 break;
dc99f600 3660 case PACKET_FANOUT:
dc99f600
DM		 3661 val = (po->fanout ?
3662 ((u32)po->fanout->id |
77f65ebd
WB		 3663 ((u32)po->fanout->type << 16) |
3664 ((u32)po->fanout->flags << 24)) :
dc99f600 3665 0);
dc99f600 3666 break;
a9b63918
WB		 3667 case PACKET_ROLLOVER_STATS:
3668 if (!po->rollover)
3669 return -EINVAL;
3670 rstats.tp_all = atomic_long_read(&po->rollover->num);
3671 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
3672 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
3673 data = &rstats;
3674 lv = sizeof(rstats);
3675 break;
5920cd3a
PC		 3676 case PACKET_TX_HAS_OFF:
3677 val = po->tp_tx_has_off;
3678 break;
d346a3fa
DB		 3679 case PACKET_QDISC_BYPASS:
3680 val = packet_use_direct_xmit(po);
3681 break;
1da177e4
LT		 3682 default:
3683 return -ENOPROTOOPT;
3684 }
3685
c06fff6e
ED		 3686 if (len > lv)
3687 len = lv;
8ae55f04
KK		 3688 if (put_user(len, optlen))
3689 return -EFAULT;
8dc41944
HX		 3690 if (copy_to_user(optval, data, len))
3691 return -EFAULT;
8ae55f04 3692 return 0;
1da177e4
LT		 3693}
3694
3695
351638e7
JP		 3696static int packet_notifier(struct notifier_block *this,
3697 unsigned long msg, void *ptr)
1da177e4
LT		 3698{
3699 struct sock *sk;
351638e7 3700 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3701 struct net *net = dev_net(dev);
1da177e4 3702
808f5114 3703 rcu_read_lock();
b67bfe0d 3704 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3705 struct packet_sock *po = pkt_sk(sk);
3706
3707 switch (msg) {
3708 case NETDEV_UNREGISTER:
1da177e4 3709 if (po->mclist)
82f17091 3710 packet_dev_mclist_delete(dev, &po->mclist);
a2efcfa0
DM		 3711 /* fallthrough */
3712
1da177e4
LT		 3713 case NETDEV_DOWN:
3714 if (dev->ifindex == po->ifindex) {
3715 spin_lock(&po->bind_lock);
3716 if (po->running) {
ce06b03e 3717 __unregister_prot_hook(sk, false);
1da177e4
LT		 3718 sk->sk_err = ENETDOWN;
3719 if (!sock_flag(sk, SOCK_DEAD))
3720 sk->sk_error_report(sk);
3721 }
3722 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3723 packet_cached_dev_reset(po);
1da177e4 3724 po->ifindex = -1;
160ff18a
BG		 3725 if (po->prot_hook.dev)
3726 dev_put(po->prot_hook.dev);
1da177e4
LT		 3727 po->prot_hook.dev = NULL;
3728 }
3729 spin_unlock(&po->bind_lock);
3730 }
3731 break;
3732 case NETDEV_UP:
808f5114 3733 if (dev->ifindex == po->ifindex) {
3734 spin_lock(&po->bind_lock);
ce06b03e
DM		 3735 if (po->num)
3736 register_prot_hook(sk);
808f5114 3737 spin_unlock(&po->bind_lock);
1da177e4 3738 }
1da177e4
LT		 3739 break;
3740 }
3741 }
808f5114 3742 rcu_read_unlock();
1da177e4
LT		 3743 return NOTIFY_DONE;
3744}
3745
3746
3747static int packet_ioctl(struct socket *sock, unsigned int cmd,
3748 unsigned long arg)
3749{
3750 struct sock *sk = sock->sk;
3751
69e3c75f 3752 switch (cmd) {
40d4e3df
ED		 3753 case SIOCOUTQ:
3754 {
3755 int amount = sk_wmem_alloc_get(sk);
31e6d363 3756
40d4e3df
ED		 3757 return put_user(amount, (int __user *)arg);
3758 }
3759 case SIOCINQ:
3760 {
3761 struct sk_buff *skb;
3762 int amount = 0;
3763
3764 spin_lock_bh(&sk->sk_receive_queue.lock);
3765 skb = skb_peek(&sk->sk_receive_queue);
3766 if (skb)
3767 amount = skb->len;
3768 spin_unlock_bh(&sk->sk_receive_queue.lock);
3769 return put_user(amount, (int __user *)arg);
3770 }
3771 case SIOCGSTAMP:
3772 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3773 case SIOCGSTAMPNS:
3774 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3775
1da177e4 3776#ifdef CONFIG_INET
40d4e3df
ED		 3777 case SIOCADDRT:
3778 case SIOCDELRT:
3779 case SIOCDARP:
3780 case SIOCGARP:
3781 case SIOCSARP:
3782 case SIOCGIFADDR:
3783 case SIOCSIFADDR:
3784 case SIOCGIFBRDADDR:
3785 case SIOCSIFBRDADDR:
3786 case SIOCGIFNETMASK:
3787 case SIOCSIFNETMASK:
3788 case SIOCGIFDSTADDR:
3789 case SIOCSIFDSTADDR:
3790 case SIOCSIFFLAGS:
40d4e3df 3791 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3792#endif
3793
40d4e3df
ED		 3794 default:
3795 return -ENOIOCTLCMD;
1da177e4
LT		 3796 }
3797 return 0;
3798}
3799
40d4e3df 3800static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3801 poll_table *wait)
3802{
3803 struct sock *sk = sock->sk;
3804 struct packet_sock *po = pkt_sk(sk);
3805 unsigned int mask = datagram_poll(file, sock, wait);
3806
3807 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3808 if (po->rx_ring.pg_vec) {
f6fb8f10 3809 if (!packet_previous_rx_frame(po, &po->rx_ring,
3810 TP_STATUS_KERNEL))
1da177e4
LT		 3811 mask |= POLLIN | POLLRDNORM;
3812 }
2ccdbaa6 3813 if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
54d7c01d 3814 po->pressure = 0;
1da177e4 3815 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3816 spin_lock_bh(&sk->sk_write_queue.lock);
3817 if (po->tx_ring.pg_vec) {
3818 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3819 mask |= POLLOUT | POLLWRNORM;
3820 }
3821 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3822 return mask;
3823}
3824
3825
3826/* Dirty? Well, I still did not learn better way to account
3827 * for user mmaps.
3828 */
3829
3830static void packet_mm_open(struct vm_area_struct *vma)
3831{
3832 struct file *file = vma->vm_file;
40d4e3df 3833 struct socket *sock = file->private_data;
1da177e4 3834 struct sock *sk = sock->sk;
1ce4f28b 3835
1da177e4
LT		 3836 if (sk)
3837 atomic_inc(&pkt_sk(sk)->mapped);
3838}
3839
3840static void packet_mm_close(struct vm_area_struct *vma)
3841{
3842 struct file *file = vma->vm_file;
40d4e3df 3843 struct socket *sock = file->private_data;
1da177e4 3844 struct sock *sk = sock->sk;
1ce4f28b 3845
1da177e4
LT		 3846 if (sk)
3847 atomic_dec(&pkt_sk(sk)->mapped);
3848}
3849
f0f37e2f 3850static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3851 .open = packet_mm_open,
3852 .close = packet_mm_close,
1da177e4
LT		 3853};
3854
0e3125c7
NH		 3855static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3856 unsigned int len)
1da177e4
LT		 3857{
3858 int i;
3859
4ebf0ae2 3860 for (i = 0; i < len; i++) {
0e3125c7 3861 if (likely(pg_vec[i].buffer)) {
c56b4d90 3862 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3863 vfree(pg_vec[i].buffer);
3864 else
3865 free_pages((unsigned long)pg_vec[i].buffer,
3866 order);
3867 pg_vec[i].buffer = NULL;
3868 }
1da177e4
LT		 3869 }
3870 kfree(pg_vec);
3871}
3872
eea49cc9 3873static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3874{
f0d4eb29 3875 char *buffer;
0e3125c7
NH		 3876 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3877 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3878
3879 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3880 if (buffer)
3881 return buffer;
3882
f0d4eb29 3883 /* __get_free_pages failed, fall back to vmalloc */
bbce5a59 3884 buffer = vzalloc((1 << order) * PAGE_SIZE);
0e3125c7
NH		 3885 if (buffer)
3886 return buffer;
3887
f0d4eb29 3888 /* vmalloc failed, lets dig into swap here */
0e3125c7 3889 gfp_flags &= ~__GFP_NORETRY;
f0d4eb29 3890 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3891 if (buffer)
3892 return buffer;
3893
f0d4eb29 3894 /* complete and utter failure */
0e3125c7 3895 return NULL;
4ebf0ae2
DM		 3896}
3897
0e3125c7 3898static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3899{
3900 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3901 struct pgv *pg_vec;
4ebf0ae2
DM		 3902 int i;
3903
0e3125c7 3904 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3905 if (unlikely(!pg_vec))
3906 goto out;
3907
3908 for (i = 0; i < block_nr; i++) {
c56b4d90 3909 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3910 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3911 goto out_free_pgvec;
3912 }
3913
3914out:
3915 return pg_vec;
3916
3917out_free_pgvec:
3918 free_pg_vec(pg_vec, order, block_nr);
3919 pg_vec = NULL;
3920 goto out;
3921}
1da177e4 3922
f6fb8f10 3923static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3924 int closing, int tx_ring)
1da177e4 3925{
0e3125c7 3926 struct pgv *pg_vec = NULL;
1da177e4 3927 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3928 int was_running, order = 0;
69e3c75f
JB		 3929 struct packet_ring_buffer *rb;
3930 struct sk_buff_head *rb_queue;
0e11c91e 3931 __be16 num;
f6fb8f10 3932 int err = -EINVAL;
3933 /* Added to avoid minimal code churn */
3934 struct tpacket_req *req = &req_u->req;
3935
3936 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3937 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3938 WARN(1, "Tx-ring is not supported.\n");
3939 goto out;
3940 }
1ce4f28b 3941
69e3c75f
JB		 3942 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3943 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3944
69e3c75f
JB		 3945 err = -EBUSY;
3946 if (!closing) {
3947 if (atomic_read(&po->mapped))
3948 goto out;
b0138408 3949 if (packet_read_pending(rb))
69e3c75f
JB		 3950 goto out;
3951 }
1da177e4 3952
69e3c75f
JB		 3953 if (req->tp_block_nr) {
3954 /* Sanity tests and some calculations */
3955 err = -EBUSY;
3956 if (unlikely(rb->pg_vec))
3957 goto out;
1da177e4 3958
bbd6ef87
PM		 3959 switch (po->tp_version) {
3960 case TPACKET_V1:
3961 po->tp_hdrlen = TPACKET_HDRLEN;
3962 break;
3963 case TPACKET_V2:
3964 po->tp_hdrlen = TPACKET2_HDRLEN;
3965 break;
f6fb8f10 3966 case TPACKET_V3:
3967 po->tp_hdrlen = TPACKET3_HDRLEN;
3968 break;
bbd6ef87
PM		 3969 }
3970
69e3c75f 3971 err = -EINVAL;
4ebf0ae2 3972 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3973 goto out;
4ebf0ae2 3974 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3975 goto out;
dc808110
ED		 3976 if (po->tp_version >= TPACKET_V3 &&
3977 (int)(req->tp_block_size -
3978 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
3979 goto out;
8913336a 3980 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3981 po->tp_reserve))
3982 goto out;
4ebf0ae2 3983 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3984 goto out;
1da177e4 3985
69e3c75f
JB		 3986 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3987 if (unlikely(rb->frames_per_block <= 0))
3988 goto out;
3989 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3990 req->tp_frame_nr))
3991 goto out;
1da177e4
LT		 3992
3993 err = -ENOMEM;
4ebf0ae2
DM		 3994 order = get_order(req->tp_block_size);
3995 pg_vec = alloc_pg_vec(req, order);
3996 if (unlikely(!pg_vec))
1da177e4 3997 goto out;
f6fb8f10 3998 switch (po->tp_version) {
3999 case TPACKET_V3:
4000 /* Transmit path is not supported. We checked
4001 * it above but just being paranoid
4002 */
4003 if (!tx_ring)
4004 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
d7cf0c34 4005 break;
f6fb8f10 4006 default:
4007 break;
4008 }
69e3c75f
JB		 4009 }
4010 /* Done */
4011 else {
4012 err = -EINVAL;
4ebf0ae2 4013 if (unlikely(req->tp_frame_nr))
69e3c75f 4014 goto out;
1da177e4
LT		 4015 }
4016
4017 lock_sock(sk);
4018
4019 /* Detach socket from network */
4020 spin_lock(&po->bind_lock);
4021 was_running = po->running;
4022 num = po->num;
4023 if (was_running) {
1da177e4 4024 po->num = 0;
ce06b03e 4025 __unregister_prot_hook(sk, false);
1da177e4
LT		 4026 }
4027 spin_unlock(&po->bind_lock);
1ce4f28b 4028
1da177e4
LT		 4029 synchronize_net();
4030
4031 err = -EBUSY;
905db440 4032 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 4033 if (closing || atomic_read(&po->mapped) == 0) {
4034 err = 0;
69e3c75f 4035 spin_lock_bh(&rb_queue->lock);
c053fd96 4036 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 4037 rb->frame_max = (req->tp_frame_nr - 1);
4038 rb->head = 0;
4039 rb->frame_size = req->tp_frame_size;
4040 spin_unlock_bh(&rb_queue->lock);
4041
c053fd96
CG		 4042 swap(rb->pg_vec_order, order);
4043 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 4044
4045 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4046 po->prot_hook.func = (po->rx_ring.pg_vec) ?
4047 tpacket_rcv : packet_rcv;
4048 skb_queue_purge(rb_queue);
1da177e4 4049 if (atomic_read(&po->mapped))
40d4e3df
ED		 4050 pr_err("packet_mmap: vma is busy: %d\n",
4051 atomic_read(&po->mapped));
1da177e4 4052 }
905db440 4053 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 4054
4055 spin_lock(&po->bind_lock);
ce06b03e 4056 if (was_running) {
1da177e4 4057 po->num = num;
ce06b03e 4058 register_prot_hook(sk);
1da177e4
LT		 4059 }
4060 spin_unlock(&po->bind_lock);
f6fb8f10 4061 if (closing && (po->tp_version > TPACKET_V2)) {
4062 /* Because we don't support block-based V3 on tx-ring */
4063 if (!tx_ring)
4064 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
4065 }
1da177e4
LT		 4066 release_sock(sk);
4067
1da177e4
LT		 4068 if (pg_vec)
4069 free_pg_vec(pg_vec, order, req->tp_block_nr);
4070out:
4071 return err;
4072}
4073
69e3c75f
JB		 4074static int packet_mmap(struct file *file, struct socket *sock,
4075 struct vm_area_struct *vma)
1da177e4
LT		 4076{
4077 struct sock *sk = sock->sk;
4078 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 4079 unsigned long size, expected_size;
4080 struct packet_ring_buffer *rb;
1da177e4
LT		 4081 unsigned long start;
4082 int err = -EINVAL;
4083 int i;
4084
4085 if (vma->vm_pgoff)
4086 return -EINVAL;
4087
905db440 4088 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 4089
4090 expected_size = 0;
4091 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4092 if (rb->pg_vec) {
4093 expected_size += rb->pg_vec_len
4094 * rb->pg_vec_pages
4095 * PAGE_SIZE;
4096 }
4097 }
4098
4099 if (expected_size == 0)
1da177e4 4100 goto out;
69e3c75f
JB		 4101
4102 size = vma->vm_end - vma->vm_start;
4103 if (size != expected_size)
1da177e4
LT		 4104 goto out;
4105
1da177e4 4106 start = vma->vm_start;
69e3c75f
JB		 4107 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4108 if (rb->pg_vec == NULL)
4109 continue;
4110
4111 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 4112 struct page *page;
4113 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 4114 int pg_num;
4115
c56b4d90
CG		 4116 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4117 page = pgv_to_page(kaddr);
69e3c75f
JB		 4118 err = vm_insert_page(vma, start, page);
4119 if (unlikely(err))
4120 goto out;
4121 start += PAGE_SIZE;
0e3125c7 4122 kaddr += PAGE_SIZE;
69e3c75f 4123 }
4ebf0ae2 4124 }
1da177e4 4125 }
69e3c75f 4126
4ebf0ae2 4127 atomic_inc(&po->mapped);
1da177e4
LT		 4128 vma->vm_ops = &packet_mmap_ops;
4129 err = 0;
4130
4131out:
905db440 4132 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 4133 return err;
4134}
1da177e4 4135
90ddc4f0 4136static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 4137 .family = PF_PACKET,
4138 .owner = THIS_MODULE,
4139 .release = packet_release,
4140 .bind = packet_bind_spkt,
4141 .connect = sock_no_connect,
4142 .socketpair = sock_no_socketpair,
4143 .accept = sock_no_accept,
4144 .getname = packet_getname_spkt,
4145 .poll = datagram_poll,
4146 .ioctl = packet_ioctl,
4147 .listen = sock_no_listen,
4148 .shutdown = sock_no_shutdown,
4149 .setsockopt = sock_no_setsockopt,
4150 .getsockopt = sock_no_getsockopt,
4151 .sendmsg = packet_sendmsg_spkt,
4152 .recvmsg = packet_recvmsg,
4153 .mmap = sock_no_mmap,
4154 .sendpage = sock_no_sendpage,
4155};
1da177e4 4156
90ddc4f0 4157static const struct proto_ops packet_ops = {
1da177e4
LT		 4158 .family = PF_PACKET,
4159 .owner = THIS_MODULE,
4160 .release = packet_release,
4161 .bind = packet_bind,
4162 .connect = sock_no_connect,
4163 .socketpair = sock_no_socketpair,
4164 .accept = sock_no_accept,
1ce4f28b 4165 .getname = packet_getname,
1da177e4
LT		 4166 .poll = packet_poll,
4167 .ioctl = packet_ioctl,
4168 .listen = sock_no_listen,
4169 .shutdown = sock_no_shutdown,
4170 .setsockopt = packet_setsockopt,
4171 .getsockopt = packet_getsockopt,
4172 .sendmsg = packet_sendmsg,
4173 .recvmsg = packet_recvmsg,
4174 .mmap = packet_mmap,
4175 .sendpage = sock_no_sendpage,
4176};
4177
ec1b4cf7 4178static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 4179 .family = PF_PACKET,
4180 .create = packet_create,
4181 .owner = THIS_MODULE,
4182};
4183
4184static struct notifier_block packet_netdev_notifier = {
40d4e3df 4185 .notifier_call = packet_notifier,
1da177e4
LT		 4186};
4187
4188#ifdef CONFIG_PROC_FS
1da177e4
LT		 4189
4190static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 4191 __acquires(RCU)
1da177e4 4192{
e372c414 4193 struct net *net = seq_file_net(seq);
808f5114 4194
4195 rcu_read_lock();
4196 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 4197}
4198
4199static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4200{
1bf40954 4201 struct net *net = seq_file_net(seq);
808f5114 4202 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 4203}
4204
4205static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 4206 __releases(RCU)
1da177e4 4207{
808f5114 4208 rcu_read_unlock();
1da177e4
LT		 4209}
4210
1ce4f28b 4211static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 4212{
4213 if (v == SEQ_START_TOKEN)
4214 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4215 else {
b7ceabd9 4216 struct sock *s = sk_entry(v);
1da177e4
LT		 4217 const struct packet_sock *po = pkt_sk(s);
4218
4219 seq_printf(seq,
71338aa7 4220 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 4221 s,
4222 atomic_read(&s->sk_refcnt),
4223 s->sk_type,
4224 ntohs(po->num),
4225 po->ifindex,
4226 po->running,
4227 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4228 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4229 sock_i_ino(s));
1da177e4
LT		 4230 }
4231
4232 return 0;
4233}
4234
56b3d975 4235static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4236 .start = packet_seq_start,
4237 .next = packet_seq_next,
4238 .stop = packet_seq_stop,
4239 .show = packet_seq_show,
4240};
4241
4242static int packet_seq_open(struct inode *inode, struct file *file)
4243{
e372c414
DL		 4244 return seq_open_net(inode, file, &packet_seq_ops,
4245 sizeof(struct seq_net_private));
1da177e4
LT		 4246}
4247
da7071d7 4248static const struct file_operations packet_seq_fops = {
1da177e4
LT		 4249 .owner = THIS_MODULE,
4250 .open = packet_seq_open,
4251 .read = seq_read,
4252 .llseek = seq_lseek,
e372c414 4253 .release = seq_release_net,
1da177e4
LT		 4254};
4255
4256#endif
4257
2c8c1e72 4258static int __net_init packet_net_init(struct net *net)
d12d01d6 4259{
0fa7fa98 4260 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4261 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4262
d4beaa66 4263 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 4264 return -ENOMEM;
4265
4266 return 0;
4267}
4268
2c8c1e72 4269static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4270{
ece31ffd 4271 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 4272}
4273
4274static struct pernet_operations packet_net_ops = {
4275 .init = packet_net_init,
4276 .exit = packet_net_exit,
4277};
4278
4279
1da177e4
LT		 4280static void __exit packet_exit(void)
4281{
1da177e4 4282 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4283 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4284 sock_unregister(PF_PACKET);
4285 proto_unregister(&packet_proto);
4286}
4287
4288static int __init packet_init(void)
4289{
4290 int rc = proto_register(&packet_proto, 0);
4291
4292 if (rc != 0)
4293 goto out;
4294
4295 sock_register(&packet_family_ops);
d12d01d6 4296 register_pernet_subsys(&packet_net_ops);
1da177e4 4297 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4298out:
4299 return rc;
4300}
4301
4302module_init(packet_init);
4303module_exit(packet_exit);
4304MODULE_LICENSE("GPL");
4305MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 1.436372 seconds and 5 git commands to generate.