projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ax25: add link layer header validation function
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
b0138408 91#include <linux/percpu.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
47dceb8e 95#include <linux/bpf.h>
1da177e4 96
2787b04b
PE		 97#include "internal.h"
98
1da177e4
LT		 99/*
100 Assumptions:
101 - if device has no dev->hard_header routine, it adds and removes ll header
102 inside itself. In this case ll header is invisible outside of device,
103 but higher levels still should reserve dev->hard_header_len.
104 Some devices are enough clever to reallocate skb, when header
105 will not fit to reserved space (tunnel), another ones are silly
106 (PPP).
107 - packet socket receives packets with pulled ll header,
108 so that SOCK_RAW should push it back.
109
110On receive:
111-----------
112
113Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 114 mac_header -> ll header
115 data -> data
1da177e4
LT		 116
117Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 118 mac_header -> ll header
119 data -> ll header
1da177e4
LT		 120
121Incoming, dev->hard_header==NULL
b0e380b1
ACM		 122 mac_header -> UNKNOWN position. It is very likely, that it points to ll
123 header. PPP makes it, that is wrong, because introduce
db0c58f9 124 assymetry between rx and tx paths.
b0e380b1 125 data -> data
1da177e4
LT		 126
127Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 128 mac_header -> data. ll header is still not built!
129 data -> data
1da177e4
LT		 130
131Resume
132 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
133
134
135On transmit:
136------------
137
138dev->hard_header != NULL
b0e380b1
ACM		 139 mac_header -> ll header
140 data -> ll header
1da177e4
LT		 141
142dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 143 mac_header -> data
144 data -> data
1da177e4
LT		 145
146 We should set nh.raw on output to correct posistion,
147 packet classifier depends on it.
148 */
149
1da177e4
LT		 150/* Private packet socket structures. */
151
0fb375fb
EB		 152/* identical to struct packet_mreq except it has
153 * a longer address field.
154 */
40d4e3df 155struct packet_mreq_max {
0fb375fb
EB		 156 int mr_ifindex;
157 unsigned short mr_type;
158 unsigned short mr_alen;
159 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 160};
a2efcfa0 161
184f489e
DB		 162union tpacket_uhdr {
163 struct tpacket_hdr *h1;
164 struct tpacket2_hdr *h2;
165 struct tpacket3_hdr *h3;
166 void *raw;
167};
168
f6fb8f10 169static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 170 int closing, int tx_ring);
171
f6fb8f10 172#define V3_ALIGNMENT (8)
173
bc59ba39 174#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 175
176#define BLK_PLUS_PRIV(sz_of_priv) \
177 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
178
f6fb8f10 179#define PGV_FROM_VMALLOC 1
69e3c75f 180
f6fb8f10 181#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
182#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
183#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
184#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
185#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
186#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
187#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
188
69e3c75f
JB		 189struct packet_sock;
190static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 191static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
192 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 193
f6fb8f10 194static void *packet_previous_frame(struct packet_sock *po,
195 struct packet_ring_buffer *rb,
196 int status);
197static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 198static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
199 struct tpacket_block_desc *);
200static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 201 struct packet_sock *);
bc59ba39 202static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 203 struct packet_sock *, unsigned int status);
bc59ba39 204static int prb_queue_frozen(struct tpacket_kbdq_core *);
205static void prb_open_block(struct tpacket_kbdq_core *,
206 struct tpacket_block_desc *);
f6fb8f10 207static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 208static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
209static void prb_init_blk_timer(struct packet_sock *,
210 struct tpacket_kbdq_core *,
211 void (*func) (unsigned long));
212static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
213static void prb_clear_rxhash(struct tpacket_kbdq_core *,
214 struct tpacket3_hdr *);
215static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
216 struct tpacket3_hdr *);
1da177e4
LT		 217static void packet_flush_mclist(struct sock *sk);
218
ffbc6111 219struct packet_skb_cb {
ffbc6111
HX		 220 union {
221 struct sockaddr_pkt pkt;
2472d761
EB		 222 union {
223 /* Trick: alias skb original length with
224 * ll.sll_family and ll.protocol in order
225 * to save room.
226 */
227 unsigned int origlen;
228 struct sockaddr_ll ll;
229 };
ffbc6111
HX		 230 } sa;
231};
232
d3869efe
DW		 233#define vio_le() virtio_legacy_is_little_endian()
234
ffbc6111 235#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 236
bc59ba39 237#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 238#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 239 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 240#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 241 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 242#define GET_NEXT_PRB_BLK_NUM(x) \
243 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
244 ((x)->kactive_blk_num+1) : 0)
245
dc99f600
DM		 246static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
247static void __fanout_link(struct sock *sk, struct packet_sock *po);
248
d346a3fa
DB		 249static int packet_direct_xmit(struct sk_buff *skb)
250{
251 struct net_device *dev = skb->dev;
d346a3fa
DB		 252 netdev_features_t features;
253 struct netdev_queue *txq;
43279500 254 int ret = NETDEV_TX_BUSY;
d346a3fa
DB		 255
256 if (unlikely(!netif_running(dev) ||
43279500
DB		 257 !netif_carrier_ok(dev)))
258 goto drop;
d346a3fa
DB		 259
260 features = netif_skb_features(skb);
261 if (skb_needs_linearize(skb, features) &&
43279500
DB		 262 __skb_linearize(skb))
263 goto drop;
d346a3fa 264
10c51b56 265 txq = skb_get_tx_queue(dev, skb);
d346a3fa 266
43279500
DB		 267 local_bh_disable();
268
269 HARD_TX_LOCK(dev, txq, smp_processor_id());
10b3ad8c 270 if (!netif_xmit_frozen_or_drv_stopped(txq))
fa2dbdc2 271 ret = netdev_start_xmit(skb, dev, txq, false);
43279500 272 HARD_TX_UNLOCK(dev, txq);
d346a3fa 273
43279500
DB		 274 local_bh_enable();
275
276 if (!dev_xmit_complete(ret))
d346a3fa 277 kfree_skb(skb);
43279500 278
d346a3fa 279 return ret;
43279500 280drop:
0f97ede4 281 atomic_long_inc(&dev->tx_dropped);
43279500
DB		 282 kfree_skb(skb);
283 return NET_XMIT_DROP;
d346a3fa
DB		 284}
285
66e56cd4
DB		 286static struct net_device *packet_cached_dev_get(struct packet_sock *po)
287{
288 struct net_device *dev;
289
290 rcu_read_lock();
291 dev = rcu_dereference(po->cached_dev);
292 if (likely(dev))
293 dev_hold(dev);
294 rcu_read_unlock();
295
296 return dev;
297}
298
299static void packet_cached_dev_assign(struct packet_sock *po,
300 struct net_device *dev)
301{
302 rcu_assign_pointer(po->cached_dev, dev);
303}
304
305static void packet_cached_dev_reset(struct packet_sock *po)
306{
307 RCU_INIT_POINTER(po->cached_dev, NULL);
308}
309
d346a3fa
DB		 310static bool packet_use_direct_xmit(const struct packet_sock *po)
311{
312 return po->xmit == packet_direct_xmit;
313}
314
0fd5d57b 315static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
d346a3fa 316{
1cbac010 317 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 318}
319
0fd5d57b
DB		 320static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
321{
322 const struct net_device_ops *ops = dev->netdev_ops;
323 u16 queue_index;
324
325 if (ops->ndo_select_queue) {
326 queue_index = ops->ndo_select_queue(dev, skb, NULL,
327 __packet_pick_tx_queue);
328 queue_index = netdev_cap_txqueue(dev, queue_index);
329 } else {
330 queue_index = __packet_pick_tx_queue(dev, skb);
331 }
332
333 skb_set_queue_mapping(skb, queue_index);
334}
335
ce06b03e
DM		 336/* register_prot_hook must be invoked with the po->bind_lock held,
337 * or from a context in which asynchronous accesses to the packet
338 * socket is not possible (packet_create()).
339 */
340static void register_prot_hook(struct sock *sk)
341{
342 struct packet_sock *po = pkt_sk(sk);
e40526cb 343
ce06b03e 344 if (!po->running) {
66e56cd4 345 if (po->fanout)
dc99f600 346 __fanout_link(sk, po);
66e56cd4 347 else
dc99f600 348 dev_add_pack(&po->prot_hook);
e40526cb 349
ce06b03e
DM		 350 sock_hold(sk);
351 po->running = 1;
352 }
353}
354
355/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
356 * held. If the sync parameter is true, we will temporarily drop
357 * the po->bind_lock and do a synchronize_net to make sure no
358 * asynchronous packet processing paths still refer to the elements
359 * of po->prot_hook. If the sync parameter is false, it is the
360 * callers responsibility to take care of this.
361 */
362static void __unregister_prot_hook(struct sock *sk, bool sync)
363{
364 struct packet_sock *po = pkt_sk(sk);
365
366 po->running = 0;
66e56cd4
DB		 367
368 if (po->fanout)
dc99f600 369 __fanout_unlink(sk, po);
66e56cd4 370 else
dc99f600 371 __dev_remove_pack(&po->prot_hook);
e40526cb 372
ce06b03e
DM		 373 __sock_put(sk);
374
375 if (sync) {
376 spin_unlock(&po->bind_lock);
377 synchronize_net();
378 spin_lock(&po->bind_lock);
379 }
380}
381
382static void unregister_prot_hook(struct sock *sk, bool sync)
383{
384 struct packet_sock *po = pkt_sk(sk);
385
386 if (po->running)
387 __unregister_prot_hook(sk, sync);
388}
389
6e58040b 390static inline struct page * __pure pgv_to_page(void *addr)
0af55bb5
CG		 391{
392 if (is_vmalloc_addr(addr))
393 return vmalloc_to_page(addr);
394 return virt_to_page(addr);
395}
396
69e3c75f 397static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 398{
184f489e 399 union tpacket_uhdr h;
1da177e4 400
69e3c75f 401 h.raw = frame;
bbd6ef87
PM		 402 switch (po->tp_version) {
403 case TPACKET_V1:
69e3c75f 404 h.h1->tp_status = status;
0af55bb5 405 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 406 break;
407 case TPACKET_V2:
69e3c75f 408 h.h2->tp_status = status;
0af55bb5 409 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 410 break;
f6fb8f10 411 case TPACKET_V3:
69e3c75f 412 default:
f6fb8f10 413 WARN(1, "TPACKET version not supported.\n");
69e3c75f 414 BUG();
bbd6ef87 415 }
69e3c75f
JB		 416
417 smp_wmb();
bbd6ef87
PM		 418}
419
69e3c75f 420static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 421{
184f489e 422 union tpacket_uhdr h;
bbd6ef87 423
69e3c75f
JB		 424 smp_rmb();
425
bbd6ef87
PM		 426 h.raw = frame;
427 switch (po->tp_version) {
428 case TPACKET_V1:
0af55bb5 429 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 430 return h.h1->tp_status;
bbd6ef87 431 case TPACKET_V2:
0af55bb5 432 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 433 return h.h2->tp_status;
f6fb8f10 434 case TPACKET_V3:
69e3c75f 435 default:
f6fb8f10 436 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 437 BUG();
438 return 0;
bbd6ef87 439 }
1da177e4 440}
69e3c75f 441
b9c32fb2
DB		 442static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
443 unsigned int flags)
7a51384c
DB		 444{
445 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
446
68a360e8
WB		 447 if (shhwtstamps &&
448 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
449 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
450 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 451
452 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 453 return TP_STATUS_TS_SOFTWARE;
7a51384c 454
b9c32fb2 455 return 0;
7a51384c
DB		 456}
457
b9c32fb2
DB		 458static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
459 struct sk_buff *skb)
2e31396f
WB		 460{
461 union tpacket_uhdr h;
462 struct timespec ts;
b9c32fb2 463 __u32 ts_status;
2e31396f 464
b9c32fb2
DB		 465 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
466 return 0;
2e31396f
WB		 467
468 h.raw = frame;
469 switch (po->tp_version) {
470 case TPACKET_V1:
471 h.h1->tp_sec = ts.tv_sec;
472 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
473 break;
474 case TPACKET_V2:
475 h.h2->tp_sec = ts.tv_sec;
476 h.h2->tp_nsec = ts.tv_nsec;
477 break;
478 case TPACKET_V3:
479 default:
480 WARN(1, "TPACKET version not supported.\n");
481 BUG();
482 }
483
484 /* one flush is safe, as both fields always lie on the same cacheline */
485 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
486 smp_wmb();
b9c32fb2
DB		 487
488 return ts_status;
2e31396f
WB		 489}
490
69e3c75f
JB		 491static void *packet_lookup_frame(struct packet_sock *po,
492 struct packet_ring_buffer *rb,
493 unsigned int position,
494 int status)
495{
496 unsigned int pg_vec_pos, frame_offset;
184f489e 497 union tpacket_uhdr h;
69e3c75f
JB		 498
499 pg_vec_pos = position / rb->frames_per_block;
500 frame_offset = position % rb->frames_per_block;
501
0e3125c7
NH		 502 h.raw = rb->pg_vec[pg_vec_pos].buffer +
503 (frame_offset * rb->frame_size);
69e3c75f
JB		 504
505 if (status != __packet_get_status(po, h.raw))
506 return NULL;
507
508 return h.raw;
509}
510
eea49cc9 511static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 512 struct packet_ring_buffer *rb,
513 int status)
514{
515 return packet_lookup_frame(po, rb, rb->head, status);
516}
517
bc59ba39 518static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 519{
520 del_timer_sync(&pkc->retire_blk_timer);
521}
522
523static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
f6fb8f10 524 struct sk_buff_head *rb_queue)
525{
bc59ba39 526 struct tpacket_kbdq_core *pkc;
f6fb8f10 527
73d0fcf2 528 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 529
ec6f809f 530 spin_lock_bh(&rb_queue->lock);
f6fb8f10 531 pkc->delete_blk_timer = 1;
ec6f809f 532 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 533
534 prb_del_retire_blk_timer(pkc);
535}
536
537static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 538 struct tpacket_kbdq_core *pkc,
f6fb8f10 539 void (*func) (unsigned long))
540{
541 init_timer(&pkc->retire_blk_timer);
542 pkc->retire_blk_timer.data = (long)po;
543 pkc->retire_blk_timer.function = func;
544 pkc->retire_blk_timer.expires = jiffies;
545}
546
e8e85cc5 547static void prb_setup_retire_blk_timer(struct packet_sock *po)
f6fb8f10 548{
bc59ba39 549 struct tpacket_kbdq_core *pkc;
f6fb8f10 550
e8e85cc5 551 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 552 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
553}
554
555static int prb_calc_retire_blk_tmo(struct packet_sock *po,
556 int blk_size_in_bytes)
557{
558 struct net_device *dev;
559 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
7cad1bac 560 struct ethtool_link_ksettings ecmd;
4bc71cb9 561 int err;
f6fb8f10 562
4bc71cb9
JP		 563 rtnl_lock();
564 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
565 if (unlikely(!dev)) {
566 rtnl_unlock();
f6fb8f10 567 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9 568 }
7cad1bac 569 err = __ethtool_get_link_ksettings(dev, &ecmd);
4bc71cb9
JP		 570 rtnl_unlock();
571 if (!err) {
4bc71cb9
JP		 572 /*
573 * If the link speed is so slow you don't really
574 * need to worry about perf anyways
575 */
7cad1bac
DD		 576 if (ecmd.base.speed < SPEED_1000 ||
577 ecmd.base.speed == SPEED_UNKNOWN) {
4bc71cb9 578 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 579 } else {
580 msec = 1;
7cad1bac 581 div = ecmd.base.speed / 1000;
f6fb8f10 582 }
583 }
584
585 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
586
587 if (div)
588 mbits /= div;
589
590 tmo = mbits * msec;
591
592 if (div)
593 return tmo+1;
594 return tmo;
595}
596
bc59ba39 597static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 598 union tpacket_req_u *req_u)
599{
600 p1->feature_req_word = req_u->req3.tp_feature_req_word;
601}
602
603static void init_prb_bdqc(struct packet_sock *po,
604 struct packet_ring_buffer *rb,
605 struct pgv *pg_vec,
e8e85cc5 606 union tpacket_req_u *req_u)
f6fb8f10 607{
22781a5b 608 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 609 struct tpacket_block_desc *pbd;
f6fb8f10 610
611 memset(p1, 0x0, sizeof(*p1));
612
613 p1->knxt_seq_num = 1;
614 p1->pkbdq = pg_vec;
bc59ba39 615 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 616 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 617 p1->kblk_size = req_u->req3.tp_block_size;
618 p1->knum_blocks = req_u->req3.tp_block_nr;
619 p1->hdrlen = po->tp_hdrlen;
620 p1->version = po->tp_version;
621 p1->last_kactive_blk_num = 0;
ee80fbf3 622 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 623 if (req_u->req3.tp_retire_blk_tov)
624 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
625 else
626 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
627 req_u->req3.tp_block_size);
628 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
629 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
630
dc808110 631 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
f6fb8f10 632 prb_init_ft_ops(p1, req_u);
e8e85cc5 633 prb_setup_retire_blk_timer(po);
f6fb8f10 634 prb_open_block(p1, pbd);
635}
636
637/* Do NOT update the last_blk_num first.
638 * Assumes sk_buff_head lock is held.
639 */
bc59ba39 640static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 641{
642 mod_timer(&pkc->retire_blk_timer,
643 jiffies + pkc->tov_in_jiffies);
644 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
645}
646
647/*
648 * Timer logic:
649 * 1) We refresh the timer only when we open a block.
650 * By doing this we don't waste cycles refreshing the timer
651 * on packet-by-packet basis.
652 *
653 * With a 1MB block-size, on a 1Gbps line, it will take
654 * i) ~8 ms to fill a block + ii) memcpy etc.
655 * In this cut we are not accounting for the memcpy time.
656 *
657 * So, if the user sets the 'tmo' to 10ms then the timer
658 * will never fire while the block is still getting filled
659 * (which is what we want). However, the user could choose
660 * to close a block early and that's fine.
661 *
662 * But when the timer does fire, we check whether or not to refresh it.
663 * Since the tmo granularity is in msecs, it is not too expensive
664 * to refresh the timer, lets say every '8' msecs.
665 * Either the user can set the 'tmo' or we can derive it based on
666 * a) line-speed and b) block-size.
667 * prb_calc_retire_blk_tmo() calculates the tmo.
668 *
669 */
670static void prb_retire_rx_blk_timer_expired(unsigned long data)
671{
672 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 673 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 674 unsigned int frozen;
bc59ba39 675 struct tpacket_block_desc *pbd;
f6fb8f10 676
677 spin_lock(&po->sk.sk_receive_queue.lock);
678
679 frozen = prb_queue_frozen(pkc);
680 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
681
682 if (unlikely(pkc->delete_blk_timer))
683 goto out;
684
685 /* We only need to plug the race when the block is partially filled.
686 * tpacket_rcv:
687 * lock(); increment BLOCK_NUM_PKTS; unlock()
688 * copy_bits() is in progress ...
689 * timer fires on other cpu:
690 * we can't retire the current block because copy_bits
691 * is in progress.
692 *
693 */
694 if (BLOCK_NUM_PKTS(pbd)) {
695 while (atomic_read(&pkc->blk_fill_in_prog)) {
696 /* Waiting for skb_copy_bits to finish... */
697 cpu_relax();
698 }
699 }
700
701 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
702 if (!frozen) {
41a50d62
AD		 703 if (!BLOCK_NUM_PKTS(pbd)) {
704 /* An empty block. Just refresh the timer. */
705 goto refresh_timer;
706 }
f6fb8f10 707 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
708 if (!prb_dispatch_next_block(pkc, po))
709 goto refresh_timer;
710 else
711 goto out;
712 } else {
713 /* Case 1. Queue was frozen because user-space was
714 * lagging behind.
715 */
716 if (prb_curr_blk_in_use(pkc, pbd)) {
717 /*
718 * Ok, user-space is still behind.
719 * So just refresh the timer.
720 */
721 goto refresh_timer;
722 } else {
723 /* Case 2. queue was frozen,user-space caught up,
724 * now the link went idle && the timer fired.
725 * We don't have a block to close.So we open this
726 * block and restart the timer.
727 * opening a block thaws the queue,restarts timer
728 * Thawing/timer-refresh is a side effect.
729 */
730 prb_open_block(pkc, pbd);
731 goto out;
732 }
733 }
734 }
735
736refresh_timer:
737 _prb_refresh_rx_retire_blk_timer(pkc);
738
739out:
740 spin_unlock(&po->sk.sk_receive_queue.lock);
741}
742
eea49cc9 743static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 744 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 745{
746 /* Flush everything minus the block header */
747
748#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
749 u8 *start, *end;
750
751 start = (u8 *)pbd1;
752
753 /* Skip the block header(we know header WILL fit in 4K) */
754 start += PAGE_SIZE;
755
756 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
757 for (; start < end; start += PAGE_SIZE)
758 flush_dcache_page(pgv_to_page(start));
759
760 smp_wmb();
761#endif
762
763 /* Now update the block status. */
764
765 BLOCK_STATUS(pbd1) = status;
766
767 /* Flush the block header */
768
769#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
770 start = (u8 *)pbd1;
771 flush_dcache_page(pgv_to_page(start));
772
773 smp_wmb();
774#endif
775}
776
777/*
778 * Side effect:
779 *
780 * 1) flush the block
781 * 2) Increment active_blk_num
782 *
783 * Note:We DONT refresh the timer on purpose.
784 * Because almost always the next block will be opened.
785 */
bc59ba39 786static void prb_close_block(struct tpacket_kbdq_core *pkc1,
787 struct tpacket_block_desc *pbd1,
f6fb8f10 788 struct packet_sock *po, unsigned int stat)
789{
790 __u32 status = TP_STATUS_USER | stat;
791
792 struct tpacket3_hdr *last_pkt;
bc59ba39 793 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
da413eec 794 struct sock *sk = &po->sk;
f6fb8f10 795
ee80fbf3 796 if (po->stats.stats3.tp_drops)
f6fb8f10 797 status |= TP_STATUS_LOSING;
798
799 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
800 last_pkt->tp_next_offset = 0;
801
802 /* Get the ts of the last pkt */
803 if (BLOCK_NUM_PKTS(pbd1)) {
804 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
805 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
806 } else {
41a50d62
AD		 807 /* Ok, we tmo'd - so get the current time.
808 *
809 * It shouldn't really happen as we don't close empty
810 * blocks. See prb_retire_rx_blk_timer_expired().
811 */
f6fb8f10 812 struct timespec ts;
813 getnstimeofday(&ts);
814 h1->ts_last_pkt.ts_sec = ts.tv_sec;
815 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
816 }
817
818 smp_wmb();
819
820 /* Flush the block */
821 prb_flush_block(pkc1, pbd1, status);
822
da413eec
DC		 823 sk->sk_data_ready(sk);
824
f6fb8f10 825 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
826}
827
eea49cc9 828static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 829{
830 pkc->reset_pending_on_curr_blk = 0;
831}
832
833/*
834 * Side effect of opening a block:
835 *
836 * 1) prb_queue is thawed.
837 * 2) retire_blk_timer is refreshed.
838 *
839 */
bc59ba39 840static void prb_open_block(struct tpacket_kbdq_core *pkc1,
841 struct tpacket_block_desc *pbd1)
f6fb8f10 842{
843 struct timespec ts;
bc59ba39 844 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 845
846 smp_rmb();
847
8da3056c
DB		 848 /* We could have just memset this but we will lose the
849 * flexibility of making the priv area sticky
850 */
f6fb8f10 851
8da3056c
DB		 852 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
853 BLOCK_NUM_PKTS(pbd1) = 0;
854 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 855
8da3056c
DB		 856 getnstimeofday(&ts);
857
858 h1->ts_first_pkt.ts_sec = ts.tv_sec;
859 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 860
8da3056c
DB		 861 pkc1->pkblk_start = (char *)pbd1;
862 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
863
864 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
865 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
866
867 pbd1->version = pkc1->version;
868 pkc1->prev = pkc1->nxt_offset;
869 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
870
871 prb_thaw_queue(pkc1);
872 _prb_refresh_rx_retire_blk_timer(pkc1);
873
874 smp_wmb();
f6fb8f10 875}
876
877/*
878 * Queue freeze logic:
879 * 1) Assume tp_block_nr = 8 blocks.
880 * 2) At time 't0', user opens Rx ring.
881 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
882 * 4) user-space is either sleeping or processing block '0'.
883 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
884 * it will close block-7,loop around and try to fill block '0'.
885 * call-flow:
886 * __packet_lookup_frame_in_block
887 * prb_retire_current_block()
888 * prb_dispatch_next_block()
889 * |->(BLOCK_STATUS == USER) evaluates to true
890 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
891 * 6) Now there are two cases:
892 * 6.1) Link goes idle right after the queue is frozen.
893 * But remember, the last open_block() refreshed the timer.
894 * When this timer expires,it will refresh itself so that we can
895 * re-open block-0 in near future.
896 * 6.2) Link is busy and keeps on receiving packets. This is a simple
897 * case and __packet_lookup_frame_in_block will check if block-0
898 * is free and can now be re-used.
899 */
eea49cc9 900static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 901 struct packet_sock *po)
902{
903 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 904 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 905}
906
907#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
908
909/*
910 * If the next block is free then we will dispatch it
911 * and return a good offset.
912 * Else, we will freeze the queue.
913 * So, caller must check the return value.
914 */
bc59ba39 915static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 916 struct packet_sock *po)
917{
bc59ba39 918 struct tpacket_block_desc *pbd;
f6fb8f10 919
920 smp_rmb();
921
922 /* 1. Get current block num */
923 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
924
925 /* 2. If this block is currently in_use then freeze the queue */
926 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
927 prb_freeze_queue(pkc, po);
928 return NULL;
929 }
930
931 /*
932 * 3.
933 * open this block and return the offset where the first packet
934 * needs to get stored.
935 */
936 prb_open_block(pkc, pbd);
937 return (void *)pkc->nxt_offset;
938}
939
bc59ba39 940static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 941 struct packet_sock *po, unsigned int status)
942{
bc59ba39 943 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 944
945 /* retire/close the current block */
946 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
947 /*
948 * Plug the case where copy_bits() is in progress on
949 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
950 * have space to copy the pkt in the current block and
951 * called prb_retire_current_block()
952 *
953 * We don't need to worry about the TMO case because
954 * the timer-handler already handled this case.
955 */
956 if (!(status & TP_STATUS_BLK_TMO)) {
957 while (atomic_read(&pkc->blk_fill_in_prog)) {
958 /* Waiting for skb_copy_bits to finish... */
959 cpu_relax();
960 }
961 }
962 prb_close_block(pkc, pbd, po, status);
963 return;
964 }
f6fb8f10 965}
966
eea49cc9 967static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 968 struct tpacket_block_desc *pbd)
f6fb8f10 969{
970 return TP_STATUS_USER & BLOCK_STATUS(pbd);
971}
972
eea49cc9 973static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 974{
975 return pkc->reset_pending_on_curr_blk;
976}
977
eea49cc9 978static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 979{
bc59ba39 980 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 981 atomic_dec(&pkc->blk_fill_in_prog);
982}
983
eea49cc9 984static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 985 struct tpacket3_hdr *ppd)
986{
3958afa1 987 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 988}
989
eea49cc9 990static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 991 struct tpacket3_hdr *ppd)
992{
993 ppd->hv1.tp_rxhash = 0;
994}
995
eea49cc9 996static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 997 struct tpacket3_hdr *ppd)
998{
df8a39de
JP		 999 if (skb_vlan_tag_present(pkc->skb)) {
1000 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
a0cdfcf3
AW		 1001 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1002 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 1003 } else {
9e67030a 1004 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 1005 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 1006 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 1007 }
1008}
1009
bc59ba39 1010static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 1011 struct tpacket3_hdr *ppd)
1012{
a0cdfcf3 1013 ppd->hv1.tp_padding = 0;
f6fb8f10 1014 prb_fill_vlan_info(pkc, ppd);
1015
1016 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1017 prb_fill_rxhash(pkc, ppd);
1018 else
1019 prb_clear_rxhash(pkc, ppd);
1020}
1021
eea49cc9 1022static void prb_fill_curr_block(char *curr,
bc59ba39 1023 struct tpacket_kbdq_core *pkc,
1024 struct tpacket_block_desc *pbd,
f6fb8f10 1025 unsigned int len)
1026{
1027 struct tpacket3_hdr *ppd;
1028
1029 ppd = (struct tpacket3_hdr *)curr;
1030 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1031 pkc->prev = curr;
1032 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1033 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1034 BLOCK_NUM_PKTS(pbd) += 1;
1035 atomic_inc(&pkc->blk_fill_in_prog);
1036 prb_run_all_ft_ops(pkc, ppd);
1037}
1038
1039/* Assumes caller has the sk->rx_queue.lock */
1040static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1041 struct sk_buff *skb,
1042 int status,
1043 unsigned int len
1044 )
1045{
bc59ba39 1046 struct tpacket_kbdq_core *pkc;
1047 struct tpacket_block_desc *pbd;
f6fb8f10 1048 char *curr, *end;
1049
e3192690 1050 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1051 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1052
1053 /* Queue is frozen when user space is lagging behind */
1054 if (prb_queue_frozen(pkc)) {
1055 /*
1056 * Check if that last block which caused the queue to freeze,
1057 * is still in_use by user-space.
1058 */
1059 if (prb_curr_blk_in_use(pkc, pbd)) {
1060 /* Can't record this packet */
1061 return NULL;
1062 } else {
1063 /*
1064 * Ok, the block was released by user-space.
1065 * Now let's open that block.
1066 * opening a block also thaws the queue.
1067 * Thawing is a side effect.
1068 */
1069 prb_open_block(pkc, pbd);
1070 }
1071 }
1072
1073 smp_mb();
1074 curr = pkc->nxt_offset;
1075 pkc->skb = skb;
e3192690 1076 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1077
1078 /* first try the current block */
1079 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1080 prb_fill_curr_block(curr, pkc, pbd, len);
1081 return (void *)curr;
1082 }
1083
1084 /* Ok, close the current block */
1085 prb_retire_current_block(pkc, po, 0);
1086
1087 /* Now, try to dispatch the next block */
1088 curr = (char *)prb_dispatch_next_block(pkc, po);
1089 if (curr) {
1090 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1091 prb_fill_curr_block(curr, pkc, pbd, len);
1092 return (void *)curr;
1093 }
1094
1095 /*
1096 * No free blocks are available.user_space hasn't caught up yet.
1097 * Queue was just frozen and now this packet will get dropped.
1098 */
1099 return NULL;
1100}
1101
eea49cc9 1102static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1103 struct sk_buff *skb,
1104 int status, unsigned int len)
1105{
1106 char *curr = NULL;
1107 switch (po->tp_version) {
1108 case TPACKET_V1:
1109 case TPACKET_V2:
1110 curr = packet_lookup_frame(po, &po->rx_ring,
1111 po->rx_ring.head, status);
1112 return curr;
1113 case TPACKET_V3:
1114 return __packet_lookup_frame_in_block(po, skb, status, len);
1115 default:
1116 WARN(1, "TPACKET version not supported\n");
1117 BUG();
99aa3473 1118 return NULL;
f6fb8f10 1119 }
1120}
1121
eea49cc9 1122static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1123 struct packet_ring_buffer *rb,
77f65ebd 1124 unsigned int idx,
f6fb8f10 1125 int status)
1126{
bc59ba39 1127 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1128 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1129
1130 if (status != BLOCK_STATUS(pbd))
1131 return NULL;
1132 return pbd;
1133}
1134
eea49cc9 1135static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1136{
1137 unsigned int prev;
1138 if (rb->prb_bdqc.kactive_blk_num)
1139 prev = rb->prb_bdqc.kactive_blk_num-1;
1140 else
1141 prev = rb->prb_bdqc.knum_blocks-1;
1142 return prev;
1143}
1144
1145/* Assumes caller has held the rx_queue.lock */
eea49cc9 1146static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1147 struct packet_ring_buffer *rb,
1148 int status)
1149{
1150 unsigned int previous = prb_previous_blk_num(rb);
1151 return prb_lookup_block(po, rb, previous, status);
1152}
1153
eea49cc9 1154static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1155 struct packet_ring_buffer *rb,
1156 int status)
1157{
1158 if (po->tp_version <= TPACKET_V2)
1159 return packet_previous_frame(po, rb, status);
1160
1161 return __prb_previous_block(po, rb, status);
1162}
1163
eea49cc9 1164static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1165 struct packet_ring_buffer *rb)
1166{
1167 switch (po->tp_version) {
1168 case TPACKET_V1:
1169 case TPACKET_V2:
1170 return packet_increment_head(rb);
1171 case TPACKET_V3:
1172 default:
1173 WARN(1, "TPACKET version not supported.\n");
1174 BUG();
1175 return;
1176 }
1177}
1178
eea49cc9 1179static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1180 struct packet_ring_buffer *rb,
1181 int status)
1182{
1183 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1184 return packet_lookup_frame(po, rb, previous, status);
1185}
1186
eea49cc9 1187static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1188{
1189 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1190}
1191
b0138408
DB		 1192static void packet_inc_pending(struct packet_ring_buffer *rb)
1193{
1194 this_cpu_inc(*rb->pending_refcnt);
1195}
1196
1197static void packet_dec_pending(struct packet_ring_buffer *rb)
1198{
1199 this_cpu_dec(*rb->pending_refcnt);
1200}
1201
1202static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1203{
1204 unsigned int refcnt = 0;
1205 int cpu;
1206
1207 /* We don't use pending refcount in rx_ring. */
1208 if (rb->pending_refcnt == NULL)
1209 return 0;
1210
1211 for_each_possible_cpu(cpu)
1212 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1213
1214 return refcnt;
1215}
1216
1217static int packet_alloc_pending(struct packet_sock *po)
1218{
1219 po->rx_ring.pending_refcnt = NULL;
1220
1221 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1222 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1223 return -ENOBUFS;
1224
1225 return 0;
1226}
1227
1228static void packet_free_pending(struct packet_sock *po)
1229{
1230 free_percpu(po->tx_ring.pending_refcnt);
1231}
1232
9954729b
WB		 1233#define ROOM_POW_OFF 2
1234#define ROOM_NONE 0x0
1235#define ROOM_LOW 0x1
1236#define ROOM_NORMAL 0x2
1237
1238static bool __tpacket_has_room(struct packet_sock *po, int pow_off)
77f65ebd 1239{
9954729b
WB		 1240 int idx, len;
1241
1242 len = po->rx_ring.frame_max + 1;
1243 idx = po->rx_ring.head;
1244 if (pow_off)
1245 idx += len >> pow_off;
1246 if (idx >= len)
1247 idx -= len;
1248 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1249}
1250
1251static bool __tpacket_v3_has_room(struct packet_sock *po, int pow_off)
1252{
1253 int idx, len;
1254
1255 len = po->rx_ring.prb_bdqc.knum_blocks;
1256 idx = po->rx_ring.prb_bdqc.kactive_blk_num;
1257 if (pow_off)
1258 idx += len >> pow_off;
1259 if (idx >= len)
1260 idx -= len;
1261 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1262}
77f65ebd 1263
2ccdbaa6 1264static int __packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
9954729b
WB		 1265{
1266 struct sock *sk = &po->sk;
1267 int ret = ROOM_NONE;
1268
1269 if (po->prot_hook.func != tpacket_rcv) {
1270 int avail = sk->sk_rcvbuf - atomic_read(&sk->sk_rmem_alloc)
2ccdbaa6 1271 - (skb ? skb->truesize : 0);
9954729b
WB		 1272 if (avail > (sk->sk_rcvbuf >> ROOM_POW_OFF))
1273 return ROOM_NORMAL;
1274 else if (avail > 0)
1275 return ROOM_LOW;
1276 else
1277 return ROOM_NONE;
1278 }
77f65ebd 1279
9954729b
WB		 1280 if (po->tp_version == TPACKET_V3) {
1281 if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1282 ret = ROOM_NORMAL;
1283 else if (__tpacket_v3_has_room(po, 0))
1284 ret = ROOM_LOW;
1285 } else {
1286 if (__tpacket_has_room(po, ROOM_POW_OFF))
1287 ret = ROOM_NORMAL;
1288 else if (__tpacket_has_room(po, 0))
1289 ret = ROOM_LOW;
1290 }
2ccdbaa6
WB		 1291
1292 return ret;
1293}
1294
1295static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1296{
1297 int ret;
1298 bool has_room;
1299
54d7c01d
WB		 1300 spin_lock_bh(&po->sk.sk_receive_queue.lock);
1301 ret = __packet_rcv_has_room(po, skb);
2ccdbaa6
WB		 1302 has_room = ret == ROOM_NORMAL;
1303 if (po->pressure == has_room)
54d7c01d
WB		 1304 po->pressure = !has_room;
1305 spin_unlock_bh(&po->sk.sk_receive_queue.lock);
77f65ebd 1306
9954729b 1307 return ret;
77f65ebd
WB		 1308}
1309
1da177e4
LT		 1310static void packet_sock_destruct(struct sock *sk)
1311{
ed85b565
RC		 1312 skb_queue_purge(&sk->sk_error_queue);
1313
547b792c
IJ		 1314 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1315 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1316
1317 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1318 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1319 return;
1320 }
1321
17ab56a2 1322 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1323}
1324
3b3a5b0a
WB		 1325static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1326{
1327 u32 rxhash;
1328 int i, count = 0;
1329
1330 rxhash = skb_get_hash(skb);
1331 for (i = 0; i < ROLLOVER_HLEN; i++)
1332 if (po->rollover->history[i] == rxhash)
1333 count++;
1334
1335 po->rollover->history[prandom_u32() % ROLLOVER_HLEN] = rxhash;
1336 return count > (ROLLOVER_HLEN >> 1);
1337}
1338
77f65ebd
WB		 1339static unsigned int fanout_demux_hash(struct packet_fanout *f,
1340 struct sk_buff *skb,
1341 unsigned int num)
dc99f600 1342{
61b905da 1343 return reciprocal_scale(skb_get_hash(skb), num);
dc99f600
DM		 1344}
1345
77f65ebd
WB		 1346static unsigned int fanout_demux_lb(struct packet_fanout *f,
1347 struct sk_buff *skb,
1348 unsigned int num)
dc99f600 1349{
468479e6 1350 unsigned int val = atomic_inc_return(&f->rr_cur);
dc99f600 1351
468479e6 1352 return val % num;
77f65ebd
WB		 1353}
1354
1355static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1356 struct sk_buff *skb,
1357 unsigned int num)
1358{
1359 return smp_processor_id() % num;
dc99f600
DM		 1360}
1361
5df0ddfb
DB		 1362static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1363 struct sk_buff *skb,
1364 unsigned int num)
1365{
f337db64 1366 return prandom_u32_max(num);
5df0ddfb
DB		 1367}
1368
77f65ebd
WB		 1369static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1370 struct sk_buff *skb,
ad377cab 1371 unsigned int idx, bool try_self,
77f65ebd 1372 unsigned int num)
95ec3eb4 1373{
4633c9e0 1374 struct packet_sock *po, *po_next, *po_skip = NULL;
a9b63918 1375 unsigned int i, j, room = ROOM_NONE;
95ec3eb4 1376
0648ab70 1377 po = pkt_sk(f->arr[idx]);
3b3a5b0a
WB		 1378
1379 if (try_self) {
1380 room = packet_rcv_has_room(po, skb);
1381 if (room == ROOM_NORMAL ||
1382 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1383 return idx;
4633c9e0 1384 po_skip = po;
3b3a5b0a 1385 }
ad377cab 1386
0648ab70 1387 i = j = min_t(int, po->rollover->sock, num - 1);
77f65ebd 1388 do {
2ccdbaa6 1389 po_next = pkt_sk(f->arr[i]);
4633c9e0 1390 if (po_next != po_skip && !po_next->pressure &&
2ccdbaa6 1391 packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
77f65ebd 1392 if (i != j)
0648ab70 1393 po->rollover->sock = i;
a9b63918
WB		 1394 atomic_long_inc(&po->rollover->num);
1395 if (room == ROOM_LOW)
1396 atomic_long_inc(&po->rollover->num_huge);
77f65ebd
WB		 1397 return i;
1398 }
ad377cab 1399
77f65ebd
WB		 1400 if (++i == num)
1401 i = 0;
1402 } while (i != j);
1403
a9b63918 1404 atomic_long_inc(&po->rollover->num_failed);
77f65ebd
WB		 1405 return idx;
1406}
1407
2d36097d
NH		 1408static unsigned int fanout_demux_qm(struct packet_fanout *f,
1409 struct sk_buff *skb,
1410 unsigned int num)
1411{
1412 return skb_get_queue_mapping(skb) % num;
1413}
1414
47dceb8e
WB		 1415static unsigned int fanout_demux_bpf(struct packet_fanout *f,
1416 struct sk_buff *skb,
1417 unsigned int num)
1418{
1419 struct bpf_prog *prog;
1420 unsigned int ret = 0;
1421
1422 rcu_read_lock();
1423 prog = rcu_dereference(f->bpf_prog);
1424 if (prog)
ff936a04 1425 ret = bpf_prog_run_clear_cb(prog, skb) % num;
47dceb8e
WB		 1426 rcu_read_unlock();
1427
1428 return ret;
1429}
1430
77f65ebd
WB		 1431static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1432{
1433 return f->flags & (flag >> 8);
95ec3eb4
DM		 1434}
1435
95ec3eb4
DM		 1436static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1437 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1438{
1439 struct packet_fanout *f = pt->af_packet_priv;
f98f4514 1440 unsigned int num = READ_ONCE(f->num_members);
19bcf9f2 1441 struct net *net = read_pnet(&f->net);
dc99f600 1442 struct packet_sock *po;
77f65ebd 1443 unsigned int idx;
dc99f600 1444
19bcf9f2 1445 if (!net_eq(dev_net(dev), net) || !num) {
dc99f600
DM		 1446 kfree_skb(skb);
1447 return 0;
1448 }
1449
3f34b24a 1450 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
19bcf9f2 1451 skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET);
3f34b24a
AD		 1452 if (!skb)
1453 return 0;
1454 }
95ec3eb4
DM		 1455 switch (f->type) {
1456 case PACKET_FANOUT_HASH:
1457 default:
77f65ebd 1458 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1459 break;
1460 case PACKET_FANOUT_LB:
77f65ebd 1461 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1462 break;
1463 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1464 idx = fanout_demux_cpu(f, skb, num);
1465 break;
5df0ddfb
DB		 1466 case PACKET_FANOUT_RND:
1467 idx = fanout_demux_rnd(f, skb, num);
1468 break;
2d36097d
NH		 1469 case PACKET_FANOUT_QM:
1470 idx = fanout_demux_qm(f, skb, num);
1471 break;
77f65ebd 1472 case PACKET_FANOUT_ROLLOVER:
ad377cab 1473 idx = fanout_demux_rollover(f, skb, 0, false, num);
95ec3eb4 1474 break;
47dceb8e 1475 case PACKET_FANOUT_CBPF:
f2e52095 1476 case PACKET_FANOUT_EBPF:
47dceb8e
WB		 1477 idx = fanout_demux_bpf(f, skb, num);
1478 break;
dc99f600
DM		 1479 }
1480
ad377cab
WB		 1481 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1482 idx = fanout_demux_rollover(f, skb, idx, true, num);
dc99f600 1483
ad377cab 1484 po = pkt_sk(f->arr[idx]);
dc99f600
DM		 1485 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1486}
1487
fff3321d
PE		 1488DEFINE_MUTEX(fanout_mutex);
1489EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1490static LIST_HEAD(fanout_list);
1491
1492static void __fanout_link(struct sock *sk, struct packet_sock *po)
1493{
1494 struct packet_fanout *f = po->fanout;
1495
1496 spin_lock(&f->lock);
1497 f->arr[f->num_members] = sk;
1498 smp_wmb();
1499 f->num_members++;
1500 spin_unlock(&f->lock);
1501}
1502
1503static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1504{
1505 struct packet_fanout *f = po->fanout;
1506 int i;
1507
1508 spin_lock(&f->lock);
1509 for (i = 0; i < f->num_members; i++) {
1510 if (f->arr[i] == sk)
1511 break;
1512 }
1513 BUG_ON(i >= f->num_members);
1514 f->arr[i] = f->arr[f->num_members - 1];
1515 f->num_members--;
1516 spin_unlock(&f->lock);
1517}
1518
d4dd8aee 1519static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1520{
161642e2
ED		 1521 if (sk->sk_family != PF_PACKET)
1522 return false;
c0de08d0 1523
161642e2 1524 return ptype->af_packet_priv == pkt_sk(sk)->fanout;
c0de08d0
EL		 1525}
1526
47dceb8e
WB		 1527static void fanout_init_data(struct packet_fanout *f)
1528{
1529 switch (f->type) {
1530 case PACKET_FANOUT_LB:
1531 atomic_set(&f->rr_cur, 0);
1532 break;
1533 case PACKET_FANOUT_CBPF:
f2e52095 1534 case PACKET_FANOUT_EBPF:
47dceb8e
WB		 1535 RCU_INIT_POINTER(f->bpf_prog, NULL);
1536 break;
1537 }
1538}
1539
1540static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
1541{
1542 struct bpf_prog *old;
1543
1544 spin_lock(&f->lock);
1545 old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
1546 rcu_assign_pointer(f->bpf_prog, new);
1547 spin_unlock(&f->lock);
1548
1549 if (old) {
1550 synchronize_net();
1551 bpf_prog_destroy(old);
1552 }
1553}
1554
1555static int fanout_set_data_cbpf(struct packet_sock *po, char __user *data,
1556 unsigned int len)
1557{
1558 struct bpf_prog *new;
1559 struct sock_fprog fprog;
1560 int ret;
1561
1562 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1563 return -EPERM;
1564 if (len != sizeof(fprog))
1565 return -EINVAL;
1566 if (copy_from_user(&fprog, data, len))
1567 return -EFAULT;
1568
bab18991 1569 ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
47dceb8e
WB		 1570 if (ret)
1571 return ret;
1572
1573 __fanout_set_data_bpf(po->fanout, new);
1574 return 0;
1575}
1576
f2e52095
WB		 1577static int fanout_set_data_ebpf(struct packet_sock *po, char __user *data,
1578 unsigned int len)
1579{
1580 struct bpf_prog *new;
1581 u32 fd;
1582
1583 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1584 return -EPERM;
1585 if (len != sizeof(fd))
1586 return -EINVAL;
1587 if (copy_from_user(&fd, data, len))
1588 return -EFAULT;
1589
1590 new = bpf_prog_get(fd);
1591 if (IS_ERR(new))
1592 return PTR_ERR(new);
1593 if (new->type != BPF_PROG_TYPE_SOCKET_FILTER) {
1594 bpf_prog_put(new);
1595 return -EINVAL;
1596 }
1597
1598 __fanout_set_data_bpf(po->fanout, new);
1599 return 0;
1600}
1601
47dceb8e
WB		 1602static int fanout_set_data(struct packet_sock *po, char __user *data,
1603 unsigned int len)
1604{
1605 switch (po->fanout->type) {
1606 case PACKET_FANOUT_CBPF:
1607 return fanout_set_data_cbpf(po, data, len);
f2e52095
WB		 1608 case PACKET_FANOUT_EBPF:
1609 return fanout_set_data_ebpf(po, data, len);
47dceb8e
WB		 1610 default:
1611 return -EINVAL;
1612 };
1613}
1614
1615static void fanout_release_data(struct packet_fanout *f)
1616{
1617 switch (f->type) {
1618 case PACKET_FANOUT_CBPF:
f2e52095 1619 case PACKET_FANOUT_EBPF:
47dceb8e
WB		 1620 __fanout_set_data_bpf(f, NULL);
1621 };
1622}
1623
7736d33f 1624static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1625{
1626 struct packet_sock *po = pkt_sk(sk);
1627 struct packet_fanout *f, *match;
7736d33f 1628 u8 type = type_flags & 0xff;
77f65ebd 1629 u8 flags = type_flags >> 8;
dc99f600
DM		 1630 int err;
1631
1632 switch (type) {
77f65ebd
WB		 1633 case PACKET_FANOUT_ROLLOVER:
1634 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1635 return -EINVAL;
dc99f600
DM		 1636 case PACKET_FANOUT_HASH:
1637 case PACKET_FANOUT_LB:
95ec3eb4 1638 case PACKET_FANOUT_CPU:
5df0ddfb 1639 case PACKET_FANOUT_RND:
2d36097d 1640 case PACKET_FANOUT_QM:
47dceb8e 1641 case PACKET_FANOUT_CBPF:
f2e52095 1642 case PACKET_FANOUT_EBPF:
dc99f600
DM		 1643 break;
1644 default:
1645 return -EINVAL;
1646 }
1647
1648 if (!po->running)
1649 return -EINVAL;
1650
1651 if (po->fanout)
1652 return -EALREADY;
1653
4633c9e0
WB		 1654 if (type == PACKET_FANOUT_ROLLOVER ||
1655 (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
0648ab70
WB		 1656 po->rollover = kzalloc(sizeof(*po->rollover), GFP_KERNEL);
1657 if (!po->rollover)
1658 return -ENOMEM;
a9b63918
WB		 1659 atomic_long_set(&po->rollover->num, 0);
1660 atomic_long_set(&po->rollover->num_huge, 0);
1661 atomic_long_set(&po->rollover->num_failed, 0);
0648ab70
WB		 1662 }
1663
dc99f600
DM		 1664 mutex_lock(&fanout_mutex);
1665 match = NULL;
1666 list_for_each_entry(f, &fanout_list, list) {
1667 if (f->id == id &&
1668 read_pnet(&f->net) == sock_net(sk)) {
1669 match = f;
1670 break;
1671 }
1672 }
afe62c68 1673 err = -EINVAL;
77f65ebd 1674 if (match && match->flags != flags)
afe62c68 1675 goto out;
dc99f600 1676 if (!match) {
afe62c68 1677 err = -ENOMEM;
dc99f600 1678 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1679 if (!match)
1680 goto out;
1681 write_pnet(&match->net, sock_net(sk));
1682 match->id = id;
1683 match->type = type;
77f65ebd 1684 match->flags = flags;
afe62c68
ED		 1685 INIT_LIST_HEAD(&match->list);
1686 spin_lock_init(&match->lock);
1687 atomic_set(&match->sk_ref, 0);
47dceb8e 1688 fanout_init_data(match);
afe62c68
ED		 1689 match->prot_hook.type = po->prot_hook.type;
1690 match->prot_hook.dev = po->prot_hook.dev;
1691 match->prot_hook.func = packet_rcv_fanout;
1692 match->prot_hook.af_packet_priv = match;
c0de08d0 1693 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1694 dev_add_pack(&match->prot_hook);
1695 list_add(&match->list, &fanout_list);
dc99f600 1696 }
afe62c68
ED		 1697 err = -EINVAL;
1698 if (match->type == type &&
1699 match->prot_hook.type == po->prot_hook.type &&
1700 match->prot_hook.dev == po->prot_hook.dev) {
1701 err = -ENOSPC;
1702 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1703 __dev_remove_pack(&po->prot_hook);
1704 po->fanout = match;
1705 atomic_inc(&match->sk_ref);
1706 __fanout_link(sk, po);
1707 err = 0;
dc99f600
DM		 1708 }
1709 }
afe62c68 1710out:
dc99f600 1711 mutex_unlock(&fanout_mutex);
0648ab70
WB		 1712 if (err) {
1713 kfree(po->rollover);
1714 po->rollover = NULL;
1715 }
dc99f600
DM		 1716 return err;
1717}
1718
1719static void fanout_release(struct sock *sk)
1720{
1721 struct packet_sock *po = pkt_sk(sk);
1722 struct packet_fanout *f;
1723
1724 f = po->fanout;
1725 if (!f)
1726 return;
1727
fff3321d 1728 mutex_lock(&fanout_mutex);
dc99f600
DM		 1729 po->fanout = NULL;
1730
dc99f600
DM		 1731 if (atomic_dec_and_test(&f->sk_ref)) {
1732 list_del(&f->list);
1733 dev_remove_pack(&f->prot_hook);
47dceb8e 1734 fanout_release_data(f);
dc99f600
DM		 1735 kfree(f);
1736 }
1737 mutex_unlock(&fanout_mutex);
0648ab70 1738
59f21118
WB		 1739 if (po->rollover)
1740 kfree_rcu(po->rollover, rcu);
dc99f600 1741}
1da177e4 1742
3c70c132
DB		 1743static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
1744 struct sk_buff *skb)
1745{
1746 /* Earlier code assumed this would be a VLAN pkt, double-check
1747 * this now that we have the actual packet in hand. We can only
1748 * do this check on Ethernet devices.
1749 */
1750 if (unlikely(dev->type != ARPHRD_ETHER))
1751 return false;
1752
1753 skb_reset_mac_header(skb);
1754 return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
1755}
1756
90ddc4f0 1757static const struct proto_ops packet_ops;
1da177e4 1758
90ddc4f0 1759static const struct proto_ops packet_ops_spkt;
1da177e4 1760
40d4e3df
ED		 1761static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1762 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1763{
1764 struct sock *sk;
1765 struct sockaddr_pkt *spkt;
1766
1767 /*
1768 * When we registered the protocol we saved the socket in the data
1769 * field for just this event.
1770 */
1771
1772 sk = pt->af_packet_priv;
1ce4f28b 1773
1da177e4
LT		 1774 /*
1775 * Yank back the headers [hope the device set this
1776 * right or kerboom...]
1777 *
1778 * Incoming packets have ll header pulled,
1779 * push it back.
1780 *
98e399f8 1781 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1782 * so that this procedure is noop.
1783 */
1784
1785 if (skb->pkt_type == PACKET_LOOPBACK)
1786 goto out;
1787
09ad9bc7 1788 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1789 goto out;
1790
40d4e3df
ED		 1791 skb = skb_share_check(skb, GFP_ATOMIC);
1792 if (skb == NULL)
1da177e4
LT		 1793 goto oom;
1794
1795 /* drop any routing info */
adf30907 1796 skb_dst_drop(skb);
1da177e4 1797
84531c24
PO		 1798 /* drop conntrack reference */
1799 nf_reset(skb);
1800
ffbc6111 1801 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1802
98e399f8 1803 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1804
1805 /*
1806 * The SOCK_PACKET socket receives _all_ frames.
1807 */
1808
1809 spkt->spkt_family = dev->type;
1810 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1811 spkt->spkt_protocol = skb->protocol;
1812
1813 /*
1814 * Charge the memory to the socket. This is done specifically
1815 * to prevent sockets using all the memory up.
1816 */
1817
40d4e3df 1818 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1819 return 0;
1820
1821out:
1822 kfree_skb(skb);
1823oom:
1824 return 0;
1825}
1826
1827
1828/*
1829 * Output a raw packet to a device layer. This bypasses all the other
1830 * protocol layers and you must therefore supply it with a complete frame
1831 */
1ce4f28b 1832
1b784140
YX		 1833static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1834 size_t len)
1da177e4
LT		 1835{
1836 struct sock *sk = sock->sk;
342dfc30 1837 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1838 struct sk_buff *skb = NULL;
1da177e4 1839 struct net_device *dev;
40d4e3df 1840 __be16 proto = 0;
1da177e4 1841 int err;
3bdc0eba 1842 int extra_len = 0;
1ce4f28b 1843
1da177e4 1844 /*
1ce4f28b 1845 * Get and verify the address.
1da177e4
LT		 1846 */
1847
40d4e3df 1848 if (saddr) {
1da177e4 1849 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1850 return -EINVAL;
1851 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1852 proto = saddr->spkt_protocol;
1853 } else
1854 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1855
1856 /*
1ce4f28b 1857 * Find the device first to size check it
1da177e4
LT		 1858 */
1859
de74e92a 1860 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1861retry:
654d1f8a
ED		 1862 rcu_read_lock();
1863 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1864 err = -ENODEV;
1865 if (dev == NULL)
1866 goto out_unlock;
1ce4f28b 1867
d5e76b0a
DM		 1868 err = -ENETDOWN;
1869 if (!(dev->flags & IFF_UP))
1870 goto out_unlock;
1871
1da177e4 1872 /*
40d4e3df
ED		 1873 * You may not queue a frame bigger than the mtu. This is the lowest level
1874 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1875 */
1ce4f28b 1876
3bdc0eba
BG		 1877 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1878 if (!netif_supports_nofcs(dev)) {
1879 err = -EPROTONOSUPPORT;
1880 goto out_unlock;
1881 }
1882 extra_len = 4; /* We're doing our own CRC */
1883 }
1884
1da177e4 1885 err = -EMSGSIZE;
3bdc0eba 1886 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1887 goto out_unlock;
1888
1a35ca80
ED		 1889 if (!skb) {
1890 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1891 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1892 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1893
1894 rcu_read_unlock();
4ce40912 1895 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1896 if (skb == NULL)
1897 return -ENOBUFS;
1898 /* FIXME: Save some space for broken drivers that write a hard
1899 * header at transmission time by themselves. PPP is the notable
1900 * one here. This should really be fixed at the driver level.
1901 */
1902 skb_reserve(skb, reserved);
1903 skb_reset_network_header(skb);
1904
1905 /* Try to align data part correctly */
1906 if (hhlen) {
1907 skb->data -= hhlen;
1908 skb->tail -= hhlen;
1909 if (len < hhlen)
1910 skb_reset_network_header(skb);
1911 }
6ce8e9ce 1912 err = memcpy_from_msg(skb_put(skb, len), msg, len);
1a35ca80
ED		 1913 if (err)
1914 goto out_free;
1915 goto retry;
1da177e4
LT		 1916 }
1917
3c70c132
DB		 1918 if (len > (dev->mtu + dev->hard_header_len + extra_len) &&
1919 !packet_extra_vlan_len_allowed(dev, skb)) {
1920 err = -EMSGSIZE;
1921 goto out_unlock;
57f89bfa 1922 }
1a35ca80 1923
1da177e4
LT		 1924 skb->protocol = proto;
1925 skb->dev = dev;
1926 skb->priority = sk->sk_priority;
2d37a186 1927 skb->mark = sk->sk_mark;
bf84a010
DB		 1928
1929 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1930
3bdc0eba
BG		 1931 if (unlikely(extra_len == 4))
1932 skb->no_fcs = 1;
1933
40893fd0 1934 skb_probe_transport_header(skb, 0);
c1aad275 1935
1da177e4 1936 dev_queue_xmit(skb);
654d1f8a 1937 rcu_read_unlock();
40d4e3df 1938 return len;
1da177e4 1939
1da177e4 1940out_unlock:
654d1f8a 1941 rcu_read_unlock();
1a35ca80
ED		 1942out_free:
1943 kfree_skb(skb);
1da177e4
LT		 1944 return err;
1945}
1da177e4 1946
ff936a04
AS		 1947static unsigned int run_filter(struct sk_buff *skb,
1948 const struct sock *sk,
1949 unsigned int res)
1da177e4
LT		 1950{
1951 struct sk_filter *filter;
fda9ef5d 1952
80f8f102
ED		 1953 rcu_read_lock();
1954 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1955 if (filter != NULL)
ff936a04 1956 res = bpf_prog_run_clear_cb(filter->prog, skb);
80f8f102 1957 rcu_read_unlock();
1da177e4 1958
dbcb5855 1959 return res;
1da177e4
LT		 1960}
1961
16cc1400
WB		 1962static int __packet_rcv_vnet(const struct sk_buff *skb,
1963 struct virtio_net_hdr *vnet_hdr)
1964{
1965 *vnet_hdr = (const struct virtio_net_hdr) { 0 };
1966
1967 if (skb_is_gso(skb)) {
1968 struct skb_shared_info *sinfo = skb_shinfo(skb);
1969
1970 /* This is a hint as to how much should be linear. */
1971 vnet_hdr->hdr_len =
1972 __cpu_to_virtio16(vio_le(), skb_headlen(skb));
1973 vnet_hdr->gso_size =
1974 __cpu_to_virtio16(vio_le(), sinfo->gso_size);
1975
1976 if (sinfo->gso_type & SKB_GSO_TCPV4)
1977 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1978 else if (sinfo->gso_type & SKB_GSO_TCPV6)
1979 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1980 else if (sinfo->gso_type & SKB_GSO_UDP)
1981 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP;
1982 else if (sinfo->gso_type & SKB_GSO_FCOE)
1983 return -EINVAL;
1984 else
1985 BUG();
1986
1987 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
1988 vnet_hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
1989 } else
1990 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE;
1991
1992 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1993 vnet_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
1994 vnet_hdr->csum_start = __cpu_to_virtio16(vio_le(),
1995 skb_checksum_start_offset(skb));
1996 vnet_hdr->csum_offset = __cpu_to_virtio16(vio_le(),
1997 skb->csum_offset);
1998 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
1999 vnet_hdr->flags = VIRTIO_NET_HDR_F_DATA_VALID;
2000 } /* else everything is zero */
2001
2002 return 0;
2003}
2004
2005static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb,
2006 size_t *len)
2007{
2008 struct virtio_net_hdr vnet_hdr;
2009
2010 if (*len < sizeof(vnet_hdr))
2011 return -EINVAL;
2012 *len -= sizeof(vnet_hdr);
2013
2014 if (__packet_rcv_vnet(skb, &vnet_hdr))
2015 return -EINVAL;
2016
2017 return memcpy_to_msg(msg, (void *)&vnet_hdr, sizeof(vnet_hdr));
2018}
2019
1da177e4 2020/*
62ab0812
ED		 2021 * This function makes lazy skb cloning in hope that most of packets
2022 * are discarded by BPF.
2023 *
2024 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
2025 * and skb->cb are mangled. It works because (and until) packets
2026 * falling here are owned by current CPU. Output packets are cloned
2027 * by dev_queue_xmit_nit(), input packets are processed by net_bh
2028 * sequencially, so that if we return skb to original state on exit,
2029 * we will not harm anyone.
1da177e4
LT		 2030 */
2031
40d4e3df
ED		 2032static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
2033 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 2034{
2035 struct sock *sk;
2036 struct sockaddr_ll *sll;
2037 struct packet_sock *po;
40d4e3df 2038 u8 *skb_head = skb->data;
1da177e4 2039 int skb_len = skb->len;
dbcb5855 2040 unsigned int snaplen, res;
1da177e4
LT		 2041
2042 if (skb->pkt_type == PACKET_LOOPBACK)
2043 goto drop;
2044
2045 sk = pt->af_packet_priv;
2046 po = pkt_sk(sk);
2047
09ad9bc7 2048 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 2049 goto drop;
2050
1da177e4
LT		 2051 skb->dev = dev;
2052
3b04ddde 2053 if (dev->header_ops) {
1da177e4 2054 /* The device has an explicit notion of ll header,
62ab0812
ED		 2055 * exported to higher levels.
2056 *
2057 * Otherwise, the device hides details of its frame
2058 * structure, so that corresponding packet head is
2059 * never delivered to user.
1da177e4
LT		 2060 */
2061 if (sk->sk_type != SOCK_DGRAM)
98e399f8 2062 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 2063 else if (skb->pkt_type == PACKET_OUTGOING) {
2064 /* Special case: outgoing packets have ll header at head */
bbe735e4 2065 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 2066 }
2067 }
2068
2069 snaplen = skb->len;
2070
dbcb5855
DM		 2071 res = run_filter(skb, sk, snaplen);
2072 if (!res)
fda9ef5d 2073 goto drop_n_restore;
dbcb5855
DM		 2074 if (snaplen > res)
2075 snaplen = res;
1da177e4 2076
0fd7bac6 2077 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 2078 goto drop_n_acct;
2079
2080 if (skb_shared(skb)) {
2081 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2082 if (nskb == NULL)
2083 goto drop_n_acct;
2084
2085 if (skb_head != skb->data) {
2086 skb->data = skb_head;
2087 skb->len = skb_len;
2088 }
abc4e4fa 2089 consume_skb(skb);
1da177e4
LT		 2090 skb = nskb;
2091 }
2092
b4772ef8 2093 sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
ffbc6111
HX		 2094
2095 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4 2096 sll->sll_hatype = dev->type;
1da177e4 2097 sll->sll_pkttype = skb->pkt_type;
8032b464 2098 if (unlikely(po->origdev))
80feaacb
PWJ		 2099 sll->sll_ifindex = orig_dev->ifindex;
2100 else
2101 sll->sll_ifindex = dev->ifindex;
1da177e4 2102
b95cce35 2103 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 2104
2472d761
EB		 2105 /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
2106 * Use their space for storing the original skb length.
2107 */
2108 PACKET_SKB_CB(skb)->sa.origlen = skb->len;
8dc41944 2109
1da177e4
LT		 2110 if (pskb_trim(skb, snaplen))
2111 goto drop_n_acct;
2112
2113 skb_set_owner_r(skb, sk);
2114 skb->dev = NULL;
adf30907 2115 skb_dst_drop(skb);
1da177e4 2116
84531c24
PO		 2117 /* drop conntrack reference */
2118 nf_reset(skb);
2119
1da177e4 2120 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 2121 po->stats.stats1.tp_packets++;
3bc3b96f 2122 sock_skb_set_dropcount(sk, skb);
1da177e4
LT		 2123 __skb_queue_tail(&sk->sk_receive_queue, skb);
2124 spin_unlock(&sk->sk_receive_queue.lock);
676d2369 2125 sk->sk_data_ready(sk);
1da177e4
LT		 2126 return 0;
2127
2128drop_n_acct:
7091fbd8 2129 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 2130 po->stats.stats1.tp_drops++;
7091fbd8
WB		 2131 atomic_inc(&sk->sk_drops);
2132 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 2133
2134drop_n_restore:
2135 if (skb_head != skb->data && skb_shared(skb)) {
2136 skb->data = skb_head;
2137 skb->len = skb_len;
2138 }
2139drop:
ead2ceb0 2140 consume_skb(skb);
1da177e4
LT		 2141 return 0;
2142}
2143
40d4e3df
ED		 2144static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
2145 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 2146{
2147 struct sock *sk;
2148 struct packet_sock *po;
2149 struct sockaddr_ll *sll;
184f489e 2150 union tpacket_uhdr h;
40d4e3df 2151 u8 *skb_head = skb->data;
1da177e4 2152 int skb_len = skb->len;
dbcb5855 2153 unsigned int snaplen, res;
f6fb8f10 2154 unsigned long status = TP_STATUS_USER;
bbd6ef87 2155 unsigned short macoff, netoff, hdrlen;
1da177e4 2156 struct sk_buff *copy_skb = NULL;
bbd6ef87 2157 struct timespec ts;
b9c32fb2 2158 __u32 ts_status;
1da177e4 2159
51846355
AW		 2160 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
2161 * We may add members to them until current aligned size without forcing
2162 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
2163 */
2164 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
2165 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
2166
1da177e4
LT		 2167 if (skb->pkt_type == PACKET_LOOPBACK)
2168 goto drop;
2169
2170 sk = pt->af_packet_priv;
2171 po = pkt_sk(sk);
2172
09ad9bc7 2173 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 2174 goto drop;
2175
3b04ddde 2176 if (dev->header_ops) {
1da177e4 2177 if (sk->sk_type != SOCK_DGRAM)
98e399f8 2178 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 2179 else if (skb->pkt_type == PACKET_OUTGOING) {
2180 /* Special case: outgoing packets have ll header at head */
bbe735e4 2181 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 2182 }
2183 }
2184
2185 snaplen = skb->len;
2186
dbcb5855
DM		 2187 res = run_filter(skb, sk, snaplen);
2188 if (!res)
fda9ef5d 2189 goto drop_n_restore;
68c2e5de
AD		 2190
2191 if (skb->ip_summed == CHECKSUM_PARTIAL)
2192 status |= TP_STATUS_CSUMNOTREADY;
682f048b
AD		 2193 else if (skb->pkt_type != PACKET_OUTGOING &&
2194 (skb->ip_summed == CHECKSUM_COMPLETE ||
2195 skb_csum_unnecessary(skb)))
2196 status |= TP_STATUS_CSUM_VALID;
68c2e5de 2197
dbcb5855
DM		 2198 if (snaplen > res)
2199 snaplen = res;
1da177e4
LT		 2200
2201 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 2202 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2203 po->tp_reserve;
1da177e4 2204 } else {
95c96174 2205 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 2206 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a 2207 (maclen < 16 ? 16 : maclen)) +
58d19b19
WB		 2208 po->tp_reserve;
2209 if (po->has_vnet_hdr)
2210 netoff += sizeof(struct virtio_net_hdr);
1da177e4
LT		 2211 macoff = netoff - maclen;
2212 }
f6fb8f10 2213 if (po->tp_version <= TPACKET_V2) {
2214 if (macoff + snaplen > po->rx_ring.frame_size) {
2215 if (po->copy_thresh &&
0fd7bac6 2216 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 2217 if (skb_shared(skb)) {
2218 copy_skb = skb_clone(skb, GFP_ATOMIC);
2219 } else {
2220 copy_skb = skb_get(skb);
2221 skb_head = skb->data;
2222 }
2223 if (copy_skb)
2224 skb_set_owner_r(copy_skb, sk);
1da177e4 2225 }
f6fb8f10 2226 snaplen = po->rx_ring.frame_size - macoff;
2227 if ((int)snaplen < 0)
2228 snaplen = 0;
1da177e4 2229 }
dc808110
ED		 2230 } else if (unlikely(macoff + snaplen >
2231 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2232 u32 nval;
2233
2234 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2235 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2236 snaplen, nval, macoff);
2237 snaplen = nval;
2238 if (unlikely((int)snaplen < 0)) {
2239 snaplen = 0;
2240 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
2241 }
1da177e4 2242 }
1da177e4 2243 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 2244 h.raw = packet_current_rx_frame(po, skb,
2245 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 2246 if (!h.raw)
58d19b19 2247 goto drop_n_account;
f6fb8f10 2248 if (po->tp_version <= TPACKET_V2) {
2249 packet_increment_rx_head(po, &po->rx_ring);
2250 /*
2251 * LOSING will be reported till you read the stats,
2252 * because it's COR - Clear On Read.
2253 * Anyways, moving it for V1/V2 only as V3 doesn't need this
2254 * at packet level.
2255 */
ee80fbf3 2256 if (po->stats.stats1.tp_drops)
f6fb8f10 2257 status |= TP_STATUS_LOSING;
2258 }
ee80fbf3 2259 po->stats.stats1.tp_packets++;
1da177e4
LT		 2260 if (copy_skb) {
2261 status |= TP_STATUS_COPY;
2262 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2263 }
1da177e4
LT		 2264 spin_unlock(&sk->sk_receive_queue.lock);
2265
58d19b19
WB		 2266 if (po->has_vnet_hdr) {
2267 if (__packet_rcv_vnet(skb, h.raw + macoff -
2268 sizeof(struct virtio_net_hdr))) {
2269 spin_lock(&sk->sk_receive_queue.lock);
2270 goto drop_n_account;
2271 }
2272 }
2273
bbd6ef87 2274 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 2275
2276 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 2277 getnstimeofday(&ts);
1da177e4 2278
b9c32fb2
DB		 2279 status |= ts_status;
2280
bbd6ef87
PM		 2281 switch (po->tp_version) {
2282 case TPACKET_V1:
2283 h.h1->tp_len = skb->len;
2284 h.h1->tp_snaplen = snaplen;
2285 h.h1->tp_mac = macoff;
2286 h.h1->tp_net = netoff;
4b457bdf
DB		 2287 h.h1->tp_sec = ts.tv_sec;
2288 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 2289 hdrlen = sizeof(*h.h1);
2290 break;
2291 case TPACKET_V2:
2292 h.h2->tp_len = skb->len;
2293 h.h2->tp_snaplen = snaplen;
2294 h.h2->tp_mac = macoff;
2295 h.h2->tp_net = netoff;
bbd6ef87
PM		 2296 h.h2->tp_sec = ts.tv_sec;
2297 h.h2->tp_nsec = ts.tv_nsec;
df8a39de
JP		 2298 if (skb_vlan_tag_present(skb)) {
2299 h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
a0cdfcf3
AW		 2300 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2301 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2302 } else {
2303 h.h2->tp_vlan_tci = 0;
a0cdfcf3 2304 h.h2->tp_vlan_tpid = 0;
a3bcc23e 2305 }
e4d26f4b 2306 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 2307 hdrlen = sizeof(*h.h2);
2308 break;
f6fb8f10 2309 case TPACKET_V3:
2310 /* tp_nxt_offset,vlan are already populated above.
2311 * So DONT clear those fields here
2312 */
2313 h.h3->tp_status |= status;
2314 h.h3->tp_len = skb->len;
2315 h.h3->tp_snaplen = snaplen;
2316 h.h3->tp_mac = macoff;
2317 h.h3->tp_net = netoff;
f6fb8f10 2318 h.h3->tp_sec = ts.tv_sec;
2319 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2320 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2321 hdrlen = sizeof(*h.h3);
2322 break;
bbd6ef87
PM		 2323 default:
2324 BUG();
2325 }
1da177e4 2326
bbd6ef87 2327 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2328 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2329 sll->sll_family = AF_PACKET;
2330 sll->sll_hatype = dev->type;
2331 sll->sll_protocol = skb->protocol;
2332 sll->sll_pkttype = skb->pkt_type;
8032b464 2333 if (unlikely(po->origdev))
80feaacb
PWJ		 2334 sll->sll_ifindex = orig_dev->ifindex;
2335 else
2336 sll->sll_ifindex = dev->ifindex;
1da177e4 2337
e16aa207 2338 smp_mb();
f0d4eb29 2339
f6dafa95 2340#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2341 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2342 u8 *start, *end;
2343
f0d4eb29
DB		 2344 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2345 macoff + snaplen);
2346
2347 for (start = h.raw; start < end; start += PAGE_SIZE)
2348 flush_dcache_page(pgv_to_page(start));
1da177e4 2349 }
f0d4eb29 2350 smp_wmb();
f6dafa95 2351#endif
f0d4eb29 2352
da413eec 2353 if (po->tp_version <= TPACKET_V2) {
f6fb8f10 2354 __packet_set_status(po, h.raw, status);
da413eec
DC		 2355 sk->sk_data_ready(sk);
2356 } else {
f6fb8f10 2357 prb_clear_blk_fill_status(&po->rx_ring);
da413eec 2358 }
1da177e4
LT		 2359
2360drop_n_restore:
2361 if (skb_head != skb->data && skb_shared(skb)) {
2362 skb->data = skb_head;
2363 skb->len = skb_len;
2364 }
2365drop:
1ce4f28b 2366 kfree_skb(skb);
1da177e4
LT		 2367 return 0;
2368
58d19b19 2369drop_n_account:
ee80fbf3 2370 po->stats.stats1.tp_drops++;
1da177e4
LT		 2371 spin_unlock(&sk->sk_receive_queue.lock);
2372
676d2369 2373 sk->sk_data_ready(sk);
acb5d75b 2374 kfree_skb(copy_skb);
1da177e4
LT		 2375 goto drop_n_restore;
2376}
2377
69e3c75f
JB		 2378static void tpacket_destruct_skb(struct sk_buff *skb)
2379{
2380 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2381
69e3c75f 2382 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2383 void *ph;
b9c32fb2
DB		 2384 __u32 ts;
2385
69e3c75f 2386 ph = skb_shinfo(skb)->destructor_arg;
b0138408 2387 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2388
2389 ts = __packet_set_timestamp(po, ph, skb);
2390 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2391 }
2392
2393 sock_wfree(skb);
2394}
2395
9c707762
WB		 2396static bool ll_header_truncated(const struct net_device *dev, int len)
2397{
2398 /* net device doesn't like empty head */
880621c2
MB		 2399 if (unlikely(len < dev->hard_header_len)) {
2400 net_warn_ratelimited("%s: packet size is too short (%d < %d)\n",
9c707762
WB		 2401 current->comm, len, dev->hard_header_len);
2402 return true;
2403 }
2404
2405 return false;
2406}
2407
c72219b7
DB		 2408static void tpacket_set_protocol(const struct net_device *dev,
2409 struct sk_buff *skb)
2410{
2411 if (dev->type == ARPHRD_ETHER) {
2412 skb_reset_mac_header(skb);
2413 skb->protocol = eth_hdr(skb)->h_proto;
2414 }
2415}
2416
16cc1400
WB		 2417static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
2418{
2419 unsigned short gso_type = 0;
2420
2421 if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2422 (__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2423 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2 >
2424 __virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len)))
2425 vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(),
2426 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2427 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2);
2428
2429 if (__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len) > len)
2430 return -EINVAL;
2431
2432 if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2433 switch (vnet_hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2434 case VIRTIO_NET_HDR_GSO_TCPV4:
2435 gso_type = SKB_GSO_TCPV4;
2436 break;
2437 case VIRTIO_NET_HDR_GSO_TCPV6:
2438 gso_type = SKB_GSO_TCPV6;
2439 break;
2440 case VIRTIO_NET_HDR_GSO_UDP:
2441 gso_type = SKB_GSO_UDP;
2442 break;
2443 default:
2444 return -EINVAL;
2445 }
2446
2447 if (vnet_hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN)
2448 gso_type |= SKB_GSO_TCP_ECN;
2449
2450 if (vnet_hdr->gso_size == 0)
2451 return -EINVAL;
2452 }
2453
2454 vnet_hdr->gso_type = gso_type; /* changes type, temporary storage */
2455 return 0;
2456}
2457
2458static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
2459 struct virtio_net_hdr *vnet_hdr)
2460{
2461 int n;
2462
2463 if (*len < sizeof(*vnet_hdr))
2464 return -EINVAL;
2465 *len -= sizeof(*vnet_hdr);
2466
2467 n = copy_from_iter(vnet_hdr, sizeof(*vnet_hdr), &msg->msg_iter);
2468 if (n != sizeof(*vnet_hdr))
2469 return -EFAULT;
2470
2471 return __packet_snd_vnet_parse(vnet_hdr, *len);
2472}
2473
2474static int packet_snd_vnet_gso(struct sk_buff *skb,
2475 struct virtio_net_hdr *vnet_hdr)
2476{
2477 if (vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2478 u16 s = __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start);
2479 u16 o = __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset);
2480
2481 if (!skb_partial_csum_set(skb, s, o))
2482 return -EINVAL;
2483 }
2484
2485 skb_shinfo(skb)->gso_size =
2486 __virtio16_to_cpu(vio_le(), vnet_hdr->gso_size);
2487 skb_shinfo(skb)->gso_type = vnet_hdr->gso_type;
2488
2489 /* Header must be checked, and gso_segs computed. */
2490 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2491 skb_shinfo(skb)->gso_segs = 0;
2492 return 0;
2493}
2494
40d4e3df 2495static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
8d39b4a6 2496 void *frame, struct net_device *dev, void *data, int tp_len,
1d036d25 2497 __be16 proto, unsigned char *addr, int hlen, int copylen)
69e3c75f 2498{
184f489e 2499 union tpacket_uhdr ph;
8d39b4a6 2500 int to_write, offset, len, nr_frags, len_max;
69e3c75f
JB		 2501 struct socket *sock = po->sk.sk_socket;
2502 struct page *page;
69e3c75f
JB		 2503 int err;
2504
2505 ph.raw = frame;
2506
2507 skb->protocol = proto;
2508 skb->dev = dev;
2509 skb->priority = po->sk.sk_priority;
2d37a186 2510 skb->mark = po->sk.sk_mark;
2e31396f 2511 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2512 skb_shinfo(skb)->destructor_arg = ph.raw;
2513
ae641949 2514 skb_reserve(skb, hlen);
69e3c75f 2515 skb_reset_network_header(skb);
c1aad275 2516
69e3c75f
JB		 2517 to_write = tp_len;
2518
2519 if (sock->type == SOCK_DGRAM) {
2520 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2521 NULL, tp_len);
2522 if (unlikely(err < 0))
2523 return -EINVAL;
1d036d25 2524 } else if (copylen) {
69e3c75f 2525 skb_push(skb, dev->hard_header_len);
1d036d25
WB		 2526 skb_put(skb, copylen - dev->hard_header_len);
2527 err = skb_store_bits(skb, 0, data, copylen);
69e3c75f
JB		 2528 if (unlikely(err))
2529 return err;
c72219b7
DB		 2530 if (!skb->protocol)
2531 tpacket_set_protocol(dev, skb);
69e3c75f 2532
1d036d25
WB		 2533 data += copylen;
2534 to_write -= copylen;
69e3c75f
JB		 2535 }
2536
69e3c75f
JB		 2537 offset = offset_in_page(data);
2538 len_max = PAGE_SIZE - offset;
2539 len = ((to_write > len_max) ? len_max : to_write);
2540
2541 skb->data_len = to_write;
2542 skb->len += to_write;
2543 skb->truesize += to_write;
2544 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2545
2546 while (likely(to_write)) {
2547 nr_frags = skb_shinfo(skb)->nr_frags;
2548
2549 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2550 pr_err("Packet exceed the number of skb frags(%lu)\n",
2551 MAX_SKB_FRAGS);
69e3c75f
JB		 2552 return -EFAULT;
2553 }
2554
0af55bb5
CG		 2555 page = pgv_to_page(data);
2556 data += len;
69e3c75f
JB		 2557 flush_dcache_page(page);
2558 get_page(page);
0af55bb5 2559 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2560 to_write -= len;
2561 offset = 0;
2562 len_max = PAGE_SIZE;
2563 len = ((to_write > len_max) ? len_max : to_write);
2564 }
2565
8fd6c80d 2566 skb_probe_transport_header(skb, 0);
efdfa2f7 2567
69e3c75f
JB		 2568 return tp_len;
2569}
2570
8d39b4a6
WB		 2571static int tpacket_parse_header(struct packet_sock *po, void *frame,
2572 int size_max, void **data)
2573{
2574 union tpacket_uhdr ph;
2575 int tp_len, off;
2576
2577 ph.raw = frame;
2578
2579 switch (po->tp_version) {
2580 case TPACKET_V2:
2581 tp_len = ph.h2->tp_len;
2582 break;
2583 default:
2584 tp_len = ph.h1->tp_len;
2585 break;
2586 }
2587 if (unlikely(tp_len > size_max)) {
2588 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2589 return -EMSGSIZE;
2590 }
2591
2592 if (unlikely(po->tp_tx_has_off)) {
2593 int off_min, off_max;
2594
2595 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2596 off_max = po->tx_ring.frame_size - tp_len;
2597 if (po->sk.sk_type == SOCK_DGRAM) {
2598 switch (po->tp_version) {
2599 case TPACKET_V2:
2600 off = ph.h2->tp_net;
2601 break;
2602 default:
2603 off = ph.h1->tp_net;
2604 break;
2605 }
2606 } else {
2607 switch (po->tp_version) {
2608 case TPACKET_V2:
2609 off = ph.h2->tp_mac;
2610 break;
2611 default:
2612 off = ph.h1->tp_mac;
2613 break;
2614 }
2615 }
2616 if (unlikely((off < off_min) || (off_max < off)))
2617 return -EINVAL;
2618 } else {
2619 off = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2620 }
2621
2622 *data = frame + off;
2623 return tp_len;
2624}
2625
69e3c75f
JB		 2626static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2627{
69e3c75f
JB		 2628 struct sk_buff *skb;
2629 struct net_device *dev;
1d036d25 2630 struct virtio_net_hdr *vnet_hdr = NULL;
69e3c75f 2631 __be16 proto;
09effa67 2632 int err, reserve = 0;
40d4e3df 2633 void *ph;
342dfc30 2634 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2635 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
69e3c75f
JB		 2636 int tp_len, size_max;
2637 unsigned char *addr;
8d39b4a6 2638 void *data;
69e3c75f 2639 int len_sum = 0;
9e67030a 2640 int status = TP_STATUS_AVAILABLE;
1d036d25 2641 int hlen, tlen, copylen = 0;
69e3c75f 2642
69e3c75f
JB		 2643 mutex_lock(&po->pg_vec_lock);
2644
66e56cd4 2645 if (likely(saddr == NULL)) {
e40526cb 2646 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2647 proto = po->num;
2648 addr = NULL;
2649 } else {
2650 err = -EINVAL;
2651 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2652 goto out;
2653 if (msg->msg_namelen < (saddr->sll_halen
2654 + offsetof(struct sockaddr_ll,
2655 sll_addr)))
2656 goto out;
69e3c75f
JB		 2657 proto = saddr->sll_protocol;
2658 addr = saddr->sll_addr;
827d9780 2659 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2660 }
2661
69e3c75f
JB		 2662 err = -ENXIO;
2663 if (unlikely(dev == NULL))
2664 goto out;
69e3c75f
JB		 2665 err = -ENETDOWN;
2666 if (unlikely(!(dev->flags & IFF_UP)))
2667 goto out_put;
2668
5cfb4c8d
DB		 2669 if (po->sk.sk_socket->type == SOCK_RAW)
2670 reserve = dev->hard_header_len;
69e3c75f 2671 size_max = po->tx_ring.frame_size
b5dd884e 2672 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2673
1d036d25 2674 if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !po->has_vnet_hdr)
5cfb4c8d 2675 size_max = dev->mtu + reserve + VLAN_HLEN;
09effa67 2676
69e3c75f
JB		 2677 do {
2678 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2679 TP_STATUS_SEND_REQUEST);
69e3c75f 2680 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2681 if (need_wait && need_resched())
2682 schedule();
69e3c75f
JB		 2683 continue;
2684 }
2685
8d39b4a6
WB		 2686 skb = NULL;
2687 tp_len = tpacket_parse_header(po, ph, size_max, &data);
2688 if (tp_len < 0)
2689 goto tpacket_error;
2690
69e3c75f 2691 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2692 hlen = LL_RESERVED_SPACE(dev);
2693 tlen = dev->needed_tailroom;
1d036d25
WB		 2694 if (po->has_vnet_hdr) {
2695 vnet_hdr = data;
2696 data += sizeof(*vnet_hdr);
2697 tp_len -= sizeof(*vnet_hdr);
2698 if (tp_len < 0 ||
2699 __packet_snd_vnet_parse(vnet_hdr, tp_len)) {
2700 tp_len = -EINVAL;
2701 goto tpacket_error;
2702 }
2703 copylen = __virtio16_to_cpu(vio_le(),
2704 vnet_hdr->hdr_len);
2705 }
2706 if (dev->hard_header_len) {
2707 if (ll_header_truncated(dev, tp_len)) {
2708 tp_len = -EINVAL;
2709 goto tpacket_error;
2710 }
2711 copylen = max_t(int, copylen, dev->hard_header_len);
2712 }
69e3c75f 2713 skb = sock_alloc_send_skb(&po->sk,
1d036d25
WB		 2714 hlen + tlen + sizeof(struct sockaddr_ll) +
2715 (copylen - dev->hard_header_len),
fbf33a28 2716 !need_wait, &err);
69e3c75f 2717
fbf33a28
KM		 2718 if (unlikely(skb == NULL)) {
2719 /* we assume the socket was initially writeable ... */
2720 if (likely(len_sum > 0))
2721 err = len_sum;
69e3c75f 2722 goto out_status;
fbf33a28 2723 }
8d39b4a6 2724 tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto,
1d036d25 2725 addr, hlen, copylen);
dbd46ab4 2726 if (likely(tp_len >= 0) &&
5cfb4c8d 2727 tp_len > dev->mtu + reserve &&
1d036d25 2728 !po->has_vnet_hdr &&
3c70c132
DB		 2729 !packet_extra_vlan_len_allowed(dev, skb))
2730 tp_len = -EMSGSIZE;
69e3c75f
JB		 2731
2732 if (unlikely(tp_len < 0)) {
8d39b4a6 2733tpacket_error:
69e3c75f
JB		 2734 if (po->tp_loss) {
2735 __packet_set_status(po, ph,
2736 TP_STATUS_AVAILABLE);
2737 packet_increment_head(&po->tx_ring);
2738 kfree_skb(skb);
2739 continue;
2740 } else {
2741 status = TP_STATUS_WRONG_FORMAT;
2742 err = tp_len;
2743 goto out_status;
2744 }
2745 }
2746
1d036d25
WB		 2747 if (po->has_vnet_hdr && packet_snd_vnet_gso(skb, vnet_hdr)) {
2748 tp_len = -EINVAL;
2749 goto tpacket_error;
2750 }
2751
0fd5d57b
DB		 2752 packet_pick_tx_queue(dev, skb);
2753
69e3c75f
JB		 2754 skb->destructor = tpacket_destruct_skb;
2755 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2756 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2757
2758 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2759 err = po->xmit(skb);
eb70df13
JP		 2760 if (unlikely(err > 0)) {
2761 err = net_xmit_errno(err);
2762 if (err && __packet_get_status(po, ph) ==
2763 TP_STATUS_AVAILABLE) {
2764 /* skb was destructed already */
2765 skb = NULL;
2766 goto out_status;
2767 }
2768 /*
2769 * skb was dropped but not destructed yet;
2770 * let's treat it like congestion or err < 0
2771 */
2772 err = 0;
2773 }
69e3c75f
JB		 2774 packet_increment_head(&po->tx_ring);
2775 len_sum += tp_len;
b0138408
DB		 2776 } while (likely((ph != NULL) ||
2777 /* Note: packet_read_pending() might be slow if we have
2778 * to call it as it's per_cpu variable, but in fast-path
2779 * we already short-circuit the loop with the first
2780 * condition, and luckily don't have to go that path
2781 * anyway.
2782 */
2783 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2784
2785 err = len_sum;
2786 goto out_put;
2787
69e3c75f
JB		 2788out_status:
2789 __packet_set_status(po, ph, status);
2790 kfree_skb(skb);
2791out_put:
e40526cb 2792 dev_put(dev);
69e3c75f
JB		 2793out:
2794 mutex_unlock(&po->pg_vec_lock);
2795 return err;
2796}
69e3c75f 2797
eea49cc9
OJ		 2798static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2799 size_t reserve, size_t len,
2800 size_t linear, int noblock,
2801 int *err)
bfd5f4a3
SS		 2802{
2803 struct sk_buff *skb;
2804
2805 /* Under a page? Don't bother with paged skb. */
2806 if (prepad + len < PAGE_SIZE || !linear)
2807 linear = len;
2808
2809 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2810 err, 0);
bfd5f4a3
SS		 2811 if (!skb)
2812 return NULL;
2813
2814 skb_reserve(skb, reserve);
2815 skb_put(skb, linear);
2816 skb->data_len = len - linear;
2817 skb->len += len - linear;
2818
2819 return skb;
2820}
2821
d346a3fa 2822static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2823{
2824 struct sock *sk = sock->sk;
342dfc30 2825 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2826 struct sk_buff *skb;
2827 struct net_device *dev;
0e11c91e 2828 __be16 proto;
1da177e4 2829 unsigned char *addr;
827d9780 2830 int err, reserve = 0;
c7d39e32 2831 struct sockcm_cookie sockc;
bfd5f4a3
SS		 2832 struct virtio_net_hdr vnet_hdr = { 0 };
2833 int offset = 0;
bfd5f4a3 2834 struct packet_sock *po = pkt_sk(sk);
ae641949 2835 int hlen, tlen;
3bdc0eba 2836 int extra_len = 0;
1da177e4
LT		 2837
2838 /*
1ce4f28b 2839 * Get and verify the address.
1da177e4 2840 */
1ce4f28b 2841
66e56cd4 2842 if (likely(saddr == NULL)) {
e40526cb 2843 dev = packet_cached_dev_get(po);
1da177e4
LT		 2844 proto = po->num;
2845 addr = NULL;
2846 } else {
2847 err = -EINVAL;
2848 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2849 goto out;
0fb375fb
EB		 2850 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2851 goto out;
1da177e4
LT		 2852 proto = saddr->sll_protocol;
2853 addr = saddr->sll_addr;
827d9780 2854 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2855 }
2856
1da177e4 2857 err = -ENXIO;
e40526cb 2858 if (unlikely(dev == NULL))
1da177e4 2859 goto out_unlock;
d5e76b0a 2860 err = -ENETDOWN;
e40526cb 2861 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2862 goto out_unlock;
2863
c7d39e32
EJ		 2864 sockc.mark = sk->sk_mark;
2865 if (msg->msg_controllen) {
2866 err = sock_cmsg_send(sk, msg, &sockc);
2867 if (unlikely(err))
2868 goto out_unlock;
2869 }
2870
e40526cb
DB		 2871 if (sock->type == SOCK_RAW)
2872 reserve = dev->hard_header_len;
bfd5f4a3 2873 if (po->has_vnet_hdr) {
16cc1400
WB		 2874 err = packet_snd_vnet_parse(msg, &len, &vnet_hdr);
2875 if (err)
bfd5f4a3 2876 goto out_unlock;
bfd5f4a3
SS		 2877 }
2878
3bdc0eba
BG		 2879 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2880 if (!netif_supports_nofcs(dev)) {
2881 err = -EPROTONOSUPPORT;
2882 goto out_unlock;
2883 }
2884 extra_len = 4; /* We're doing our own CRC */
2885 }
2886
1da177e4 2887 err = -EMSGSIZE;
16cc1400
WB		 2888 if (!vnet_hdr.gso_type &&
2889 (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2890 goto out_unlock;
2891
bfd5f4a3 2892 err = -ENOBUFS;
ae641949
HX		 2893 hlen = LL_RESERVED_SPACE(dev);
2894 tlen = dev->needed_tailroom;
dc9e5153 2895 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len,
d3869efe 2896 __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len),
bfd5f4a3 2897 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2898 if (skb == NULL)
1da177e4
LT		 2899 goto out_unlock;
2900
bfd5f4a3 2901 skb_set_network_header(skb, reserve);
1da177e4 2902
0c4e8581 2903 err = -EINVAL;
9c707762
WB		 2904 if (sock->type == SOCK_DGRAM) {
2905 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
46d2cfb1 2906 if (unlikely(offset < 0))
9c707762
WB		 2907 goto out_free;
2908 } else {
2909 if (ll_header_truncated(dev, len))
2910 goto out_free;
2911 }
1da177e4
LT		 2912
2913 /* Returns -EFAULT on error */
c0371da6 2914 err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
1da177e4
LT		 2915 if (err)
2916 goto out_free;
bf84a010
DB		 2917
2918 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2919
16cc1400 2920 if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
3c70c132
DB		 2921 !packet_extra_vlan_len_allowed(dev, skb)) {
2922 err = -EMSGSIZE;
2923 goto out_free;
57f89bfa
BG		 2924 }
2925
09effa67
DM		 2926 skb->protocol = proto;
2927 skb->dev = dev;
1da177e4 2928 skb->priority = sk->sk_priority;
c7d39e32 2929 skb->mark = sockc.mark;
0fd5d57b
DB		 2930
2931 packet_pick_tx_queue(dev, skb);
1da177e4 2932
bfd5f4a3 2933 if (po->has_vnet_hdr) {
16cc1400
WB		 2934 err = packet_snd_vnet_gso(skb, &vnet_hdr);
2935 if (err)
2936 goto out_free;
2937 len += sizeof(vnet_hdr);
bfd5f4a3
SS		 2938 }
2939
8fd6c80d
DB		 2940 skb_probe_transport_header(skb, reserve);
2941
3bdc0eba
BG		 2942 if (unlikely(extra_len == 4))
2943 skb->no_fcs = 1;
2944
d346a3fa 2945 err = po->xmit(skb);
1da177e4
LT		 2946 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2947 goto out_unlock;
2948
e40526cb 2949 dev_put(dev);
1da177e4 2950
40d4e3df 2951 return len;
1da177e4
LT		 2952
2953out_free:
2954 kfree_skb(skb);
2955out_unlock:
e40526cb 2956 if (dev)
1da177e4
LT		 2957 dev_put(dev);
2958out:
2959 return err;
2960}
2961
1b784140 2962static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
69e3c75f 2963{
69e3c75f
JB		 2964 struct sock *sk = sock->sk;
2965 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2966
69e3c75f
JB		 2967 if (po->tx_ring.pg_vec)
2968 return tpacket_snd(po, msg);
2969 else
69e3c75f
JB		 2970 return packet_snd(sock, msg, len);
2971}
2972
1da177e4
LT		 2973/*
2974 * Close a PACKET socket. This is fairly simple. We immediately go
2975 * to 'closed' state and remove our protocol entry in the device list.
2976 */
2977
2978static int packet_release(struct socket *sock)
2979{
2980 struct sock *sk = sock->sk;
2981 struct packet_sock *po;
d12d01d6 2982 struct net *net;
f6fb8f10 2983 union tpacket_req_u req_u;
1da177e4
LT		 2984
2985 if (!sk)
2986 return 0;
2987
3b1e0a65 2988 net = sock_net(sk);
1da177e4
LT		 2989 po = pkt_sk(sk);
2990
0fa7fa98 2991 mutex_lock(&net->packet.sklist_lock);
808f5114 2992 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2993 mutex_unlock(&net->packet.sklist_lock);
2994
2995 preempt_disable();
920de804 2996 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2997 preempt_enable();
1da177e4 2998
808f5114 2999 spin_lock(&po->bind_lock);
ce06b03e 3000 unregister_prot_hook(sk, false);
66e56cd4
DB		 3001 packet_cached_dev_reset(po);
3002
160ff18a
BG		 3003 if (po->prot_hook.dev) {
3004 dev_put(po->prot_hook.dev);
3005 po->prot_hook.dev = NULL;
3006 }
808f5114 3007 spin_unlock(&po->bind_lock);
1da177e4 3008
1da177e4 3009 packet_flush_mclist(sk);
1da177e4 3010
9665d5d6
PS		 3011 if (po->rx_ring.pg_vec) {
3012 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 3013 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 3014 }
69e3c75f 3015
9665d5d6
PS		 3016 if (po->tx_ring.pg_vec) {
3017 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 3018 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 3019 }
1da177e4 3020
dc99f600
DM		 3021 fanout_release(sk);
3022
808f5114 3023 synchronize_net();
1da177e4
LT		 3024 /*
3025 * Now the socket is dead. No more input will appear.
3026 */
1da177e4
LT		 3027 sock_orphan(sk);
3028 sock->sk = NULL;
3029
3030 /* Purge queues */
3031
3032 skb_queue_purge(&sk->sk_receive_queue);
b0138408 3033 packet_free_pending(po);
17ab56a2 3034 sk_refcnt_debug_release(sk);
1da177e4
LT		 3035
3036 sock_put(sk);
3037 return 0;
3038}
3039
3040/*
3041 * Attach a packet hook.
3042 */
3043
30f7ea1c
FR		 3044static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
3045 __be16 proto)
1da177e4
LT		 3046{
3047 struct packet_sock *po = pkt_sk(sk);
158cd4af 3048 struct net_device *dev_curr;
902fefb8
DB		 3049 __be16 proto_curr;
3050 bool need_rehook;
30f7ea1c
FR		 3051 struct net_device *dev = NULL;
3052 int ret = 0;
3053 bool unlisted = false;
dc99f600 3054
30f7ea1c 3055 if (po->fanout)
dc99f600 3056 return -EINVAL;
1da177e4
LT		 3057
3058 lock_sock(sk);
1da177e4 3059 spin_lock(&po->bind_lock);
30f7ea1c
FR		 3060 rcu_read_lock();
3061
3062 if (name) {
3063 dev = dev_get_by_name_rcu(sock_net(sk), name);
3064 if (!dev) {
3065 ret = -ENODEV;
3066 goto out_unlock;
3067 }
3068 } else if (ifindex) {
3069 dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
3070 if (!dev) {
3071 ret = -ENODEV;
3072 goto out_unlock;
3073 }
3074 }
3075
3076 if (dev)
3077 dev_hold(dev);
66e56cd4 3078
902fefb8
DB		 3079 proto_curr = po->prot_hook.type;
3080 dev_curr = po->prot_hook.dev;
3081
3082 need_rehook = proto_curr != proto || dev_curr != dev;
3083
3084 if (need_rehook) {
30f7ea1c
FR		 3085 if (po->running) {
3086 rcu_read_unlock();
3087 __unregister_prot_hook(sk, true);
3088 rcu_read_lock();
3089 dev_curr = po->prot_hook.dev;
3090 if (dev)
3091 unlisted = !dev_get_by_index_rcu(sock_net(sk),
3092 dev->ifindex);
3093 }
1da177e4 3094
902fefb8
DB		 3095 po->num = proto;
3096 po->prot_hook.type = proto;
902fefb8 3097
30f7ea1c
FR		 3098 if (unlikely(unlisted)) {
3099 dev_put(dev);
3100 po->prot_hook.dev = NULL;
3101 po->ifindex = -1;
3102 packet_cached_dev_reset(po);
3103 } else {
3104 po->prot_hook.dev = dev;
3105 po->ifindex = dev ? dev->ifindex : 0;
3106 packet_cached_dev_assign(po, dev);
3107 }
902fefb8 3108 }
158cd4af
LW		 3109 if (dev_curr)
3110 dev_put(dev_curr);
66e56cd4 3111
902fefb8 3112 if (proto == 0 || !need_rehook)
1da177e4
LT		 3113 goto out_unlock;
3114
30f7ea1c 3115 if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
ce06b03e 3116 register_prot_hook(sk);
be85d4ad
UT		 3117 } else {
3118 sk->sk_err = ENETDOWN;
3119 if (!sock_flag(sk, SOCK_DEAD))
3120 sk->sk_error_report(sk);
1da177e4
LT		 3121 }
3122
3123out_unlock:
30f7ea1c 3124 rcu_read_unlock();
1da177e4
LT		 3125 spin_unlock(&po->bind_lock);
3126 release_sock(sk);
30f7ea1c 3127 return ret;
1da177e4
LT		 3128}
3129
3130/*
3131 * Bind a packet socket to a device
3132 */
3133
40d4e3df
ED		 3134static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
3135 int addr_len)
1da177e4 3136{
40d4e3df 3137 struct sock *sk = sock->sk;
1da177e4 3138 char name[15];
1ce4f28b 3139
1da177e4
LT		 3140 /*
3141 * Check legality
3142 */
1ce4f28b 3143
8ae55f04 3144 if (addr_len != sizeof(struct sockaddr))
1da177e4 3145 return -EINVAL;
40d4e3df 3146 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 3147
30f7ea1c 3148 return packet_do_bind(sk, name, 0, pkt_sk(sk)->num);
1da177e4 3149}
1da177e4
LT		 3150
3151static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
3152{
40d4e3df
ED		 3153 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
3154 struct sock *sk = sock->sk;
1da177e4
LT		 3155
3156 /*
3157 * Check legality
3158 */
1ce4f28b 3159
1da177e4
LT		 3160 if (addr_len < sizeof(struct sockaddr_ll))
3161 return -EINVAL;
3162 if (sll->sll_family != AF_PACKET)
3163 return -EINVAL;
3164
30f7ea1c
FR		 3165 return packet_do_bind(sk, NULL, sll->sll_ifindex,
3166 sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 3167}
3168
3169static struct proto packet_proto = {
3170 .name = "PACKET",
3171 .owner = THIS_MODULE,
3172 .obj_size = sizeof(struct packet_sock),
3173};
3174
3175/*
1ce4f28b 3176 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 3177 */
3178
3f378b68
EP		 3179static int packet_create(struct net *net, struct socket *sock, int protocol,
3180 int kern)
1da177e4
LT		 3181{
3182 struct sock *sk;
3183 struct packet_sock *po;
0e11c91e 3184 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 3185 int err;
3186
df008c91 3187 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 3188 return -EPERM;
be02097c
DM		 3189 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
3190 sock->type != SOCK_PACKET)
1da177e4
LT		 3191 return -ESOCKTNOSUPPORT;
3192
3193 sock->state = SS_UNCONNECTED;
3194
3195 err = -ENOBUFS;
11aa9c28 3196 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
1da177e4
LT		 3197 if (sk == NULL)
3198 goto out;
3199
3200 sock->ops = &packet_ops;
1da177e4
LT		 3201 if (sock->type == SOCK_PACKET)
3202 sock->ops = &packet_ops_spkt;
be02097c 3203
1da177e4
LT		 3204 sock_init_data(sock, sk);
3205
3206 po = pkt_sk(sk);
3207 sk->sk_family = PF_PACKET;
0e11c91e 3208 po->num = proto;
d346a3fa 3209 po->xmit = dev_queue_xmit;
66e56cd4 3210
b0138408
DB		 3211 err = packet_alloc_pending(po);
3212 if (err)
3213 goto out2;
3214
66e56cd4 3215 packet_cached_dev_reset(po);
1da177e4
LT		 3216
3217 sk->sk_destruct = packet_sock_destruct;
17ab56a2 3218 sk_refcnt_debug_inc(sk);
1da177e4
LT		 3219
3220 /*
3221 * Attach a protocol block
3222 */
3223
3224 spin_lock_init(&po->bind_lock);
905db440 3225 mutex_init(&po->pg_vec_lock);
0648ab70 3226 po->rollover = NULL;
1da177e4 3227 po->prot_hook.func = packet_rcv;
be02097c 3228
1da177e4
LT		 3229 if (sock->type == SOCK_PACKET)
3230 po->prot_hook.func = packet_rcv_spkt;
be02097c 3231
1da177e4
LT		 3232 po->prot_hook.af_packet_priv = sk;
3233
0e11c91e
AV		 3234 if (proto) {
3235 po->prot_hook.type = proto;
ce06b03e 3236 register_prot_hook(sk);
1da177e4
LT		 3237 }
3238
0fa7fa98 3239 mutex_lock(&net->packet.sklist_lock);
808f5114 3240 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 3241 mutex_unlock(&net->packet.sklist_lock);
3242
3243 preempt_disable();
3680453c 3244 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 3245 preempt_enable();
808f5114 3246
40d4e3df 3247 return 0;
b0138408
DB		 3248out2:
3249 sk_free(sk);
1da177e4
LT		 3250out:
3251 return err;
3252}
3253
3254/*
3255 * Pull a packet from our receive queue and hand it to the user.
3256 * If necessary we block.
3257 */
3258
1b784140
YX		 3259static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
3260 int flags)
1da177e4
LT		 3261{
3262 struct sock *sk = sock->sk;
3263 struct sk_buff *skb;
3264 int copied, err;
bfd5f4a3 3265 int vnet_hdr_len = 0;
2472d761 3266 unsigned int origlen = 0;
1da177e4
LT		 3267
3268 err = -EINVAL;
ed85b565 3269 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 3270 goto out;
3271
3272#if 0
3273 /* What error should we return now? EUNATTACH? */
3274 if (pkt_sk(sk)->ifindex < 0)
3275 return -ENODEV;
3276#endif
3277
ed85b565 3278 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 3279 err = sock_recv_errqueue(sk, msg, len,
3280 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 3281 goto out;
3282 }
3283
1da177e4
LT		 3284 /*
3285 * Call the generic datagram receiver. This handles all sorts
3286 * of horrible races and re-entrancy so we can forget about it
3287 * in the protocol layers.
3288 *
3289 * Now it will return ENETDOWN, if device have just gone down,
3290 * but then it will block.
3291 */
3292
40d4e3df 3293 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 3294
3295 /*
1ce4f28b 3296 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 3297 * handles the blocking we don't see and worry about blocking
3298 * retries.
3299 */
3300
8ae55f04 3301 if (skb == NULL)
1da177e4
LT		 3302 goto out;
3303
2ccdbaa6
WB		 3304 if (pkt_sk(sk)->pressure)
3305 packet_rcv_has_room(pkt_sk(sk), NULL);
3306
bfd5f4a3 3307 if (pkt_sk(sk)->has_vnet_hdr) {
16cc1400
WB		 3308 err = packet_rcv_vnet(msg, skb, &len);
3309 if (err)
bfd5f4a3 3310 goto out_free;
16cc1400 3311 vnet_hdr_len = sizeof(struct virtio_net_hdr);
bfd5f4a3
SS		 3312 }
3313
f3d33426
HFS		 3314 /* You lose any data beyond the buffer you gave. If it worries
3315 * a user program they can ask the device for its MTU
3316 * anyway.
1da177e4 3317 */
1da177e4 3318 copied = skb->len;
40d4e3df
ED		 3319 if (copied > len) {
3320 copied = len;
3321 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 3322 }
3323
51f3d02b 3324 err = skb_copy_datagram_msg(skb, 0, msg, copied);
1da177e4
LT		 3325 if (err)
3326 goto out_free;
3327
2472d761
EB		 3328 if (sock->type != SOCK_PACKET) {
3329 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3330
3331 /* Original length was stored in sockaddr_ll fields */
3332 origlen = PACKET_SKB_CB(skb)->sa.origlen;
3333 sll->sll_family = AF_PACKET;
3334 sll->sll_protocol = skb->protocol;
3335 }
3336
3b885787 3337 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 3338
f3d33426
HFS		 3339 if (msg->msg_name) {
3340 /* If the address length field is there to be filled
3341 * in, we fill it in now.
3342 */
3343 if (sock->type == SOCK_PACKET) {
342dfc30 3344 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426
HFS		 3345 msg->msg_namelen = sizeof(struct sockaddr_pkt);
3346 } else {
3347 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2472d761 3348
f3d33426
HFS		 3349 msg->msg_namelen = sll->sll_halen +
3350 offsetof(struct sockaddr_ll, sll_addr);
3351 }
ffbc6111
HX		 3352 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
3353 msg->msg_namelen);
f3d33426 3354 }
1da177e4 3355
8dc41944 3356 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 3357 struct tpacket_auxdata aux;
3358
3359 aux.tp_status = TP_STATUS_USER;
3360 if (skb->ip_summed == CHECKSUM_PARTIAL)
3361 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
682f048b
AD		 3362 else if (skb->pkt_type != PACKET_OUTGOING &&
3363 (skb->ip_summed == CHECKSUM_COMPLETE ||
3364 skb_csum_unnecessary(skb)))
3365 aux.tp_status |= TP_STATUS_CSUM_VALID;
3366
2472d761 3367 aux.tp_len = origlen;
ffbc6111
HX		 3368 aux.tp_snaplen = skb->len;
3369 aux.tp_mac = 0;
bbe735e4 3370 aux.tp_net = skb_network_offset(skb);
df8a39de
JP		 3371 if (skb_vlan_tag_present(skb)) {
3372 aux.tp_vlan_tci = skb_vlan_tag_get(skb);
a0cdfcf3
AW		 3373 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3374 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 3375 } else {
3376 aux.tp_vlan_tci = 0;
a0cdfcf3 3377 aux.tp_vlan_tpid = 0;
a3bcc23e 3378 }
ffbc6111 3379 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 3380 }
3381
1da177e4
LT		 3382 /*
3383 * Free or return the buffer as appropriate. Again this
3384 * hides all the races and re-entrancy issues from us.
3385 */
bfd5f4a3 3386 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 3387
3388out_free:
3389 skb_free_datagram(sk, skb);
3390out:
3391 return err;
3392}
3393
1da177e4
LT		 3394static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3395 int *uaddr_len, int peer)
3396{
3397 struct net_device *dev;
3398 struct sock *sk = sock->sk;
3399
3400 if (peer)
3401 return -EOPNOTSUPP;
3402
3403 uaddr->sa_family = AF_PACKET;
2dc85bf3 3404 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 3405 rcu_read_lock();
3406 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3407 if (dev)
2dc85bf3 3408 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 3409 rcu_read_unlock();
1da177e4
LT		 3410 *uaddr_len = sizeof(*uaddr);
3411
3412 return 0;
3413}
1da177e4
LT		 3414
3415static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3416 int *uaddr_len, int peer)
3417{
3418 struct net_device *dev;
3419 struct sock *sk = sock->sk;
3420 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3421 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3422
3423 if (peer)
3424 return -EOPNOTSUPP;
3425
3426 sll->sll_family = AF_PACKET;
3427 sll->sll_ifindex = po->ifindex;
3428 sll->sll_protocol = po->num;
67286640 3429 sll->sll_pkttype = 0;
654d1f8a
ED		 3430 rcu_read_lock();
3431 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3432 if (dev) {
3433 sll->sll_hatype = dev->type;
3434 sll->sll_halen = dev->addr_len;
3435 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3436 } else {
3437 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3438 sll->sll_halen = 0;
3439 }
654d1f8a 3440 rcu_read_unlock();
0fb375fb 3441 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3442
3443 return 0;
3444}
3445
2aeb0b88
WC		 3446static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3447 int what)
1da177e4
LT		 3448{
3449 switch (i->type) {
3450 case PACKET_MR_MULTICAST:
1162563f
JP		 3451 if (i->alen != dev->addr_len)
3452 return -EINVAL;
1da177e4 3453 if (what > 0)
22bedad3 3454 return dev_mc_add(dev, i->addr);
1da177e4 3455 else
22bedad3 3456 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3457 break;
3458 case PACKET_MR_PROMISC:
2aeb0b88 3459 return dev_set_promiscuity(dev, what);
1da177e4 3460 case PACKET_MR_ALLMULTI:
2aeb0b88 3461 return dev_set_allmulti(dev, what);
d95ed927 3462 case PACKET_MR_UNICAST:
1162563f
JP		 3463 if (i->alen != dev->addr_len)
3464 return -EINVAL;
d95ed927 3465 if (what > 0)
a748ee24 3466 return dev_uc_add(dev, i->addr);
d95ed927 3467 else
a748ee24 3468 return dev_uc_del(dev, i->addr);
d95ed927 3469 break;
40d4e3df
ED		 3470 default:
3471 break;
1da177e4 3472 }
2aeb0b88 3473 return 0;
1da177e4
LT		 3474}
3475
82f17091
FR		 3476static void packet_dev_mclist_delete(struct net_device *dev,
3477 struct packet_mclist **mlp)
1da177e4 3478{
82f17091
FR		 3479 struct packet_mclist *ml;
3480
3481 while ((ml = *mlp) != NULL) {
3482 if (ml->ifindex == dev->ifindex) {
3483 packet_dev_mc(dev, ml, -1);
3484 *mlp = ml->next;
3485 kfree(ml);
3486 } else
3487 mlp = &ml->next;
1da177e4
LT		 3488 }
3489}
3490
0fb375fb 3491static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3492{
3493 struct packet_sock *po = pkt_sk(sk);
3494 struct packet_mclist *ml, *i;
3495 struct net_device *dev;
3496 int err;
3497
3498 rtnl_lock();
3499
3500 err = -ENODEV;
3b1e0a65 3501 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3502 if (!dev)
3503 goto done;
3504
3505 err = -EINVAL;
1162563f 3506 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3507 goto done;
3508
3509 err = -ENOBUFS;
8b3a7005 3510 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3511 if (i == NULL)
3512 goto done;
3513
3514 err = 0;
3515 for (ml = po->mclist; ml; ml = ml->next) {
3516 if (ml->ifindex == mreq->mr_ifindex &&
3517 ml->type == mreq->mr_type &&
3518 ml->alen == mreq->mr_alen &&
3519 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3520 ml->count++;
3521 /* Free the new element ... */
3522 kfree(i);
3523 goto done;
3524 }
3525 }
3526
3527 i->type = mreq->mr_type;
3528 i->ifindex = mreq->mr_ifindex;
3529 i->alen = mreq->mr_alen;
3530 memcpy(i->addr, mreq->mr_address, i->alen);
3531 i->count = 1;
3532 i->next = po->mclist;
3533 po->mclist = i;
2aeb0b88
WC		 3534 err = packet_dev_mc(dev, i, 1);
3535 if (err) {
3536 po->mclist = i->next;
3537 kfree(i);
3538 }
1da177e4
LT		 3539
3540done:
3541 rtnl_unlock();
3542 return err;
3543}
3544
0fb375fb 3545static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3546{
3547 struct packet_mclist *ml, **mlp;
3548
3549 rtnl_lock();
3550
3551 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3552 if (ml->ifindex == mreq->mr_ifindex &&
3553 ml->type == mreq->mr_type &&
3554 ml->alen == mreq->mr_alen &&
3555 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3556 if (--ml->count == 0) {
3557 struct net_device *dev;
3558 *mlp = ml->next;
ad959e76
ED		 3559 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3560 if (dev)
1da177e4 3561 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3562 kfree(ml);
3563 }
82f17091 3564 break;
1da177e4
LT		 3565 }
3566 }
3567 rtnl_unlock();
82f17091 3568 return 0;
1da177e4
LT		 3569}
3570
3571static void packet_flush_mclist(struct sock *sk)
3572{
3573 struct packet_sock *po = pkt_sk(sk);
3574 struct packet_mclist *ml;
3575
3576 if (!po->mclist)
3577 return;
3578
3579 rtnl_lock();
3580 while ((ml = po->mclist) != NULL) {
3581 struct net_device *dev;
3582
3583 po->mclist = ml->next;
ad959e76
ED		 3584 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3585 if (dev != NULL)
1da177e4 3586 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3587 kfree(ml);
3588 }
3589 rtnl_unlock();
3590}
1da177e4
LT		 3591
3592static int
b7058842 3593packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3594{
3595 struct sock *sk = sock->sk;
8dc41944 3596 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3597 int ret;
3598
3599 if (level != SOL_PACKET)
3600 return -ENOPROTOOPT;
3601
69e3c75f 3602 switch (optname) {
1ce4f28b 3603 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3604 case PACKET_DROP_MEMBERSHIP:
3605 {
0fb375fb
EB		 3606 struct packet_mreq_max mreq;
3607 int len = optlen;
3608 memset(&mreq, 0, sizeof(mreq));
3609 if (len < sizeof(struct packet_mreq))
1da177e4 3610 return -EINVAL;
0fb375fb
EB		 3611 if (len > sizeof(mreq))
3612 len = sizeof(mreq);
40d4e3df 3613 if (copy_from_user(&mreq, optval, len))
1da177e4 3614 return -EFAULT;
0fb375fb
EB		 3615 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3616 return -EINVAL;
1da177e4
LT		 3617 if (optname == PACKET_ADD_MEMBERSHIP)
3618 ret = packet_mc_add(sk, &mreq);
3619 else
3620 ret = packet_mc_drop(sk, &mreq);
3621 return ret;
3622 }
a2efcfa0 3623
1da177e4 3624 case PACKET_RX_RING:
69e3c75f 3625 case PACKET_TX_RING:
1da177e4 3626 {
f6fb8f10 3627 union tpacket_req_u req_u;
3628 int len;
1da177e4 3629
f6fb8f10 3630 switch (po->tp_version) {
3631 case TPACKET_V1:
3632 case TPACKET_V2:
3633 len = sizeof(req_u.req);
3634 break;
3635 case TPACKET_V3:
3636 default:
3637 len = sizeof(req_u.req3);
3638 break;
3639 }
3640 if (optlen < len)
1da177e4 3641 return -EINVAL;
f6fb8f10 3642 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3643 return -EFAULT;
f6fb8f10 3644 return packet_set_ring(sk, &req_u, 0,
3645 optname == PACKET_TX_RING);
1da177e4
LT		 3646 }
3647 case PACKET_COPY_THRESH:
3648 {
3649 int val;
3650
40d4e3df 3651 if (optlen != sizeof(val))
1da177e4 3652 return -EINVAL;
40d4e3df 3653 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3654 return -EFAULT;
3655
3656 pkt_sk(sk)->copy_thresh = val;
3657 return 0;
3658 }
bbd6ef87
PM		 3659 case PACKET_VERSION:
3660 {
3661 int val;
3662
3663 if (optlen != sizeof(val))
3664 return -EINVAL;
69e3c75f 3665 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3666 return -EBUSY;
3667 if (copy_from_user(&val, optval, sizeof(val)))
3668 return -EFAULT;
3669 switch (val) {
3670 case TPACKET_V1:
3671 case TPACKET_V2:
f6fb8f10 3672 case TPACKET_V3:
bbd6ef87
PM		 3673 po->tp_version = val;
3674 return 0;
3675 default:
3676 return -EINVAL;
3677 }
3678 }
8913336a
PM		 3679 case PACKET_RESERVE:
3680 {
3681 unsigned int val;
3682
3683 if (optlen != sizeof(val))
3684 return -EINVAL;
69e3c75f 3685 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3686 return -EBUSY;
3687 if (copy_from_user(&val, optval, sizeof(val)))
3688 return -EFAULT;
3689 po->tp_reserve = val;
3690 return 0;
3691 }
69e3c75f
JB		 3692 case PACKET_LOSS:
3693 {
3694 unsigned int val;
3695
3696 if (optlen != sizeof(val))
3697 return -EINVAL;
3698 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3699 return -EBUSY;
3700 if (copy_from_user(&val, optval, sizeof(val)))
3701 return -EFAULT;
3702 po->tp_loss = !!val;
3703 return 0;
3704 }
8dc41944
HX		 3705 case PACKET_AUXDATA:
3706 {
3707 int val;
3708
3709 if (optlen < sizeof(val))
3710 return -EINVAL;
3711 if (copy_from_user(&val, optval, sizeof(val)))
3712 return -EFAULT;
3713
3714 po->auxdata = !!val;
3715 return 0;
3716 }
80feaacb
PWJ		 3717 case PACKET_ORIGDEV:
3718 {
3719 int val;
3720
3721 if (optlen < sizeof(val))
3722 return -EINVAL;
3723 if (copy_from_user(&val, optval, sizeof(val)))
3724 return -EFAULT;
3725
3726 po->origdev = !!val;
3727 return 0;
3728 }
bfd5f4a3
SS		 3729 case PACKET_VNET_HDR:
3730 {
3731 int val;
3732
3733 if (sock->type != SOCK_RAW)
3734 return -EINVAL;
3735 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3736 return -EBUSY;
3737 if (optlen < sizeof(val))
3738 return -EINVAL;
3739 if (copy_from_user(&val, optval, sizeof(val)))
3740 return -EFAULT;
3741
3742 po->has_vnet_hdr = !!val;
3743 return 0;
3744 }
614f60fa
SM		 3745 case PACKET_TIMESTAMP:
3746 {
3747 int val;
3748
3749 if (optlen != sizeof(val))
3750 return -EINVAL;
3751 if (copy_from_user(&val, optval, sizeof(val)))
3752 return -EFAULT;
3753
3754 po->tp_tstamp = val;
3755 return 0;
3756 }
dc99f600
DM		 3757 case PACKET_FANOUT:
3758 {
3759 int val;
3760
3761 if (optlen != sizeof(val))
3762 return -EINVAL;
3763 if (copy_from_user(&val, optval, sizeof(val)))
3764 return -EFAULT;
3765
3766 return fanout_add(sk, val & 0xffff, val >> 16);
3767 }
47dceb8e
WB		 3768 case PACKET_FANOUT_DATA:
3769 {
3770 if (!po->fanout)
3771 return -EINVAL;
3772
3773 return fanout_set_data(po, optval, optlen);
3774 }
5920cd3a
PC		 3775 case PACKET_TX_HAS_OFF:
3776 {
3777 unsigned int val;
3778
3779 if (optlen != sizeof(val))
3780 return -EINVAL;
3781 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3782 return -EBUSY;
3783 if (copy_from_user(&val, optval, sizeof(val)))
3784 return -EFAULT;
3785 po->tp_tx_has_off = !!val;
3786 return 0;
3787 }
d346a3fa
DB		 3788 case PACKET_QDISC_BYPASS:
3789 {
3790 int val;
3791
3792 if (optlen != sizeof(val))
3793 return -EINVAL;
3794 if (copy_from_user(&val, optval, sizeof(val)))
3795 return -EFAULT;
3796
3797 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3798 return 0;
3799 }
1da177e4
LT		 3800 default:
3801 return -ENOPROTOOPT;
3802 }
3803}
3804
3805static int packet_getsockopt(struct socket *sock, int level, int optname,
3806 char __user *optval, int __user *optlen)
3807{
3808 int len;
c06fff6e 3809 int val, lv = sizeof(val);
1da177e4
LT		 3810 struct sock *sk = sock->sk;
3811 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3812 void *data = &val;
ee80fbf3 3813 union tpacket_stats_u st;
a9b63918 3814 struct tpacket_rollover_stats rstats;
1da177e4
LT		 3815
3816 if (level != SOL_PACKET)
3817 return -ENOPROTOOPT;
3818
8ae55f04
KK		 3819 if (get_user(len, optlen))
3820 return -EFAULT;
1da177e4
LT		 3821
3822 if (len < 0)
3823 return -EINVAL;
1ce4f28b 3824
69e3c75f 3825 switch (optname) {
1da177e4 3826 case PACKET_STATISTICS:
1da177e4 3827 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3828 memcpy(&st, &po->stats, sizeof(st));
3829 memset(&po->stats, 0, sizeof(po->stats));
3830 spin_unlock_bh(&sk->sk_receive_queue.lock);
3831
f6fb8f10 3832 if (po->tp_version == TPACKET_V3) {
c06fff6e 3833 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3834 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3835 data = &st.stats3;
f6fb8f10 3836 } else {
c06fff6e 3837 lv = sizeof(struct tpacket_stats);
8bcdeaff 3838 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3839 data = &st.stats1;
f6fb8f10 3840 }
ee80fbf3 3841
8dc41944
HX		 3842 break;
3843 case PACKET_AUXDATA:
8dc41944 3844 val = po->auxdata;
80feaacb
PWJ		 3845 break;
3846 case PACKET_ORIGDEV:
80feaacb 3847 val = po->origdev;
bfd5f4a3
SS		 3848 break;
3849 case PACKET_VNET_HDR:
bfd5f4a3 3850 val = po->has_vnet_hdr;
1da177e4 3851 break;
bbd6ef87 3852 case PACKET_VERSION:
bbd6ef87 3853 val = po->tp_version;
bbd6ef87
PM		 3854 break;
3855 case PACKET_HDRLEN:
3856 if (len > sizeof(int))
3857 len = sizeof(int);
3858 if (copy_from_user(&val, optval, len))
3859 return -EFAULT;
3860 switch (val) {
3861 case TPACKET_V1:
3862 val = sizeof(struct tpacket_hdr);
3863 break;
3864 case TPACKET_V2:
3865 val = sizeof(struct tpacket2_hdr);
3866 break;
f6fb8f10 3867 case TPACKET_V3:
3868 val = sizeof(struct tpacket3_hdr);
3869 break;
bbd6ef87
PM		 3870 default:
3871 return -EINVAL;
3872 }
bbd6ef87 3873 break;
8913336a 3874 case PACKET_RESERVE:
8913336a 3875 val = po->tp_reserve;
8913336a 3876 break;
69e3c75f 3877 case PACKET_LOSS:
69e3c75f 3878 val = po->tp_loss;
69e3c75f 3879 break;
614f60fa 3880 case PACKET_TIMESTAMP:
614f60fa 3881 val = po->tp_tstamp;
614f60fa 3882 break;
dc99f600 3883 case PACKET_FANOUT:
dc99f600
DM		 3884 val = (po->fanout ?
3885 ((u32)po->fanout->id |
77f65ebd
WB		 3886 ((u32)po->fanout->type << 16) |
3887 ((u32)po->fanout->flags << 24)) :
dc99f600 3888 0);
dc99f600 3889 break;
a9b63918
WB		 3890 case PACKET_ROLLOVER_STATS:
3891 if (!po->rollover)
3892 return -EINVAL;
3893 rstats.tp_all = atomic_long_read(&po->rollover->num);
3894 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
3895 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
3896 data = &rstats;
3897 lv = sizeof(rstats);
3898 break;
5920cd3a
PC		 3899 case PACKET_TX_HAS_OFF:
3900 val = po->tp_tx_has_off;
3901 break;
d346a3fa
DB		 3902 case PACKET_QDISC_BYPASS:
3903 val = packet_use_direct_xmit(po);
3904 break;
1da177e4
LT		 3905 default:
3906 return -ENOPROTOOPT;
3907 }
3908
c06fff6e
ED		 3909 if (len > lv)
3910 len = lv;
8ae55f04
KK		 3911 if (put_user(len, optlen))
3912 return -EFAULT;
8dc41944
HX		 3913 if (copy_to_user(optval, data, len))
3914 return -EFAULT;
8ae55f04 3915 return 0;
1da177e4
LT		 3916}
3917
3918
351638e7
JP		 3919static int packet_notifier(struct notifier_block *this,
3920 unsigned long msg, void *ptr)
1da177e4
LT		 3921{
3922 struct sock *sk;
351638e7 3923 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3924 struct net *net = dev_net(dev);
1da177e4 3925
808f5114 3926 rcu_read_lock();
b67bfe0d 3927 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3928 struct packet_sock *po = pkt_sk(sk);
3929
3930 switch (msg) {
3931 case NETDEV_UNREGISTER:
1da177e4 3932 if (po->mclist)
82f17091 3933 packet_dev_mclist_delete(dev, &po->mclist);
a2efcfa0
DM		 3934 /* fallthrough */
3935
1da177e4
LT		 3936 case NETDEV_DOWN:
3937 if (dev->ifindex == po->ifindex) {
3938 spin_lock(&po->bind_lock);
3939 if (po->running) {
ce06b03e 3940 __unregister_prot_hook(sk, false);
1da177e4
LT		 3941 sk->sk_err = ENETDOWN;
3942 if (!sock_flag(sk, SOCK_DEAD))
3943 sk->sk_error_report(sk);
3944 }
3945 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3946 packet_cached_dev_reset(po);
1da177e4 3947 po->ifindex = -1;
160ff18a
BG		 3948 if (po->prot_hook.dev)
3949 dev_put(po->prot_hook.dev);
1da177e4
LT		 3950 po->prot_hook.dev = NULL;
3951 }
3952 spin_unlock(&po->bind_lock);
3953 }
3954 break;
3955 case NETDEV_UP:
808f5114 3956 if (dev->ifindex == po->ifindex) {
3957 spin_lock(&po->bind_lock);
ce06b03e
DM		 3958 if (po->num)
3959 register_prot_hook(sk);
808f5114 3960 spin_unlock(&po->bind_lock);
1da177e4 3961 }
1da177e4
LT		 3962 break;
3963 }
3964 }
808f5114 3965 rcu_read_unlock();
1da177e4
LT		 3966 return NOTIFY_DONE;
3967}
3968
3969
3970static int packet_ioctl(struct socket *sock, unsigned int cmd,
3971 unsigned long arg)
3972{
3973 struct sock *sk = sock->sk;
3974
69e3c75f 3975 switch (cmd) {
40d4e3df
ED		 3976 case SIOCOUTQ:
3977 {
3978 int amount = sk_wmem_alloc_get(sk);
31e6d363 3979
40d4e3df
ED		 3980 return put_user(amount, (int __user *)arg);
3981 }
3982 case SIOCINQ:
3983 {
3984 struct sk_buff *skb;
3985 int amount = 0;
3986
3987 spin_lock_bh(&sk->sk_receive_queue.lock);
3988 skb = skb_peek(&sk->sk_receive_queue);
3989 if (skb)
3990 amount = skb->len;
3991 spin_unlock_bh(&sk->sk_receive_queue.lock);
3992 return put_user(amount, (int __user *)arg);
3993 }
3994 case SIOCGSTAMP:
3995 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3996 case SIOCGSTAMPNS:
3997 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3998
1da177e4 3999#ifdef CONFIG_INET
40d4e3df
ED		 4000 case SIOCADDRT:
4001 case SIOCDELRT:
4002 case SIOCDARP:
4003 case SIOCGARP:
4004 case SIOCSARP:
4005 case SIOCGIFADDR:
4006 case SIOCSIFADDR:
4007 case SIOCGIFBRDADDR:
4008 case SIOCSIFBRDADDR:
4009 case SIOCGIFNETMASK:
4010 case SIOCSIFNETMASK:
4011 case SIOCGIFDSTADDR:
4012 case SIOCSIFDSTADDR:
4013 case SIOCSIFFLAGS:
40d4e3df 4014 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 4015#endif
4016
40d4e3df
ED		 4017 default:
4018 return -ENOIOCTLCMD;
1da177e4
LT		 4019 }
4020 return 0;
4021}
4022
40d4e3df 4023static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 4024 poll_table *wait)
4025{
4026 struct sock *sk = sock->sk;
4027 struct packet_sock *po = pkt_sk(sk);
4028 unsigned int mask = datagram_poll(file, sock, wait);
4029
4030 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 4031 if (po->rx_ring.pg_vec) {
f6fb8f10 4032 if (!packet_previous_rx_frame(po, &po->rx_ring,
4033 TP_STATUS_KERNEL))
1da177e4
LT		 4034 mask |= POLLIN | POLLRDNORM;
4035 }
2ccdbaa6 4036 if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
54d7c01d 4037 po->pressure = 0;
1da177e4 4038 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 4039 spin_lock_bh(&sk->sk_write_queue.lock);
4040 if (po->tx_ring.pg_vec) {
4041 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
4042 mask |= POLLOUT | POLLWRNORM;
4043 }
4044 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 4045 return mask;
4046}
4047
4048
4049/* Dirty? Well, I still did not learn better way to account
4050 * for user mmaps.
4051 */
4052
4053static void packet_mm_open(struct vm_area_struct *vma)
4054{
4055 struct file *file = vma->vm_file;
40d4e3df 4056 struct socket *sock = file->private_data;
1da177e4 4057 struct sock *sk = sock->sk;
1ce4f28b 4058
1da177e4
LT		 4059 if (sk)
4060 atomic_inc(&pkt_sk(sk)->mapped);
4061}
4062
4063static void packet_mm_close(struct vm_area_struct *vma)
4064{
4065 struct file *file = vma->vm_file;
40d4e3df 4066 struct socket *sock = file->private_data;
1da177e4 4067 struct sock *sk = sock->sk;
1ce4f28b 4068
1da177e4
LT		 4069 if (sk)
4070 atomic_dec(&pkt_sk(sk)->mapped);
4071}
4072
f0f37e2f 4073static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 4074 .open = packet_mm_open,
4075 .close = packet_mm_close,
1da177e4
LT		 4076};
4077
0e3125c7
NH		 4078static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
4079 unsigned int len)
1da177e4
LT		 4080{
4081 int i;
4082
4ebf0ae2 4083 for (i = 0; i < len; i++) {
0e3125c7 4084 if (likely(pg_vec[i].buffer)) {
c56b4d90 4085 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 4086 vfree(pg_vec[i].buffer);
4087 else
4088 free_pages((unsigned long)pg_vec[i].buffer,
4089 order);
4090 pg_vec[i].buffer = NULL;
4091 }
1da177e4
LT		 4092 }
4093 kfree(pg_vec);
4094}
4095
eea49cc9 4096static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 4097{
f0d4eb29 4098 char *buffer;
0e3125c7
NH		 4099 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
4100 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
4101
4102 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 4103 if (buffer)
4104 return buffer;
4105
f0d4eb29 4106 /* __get_free_pages failed, fall back to vmalloc */
bbce5a59 4107 buffer = vzalloc((1 << order) * PAGE_SIZE);
0e3125c7
NH		 4108 if (buffer)
4109 return buffer;
4110
f0d4eb29 4111 /* vmalloc failed, lets dig into swap here */
0e3125c7 4112 gfp_flags &= ~__GFP_NORETRY;
f0d4eb29 4113 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 4114 if (buffer)
4115 return buffer;
4116
f0d4eb29 4117 /* complete and utter failure */
0e3125c7 4118 return NULL;
4ebf0ae2
DM		 4119}
4120
0e3125c7 4121static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 4122{
4123 unsigned int block_nr = req->tp_block_nr;
0e3125c7 4124 struct pgv *pg_vec;
4ebf0ae2
DM		 4125 int i;
4126
0e3125c7 4127 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 4128 if (unlikely(!pg_vec))
4129 goto out;
4130
4131 for (i = 0; i < block_nr; i++) {
c56b4d90 4132 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 4133 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 4134 goto out_free_pgvec;
4135 }
4136
4137out:
4138 return pg_vec;
4139
4140out_free_pgvec:
4141 free_pg_vec(pg_vec, order, block_nr);
4142 pg_vec = NULL;
4143 goto out;
4144}
1da177e4 4145
f6fb8f10 4146static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 4147 int closing, int tx_ring)
1da177e4 4148{
0e3125c7 4149 struct pgv *pg_vec = NULL;
1da177e4 4150 struct packet_sock *po = pkt_sk(sk);
0e11c91e 4151 int was_running, order = 0;
69e3c75f
JB		 4152 struct packet_ring_buffer *rb;
4153 struct sk_buff_head *rb_queue;
0e11c91e 4154 __be16 num;
f6fb8f10 4155 int err = -EINVAL;
4156 /* Added to avoid minimal code churn */
4157 struct tpacket_req *req = &req_u->req;
4158
4159 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
4160 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
4161 WARN(1, "Tx-ring is not supported.\n");
4162 goto out;
4163 }
1ce4f28b 4164
69e3c75f
JB		 4165 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
4166 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 4167
69e3c75f
JB		 4168 err = -EBUSY;
4169 if (!closing) {
4170 if (atomic_read(&po->mapped))
4171 goto out;
b0138408 4172 if (packet_read_pending(rb))
69e3c75f
JB		 4173 goto out;
4174 }
1da177e4 4175
69e3c75f
JB		 4176 if (req->tp_block_nr) {
4177 /* Sanity tests and some calculations */
4178 err = -EBUSY;
4179 if (unlikely(rb->pg_vec))
4180 goto out;
1da177e4 4181
bbd6ef87
PM		 4182 switch (po->tp_version) {
4183 case TPACKET_V1:
4184 po->tp_hdrlen = TPACKET_HDRLEN;
4185 break;
4186 case TPACKET_V2:
4187 po->tp_hdrlen = TPACKET2_HDRLEN;
4188 break;
f6fb8f10 4189 case TPACKET_V3:
4190 po->tp_hdrlen = TPACKET3_HDRLEN;
4191 break;
bbd6ef87
PM		 4192 }
4193
69e3c75f 4194 err = -EINVAL;
4ebf0ae2 4195 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 4196 goto out;
90836b67 4197 if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
69e3c75f 4198 goto out;
dc808110
ED		 4199 if (po->tp_version >= TPACKET_V3 &&
4200 (int)(req->tp_block_size -
4201 BLK_PLUS_PRIV(req_u->req3.tp_sizeof_priv)) <= 0)
4202 goto out;
8913336a 4203 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 4204 po->tp_reserve))
4205 goto out;
4ebf0ae2 4206 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 4207 goto out;
1da177e4 4208
4194b491
TK		 4209 rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
4210 if (unlikely(rb->frames_per_block == 0))
69e3c75f
JB		 4211 goto out;
4212 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
4213 req->tp_frame_nr))
4214 goto out;
1da177e4
LT		 4215
4216 err = -ENOMEM;
4ebf0ae2
DM		 4217 order = get_order(req->tp_block_size);
4218 pg_vec = alloc_pg_vec(req, order);
4219 if (unlikely(!pg_vec))
1da177e4 4220 goto out;
f6fb8f10 4221 switch (po->tp_version) {
4222 case TPACKET_V3:
4223 /* Transmit path is not supported. We checked
4224 * it above but just being paranoid
4225 */
4226 if (!tx_ring)
e8e85cc5 4227 init_prb_bdqc(po, rb, pg_vec, req_u);
d7cf0c34 4228 break;
f6fb8f10 4229 default:
4230 break;
4231 }
69e3c75f
JB		 4232 }
4233 /* Done */
4234 else {
4235 err = -EINVAL;
4ebf0ae2 4236 if (unlikely(req->tp_frame_nr))
69e3c75f 4237 goto out;
1da177e4
LT		 4238 }
4239
4240 lock_sock(sk);
4241
4242 /* Detach socket from network */
4243 spin_lock(&po->bind_lock);
4244 was_running = po->running;
4245 num = po->num;
4246 if (was_running) {
1da177e4 4247 po->num = 0;
ce06b03e 4248 __unregister_prot_hook(sk, false);
1da177e4
LT		 4249 }
4250 spin_unlock(&po->bind_lock);
1ce4f28b 4251
1da177e4
LT		 4252 synchronize_net();
4253
4254 err = -EBUSY;
905db440 4255 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 4256 if (closing || atomic_read(&po->mapped) == 0) {
4257 err = 0;
69e3c75f 4258 spin_lock_bh(&rb_queue->lock);
c053fd96 4259 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 4260 rb->frame_max = (req->tp_frame_nr - 1);
4261 rb->head = 0;
4262 rb->frame_size = req->tp_frame_size;
4263 spin_unlock_bh(&rb_queue->lock);
4264
c053fd96
CG		 4265 swap(rb->pg_vec_order, order);
4266 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 4267
4268 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4269 po->prot_hook.func = (po->rx_ring.pg_vec) ?
4270 tpacket_rcv : packet_rcv;
4271 skb_queue_purge(rb_queue);
1da177e4 4272 if (atomic_read(&po->mapped))
40d4e3df
ED		 4273 pr_err("packet_mmap: vma is busy: %d\n",
4274 atomic_read(&po->mapped));
1da177e4 4275 }
905db440 4276 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 4277
4278 spin_lock(&po->bind_lock);
ce06b03e 4279 if (was_running) {
1da177e4 4280 po->num = num;
ce06b03e 4281 register_prot_hook(sk);
1da177e4
LT		 4282 }
4283 spin_unlock(&po->bind_lock);
f6fb8f10 4284 if (closing && (po->tp_version > TPACKET_V2)) {
4285 /* Because we don't support block-based V3 on tx-ring */
4286 if (!tx_ring)
73d0fcf2 4287 prb_shutdown_retire_blk_timer(po, rb_queue);
f6fb8f10 4288 }
1da177e4
LT		 4289 release_sock(sk);
4290
1da177e4
LT		 4291 if (pg_vec)
4292 free_pg_vec(pg_vec, order, req->tp_block_nr);
4293out:
4294 return err;
4295}
4296
69e3c75f
JB		 4297static int packet_mmap(struct file *file, struct socket *sock,
4298 struct vm_area_struct *vma)
1da177e4
LT		 4299{
4300 struct sock *sk = sock->sk;
4301 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 4302 unsigned long size, expected_size;
4303 struct packet_ring_buffer *rb;
1da177e4
LT		 4304 unsigned long start;
4305 int err = -EINVAL;
4306 int i;
4307
4308 if (vma->vm_pgoff)
4309 return -EINVAL;
4310
905db440 4311 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 4312
4313 expected_size = 0;
4314 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4315 if (rb->pg_vec) {
4316 expected_size += rb->pg_vec_len
4317 * rb->pg_vec_pages
4318 * PAGE_SIZE;
4319 }
4320 }
4321
4322 if (expected_size == 0)
1da177e4 4323 goto out;
69e3c75f
JB		 4324
4325 size = vma->vm_end - vma->vm_start;
4326 if (size != expected_size)
1da177e4
LT		 4327 goto out;
4328
1da177e4 4329 start = vma->vm_start;
69e3c75f
JB		 4330 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4331 if (rb->pg_vec == NULL)
4332 continue;
4333
4334 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 4335 struct page *page;
4336 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 4337 int pg_num;
4338
c56b4d90
CG		 4339 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4340 page = pgv_to_page(kaddr);
69e3c75f
JB		 4341 err = vm_insert_page(vma, start, page);
4342 if (unlikely(err))
4343 goto out;
4344 start += PAGE_SIZE;
0e3125c7 4345 kaddr += PAGE_SIZE;
69e3c75f 4346 }
4ebf0ae2 4347 }
1da177e4 4348 }
69e3c75f 4349
4ebf0ae2 4350 atomic_inc(&po->mapped);
1da177e4
LT		 4351 vma->vm_ops = &packet_mmap_ops;
4352 err = 0;
4353
4354out:
905db440 4355 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 4356 return err;
4357}
1da177e4 4358
90ddc4f0 4359static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 4360 .family = PF_PACKET,
4361 .owner = THIS_MODULE,
4362 .release = packet_release,
4363 .bind = packet_bind_spkt,
4364 .connect = sock_no_connect,
4365 .socketpair = sock_no_socketpair,
4366 .accept = sock_no_accept,
4367 .getname = packet_getname_spkt,
4368 .poll = datagram_poll,
4369 .ioctl = packet_ioctl,
4370 .listen = sock_no_listen,
4371 .shutdown = sock_no_shutdown,
4372 .setsockopt = sock_no_setsockopt,
4373 .getsockopt = sock_no_getsockopt,
4374 .sendmsg = packet_sendmsg_spkt,
4375 .recvmsg = packet_recvmsg,
4376 .mmap = sock_no_mmap,
4377 .sendpage = sock_no_sendpage,
4378};
1da177e4 4379
90ddc4f0 4380static const struct proto_ops packet_ops = {
1da177e4
LT		 4381 .family = PF_PACKET,
4382 .owner = THIS_MODULE,
4383 .release = packet_release,
4384 .bind = packet_bind,
4385 .connect = sock_no_connect,
4386 .socketpair = sock_no_socketpair,
4387 .accept = sock_no_accept,
1ce4f28b 4388 .getname = packet_getname,
1da177e4
LT		 4389 .poll = packet_poll,
4390 .ioctl = packet_ioctl,
4391 .listen = sock_no_listen,
4392 .shutdown = sock_no_shutdown,
4393 .setsockopt = packet_setsockopt,
4394 .getsockopt = packet_getsockopt,
4395 .sendmsg = packet_sendmsg,
4396 .recvmsg = packet_recvmsg,
4397 .mmap = packet_mmap,
4398 .sendpage = sock_no_sendpage,
4399};
4400
ec1b4cf7 4401static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 4402 .family = PF_PACKET,
4403 .create = packet_create,
4404 .owner = THIS_MODULE,
4405};
4406
4407static struct notifier_block packet_netdev_notifier = {
40d4e3df 4408 .notifier_call = packet_notifier,
1da177e4
LT		 4409};
4410
4411#ifdef CONFIG_PROC_FS
1da177e4
LT		 4412
4413static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 4414 __acquires(RCU)
1da177e4 4415{
e372c414 4416 struct net *net = seq_file_net(seq);
808f5114 4417
4418 rcu_read_lock();
4419 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 4420}
4421
4422static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4423{
1bf40954 4424 struct net *net = seq_file_net(seq);
808f5114 4425 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 4426}
4427
4428static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 4429 __releases(RCU)
1da177e4 4430{
808f5114 4431 rcu_read_unlock();
1da177e4
LT		 4432}
4433
1ce4f28b 4434static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 4435{
4436 if (v == SEQ_START_TOKEN)
4437 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4438 else {
b7ceabd9 4439 struct sock *s = sk_entry(v);
1da177e4
LT		 4440 const struct packet_sock *po = pkt_sk(s);
4441
4442 seq_printf(seq,
71338aa7 4443 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 4444 s,
4445 atomic_read(&s->sk_refcnt),
4446 s->sk_type,
4447 ntohs(po->num),
4448 po->ifindex,
4449 po->running,
4450 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4451 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4452 sock_i_ino(s));
1da177e4
LT		 4453 }
4454
4455 return 0;
4456}
4457
56b3d975 4458static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4459 .start = packet_seq_start,
4460 .next = packet_seq_next,
4461 .stop = packet_seq_stop,
4462 .show = packet_seq_show,
4463};
4464
4465static int packet_seq_open(struct inode *inode, struct file *file)
4466{
e372c414
DL		 4467 return seq_open_net(inode, file, &packet_seq_ops,
4468 sizeof(struct seq_net_private));
1da177e4
LT		 4469}
4470
da7071d7 4471static const struct file_operations packet_seq_fops = {
1da177e4
LT		 4472 .owner = THIS_MODULE,
4473 .open = packet_seq_open,
4474 .read = seq_read,
4475 .llseek = seq_lseek,
e372c414 4476 .release = seq_release_net,
1da177e4
LT		 4477};
4478
4479#endif
4480
2c8c1e72 4481static int __net_init packet_net_init(struct net *net)
d12d01d6 4482{
0fa7fa98 4483 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4484 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4485
d4beaa66 4486 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 4487 return -ENOMEM;
4488
4489 return 0;
4490}
4491
2c8c1e72 4492static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4493{
ece31ffd 4494 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 4495}
4496
4497static struct pernet_operations packet_net_ops = {
4498 .init = packet_net_init,
4499 .exit = packet_net_exit,
4500};
4501
4502
1da177e4
LT		 4503static void __exit packet_exit(void)
4504{
1da177e4 4505 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4506 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4507 sock_unregister(PF_PACKET);
4508 proto_unregister(&packet_proto);
4509}
4510
4511static int __init packet_init(void)
4512{
4513 int rc = proto_register(&packet_proto, 0);
4514
4515 if (rc != 0)
4516 goto out;
4517
4518 sock_register(&packet_family_ops);
d12d01d6 4519 register_pernet_subsys(&packet_net_ops);
1da177e4 4520 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4521out:
4522 return rc;
4523}
4524
4525module_init(packet_init);
4526module_exit(packet_exit);
4527MODULE_LICENSE("GPL");
4528MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 1.772075 seconds and 5 git commands to generate.