projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'dmaengine-fixes-3.14-rc4' of git://git.kernel.org/pub/scm/linux/kernel...
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
b0138408 91#include <linux/percpu.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
d346a3fa
DB		 240static int packet_direct_xmit(struct sk_buff *skb)
241{
242 struct net_device *dev = skb->dev;
243 const struct net_device_ops *ops = dev->netdev_ops;
244 netdev_features_t features;
245 struct netdev_queue *txq;
246 u16 queue_map;
247 int ret;
248
249 if (unlikely(!netif_running(dev) ||
250 !netif_carrier_ok(dev))) {
251 kfree_skb(skb);
252 return NET_XMIT_DROP;
253 }
254
255 features = netif_skb_features(skb);
256 if (skb_needs_linearize(skb, features) &&
257 __skb_linearize(skb)) {
258 kfree_skb(skb);
259 return NET_XMIT_DROP;
260 }
261
262 queue_map = skb_get_queue_mapping(skb);
263 txq = netdev_get_tx_queue(dev, queue_map);
264
265 __netif_tx_lock_bh(txq);
266 if (unlikely(netif_xmit_frozen_or_stopped(txq))) {
267 ret = NETDEV_TX_BUSY;
268 kfree_skb(skb);
269 goto out;
270 }
271
272 ret = ops->ndo_start_xmit(skb, dev);
273 if (likely(dev_xmit_complete(ret)))
274 txq_trans_update(txq);
275 else
276 kfree_skb(skb);
277out:
278 __netif_tx_unlock_bh(txq);
279 return ret;
280}
281
66e56cd4
DB		 282static struct net_device *packet_cached_dev_get(struct packet_sock *po)
283{
284 struct net_device *dev;
285
286 rcu_read_lock();
287 dev = rcu_dereference(po->cached_dev);
288 if (likely(dev))
289 dev_hold(dev);
290 rcu_read_unlock();
291
292 return dev;
293}
294
295static void packet_cached_dev_assign(struct packet_sock *po,
296 struct net_device *dev)
297{
298 rcu_assign_pointer(po->cached_dev, dev);
299}
300
301static void packet_cached_dev_reset(struct packet_sock *po)
302{
303 RCU_INIT_POINTER(po->cached_dev, NULL);
304}
305
d346a3fa
DB		 306static bool packet_use_direct_xmit(const struct packet_sock *po)
307{
308 return po->xmit == packet_direct_xmit;
309}
310
0fd5d57b 311static u16 __packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
d346a3fa 312{
1cbac010 313 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 314}
315
0fd5d57b
DB		 316static void packet_pick_tx_queue(struct net_device *dev, struct sk_buff *skb)
317{
318 const struct net_device_ops *ops = dev->netdev_ops;
319 u16 queue_index;
320
321 if (ops->ndo_select_queue) {
322 queue_index = ops->ndo_select_queue(dev, skb, NULL,
323 __packet_pick_tx_queue);
324 queue_index = netdev_cap_txqueue(dev, queue_index);
325 } else {
326 queue_index = __packet_pick_tx_queue(dev, skb);
327 }
328
329 skb_set_queue_mapping(skb, queue_index);
330}
331
ce06b03e
DM		 332/* register_prot_hook must be invoked with the po->bind_lock held,
333 * or from a context in which asynchronous accesses to the packet
334 * socket is not possible (packet_create()).
335 */
336static void register_prot_hook(struct sock *sk)
337{
338 struct packet_sock *po = pkt_sk(sk);
e40526cb 339
ce06b03e 340 if (!po->running) {
66e56cd4 341 if (po->fanout)
dc99f600 342 __fanout_link(sk, po);
66e56cd4 343 else
dc99f600 344 dev_add_pack(&po->prot_hook);
e40526cb 345
ce06b03e
DM		 346 sock_hold(sk);
347 po->running = 1;
348 }
349}
350
351/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
352 * held. If the sync parameter is true, we will temporarily drop
353 * the po->bind_lock and do a synchronize_net to make sure no
354 * asynchronous packet processing paths still refer to the elements
355 * of po->prot_hook. If the sync parameter is false, it is the
356 * callers responsibility to take care of this.
357 */
358static void __unregister_prot_hook(struct sock *sk, bool sync)
359{
360 struct packet_sock *po = pkt_sk(sk);
361
362 po->running = 0;
66e56cd4
DB		 363
364 if (po->fanout)
dc99f600 365 __fanout_unlink(sk, po);
66e56cd4 366 else
dc99f600 367 __dev_remove_pack(&po->prot_hook);
e40526cb 368
ce06b03e
DM		 369 __sock_put(sk);
370
371 if (sync) {
372 spin_unlock(&po->bind_lock);
373 synchronize_net();
374 spin_lock(&po->bind_lock);
375 }
376}
377
378static void unregister_prot_hook(struct sock *sk, bool sync)
379{
380 struct packet_sock *po = pkt_sk(sk);
381
382 if (po->running)
383 __unregister_prot_hook(sk, sync);
384}
385
f6dafa95 386static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 387{
388 if (is_vmalloc_addr(addr))
389 return vmalloc_to_page(addr);
390 return virt_to_page(addr);
391}
392
69e3c75f 393static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 394{
184f489e 395 union tpacket_uhdr h;
1da177e4 396
69e3c75f 397 h.raw = frame;
bbd6ef87
PM		 398 switch (po->tp_version) {
399 case TPACKET_V1:
69e3c75f 400 h.h1->tp_status = status;
0af55bb5 401 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 402 break;
403 case TPACKET_V2:
69e3c75f 404 h.h2->tp_status = status;
0af55bb5 405 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 406 break;
f6fb8f10 407 case TPACKET_V3:
69e3c75f 408 default:
f6fb8f10 409 WARN(1, "TPACKET version not supported.\n");
69e3c75f 410 BUG();
bbd6ef87 411 }
69e3c75f
JB		 412
413 smp_wmb();
bbd6ef87
PM		 414}
415
69e3c75f 416static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 417{
184f489e 418 union tpacket_uhdr h;
bbd6ef87 419
69e3c75f
JB		 420 smp_rmb();
421
bbd6ef87
PM		 422 h.raw = frame;
423 switch (po->tp_version) {
424 case TPACKET_V1:
0af55bb5 425 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 426 return h.h1->tp_status;
bbd6ef87 427 case TPACKET_V2:
0af55bb5 428 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 429 return h.h2->tp_status;
f6fb8f10 430 case TPACKET_V3:
69e3c75f 431 default:
f6fb8f10 432 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 433 BUG();
434 return 0;
bbd6ef87 435 }
1da177e4 436}
69e3c75f 437
b9c32fb2
DB		 438static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
439 unsigned int flags)
7a51384c
DB		 440{
441 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
442
443 if (shhwtstamps) {
444 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
445 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
b9c32fb2 446 return TP_STATUS_TS_SYS_HARDWARE;
7a51384c
DB		 447 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
448 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
b9c32fb2 449 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 450 }
451
452 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 453 return TP_STATUS_TS_SOFTWARE;
7a51384c 454
b9c32fb2 455 return 0;
7a51384c
DB		 456}
457
b9c32fb2
DB		 458static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
459 struct sk_buff *skb)
2e31396f
WB		 460{
461 union tpacket_uhdr h;
462 struct timespec ts;
b9c32fb2 463 __u32 ts_status;
2e31396f 464
b9c32fb2
DB		 465 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
466 return 0;
2e31396f
WB		 467
468 h.raw = frame;
469 switch (po->tp_version) {
470 case TPACKET_V1:
471 h.h1->tp_sec = ts.tv_sec;
472 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
473 break;
474 case TPACKET_V2:
475 h.h2->tp_sec = ts.tv_sec;
476 h.h2->tp_nsec = ts.tv_nsec;
477 break;
478 case TPACKET_V3:
479 default:
480 WARN(1, "TPACKET version not supported.\n");
481 BUG();
482 }
483
484 /* one flush is safe, as both fields always lie on the same cacheline */
485 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
486 smp_wmb();
b9c32fb2
DB		 487
488 return ts_status;
2e31396f
WB		 489}
490
69e3c75f
JB		 491static void *packet_lookup_frame(struct packet_sock *po,
492 struct packet_ring_buffer *rb,
493 unsigned int position,
494 int status)
495{
496 unsigned int pg_vec_pos, frame_offset;
184f489e 497 union tpacket_uhdr h;
69e3c75f
JB		 498
499 pg_vec_pos = position / rb->frames_per_block;
500 frame_offset = position % rb->frames_per_block;
501
0e3125c7
NH		 502 h.raw = rb->pg_vec[pg_vec_pos].buffer +
503 (frame_offset * rb->frame_size);
69e3c75f
JB		 504
505 if (status != __packet_get_status(po, h.raw))
506 return NULL;
507
508 return h.raw;
509}
510
eea49cc9 511static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 512 struct packet_ring_buffer *rb,
513 int status)
514{
515 return packet_lookup_frame(po, rb, rb->head, status);
516}
517
bc59ba39 518static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 519{
520 del_timer_sync(&pkc->retire_blk_timer);
521}
522
523static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
524 int tx_ring,
525 struct sk_buff_head *rb_queue)
526{
bc59ba39 527 struct tpacket_kbdq_core *pkc;
f6fb8f10 528
22781a5b
DJ		 529 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
530 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 531
ec6f809f 532 spin_lock_bh(&rb_queue->lock);
f6fb8f10 533 pkc->delete_blk_timer = 1;
ec6f809f 534 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 535
536 prb_del_retire_blk_timer(pkc);
537}
538
539static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 540 struct tpacket_kbdq_core *pkc,
f6fb8f10 541 void (*func) (unsigned long))
542{
543 init_timer(&pkc->retire_blk_timer);
544 pkc->retire_blk_timer.data = (long)po;
545 pkc->retire_blk_timer.function = func;
546 pkc->retire_blk_timer.expires = jiffies;
547}
548
549static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
550{
bc59ba39 551 struct tpacket_kbdq_core *pkc;
f6fb8f10 552
553 if (tx_ring)
554 BUG();
555
22781a5b
DJ		 556 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
557 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 558 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
559}
560
561static int prb_calc_retire_blk_tmo(struct packet_sock *po,
562 int blk_size_in_bytes)
563{
564 struct net_device *dev;
565 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 566 struct ethtool_cmd ecmd;
567 int err;
e440cf2c 568 u32 speed;
f6fb8f10 569
4bc71cb9
JP		 570 rtnl_lock();
571 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
572 if (unlikely(!dev)) {
573 rtnl_unlock();
f6fb8f10 574 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 575 }
576 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 577 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 578 rtnl_unlock();
579 if (!err) {
4bc71cb9
JP		 580 /*
581 * If the link speed is so slow you don't really
582 * need to worry about perf anyways
583 */
e440cf2c 584 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 585 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 586 } else {
587 msec = 1;
588 div = speed / 1000;
f6fb8f10 589 }
590 }
591
592 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
593
594 if (div)
595 mbits /= div;
596
597 tmo = mbits * msec;
598
599 if (div)
600 return tmo+1;
601 return tmo;
602}
603
bc59ba39 604static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 605 union tpacket_req_u *req_u)
606{
607 p1->feature_req_word = req_u->req3.tp_feature_req_word;
608}
609
610static void init_prb_bdqc(struct packet_sock *po,
611 struct packet_ring_buffer *rb,
612 struct pgv *pg_vec,
613 union tpacket_req_u *req_u, int tx_ring)
614{
22781a5b 615 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 616 struct tpacket_block_desc *pbd;
f6fb8f10 617
618 memset(p1, 0x0, sizeof(*p1));
619
620 p1->knxt_seq_num = 1;
621 p1->pkbdq = pg_vec;
bc59ba39 622 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 623 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 624 p1->kblk_size = req_u->req3.tp_block_size;
625 p1->knum_blocks = req_u->req3.tp_block_nr;
626 p1->hdrlen = po->tp_hdrlen;
627 p1->version = po->tp_version;
628 p1->last_kactive_blk_num = 0;
ee80fbf3 629 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 630 if (req_u->req3.tp_retire_blk_tov)
631 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
632 else
633 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
634 req_u->req3.tp_block_size);
635 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
636 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
637
638 prb_init_ft_ops(p1, req_u);
639 prb_setup_retire_blk_timer(po, tx_ring);
640 prb_open_block(p1, pbd);
641}
642
643/* Do NOT update the last_blk_num first.
644 * Assumes sk_buff_head lock is held.
645 */
bc59ba39 646static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 647{
648 mod_timer(&pkc->retire_blk_timer,
649 jiffies + pkc->tov_in_jiffies);
650 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
651}
652
653/*
654 * Timer logic:
655 * 1) We refresh the timer only when we open a block.
656 * By doing this we don't waste cycles refreshing the timer
657 * on packet-by-packet basis.
658 *
659 * With a 1MB block-size, on a 1Gbps line, it will take
660 * i) ~8 ms to fill a block + ii) memcpy etc.
661 * In this cut we are not accounting for the memcpy time.
662 *
663 * So, if the user sets the 'tmo' to 10ms then the timer
664 * will never fire while the block is still getting filled
665 * (which is what we want). However, the user could choose
666 * to close a block early and that's fine.
667 *
668 * But when the timer does fire, we check whether or not to refresh it.
669 * Since the tmo granularity is in msecs, it is not too expensive
670 * to refresh the timer, lets say every '8' msecs.
671 * Either the user can set the 'tmo' or we can derive it based on
672 * a) line-speed and b) block-size.
673 * prb_calc_retire_blk_tmo() calculates the tmo.
674 *
675 */
676static void prb_retire_rx_blk_timer_expired(unsigned long data)
677{
678 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 679 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 680 unsigned int frozen;
bc59ba39 681 struct tpacket_block_desc *pbd;
f6fb8f10 682
683 spin_lock(&po->sk.sk_receive_queue.lock);
684
685 frozen = prb_queue_frozen(pkc);
686 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
687
688 if (unlikely(pkc->delete_blk_timer))
689 goto out;
690
691 /* We only need to plug the race when the block is partially filled.
692 * tpacket_rcv:
693 * lock(); increment BLOCK_NUM_PKTS; unlock()
694 * copy_bits() is in progress ...
695 * timer fires on other cpu:
696 * we can't retire the current block because copy_bits
697 * is in progress.
698 *
699 */
700 if (BLOCK_NUM_PKTS(pbd)) {
701 while (atomic_read(&pkc->blk_fill_in_prog)) {
702 /* Waiting for skb_copy_bits to finish... */
703 cpu_relax();
704 }
705 }
706
707 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
708 if (!frozen) {
709 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
710 if (!prb_dispatch_next_block(pkc, po))
711 goto refresh_timer;
712 else
713 goto out;
714 } else {
715 /* Case 1. Queue was frozen because user-space was
716 * lagging behind.
717 */
718 if (prb_curr_blk_in_use(pkc, pbd)) {
719 /*
720 * Ok, user-space is still behind.
721 * So just refresh the timer.
722 */
723 goto refresh_timer;
724 } else {
725 /* Case 2. queue was frozen,user-space caught up,
726 * now the link went idle && the timer fired.
727 * We don't have a block to close.So we open this
728 * block and restart the timer.
729 * opening a block thaws the queue,restarts timer
730 * Thawing/timer-refresh is a side effect.
731 */
732 prb_open_block(pkc, pbd);
733 goto out;
734 }
735 }
736 }
737
738refresh_timer:
739 _prb_refresh_rx_retire_blk_timer(pkc);
740
741out:
742 spin_unlock(&po->sk.sk_receive_queue.lock);
743}
744
eea49cc9 745static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 746 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 747{
748 /* Flush everything minus the block header */
749
750#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
751 u8 *start, *end;
752
753 start = (u8 *)pbd1;
754
755 /* Skip the block header(we know header WILL fit in 4K) */
756 start += PAGE_SIZE;
757
758 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
759 for (; start < end; start += PAGE_SIZE)
760 flush_dcache_page(pgv_to_page(start));
761
762 smp_wmb();
763#endif
764
765 /* Now update the block status. */
766
767 BLOCK_STATUS(pbd1) = status;
768
769 /* Flush the block header */
770
771#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
772 start = (u8 *)pbd1;
773 flush_dcache_page(pgv_to_page(start));
774
775 smp_wmb();
776#endif
777}
778
779/*
780 * Side effect:
781 *
782 * 1) flush the block
783 * 2) Increment active_blk_num
784 *
785 * Note:We DONT refresh the timer on purpose.
786 * Because almost always the next block will be opened.
787 */
bc59ba39 788static void prb_close_block(struct tpacket_kbdq_core *pkc1,
789 struct tpacket_block_desc *pbd1,
f6fb8f10 790 struct packet_sock *po, unsigned int stat)
791{
792 __u32 status = TP_STATUS_USER | stat;
793
794 struct tpacket3_hdr *last_pkt;
bc59ba39 795 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 796
ee80fbf3 797 if (po->stats.stats3.tp_drops)
f6fb8f10 798 status |= TP_STATUS_LOSING;
799
800 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
801 last_pkt->tp_next_offset = 0;
802
803 /* Get the ts of the last pkt */
804 if (BLOCK_NUM_PKTS(pbd1)) {
805 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
806 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
807 } else {
808 /* Ok, we tmo'd - so get the current time */
809 struct timespec ts;
810 getnstimeofday(&ts);
811 h1->ts_last_pkt.ts_sec = ts.tv_sec;
812 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
813 }
814
815 smp_wmb();
816
817 /* Flush the block */
818 prb_flush_block(pkc1, pbd1, status);
819
820 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
821}
822
eea49cc9 823static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 824{
825 pkc->reset_pending_on_curr_blk = 0;
826}
827
828/*
829 * Side effect of opening a block:
830 *
831 * 1) prb_queue is thawed.
832 * 2) retire_blk_timer is refreshed.
833 *
834 */
bc59ba39 835static void prb_open_block(struct tpacket_kbdq_core *pkc1,
836 struct tpacket_block_desc *pbd1)
f6fb8f10 837{
838 struct timespec ts;
bc59ba39 839 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 840
841 smp_rmb();
842
8da3056c
DB		 843 /* We could have just memset this but we will lose the
844 * flexibility of making the priv area sticky
845 */
f6fb8f10 846
8da3056c
DB		 847 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
848 BLOCK_NUM_PKTS(pbd1) = 0;
849 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 850
8da3056c
DB		 851 getnstimeofday(&ts);
852
853 h1->ts_first_pkt.ts_sec = ts.tv_sec;
854 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 855
8da3056c
DB		 856 pkc1->pkblk_start = (char *)pbd1;
857 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
858
859 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
860 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
861
862 pbd1->version = pkc1->version;
863 pkc1->prev = pkc1->nxt_offset;
864 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
865
866 prb_thaw_queue(pkc1);
867 _prb_refresh_rx_retire_blk_timer(pkc1);
868
869 smp_wmb();
f6fb8f10 870}
871
872/*
873 * Queue freeze logic:
874 * 1) Assume tp_block_nr = 8 blocks.
875 * 2) At time 't0', user opens Rx ring.
876 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
877 * 4) user-space is either sleeping or processing block '0'.
878 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
879 * it will close block-7,loop around and try to fill block '0'.
880 * call-flow:
881 * __packet_lookup_frame_in_block
882 * prb_retire_current_block()
883 * prb_dispatch_next_block()
884 * |->(BLOCK_STATUS == USER) evaluates to true
885 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
886 * 6) Now there are two cases:
887 * 6.1) Link goes idle right after the queue is frozen.
888 * But remember, the last open_block() refreshed the timer.
889 * When this timer expires,it will refresh itself so that we can
890 * re-open block-0 in near future.
891 * 6.2) Link is busy and keeps on receiving packets. This is a simple
892 * case and __packet_lookup_frame_in_block will check if block-0
893 * is free and can now be re-used.
894 */
eea49cc9 895static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 896 struct packet_sock *po)
897{
898 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 899 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 900}
901
902#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
903
904/*
905 * If the next block is free then we will dispatch it
906 * and return a good offset.
907 * Else, we will freeze the queue.
908 * So, caller must check the return value.
909 */
bc59ba39 910static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 911 struct packet_sock *po)
912{
bc59ba39 913 struct tpacket_block_desc *pbd;
f6fb8f10 914
915 smp_rmb();
916
917 /* 1. Get current block num */
918 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
919
920 /* 2. If this block is currently in_use then freeze the queue */
921 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
922 prb_freeze_queue(pkc, po);
923 return NULL;
924 }
925
926 /*
927 * 3.
928 * open this block and return the offset where the first packet
929 * needs to get stored.
930 */
931 prb_open_block(pkc, pbd);
932 return (void *)pkc->nxt_offset;
933}
934
bc59ba39 935static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 936 struct packet_sock *po, unsigned int status)
937{
bc59ba39 938 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 939
940 /* retire/close the current block */
941 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
942 /*
943 * Plug the case where copy_bits() is in progress on
944 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
945 * have space to copy the pkt in the current block and
946 * called prb_retire_current_block()
947 *
948 * We don't need to worry about the TMO case because
949 * the timer-handler already handled this case.
950 */
951 if (!(status & TP_STATUS_BLK_TMO)) {
952 while (atomic_read(&pkc->blk_fill_in_prog)) {
953 /* Waiting for skb_copy_bits to finish... */
954 cpu_relax();
955 }
956 }
957 prb_close_block(pkc, pbd, po, status);
958 return;
959 }
f6fb8f10 960}
961
eea49cc9 962static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 963 struct tpacket_block_desc *pbd)
f6fb8f10 964{
965 return TP_STATUS_USER & BLOCK_STATUS(pbd);
966}
967
eea49cc9 968static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 969{
970 return pkc->reset_pending_on_curr_blk;
971}
972
eea49cc9 973static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 974{
bc59ba39 975 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 976 atomic_dec(&pkc->blk_fill_in_prog);
977}
978
eea49cc9 979static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 980 struct tpacket3_hdr *ppd)
981{
3958afa1 982 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 983}
984
eea49cc9 985static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 986 struct tpacket3_hdr *ppd)
987{
988 ppd->hv1.tp_rxhash = 0;
989}
990
eea49cc9 991static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 992 struct tpacket3_hdr *ppd)
993{
994 if (vlan_tx_tag_present(pkc->skb)) {
995 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
a0cdfcf3
AW		 996 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
997 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 998 } else {
9e67030a 999 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 1000 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 1001 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 1002 }
1003}
1004
bc59ba39 1005static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 1006 struct tpacket3_hdr *ppd)
1007{
a0cdfcf3 1008 ppd->hv1.tp_padding = 0;
f6fb8f10 1009 prb_fill_vlan_info(pkc, ppd);
1010
1011 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1012 prb_fill_rxhash(pkc, ppd);
1013 else
1014 prb_clear_rxhash(pkc, ppd);
1015}
1016
eea49cc9 1017static void prb_fill_curr_block(char *curr,
bc59ba39 1018 struct tpacket_kbdq_core *pkc,
1019 struct tpacket_block_desc *pbd,
f6fb8f10 1020 unsigned int len)
1021{
1022 struct tpacket3_hdr *ppd;
1023
1024 ppd = (struct tpacket3_hdr *)curr;
1025 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1026 pkc->prev = curr;
1027 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1028 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1029 BLOCK_NUM_PKTS(pbd) += 1;
1030 atomic_inc(&pkc->blk_fill_in_prog);
1031 prb_run_all_ft_ops(pkc, ppd);
1032}
1033
1034/* Assumes caller has the sk->rx_queue.lock */
1035static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1036 struct sk_buff *skb,
1037 int status,
1038 unsigned int len
1039 )
1040{
bc59ba39 1041 struct tpacket_kbdq_core *pkc;
1042 struct tpacket_block_desc *pbd;
f6fb8f10 1043 char *curr, *end;
1044
e3192690 1045 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1046 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1047
1048 /* Queue is frozen when user space is lagging behind */
1049 if (prb_queue_frozen(pkc)) {
1050 /*
1051 * Check if that last block which caused the queue to freeze,
1052 * is still in_use by user-space.
1053 */
1054 if (prb_curr_blk_in_use(pkc, pbd)) {
1055 /* Can't record this packet */
1056 return NULL;
1057 } else {
1058 /*
1059 * Ok, the block was released by user-space.
1060 * Now let's open that block.
1061 * opening a block also thaws the queue.
1062 * Thawing is a side effect.
1063 */
1064 prb_open_block(pkc, pbd);
1065 }
1066 }
1067
1068 smp_mb();
1069 curr = pkc->nxt_offset;
1070 pkc->skb = skb;
e3192690 1071 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1072
1073 /* first try the current block */
1074 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1075 prb_fill_curr_block(curr, pkc, pbd, len);
1076 return (void *)curr;
1077 }
1078
1079 /* Ok, close the current block */
1080 prb_retire_current_block(pkc, po, 0);
1081
1082 /* Now, try to dispatch the next block */
1083 curr = (char *)prb_dispatch_next_block(pkc, po);
1084 if (curr) {
1085 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1086 prb_fill_curr_block(curr, pkc, pbd, len);
1087 return (void *)curr;
1088 }
1089
1090 /*
1091 * No free blocks are available.user_space hasn't caught up yet.
1092 * Queue was just frozen and now this packet will get dropped.
1093 */
1094 return NULL;
1095}
1096
eea49cc9 1097static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1098 struct sk_buff *skb,
1099 int status, unsigned int len)
1100{
1101 char *curr = NULL;
1102 switch (po->tp_version) {
1103 case TPACKET_V1:
1104 case TPACKET_V2:
1105 curr = packet_lookup_frame(po, &po->rx_ring,
1106 po->rx_ring.head, status);
1107 return curr;
1108 case TPACKET_V3:
1109 return __packet_lookup_frame_in_block(po, skb, status, len);
1110 default:
1111 WARN(1, "TPACKET version not supported\n");
1112 BUG();
99aa3473 1113 return NULL;
f6fb8f10 1114 }
1115}
1116
eea49cc9 1117static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1118 struct packet_ring_buffer *rb,
77f65ebd 1119 unsigned int idx,
f6fb8f10 1120 int status)
1121{
bc59ba39 1122 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1123 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1124
1125 if (status != BLOCK_STATUS(pbd))
1126 return NULL;
1127 return pbd;
1128}
1129
eea49cc9 1130static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1131{
1132 unsigned int prev;
1133 if (rb->prb_bdqc.kactive_blk_num)
1134 prev = rb->prb_bdqc.kactive_blk_num-1;
1135 else
1136 prev = rb->prb_bdqc.knum_blocks-1;
1137 return prev;
1138}
1139
1140/* Assumes caller has held the rx_queue.lock */
eea49cc9 1141static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1142 struct packet_ring_buffer *rb,
1143 int status)
1144{
1145 unsigned int previous = prb_previous_blk_num(rb);
1146 return prb_lookup_block(po, rb, previous, status);
1147}
1148
eea49cc9 1149static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1150 struct packet_ring_buffer *rb,
1151 int status)
1152{
1153 if (po->tp_version <= TPACKET_V2)
1154 return packet_previous_frame(po, rb, status);
1155
1156 return __prb_previous_block(po, rb, status);
1157}
1158
eea49cc9 1159static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1160 struct packet_ring_buffer *rb)
1161{
1162 switch (po->tp_version) {
1163 case TPACKET_V1:
1164 case TPACKET_V2:
1165 return packet_increment_head(rb);
1166 case TPACKET_V3:
1167 default:
1168 WARN(1, "TPACKET version not supported.\n");
1169 BUG();
1170 return;
1171 }
1172}
1173
eea49cc9 1174static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1175 struct packet_ring_buffer *rb,
1176 int status)
1177{
1178 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1179 return packet_lookup_frame(po, rb, previous, status);
1180}
1181
eea49cc9 1182static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1183{
1184 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1185}
1186
b0138408
DB		 1187static void packet_inc_pending(struct packet_ring_buffer *rb)
1188{
1189 this_cpu_inc(*rb->pending_refcnt);
1190}
1191
1192static void packet_dec_pending(struct packet_ring_buffer *rb)
1193{
1194 this_cpu_dec(*rb->pending_refcnt);
1195}
1196
1197static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1198{
1199 unsigned int refcnt = 0;
1200 int cpu;
1201
1202 /* We don't use pending refcount in rx_ring. */
1203 if (rb->pending_refcnt == NULL)
1204 return 0;
1205
1206 for_each_possible_cpu(cpu)
1207 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1208
1209 return refcnt;
1210}
1211
1212static int packet_alloc_pending(struct packet_sock *po)
1213{
1214 po->rx_ring.pending_refcnt = NULL;
1215
1216 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1217 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1218 return -ENOBUFS;
1219
1220 return 0;
1221}
1222
1223static void packet_free_pending(struct packet_sock *po)
1224{
1225 free_percpu(po->tx_ring.pending_refcnt);
1226}
1227
77f65ebd
WB		 1228static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1229{
1230 struct sock *sk = &po->sk;
1231 bool has_room;
1232
1233 if (po->prot_hook.func != tpacket_rcv)
1234 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1235 <= sk->sk_rcvbuf;
1236
1237 spin_lock(&sk->sk_receive_queue.lock);
1238 if (po->tp_version == TPACKET_V3)
1239 has_room = prb_lookup_block(po, &po->rx_ring,
1240 po->rx_ring.prb_bdqc.kactive_blk_num,
1241 TP_STATUS_KERNEL);
1242 else
1243 has_room = packet_lookup_frame(po, &po->rx_ring,
1244 po->rx_ring.head,
1245 TP_STATUS_KERNEL);
1246 spin_unlock(&sk->sk_receive_queue.lock);
1247
1248 return has_room;
1249}
1250
1da177e4
LT		 1251static void packet_sock_destruct(struct sock *sk)
1252{
ed85b565
RC		 1253 skb_queue_purge(&sk->sk_error_queue);
1254
547b792c
IJ		 1255 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1256 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1257
1258 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1259 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1260 return;
1261 }
1262
17ab56a2 1263 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1264}
1265
dc99f600
DM		 1266static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1267{
1268 int x = atomic_read(&f->rr_cur) + 1;
1269
1270 if (x >= num)
1271 x = 0;
1272
1273 return x;
1274}
1275
77f65ebd
WB		 1276static unsigned int fanout_demux_hash(struct packet_fanout *f,
1277 struct sk_buff *skb,
1278 unsigned int num)
dc99f600 1279{
89770b0a 1280 return reciprocal_scale(skb->rxhash, num);
dc99f600
DM		 1281}
1282
77f65ebd
WB		 1283static unsigned int fanout_demux_lb(struct packet_fanout *f,
1284 struct sk_buff *skb,
1285 unsigned int num)
dc99f600
DM		 1286{
1287 int cur, old;
1288
1289 cur = atomic_read(&f->rr_cur);
1290 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1291 fanout_rr_next(f, num))) != cur)
1292 cur = old;
77f65ebd
WB		 1293 return cur;
1294}
1295
1296static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1297 struct sk_buff *skb,
1298 unsigned int num)
1299{
1300 return smp_processor_id() % num;
dc99f600
DM		 1301}
1302
5df0ddfb
DB		 1303static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1304 struct sk_buff *skb,
1305 unsigned int num)
1306{
f337db64 1307 return prandom_u32_max(num);
5df0ddfb
DB		 1308}
1309
77f65ebd
WB		 1310static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1311 struct sk_buff *skb,
1312 unsigned int idx, unsigned int skip,
1313 unsigned int num)
95ec3eb4 1314{
77f65ebd 1315 unsigned int i, j;
95ec3eb4 1316
77f65ebd
WB		 1317 i = j = min_t(int, f->next[idx], num - 1);
1318 do {
1319 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1320 if (i != j)
1321 f->next[idx] = i;
1322 return i;
1323 }
1324 if (++i == num)
1325 i = 0;
1326 } while (i != j);
1327
1328 return idx;
1329}
1330
2d36097d
NH		 1331static unsigned int fanout_demux_qm(struct packet_fanout *f,
1332 struct sk_buff *skb,
1333 unsigned int num)
1334{
1335 return skb_get_queue_mapping(skb) % num;
1336}
1337
77f65ebd
WB		 1338static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1339{
1340 return f->flags & (flag >> 8);
95ec3eb4
DM		 1341}
1342
95ec3eb4
DM		 1343static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1344 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1345{
1346 struct packet_fanout *f = pt->af_packet_priv;
1347 unsigned int num = f->num_members;
1348 struct packet_sock *po;
77f65ebd 1349 unsigned int idx;
dc99f600
DM		 1350
1351 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1352 !num) {
1353 kfree_skb(skb);
1354 return 0;
1355 }
1356
95ec3eb4
DM		 1357 switch (f->type) {
1358 case PACKET_FANOUT_HASH:
1359 default:
77f65ebd 1360 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1361 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1362 if (!skb)
1363 return 0;
1364 }
3958afa1 1365 skb_get_hash(skb);
77f65ebd 1366 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1367 break;
1368 case PACKET_FANOUT_LB:
77f65ebd 1369 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1370 break;
1371 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1372 idx = fanout_demux_cpu(f, skb, num);
1373 break;
5df0ddfb
DB		 1374 case PACKET_FANOUT_RND:
1375 idx = fanout_demux_rnd(f, skb, num);
1376 break;
2d36097d
NH		 1377 case PACKET_FANOUT_QM:
1378 idx = fanout_demux_qm(f, skb, num);
1379 break;
77f65ebd
WB		 1380 case PACKET_FANOUT_ROLLOVER:
1381 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1382 break;
dc99f600
DM		 1383 }
1384
77f65ebd
WB		 1385 po = pkt_sk(f->arr[idx]);
1386 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1387 unlikely(!packet_rcv_has_room(po, skb))) {
1388 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1389 po = pkt_sk(f->arr[idx]);
1390 }
dc99f600
DM		 1391
1392 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1393}
1394
fff3321d
PE		 1395DEFINE_MUTEX(fanout_mutex);
1396EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1397static LIST_HEAD(fanout_list);
1398
1399static void __fanout_link(struct sock *sk, struct packet_sock *po)
1400{
1401 struct packet_fanout *f = po->fanout;
1402
1403 spin_lock(&f->lock);
1404 f->arr[f->num_members] = sk;
1405 smp_wmb();
1406 f->num_members++;
1407 spin_unlock(&f->lock);
1408}
1409
1410static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1411{
1412 struct packet_fanout *f = po->fanout;
1413 int i;
1414
1415 spin_lock(&f->lock);
1416 for (i = 0; i < f->num_members; i++) {
1417 if (f->arr[i] == sk)
1418 break;
1419 }
1420 BUG_ON(i >= f->num_members);
1421 f->arr[i] = f->arr[f->num_members - 1];
1422 f->num_members--;
1423 spin_unlock(&f->lock);
1424}
1425
d4dd8aee 1426static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1427{
d4dd8aee 1428 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
c0de08d0
EL		 1429 return true;
1430
1431 return false;
1432}
1433
7736d33f 1434static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1435{
1436 struct packet_sock *po = pkt_sk(sk);
1437 struct packet_fanout *f, *match;
7736d33f 1438 u8 type = type_flags & 0xff;
77f65ebd 1439 u8 flags = type_flags >> 8;
dc99f600
DM		 1440 int err;
1441
1442 switch (type) {
77f65ebd
WB		 1443 case PACKET_FANOUT_ROLLOVER:
1444 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1445 return -EINVAL;
dc99f600
DM		 1446 case PACKET_FANOUT_HASH:
1447 case PACKET_FANOUT_LB:
95ec3eb4 1448 case PACKET_FANOUT_CPU:
5df0ddfb 1449 case PACKET_FANOUT_RND:
2d36097d 1450 case PACKET_FANOUT_QM:
dc99f600
DM		 1451 break;
1452 default:
1453 return -EINVAL;
1454 }
1455
1456 if (!po->running)
1457 return -EINVAL;
1458
1459 if (po->fanout)
1460 return -EALREADY;
1461
1462 mutex_lock(&fanout_mutex);
1463 match = NULL;
1464 list_for_each_entry(f, &fanout_list, list) {
1465 if (f->id == id &&
1466 read_pnet(&f->net) == sock_net(sk)) {
1467 match = f;
1468 break;
1469 }
1470 }
afe62c68 1471 err = -EINVAL;
77f65ebd 1472 if (match && match->flags != flags)
afe62c68 1473 goto out;
dc99f600 1474 if (!match) {
afe62c68 1475 err = -ENOMEM;
dc99f600 1476 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1477 if (!match)
1478 goto out;
1479 write_pnet(&match->net, sock_net(sk));
1480 match->id = id;
1481 match->type = type;
77f65ebd 1482 match->flags = flags;
afe62c68
ED		 1483 atomic_set(&match->rr_cur, 0);
1484 INIT_LIST_HEAD(&match->list);
1485 spin_lock_init(&match->lock);
1486 atomic_set(&match->sk_ref, 0);
1487 match->prot_hook.type = po->prot_hook.type;
1488 match->prot_hook.dev = po->prot_hook.dev;
1489 match->prot_hook.func = packet_rcv_fanout;
1490 match->prot_hook.af_packet_priv = match;
c0de08d0 1491 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1492 dev_add_pack(&match->prot_hook);
1493 list_add(&match->list, &fanout_list);
dc99f600 1494 }
afe62c68
ED		 1495 err = -EINVAL;
1496 if (match->type == type &&
1497 match->prot_hook.type == po->prot_hook.type &&
1498 match->prot_hook.dev == po->prot_hook.dev) {
1499 err = -ENOSPC;
1500 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1501 __dev_remove_pack(&po->prot_hook);
1502 po->fanout = match;
1503 atomic_inc(&match->sk_ref);
1504 __fanout_link(sk, po);
1505 err = 0;
dc99f600
DM		 1506 }
1507 }
afe62c68 1508out:
dc99f600
DM		 1509 mutex_unlock(&fanout_mutex);
1510 return err;
1511}
1512
1513static void fanout_release(struct sock *sk)
1514{
1515 struct packet_sock *po = pkt_sk(sk);
1516 struct packet_fanout *f;
1517
1518 f = po->fanout;
1519 if (!f)
1520 return;
1521
fff3321d 1522 mutex_lock(&fanout_mutex);
dc99f600
DM		 1523 po->fanout = NULL;
1524
dc99f600
DM		 1525 if (atomic_dec_and_test(&f->sk_ref)) {
1526 list_del(&f->list);
1527 dev_remove_pack(&f->prot_hook);
1528 kfree(f);
1529 }
1530 mutex_unlock(&fanout_mutex);
1531}
1da177e4 1532
90ddc4f0 1533static const struct proto_ops packet_ops;
1da177e4 1534
90ddc4f0 1535static const struct proto_ops packet_ops_spkt;
1da177e4 1536
40d4e3df
ED		 1537static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1538 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1539{
1540 struct sock *sk;
1541 struct sockaddr_pkt *spkt;
1542
1543 /*
1544 * When we registered the protocol we saved the socket in the data
1545 * field for just this event.
1546 */
1547
1548 sk = pt->af_packet_priv;
1ce4f28b 1549
1da177e4
LT		 1550 /*
1551 * Yank back the headers [hope the device set this
1552 * right or kerboom...]
1553 *
1554 * Incoming packets have ll header pulled,
1555 * push it back.
1556 *
98e399f8 1557 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1558 * so that this procedure is noop.
1559 */
1560
1561 if (skb->pkt_type == PACKET_LOOPBACK)
1562 goto out;
1563
09ad9bc7 1564 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1565 goto out;
1566
40d4e3df
ED		 1567 skb = skb_share_check(skb, GFP_ATOMIC);
1568 if (skb == NULL)
1da177e4
LT		 1569 goto oom;
1570
1571 /* drop any routing info */
adf30907 1572 skb_dst_drop(skb);
1da177e4 1573
84531c24
PO		 1574 /* drop conntrack reference */
1575 nf_reset(skb);
1576
ffbc6111 1577 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1578
98e399f8 1579 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1580
1581 /*
1582 * The SOCK_PACKET socket receives _all_ frames.
1583 */
1584
1585 spkt->spkt_family = dev->type;
1586 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1587 spkt->spkt_protocol = skb->protocol;
1588
1589 /*
1590 * Charge the memory to the socket. This is done specifically
1591 * to prevent sockets using all the memory up.
1592 */
1593
40d4e3df 1594 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1595 return 0;
1596
1597out:
1598 kfree_skb(skb);
1599oom:
1600 return 0;
1601}
1602
1603
1604/*
1605 * Output a raw packet to a device layer. This bypasses all the other
1606 * protocol layers and you must therefore supply it with a complete frame
1607 */
1ce4f28b 1608
1da177e4
LT		 1609static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1610 struct msghdr *msg, size_t len)
1611{
1612 struct sock *sk = sock->sk;
342dfc30 1613 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1614 struct sk_buff *skb = NULL;
1da177e4 1615 struct net_device *dev;
40d4e3df 1616 __be16 proto = 0;
1da177e4 1617 int err;
3bdc0eba 1618 int extra_len = 0;
1ce4f28b 1619
1da177e4 1620 /*
1ce4f28b 1621 * Get and verify the address.
1da177e4
LT		 1622 */
1623
40d4e3df 1624 if (saddr) {
1da177e4 1625 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1626 return -EINVAL;
1627 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1628 proto = saddr->spkt_protocol;
1629 } else
1630 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1631
1632 /*
1ce4f28b 1633 * Find the device first to size check it
1da177e4
LT		 1634 */
1635
de74e92a 1636 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1637retry:
654d1f8a
ED		 1638 rcu_read_lock();
1639 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1640 err = -ENODEV;
1641 if (dev == NULL)
1642 goto out_unlock;
1ce4f28b 1643
d5e76b0a
DM		 1644 err = -ENETDOWN;
1645 if (!(dev->flags & IFF_UP))
1646 goto out_unlock;
1647
1da177e4 1648 /*
40d4e3df
ED		 1649 * You may not queue a frame bigger than the mtu. This is the lowest level
1650 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1651 */
1ce4f28b 1652
3bdc0eba
BG		 1653 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1654 if (!netif_supports_nofcs(dev)) {
1655 err = -EPROTONOSUPPORT;
1656 goto out_unlock;
1657 }
1658 extra_len = 4; /* We're doing our own CRC */
1659 }
1660
1da177e4 1661 err = -EMSGSIZE;
3bdc0eba 1662 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1663 goto out_unlock;
1664
1a35ca80
ED		 1665 if (!skb) {
1666 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1667 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1668 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1669
1670 rcu_read_unlock();
4ce40912 1671 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1672 if (skb == NULL)
1673 return -ENOBUFS;
1674 /* FIXME: Save some space for broken drivers that write a hard
1675 * header at transmission time by themselves. PPP is the notable
1676 * one here. This should really be fixed at the driver level.
1677 */
1678 skb_reserve(skb, reserved);
1679 skb_reset_network_header(skb);
1680
1681 /* Try to align data part correctly */
1682 if (hhlen) {
1683 skb->data -= hhlen;
1684 skb->tail -= hhlen;
1685 if (len < hhlen)
1686 skb_reset_network_header(skb);
1687 }
1688 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1689 if (err)
1690 goto out_free;
1691 goto retry;
1da177e4
LT		 1692 }
1693
3bdc0eba 1694 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1695 /* Earlier code assumed this would be a VLAN pkt,
1696 * double-check this now that we have the actual
1697 * packet in hand.
1698 */
1699 struct ethhdr *ehdr;
1700 skb_reset_mac_header(skb);
1701 ehdr = eth_hdr(skb);
1702 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1703 err = -EMSGSIZE;
1704 goto out_unlock;
1705 }
1706 }
1a35ca80 1707
1da177e4
LT		 1708 skb->protocol = proto;
1709 skb->dev = dev;
1710 skb->priority = sk->sk_priority;
2d37a186 1711 skb->mark = sk->sk_mark;
bf84a010
DB		 1712
1713 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1714
3bdc0eba
BG		 1715 if (unlikely(extra_len == 4))
1716 skb->no_fcs = 1;
1717
40893fd0 1718 skb_probe_transport_header(skb, 0);
c1aad275 1719
1da177e4 1720 dev_queue_xmit(skb);
654d1f8a 1721 rcu_read_unlock();
40d4e3df 1722 return len;
1da177e4 1723
1da177e4 1724out_unlock:
654d1f8a 1725 rcu_read_unlock();
1a35ca80
ED		 1726out_free:
1727 kfree_skb(skb);
1da177e4
LT		 1728 return err;
1729}
1da177e4 1730
eea49cc9 1731static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1732 const struct sock *sk,
dbcb5855 1733 unsigned int res)
1da177e4
LT		 1734{
1735 struct sk_filter *filter;
fda9ef5d 1736
80f8f102
ED		 1737 rcu_read_lock();
1738 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1739 if (filter != NULL)
0a14842f 1740 res = SK_RUN_FILTER(filter, skb);
80f8f102 1741 rcu_read_unlock();
1da177e4 1742
dbcb5855 1743 return res;
1da177e4
LT		 1744}
1745
1746/*
62ab0812
ED		 1747 * This function makes lazy skb cloning in hope that most of packets
1748 * are discarded by BPF.
1749 *
1750 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1751 * and skb->cb are mangled. It works because (and until) packets
1752 * falling here are owned by current CPU. Output packets are cloned
1753 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1754 * sequencially, so that if we return skb to original state on exit,
1755 * we will not harm anyone.
1da177e4
LT		 1756 */
1757
40d4e3df
ED		 1758static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1759 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1760{
1761 struct sock *sk;
1762 struct sockaddr_ll *sll;
1763 struct packet_sock *po;
40d4e3df 1764 u8 *skb_head = skb->data;
1da177e4 1765 int skb_len = skb->len;
dbcb5855 1766 unsigned int snaplen, res;
1da177e4
LT		 1767
1768 if (skb->pkt_type == PACKET_LOOPBACK)
1769 goto drop;
1770
1771 sk = pt->af_packet_priv;
1772 po = pkt_sk(sk);
1773
09ad9bc7 1774 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1775 goto drop;
1776
1da177e4
LT		 1777 skb->dev = dev;
1778
3b04ddde 1779 if (dev->header_ops) {
1da177e4 1780 /* The device has an explicit notion of ll header,
62ab0812
ED		 1781 * exported to higher levels.
1782 *
1783 * Otherwise, the device hides details of its frame
1784 * structure, so that corresponding packet head is
1785 * never delivered to user.
1da177e4
LT		 1786 */
1787 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1788 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1789 else if (skb->pkt_type == PACKET_OUTGOING) {
1790 /* Special case: outgoing packets have ll header at head */
bbe735e4 1791 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1792 }
1793 }
1794
1795 snaplen = skb->len;
1796
dbcb5855
DM		 1797 res = run_filter(skb, sk, snaplen);
1798 if (!res)
fda9ef5d 1799 goto drop_n_restore;
dbcb5855
DM		 1800 if (snaplen > res)
1801 snaplen = res;
1da177e4 1802
0fd7bac6 1803 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1804 goto drop_n_acct;
1805
1806 if (skb_shared(skb)) {
1807 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1808 if (nskb == NULL)
1809 goto drop_n_acct;
1810
1811 if (skb_head != skb->data) {
1812 skb->data = skb_head;
1813 skb->len = skb_len;
1814 }
abc4e4fa 1815 consume_skb(skb);
1da177e4
LT		 1816 skb = nskb;
1817 }
1818
ffbc6111
HX		 1819 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1820 sizeof(skb->cb));
1821
1822 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1823 sll->sll_family = AF_PACKET;
1824 sll->sll_hatype = dev->type;
1825 sll->sll_protocol = skb->protocol;
1826 sll->sll_pkttype = skb->pkt_type;
8032b464 1827 if (unlikely(po->origdev))
80feaacb
PWJ		 1828 sll->sll_ifindex = orig_dev->ifindex;
1829 else
1830 sll->sll_ifindex = dev->ifindex;
1da177e4 1831
b95cce35 1832 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1833
ffbc6111 1834 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1835
1da177e4
LT		 1836 if (pskb_trim(skb, snaplen))
1837 goto drop_n_acct;
1838
1839 skb_set_owner_r(skb, sk);
1840 skb->dev = NULL;
adf30907 1841 skb_dst_drop(skb);
1da177e4 1842
84531c24
PO		 1843 /* drop conntrack reference */
1844 nf_reset(skb);
1845
1da177e4 1846 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1847 po->stats.stats1.tp_packets++;
3b885787 1848 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1849 __skb_queue_tail(&sk->sk_receive_queue, skb);
1850 spin_unlock(&sk->sk_receive_queue.lock);
1851 sk->sk_data_ready(sk, skb->len);
1852 return 0;
1853
1854drop_n_acct:
7091fbd8 1855 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1856 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1857 atomic_inc(&sk->sk_drops);
1858 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1859
1860drop_n_restore:
1861 if (skb_head != skb->data && skb_shared(skb)) {
1862 skb->data = skb_head;
1863 skb->len = skb_len;
1864 }
1865drop:
ead2ceb0 1866 consume_skb(skb);
1da177e4
LT		 1867 return 0;
1868}
1869
40d4e3df
ED		 1870static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1871 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1872{
1873 struct sock *sk;
1874 struct packet_sock *po;
1875 struct sockaddr_ll *sll;
184f489e 1876 union tpacket_uhdr h;
40d4e3df 1877 u8 *skb_head = skb->data;
1da177e4 1878 int skb_len = skb->len;
dbcb5855 1879 unsigned int snaplen, res;
f6fb8f10 1880 unsigned long status = TP_STATUS_USER;
bbd6ef87 1881 unsigned short macoff, netoff, hdrlen;
1da177e4 1882 struct sk_buff *copy_skb = NULL;
bbd6ef87 1883 struct timespec ts;
b9c32fb2 1884 __u32 ts_status;
1da177e4 1885
51846355
AW		 1886 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1887 * We may add members to them until current aligned size without forcing
1888 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1889 */
1890 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1891 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1892
1da177e4
LT		 1893 if (skb->pkt_type == PACKET_LOOPBACK)
1894 goto drop;
1895
1896 sk = pt->af_packet_priv;
1897 po = pkt_sk(sk);
1898
09ad9bc7 1899 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1900 goto drop;
1901
3b04ddde 1902 if (dev->header_ops) {
1da177e4 1903 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1904 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1905 else if (skb->pkt_type == PACKET_OUTGOING) {
1906 /* Special case: outgoing packets have ll header at head */
bbe735e4 1907 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1908 }
1909 }
1910
8dc41944
HX		 1911 if (skb->ip_summed == CHECKSUM_PARTIAL)
1912 status |= TP_STATUS_CSUMNOTREADY;
1913
1da177e4
LT		 1914 snaplen = skb->len;
1915
dbcb5855
DM		 1916 res = run_filter(skb, sk, snaplen);
1917 if (!res)
fda9ef5d 1918 goto drop_n_restore;
dbcb5855
DM		 1919 if (snaplen > res)
1920 snaplen = res;
1da177e4
LT		 1921
1922 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1923 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1924 po->tp_reserve;
1da177e4 1925 } else {
95c96174 1926 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1927 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1928 (maclen < 16 ? 16 : maclen)) +
1929 po->tp_reserve;
1da177e4
LT		 1930 macoff = netoff - maclen;
1931 }
f6fb8f10 1932 if (po->tp_version <= TPACKET_V2) {
1933 if (macoff + snaplen > po->rx_ring.frame_size) {
1934 if (po->copy_thresh &&
0fd7bac6 1935 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1936 if (skb_shared(skb)) {
1937 copy_skb = skb_clone(skb, GFP_ATOMIC);
1938 } else {
1939 copy_skb = skb_get(skb);
1940 skb_head = skb->data;
1941 }
1942 if (copy_skb)
1943 skb_set_owner_r(copy_skb, sk);
1da177e4 1944 }
f6fb8f10 1945 snaplen = po->rx_ring.frame_size - macoff;
1946 if ((int)snaplen < 0)
1947 snaplen = 0;
1da177e4 1948 }
1da177e4 1949 }
1da177e4 1950 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1951 h.raw = packet_current_rx_frame(po, skb,
1952 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1953 if (!h.raw)
1da177e4 1954 goto ring_is_full;
f6fb8f10 1955 if (po->tp_version <= TPACKET_V2) {
1956 packet_increment_rx_head(po, &po->rx_ring);
1957 /*
1958 * LOSING will be reported till you read the stats,
1959 * because it's COR - Clear On Read.
1960 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1961 * at packet level.
1962 */
ee80fbf3 1963 if (po->stats.stats1.tp_drops)
f6fb8f10 1964 status |= TP_STATUS_LOSING;
1965 }
ee80fbf3 1966 po->stats.stats1.tp_packets++;
1da177e4
LT		 1967 if (copy_skb) {
1968 status |= TP_STATUS_COPY;
1969 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1970 }
1da177e4
LT		 1971 spin_unlock(&sk->sk_receive_queue.lock);
1972
bbd6ef87 1973 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1974
1975 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1976 getnstimeofday(&ts);
1da177e4 1977
b9c32fb2
DB		 1978 status |= ts_status;
1979
bbd6ef87
PM		 1980 switch (po->tp_version) {
1981 case TPACKET_V1:
1982 h.h1->tp_len = skb->len;
1983 h.h1->tp_snaplen = snaplen;
1984 h.h1->tp_mac = macoff;
1985 h.h1->tp_net = netoff;
4b457bdf
DB		 1986 h.h1->tp_sec = ts.tv_sec;
1987 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1988 hdrlen = sizeof(*h.h1);
1989 break;
1990 case TPACKET_V2:
1991 h.h2->tp_len = skb->len;
1992 h.h2->tp_snaplen = snaplen;
1993 h.h2->tp_mac = macoff;
1994 h.h2->tp_net = netoff;
bbd6ef87
PM		 1995 h.h2->tp_sec = ts.tv_sec;
1996 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1997 if (vlan_tx_tag_present(skb)) {
1998 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 1999 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2000 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2001 } else {
2002 h.h2->tp_vlan_tci = 0;
a0cdfcf3 2003 h.h2->tp_vlan_tpid = 0;
a3bcc23e 2004 }
e4d26f4b 2005 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 2006 hdrlen = sizeof(*h.h2);
2007 break;
f6fb8f10 2008 case TPACKET_V3:
2009 /* tp_nxt_offset,vlan are already populated above.
2010 * So DONT clear those fields here
2011 */
2012 h.h3->tp_status |= status;
2013 h.h3->tp_len = skb->len;
2014 h.h3->tp_snaplen = snaplen;
2015 h.h3->tp_mac = macoff;
2016 h.h3->tp_net = netoff;
f6fb8f10 2017 h.h3->tp_sec = ts.tv_sec;
2018 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2019 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2020 hdrlen = sizeof(*h.h3);
2021 break;
bbd6ef87
PM		 2022 default:
2023 BUG();
2024 }
1da177e4 2025
bbd6ef87 2026 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2027 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2028 sll->sll_family = AF_PACKET;
2029 sll->sll_hatype = dev->type;
2030 sll->sll_protocol = skb->protocol;
2031 sll->sll_pkttype = skb->pkt_type;
8032b464 2032 if (unlikely(po->origdev))
80feaacb
PWJ		 2033 sll->sll_ifindex = orig_dev->ifindex;
2034 else
2035 sll->sll_ifindex = dev->ifindex;
1da177e4 2036
e16aa207 2037 smp_mb();
f0d4eb29 2038
f6dafa95 2039#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2040 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2041 u8 *start, *end;
2042
f0d4eb29
DB		 2043 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2044 macoff + snaplen);
2045
2046 for (start = h.raw; start < end; start += PAGE_SIZE)
2047 flush_dcache_page(pgv_to_page(start));
1da177e4 2048 }
f0d4eb29 2049 smp_wmb();
f6dafa95 2050#endif
f0d4eb29 2051
f6fb8f10 2052 if (po->tp_version <= TPACKET_V2)
2053 __packet_set_status(po, h.raw, status);
2054 else
2055 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 2056
2057 sk->sk_data_ready(sk, 0);
2058
2059drop_n_restore:
2060 if (skb_head != skb->data && skb_shared(skb)) {
2061 skb->data = skb_head;
2062 skb->len = skb_len;
2063 }
2064drop:
1ce4f28b 2065 kfree_skb(skb);
1da177e4
LT		 2066 return 0;
2067
2068ring_is_full:
ee80fbf3 2069 po->stats.stats1.tp_drops++;
1da177e4
LT		 2070 spin_unlock(&sk->sk_receive_queue.lock);
2071
2072 sk->sk_data_ready(sk, 0);
acb5d75b 2073 kfree_skb(copy_skb);
1da177e4
LT		 2074 goto drop_n_restore;
2075}
2076
69e3c75f
JB		 2077static void tpacket_destruct_skb(struct sk_buff *skb)
2078{
2079 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2080
69e3c75f 2081 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2082 void *ph;
b9c32fb2
DB		 2083 __u32 ts;
2084
69e3c75f 2085 ph = skb_shinfo(skb)->destructor_arg;
b0138408 2086 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2087
2088 ts = __packet_set_timestamp(po, ph, skb);
2089 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2090 }
2091
2092 sock_wfree(skb);
2093}
2094
40d4e3df
ED		 2095static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2096 void *frame, struct net_device *dev, int size_max,
ae641949 2097 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 2098{
184f489e 2099 union tpacket_uhdr ph;
09effa67 2100 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 2101 struct socket *sock = po->sk.sk_socket;
2102 struct page *page;
2103 void *data;
2104 int err;
2105
2106 ph.raw = frame;
2107
2108 skb->protocol = proto;
2109 skb->dev = dev;
2110 skb->priority = po->sk.sk_priority;
2d37a186 2111 skb->mark = po->sk.sk_mark;
2e31396f 2112 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2113 skb_shinfo(skb)->destructor_arg = ph.raw;
2114
2115 switch (po->tp_version) {
2116 case TPACKET_V2:
2117 tp_len = ph.h2->tp_len;
2118 break;
2119 default:
2120 tp_len = ph.h1->tp_len;
2121 break;
2122 }
09effa67
DM		 2123 if (unlikely(tp_len > size_max)) {
2124 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2125 return -EMSGSIZE;
2126 }
69e3c75f 2127
ae641949 2128 skb_reserve(skb, hlen);
69e3c75f 2129 skb_reset_network_header(skb);
c1aad275 2130
d346a3fa
DB		 2131 if (!packet_use_direct_xmit(po))
2132 skb_probe_transport_header(skb, 0);
2133 if (unlikely(po->tp_tx_has_off)) {
5920cd3a
PC		 2134 int off_min, off_max, off;
2135 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2136 off_max = po->tx_ring.frame_size - tp_len;
2137 if (sock->type == SOCK_DGRAM) {
2138 switch (po->tp_version) {
2139 case TPACKET_V2:
2140 off = ph.h2->tp_net;
2141 break;
2142 default:
2143 off = ph.h1->tp_net;
2144 break;
2145 }
2146 } else {
2147 switch (po->tp_version) {
2148 case TPACKET_V2:
2149 off = ph.h2->tp_mac;
2150 break;
2151 default:
2152 off = ph.h1->tp_mac;
2153 break;
2154 }
2155 }
2156 if (unlikely((off < off_min) || (off_max < off)))
2157 return -EINVAL;
2158 data = ph.raw + off;
2159 } else {
2160 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2161 }
69e3c75f
JB		 2162 to_write = tp_len;
2163
2164 if (sock->type == SOCK_DGRAM) {
2165 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2166 NULL, tp_len);
2167 if (unlikely(err < 0))
2168 return -EINVAL;
40d4e3df 2169 } else if (dev->hard_header_len) {
69e3c75f
JB		 2170 /* net device doesn't like empty head */
2171 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2172 pr_err("packet size is too short (%d < %d)\n",
2173 tp_len, dev->hard_header_len);
69e3c75f
JB		 2174 return -EINVAL;
2175 }
2176
2177 skb_push(skb, dev->hard_header_len);
2178 err = skb_store_bits(skb, 0, data,
2179 dev->hard_header_len);
2180 if (unlikely(err))
2181 return err;
2182
2183 data += dev->hard_header_len;
2184 to_write -= dev->hard_header_len;
2185 }
2186
69e3c75f
JB		 2187 offset = offset_in_page(data);
2188 len_max = PAGE_SIZE - offset;
2189 len = ((to_write > len_max) ? len_max : to_write);
2190
2191 skb->data_len = to_write;
2192 skb->len += to_write;
2193 skb->truesize += to_write;
2194 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2195
2196 while (likely(to_write)) {
2197 nr_frags = skb_shinfo(skb)->nr_frags;
2198
2199 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2200 pr_err("Packet exceed the number of skb frags(%lu)\n",
2201 MAX_SKB_FRAGS);
69e3c75f
JB		 2202 return -EFAULT;
2203 }
2204
0af55bb5
CG		 2205 page = pgv_to_page(data);
2206 data += len;
69e3c75f
JB		 2207 flush_dcache_page(page);
2208 get_page(page);
0af55bb5 2209 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2210 to_write -= len;
2211 offset = 0;
2212 len_max = PAGE_SIZE;
2213 len = ((to_write > len_max) ? len_max : to_write);
2214 }
2215
2216 return tp_len;
2217}
2218
2219static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2220{
69e3c75f
JB		 2221 struct sk_buff *skb;
2222 struct net_device *dev;
2223 __be16 proto;
09effa67 2224 int err, reserve = 0;
40d4e3df 2225 void *ph;
342dfc30 2226 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2227 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
69e3c75f
JB		 2228 int tp_len, size_max;
2229 unsigned char *addr;
2230 int len_sum = 0;
9e67030a 2231 int status = TP_STATUS_AVAILABLE;
ae641949 2232 int hlen, tlen;
69e3c75f 2233
69e3c75f
JB		 2234 mutex_lock(&po->pg_vec_lock);
2235
66e56cd4 2236 if (likely(saddr == NULL)) {
e40526cb 2237 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2238 proto = po->num;
2239 addr = NULL;
2240 } else {
2241 err = -EINVAL;
2242 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2243 goto out;
2244 if (msg->msg_namelen < (saddr->sll_halen
2245 + offsetof(struct sockaddr_ll,
2246 sll_addr)))
2247 goto out;
69e3c75f
JB		 2248 proto = saddr->sll_protocol;
2249 addr = saddr->sll_addr;
827d9780 2250 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2251 }
2252
69e3c75f
JB		 2253 err = -ENXIO;
2254 if (unlikely(dev == NULL))
2255 goto out;
69e3c75f
JB		 2256 err = -ENETDOWN;
2257 if (unlikely(!(dev->flags & IFF_UP)))
2258 goto out_put;
2259
e40526cb
DB		 2260 reserve = dev->hard_header_len;
2261
69e3c75f 2262 size_max = po->tx_ring.frame_size
b5dd884e 2263 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2264
09effa67
DM		 2265 if (size_max > dev->mtu + reserve)
2266 size_max = dev->mtu + reserve;
2267
69e3c75f
JB		 2268 do {
2269 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2270 TP_STATUS_SEND_REQUEST);
69e3c75f 2271 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2272 if (need_wait && need_resched())
2273 schedule();
69e3c75f
JB		 2274 continue;
2275 }
2276
2277 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2278 hlen = LL_RESERVED_SPACE(dev);
2279 tlen = dev->needed_tailroom;
69e3c75f 2280 skb = sock_alloc_send_skb(&po->sk,
ae641949 2281 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2282 0, &err);
2283
2284 if (unlikely(skb == NULL))
2285 goto out_status;
2286
2287 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2288 addr, hlen);
69e3c75f
JB		 2289
2290 if (unlikely(tp_len < 0)) {
2291 if (po->tp_loss) {
2292 __packet_set_status(po, ph,
2293 TP_STATUS_AVAILABLE);
2294 packet_increment_head(&po->tx_ring);
2295 kfree_skb(skb);
2296 continue;
2297 } else {
2298 status = TP_STATUS_WRONG_FORMAT;
2299 err = tp_len;
2300 goto out_status;
2301 }
2302 }
2303
0fd5d57b
DB		 2304 packet_pick_tx_queue(dev, skb);
2305
69e3c75f
JB		 2306 skb->destructor = tpacket_destruct_skb;
2307 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2308 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2309
2310 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2311 err = po->xmit(skb);
eb70df13
JP		 2312 if (unlikely(err > 0)) {
2313 err = net_xmit_errno(err);
2314 if (err && __packet_get_status(po, ph) ==
2315 TP_STATUS_AVAILABLE) {
2316 /* skb was destructed already */
2317 skb = NULL;
2318 goto out_status;
2319 }
2320 /*
2321 * skb was dropped but not destructed yet;
2322 * let's treat it like congestion or err < 0
2323 */
2324 err = 0;
2325 }
69e3c75f
JB		 2326 packet_increment_head(&po->tx_ring);
2327 len_sum += tp_len;
b0138408
DB		 2328 } while (likely((ph != NULL) ||
2329 /* Note: packet_read_pending() might be slow if we have
2330 * to call it as it's per_cpu variable, but in fast-path
2331 * we already short-circuit the loop with the first
2332 * condition, and luckily don't have to go that path
2333 * anyway.
2334 */
2335 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2336
2337 err = len_sum;
2338 goto out_put;
2339
69e3c75f
JB		 2340out_status:
2341 __packet_set_status(po, ph, status);
2342 kfree_skb(skb);
2343out_put:
e40526cb 2344 dev_put(dev);
69e3c75f
JB		 2345out:
2346 mutex_unlock(&po->pg_vec_lock);
2347 return err;
2348}
69e3c75f 2349
eea49cc9
OJ		 2350static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2351 size_t reserve, size_t len,
2352 size_t linear, int noblock,
2353 int *err)
bfd5f4a3
SS		 2354{
2355 struct sk_buff *skb;
2356
2357 /* Under a page? Don't bother with paged skb. */
2358 if (prepad + len < PAGE_SIZE || !linear)
2359 linear = len;
2360
2361 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2362 err, 0);
bfd5f4a3
SS		 2363 if (!skb)
2364 return NULL;
2365
2366 skb_reserve(skb, reserve);
2367 skb_put(skb, linear);
2368 skb->data_len = len - linear;
2369 skb->len += len - linear;
2370
2371 return skb;
2372}
2373
d346a3fa 2374static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2375{
2376 struct sock *sk = sock->sk;
342dfc30 2377 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2378 struct sk_buff *skb;
2379 struct net_device *dev;
0e11c91e 2380 __be16 proto;
1da177e4 2381 unsigned char *addr;
827d9780 2382 int err, reserve = 0;
bfd5f4a3
SS		 2383 struct virtio_net_hdr vnet_hdr = { 0 };
2384 int offset = 0;
2385 int vnet_hdr_len;
2386 struct packet_sock *po = pkt_sk(sk);
2387 unsigned short gso_type = 0;
ae641949 2388 int hlen, tlen;
3bdc0eba 2389 int extra_len = 0;
1da177e4
LT		 2390
2391 /*
1ce4f28b 2392 * Get and verify the address.
1da177e4 2393 */
1ce4f28b 2394
66e56cd4 2395 if (likely(saddr == NULL)) {
e40526cb 2396 dev = packet_cached_dev_get(po);
1da177e4
LT		 2397 proto = po->num;
2398 addr = NULL;
2399 } else {
2400 err = -EINVAL;
2401 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2402 goto out;
0fb375fb
EB		 2403 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2404 goto out;
1da177e4
LT		 2405 proto = saddr->sll_protocol;
2406 addr = saddr->sll_addr;
827d9780 2407 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2408 }
2409
1da177e4 2410 err = -ENXIO;
e40526cb 2411 if (unlikely(dev == NULL))
1da177e4 2412 goto out_unlock;
d5e76b0a 2413 err = -ENETDOWN;
e40526cb 2414 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2415 goto out_unlock;
2416
e40526cb
DB		 2417 if (sock->type == SOCK_RAW)
2418 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2419 if (po->has_vnet_hdr) {
2420 vnet_hdr_len = sizeof(vnet_hdr);
2421
2422 err = -EINVAL;
2423 if (len < vnet_hdr_len)
2424 goto out_unlock;
2425
2426 len -= vnet_hdr_len;
2427
2428 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2429 vnet_hdr_len);
2430 if (err < 0)
2431 goto out_unlock;
2432
2433 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2434 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2435 vnet_hdr.hdr_len))
2436 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2437 vnet_hdr.csum_offset + 2;
2438
2439 err = -EINVAL;
2440 if (vnet_hdr.hdr_len > len)
2441 goto out_unlock;
2442
2443 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2444 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2445 case VIRTIO_NET_HDR_GSO_TCPV4:
2446 gso_type = SKB_GSO_TCPV4;
2447 break;
2448 case VIRTIO_NET_HDR_GSO_TCPV6:
2449 gso_type = SKB_GSO_TCPV6;
2450 break;
2451 case VIRTIO_NET_HDR_GSO_UDP:
2452 gso_type = SKB_GSO_UDP;
2453 break;
2454 default:
2455 goto out_unlock;
2456 }
2457
2458 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2459 gso_type |= SKB_GSO_TCP_ECN;
2460
2461 if (vnet_hdr.gso_size == 0)
2462 goto out_unlock;
2463
2464 }
2465 }
2466
3bdc0eba
BG		 2467 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2468 if (!netif_supports_nofcs(dev)) {
2469 err = -EPROTONOSUPPORT;
2470 goto out_unlock;
2471 }
2472 extra_len = 4; /* We're doing our own CRC */
2473 }
2474
1da177e4 2475 err = -EMSGSIZE;
3bdc0eba 2476 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2477 goto out_unlock;
2478
bfd5f4a3 2479 err = -ENOBUFS;
ae641949
HX		 2480 hlen = LL_RESERVED_SPACE(dev);
2481 tlen = dev->needed_tailroom;
2482 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2483 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2484 if (skb == NULL)
1da177e4
LT		 2485 goto out_unlock;
2486
bfd5f4a3 2487 skb_set_network_header(skb, reserve);
1da177e4 2488
0c4e8581
SH		 2489 err = -EINVAL;
2490 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2491 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2492 goto out_free;
1da177e4
LT		 2493
2494 /* Returns -EFAULT on error */
bfd5f4a3 2495 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2496 if (err)
2497 goto out_free;
bf84a010
DB		 2498
2499 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2500
3bdc0eba 2501 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2502 /* Earlier code assumed this would be a VLAN pkt,
2503 * double-check this now that we have the actual
2504 * packet in hand.
2505 */
2506 struct ethhdr *ehdr;
2507 skb_reset_mac_header(skb);
2508 ehdr = eth_hdr(skb);
2509 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2510 err = -EMSGSIZE;
2511 goto out_free;
2512 }
57f89bfa
BG		 2513 }
2514
09effa67
DM		 2515 skb->protocol = proto;
2516 skb->dev = dev;
1da177e4 2517 skb->priority = sk->sk_priority;
2d37a186 2518 skb->mark = sk->sk_mark;
0fd5d57b
DB		 2519
2520 packet_pick_tx_queue(dev, skb);
1da177e4 2521
bfd5f4a3
SS		 2522 if (po->has_vnet_hdr) {
2523 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2524 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2525 vnet_hdr.csum_offset)) {
2526 err = -EINVAL;
2527 goto out_free;
2528 }
2529 }
2530
2531 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2532 skb_shinfo(skb)->gso_type = gso_type;
2533
2534 /* Header must be checked, and gso_segs computed. */
2535 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2536 skb_shinfo(skb)->gso_segs = 0;
2537
2538 len += vnet_hdr_len;
2539 }
2540
d346a3fa
DB		 2541 if (!packet_use_direct_xmit(po))
2542 skb_probe_transport_header(skb, reserve);
3bdc0eba
BG		 2543 if (unlikely(extra_len == 4))
2544 skb->no_fcs = 1;
2545
d346a3fa 2546 err = po->xmit(skb);
1da177e4
LT		 2547 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2548 goto out_unlock;
2549
e40526cb 2550 dev_put(dev);
1da177e4 2551
40d4e3df 2552 return len;
1da177e4
LT		 2553
2554out_free:
2555 kfree_skb(skb);
2556out_unlock:
e40526cb 2557 if (dev)
1da177e4
LT		 2558 dev_put(dev);
2559out:
2560 return err;
2561}
2562
69e3c75f
JB		 2563static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2564 struct msghdr *msg, size_t len)
2565{
69e3c75f
JB		 2566 struct sock *sk = sock->sk;
2567 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2568
69e3c75f
JB		 2569 if (po->tx_ring.pg_vec)
2570 return tpacket_snd(po, msg);
2571 else
69e3c75f
JB		 2572 return packet_snd(sock, msg, len);
2573}
2574
1da177e4
LT		 2575/*
2576 * Close a PACKET socket. This is fairly simple. We immediately go
2577 * to 'closed' state and remove our protocol entry in the device list.
2578 */
2579
2580static int packet_release(struct socket *sock)
2581{
2582 struct sock *sk = sock->sk;
2583 struct packet_sock *po;
d12d01d6 2584 struct net *net;
f6fb8f10 2585 union tpacket_req_u req_u;
1da177e4
LT		 2586
2587 if (!sk)
2588 return 0;
2589
3b1e0a65 2590 net = sock_net(sk);
1da177e4
LT		 2591 po = pkt_sk(sk);
2592
0fa7fa98 2593 mutex_lock(&net->packet.sklist_lock);
808f5114 2594 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2595 mutex_unlock(&net->packet.sklist_lock);
2596
2597 preempt_disable();
920de804 2598 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2599 preempt_enable();
1da177e4 2600
808f5114 2601 spin_lock(&po->bind_lock);
ce06b03e 2602 unregister_prot_hook(sk, false);
66e56cd4
DB		 2603 packet_cached_dev_reset(po);
2604
160ff18a
BG		 2605 if (po->prot_hook.dev) {
2606 dev_put(po->prot_hook.dev);
2607 po->prot_hook.dev = NULL;
2608 }
808f5114 2609 spin_unlock(&po->bind_lock);
1da177e4 2610
1da177e4 2611 packet_flush_mclist(sk);
1da177e4 2612
9665d5d6
PS		 2613 if (po->rx_ring.pg_vec) {
2614 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2615 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2616 }
69e3c75f 2617
9665d5d6
PS		 2618 if (po->tx_ring.pg_vec) {
2619 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2620 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2621 }
1da177e4 2622
dc99f600
DM		 2623 fanout_release(sk);
2624
808f5114 2625 synchronize_net();
1da177e4
LT		 2626 /*
2627 * Now the socket is dead. No more input will appear.
2628 */
1da177e4
LT		 2629 sock_orphan(sk);
2630 sock->sk = NULL;
2631
2632 /* Purge queues */
2633
2634 skb_queue_purge(&sk->sk_receive_queue);
b0138408 2635 packet_free_pending(po);
17ab56a2 2636 sk_refcnt_debug_release(sk);
1da177e4
LT		 2637
2638 sock_put(sk);
2639 return 0;
2640}
2641
2642/*
2643 * Attach a packet hook.
2644 */
2645
902fefb8 2646static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
1da177e4
LT		 2647{
2648 struct packet_sock *po = pkt_sk(sk);
902fefb8
DB		 2649 const struct net_device *dev_curr;
2650 __be16 proto_curr;
2651 bool need_rehook;
dc99f600 2652
aef950b4
WY		 2653 if (po->fanout) {
2654 if (dev)
2655 dev_put(dev);
2656
dc99f600 2657 return -EINVAL;
aef950b4 2658 }
1da177e4
LT		 2659
2660 lock_sock(sk);
1da177e4 2661 spin_lock(&po->bind_lock);
66e56cd4 2662
902fefb8
DB		 2663 proto_curr = po->prot_hook.type;
2664 dev_curr = po->prot_hook.dev;
2665
2666 need_rehook = proto_curr != proto || dev_curr != dev;
2667
2668 if (need_rehook) {
2669 unregister_prot_hook(sk, true);
1da177e4 2670
902fefb8
DB		 2671 po->num = proto;
2672 po->prot_hook.type = proto;
1da177e4 2673
902fefb8
DB		 2674 if (po->prot_hook.dev)
2675 dev_put(po->prot_hook.dev);
2676
2677 po->prot_hook.dev = dev;
2678
2679 po->ifindex = dev ? dev->ifindex : 0;
2680 packet_cached_dev_assign(po, dev);
2681 }
66e56cd4 2682
902fefb8 2683 if (proto == 0 || !need_rehook)
1da177e4
LT		 2684 goto out_unlock;
2685
be85d4ad 2686 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2687 register_prot_hook(sk);
be85d4ad
UT		 2688 } else {
2689 sk->sk_err = ENETDOWN;
2690 if (!sock_flag(sk, SOCK_DEAD))
2691 sk->sk_error_report(sk);
1da177e4
LT		 2692 }
2693
2694out_unlock:
2695 spin_unlock(&po->bind_lock);
2696 release_sock(sk);
2697 return 0;
2698}
2699
2700/*
2701 * Bind a packet socket to a device
2702 */
2703
40d4e3df
ED		 2704static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2705 int addr_len)
1da177e4 2706{
40d4e3df 2707 struct sock *sk = sock->sk;
1da177e4
LT		 2708 char name[15];
2709 struct net_device *dev;
2710 int err = -ENODEV;
1ce4f28b 2711
1da177e4
LT		 2712 /*
2713 * Check legality
2714 */
1ce4f28b 2715
8ae55f04 2716 if (addr_len != sizeof(struct sockaddr))
1da177e4 2717 return -EINVAL;
40d4e3df 2718 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2719
3b1e0a65 2720 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2721 if (dev)
1da177e4 2722 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2723 return err;
2724}
1da177e4
LT		 2725
2726static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2727{
40d4e3df
ED		 2728 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2729 struct sock *sk = sock->sk;
1da177e4
LT		 2730 struct net_device *dev = NULL;
2731 int err;
2732
2733
2734 /*
2735 * Check legality
2736 */
1ce4f28b 2737
1da177e4
LT		 2738 if (addr_len < sizeof(struct sockaddr_ll))
2739 return -EINVAL;
2740 if (sll->sll_family != AF_PACKET)
2741 return -EINVAL;
2742
2743 if (sll->sll_ifindex) {
2744 err = -ENODEV;
3b1e0a65 2745 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2746 if (dev == NULL)
2747 goto out;
2748 }
2749 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2750
2751out:
2752 return err;
2753}
2754
2755static struct proto packet_proto = {
2756 .name = "PACKET",
2757 .owner = THIS_MODULE,
2758 .obj_size = sizeof(struct packet_sock),
2759};
2760
2761/*
1ce4f28b 2762 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2763 */
2764
3f378b68
EP		 2765static int packet_create(struct net *net, struct socket *sock, int protocol,
2766 int kern)
1da177e4
LT		 2767{
2768 struct sock *sk;
2769 struct packet_sock *po;
0e11c91e 2770 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2771 int err;
2772
df008c91 2773 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2774 return -EPERM;
be02097c
DM		 2775 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2776 sock->type != SOCK_PACKET)
1da177e4
LT		 2777 return -ESOCKTNOSUPPORT;
2778
2779 sock->state = SS_UNCONNECTED;
2780
2781 err = -ENOBUFS;
6257ff21 2782 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2783 if (sk == NULL)
2784 goto out;
2785
2786 sock->ops = &packet_ops;
1da177e4
LT		 2787 if (sock->type == SOCK_PACKET)
2788 sock->ops = &packet_ops_spkt;
be02097c 2789
1da177e4
LT		 2790 sock_init_data(sock, sk);
2791
2792 po = pkt_sk(sk);
2793 sk->sk_family = PF_PACKET;
0e11c91e 2794 po->num = proto;
d346a3fa 2795 po->xmit = dev_queue_xmit;
66e56cd4 2796
b0138408
DB		 2797 err = packet_alloc_pending(po);
2798 if (err)
2799 goto out2;
2800
66e56cd4 2801 packet_cached_dev_reset(po);
1da177e4
LT		 2802
2803 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2804 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2805
2806 /*
2807 * Attach a protocol block
2808 */
2809
2810 spin_lock_init(&po->bind_lock);
905db440 2811 mutex_init(&po->pg_vec_lock);
1da177e4 2812 po->prot_hook.func = packet_rcv;
be02097c 2813
1da177e4
LT		 2814 if (sock->type == SOCK_PACKET)
2815 po->prot_hook.func = packet_rcv_spkt;
be02097c 2816
1da177e4
LT		 2817 po->prot_hook.af_packet_priv = sk;
2818
0e11c91e
AV		 2819 if (proto) {
2820 po->prot_hook.type = proto;
ce06b03e 2821 register_prot_hook(sk);
1da177e4
LT		 2822 }
2823
0fa7fa98 2824 mutex_lock(&net->packet.sklist_lock);
808f5114 2825 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2826 mutex_unlock(&net->packet.sklist_lock);
2827
2828 preempt_disable();
3680453c 2829 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2830 preempt_enable();
808f5114 2831
40d4e3df 2832 return 0;
b0138408
DB		 2833out2:
2834 sk_free(sk);
1da177e4
LT		 2835out:
2836 return err;
2837}
2838
2839/*
2840 * Pull a packet from our receive queue and hand it to the user.
2841 * If necessary we block.
2842 */
2843
2844static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2845 struct msghdr *msg, size_t len, int flags)
2846{
2847 struct sock *sk = sock->sk;
2848 struct sk_buff *skb;
2849 int copied, err;
bfd5f4a3 2850 int vnet_hdr_len = 0;
1da177e4
LT		 2851
2852 err = -EINVAL;
ed85b565 2853 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2854 goto out;
2855
2856#if 0
2857 /* What error should we return now? EUNATTACH? */
2858 if (pkt_sk(sk)->ifindex < 0)
2859 return -ENODEV;
2860#endif
2861
ed85b565 2862 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2863 err = sock_recv_errqueue(sk, msg, len,
2864 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2865 goto out;
2866 }
2867
1da177e4
LT		 2868 /*
2869 * Call the generic datagram receiver. This handles all sorts
2870 * of horrible races and re-entrancy so we can forget about it
2871 * in the protocol layers.
2872 *
2873 * Now it will return ENETDOWN, if device have just gone down,
2874 * but then it will block.
2875 */
2876
40d4e3df 2877 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2878
2879 /*
1ce4f28b 2880 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2881 * handles the blocking we don't see and worry about blocking
2882 * retries.
2883 */
2884
8ae55f04 2885 if (skb == NULL)
1da177e4
LT		 2886 goto out;
2887
bfd5f4a3
SS		 2888 if (pkt_sk(sk)->has_vnet_hdr) {
2889 struct virtio_net_hdr vnet_hdr = { 0 };
2890
2891 err = -EINVAL;
2892 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2893 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2894 goto out_free;
2895
1f18b717
MK		 2896 len -= vnet_hdr_len;
2897
bfd5f4a3
SS		 2898 if (skb_is_gso(skb)) {
2899 struct skb_shared_info *sinfo = skb_shinfo(skb);
2900
2901 /* This is a hint as to how much should be linear. */
2902 vnet_hdr.hdr_len = skb_headlen(skb);
2903 vnet_hdr.gso_size = sinfo->gso_size;
2904 if (sinfo->gso_type & SKB_GSO_TCPV4)
2905 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2906 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2907 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2908 else if (sinfo->gso_type & SKB_GSO_UDP)
2909 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2910 else if (sinfo->gso_type & SKB_GSO_FCOE)
2911 goto out_free;
2912 else
2913 BUG();
2914 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2915 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2916 } else
2917 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2918
2919 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2920 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2921 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2922 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2923 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2924 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2925 } /* else everything is zero */
2926
2927 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2928 vnet_hdr_len);
2929 if (err < 0)
2930 goto out_free;
2931 }
2932
f3d33426
HFS		 2933 /* You lose any data beyond the buffer you gave. If it worries
2934 * a user program they can ask the device for its MTU
2935 * anyway.
1da177e4 2936 */
1da177e4 2937 copied = skb->len;
40d4e3df
ED		 2938 if (copied > len) {
2939 copied = len;
2940 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2941 }
2942
2943 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2944 if (err)
2945 goto out_free;
2946
3b885787 2947 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2948
f3d33426
HFS		 2949 if (msg->msg_name) {
2950 /* If the address length field is there to be filled
2951 * in, we fill it in now.
2952 */
2953 if (sock->type == SOCK_PACKET) {
342dfc30 2954 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426
HFS		 2955 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2956 } else {
2957 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2958 msg->msg_namelen = sll->sll_halen +
2959 offsetof(struct sockaddr_ll, sll_addr);
2960 }
ffbc6111
HX		 2961 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2962 msg->msg_namelen);
f3d33426 2963 }
1da177e4 2964
8dc41944 2965 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2966 struct tpacket_auxdata aux;
2967
2968 aux.tp_status = TP_STATUS_USER;
2969 if (skb->ip_summed == CHECKSUM_PARTIAL)
2970 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2971 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2972 aux.tp_snaplen = skb->len;
2973 aux.tp_mac = 0;
bbe735e4 2974 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2975 if (vlan_tx_tag_present(skb)) {
2976 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 2977 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
2978 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2979 } else {
2980 aux.tp_vlan_tci = 0;
a0cdfcf3 2981 aux.tp_vlan_tpid = 0;
a3bcc23e 2982 }
ffbc6111 2983 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2984 }
2985
1da177e4
LT		 2986 /*
2987 * Free or return the buffer as appropriate. Again this
2988 * hides all the races and re-entrancy issues from us.
2989 */
bfd5f4a3 2990 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2991
2992out_free:
2993 skb_free_datagram(sk, skb);
2994out:
2995 return err;
2996}
2997
1da177e4
LT		 2998static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2999 int *uaddr_len, int peer)
3000{
3001 struct net_device *dev;
3002 struct sock *sk = sock->sk;
3003
3004 if (peer)
3005 return -EOPNOTSUPP;
3006
3007 uaddr->sa_family = AF_PACKET;
2dc85bf3 3008 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 3009 rcu_read_lock();
3010 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3011 if (dev)
2dc85bf3 3012 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 3013 rcu_read_unlock();
1da177e4
LT		 3014 *uaddr_len = sizeof(*uaddr);
3015
3016 return 0;
3017}
1da177e4
LT		 3018
3019static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3020 int *uaddr_len, int peer)
3021{
3022 struct net_device *dev;
3023 struct sock *sk = sock->sk;
3024 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3025 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3026
3027 if (peer)
3028 return -EOPNOTSUPP;
3029
3030 sll->sll_family = AF_PACKET;
3031 sll->sll_ifindex = po->ifindex;
3032 sll->sll_protocol = po->num;
67286640 3033 sll->sll_pkttype = 0;
654d1f8a
ED		 3034 rcu_read_lock();
3035 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3036 if (dev) {
3037 sll->sll_hatype = dev->type;
3038 sll->sll_halen = dev->addr_len;
3039 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3040 } else {
3041 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3042 sll->sll_halen = 0;
3043 }
654d1f8a 3044 rcu_read_unlock();
0fb375fb 3045 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3046
3047 return 0;
3048}
3049
2aeb0b88
WC		 3050static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3051 int what)
1da177e4
LT		 3052{
3053 switch (i->type) {
3054 case PACKET_MR_MULTICAST:
1162563f
JP		 3055 if (i->alen != dev->addr_len)
3056 return -EINVAL;
1da177e4 3057 if (what > 0)
22bedad3 3058 return dev_mc_add(dev, i->addr);
1da177e4 3059 else
22bedad3 3060 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3061 break;
3062 case PACKET_MR_PROMISC:
2aeb0b88 3063 return dev_set_promiscuity(dev, what);
1da177e4
LT		 3064 break;
3065 case PACKET_MR_ALLMULTI:
2aeb0b88 3066 return dev_set_allmulti(dev, what);
1da177e4 3067 break;
d95ed927 3068 case PACKET_MR_UNICAST:
1162563f
JP		 3069 if (i->alen != dev->addr_len)
3070 return -EINVAL;
d95ed927 3071 if (what > 0)
a748ee24 3072 return dev_uc_add(dev, i->addr);
d95ed927 3073 else
a748ee24 3074 return dev_uc_del(dev, i->addr);
d95ed927 3075 break;
40d4e3df
ED		 3076 default:
3077 break;
1da177e4 3078 }
2aeb0b88 3079 return 0;
1da177e4
LT		 3080}
3081
3082static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
3083{
40d4e3df 3084 for ( ; i; i = i->next) {
1da177e4
LT		 3085 if (i->ifindex == dev->ifindex)
3086 packet_dev_mc(dev, i, what);
3087 }
3088}
3089
0fb375fb 3090static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3091{
3092 struct packet_sock *po = pkt_sk(sk);
3093 struct packet_mclist *ml, *i;
3094 struct net_device *dev;
3095 int err;
3096
3097 rtnl_lock();
3098
3099 err = -ENODEV;
3b1e0a65 3100 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3101 if (!dev)
3102 goto done;
3103
3104 err = -EINVAL;
1162563f 3105 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3106 goto done;
3107
3108 err = -ENOBUFS;
8b3a7005 3109 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3110 if (i == NULL)
3111 goto done;
3112
3113 err = 0;
3114 for (ml = po->mclist; ml; ml = ml->next) {
3115 if (ml->ifindex == mreq->mr_ifindex &&
3116 ml->type == mreq->mr_type &&
3117 ml->alen == mreq->mr_alen &&
3118 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3119 ml->count++;
3120 /* Free the new element ... */
3121 kfree(i);
3122 goto done;
3123 }
3124 }
3125
3126 i->type = mreq->mr_type;
3127 i->ifindex = mreq->mr_ifindex;
3128 i->alen = mreq->mr_alen;
3129 memcpy(i->addr, mreq->mr_address, i->alen);
3130 i->count = 1;
3131 i->next = po->mclist;
3132 po->mclist = i;
2aeb0b88
WC		 3133 err = packet_dev_mc(dev, i, 1);
3134 if (err) {
3135 po->mclist = i->next;
3136 kfree(i);
3137 }
1da177e4
LT		 3138
3139done:
3140 rtnl_unlock();
3141 return err;
3142}
3143
0fb375fb 3144static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3145{
3146 struct packet_mclist *ml, **mlp;
3147
3148 rtnl_lock();
3149
3150 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3151 if (ml->ifindex == mreq->mr_ifindex &&
3152 ml->type == mreq->mr_type &&
3153 ml->alen == mreq->mr_alen &&
3154 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3155 if (--ml->count == 0) {
3156 struct net_device *dev;
3157 *mlp = ml->next;
ad959e76
ED		 3158 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3159 if (dev)
1da177e4 3160 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3161 kfree(ml);
3162 }
3163 rtnl_unlock();
3164 return 0;
3165 }
3166 }
3167 rtnl_unlock();
3168 return -EADDRNOTAVAIL;
3169}
3170
3171static void packet_flush_mclist(struct sock *sk)
3172{
3173 struct packet_sock *po = pkt_sk(sk);
3174 struct packet_mclist *ml;
3175
3176 if (!po->mclist)
3177 return;
3178
3179 rtnl_lock();
3180 while ((ml = po->mclist) != NULL) {
3181 struct net_device *dev;
3182
3183 po->mclist = ml->next;
ad959e76
ED		 3184 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3185 if (dev != NULL)
1da177e4 3186 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3187 kfree(ml);
3188 }
3189 rtnl_unlock();
3190}
1da177e4
LT		 3191
3192static int
b7058842 3193packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3194{
3195 struct sock *sk = sock->sk;
8dc41944 3196 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3197 int ret;
3198
3199 if (level != SOL_PACKET)
3200 return -ENOPROTOOPT;
3201
69e3c75f 3202 switch (optname) {
1ce4f28b 3203 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3204 case PACKET_DROP_MEMBERSHIP:
3205 {
0fb375fb
EB		 3206 struct packet_mreq_max mreq;
3207 int len = optlen;
3208 memset(&mreq, 0, sizeof(mreq));
3209 if (len < sizeof(struct packet_mreq))
1da177e4 3210 return -EINVAL;
0fb375fb
EB		 3211 if (len > sizeof(mreq))
3212 len = sizeof(mreq);
40d4e3df 3213 if (copy_from_user(&mreq, optval, len))
1da177e4 3214 return -EFAULT;
0fb375fb
EB		 3215 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3216 return -EINVAL;
1da177e4
LT		 3217 if (optname == PACKET_ADD_MEMBERSHIP)
3218 ret = packet_mc_add(sk, &mreq);
3219 else
3220 ret = packet_mc_drop(sk, &mreq);
3221 return ret;
3222 }
a2efcfa0 3223
1da177e4 3224 case PACKET_RX_RING:
69e3c75f 3225 case PACKET_TX_RING:
1da177e4 3226 {
f6fb8f10 3227 union tpacket_req_u req_u;
3228 int len;
1da177e4 3229
f6fb8f10 3230 switch (po->tp_version) {
3231 case TPACKET_V1:
3232 case TPACKET_V2:
3233 len = sizeof(req_u.req);
3234 break;
3235 case TPACKET_V3:
3236 default:
3237 len = sizeof(req_u.req3);
3238 break;
3239 }
3240 if (optlen < len)
1da177e4 3241 return -EINVAL;
bfd5f4a3
SS		 3242 if (pkt_sk(sk)->has_vnet_hdr)
3243 return -EINVAL;
f6fb8f10 3244 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3245 return -EFAULT;
f6fb8f10 3246 return packet_set_ring(sk, &req_u, 0,
3247 optname == PACKET_TX_RING);
1da177e4
LT		 3248 }
3249 case PACKET_COPY_THRESH:
3250 {
3251 int val;
3252
40d4e3df 3253 if (optlen != sizeof(val))
1da177e4 3254 return -EINVAL;
40d4e3df 3255 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3256 return -EFAULT;
3257
3258 pkt_sk(sk)->copy_thresh = val;
3259 return 0;
3260 }
bbd6ef87
PM		 3261 case PACKET_VERSION:
3262 {
3263 int val;
3264
3265 if (optlen != sizeof(val))
3266 return -EINVAL;
69e3c75f 3267 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3268 return -EBUSY;
3269 if (copy_from_user(&val, optval, sizeof(val)))
3270 return -EFAULT;
3271 switch (val) {
3272 case TPACKET_V1:
3273 case TPACKET_V2:
f6fb8f10 3274 case TPACKET_V3:
bbd6ef87
PM		 3275 po->tp_version = val;
3276 return 0;
3277 default:
3278 return -EINVAL;
3279 }
3280 }
8913336a
PM		 3281 case PACKET_RESERVE:
3282 {
3283 unsigned int val;
3284
3285 if (optlen != sizeof(val))
3286 return -EINVAL;
69e3c75f 3287 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3288 return -EBUSY;
3289 if (copy_from_user(&val, optval, sizeof(val)))
3290 return -EFAULT;
3291 po->tp_reserve = val;
3292 return 0;
3293 }
69e3c75f
JB		 3294 case PACKET_LOSS:
3295 {
3296 unsigned int val;
3297
3298 if (optlen != sizeof(val))
3299 return -EINVAL;
3300 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3301 return -EBUSY;
3302 if (copy_from_user(&val, optval, sizeof(val)))
3303 return -EFAULT;
3304 po->tp_loss = !!val;
3305 return 0;
3306 }
8dc41944
HX		 3307 case PACKET_AUXDATA:
3308 {
3309 int val;
3310
3311 if (optlen < sizeof(val))
3312 return -EINVAL;
3313 if (copy_from_user(&val, optval, sizeof(val)))
3314 return -EFAULT;
3315
3316 po->auxdata = !!val;
3317 return 0;
3318 }
80feaacb
PWJ		 3319 case PACKET_ORIGDEV:
3320 {
3321 int val;
3322
3323 if (optlen < sizeof(val))
3324 return -EINVAL;
3325 if (copy_from_user(&val, optval, sizeof(val)))
3326 return -EFAULT;
3327
3328 po->origdev = !!val;
3329 return 0;
3330 }
bfd5f4a3
SS		 3331 case PACKET_VNET_HDR:
3332 {
3333 int val;
3334
3335 if (sock->type != SOCK_RAW)
3336 return -EINVAL;
3337 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3338 return -EBUSY;
3339 if (optlen < sizeof(val))
3340 return -EINVAL;
3341 if (copy_from_user(&val, optval, sizeof(val)))
3342 return -EFAULT;
3343
3344 po->has_vnet_hdr = !!val;
3345 return 0;
3346 }
614f60fa
SM		 3347 case PACKET_TIMESTAMP:
3348 {
3349 int val;
3350
3351 if (optlen != sizeof(val))
3352 return -EINVAL;
3353 if (copy_from_user(&val, optval, sizeof(val)))
3354 return -EFAULT;
3355
3356 po->tp_tstamp = val;
3357 return 0;
3358 }
dc99f600
DM		 3359 case PACKET_FANOUT:
3360 {
3361 int val;
3362
3363 if (optlen != sizeof(val))
3364 return -EINVAL;
3365 if (copy_from_user(&val, optval, sizeof(val)))
3366 return -EFAULT;
3367
3368 return fanout_add(sk, val & 0xffff, val >> 16);
3369 }
5920cd3a
PC		 3370 case PACKET_TX_HAS_OFF:
3371 {
3372 unsigned int val;
3373
3374 if (optlen != sizeof(val))
3375 return -EINVAL;
3376 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3377 return -EBUSY;
3378 if (copy_from_user(&val, optval, sizeof(val)))
3379 return -EFAULT;
3380 po->tp_tx_has_off = !!val;
3381 return 0;
3382 }
d346a3fa
DB		 3383 case PACKET_QDISC_BYPASS:
3384 {
3385 int val;
3386
3387 if (optlen != sizeof(val))
3388 return -EINVAL;
3389 if (copy_from_user(&val, optval, sizeof(val)))
3390 return -EFAULT;
3391
3392 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3393 return 0;
3394 }
1da177e4
LT		 3395 default:
3396 return -ENOPROTOOPT;
3397 }
3398}
3399
3400static int packet_getsockopt(struct socket *sock, int level, int optname,
3401 char __user *optval, int __user *optlen)
3402{
3403 int len;
c06fff6e 3404 int val, lv = sizeof(val);
1da177e4
LT		 3405 struct sock *sk = sock->sk;
3406 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3407 void *data = &val;
ee80fbf3 3408 union tpacket_stats_u st;
1da177e4
LT		 3409
3410 if (level != SOL_PACKET)
3411 return -ENOPROTOOPT;
3412
8ae55f04
KK		 3413 if (get_user(len, optlen))
3414 return -EFAULT;
1da177e4
LT		 3415
3416 if (len < 0)
3417 return -EINVAL;
1ce4f28b 3418
69e3c75f 3419 switch (optname) {
1da177e4 3420 case PACKET_STATISTICS:
1da177e4 3421 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3422 memcpy(&st, &po->stats, sizeof(st));
3423 memset(&po->stats, 0, sizeof(po->stats));
3424 spin_unlock_bh(&sk->sk_receive_queue.lock);
3425
f6fb8f10 3426 if (po->tp_version == TPACKET_V3) {
c06fff6e 3427 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3428 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3429 data = &st.stats3;
f6fb8f10 3430 } else {
c06fff6e 3431 lv = sizeof(struct tpacket_stats);
8bcdeaff 3432 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3433 data = &st.stats1;
f6fb8f10 3434 }
ee80fbf3 3435
8dc41944
HX		 3436 break;
3437 case PACKET_AUXDATA:
8dc41944 3438 val = po->auxdata;
80feaacb
PWJ		 3439 break;
3440 case PACKET_ORIGDEV:
80feaacb 3441 val = po->origdev;
bfd5f4a3
SS		 3442 break;
3443 case PACKET_VNET_HDR:
bfd5f4a3 3444 val = po->has_vnet_hdr;
1da177e4 3445 break;
bbd6ef87 3446 case PACKET_VERSION:
bbd6ef87 3447 val = po->tp_version;
bbd6ef87
PM		 3448 break;
3449 case PACKET_HDRLEN:
3450 if (len > sizeof(int))
3451 len = sizeof(int);
3452 if (copy_from_user(&val, optval, len))
3453 return -EFAULT;
3454 switch (val) {
3455 case TPACKET_V1:
3456 val = sizeof(struct tpacket_hdr);
3457 break;
3458 case TPACKET_V2:
3459 val = sizeof(struct tpacket2_hdr);
3460 break;
f6fb8f10 3461 case TPACKET_V3:
3462 val = sizeof(struct tpacket3_hdr);
3463 break;
bbd6ef87
PM		 3464 default:
3465 return -EINVAL;
3466 }
bbd6ef87 3467 break;
8913336a 3468 case PACKET_RESERVE:
8913336a 3469 val = po->tp_reserve;
8913336a 3470 break;
69e3c75f 3471 case PACKET_LOSS:
69e3c75f 3472 val = po->tp_loss;
69e3c75f 3473 break;
614f60fa 3474 case PACKET_TIMESTAMP:
614f60fa 3475 val = po->tp_tstamp;
614f60fa 3476 break;
dc99f600 3477 case PACKET_FANOUT:
dc99f600
DM		 3478 val = (po->fanout ?
3479 ((u32)po->fanout->id |
77f65ebd
WB		 3480 ((u32)po->fanout->type << 16) |
3481 ((u32)po->fanout->flags << 24)) :
dc99f600 3482 0);
dc99f600 3483 break;
5920cd3a
PC		 3484 case PACKET_TX_HAS_OFF:
3485 val = po->tp_tx_has_off;
3486 break;
d346a3fa
DB		 3487 case PACKET_QDISC_BYPASS:
3488 val = packet_use_direct_xmit(po);
3489 break;
1da177e4
LT		 3490 default:
3491 return -ENOPROTOOPT;
3492 }
3493
c06fff6e
ED		 3494 if (len > lv)
3495 len = lv;
8ae55f04
KK		 3496 if (put_user(len, optlen))
3497 return -EFAULT;
8dc41944
HX		 3498 if (copy_to_user(optval, data, len))
3499 return -EFAULT;
8ae55f04 3500 return 0;
1da177e4
LT		 3501}
3502
3503
351638e7
JP		 3504static int packet_notifier(struct notifier_block *this,
3505 unsigned long msg, void *ptr)
1da177e4
LT		 3506{
3507 struct sock *sk;
351638e7 3508 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3509 struct net *net = dev_net(dev);
1da177e4 3510
808f5114 3511 rcu_read_lock();
b67bfe0d 3512 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3513 struct packet_sock *po = pkt_sk(sk);
3514
3515 switch (msg) {
3516 case NETDEV_UNREGISTER:
1da177e4
LT		 3517 if (po->mclist)
3518 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3519 /* fallthrough */
3520
1da177e4
LT		 3521 case NETDEV_DOWN:
3522 if (dev->ifindex == po->ifindex) {
3523 spin_lock(&po->bind_lock);
3524 if (po->running) {
ce06b03e 3525 __unregister_prot_hook(sk, false);
1da177e4
LT		 3526 sk->sk_err = ENETDOWN;
3527 if (!sock_flag(sk, SOCK_DEAD))
3528 sk->sk_error_report(sk);
3529 }
3530 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3531 packet_cached_dev_reset(po);
1da177e4 3532 po->ifindex = -1;
160ff18a
BG		 3533 if (po->prot_hook.dev)
3534 dev_put(po->prot_hook.dev);
1da177e4
LT		 3535 po->prot_hook.dev = NULL;
3536 }
3537 spin_unlock(&po->bind_lock);
3538 }
3539 break;
3540 case NETDEV_UP:
808f5114 3541 if (dev->ifindex == po->ifindex) {
3542 spin_lock(&po->bind_lock);
ce06b03e
DM		 3543 if (po->num)
3544 register_prot_hook(sk);
808f5114 3545 spin_unlock(&po->bind_lock);
1da177e4 3546 }
1da177e4
LT		 3547 break;
3548 }
3549 }
808f5114 3550 rcu_read_unlock();
1da177e4
LT		 3551 return NOTIFY_DONE;
3552}
3553
3554
3555static int packet_ioctl(struct socket *sock, unsigned int cmd,
3556 unsigned long arg)
3557{
3558 struct sock *sk = sock->sk;
3559
69e3c75f 3560 switch (cmd) {
40d4e3df
ED		 3561 case SIOCOUTQ:
3562 {
3563 int amount = sk_wmem_alloc_get(sk);
31e6d363 3564
40d4e3df
ED		 3565 return put_user(amount, (int __user *)arg);
3566 }
3567 case SIOCINQ:
3568 {
3569 struct sk_buff *skb;
3570 int amount = 0;
3571
3572 spin_lock_bh(&sk->sk_receive_queue.lock);
3573 skb = skb_peek(&sk->sk_receive_queue);
3574 if (skb)
3575 amount = skb->len;
3576 spin_unlock_bh(&sk->sk_receive_queue.lock);
3577 return put_user(amount, (int __user *)arg);
3578 }
3579 case SIOCGSTAMP:
3580 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3581 case SIOCGSTAMPNS:
3582 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3583
1da177e4 3584#ifdef CONFIG_INET
40d4e3df
ED		 3585 case SIOCADDRT:
3586 case SIOCDELRT:
3587 case SIOCDARP:
3588 case SIOCGARP:
3589 case SIOCSARP:
3590 case SIOCGIFADDR:
3591 case SIOCSIFADDR:
3592 case SIOCGIFBRDADDR:
3593 case SIOCSIFBRDADDR:
3594 case SIOCGIFNETMASK:
3595 case SIOCSIFNETMASK:
3596 case SIOCGIFDSTADDR:
3597 case SIOCSIFDSTADDR:
3598 case SIOCSIFFLAGS:
40d4e3df 3599 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3600#endif
3601
40d4e3df
ED		 3602 default:
3603 return -ENOIOCTLCMD;
1da177e4
LT		 3604 }
3605 return 0;
3606}
3607
40d4e3df 3608static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3609 poll_table *wait)
3610{
3611 struct sock *sk = sock->sk;
3612 struct packet_sock *po = pkt_sk(sk);
3613 unsigned int mask = datagram_poll(file, sock, wait);
3614
3615 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3616 if (po->rx_ring.pg_vec) {
f6fb8f10 3617 if (!packet_previous_rx_frame(po, &po->rx_ring,
3618 TP_STATUS_KERNEL))
1da177e4
LT		 3619 mask |= POLLIN | POLLRDNORM;
3620 }
3621 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3622 spin_lock_bh(&sk->sk_write_queue.lock);
3623 if (po->tx_ring.pg_vec) {
3624 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3625 mask |= POLLOUT | POLLWRNORM;
3626 }
3627 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3628 return mask;
3629}
3630
3631
3632/* Dirty? Well, I still did not learn better way to account
3633 * for user mmaps.
3634 */
3635
3636static void packet_mm_open(struct vm_area_struct *vma)
3637{
3638 struct file *file = vma->vm_file;
40d4e3df 3639 struct socket *sock = file->private_data;
1da177e4 3640 struct sock *sk = sock->sk;
1ce4f28b 3641
1da177e4
LT		 3642 if (sk)
3643 atomic_inc(&pkt_sk(sk)->mapped);
3644}
3645
3646static void packet_mm_close(struct vm_area_struct *vma)
3647{
3648 struct file *file = vma->vm_file;
40d4e3df 3649 struct socket *sock = file->private_data;
1da177e4 3650 struct sock *sk = sock->sk;
1ce4f28b 3651
1da177e4
LT		 3652 if (sk)
3653 atomic_dec(&pkt_sk(sk)->mapped);
3654}
3655
f0f37e2f 3656static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3657 .open = packet_mm_open,
3658 .close = packet_mm_close,
1da177e4
LT		 3659};
3660
0e3125c7
NH		 3661static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3662 unsigned int len)
1da177e4
LT		 3663{
3664 int i;
3665
4ebf0ae2 3666 for (i = 0; i < len; i++) {
0e3125c7 3667 if (likely(pg_vec[i].buffer)) {
c56b4d90 3668 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3669 vfree(pg_vec[i].buffer);
3670 else
3671 free_pages((unsigned long)pg_vec[i].buffer,
3672 order);
3673 pg_vec[i].buffer = NULL;
3674 }
1da177e4
LT		 3675 }
3676 kfree(pg_vec);
3677}
3678
eea49cc9 3679static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3680{
f0d4eb29 3681 char *buffer;
0e3125c7
NH		 3682 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3683 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3684
3685 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3686 if (buffer)
3687 return buffer;
3688
f0d4eb29 3689 /* __get_free_pages failed, fall back to vmalloc */
bbce5a59 3690 buffer = vzalloc((1 << order) * PAGE_SIZE);
0e3125c7
NH		 3691 if (buffer)
3692 return buffer;
3693
f0d4eb29 3694 /* vmalloc failed, lets dig into swap here */
0e3125c7 3695 gfp_flags &= ~__GFP_NORETRY;
f0d4eb29 3696 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3697 if (buffer)
3698 return buffer;
3699
f0d4eb29 3700 /* complete and utter failure */
0e3125c7 3701 return NULL;
4ebf0ae2
DM		 3702}
3703
0e3125c7 3704static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3705{
3706 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3707 struct pgv *pg_vec;
4ebf0ae2
DM		 3708 int i;
3709
0e3125c7 3710 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3711 if (unlikely(!pg_vec))
3712 goto out;
3713
3714 for (i = 0; i < block_nr; i++) {
c56b4d90 3715 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3716 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3717 goto out_free_pgvec;
3718 }
3719
3720out:
3721 return pg_vec;
3722
3723out_free_pgvec:
3724 free_pg_vec(pg_vec, order, block_nr);
3725 pg_vec = NULL;
3726 goto out;
3727}
1da177e4 3728
f6fb8f10 3729static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3730 int closing, int tx_ring)
1da177e4 3731{
0e3125c7 3732 struct pgv *pg_vec = NULL;
1da177e4 3733 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3734 int was_running, order = 0;
69e3c75f
JB		 3735 struct packet_ring_buffer *rb;
3736 struct sk_buff_head *rb_queue;
0e11c91e 3737 __be16 num;
f6fb8f10 3738 int err = -EINVAL;
3739 /* Added to avoid minimal code churn */
3740 struct tpacket_req *req = &req_u->req;
3741
3742 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3743 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3744 WARN(1, "Tx-ring is not supported.\n");
3745 goto out;
3746 }
1ce4f28b 3747
69e3c75f
JB		 3748 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3749 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3750
69e3c75f
JB		 3751 err = -EBUSY;
3752 if (!closing) {
3753 if (atomic_read(&po->mapped))
3754 goto out;
b0138408 3755 if (packet_read_pending(rb))
69e3c75f
JB		 3756 goto out;
3757 }
1da177e4 3758
69e3c75f
JB		 3759 if (req->tp_block_nr) {
3760 /* Sanity tests and some calculations */
3761 err = -EBUSY;
3762 if (unlikely(rb->pg_vec))
3763 goto out;
1da177e4 3764
bbd6ef87
PM		 3765 switch (po->tp_version) {
3766 case TPACKET_V1:
3767 po->tp_hdrlen = TPACKET_HDRLEN;
3768 break;
3769 case TPACKET_V2:
3770 po->tp_hdrlen = TPACKET2_HDRLEN;
3771 break;
f6fb8f10 3772 case TPACKET_V3:
3773 po->tp_hdrlen = TPACKET3_HDRLEN;
3774 break;
bbd6ef87
PM		 3775 }
3776
69e3c75f 3777 err = -EINVAL;
4ebf0ae2 3778 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3779 goto out;
4ebf0ae2 3780 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3781 goto out;
8913336a 3782 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3783 po->tp_reserve))
3784 goto out;
4ebf0ae2 3785 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3786 goto out;
1da177e4 3787
69e3c75f
JB		 3788 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3789 if (unlikely(rb->frames_per_block <= 0))
3790 goto out;
3791 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3792 req->tp_frame_nr))
3793 goto out;
1da177e4
LT		 3794
3795 err = -ENOMEM;
4ebf0ae2
DM		 3796 order = get_order(req->tp_block_size);
3797 pg_vec = alloc_pg_vec(req, order);
3798 if (unlikely(!pg_vec))
1da177e4 3799 goto out;
f6fb8f10 3800 switch (po->tp_version) {
3801 case TPACKET_V3:
3802 /* Transmit path is not supported. We checked
3803 * it above but just being paranoid
3804 */
3805 if (!tx_ring)
3806 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
d7cf0c34 3807 break;
f6fb8f10 3808 default:
3809 break;
3810 }
69e3c75f
JB		 3811 }
3812 /* Done */
3813 else {
3814 err = -EINVAL;
4ebf0ae2 3815 if (unlikely(req->tp_frame_nr))
69e3c75f 3816 goto out;
1da177e4
LT		 3817 }
3818
3819 lock_sock(sk);
3820
3821 /* Detach socket from network */
3822 spin_lock(&po->bind_lock);
3823 was_running = po->running;
3824 num = po->num;
3825 if (was_running) {
1da177e4 3826 po->num = 0;
ce06b03e 3827 __unregister_prot_hook(sk, false);
1da177e4
LT		 3828 }
3829 spin_unlock(&po->bind_lock);
1ce4f28b 3830
1da177e4
LT		 3831 synchronize_net();
3832
3833 err = -EBUSY;
905db440 3834 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3835 if (closing || atomic_read(&po->mapped) == 0) {
3836 err = 0;
69e3c75f 3837 spin_lock_bh(&rb_queue->lock);
c053fd96 3838 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3839 rb->frame_max = (req->tp_frame_nr - 1);
3840 rb->head = 0;
3841 rb->frame_size = req->tp_frame_size;
3842 spin_unlock_bh(&rb_queue->lock);
3843
c053fd96
CG		 3844 swap(rb->pg_vec_order, order);
3845 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3846
3847 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3848 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3849 tpacket_rcv : packet_rcv;
3850 skb_queue_purge(rb_queue);
1da177e4 3851 if (atomic_read(&po->mapped))
40d4e3df
ED		 3852 pr_err("packet_mmap: vma is busy: %d\n",
3853 atomic_read(&po->mapped));
1da177e4 3854 }
905db440 3855 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3856
3857 spin_lock(&po->bind_lock);
ce06b03e 3858 if (was_running) {
1da177e4 3859 po->num = num;
ce06b03e 3860 register_prot_hook(sk);
1da177e4
LT		 3861 }
3862 spin_unlock(&po->bind_lock);
f6fb8f10 3863 if (closing && (po->tp_version > TPACKET_V2)) {
3864 /* Because we don't support block-based V3 on tx-ring */
3865 if (!tx_ring)
3866 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3867 }
1da177e4
LT		 3868 release_sock(sk);
3869
1da177e4
LT		 3870 if (pg_vec)
3871 free_pg_vec(pg_vec, order, req->tp_block_nr);
3872out:
3873 return err;
3874}
3875
69e3c75f
JB		 3876static int packet_mmap(struct file *file, struct socket *sock,
3877 struct vm_area_struct *vma)
1da177e4
LT		 3878{
3879 struct sock *sk = sock->sk;
3880 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3881 unsigned long size, expected_size;
3882 struct packet_ring_buffer *rb;
1da177e4
LT		 3883 unsigned long start;
3884 int err = -EINVAL;
3885 int i;
3886
3887 if (vma->vm_pgoff)
3888 return -EINVAL;
3889
905db440 3890 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3891
3892 expected_size = 0;
3893 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3894 if (rb->pg_vec) {
3895 expected_size += rb->pg_vec_len
3896 * rb->pg_vec_pages
3897 * PAGE_SIZE;
3898 }
3899 }
3900
3901 if (expected_size == 0)
1da177e4 3902 goto out;
69e3c75f
JB		 3903
3904 size = vma->vm_end - vma->vm_start;
3905 if (size != expected_size)
1da177e4
LT		 3906 goto out;
3907
1da177e4 3908 start = vma->vm_start;
69e3c75f
JB		 3909 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3910 if (rb->pg_vec == NULL)
3911 continue;
3912
3913 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3914 struct page *page;
3915 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3916 int pg_num;
3917
c56b4d90
CG		 3918 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3919 page = pgv_to_page(kaddr);
69e3c75f
JB		 3920 err = vm_insert_page(vma, start, page);
3921 if (unlikely(err))
3922 goto out;
3923 start += PAGE_SIZE;
0e3125c7 3924 kaddr += PAGE_SIZE;
69e3c75f 3925 }
4ebf0ae2 3926 }
1da177e4 3927 }
69e3c75f 3928
4ebf0ae2 3929 atomic_inc(&po->mapped);
1da177e4
LT		 3930 vma->vm_ops = &packet_mmap_ops;
3931 err = 0;
3932
3933out:
905db440 3934 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3935 return err;
3936}
1da177e4 3937
90ddc4f0 3938static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3939 .family = PF_PACKET,
3940 .owner = THIS_MODULE,
3941 .release = packet_release,
3942 .bind = packet_bind_spkt,
3943 .connect = sock_no_connect,
3944 .socketpair = sock_no_socketpair,
3945 .accept = sock_no_accept,
3946 .getname = packet_getname_spkt,
3947 .poll = datagram_poll,
3948 .ioctl = packet_ioctl,
3949 .listen = sock_no_listen,
3950 .shutdown = sock_no_shutdown,
3951 .setsockopt = sock_no_setsockopt,
3952 .getsockopt = sock_no_getsockopt,
3953 .sendmsg = packet_sendmsg_spkt,
3954 .recvmsg = packet_recvmsg,
3955 .mmap = sock_no_mmap,
3956 .sendpage = sock_no_sendpage,
3957};
1da177e4 3958
90ddc4f0 3959static const struct proto_ops packet_ops = {
1da177e4
LT		 3960 .family = PF_PACKET,
3961 .owner = THIS_MODULE,
3962 .release = packet_release,
3963 .bind = packet_bind,
3964 .connect = sock_no_connect,
3965 .socketpair = sock_no_socketpair,
3966 .accept = sock_no_accept,
1ce4f28b 3967 .getname = packet_getname,
1da177e4
LT		 3968 .poll = packet_poll,
3969 .ioctl = packet_ioctl,
3970 .listen = sock_no_listen,
3971 .shutdown = sock_no_shutdown,
3972 .setsockopt = packet_setsockopt,
3973 .getsockopt = packet_getsockopt,
3974 .sendmsg = packet_sendmsg,
3975 .recvmsg = packet_recvmsg,
3976 .mmap = packet_mmap,
3977 .sendpage = sock_no_sendpage,
3978};
3979
ec1b4cf7 3980static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3981 .family = PF_PACKET,
3982 .create = packet_create,
3983 .owner = THIS_MODULE,
3984};
3985
3986static struct notifier_block packet_netdev_notifier = {
40d4e3df 3987 .notifier_call = packet_notifier,
1da177e4
LT		 3988};
3989
3990#ifdef CONFIG_PROC_FS
1da177e4
LT		 3991
3992static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3993 __acquires(RCU)
1da177e4 3994{
e372c414 3995 struct net *net = seq_file_net(seq);
808f5114 3996
3997 rcu_read_lock();
3998 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3999}
4000
4001static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4002{
1bf40954 4003 struct net *net = seq_file_net(seq);
808f5114 4004 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 4005}
4006
4007static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 4008 __releases(RCU)
1da177e4 4009{
808f5114 4010 rcu_read_unlock();
1da177e4
LT		 4011}
4012
1ce4f28b 4013static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 4014{
4015 if (v == SEQ_START_TOKEN)
4016 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4017 else {
b7ceabd9 4018 struct sock *s = sk_entry(v);
1da177e4
LT		 4019 const struct packet_sock *po = pkt_sk(s);
4020
4021 seq_printf(seq,
71338aa7 4022 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 4023 s,
4024 atomic_read(&s->sk_refcnt),
4025 s->sk_type,
4026 ntohs(po->num),
4027 po->ifindex,
4028 po->running,
4029 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4030 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4031 sock_i_ino(s));
1da177e4
LT		 4032 }
4033
4034 return 0;
4035}
4036
56b3d975 4037static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4038 .start = packet_seq_start,
4039 .next = packet_seq_next,
4040 .stop = packet_seq_stop,
4041 .show = packet_seq_show,
4042};
4043
4044static int packet_seq_open(struct inode *inode, struct file *file)
4045{
e372c414
DL		 4046 return seq_open_net(inode, file, &packet_seq_ops,
4047 sizeof(struct seq_net_private));
1da177e4
LT		 4048}
4049
da7071d7 4050static const struct file_operations packet_seq_fops = {
1da177e4
LT		 4051 .owner = THIS_MODULE,
4052 .open = packet_seq_open,
4053 .read = seq_read,
4054 .llseek = seq_lseek,
e372c414 4055 .release = seq_release_net,
1da177e4
LT		 4056};
4057
4058#endif
4059
2c8c1e72 4060static int __net_init packet_net_init(struct net *net)
d12d01d6 4061{
0fa7fa98 4062 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4063 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4064
d4beaa66 4065 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 4066 return -ENOMEM;
4067
4068 return 0;
4069}
4070
2c8c1e72 4071static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4072{
ece31ffd 4073 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 4074}
4075
4076static struct pernet_operations packet_net_ops = {
4077 .init = packet_net_init,
4078 .exit = packet_net_exit,
4079};
4080
4081
1da177e4
LT		 4082static void __exit packet_exit(void)
4083{
1da177e4 4084 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4085 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4086 sock_unregister(PF_PACKET);
4087 proto_unregister(&packet_proto);
4088}
4089
4090static int __init packet_init(void)
4091{
4092 int rc = proto_register(&packet_proto, 0);
4093
4094 if (rc != 0)
4095 goto out;
4096
4097 sock_register(&packet_family_ops);
d12d01d6 4098 register_pernet_subsys(&packet_net_ops);
1da177e4 4099 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4100out:
4101 return rc;
4102}
4103
4104module_init(packet_init);
4105module_exit(packet_exit);
4106MODULE_LICENSE("GPL");
4107MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 1.037127 seconds and 5 git commands to generate.