projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
packet: if hw/sw ts enabled in rx/tx ring, report which ts we got
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
1da177e4
LT		 91
92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
ce06b03e
DM		 240/* register_prot_hook must be invoked with the po->bind_lock held,
241 * or from a context in which asynchronous accesses to the packet
242 * socket is not possible (packet_create()).
243 */
244static void register_prot_hook(struct sock *sk)
245{
246 struct packet_sock *po = pkt_sk(sk);
247 if (!po->running) {
dc99f600
DM		 248 if (po->fanout)
249 __fanout_link(sk, po);
250 else
251 dev_add_pack(&po->prot_hook);
ce06b03e
DM		 252 sock_hold(sk);
253 po->running = 1;
254 }
255}
256
257/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
258 * held. If the sync parameter is true, we will temporarily drop
259 * the po->bind_lock and do a synchronize_net to make sure no
260 * asynchronous packet processing paths still refer to the elements
261 * of po->prot_hook. If the sync parameter is false, it is the
262 * callers responsibility to take care of this.
263 */
264static void __unregister_prot_hook(struct sock *sk, bool sync)
265{
266 struct packet_sock *po = pkt_sk(sk);
267
268 po->running = 0;
dc99f600
DM		 269 if (po->fanout)
270 __fanout_unlink(sk, po);
271 else
272 __dev_remove_pack(&po->prot_hook);
ce06b03e
DM		 273 __sock_put(sk);
274
275 if (sync) {
276 spin_unlock(&po->bind_lock);
277 synchronize_net();
278 spin_lock(&po->bind_lock);
279 }
280}
281
282static void unregister_prot_hook(struct sock *sk, bool sync)
283{
284 struct packet_sock *po = pkt_sk(sk);
285
286 if (po->running)
287 __unregister_prot_hook(sk, sync);
288}
289
f6dafa95 290static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 291{
292 if (is_vmalloc_addr(addr))
293 return vmalloc_to_page(addr);
294 return virt_to_page(addr);
295}
296
69e3c75f 297static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 298{
184f489e 299 union tpacket_uhdr h;
1da177e4 300
69e3c75f 301 h.raw = frame;
bbd6ef87
PM		 302 switch (po->tp_version) {
303 case TPACKET_V1:
69e3c75f 304 h.h1->tp_status = status;
0af55bb5 305 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 306 break;
307 case TPACKET_V2:
69e3c75f 308 h.h2->tp_status = status;
0af55bb5 309 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 310 break;
f6fb8f10 311 case TPACKET_V3:
69e3c75f 312 default:
f6fb8f10 313 WARN(1, "TPACKET version not supported.\n");
69e3c75f 314 BUG();
bbd6ef87 315 }
69e3c75f
JB		 316
317 smp_wmb();
bbd6ef87
PM		 318}
319
69e3c75f 320static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 321{
184f489e 322 union tpacket_uhdr h;
bbd6ef87 323
69e3c75f
JB		 324 smp_rmb();
325
bbd6ef87
PM		 326 h.raw = frame;
327 switch (po->tp_version) {
328 case TPACKET_V1:
0af55bb5 329 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 330 return h.h1->tp_status;
bbd6ef87 331 case TPACKET_V2:
0af55bb5 332 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 333 return h.h2->tp_status;
f6fb8f10 334 case TPACKET_V3:
69e3c75f 335 default:
f6fb8f10 336 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 337 BUG();
338 return 0;
bbd6ef87 339 }
1da177e4 340}
69e3c75f 341
b9c32fb2
DB		 342static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
343 unsigned int flags)
7a51384c
DB		 344{
345 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
346
347 if (shhwtstamps) {
348 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
349 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
b9c32fb2 350 return TP_STATUS_TS_SYS_HARDWARE;
7a51384c
DB		 351 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
352 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
b9c32fb2 353 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 354 }
355
356 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 357 return TP_STATUS_TS_SOFTWARE;
7a51384c 358
b9c32fb2 359 return 0;
7a51384c
DB		 360}
361
b9c32fb2
DB		 362static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
363 struct sk_buff *skb)
2e31396f
WB		 364{
365 union tpacket_uhdr h;
366 struct timespec ts;
b9c32fb2 367 __u32 ts_status;
2e31396f 368
b9c32fb2
DB		 369 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
370 return 0;
2e31396f
WB		 371
372 h.raw = frame;
373 switch (po->tp_version) {
374 case TPACKET_V1:
375 h.h1->tp_sec = ts.tv_sec;
376 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
377 break;
378 case TPACKET_V2:
379 h.h2->tp_sec = ts.tv_sec;
380 h.h2->tp_nsec = ts.tv_nsec;
381 break;
382 case TPACKET_V3:
383 default:
384 WARN(1, "TPACKET version not supported.\n");
385 BUG();
386 }
387
388 /* one flush is safe, as both fields always lie on the same cacheline */
389 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
390 smp_wmb();
b9c32fb2
DB		 391
392 return ts_status;
2e31396f
WB		 393}
394
69e3c75f
JB		 395static void *packet_lookup_frame(struct packet_sock *po,
396 struct packet_ring_buffer *rb,
397 unsigned int position,
398 int status)
399{
400 unsigned int pg_vec_pos, frame_offset;
184f489e 401 union tpacket_uhdr h;
69e3c75f
JB		 402
403 pg_vec_pos = position / rb->frames_per_block;
404 frame_offset = position % rb->frames_per_block;
405
0e3125c7
NH		 406 h.raw = rb->pg_vec[pg_vec_pos].buffer +
407 (frame_offset * rb->frame_size);
69e3c75f
JB		 408
409 if (status != __packet_get_status(po, h.raw))
410 return NULL;
411
412 return h.raw;
413}
414
eea49cc9 415static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 416 struct packet_ring_buffer *rb,
417 int status)
418{
419 return packet_lookup_frame(po, rb, rb->head, status);
420}
421
bc59ba39 422static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 423{
424 del_timer_sync(&pkc->retire_blk_timer);
425}
426
427static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
428 int tx_ring,
429 struct sk_buff_head *rb_queue)
430{
bc59ba39 431 struct tpacket_kbdq_core *pkc;
f6fb8f10 432
433 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
434
435 spin_lock(&rb_queue->lock);
436 pkc->delete_blk_timer = 1;
437 spin_unlock(&rb_queue->lock);
438
439 prb_del_retire_blk_timer(pkc);
440}
441
442static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 443 struct tpacket_kbdq_core *pkc,
f6fb8f10 444 void (*func) (unsigned long))
445{
446 init_timer(&pkc->retire_blk_timer);
447 pkc->retire_blk_timer.data = (long)po;
448 pkc->retire_blk_timer.function = func;
449 pkc->retire_blk_timer.expires = jiffies;
450}
451
452static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
453{
bc59ba39 454 struct tpacket_kbdq_core *pkc;
f6fb8f10 455
456 if (tx_ring)
457 BUG();
458
459 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
460 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
461}
462
463static int prb_calc_retire_blk_tmo(struct packet_sock *po,
464 int blk_size_in_bytes)
465{
466 struct net_device *dev;
467 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 468 struct ethtool_cmd ecmd;
469 int err;
e440cf2c 470 u32 speed;
f6fb8f10 471
4bc71cb9
JP		 472 rtnl_lock();
473 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
474 if (unlikely(!dev)) {
475 rtnl_unlock();
f6fb8f10 476 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 477 }
478 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 479 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 480 rtnl_unlock();
481 if (!err) {
4bc71cb9
JP		 482 /*
483 * If the link speed is so slow you don't really
484 * need to worry about perf anyways
485 */
e440cf2c 486 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 487 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 488 } else {
489 msec = 1;
490 div = speed / 1000;
f6fb8f10 491 }
492 }
493
494 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
495
496 if (div)
497 mbits /= div;
498
499 tmo = mbits * msec;
500
501 if (div)
502 return tmo+1;
503 return tmo;
504}
505
bc59ba39 506static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 507 union tpacket_req_u *req_u)
508{
509 p1->feature_req_word = req_u->req3.tp_feature_req_word;
510}
511
512static void init_prb_bdqc(struct packet_sock *po,
513 struct packet_ring_buffer *rb,
514 struct pgv *pg_vec,
515 union tpacket_req_u *req_u, int tx_ring)
516{
bc59ba39 517 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
518 struct tpacket_block_desc *pbd;
f6fb8f10 519
520 memset(p1, 0x0, sizeof(*p1));
521
522 p1->knxt_seq_num = 1;
523 p1->pkbdq = pg_vec;
bc59ba39 524 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 525 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 526 p1->kblk_size = req_u->req3.tp_block_size;
527 p1->knum_blocks = req_u->req3.tp_block_nr;
528 p1->hdrlen = po->tp_hdrlen;
529 p1->version = po->tp_version;
530 p1->last_kactive_blk_num = 0;
531 po->stats_u.stats3.tp_freeze_q_cnt = 0;
532 if (req_u->req3.tp_retire_blk_tov)
533 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
534 else
535 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
536 req_u->req3.tp_block_size);
537 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
538 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
539
540 prb_init_ft_ops(p1, req_u);
541 prb_setup_retire_blk_timer(po, tx_ring);
542 prb_open_block(p1, pbd);
543}
544
545/* Do NOT update the last_blk_num first.
546 * Assumes sk_buff_head lock is held.
547 */
bc59ba39 548static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 549{
550 mod_timer(&pkc->retire_blk_timer,
551 jiffies + pkc->tov_in_jiffies);
552 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
553}
554
555/*
556 * Timer logic:
557 * 1) We refresh the timer only when we open a block.
558 * By doing this we don't waste cycles refreshing the timer
559 * on packet-by-packet basis.
560 *
561 * With a 1MB block-size, on a 1Gbps line, it will take
562 * i) ~8 ms to fill a block + ii) memcpy etc.
563 * In this cut we are not accounting for the memcpy time.
564 *
565 * So, if the user sets the 'tmo' to 10ms then the timer
566 * will never fire while the block is still getting filled
567 * (which is what we want). However, the user could choose
568 * to close a block early and that's fine.
569 *
570 * But when the timer does fire, we check whether or not to refresh it.
571 * Since the tmo granularity is in msecs, it is not too expensive
572 * to refresh the timer, lets say every '8' msecs.
573 * Either the user can set the 'tmo' or we can derive it based on
574 * a) line-speed and b) block-size.
575 * prb_calc_retire_blk_tmo() calculates the tmo.
576 *
577 */
578static void prb_retire_rx_blk_timer_expired(unsigned long data)
579{
580 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 581 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 582 unsigned int frozen;
bc59ba39 583 struct tpacket_block_desc *pbd;
f6fb8f10 584
585 spin_lock(&po->sk.sk_receive_queue.lock);
586
587 frozen = prb_queue_frozen(pkc);
588 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
589
590 if (unlikely(pkc->delete_blk_timer))
591 goto out;
592
593 /* We only need to plug the race when the block is partially filled.
594 * tpacket_rcv:
595 * lock(); increment BLOCK_NUM_PKTS; unlock()
596 * copy_bits() is in progress ...
597 * timer fires on other cpu:
598 * we can't retire the current block because copy_bits
599 * is in progress.
600 *
601 */
602 if (BLOCK_NUM_PKTS(pbd)) {
603 while (atomic_read(&pkc->blk_fill_in_prog)) {
604 /* Waiting for skb_copy_bits to finish... */
605 cpu_relax();
606 }
607 }
608
609 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
610 if (!frozen) {
611 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
612 if (!prb_dispatch_next_block(pkc, po))
613 goto refresh_timer;
614 else
615 goto out;
616 } else {
617 /* Case 1. Queue was frozen because user-space was
618 * lagging behind.
619 */
620 if (prb_curr_blk_in_use(pkc, pbd)) {
621 /*
622 * Ok, user-space is still behind.
623 * So just refresh the timer.
624 */
625 goto refresh_timer;
626 } else {
627 /* Case 2. queue was frozen,user-space caught up,
628 * now the link went idle && the timer fired.
629 * We don't have a block to close.So we open this
630 * block and restart the timer.
631 * opening a block thaws the queue,restarts timer
632 * Thawing/timer-refresh is a side effect.
633 */
634 prb_open_block(pkc, pbd);
635 goto out;
636 }
637 }
638 }
639
640refresh_timer:
641 _prb_refresh_rx_retire_blk_timer(pkc);
642
643out:
644 spin_unlock(&po->sk.sk_receive_queue.lock);
645}
646
eea49cc9 647static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 648 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 649{
650 /* Flush everything minus the block header */
651
652#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
653 u8 *start, *end;
654
655 start = (u8 *)pbd1;
656
657 /* Skip the block header(we know header WILL fit in 4K) */
658 start += PAGE_SIZE;
659
660 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
661 for (; start < end; start += PAGE_SIZE)
662 flush_dcache_page(pgv_to_page(start));
663
664 smp_wmb();
665#endif
666
667 /* Now update the block status. */
668
669 BLOCK_STATUS(pbd1) = status;
670
671 /* Flush the block header */
672
673#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
674 start = (u8 *)pbd1;
675 flush_dcache_page(pgv_to_page(start));
676
677 smp_wmb();
678#endif
679}
680
681/*
682 * Side effect:
683 *
684 * 1) flush the block
685 * 2) Increment active_blk_num
686 *
687 * Note:We DONT refresh the timer on purpose.
688 * Because almost always the next block will be opened.
689 */
bc59ba39 690static void prb_close_block(struct tpacket_kbdq_core *pkc1,
691 struct tpacket_block_desc *pbd1,
f6fb8f10 692 struct packet_sock *po, unsigned int stat)
693{
694 __u32 status = TP_STATUS_USER | stat;
695
696 struct tpacket3_hdr *last_pkt;
bc59ba39 697 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 698
699 if (po->stats.tp_drops)
700 status |= TP_STATUS_LOSING;
701
702 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
703 last_pkt->tp_next_offset = 0;
704
705 /* Get the ts of the last pkt */
706 if (BLOCK_NUM_PKTS(pbd1)) {
707 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
708 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
709 } else {
710 /* Ok, we tmo'd - so get the current time */
711 struct timespec ts;
712 getnstimeofday(&ts);
713 h1->ts_last_pkt.ts_sec = ts.tv_sec;
714 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
715 }
716
717 smp_wmb();
718
719 /* Flush the block */
720 prb_flush_block(pkc1, pbd1, status);
721
722 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
723}
724
eea49cc9 725static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 726{
727 pkc->reset_pending_on_curr_blk = 0;
728}
729
730/*
731 * Side effect of opening a block:
732 *
733 * 1) prb_queue is thawed.
734 * 2) retire_blk_timer is refreshed.
735 *
736 */
bc59ba39 737static void prb_open_block(struct tpacket_kbdq_core *pkc1,
738 struct tpacket_block_desc *pbd1)
f6fb8f10 739{
740 struct timespec ts;
bc59ba39 741 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 742
743 smp_rmb();
744
745 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd1))) {
746
747 /* We could have just memset this but we will lose the
748 * flexibility of making the priv area sticky
749 */
750 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
751 BLOCK_NUM_PKTS(pbd1) = 0;
752 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
753 getnstimeofday(&ts);
754 h1->ts_first_pkt.ts_sec = ts.tv_sec;
755 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
756 pkc1->pkblk_start = (char *)pbd1;
e3192690 757 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 758 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
759 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
760 pbd1->version = pkc1->version;
761 pkc1->prev = pkc1->nxt_offset;
762 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
763 prb_thaw_queue(pkc1);
764 _prb_refresh_rx_retire_blk_timer(pkc1);
765
766 smp_wmb();
767
768 return;
769 }
770
771 WARN(1, "ERROR block:%p is NOT FREE status:%d kactive_blk_num:%d\n",
772 pbd1, BLOCK_STATUS(pbd1), pkc1->kactive_blk_num);
773 dump_stack();
774 BUG();
775}
776
777/*
778 * Queue freeze logic:
779 * 1) Assume tp_block_nr = 8 blocks.
780 * 2) At time 't0', user opens Rx ring.
781 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
782 * 4) user-space is either sleeping or processing block '0'.
783 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
784 * it will close block-7,loop around and try to fill block '0'.
785 * call-flow:
786 * __packet_lookup_frame_in_block
787 * prb_retire_current_block()
788 * prb_dispatch_next_block()
789 * |->(BLOCK_STATUS == USER) evaluates to true
790 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
791 * 6) Now there are two cases:
792 * 6.1) Link goes idle right after the queue is frozen.
793 * But remember, the last open_block() refreshed the timer.
794 * When this timer expires,it will refresh itself so that we can
795 * re-open block-0 in near future.
796 * 6.2) Link is busy and keeps on receiving packets. This is a simple
797 * case and __packet_lookup_frame_in_block will check if block-0
798 * is free and can now be re-used.
799 */
eea49cc9 800static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 801 struct packet_sock *po)
802{
803 pkc->reset_pending_on_curr_blk = 1;
804 po->stats_u.stats3.tp_freeze_q_cnt++;
805}
806
807#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
808
809/*
810 * If the next block is free then we will dispatch it
811 * and return a good offset.
812 * Else, we will freeze the queue.
813 * So, caller must check the return value.
814 */
bc59ba39 815static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 816 struct packet_sock *po)
817{
bc59ba39 818 struct tpacket_block_desc *pbd;
f6fb8f10 819
820 smp_rmb();
821
822 /* 1. Get current block num */
823 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
824
825 /* 2. If this block is currently in_use then freeze the queue */
826 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
827 prb_freeze_queue(pkc, po);
828 return NULL;
829 }
830
831 /*
832 * 3.
833 * open this block and return the offset where the first packet
834 * needs to get stored.
835 */
836 prb_open_block(pkc, pbd);
837 return (void *)pkc->nxt_offset;
838}
839
bc59ba39 840static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 841 struct packet_sock *po, unsigned int status)
842{
bc59ba39 843 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 844
845 /* retire/close the current block */
846 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
847 /*
848 * Plug the case where copy_bits() is in progress on
849 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
850 * have space to copy the pkt in the current block and
851 * called prb_retire_current_block()
852 *
853 * We don't need to worry about the TMO case because
854 * the timer-handler already handled this case.
855 */
856 if (!(status & TP_STATUS_BLK_TMO)) {
857 while (atomic_read(&pkc->blk_fill_in_prog)) {
858 /* Waiting for skb_copy_bits to finish... */
859 cpu_relax();
860 }
861 }
862 prb_close_block(pkc, pbd, po, status);
863 return;
864 }
865
866 WARN(1, "ERROR-pbd[%d]:%p\n", pkc->kactive_blk_num, pbd);
867 dump_stack();
868 BUG();
869}
870
eea49cc9 871static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 872 struct tpacket_block_desc *pbd)
f6fb8f10 873{
874 return TP_STATUS_USER & BLOCK_STATUS(pbd);
875}
876
eea49cc9 877static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 878{
879 return pkc->reset_pending_on_curr_blk;
880}
881
eea49cc9 882static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 883{
bc59ba39 884 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 885 atomic_dec(&pkc->blk_fill_in_prog);
886}
887
eea49cc9 888static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 889 struct tpacket3_hdr *ppd)
890{
891 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
892}
893
eea49cc9 894static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 895 struct tpacket3_hdr *ppd)
896{
897 ppd->hv1.tp_rxhash = 0;
898}
899
eea49cc9 900static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 901 struct tpacket3_hdr *ppd)
902{
903 if (vlan_tx_tag_present(pkc->skb)) {
904 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
905 ppd->tp_status = TP_STATUS_VLAN_VALID;
906 } else {
9e67030a 907 ppd->hv1.tp_vlan_tci = 0;
908 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 909 }
910}
911
bc59ba39 912static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 913 struct tpacket3_hdr *ppd)
914{
915 prb_fill_vlan_info(pkc, ppd);
916
917 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
918 prb_fill_rxhash(pkc, ppd);
919 else
920 prb_clear_rxhash(pkc, ppd);
921}
922
eea49cc9 923static void prb_fill_curr_block(char *curr,
bc59ba39 924 struct tpacket_kbdq_core *pkc,
925 struct tpacket_block_desc *pbd,
f6fb8f10 926 unsigned int len)
927{
928 struct tpacket3_hdr *ppd;
929
930 ppd = (struct tpacket3_hdr *)curr;
931 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
932 pkc->prev = curr;
933 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
934 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
935 BLOCK_NUM_PKTS(pbd) += 1;
936 atomic_inc(&pkc->blk_fill_in_prog);
937 prb_run_all_ft_ops(pkc, ppd);
938}
939
940/* Assumes caller has the sk->rx_queue.lock */
941static void *__packet_lookup_frame_in_block(struct packet_sock *po,
942 struct sk_buff *skb,
943 int status,
944 unsigned int len
945 )
946{
bc59ba39 947 struct tpacket_kbdq_core *pkc;
948 struct tpacket_block_desc *pbd;
f6fb8f10 949 char *curr, *end;
950
e3192690 951 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 952 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
953
954 /* Queue is frozen when user space is lagging behind */
955 if (prb_queue_frozen(pkc)) {
956 /*
957 * Check if that last block which caused the queue to freeze,
958 * is still in_use by user-space.
959 */
960 if (prb_curr_blk_in_use(pkc, pbd)) {
961 /* Can't record this packet */
962 return NULL;
963 } else {
964 /*
965 * Ok, the block was released by user-space.
966 * Now let's open that block.
967 * opening a block also thaws the queue.
968 * Thawing is a side effect.
969 */
970 prb_open_block(pkc, pbd);
971 }
972 }
973
974 smp_mb();
975 curr = pkc->nxt_offset;
976 pkc->skb = skb;
e3192690 977 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 978
979 /* first try the current block */
980 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
981 prb_fill_curr_block(curr, pkc, pbd, len);
982 return (void *)curr;
983 }
984
985 /* Ok, close the current block */
986 prb_retire_current_block(pkc, po, 0);
987
988 /* Now, try to dispatch the next block */
989 curr = (char *)prb_dispatch_next_block(pkc, po);
990 if (curr) {
991 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
992 prb_fill_curr_block(curr, pkc, pbd, len);
993 return (void *)curr;
994 }
995
996 /*
997 * No free blocks are available.user_space hasn't caught up yet.
998 * Queue was just frozen and now this packet will get dropped.
999 */
1000 return NULL;
1001}
1002
eea49cc9 1003static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1004 struct sk_buff *skb,
1005 int status, unsigned int len)
1006{
1007 char *curr = NULL;
1008 switch (po->tp_version) {
1009 case TPACKET_V1:
1010 case TPACKET_V2:
1011 curr = packet_lookup_frame(po, &po->rx_ring,
1012 po->rx_ring.head, status);
1013 return curr;
1014 case TPACKET_V3:
1015 return __packet_lookup_frame_in_block(po, skb, status, len);
1016 default:
1017 WARN(1, "TPACKET version not supported\n");
1018 BUG();
99aa3473 1019 return NULL;
f6fb8f10 1020 }
1021}
1022
eea49cc9 1023static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1024 struct packet_ring_buffer *rb,
77f65ebd 1025 unsigned int idx,
f6fb8f10 1026 int status)
1027{
bc59ba39 1028 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1029 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1030
1031 if (status != BLOCK_STATUS(pbd))
1032 return NULL;
1033 return pbd;
1034}
1035
eea49cc9 1036static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1037{
1038 unsigned int prev;
1039 if (rb->prb_bdqc.kactive_blk_num)
1040 prev = rb->prb_bdqc.kactive_blk_num-1;
1041 else
1042 prev = rb->prb_bdqc.knum_blocks-1;
1043 return prev;
1044}
1045
1046/* Assumes caller has held the rx_queue.lock */
eea49cc9 1047static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1048 struct packet_ring_buffer *rb,
1049 int status)
1050{
1051 unsigned int previous = prb_previous_blk_num(rb);
1052 return prb_lookup_block(po, rb, previous, status);
1053}
1054
eea49cc9 1055static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1056 struct packet_ring_buffer *rb,
1057 int status)
1058{
1059 if (po->tp_version <= TPACKET_V2)
1060 return packet_previous_frame(po, rb, status);
1061
1062 return __prb_previous_block(po, rb, status);
1063}
1064
eea49cc9 1065static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1066 struct packet_ring_buffer *rb)
1067{
1068 switch (po->tp_version) {
1069 case TPACKET_V1:
1070 case TPACKET_V2:
1071 return packet_increment_head(rb);
1072 case TPACKET_V3:
1073 default:
1074 WARN(1, "TPACKET version not supported.\n");
1075 BUG();
1076 return;
1077 }
1078}
1079
eea49cc9 1080static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1081 struct packet_ring_buffer *rb,
1082 int status)
1083{
1084 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1085 return packet_lookup_frame(po, rb, previous, status);
1086}
1087
eea49cc9 1088static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1089{
1090 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1091}
1092
77f65ebd
WB		 1093static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1094{
1095 struct sock *sk = &po->sk;
1096 bool has_room;
1097
1098 if (po->prot_hook.func != tpacket_rcv)
1099 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1100 <= sk->sk_rcvbuf;
1101
1102 spin_lock(&sk->sk_receive_queue.lock);
1103 if (po->tp_version == TPACKET_V3)
1104 has_room = prb_lookup_block(po, &po->rx_ring,
1105 po->rx_ring.prb_bdqc.kactive_blk_num,
1106 TP_STATUS_KERNEL);
1107 else
1108 has_room = packet_lookup_frame(po, &po->rx_ring,
1109 po->rx_ring.head,
1110 TP_STATUS_KERNEL);
1111 spin_unlock(&sk->sk_receive_queue.lock);
1112
1113 return has_room;
1114}
1115
1da177e4
LT		 1116static void packet_sock_destruct(struct sock *sk)
1117{
ed85b565
RC		 1118 skb_queue_purge(&sk->sk_error_queue);
1119
547b792c
IJ		 1120 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1121 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1122
1123 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1124 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1125 return;
1126 }
1127
17ab56a2 1128 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1129}
1130
dc99f600
DM		 1131static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1132{
1133 int x = atomic_read(&f->rr_cur) + 1;
1134
1135 if (x >= num)
1136 x = 0;
1137
1138 return x;
1139}
1140
77f65ebd
WB		 1141static unsigned int fanout_demux_hash(struct packet_fanout *f,
1142 struct sk_buff *skb,
1143 unsigned int num)
dc99f600 1144{
77f65ebd 1145 return (((u64)skb->rxhash) * num) >> 32;
dc99f600
DM		 1146}
1147
77f65ebd
WB		 1148static unsigned int fanout_demux_lb(struct packet_fanout *f,
1149 struct sk_buff *skb,
1150 unsigned int num)
dc99f600
DM		 1151{
1152 int cur, old;
1153
1154 cur = atomic_read(&f->rr_cur);
1155 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1156 fanout_rr_next(f, num))) != cur)
1157 cur = old;
77f65ebd
WB		 1158 return cur;
1159}
1160
1161static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1162 struct sk_buff *skb,
1163 unsigned int num)
1164{
1165 return smp_processor_id() % num;
dc99f600
DM		 1166}
1167
77f65ebd
WB		 1168static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1169 struct sk_buff *skb,
1170 unsigned int idx, unsigned int skip,
1171 unsigned int num)
95ec3eb4 1172{
77f65ebd 1173 unsigned int i, j;
95ec3eb4 1174
77f65ebd
WB		 1175 i = j = min_t(int, f->next[idx], num - 1);
1176 do {
1177 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1178 if (i != j)
1179 f->next[idx] = i;
1180 return i;
1181 }
1182 if (++i == num)
1183 i = 0;
1184 } while (i != j);
1185
1186 return idx;
1187}
1188
1189static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1190{
1191 return f->flags & (flag >> 8);
95ec3eb4
DM		 1192}
1193
95ec3eb4
DM		 1194static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1195 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1196{
1197 struct packet_fanout *f = pt->af_packet_priv;
1198 unsigned int num = f->num_members;
1199 struct packet_sock *po;
77f65ebd 1200 unsigned int idx;
dc99f600
DM		 1201
1202 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1203 !num) {
1204 kfree_skb(skb);
1205 return 0;
1206 }
1207
95ec3eb4
DM		 1208 switch (f->type) {
1209 case PACKET_FANOUT_HASH:
1210 default:
77f65ebd 1211 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1212 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1213 if (!skb)
1214 return 0;
1215 }
1216 skb_get_rxhash(skb);
77f65ebd 1217 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1218 break;
1219 case PACKET_FANOUT_LB:
77f65ebd 1220 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1221 break;
1222 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1223 idx = fanout_demux_cpu(f, skb, num);
1224 break;
1225 case PACKET_FANOUT_ROLLOVER:
1226 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1227 break;
dc99f600
DM		 1228 }
1229
77f65ebd
WB		 1230 po = pkt_sk(f->arr[idx]);
1231 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1232 unlikely(!packet_rcv_has_room(po, skb))) {
1233 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1234 po = pkt_sk(f->arr[idx]);
1235 }
dc99f600
DM		 1236
1237 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1238}
1239
fff3321d
PE		 1240DEFINE_MUTEX(fanout_mutex);
1241EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1242static LIST_HEAD(fanout_list);
1243
1244static void __fanout_link(struct sock *sk, struct packet_sock *po)
1245{
1246 struct packet_fanout *f = po->fanout;
1247
1248 spin_lock(&f->lock);
1249 f->arr[f->num_members] = sk;
1250 smp_wmb();
1251 f->num_members++;
1252 spin_unlock(&f->lock);
1253}
1254
1255static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1256{
1257 struct packet_fanout *f = po->fanout;
1258 int i;
1259
1260 spin_lock(&f->lock);
1261 for (i = 0; i < f->num_members; i++) {
1262 if (f->arr[i] == sk)
1263 break;
1264 }
1265 BUG_ON(i >= f->num_members);
1266 f->arr[i] = f->arr[f->num_members - 1];
1267 f->num_members--;
1268 spin_unlock(&f->lock);
1269}
1270
a0dfb263 1271static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
c0de08d0
EL		 1272{
1273 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1274 return true;
1275
1276 return false;
1277}
1278
7736d33f 1279static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1280{
1281 struct packet_sock *po = pkt_sk(sk);
1282 struct packet_fanout *f, *match;
7736d33f 1283 u8 type = type_flags & 0xff;
77f65ebd 1284 u8 flags = type_flags >> 8;
dc99f600
DM		 1285 int err;
1286
1287 switch (type) {
77f65ebd
WB		 1288 case PACKET_FANOUT_ROLLOVER:
1289 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1290 return -EINVAL;
dc99f600
DM		 1291 case PACKET_FANOUT_HASH:
1292 case PACKET_FANOUT_LB:
95ec3eb4 1293 case PACKET_FANOUT_CPU:
dc99f600
DM		 1294 break;
1295 default:
1296 return -EINVAL;
1297 }
1298
1299 if (!po->running)
1300 return -EINVAL;
1301
1302 if (po->fanout)
1303 return -EALREADY;
1304
1305 mutex_lock(&fanout_mutex);
1306 match = NULL;
1307 list_for_each_entry(f, &fanout_list, list) {
1308 if (f->id == id &&
1309 read_pnet(&f->net) == sock_net(sk)) {
1310 match = f;
1311 break;
1312 }
1313 }
afe62c68 1314 err = -EINVAL;
77f65ebd 1315 if (match && match->flags != flags)
afe62c68 1316 goto out;
dc99f600 1317 if (!match) {
afe62c68 1318 err = -ENOMEM;
dc99f600 1319 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1320 if (!match)
1321 goto out;
1322 write_pnet(&match->net, sock_net(sk));
1323 match->id = id;
1324 match->type = type;
77f65ebd 1325 match->flags = flags;
afe62c68
ED		 1326 atomic_set(&match->rr_cur, 0);
1327 INIT_LIST_HEAD(&match->list);
1328 spin_lock_init(&match->lock);
1329 atomic_set(&match->sk_ref, 0);
1330 match->prot_hook.type = po->prot_hook.type;
1331 match->prot_hook.dev = po->prot_hook.dev;
1332 match->prot_hook.func = packet_rcv_fanout;
1333 match->prot_hook.af_packet_priv = match;
c0de08d0 1334 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1335 dev_add_pack(&match->prot_hook);
1336 list_add(&match->list, &fanout_list);
dc99f600 1337 }
afe62c68
ED		 1338 err = -EINVAL;
1339 if (match->type == type &&
1340 match->prot_hook.type == po->prot_hook.type &&
1341 match->prot_hook.dev == po->prot_hook.dev) {
1342 err = -ENOSPC;
1343 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1344 __dev_remove_pack(&po->prot_hook);
1345 po->fanout = match;
1346 atomic_inc(&match->sk_ref);
1347 __fanout_link(sk, po);
1348 err = 0;
dc99f600
DM		 1349 }
1350 }
afe62c68 1351out:
dc99f600
DM		 1352 mutex_unlock(&fanout_mutex);
1353 return err;
1354}
1355
1356static void fanout_release(struct sock *sk)
1357{
1358 struct packet_sock *po = pkt_sk(sk);
1359 struct packet_fanout *f;
1360
1361 f = po->fanout;
1362 if (!f)
1363 return;
1364
fff3321d 1365 mutex_lock(&fanout_mutex);
dc99f600
DM		 1366 po->fanout = NULL;
1367
dc99f600
DM		 1368 if (atomic_dec_and_test(&f->sk_ref)) {
1369 list_del(&f->list);
1370 dev_remove_pack(&f->prot_hook);
1371 kfree(f);
1372 }
1373 mutex_unlock(&fanout_mutex);
1374}
1da177e4 1375
90ddc4f0 1376static const struct proto_ops packet_ops;
1da177e4 1377
90ddc4f0 1378static const struct proto_ops packet_ops_spkt;
1da177e4 1379
40d4e3df
ED		 1380static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1381 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1382{
1383 struct sock *sk;
1384 struct sockaddr_pkt *spkt;
1385
1386 /*
1387 * When we registered the protocol we saved the socket in the data
1388 * field for just this event.
1389 */
1390
1391 sk = pt->af_packet_priv;
1ce4f28b 1392
1da177e4
LT		 1393 /*
1394 * Yank back the headers [hope the device set this
1395 * right or kerboom...]
1396 *
1397 * Incoming packets have ll header pulled,
1398 * push it back.
1399 *
98e399f8 1400 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1401 * so that this procedure is noop.
1402 */
1403
1404 if (skb->pkt_type == PACKET_LOOPBACK)
1405 goto out;
1406
09ad9bc7 1407 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1408 goto out;
1409
40d4e3df
ED		 1410 skb = skb_share_check(skb, GFP_ATOMIC);
1411 if (skb == NULL)
1da177e4
LT		 1412 goto oom;
1413
1414 /* drop any routing info */
adf30907 1415 skb_dst_drop(skb);
1da177e4 1416
84531c24
PO		 1417 /* drop conntrack reference */
1418 nf_reset(skb);
1419
ffbc6111 1420 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1421
98e399f8 1422 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1423
1424 /*
1425 * The SOCK_PACKET socket receives _all_ frames.
1426 */
1427
1428 spkt->spkt_family = dev->type;
1429 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1430 spkt->spkt_protocol = skb->protocol;
1431
1432 /*
1433 * Charge the memory to the socket. This is done specifically
1434 * to prevent sockets using all the memory up.
1435 */
1436
40d4e3df 1437 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1438 return 0;
1439
1440out:
1441 kfree_skb(skb);
1442oom:
1443 return 0;
1444}
1445
1446
1447/*
1448 * Output a raw packet to a device layer. This bypasses all the other
1449 * protocol layers and you must therefore supply it with a complete frame
1450 */
1ce4f28b 1451
1da177e4
LT		 1452static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1453 struct msghdr *msg, size_t len)
1454{
1455 struct sock *sk = sock->sk;
40d4e3df 1456 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1457 struct sk_buff *skb = NULL;
1da177e4 1458 struct net_device *dev;
40d4e3df 1459 __be16 proto = 0;
1da177e4 1460 int err;
3bdc0eba 1461 int extra_len = 0;
1ce4f28b 1462
1da177e4 1463 /*
1ce4f28b 1464 * Get and verify the address.
1da177e4
LT		 1465 */
1466
40d4e3df 1467 if (saddr) {
1da177e4 1468 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1469 return -EINVAL;
1470 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1471 proto = saddr->spkt_protocol;
1472 } else
1473 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1474
1475 /*
1ce4f28b 1476 * Find the device first to size check it
1da177e4
LT		 1477 */
1478
de74e92a 1479 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1480retry:
654d1f8a
ED		 1481 rcu_read_lock();
1482 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1483 err = -ENODEV;
1484 if (dev == NULL)
1485 goto out_unlock;
1ce4f28b 1486
d5e76b0a
DM		 1487 err = -ENETDOWN;
1488 if (!(dev->flags & IFF_UP))
1489 goto out_unlock;
1490
1da177e4 1491 /*
40d4e3df
ED		 1492 * You may not queue a frame bigger than the mtu. This is the lowest level
1493 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1494 */
1ce4f28b 1495
3bdc0eba
BG		 1496 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1497 if (!netif_supports_nofcs(dev)) {
1498 err = -EPROTONOSUPPORT;
1499 goto out_unlock;
1500 }
1501 extra_len = 4; /* We're doing our own CRC */
1502 }
1503
1da177e4 1504 err = -EMSGSIZE;
3bdc0eba 1505 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1506 goto out_unlock;
1507
1a35ca80
ED		 1508 if (!skb) {
1509 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1510 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1511 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1512
1513 rcu_read_unlock();
4ce40912 1514 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1515 if (skb == NULL)
1516 return -ENOBUFS;
1517 /* FIXME: Save some space for broken drivers that write a hard
1518 * header at transmission time by themselves. PPP is the notable
1519 * one here. This should really be fixed at the driver level.
1520 */
1521 skb_reserve(skb, reserved);
1522 skb_reset_network_header(skb);
1523
1524 /* Try to align data part correctly */
1525 if (hhlen) {
1526 skb->data -= hhlen;
1527 skb->tail -= hhlen;
1528 if (len < hhlen)
1529 skb_reset_network_header(skb);
1530 }
1531 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1532 if (err)
1533 goto out_free;
1534 goto retry;
1da177e4
LT		 1535 }
1536
3bdc0eba 1537 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1538 /* Earlier code assumed this would be a VLAN pkt,
1539 * double-check this now that we have the actual
1540 * packet in hand.
1541 */
1542 struct ethhdr *ehdr;
1543 skb_reset_mac_header(skb);
1544 ehdr = eth_hdr(skb);
1545 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1546 err = -EMSGSIZE;
1547 goto out_unlock;
1548 }
1549 }
1a35ca80 1550
1da177e4
LT		 1551 skb->protocol = proto;
1552 skb->dev = dev;
1553 skb->priority = sk->sk_priority;
2d37a186 1554 skb->mark = sk->sk_mark;
bf84a010
DB		 1555
1556 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1557
3bdc0eba
BG		 1558 if (unlikely(extra_len == 4))
1559 skb->no_fcs = 1;
1560
40893fd0 1561 skb_probe_transport_header(skb, 0);
c1aad275 1562
1da177e4 1563 dev_queue_xmit(skb);
654d1f8a 1564 rcu_read_unlock();
40d4e3df 1565 return len;
1da177e4 1566
1da177e4 1567out_unlock:
654d1f8a 1568 rcu_read_unlock();
1a35ca80
ED		 1569out_free:
1570 kfree_skb(skb);
1da177e4
LT		 1571 return err;
1572}
1da177e4 1573
eea49cc9 1574static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1575 const struct sock *sk,
dbcb5855 1576 unsigned int res)
1da177e4
LT		 1577{
1578 struct sk_filter *filter;
fda9ef5d 1579
80f8f102
ED		 1580 rcu_read_lock();
1581 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1582 if (filter != NULL)
0a14842f 1583 res = SK_RUN_FILTER(filter, skb);
80f8f102 1584 rcu_read_unlock();
1da177e4 1585
dbcb5855 1586 return res;
1da177e4
LT		 1587}
1588
1589/*
62ab0812
ED		 1590 * This function makes lazy skb cloning in hope that most of packets
1591 * are discarded by BPF.
1592 *
1593 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1594 * and skb->cb are mangled. It works because (and until) packets
1595 * falling here are owned by current CPU. Output packets are cloned
1596 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1597 * sequencially, so that if we return skb to original state on exit,
1598 * we will not harm anyone.
1da177e4
LT		 1599 */
1600
40d4e3df
ED		 1601static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1602 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1603{
1604 struct sock *sk;
1605 struct sockaddr_ll *sll;
1606 struct packet_sock *po;
40d4e3df 1607 u8 *skb_head = skb->data;
1da177e4 1608 int skb_len = skb->len;
dbcb5855 1609 unsigned int snaplen, res;
1da177e4
LT		 1610
1611 if (skb->pkt_type == PACKET_LOOPBACK)
1612 goto drop;
1613
1614 sk = pt->af_packet_priv;
1615 po = pkt_sk(sk);
1616
09ad9bc7 1617 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1618 goto drop;
1619
1da177e4
LT		 1620 skb->dev = dev;
1621
3b04ddde 1622 if (dev->header_ops) {
1da177e4 1623 /* The device has an explicit notion of ll header,
62ab0812
ED		 1624 * exported to higher levels.
1625 *
1626 * Otherwise, the device hides details of its frame
1627 * structure, so that corresponding packet head is
1628 * never delivered to user.
1da177e4
LT		 1629 */
1630 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1631 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1632 else if (skb->pkt_type == PACKET_OUTGOING) {
1633 /* Special case: outgoing packets have ll header at head */
bbe735e4 1634 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1635 }
1636 }
1637
1638 snaplen = skb->len;
1639
dbcb5855
DM		 1640 res = run_filter(skb, sk, snaplen);
1641 if (!res)
fda9ef5d 1642 goto drop_n_restore;
dbcb5855
DM		 1643 if (snaplen > res)
1644 snaplen = res;
1da177e4 1645
0fd7bac6 1646 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1647 goto drop_n_acct;
1648
1649 if (skb_shared(skb)) {
1650 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1651 if (nskb == NULL)
1652 goto drop_n_acct;
1653
1654 if (skb_head != skb->data) {
1655 skb->data = skb_head;
1656 skb->len = skb_len;
1657 }
abc4e4fa 1658 consume_skb(skb);
1da177e4
LT		 1659 skb = nskb;
1660 }
1661
ffbc6111
HX		 1662 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1663 sizeof(skb->cb));
1664
1665 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1666 sll->sll_family = AF_PACKET;
1667 sll->sll_hatype = dev->type;
1668 sll->sll_protocol = skb->protocol;
1669 sll->sll_pkttype = skb->pkt_type;
8032b464 1670 if (unlikely(po->origdev))
80feaacb
PWJ		 1671 sll->sll_ifindex = orig_dev->ifindex;
1672 else
1673 sll->sll_ifindex = dev->ifindex;
1da177e4 1674
b95cce35 1675 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1676
ffbc6111 1677 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1678
1da177e4
LT		 1679 if (pskb_trim(skb, snaplen))
1680 goto drop_n_acct;
1681
1682 skb_set_owner_r(skb, sk);
1683 skb->dev = NULL;
adf30907 1684 skb_dst_drop(skb);
1da177e4 1685
84531c24
PO		 1686 /* drop conntrack reference */
1687 nf_reset(skb);
1688
1da177e4
LT		 1689 spin_lock(&sk->sk_receive_queue.lock);
1690 po->stats.tp_packets++;
3b885787 1691 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1692 __skb_queue_tail(&sk->sk_receive_queue, skb);
1693 spin_unlock(&sk->sk_receive_queue.lock);
1694 sk->sk_data_ready(sk, skb->len);
1695 return 0;
1696
1697drop_n_acct:
7091fbd8
WB		 1698 spin_lock(&sk->sk_receive_queue.lock);
1699 po->stats.tp_drops++;
1700 atomic_inc(&sk->sk_drops);
1701 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1702
1703drop_n_restore:
1704 if (skb_head != skb->data && skb_shared(skb)) {
1705 skb->data = skb_head;
1706 skb->len = skb_len;
1707 }
1708drop:
ead2ceb0 1709 consume_skb(skb);
1da177e4
LT		 1710 return 0;
1711}
1712
40d4e3df
ED		 1713static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1714 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1715{
1716 struct sock *sk;
1717 struct packet_sock *po;
1718 struct sockaddr_ll *sll;
184f489e 1719 union tpacket_uhdr h;
40d4e3df 1720 u8 *skb_head = skb->data;
1da177e4 1721 int skb_len = skb->len;
dbcb5855 1722 unsigned int snaplen, res;
f6fb8f10 1723 unsigned long status = TP_STATUS_USER;
bbd6ef87 1724 unsigned short macoff, netoff, hdrlen;
1da177e4 1725 struct sk_buff *copy_skb = NULL;
bbd6ef87 1726 struct timespec ts;
b9c32fb2 1727 __u32 ts_status;
1da177e4
LT		 1728
1729 if (skb->pkt_type == PACKET_LOOPBACK)
1730 goto drop;
1731
1732 sk = pt->af_packet_priv;
1733 po = pkt_sk(sk);
1734
09ad9bc7 1735 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1736 goto drop;
1737
3b04ddde 1738 if (dev->header_ops) {
1da177e4 1739 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1740 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1741 else if (skb->pkt_type == PACKET_OUTGOING) {
1742 /* Special case: outgoing packets have ll header at head */
bbe735e4 1743 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1744 }
1745 }
1746
8dc41944
HX		 1747 if (skb->ip_summed == CHECKSUM_PARTIAL)
1748 status |= TP_STATUS_CSUMNOTREADY;
1749
1da177e4
LT		 1750 snaplen = skb->len;
1751
dbcb5855
DM		 1752 res = run_filter(skb, sk, snaplen);
1753 if (!res)
fda9ef5d 1754 goto drop_n_restore;
dbcb5855
DM		 1755 if (snaplen > res)
1756 snaplen = res;
1da177e4
LT		 1757
1758 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1759 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1760 po->tp_reserve;
1da177e4 1761 } else {
95c96174 1762 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1763 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1764 (maclen < 16 ? 16 : maclen)) +
1765 po->tp_reserve;
1da177e4
LT		 1766 macoff = netoff - maclen;
1767 }
f6fb8f10 1768 if (po->tp_version <= TPACKET_V2) {
1769 if (macoff + snaplen > po->rx_ring.frame_size) {
1770 if (po->copy_thresh &&
0fd7bac6 1771 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1772 if (skb_shared(skb)) {
1773 copy_skb = skb_clone(skb, GFP_ATOMIC);
1774 } else {
1775 copy_skb = skb_get(skb);
1776 skb_head = skb->data;
1777 }
1778 if (copy_skb)
1779 skb_set_owner_r(copy_skb, sk);
1da177e4 1780 }
f6fb8f10 1781 snaplen = po->rx_ring.frame_size - macoff;
1782 if ((int)snaplen < 0)
1783 snaplen = 0;
1da177e4 1784 }
1da177e4 1785 }
1da177e4 1786 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1787 h.raw = packet_current_rx_frame(po, skb,
1788 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1789 if (!h.raw)
1da177e4 1790 goto ring_is_full;
f6fb8f10 1791 if (po->tp_version <= TPACKET_V2) {
1792 packet_increment_rx_head(po, &po->rx_ring);
1793 /*
1794 * LOSING will be reported till you read the stats,
1795 * because it's COR - Clear On Read.
1796 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1797 * at packet level.
1798 */
1799 if (po->stats.tp_drops)
1800 status |= TP_STATUS_LOSING;
1801 }
1da177e4
LT		 1802 po->stats.tp_packets++;
1803 if (copy_skb) {
1804 status |= TP_STATUS_COPY;
1805 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1806 }
1da177e4
LT		 1807 spin_unlock(&sk->sk_receive_queue.lock);
1808
bbd6ef87 1809 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1810
1811 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1812 getnstimeofday(&ts);
1da177e4 1813
b9c32fb2
DB		 1814 status |= ts_status;
1815
bbd6ef87
PM		 1816 switch (po->tp_version) {
1817 case TPACKET_V1:
1818 h.h1->tp_len = skb->len;
1819 h.h1->tp_snaplen = snaplen;
1820 h.h1->tp_mac = macoff;
1821 h.h1->tp_net = netoff;
4b457bdf
DB		 1822 h.h1->tp_sec = ts.tv_sec;
1823 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1824 hdrlen = sizeof(*h.h1);
1825 break;
1826 case TPACKET_V2:
1827 h.h2->tp_len = skb->len;
1828 h.h2->tp_snaplen = snaplen;
1829 h.h2->tp_mac = macoff;
1830 h.h2->tp_net = netoff;
bbd6ef87
PM		 1831 h.h2->tp_sec = ts.tv_sec;
1832 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1833 if (vlan_tx_tag_present(skb)) {
1834 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1835 status |= TP_STATUS_VLAN_VALID;
1836 } else {
1837 h.h2->tp_vlan_tci = 0;
1838 }
13fcb7bd 1839 h.h2->tp_padding = 0;
bbd6ef87
PM		 1840 hdrlen = sizeof(*h.h2);
1841 break;
f6fb8f10 1842 case TPACKET_V3:
1843 /* tp_nxt_offset,vlan are already populated above.
1844 * So DONT clear those fields here
1845 */
1846 h.h3->tp_status |= status;
1847 h.h3->tp_len = skb->len;
1848 h.h3->tp_snaplen = snaplen;
1849 h.h3->tp_mac = macoff;
1850 h.h3->tp_net = netoff;
f6fb8f10 1851 h.h3->tp_sec = ts.tv_sec;
1852 h.h3->tp_nsec = ts.tv_nsec;
1853 hdrlen = sizeof(*h.h3);
1854 break;
bbd6ef87
PM		 1855 default:
1856 BUG();
1857 }
1da177e4 1858
bbd6ef87 1859 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1860 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1861 sll->sll_family = AF_PACKET;
1862 sll->sll_hatype = dev->type;
1863 sll->sll_protocol = skb->protocol;
1864 sll->sll_pkttype = skb->pkt_type;
8032b464 1865 if (unlikely(po->origdev))
80feaacb
PWJ		 1866 sll->sll_ifindex = orig_dev->ifindex;
1867 else
1868 sll->sll_ifindex = dev->ifindex;
1da177e4 1869
e16aa207 1870 smp_mb();
f6dafa95 1871#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1872 {
0af55bb5
CG		 1873 u8 *start, *end;
1874
f6fb8f10 1875 if (po->tp_version <= TPACKET_V2) {
1876 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1877 + macoff + snaplen);
1878 for (start = h.raw; start < end; start += PAGE_SIZE)
1879 flush_dcache_page(pgv_to_page(start));
1880 }
cc9f01b2 1881 smp_wmb();
1da177e4 1882 }
f6dafa95 1883#endif
f6fb8f10 1884 if (po->tp_version <= TPACKET_V2)
1885 __packet_set_status(po, h.raw, status);
1886 else
1887 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1888
1889 sk->sk_data_ready(sk, 0);
1890
1891drop_n_restore:
1892 if (skb_head != skb->data && skb_shared(skb)) {
1893 skb->data = skb_head;
1894 skb->len = skb_len;
1895 }
1896drop:
1ce4f28b 1897 kfree_skb(skb);
1da177e4
LT		 1898 return 0;
1899
1900ring_is_full:
1901 po->stats.tp_drops++;
1902 spin_unlock(&sk->sk_receive_queue.lock);
1903
1904 sk->sk_data_ready(sk, 0);
acb5d75b 1905 kfree_skb(copy_skb);
1da177e4
LT		 1906 goto drop_n_restore;
1907}
1908
69e3c75f
JB		 1909static void tpacket_destruct_skb(struct sk_buff *skb)
1910{
1911 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1912 void *ph;
1da177e4 1913
69e3c75f 1914 if (likely(po->tx_ring.pg_vec)) {
b9c32fb2
DB		 1915 __u32 ts;
1916
69e3c75f 1917 ph = skb_shinfo(skb)->destructor_arg;
69e3c75f
JB		 1918 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1919 atomic_dec(&po->tx_ring.pending);
b9c32fb2
DB		 1920
1921 ts = __packet_set_timestamp(po, ph, skb);
1922 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 1923 }
1924
1925 sock_wfree(skb);
1926}
1927
40d4e3df
ED		 1928static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1929 void *frame, struct net_device *dev, int size_max,
ae641949 1930 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 1931{
184f489e 1932 union tpacket_uhdr ph;
69e3c75f
JB		 1933 int to_write, offset, len, tp_len, nr_frags, len_max;
1934 struct socket *sock = po->sk.sk_socket;
1935 struct page *page;
1936 void *data;
1937 int err;
1938
1939 ph.raw = frame;
1940
1941 skb->protocol = proto;
1942 skb->dev = dev;
1943 skb->priority = po->sk.sk_priority;
2d37a186 1944 skb->mark = po->sk.sk_mark;
2e31396f 1945 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 1946 skb_shinfo(skb)->destructor_arg = ph.raw;
1947
1948 switch (po->tp_version) {
1949 case TPACKET_V2:
1950 tp_len = ph.h2->tp_len;
1951 break;
1952 default:
1953 tp_len = ph.h1->tp_len;
1954 break;
1955 }
1956 if (unlikely(tp_len > size_max)) {
40d4e3df 1957 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
69e3c75f
JB		 1958 return -EMSGSIZE;
1959 }
1960
ae641949 1961 skb_reserve(skb, hlen);
69e3c75f 1962 skb_reset_network_header(skb);
40893fd0 1963 skb_probe_transport_header(skb, 0);
c1aad275 1964
5920cd3a
PC		 1965 if (po->tp_tx_has_off) {
1966 int off_min, off_max, off;
1967 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
1968 off_max = po->tx_ring.frame_size - tp_len;
1969 if (sock->type == SOCK_DGRAM) {
1970 switch (po->tp_version) {
1971 case TPACKET_V2:
1972 off = ph.h2->tp_net;
1973 break;
1974 default:
1975 off = ph.h1->tp_net;
1976 break;
1977 }
1978 } else {
1979 switch (po->tp_version) {
1980 case TPACKET_V2:
1981 off = ph.h2->tp_mac;
1982 break;
1983 default:
1984 off = ph.h1->tp_mac;
1985 break;
1986 }
1987 }
1988 if (unlikely((off < off_min) || (off_max < off)))
1989 return -EINVAL;
1990 data = ph.raw + off;
1991 } else {
1992 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
1993 }
69e3c75f
JB		 1994 to_write = tp_len;
1995
1996 if (sock->type == SOCK_DGRAM) {
1997 err = dev_hard_header(skb, dev, ntohs(proto), addr,
1998 NULL, tp_len);
1999 if (unlikely(err < 0))
2000 return -EINVAL;
40d4e3df 2001 } else if (dev->hard_header_len) {
69e3c75f
JB		 2002 /* net device doesn't like empty head */
2003 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2004 pr_err("packet size is too short (%d < %d)\n",
2005 tp_len, dev->hard_header_len);
69e3c75f
JB		 2006 return -EINVAL;
2007 }
2008
2009 skb_push(skb, dev->hard_header_len);
2010 err = skb_store_bits(skb, 0, data,
2011 dev->hard_header_len);
2012 if (unlikely(err))
2013 return err;
2014
2015 data += dev->hard_header_len;
2016 to_write -= dev->hard_header_len;
2017 }
2018
69e3c75f
JB		 2019 offset = offset_in_page(data);
2020 len_max = PAGE_SIZE - offset;
2021 len = ((to_write > len_max) ? len_max : to_write);
2022
2023 skb->data_len = to_write;
2024 skb->len += to_write;
2025 skb->truesize += to_write;
2026 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2027
2028 while (likely(to_write)) {
2029 nr_frags = skb_shinfo(skb)->nr_frags;
2030
2031 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2032 pr_err("Packet exceed the number of skb frags(%lu)\n",
2033 MAX_SKB_FRAGS);
69e3c75f
JB		 2034 return -EFAULT;
2035 }
2036
0af55bb5
CG		 2037 page = pgv_to_page(data);
2038 data += len;
69e3c75f
JB		 2039 flush_dcache_page(page);
2040 get_page(page);
0af55bb5 2041 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2042 to_write -= len;
2043 offset = 0;
2044 len_max = PAGE_SIZE;
2045 len = ((to_write > len_max) ? len_max : to_write);
2046 }
2047
2048 return tp_len;
2049}
2050
2051static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2052{
69e3c75f
JB		 2053 struct sk_buff *skb;
2054 struct net_device *dev;
2055 __be16 proto;
827d9780
BG		 2056 bool need_rls_dev = false;
2057 int err, reserve = 0;
40d4e3df
ED		 2058 void *ph;
2059 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2060 int tp_len, size_max;
2061 unsigned char *addr;
2062 int len_sum = 0;
9e67030a 2063 int status = TP_STATUS_AVAILABLE;
ae641949 2064 int hlen, tlen;
69e3c75f 2065
69e3c75f
JB		 2066 mutex_lock(&po->pg_vec_lock);
2067
69e3c75f 2068 if (saddr == NULL) {
827d9780 2069 dev = po->prot_hook.dev;
69e3c75f
JB		 2070 proto = po->num;
2071 addr = NULL;
2072 } else {
2073 err = -EINVAL;
2074 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2075 goto out;
2076 if (msg->msg_namelen < (saddr->sll_halen
2077 + offsetof(struct sockaddr_ll,
2078 sll_addr)))
2079 goto out;
69e3c75f
JB		 2080 proto = saddr->sll_protocol;
2081 addr = saddr->sll_addr;
827d9780
BG		 2082 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2083 need_rls_dev = true;
69e3c75f
JB		 2084 }
2085
69e3c75f
JB		 2086 err = -ENXIO;
2087 if (unlikely(dev == NULL))
2088 goto out;
2089
2090 reserve = dev->hard_header_len;
2091
2092 err = -ENETDOWN;
2093 if (unlikely(!(dev->flags & IFF_UP)))
2094 goto out_put;
2095
2096 size_max = po->tx_ring.frame_size
b5dd884e 2097 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f
JB		 2098
2099 if (size_max > dev->mtu + reserve)
2100 size_max = dev->mtu + reserve;
2101
2102 do {
2103 ph = packet_current_frame(po, &po->tx_ring,
2104 TP_STATUS_SEND_REQUEST);
2105
2106 if (unlikely(ph == NULL)) {
2107 schedule();
2108 continue;
2109 }
2110
2111 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2112 hlen = LL_RESERVED_SPACE(dev);
2113 tlen = dev->needed_tailroom;
69e3c75f 2114 skb = sock_alloc_send_skb(&po->sk,
ae641949 2115 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2116 0, &err);
2117
2118 if (unlikely(skb == NULL))
2119 goto out_status;
2120
2121 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2122 addr, hlen);
69e3c75f
JB		 2123
2124 if (unlikely(tp_len < 0)) {
2125 if (po->tp_loss) {
2126 __packet_set_status(po, ph,
2127 TP_STATUS_AVAILABLE);
2128 packet_increment_head(&po->tx_ring);
2129 kfree_skb(skb);
2130 continue;
2131 } else {
2132 status = TP_STATUS_WRONG_FORMAT;
2133 err = tp_len;
2134 goto out_status;
2135 }
2136 }
2137
2138 skb->destructor = tpacket_destruct_skb;
2139 __packet_set_status(po, ph, TP_STATUS_SENDING);
2140 atomic_inc(&po->tx_ring.pending);
2141
2142 status = TP_STATUS_SEND_REQUEST;
2143 err = dev_queue_xmit(skb);
eb70df13
JP		 2144 if (unlikely(err > 0)) {
2145 err = net_xmit_errno(err);
2146 if (err && __packet_get_status(po, ph) ==
2147 TP_STATUS_AVAILABLE) {
2148 /* skb was destructed already */
2149 skb = NULL;
2150 goto out_status;
2151 }
2152 /*
2153 * skb was dropped but not destructed yet;
2154 * let's treat it like congestion or err < 0
2155 */
2156 err = 0;
2157 }
69e3c75f
JB		 2158 packet_increment_head(&po->tx_ring);
2159 len_sum += tp_len;
f64f9e71
JP		 2160 } while (likely((ph != NULL) ||
2161 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2162 (atomic_read(&po->tx_ring.pending))))
2163 );
69e3c75f
JB		 2164
2165 err = len_sum;
2166 goto out_put;
2167
69e3c75f
JB		 2168out_status:
2169 __packet_set_status(po, ph, status);
2170 kfree_skb(skb);
2171out_put:
827d9780
BG		 2172 if (need_rls_dev)
2173 dev_put(dev);
69e3c75f
JB		 2174out:
2175 mutex_unlock(&po->pg_vec_lock);
2176 return err;
2177}
69e3c75f 2178
eea49cc9
OJ		 2179static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2180 size_t reserve, size_t len,
2181 size_t linear, int noblock,
2182 int *err)
bfd5f4a3
SS		 2183{
2184 struct sk_buff *skb;
2185
2186 /* Under a page? Don't bother with paged skb. */
2187 if (prepad + len < PAGE_SIZE || !linear)
2188 linear = len;
2189
2190 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2191 err);
2192 if (!skb)
2193 return NULL;
2194
2195 skb_reserve(skb, reserve);
2196 skb_put(skb, linear);
2197 skb->data_len = len - linear;
2198 skb->len += len - linear;
2199
2200 return skb;
2201}
2202
69e3c75f 2203static int packet_snd(struct socket *sock,
1da177e4
LT		 2204 struct msghdr *msg, size_t len)
2205{
2206 struct sock *sk = sock->sk;
40d4e3df 2207 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2208 struct sk_buff *skb;
2209 struct net_device *dev;
0e11c91e 2210 __be16 proto;
827d9780 2211 bool need_rls_dev = false;
1da177e4 2212 unsigned char *addr;
827d9780 2213 int err, reserve = 0;
bfd5f4a3
SS		 2214 struct virtio_net_hdr vnet_hdr = { 0 };
2215 int offset = 0;
2216 int vnet_hdr_len;
2217 struct packet_sock *po = pkt_sk(sk);
2218 unsigned short gso_type = 0;
ae641949 2219 int hlen, tlen;
3bdc0eba 2220 int extra_len = 0;
1da177e4
LT		 2221
2222 /*
1ce4f28b 2223 * Get and verify the address.
1da177e4 2224 */
1ce4f28b 2225
1da177e4 2226 if (saddr == NULL) {
827d9780 2227 dev = po->prot_hook.dev;
1da177e4
LT		 2228 proto = po->num;
2229 addr = NULL;
2230 } else {
2231 err = -EINVAL;
2232 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2233 goto out;
0fb375fb
EB		 2234 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2235 goto out;
1da177e4
LT		 2236 proto = saddr->sll_protocol;
2237 addr = saddr->sll_addr;
827d9780
BG		 2238 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2239 need_rls_dev = true;
1da177e4
LT		 2240 }
2241
1da177e4
LT		 2242 err = -ENXIO;
2243 if (dev == NULL)
2244 goto out_unlock;
2245 if (sock->type == SOCK_RAW)
2246 reserve = dev->hard_header_len;
2247
d5e76b0a
DM		 2248 err = -ENETDOWN;
2249 if (!(dev->flags & IFF_UP))
2250 goto out_unlock;
2251
bfd5f4a3
SS		 2252 if (po->has_vnet_hdr) {
2253 vnet_hdr_len = sizeof(vnet_hdr);
2254
2255 err = -EINVAL;
2256 if (len < vnet_hdr_len)
2257 goto out_unlock;
2258
2259 len -= vnet_hdr_len;
2260
2261 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2262 vnet_hdr_len);
2263 if (err < 0)
2264 goto out_unlock;
2265
2266 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2267 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2268 vnet_hdr.hdr_len))
2269 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2270 vnet_hdr.csum_offset + 2;
2271
2272 err = -EINVAL;
2273 if (vnet_hdr.hdr_len > len)
2274 goto out_unlock;
2275
2276 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2277 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2278 case VIRTIO_NET_HDR_GSO_TCPV4:
2279 gso_type = SKB_GSO_TCPV4;
2280 break;
2281 case VIRTIO_NET_HDR_GSO_TCPV6:
2282 gso_type = SKB_GSO_TCPV6;
2283 break;
2284 case VIRTIO_NET_HDR_GSO_UDP:
2285 gso_type = SKB_GSO_UDP;
2286 break;
2287 default:
2288 goto out_unlock;
2289 }
2290
2291 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2292 gso_type |= SKB_GSO_TCP_ECN;
2293
2294 if (vnet_hdr.gso_size == 0)
2295 goto out_unlock;
2296
2297 }
2298 }
2299
3bdc0eba
BG		 2300 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2301 if (!netif_supports_nofcs(dev)) {
2302 err = -EPROTONOSUPPORT;
2303 goto out_unlock;
2304 }
2305 extra_len = 4; /* We're doing our own CRC */
2306 }
2307
1da177e4 2308 err = -EMSGSIZE;
3bdc0eba 2309 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2310 goto out_unlock;
2311
bfd5f4a3 2312 err = -ENOBUFS;
ae641949
HX		 2313 hlen = LL_RESERVED_SPACE(dev);
2314 tlen = dev->needed_tailroom;
2315 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2316 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2317 if (skb == NULL)
1da177e4
LT		 2318 goto out_unlock;
2319
bfd5f4a3 2320 skb_set_network_header(skb, reserve);
1da177e4 2321
0c4e8581
SH		 2322 err = -EINVAL;
2323 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2324 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2325 goto out_free;
1da177e4
LT		 2326
2327 /* Returns -EFAULT on error */
bfd5f4a3 2328 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2329 if (err)
2330 goto out_free;
bf84a010
DB		 2331
2332 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2333
3bdc0eba 2334 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
57f89bfa
BG		 2335 /* Earlier code assumed this would be a VLAN pkt,
2336 * double-check this now that we have the actual
2337 * packet in hand.
2338 */
2339 struct ethhdr *ehdr;
2340 skb_reset_mac_header(skb);
2341 ehdr = eth_hdr(skb);
2342 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2343 err = -EMSGSIZE;
2344 goto out_free;
2345 }
2346 }
2347
1da177e4
LT		 2348 skb->protocol = proto;
2349 skb->dev = dev;
2350 skb->priority = sk->sk_priority;
2d37a186 2351 skb->mark = sk->sk_mark;
1da177e4 2352
bfd5f4a3
SS		 2353 if (po->has_vnet_hdr) {
2354 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2355 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2356 vnet_hdr.csum_offset)) {
2357 err = -EINVAL;
2358 goto out_free;
2359 }
2360 }
2361
2362 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2363 skb_shinfo(skb)->gso_type = gso_type;
2364
2365 /* Header must be checked, and gso_segs computed. */
2366 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2367 skb_shinfo(skb)->gso_segs = 0;
2368
2369 len += vnet_hdr_len;
2370 }
2371
40893fd0 2372 skb_probe_transport_header(skb, reserve);
c1aad275 2373
3bdc0eba
BG		 2374 if (unlikely(extra_len == 4))
2375 skb->no_fcs = 1;
2376
1da177e4
LT		 2377 /*
2378 * Now send it
2379 */
2380
2381 err = dev_queue_xmit(skb);
2382 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2383 goto out_unlock;
2384
827d9780
BG		 2385 if (need_rls_dev)
2386 dev_put(dev);
1da177e4 2387
40d4e3df 2388 return len;
1da177e4
LT		 2389
2390out_free:
2391 kfree_skb(skb);
2392out_unlock:
827d9780 2393 if (dev && need_rls_dev)
1da177e4
LT		 2394 dev_put(dev);
2395out:
2396 return err;
2397}
2398
69e3c75f
JB		 2399static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2400 struct msghdr *msg, size_t len)
2401{
69e3c75f
JB		 2402 struct sock *sk = sock->sk;
2403 struct packet_sock *po = pkt_sk(sk);
2404 if (po->tx_ring.pg_vec)
2405 return tpacket_snd(po, msg);
2406 else
69e3c75f
JB		 2407 return packet_snd(sock, msg, len);
2408}
2409
1da177e4
LT		 2410/*
2411 * Close a PACKET socket. This is fairly simple. We immediately go
2412 * to 'closed' state and remove our protocol entry in the device list.
2413 */
2414
2415static int packet_release(struct socket *sock)
2416{
2417 struct sock *sk = sock->sk;
2418 struct packet_sock *po;
d12d01d6 2419 struct net *net;
f6fb8f10 2420 union tpacket_req_u req_u;
1da177e4
LT		 2421
2422 if (!sk)
2423 return 0;
2424
3b1e0a65 2425 net = sock_net(sk);
1da177e4
LT		 2426 po = pkt_sk(sk);
2427
0fa7fa98 2428 mutex_lock(&net->packet.sklist_lock);
808f5114 2429 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2430 mutex_unlock(&net->packet.sklist_lock);
2431
2432 preempt_disable();
920de804 2433 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2434 preempt_enable();
1da177e4 2435
808f5114 2436 spin_lock(&po->bind_lock);
ce06b03e 2437 unregister_prot_hook(sk, false);
160ff18a
BG		 2438 if (po->prot_hook.dev) {
2439 dev_put(po->prot_hook.dev);
2440 po->prot_hook.dev = NULL;
2441 }
808f5114 2442 spin_unlock(&po->bind_lock);
1da177e4 2443
1da177e4 2444 packet_flush_mclist(sk);
1da177e4 2445
9665d5d6
PS		 2446 if (po->rx_ring.pg_vec) {
2447 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2448 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2449 }
69e3c75f 2450
9665d5d6
PS		 2451 if (po->tx_ring.pg_vec) {
2452 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2453 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2454 }
1da177e4 2455
dc99f600
DM		 2456 fanout_release(sk);
2457
808f5114 2458 synchronize_net();
1da177e4
LT		 2459 /*
2460 * Now the socket is dead. No more input will appear.
2461 */
1da177e4
LT		 2462 sock_orphan(sk);
2463 sock->sk = NULL;
2464
2465 /* Purge queues */
2466
2467 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2468 sk_refcnt_debug_release(sk);
1da177e4
LT		 2469
2470 sock_put(sk);
2471 return 0;
2472}
2473
2474/*
2475 * Attach a packet hook.
2476 */
2477
0e11c91e 2478static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2479{
2480 struct packet_sock *po = pkt_sk(sk);
dc99f600 2481
aef950b4
WY		 2482 if (po->fanout) {
2483 if (dev)
2484 dev_put(dev);
2485
dc99f600 2486 return -EINVAL;
aef950b4 2487 }
1da177e4
LT		 2488
2489 lock_sock(sk);
2490
2491 spin_lock(&po->bind_lock);
ce06b03e 2492 unregister_prot_hook(sk, true);
1da177e4
LT		 2493 po->num = protocol;
2494 po->prot_hook.type = protocol;
160ff18a
BG		 2495 if (po->prot_hook.dev)
2496 dev_put(po->prot_hook.dev);
1da177e4
LT		 2497 po->prot_hook.dev = dev;
2498
2499 po->ifindex = dev ? dev->ifindex : 0;
2500
2501 if (protocol == 0)
2502 goto out_unlock;
2503
be85d4ad 2504 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2505 register_prot_hook(sk);
be85d4ad
UT		 2506 } else {
2507 sk->sk_err = ENETDOWN;
2508 if (!sock_flag(sk, SOCK_DEAD))
2509 sk->sk_error_report(sk);
1da177e4
LT		 2510 }
2511
2512out_unlock:
2513 spin_unlock(&po->bind_lock);
2514 release_sock(sk);
2515 return 0;
2516}
2517
2518/*
2519 * Bind a packet socket to a device
2520 */
2521
40d4e3df
ED		 2522static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2523 int addr_len)
1da177e4 2524{
40d4e3df 2525 struct sock *sk = sock->sk;
1da177e4
LT		 2526 char name[15];
2527 struct net_device *dev;
2528 int err = -ENODEV;
1ce4f28b 2529
1da177e4
LT		 2530 /*
2531 * Check legality
2532 */
1ce4f28b 2533
8ae55f04 2534 if (addr_len != sizeof(struct sockaddr))
1da177e4 2535 return -EINVAL;
40d4e3df 2536 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2537
3b1e0a65 2538 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2539 if (dev)
1da177e4 2540 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2541 return err;
2542}
1da177e4
LT		 2543
2544static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2545{
40d4e3df
ED		 2546 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2547 struct sock *sk = sock->sk;
1da177e4
LT		 2548 struct net_device *dev = NULL;
2549 int err;
2550
2551
2552 /*
2553 * Check legality
2554 */
1ce4f28b 2555
1da177e4
LT		 2556 if (addr_len < sizeof(struct sockaddr_ll))
2557 return -EINVAL;
2558 if (sll->sll_family != AF_PACKET)
2559 return -EINVAL;
2560
2561 if (sll->sll_ifindex) {
2562 err = -ENODEV;
3b1e0a65 2563 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2564 if (dev == NULL)
2565 goto out;
2566 }
2567 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2568
2569out:
2570 return err;
2571}
2572
2573static struct proto packet_proto = {
2574 .name = "PACKET",
2575 .owner = THIS_MODULE,
2576 .obj_size = sizeof(struct packet_sock),
2577};
2578
2579/*
1ce4f28b 2580 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2581 */
2582
3f378b68
EP		 2583static int packet_create(struct net *net, struct socket *sock, int protocol,
2584 int kern)
1da177e4
LT		 2585{
2586 struct sock *sk;
2587 struct packet_sock *po;
0e11c91e 2588 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2589 int err;
2590
df008c91 2591 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2592 return -EPERM;
be02097c
DM		 2593 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2594 sock->type != SOCK_PACKET)
1da177e4
LT		 2595 return -ESOCKTNOSUPPORT;
2596
2597 sock->state = SS_UNCONNECTED;
2598
2599 err = -ENOBUFS;
6257ff21 2600 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2601 if (sk == NULL)
2602 goto out;
2603
2604 sock->ops = &packet_ops;
1da177e4
LT		 2605 if (sock->type == SOCK_PACKET)
2606 sock->ops = &packet_ops_spkt;
be02097c 2607
1da177e4
LT		 2608 sock_init_data(sock, sk);
2609
2610 po = pkt_sk(sk);
2611 sk->sk_family = PF_PACKET;
0e11c91e 2612 po->num = proto;
1da177e4
LT		 2613
2614 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2615 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2616
2617 /*
2618 * Attach a protocol block
2619 */
2620
2621 spin_lock_init(&po->bind_lock);
905db440 2622 mutex_init(&po->pg_vec_lock);
1da177e4 2623 po->prot_hook.func = packet_rcv;
be02097c 2624
1da177e4
LT		 2625 if (sock->type == SOCK_PACKET)
2626 po->prot_hook.func = packet_rcv_spkt;
be02097c 2627
1da177e4
LT		 2628 po->prot_hook.af_packet_priv = sk;
2629
0e11c91e
AV		 2630 if (proto) {
2631 po->prot_hook.type = proto;
ce06b03e 2632 register_prot_hook(sk);
1da177e4
LT		 2633 }
2634
0fa7fa98 2635 mutex_lock(&net->packet.sklist_lock);
808f5114 2636 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2637 mutex_unlock(&net->packet.sklist_lock);
2638
2639 preempt_disable();
3680453c 2640 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2641 preempt_enable();
808f5114 2642
40d4e3df 2643 return 0;
1da177e4
LT		 2644out:
2645 return err;
2646}
2647
ed85b565
RC		 2648static int packet_recv_error(struct sock *sk, struct msghdr *msg, int len)
2649{
2650 struct sock_exterr_skb *serr;
2651 struct sk_buff *skb, *skb2;
2652 int copied, err;
2653
2654 err = -EAGAIN;
2655 skb = skb_dequeue(&sk->sk_error_queue);
2656 if (skb == NULL)
2657 goto out;
2658
2659 copied = skb->len;
2660 if (copied > len) {
2661 msg->msg_flags |= MSG_TRUNC;
2662 copied = len;
2663 }
2664 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2665 if (err)
2666 goto out_free_skb;
2667
2668 sock_recv_timestamp(msg, sk, skb);
2669
2670 serr = SKB_EXT_ERR(skb);
2671 put_cmsg(msg, SOL_PACKET, PACKET_TX_TIMESTAMP,
2672 sizeof(serr->ee), &serr->ee);
2673
2674 msg->msg_flags |= MSG_ERRQUEUE;
2675 err = copied;
2676
2677 /* Reset and regenerate socket error */
2678 spin_lock_bh(&sk->sk_error_queue.lock);
2679 sk->sk_err = 0;
2680 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) {
2681 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno;
2682 spin_unlock_bh(&sk->sk_error_queue.lock);
2683 sk->sk_error_report(sk);
2684 } else
2685 spin_unlock_bh(&sk->sk_error_queue.lock);
2686
2687out_free_skb:
2688 kfree_skb(skb);
2689out:
2690 return err;
2691}
2692
1da177e4
LT		 2693/*
2694 * Pull a packet from our receive queue and hand it to the user.
2695 * If necessary we block.
2696 */
2697
2698static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2699 struct msghdr *msg, size_t len, int flags)
2700{
2701 struct sock *sk = sock->sk;
2702 struct sk_buff *skb;
2703 int copied, err;
0fb375fb 2704 struct sockaddr_ll *sll;
bfd5f4a3 2705 int vnet_hdr_len = 0;
1da177e4
LT		 2706
2707 err = -EINVAL;
ed85b565 2708 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2709 goto out;
2710
2711#if 0
2712 /* What error should we return now? EUNATTACH? */
2713 if (pkt_sk(sk)->ifindex < 0)
2714 return -ENODEV;
2715#endif
2716
ed85b565
RC		 2717 if (flags & MSG_ERRQUEUE) {
2718 err = packet_recv_error(sk, msg, len);
2719 goto out;
2720 }
2721
1da177e4
LT		 2722 /*
2723 * Call the generic datagram receiver. This handles all sorts
2724 * of horrible races and re-entrancy so we can forget about it
2725 * in the protocol layers.
2726 *
2727 * Now it will return ENETDOWN, if device have just gone down,
2728 * but then it will block.
2729 */
2730
40d4e3df 2731 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2732
2733 /*
1ce4f28b 2734 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2735 * handles the blocking we don't see and worry about blocking
2736 * retries.
2737 */
2738
8ae55f04 2739 if (skb == NULL)
1da177e4
LT		 2740 goto out;
2741
bfd5f4a3
SS		 2742 if (pkt_sk(sk)->has_vnet_hdr) {
2743 struct virtio_net_hdr vnet_hdr = { 0 };
2744
2745 err = -EINVAL;
2746 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2747 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2748 goto out_free;
2749
1f18b717
MK		 2750 len -= vnet_hdr_len;
2751
bfd5f4a3
SS		 2752 if (skb_is_gso(skb)) {
2753 struct skb_shared_info *sinfo = skb_shinfo(skb);
2754
2755 /* This is a hint as to how much should be linear. */
2756 vnet_hdr.hdr_len = skb_headlen(skb);
2757 vnet_hdr.gso_size = sinfo->gso_size;
2758 if (sinfo->gso_type & SKB_GSO_TCPV4)
2759 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2760 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2761 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2762 else if (sinfo->gso_type & SKB_GSO_UDP)
2763 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2764 else if (sinfo->gso_type & SKB_GSO_FCOE)
2765 goto out_free;
2766 else
2767 BUG();
2768 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2769 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2770 } else
2771 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2772
2773 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2774 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2775 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2776 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2777 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2778 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2779 } /* else everything is zero */
2780
2781 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2782 vnet_hdr_len);
2783 if (err < 0)
2784 goto out_free;
2785 }
2786
0fb375fb
EB		 2787 /*
2788 * If the address length field is there to be filled in, we fill
2789 * it in now.
2790 */
2791
ffbc6111 2792 sll = &PACKET_SKB_CB(skb)->sa.ll;
0fb375fb
EB		 2793 if (sock->type == SOCK_PACKET)
2794 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2795 else
2796 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
2797
1da177e4
LT		 2798 /*
2799 * You lose any data beyond the buffer you gave. If it worries a
2800 * user program they can ask the device for its MTU anyway.
2801 */
2802
2803 copied = skb->len;
40d4e3df
ED		 2804 if (copied > len) {
2805 copied = len;
2806 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2807 }
2808
2809 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2810 if (err)
2811 goto out_free;
2812
3b885787 2813 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4
LT		 2814
2815 if (msg->msg_name)
ffbc6111
HX		 2816 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2817 msg->msg_namelen);
1da177e4 2818
8dc41944 2819 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2820 struct tpacket_auxdata aux;
2821
2822 aux.tp_status = TP_STATUS_USER;
2823 if (skb->ip_summed == CHECKSUM_PARTIAL)
2824 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2825 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2826 aux.tp_snaplen = skb->len;
2827 aux.tp_mac = 0;
bbe735e4 2828 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2829 if (vlan_tx_tag_present(skb)) {
2830 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2831 aux.tp_status |= TP_STATUS_VLAN_VALID;
2832 } else {
2833 aux.tp_vlan_tci = 0;
2834 }
13fcb7bd 2835 aux.tp_padding = 0;
ffbc6111 2836 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2837 }
2838
1da177e4
LT		 2839 /*
2840 * Free or return the buffer as appropriate. Again this
2841 * hides all the races and re-entrancy issues from us.
2842 */
bfd5f4a3 2843 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2844
2845out_free:
2846 skb_free_datagram(sk, skb);
2847out:
2848 return err;
2849}
2850
1da177e4
LT		 2851static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2852 int *uaddr_len, int peer)
2853{
2854 struct net_device *dev;
2855 struct sock *sk = sock->sk;
2856
2857 if (peer)
2858 return -EOPNOTSUPP;
2859
2860 uaddr->sa_family = AF_PACKET;
654d1f8a
ED		 2861 rcu_read_lock();
2862 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2863 if (dev)
67286640 2864 strncpy(uaddr->sa_data, dev->name, 14);
654d1f8a 2865 else
1da177e4 2866 memset(uaddr->sa_data, 0, 14);
654d1f8a 2867 rcu_read_unlock();
1da177e4
LT		 2868 *uaddr_len = sizeof(*uaddr);
2869
2870 return 0;
2871}
1da177e4
LT		 2872
2873static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2874 int *uaddr_len, int peer)
2875{
2876 struct net_device *dev;
2877 struct sock *sk = sock->sk;
2878 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2879 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2880
2881 if (peer)
2882 return -EOPNOTSUPP;
2883
2884 sll->sll_family = AF_PACKET;
2885 sll->sll_ifindex = po->ifindex;
2886 sll->sll_protocol = po->num;
67286640 2887 sll->sll_pkttype = 0;
654d1f8a
ED		 2888 rcu_read_lock();
2889 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2890 if (dev) {
2891 sll->sll_hatype = dev->type;
2892 sll->sll_halen = dev->addr_len;
2893 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2894 } else {
2895 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2896 sll->sll_halen = 0;
2897 }
654d1f8a 2898 rcu_read_unlock();
0fb375fb 2899 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2900
2901 return 0;
2902}
2903
2aeb0b88
WC		 2904static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2905 int what)
1da177e4
LT		 2906{
2907 switch (i->type) {
2908 case PACKET_MR_MULTICAST:
1162563f
JP		 2909 if (i->alen != dev->addr_len)
2910 return -EINVAL;
1da177e4 2911 if (what > 0)
22bedad3 2912 return dev_mc_add(dev, i->addr);
1da177e4 2913 else
22bedad3 2914 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2915 break;
2916 case PACKET_MR_PROMISC:
2aeb0b88 2917 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2918 break;
2919 case PACKET_MR_ALLMULTI:
2aeb0b88 2920 return dev_set_allmulti(dev, what);
1da177e4 2921 break;
d95ed927 2922 case PACKET_MR_UNICAST:
1162563f
JP		 2923 if (i->alen != dev->addr_len)
2924 return -EINVAL;
d95ed927 2925 if (what > 0)
a748ee24 2926 return dev_uc_add(dev, i->addr);
d95ed927 2927 else
a748ee24 2928 return dev_uc_del(dev, i->addr);
d95ed927 2929 break;
40d4e3df
ED		 2930 default:
2931 break;
1da177e4 2932 }
2aeb0b88 2933 return 0;
1da177e4
LT		 2934}
2935
2936static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2937{
40d4e3df 2938 for ( ; i; i = i->next) {
1da177e4
LT		 2939 if (i->ifindex == dev->ifindex)
2940 packet_dev_mc(dev, i, what);
2941 }
2942}
2943
0fb375fb 2944static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2945{
2946 struct packet_sock *po = pkt_sk(sk);
2947 struct packet_mclist *ml, *i;
2948 struct net_device *dev;
2949 int err;
2950
2951 rtnl_lock();
2952
2953 err = -ENODEV;
3b1e0a65 2954 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2955 if (!dev)
2956 goto done;
2957
2958 err = -EINVAL;
1162563f 2959 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2960 goto done;
2961
2962 err = -ENOBUFS;
8b3a7005 2963 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 2964 if (i == NULL)
2965 goto done;
2966
2967 err = 0;
2968 for (ml = po->mclist; ml; ml = ml->next) {
2969 if (ml->ifindex == mreq->mr_ifindex &&
2970 ml->type == mreq->mr_type &&
2971 ml->alen == mreq->mr_alen &&
2972 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2973 ml->count++;
2974 /* Free the new element ... */
2975 kfree(i);
2976 goto done;
2977 }
2978 }
2979
2980 i->type = mreq->mr_type;
2981 i->ifindex = mreq->mr_ifindex;
2982 i->alen = mreq->mr_alen;
2983 memcpy(i->addr, mreq->mr_address, i->alen);
2984 i->count = 1;
2985 i->next = po->mclist;
2986 po->mclist = i;
2aeb0b88
WC		 2987 err = packet_dev_mc(dev, i, 1);
2988 if (err) {
2989 po->mclist = i->next;
2990 kfree(i);
2991 }
1da177e4
LT		 2992
2993done:
2994 rtnl_unlock();
2995 return err;
2996}
2997
0fb375fb 2998static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2999{
3000 struct packet_mclist *ml, **mlp;
3001
3002 rtnl_lock();
3003
3004 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3005 if (ml->ifindex == mreq->mr_ifindex &&
3006 ml->type == mreq->mr_type &&
3007 ml->alen == mreq->mr_alen &&
3008 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3009 if (--ml->count == 0) {
3010 struct net_device *dev;
3011 *mlp = ml->next;
ad959e76
ED		 3012 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3013 if (dev)
1da177e4 3014 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3015 kfree(ml);
3016 }
3017 rtnl_unlock();
3018 return 0;
3019 }
3020 }
3021 rtnl_unlock();
3022 return -EADDRNOTAVAIL;
3023}
3024
3025static void packet_flush_mclist(struct sock *sk)
3026{
3027 struct packet_sock *po = pkt_sk(sk);
3028 struct packet_mclist *ml;
3029
3030 if (!po->mclist)
3031 return;
3032
3033 rtnl_lock();
3034 while ((ml = po->mclist) != NULL) {
3035 struct net_device *dev;
3036
3037 po->mclist = ml->next;
ad959e76
ED		 3038 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3039 if (dev != NULL)
1da177e4 3040 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3041 kfree(ml);
3042 }
3043 rtnl_unlock();
3044}
1da177e4
LT		 3045
3046static int
b7058842 3047packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3048{
3049 struct sock *sk = sock->sk;
8dc41944 3050 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3051 int ret;
3052
3053 if (level != SOL_PACKET)
3054 return -ENOPROTOOPT;
3055
69e3c75f 3056 switch (optname) {
1ce4f28b 3057 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3058 case PACKET_DROP_MEMBERSHIP:
3059 {
0fb375fb
EB		 3060 struct packet_mreq_max mreq;
3061 int len = optlen;
3062 memset(&mreq, 0, sizeof(mreq));
3063 if (len < sizeof(struct packet_mreq))
1da177e4 3064 return -EINVAL;
0fb375fb
EB		 3065 if (len > sizeof(mreq))
3066 len = sizeof(mreq);
40d4e3df 3067 if (copy_from_user(&mreq, optval, len))
1da177e4 3068 return -EFAULT;
0fb375fb
EB		 3069 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3070 return -EINVAL;
1da177e4
LT		 3071 if (optname == PACKET_ADD_MEMBERSHIP)
3072 ret = packet_mc_add(sk, &mreq);
3073 else
3074 ret = packet_mc_drop(sk, &mreq);
3075 return ret;
3076 }
a2efcfa0 3077
1da177e4 3078 case PACKET_RX_RING:
69e3c75f 3079 case PACKET_TX_RING:
1da177e4 3080 {
f6fb8f10 3081 union tpacket_req_u req_u;
3082 int len;
1da177e4 3083
f6fb8f10 3084 switch (po->tp_version) {
3085 case TPACKET_V1:
3086 case TPACKET_V2:
3087 len = sizeof(req_u.req);
3088 break;
3089 case TPACKET_V3:
3090 default:
3091 len = sizeof(req_u.req3);
3092 break;
3093 }
3094 if (optlen < len)
1da177e4 3095 return -EINVAL;
bfd5f4a3
SS		 3096 if (pkt_sk(sk)->has_vnet_hdr)
3097 return -EINVAL;
f6fb8f10 3098 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3099 return -EFAULT;
f6fb8f10 3100 return packet_set_ring(sk, &req_u, 0,
3101 optname == PACKET_TX_RING);
1da177e4
LT		 3102 }
3103 case PACKET_COPY_THRESH:
3104 {
3105 int val;
3106
40d4e3df 3107 if (optlen != sizeof(val))
1da177e4 3108 return -EINVAL;
40d4e3df 3109 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3110 return -EFAULT;
3111
3112 pkt_sk(sk)->copy_thresh = val;
3113 return 0;
3114 }
bbd6ef87
PM		 3115 case PACKET_VERSION:
3116 {
3117 int val;
3118
3119 if (optlen != sizeof(val))
3120 return -EINVAL;
69e3c75f 3121 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3122 return -EBUSY;
3123 if (copy_from_user(&val, optval, sizeof(val)))
3124 return -EFAULT;
3125 switch (val) {
3126 case TPACKET_V1:
3127 case TPACKET_V2:
f6fb8f10 3128 case TPACKET_V3:
bbd6ef87
PM		 3129 po->tp_version = val;
3130 return 0;
3131 default:
3132 return -EINVAL;
3133 }
3134 }
8913336a
PM		 3135 case PACKET_RESERVE:
3136 {
3137 unsigned int val;
3138
3139 if (optlen != sizeof(val))
3140 return -EINVAL;
69e3c75f 3141 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3142 return -EBUSY;
3143 if (copy_from_user(&val, optval, sizeof(val)))
3144 return -EFAULT;
3145 po->tp_reserve = val;
3146 return 0;
3147 }
69e3c75f
JB		 3148 case PACKET_LOSS:
3149 {
3150 unsigned int val;
3151
3152 if (optlen != sizeof(val))
3153 return -EINVAL;
3154 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3155 return -EBUSY;
3156 if (copy_from_user(&val, optval, sizeof(val)))
3157 return -EFAULT;
3158 po->tp_loss = !!val;
3159 return 0;
3160 }
8dc41944
HX		 3161 case PACKET_AUXDATA:
3162 {
3163 int val;
3164
3165 if (optlen < sizeof(val))
3166 return -EINVAL;
3167 if (copy_from_user(&val, optval, sizeof(val)))
3168 return -EFAULT;
3169
3170 po->auxdata = !!val;
3171 return 0;
3172 }
80feaacb
PWJ		 3173 case PACKET_ORIGDEV:
3174 {
3175 int val;
3176
3177 if (optlen < sizeof(val))
3178 return -EINVAL;
3179 if (copy_from_user(&val, optval, sizeof(val)))
3180 return -EFAULT;
3181
3182 po->origdev = !!val;
3183 return 0;
3184 }
bfd5f4a3
SS		 3185 case PACKET_VNET_HDR:
3186 {
3187 int val;
3188
3189 if (sock->type != SOCK_RAW)
3190 return -EINVAL;
3191 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3192 return -EBUSY;
3193 if (optlen < sizeof(val))
3194 return -EINVAL;
3195 if (copy_from_user(&val, optval, sizeof(val)))
3196 return -EFAULT;
3197
3198 po->has_vnet_hdr = !!val;
3199 return 0;
3200 }
614f60fa
SM		 3201 case PACKET_TIMESTAMP:
3202 {
3203 int val;
3204
3205 if (optlen != sizeof(val))
3206 return -EINVAL;
3207 if (copy_from_user(&val, optval, sizeof(val)))
3208 return -EFAULT;
3209
3210 po->tp_tstamp = val;
3211 return 0;
3212 }
dc99f600
DM		 3213 case PACKET_FANOUT:
3214 {
3215 int val;
3216
3217 if (optlen != sizeof(val))
3218 return -EINVAL;
3219 if (copy_from_user(&val, optval, sizeof(val)))
3220 return -EFAULT;
3221
3222 return fanout_add(sk, val & 0xffff, val >> 16);
3223 }
5920cd3a
PC		 3224 case PACKET_TX_HAS_OFF:
3225 {
3226 unsigned int val;
3227
3228 if (optlen != sizeof(val))
3229 return -EINVAL;
3230 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3231 return -EBUSY;
3232 if (copy_from_user(&val, optval, sizeof(val)))
3233 return -EFAULT;
3234 po->tp_tx_has_off = !!val;
3235 return 0;
3236 }
1da177e4
LT		 3237 default:
3238 return -ENOPROTOOPT;
3239 }
3240}
3241
3242static int packet_getsockopt(struct socket *sock, int level, int optname,
3243 char __user *optval, int __user *optlen)
3244{
3245 int len;
c06fff6e 3246 int val, lv = sizeof(val);
1da177e4
LT		 3247 struct sock *sk = sock->sk;
3248 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3249 void *data = &val;
8dc41944 3250 struct tpacket_stats st;
f6fb8f10 3251 union tpacket_stats_u st_u;
1da177e4
LT		 3252
3253 if (level != SOL_PACKET)
3254 return -ENOPROTOOPT;
3255
8ae55f04
KK		 3256 if (get_user(len, optlen))
3257 return -EFAULT;
1da177e4
LT		 3258
3259 if (len < 0)
3260 return -EINVAL;
1ce4f28b 3261
69e3c75f 3262 switch (optname) {
1da177e4 3263 case PACKET_STATISTICS:
1da177e4 3264 spin_lock_bh(&sk->sk_receive_queue.lock);
f6fb8f10 3265 if (po->tp_version == TPACKET_V3) {
c06fff6e 3266 lv = sizeof(struct tpacket_stats_v3);
f6fb8f10 3267 memcpy(&st_u.stats3, &po->stats,
c06fff6e 3268 sizeof(struct tpacket_stats));
f6fb8f10 3269 st_u.stats3.tp_freeze_q_cnt =
c06fff6e 3270 po->stats_u.stats3.tp_freeze_q_cnt;
f6fb8f10 3271 st_u.stats3.tp_packets += po->stats.tp_drops;
3272 data = &st_u.stats3;
3273 } else {
c06fff6e 3274 lv = sizeof(struct tpacket_stats);
f6fb8f10 3275 st = po->stats;
3276 st.tp_packets += st.tp_drops;
3277 data = &st;
3278 }
1da177e4
LT		 3279 memset(&po->stats, 0, sizeof(st));
3280 spin_unlock_bh(&sk->sk_receive_queue.lock);
8dc41944
HX		 3281 break;
3282 case PACKET_AUXDATA:
8dc41944 3283 val = po->auxdata;
80feaacb
PWJ		 3284 break;
3285 case PACKET_ORIGDEV:
80feaacb 3286 val = po->origdev;
bfd5f4a3
SS		 3287 break;
3288 case PACKET_VNET_HDR:
bfd5f4a3 3289 val = po->has_vnet_hdr;
1da177e4 3290 break;
bbd6ef87 3291 case PACKET_VERSION:
bbd6ef87 3292 val = po->tp_version;
bbd6ef87
PM		 3293 break;
3294 case PACKET_HDRLEN:
3295 if (len > sizeof(int))
3296 len = sizeof(int);
3297 if (copy_from_user(&val, optval, len))
3298 return -EFAULT;
3299 switch (val) {
3300 case TPACKET_V1:
3301 val = sizeof(struct tpacket_hdr);
3302 break;
3303 case TPACKET_V2:
3304 val = sizeof(struct tpacket2_hdr);
3305 break;
f6fb8f10 3306 case TPACKET_V3:
3307 val = sizeof(struct tpacket3_hdr);
3308 break;
bbd6ef87
PM		 3309 default:
3310 return -EINVAL;
3311 }
bbd6ef87 3312 break;
8913336a 3313 case PACKET_RESERVE:
8913336a 3314 val = po->tp_reserve;
8913336a 3315 break;
69e3c75f 3316 case PACKET_LOSS:
69e3c75f 3317 val = po->tp_loss;
69e3c75f 3318 break;
614f60fa 3319 case PACKET_TIMESTAMP:
614f60fa 3320 val = po->tp_tstamp;
614f60fa 3321 break;
dc99f600 3322 case PACKET_FANOUT:
dc99f600
DM		 3323 val = (po->fanout ?
3324 ((u32)po->fanout->id |
77f65ebd
WB		 3325 ((u32)po->fanout->type << 16) |
3326 ((u32)po->fanout->flags << 24)) :
dc99f600 3327 0);
dc99f600 3328 break;
5920cd3a
PC		 3329 case PACKET_TX_HAS_OFF:
3330 val = po->tp_tx_has_off;
3331 break;
1da177e4
LT		 3332 default:
3333 return -ENOPROTOOPT;
3334 }
3335
c06fff6e
ED		 3336 if (len > lv)
3337 len = lv;
8ae55f04
KK		 3338 if (put_user(len, optlen))
3339 return -EFAULT;
8dc41944
HX		 3340 if (copy_to_user(optval, data, len))
3341 return -EFAULT;
8ae55f04 3342 return 0;
1da177e4
LT		 3343}
3344
3345
3346static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
3347{
3348 struct sock *sk;
ad930650 3349 struct net_device *dev = data;
c346dca1 3350 struct net *net = dev_net(dev);
1da177e4 3351
808f5114 3352 rcu_read_lock();
b67bfe0d 3353 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3354 struct packet_sock *po = pkt_sk(sk);
3355
3356 switch (msg) {
3357 case NETDEV_UNREGISTER:
1da177e4
LT		 3358 if (po->mclist)
3359 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3360 /* fallthrough */
3361
1da177e4
LT		 3362 case NETDEV_DOWN:
3363 if (dev->ifindex == po->ifindex) {
3364 spin_lock(&po->bind_lock);
3365 if (po->running) {
ce06b03e 3366 __unregister_prot_hook(sk, false);
1da177e4
LT		 3367 sk->sk_err = ENETDOWN;
3368 if (!sock_flag(sk, SOCK_DEAD))
3369 sk->sk_error_report(sk);
3370 }
3371 if (msg == NETDEV_UNREGISTER) {
3372 po->ifindex = -1;
160ff18a
BG		 3373 if (po->prot_hook.dev)
3374 dev_put(po->prot_hook.dev);
1da177e4
LT		 3375 po->prot_hook.dev = NULL;
3376 }
3377 spin_unlock(&po->bind_lock);
3378 }
3379 break;
3380 case NETDEV_UP:
808f5114 3381 if (dev->ifindex == po->ifindex) {
3382 spin_lock(&po->bind_lock);
ce06b03e
DM		 3383 if (po->num)
3384 register_prot_hook(sk);
808f5114 3385 spin_unlock(&po->bind_lock);
1da177e4 3386 }
1da177e4
LT		 3387 break;
3388 }
3389 }
808f5114 3390 rcu_read_unlock();
1da177e4
LT		 3391 return NOTIFY_DONE;
3392}
3393
3394
3395static int packet_ioctl(struct socket *sock, unsigned int cmd,
3396 unsigned long arg)
3397{
3398 struct sock *sk = sock->sk;
3399
69e3c75f 3400 switch (cmd) {
40d4e3df
ED		 3401 case SIOCOUTQ:
3402 {
3403 int amount = sk_wmem_alloc_get(sk);
31e6d363 3404
40d4e3df
ED		 3405 return put_user(amount, (int __user *)arg);
3406 }
3407 case SIOCINQ:
3408 {
3409 struct sk_buff *skb;
3410 int amount = 0;
3411
3412 spin_lock_bh(&sk->sk_receive_queue.lock);
3413 skb = skb_peek(&sk->sk_receive_queue);
3414 if (skb)
3415 amount = skb->len;
3416 spin_unlock_bh(&sk->sk_receive_queue.lock);
3417 return put_user(amount, (int __user *)arg);
3418 }
3419 case SIOCGSTAMP:
3420 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3421 case SIOCGSTAMPNS:
3422 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3423
1da177e4 3424#ifdef CONFIG_INET
40d4e3df
ED		 3425 case SIOCADDRT:
3426 case SIOCDELRT:
3427 case SIOCDARP:
3428 case SIOCGARP:
3429 case SIOCSARP:
3430 case SIOCGIFADDR:
3431 case SIOCSIFADDR:
3432 case SIOCGIFBRDADDR:
3433 case SIOCSIFBRDADDR:
3434 case SIOCGIFNETMASK:
3435 case SIOCSIFNETMASK:
3436 case SIOCGIFDSTADDR:
3437 case SIOCSIFDSTADDR:
3438 case SIOCSIFFLAGS:
40d4e3df 3439 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3440#endif
3441
40d4e3df
ED		 3442 default:
3443 return -ENOIOCTLCMD;
1da177e4
LT		 3444 }
3445 return 0;
3446}
3447
40d4e3df 3448static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3449 poll_table *wait)
3450{
3451 struct sock *sk = sock->sk;
3452 struct packet_sock *po = pkt_sk(sk);
3453 unsigned int mask = datagram_poll(file, sock, wait);
3454
3455 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3456 if (po->rx_ring.pg_vec) {
f6fb8f10 3457 if (!packet_previous_rx_frame(po, &po->rx_ring,
3458 TP_STATUS_KERNEL))
1da177e4
LT		 3459 mask |= POLLIN | POLLRDNORM;
3460 }
3461 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3462 spin_lock_bh(&sk->sk_write_queue.lock);
3463 if (po->tx_ring.pg_vec) {
3464 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3465 mask |= POLLOUT | POLLWRNORM;
3466 }
3467 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3468 return mask;
3469}
3470
3471
3472/* Dirty? Well, I still did not learn better way to account
3473 * for user mmaps.
3474 */
3475
3476static void packet_mm_open(struct vm_area_struct *vma)
3477{
3478 struct file *file = vma->vm_file;
40d4e3df 3479 struct socket *sock = file->private_data;
1da177e4 3480 struct sock *sk = sock->sk;
1ce4f28b 3481
1da177e4
LT		 3482 if (sk)
3483 atomic_inc(&pkt_sk(sk)->mapped);
3484}
3485
3486static void packet_mm_close(struct vm_area_struct *vma)
3487{
3488 struct file *file = vma->vm_file;
40d4e3df 3489 struct socket *sock = file->private_data;
1da177e4 3490 struct sock *sk = sock->sk;
1ce4f28b 3491
1da177e4
LT		 3492 if (sk)
3493 atomic_dec(&pkt_sk(sk)->mapped);
3494}
3495
f0f37e2f 3496static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3497 .open = packet_mm_open,
3498 .close = packet_mm_close,
1da177e4
LT		 3499};
3500
0e3125c7
NH		 3501static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3502 unsigned int len)
1da177e4
LT		 3503{
3504 int i;
3505
4ebf0ae2 3506 for (i = 0; i < len; i++) {
0e3125c7 3507 if (likely(pg_vec[i].buffer)) {
c56b4d90 3508 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3509 vfree(pg_vec[i].buffer);
3510 else
3511 free_pages((unsigned long)pg_vec[i].buffer,
3512 order);
3513 pg_vec[i].buffer = NULL;
3514 }
1da177e4
LT		 3515 }
3516 kfree(pg_vec);
3517}
3518
eea49cc9 3519static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3520{
0e3125c7
NH		 3521 char *buffer = NULL;
3522 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3523 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3524
3525 buffer = (char *) __get_free_pages(gfp_flags, order);
3526
3527 if (buffer)
3528 return buffer;
3529
3530 /*
3531 * __get_free_pages failed, fall back to vmalloc
3532 */
bbce5a59 3533 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3534
0e3125c7
NH		 3535 if (buffer)
3536 return buffer;
3537
3538 /*
3539 * vmalloc failed, lets dig into swap here
3540 */
0e3125c7
NH		 3541 gfp_flags &= ~__GFP_NORETRY;
3542 buffer = (char *)__get_free_pages(gfp_flags, order);
3543 if (buffer)
3544 return buffer;
3545
3546 /*
3547 * complete and utter failure
3548 */
3549 return NULL;
4ebf0ae2
DM		 3550}
3551
0e3125c7 3552static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3553{
3554 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3555 struct pgv *pg_vec;
4ebf0ae2
DM		 3556 int i;
3557
0e3125c7 3558 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3559 if (unlikely(!pg_vec))
3560 goto out;
3561
3562 for (i = 0; i < block_nr; i++) {
c56b4d90 3563 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3564 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3565 goto out_free_pgvec;
3566 }
3567
3568out:
3569 return pg_vec;
3570
3571out_free_pgvec:
3572 free_pg_vec(pg_vec, order, block_nr);
3573 pg_vec = NULL;
3574 goto out;
3575}
1da177e4 3576
f6fb8f10 3577static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3578 int closing, int tx_ring)
1da177e4 3579{
0e3125c7 3580 struct pgv *pg_vec = NULL;
1da177e4 3581 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3582 int was_running, order = 0;
69e3c75f
JB		 3583 struct packet_ring_buffer *rb;
3584 struct sk_buff_head *rb_queue;
0e11c91e 3585 __be16 num;
f6fb8f10 3586 int err = -EINVAL;
3587 /* Added to avoid minimal code churn */
3588 struct tpacket_req *req = &req_u->req;
3589
3590 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3591 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3592 WARN(1, "Tx-ring is not supported.\n");
3593 goto out;
3594 }
1ce4f28b 3595
69e3c75f
JB		 3596 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3597 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3598
69e3c75f
JB		 3599 err = -EBUSY;
3600 if (!closing) {
3601 if (atomic_read(&po->mapped))
3602 goto out;
3603 if (atomic_read(&rb->pending))
3604 goto out;
3605 }
1da177e4 3606
69e3c75f
JB		 3607 if (req->tp_block_nr) {
3608 /* Sanity tests and some calculations */
3609 err = -EBUSY;
3610 if (unlikely(rb->pg_vec))
3611 goto out;
1da177e4 3612
bbd6ef87
PM		 3613 switch (po->tp_version) {
3614 case TPACKET_V1:
3615 po->tp_hdrlen = TPACKET_HDRLEN;
3616 break;
3617 case TPACKET_V2:
3618 po->tp_hdrlen = TPACKET2_HDRLEN;
3619 break;
f6fb8f10 3620 case TPACKET_V3:
3621 po->tp_hdrlen = TPACKET3_HDRLEN;
3622 break;
bbd6ef87
PM		 3623 }
3624
69e3c75f 3625 err = -EINVAL;
4ebf0ae2 3626 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3627 goto out;
4ebf0ae2 3628 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3629 goto out;
8913336a 3630 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3631 po->tp_reserve))
3632 goto out;
4ebf0ae2 3633 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3634 goto out;
1da177e4 3635
69e3c75f
JB		 3636 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3637 if (unlikely(rb->frames_per_block <= 0))
3638 goto out;
3639 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3640 req->tp_frame_nr))
3641 goto out;
1da177e4
LT		 3642
3643 err = -ENOMEM;
4ebf0ae2
DM		 3644 order = get_order(req->tp_block_size);
3645 pg_vec = alloc_pg_vec(req, order);
3646 if (unlikely(!pg_vec))
1da177e4 3647 goto out;
f6fb8f10 3648 switch (po->tp_version) {
3649 case TPACKET_V3:
3650 /* Transmit path is not supported. We checked
3651 * it above but just being paranoid
3652 */
3653 if (!tx_ring)
3654 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3655 break;
3656 default:
3657 break;
3658 }
69e3c75f
JB		 3659 }
3660 /* Done */
3661 else {
3662 err = -EINVAL;
4ebf0ae2 3663 if (unlikely(req->tp_frame_nr))
69e3c75f 3664 goto out;
1da177e4
LT		 3665 }
3666
3667 lock_sock(sk);
3668
3669 /* Detach socket from network */
3670 spin_lock(&po->bind_lock);
3671 was_running = po->running;
3672 num = po->num;
3673 if (was_running) {
1da177e4 3674 po->num = 0;
ce06b03e 3675 __unregister_prot_hook(sk, false);
1da177e4
LT		 3676 }
3677 spin_unlock(&po->bind_lock);
1ce4f28b 3678
1da177e4
LT		 3679 synchronize_net();
3680
3681 err = -EBUSY;
905db440 3682 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3683 if (closing || atomic_read(&po->mapped) == 0) {
3684 err = 0;
69e3c75f 3685 spin_lock_bh(&rb_queue->lock);
c053fd96 3686 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3687 rb->frame_max = (req->tp_frame_nr - 1);
3688 rb->head = 0;
3689 rb->frame_size = req->tp_frame_size;
3690 spin_unlock_bh(&rb_queue->lock);
3691
c053fd96
CG		 3692 swap(rb->pg_vec_order, order);
3693 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3694
3695 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3696 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3697 tpacket_rcv : packet_rcv;
3698 skb_queue_purge(rb_queue);
1da177e4 3699 if (atomic_read(&po->mapped))
40d4e3df
ED		 3700 pr_err("packet_mmap: vma is busy: %d\n",
3701 atomic_read(&po->mapped));
1da177e4 3702 }
905db440 3703 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3704
3705 spin_lock(&po->bind_lock);
ce06b03e 3706 if (was_running) {
1da177e4 3707 po->num = num;
ce06b03e 3708 register_prot_hook(sk);
1da177e4
LT		 3709 }
3710 spin_unlock(&po->bind_lock);
f6fb8f10 3711 if (closing && (po->tp_version > TPACKET_V2)) {
3712 /* Because we don't support block-based V3 on tx-ring */
3713 if (!tx_ring)
3714 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3715 }
1da177e4
LT		 3716 release_sock(sk);
3717
1da177e4
LT		 3718 if (pg_vec)
3719 free_pg_vec(pg_vec, order, req->tp_block_nr);
3720out:
3721 return err;
3722}
3723
69e3c75f
JB		 3724static int packet_mmap(struct file *file, struct socket *sock,
3725 struct vm_area_struct *vma)
1da177e4
LT		 3726{
3727 struct sock *sk = sock->sk;
3728 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3729 unsigned long size, expected_size;
3730 struct packet_ring_buffer *rb;
1da177e4
LT		 3731 unsigned long start;
3732 int err = -EINVAL;
3733 int i;
3734
3735 if (vma->vm_pgoff)
3736 return -EINVAL;
3737
905db440 3738 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3739
3740 expected_size = 0;
3741 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3742 if (rb->pg_vec) {
3743 expected_size += rb->pg_vec_len
3744 * rb->pg_vec_pages
3745 * PAGE_SIZE;
3746 }
3747 }
3748
3749 if (expected_size == 0)
1da177e4 3750 goto out;
69e3c75f
JB		 3751
3752 size = vma->vm_end - vma->vm_start;
3753 if (size != expected_size)
1da177e4
LT		 3754 goto out;
3755
1da177e4 3756 start = vma->vm_start;
69e3c75f
JB		 3757 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3758 if (rb->pg_vec == NULL)
3759 continue;
3760
3761 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3762 struct page *page;
3763 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3764 int pg_num;
3765
c56b4d90
CG		 3766 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3767 page = pgv_to_page(kaddr);
69e3c75f
JB		 3768 err = vm_insert_page(vma, start, page);
3769 if (unlikely(err))
3770 goto out;
3771 start += PAGE_SIZE;
0e3125c7 3772 kaddr += PAGE_SIZE;
69e3c75f 3773 }
4ebf0ae2 3774 }
1da177e4 3775 }
69e3c75f 3776
4ebf0ae2 3777 atomic_inc(&po->mapped);
1da177e4
LT		 3778 vma->vm_ops = &packet_mmap_ops;
3779 err = 0;
3780
3781out:
905db440 3782 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3783 return err;
3784}
1da177e4 3785
90ddc4f0 3786static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3787 .family = PF_PACKET,
3788 .owner = THIS_MODULE,
3789 .release = packet_release,
3790 .bind = packet_bind_spkt,
3791 .connect = sock_no_connect,
3792 .socketpair = sock_no_socketpair,
3793 .accept = sock_no_accept,
3794 .getname = packet_getname_spkt,
3795 .poll = datagram_poll,
3796 .ioctl = packet_ioctl,
3797 .listen = sock_no_listen,
3798 .shutdown = sock_no_shutdown,
3799 .setsockopt = sock_no_setsockopt,
3800 .getsockopt = sock_no_getsockopt,
3801 .sendmsg = packet_sendmsg_spkt,
3802 .recvmsg = packet_recvmsg,
3803 .mmap = sock_no_mmap,
3804 .sendpage = sock_no_sendpage,
3805};
1da177e4 3806
90ddc4f0 3807static const struct proto_ops packet_ops = {
1da177e4
LT		 3808 .family = PF_PACKET,
3809 .owner = THIS_MODULE,
3810 .release = packet_release,
3811 .bind = packet_bind,
3812 .connect = sock_no_connect,
3813 .socketpair = sock_no_socketpair,
3814 .accept = sock_no_accept,
1ce4f28b 3815 .getname = packet_getname,
1da177e4
LT		 3816 .poll = packet_poll,
3817 .ioctl = packet_ioctl,
3818 .listen = sock_no_listen,
3819 .shutdown = sock_no_shutdown,
3820 .setsockopt = packet_setsockopt,
3821 .getsockopt = packet_getsockopt,
3822 .sendmsg = packet_sendmsg,
3823 .recvmsg = packet_recvmsg,
3824 .mmap = packet_mmap,
3825 .sendpage = sock_no_sendpage,
3826};
3827
ec1b4cf7 3828static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3829 .family = PF_PACKET,
3830 .create = packet_create,
3831 .owner = THIS_MODULE,
3832};
3833
3834static struct notifier_block packet_netdev_notifier = {
40d4e3df 3835 .notifier_call = packet_notifier,
1da177e4
LT		 3836};
3837
3838#ifdef CONFIG_PROC_FS
1da177e4
LT		 3839
3840static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3841 __acquires(RCU)
1da177e4 3842{
e372c414 3843 struct net *net = seq_file_net(seq);
808f5114 3844
3845 rcu_read_lock();
3846 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3847}
3848
3849static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3850{
1bf40954 3851 struct net *net = seq_file_net(seq);
808f5114 3852 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3853}
3854
3855static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3856 __releases(RCU)
1da177e4 3857{
808f5114 3858 rcu_read_unlock();
1da177e4
LT		 3859}
3860
1ce4f28b 3861static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3862{
3863 if (v == SEQ_START_TOKEN)
3864 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3865 else {
b7ceabd9 3866 struct sock *s = sk_entry(v);
1da177e4
LT		 3867 const struct packet_sock *po = pkt_sk(s);
3868
3869 seq_printf(seq,
71338aa7 3870 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3871 s,
3872 atomic_read(&s->sk_refcnt),
3873 s->sk_type,
3874 ntohs(po->num),
3875 po->ifindex,
3876 po->running,
3877 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 3878 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 3879 sock_i_ino(s));
1da177e4
LT		 3880 }
3881
3882 return 0;
3883}
3884
56b3d975 3885static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3886 .start = packet_seq_start,
3887 .next = packet_seq_next,
3888 .stop = packet_seq_stop,
3889 .show = packet_seq_show,
3890};
3891
3892static int packet_seq_open(struct inode *inode, struct file *file)
3893{
e372c414
DL		 3894 return seq_open_net(inode, file, &packet_seq_ops,
3895 sizeof(struct seq_net_private));
1da177e4
LT		 3896}
3897
da7071d7 3898static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3899 .owner = THIS_MODULE,
3900 .open = packet_seq_open,
3901 .read = seq_read,
3902 .llseek = seq_lseek,
e372c414 3903 .release = seq_release_net,
1da177e4
LT		 3904};
3905
3906#endif
3907
2c8c1e72 3908static int __net_init packet_net_init(struct net *net)
d12d01d6 3909{
0fa7fa98 3910 mutex_init(&net->packet.sklist_lock);
2aaef4e4 3911 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 3912
d4beaa66 3913 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 3914 return -ENOMEM;
3915
3916 return 0;
3917}
3918
2c8c1e72 3919static void __net_exit packet_net_exit(struct net *net)
d12d01d6 3920{
ece31ffd 3921 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 3922}
3923
3924static struct pernet_operations packet_net_ops = {
3925 .init = packet_net_init,
3926 .exit = packet_net_exit,
3927};
3928
3929
1da177e4
LT		 3930static void __exit packet_exit(void)
3931{
1da177e4 3932 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3933 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3934 sock_unregister(PF_PACKET);
3935 proto_unregister(&packet_proto);
3936}
3937
3938static int __init packet_init(void)
3939{
3940 int rc = proto_register(&packet_proto, 0);
3941
3942 if (rc != 0)
3943 goto out;
3944
3945 sock_register(&packet_family_ops);
d12d01d6 3946 register_pernet_subsys(&packet_net_ops);
1da177e4 3947 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3948out:
3949 return rc;
3950}
3951
3952module_init(packet_init);
3953module_exit(packet_exit);
3954MODULE_LICENSE("GPL");
3955MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 1.042545 seconds and 5 git commands to generate.