projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
af_packet: when sending ethernet frames, parse header for skb->protocol
[deliverable/linux.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
0f75b09c 91#include <linux/if_arp.h>
1da177e4
LT		 92
93#ifdef CONFIG_INET
94#include <net/inet_common.h>
95#endif
96
2787b04b
PE		 97#include "internal.h"
98
1da177e4
LT		 99/*
100 Assumptions:
101 - if device has no dev->hard_header routine, it adds and removes ll header
102 inside itself. In this case ll header is invisible outside of device,
103 but higher levels still should reserve dev->hard_header_len.
104 Some devices are enough clever to reallocate skb, when header
105 will not fit to reserved space (tunnel), another ones are silly
106 (PPP).
107 - packet socket receives packets with pulled ll header,
108 so that SOCK_RAW should push it back.
109
110On receive:
111-----------
112
113Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 114 mac_header -> ll header
115 data -> data
1da177e4
LT		 116
117Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 118 mac_header -> ll header
119 data -> ll header
1da177e4
LT		 120
121Incoming, dev->hard_header==NULL
b0e380b1
ACM		 122 mac_header -> UNKNOWN position. It is very likely, that it points to ll
123 header. PPP makes it, that is wrong, because introduce
db0c58f9 124 assymetry between rx and tx paths.
b0e380b1 125 data -> data
1da177e4
LT		 126
127Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 128 mac_header -> data. ll header is still not built!
129 data -> data
1da177e4
LT		 130
131Resume
132 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
133
134
135On transmit:
136------------
137
138dev->hard_header != NULL
b0e380b1
ACM		 139 mac_header -> ll header
140 data -> ll header
1da177e4
LT		 141
142dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 143 mac_header -> data
144 data -> data
1da177e4
LT		 145
146 We should set nh.raw on output to correct posistion,
147 packet classifier depends on it.
148 */
149
1da177e4
LT		 150/* Private packet socket structures. */
151
0fb375fb
EB		 152/* identical to struct packet_mreq except it has
153 * a longer address field.
154 */
40d4e3df 155struct packet_mreq_max {
0fb375fb
EB		 156 int mr_ifindex;
157 unsigned short mr_type;
158 unsigned short mr_alen;
159 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 160};
a2efcfa0 161
184f489e
DB		 162union tpacket_uhdr {
163 struct tpacket_hdr *h1;
164 struct tpacket2_hdr *h2;
165 struct tpacket3_hdr *h3;
166 void *raw;
167};
168
f6fb8f10 169static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 170 int closing, int tx_ring);
171
f6fb8f10 172#define V3_ALIGNMENT (8)
173
bc59ba39 174#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 175
176#define BLK_PLUS_PRIV(sz_of_priv) \
177 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
178
f6fb8f10 179#define PGV_FROM_VMALLOC 1
69e3c75f 180
f6fb8f10 181#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
182#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
183#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
184#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
185#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
186#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
187#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
188
69e3c75f
JB		 189struct packet_sock;
190static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 191static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
192 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 193
f6fb8f10 194static void *packet_previous_frame(struct packet_sock *po,
195 struct packet_ring_buffer *rb,
196 int status);
197static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 198static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
199 struct tpacket_block_desc *);
200static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 201 struct packet_sock *);
bc59ba39 202static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 203 struct packet_sock *, unsigned int status);
bc59ba39 204static int prb_queue_frozen(struct tpacket_kbdq_core *);
205static void prb_open_block(struct tpacket_kbdq_core *,
206 struct tpacket_block_desc *);
f6fb8f10 207static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 208static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
209static void prb_init_blk_timer(struct packet_sock *,
210 struct tpacket_kbdq_core *,
211 void (*func) (unsigned long));
212static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
213static void prb_clear_rxhash(struct tpacket_kbdq_core *,
214 struct tpacket3_hdr *);
215static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
216 struct tpacket3_hdr *);
1da177e4
LT		 217static void packet_flush_mclist(struct sock *sk);
218
ffbc6111
HX		 219struct packet_skb_cb {
220 unsigned int origlen;
221 union {
222 struct sockaddr_pkt pkt;
223 struct sockaddr_ll ll;
224 } sa;
225};
226
227#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 228
bc59ba39 229#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 230#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 231 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 232#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 233 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 234#define GET_NEXT_PRB_BLK_NUM(x) \
235 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
236 ((x)->kactive_blk_num+1) : 0)
237
dc99f600
DM		 238static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
239static void __fanout_link(struct sock *sk, struct packet_sock *po);
240
ce06b03e
DM		 241/* register_prot_hook must be invoked with the po->bind_lock held,
242 * or from a context in which asynchronous accesses to the packet
243 * socket is not possible (packet_create()).
244 */
245static void register_prot_hook(struct sock *sk)
246{
247 struct packet_sock *po = pkt_sk(sk);
248 if (!po->running) {
dc99f600
DM		 249 if (po->fanout)
250 __fanout_link(sk, po);
251 else
252 dev_add_pack(&po->prot_hook);
ce06b03e
DM		 253 sock_hold(sk);
254 po->running = 1;
255 }
256}
257
258/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
259 * held. If the sync parameter is true, we will temporarily drop
260 * the po->bind_lock and do a synchronize_net to make sure no
261 * asynchronous packet processing paths still refer to the elements
262 * of po->prot_hook. If the sync parameter is false, it is the
263 * callers responsibility to take care of this.
264 */
265static void __unregister_prot_hook(struct sock *sk, bool sync)
266{
267 struct packet_sock *po = pkt_sk(sk);
268
269 po->running = 0;
dc99f600
DM		 270 if (po->fanout)
271 __fanout_unlink(sk, po);
272 else
273 __dev_remove_pack(&po->prot_hook);
ce06b03e
DM		 274 __sock_put(sk);
275
276 if (sync) {
277 spin_unlock(&po->bind_lock);
278 synchronize_net();
279 spin_lock(&po->bind_lock);
280 }
281}
282
283static void unregister_prot_hook(struct sock *sk, bool sync)
284{
285 struct packet_sock *po = pkt_sk(sk);
286
287 if (po->running)
288 __unregister_prot_hook(sk, sync);
289}
290
f6dafa95 291static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 292{
293 if (is_vmalloc_addr(addr))
294 return vmalloc_to_page(addr);
295 return virt_to_page(addr);
296}
297
69e3c75f 298static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 299{
184f489e 300 union tpacket_uhdr h;
1da177e4 301
69e3c75f 302 h.raw = frame;
bbd6ef87
PM		 303 switch (po->tp_version) {
304 case TPACKET_V1:
69e3c75f 305 h.h1->tp_status = status;
0af55bb5 306 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 307 break;
308 case TPACKET_V2:
69e3c75f 309 h.h2->tp_status = status;
0af55bb5 310 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 311 break;
f6fb8f10 312 case TPACKET_V3:
69e3c75f 313 default:
f6fb8f10 314 WARN(1, "TPACKET version not supported.\n");
69e3c75f 315 BUG();
bbd6ef87 316 }
69e3c75f
JB		 317
318 smp_wmb();
bbd6ef87
PM		 319}
320
69e3c75f 321static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 322{
184f489e 323 union tpacket_uhdr h;
bbd6ef87 324
69e3c75f
JB		 325 smp_rmb();
326
bbd6ef87
PM		 327 h.raw = frame;
328 switch (po->tp_version) {
329 case TPACKET_V1:
0af55bb5 330 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 331 return h.h1->tp_status;
bbd6ef87 332 case TPACKET_V2:
0af55bb5 333 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 334 return h.h2->tp_status;
f6fb8f10 335 case TPACKET_V3:
69e3c75f 336 default:
f6fb8f10 337 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 338 BUG();
339 return 0;
bbd6ef87 340 }
1da177e4 341}
69e3c75f 342
b9c32fb2
DB		 343static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
344 unsigned int flags)
7a51384c
DB		 345{
346 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
347
348 if (shhwtstamps) {
349 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
350 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
b9c32fb2 351 return TP_STATUS_TS_SYS_HARDWARE;
7a51384c
DB		 352 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
353 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
b9c32fb2 354 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 355 }
356
357 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 358 return TP_STATUS_TS_SOFTWARE;
7a51384c 359
b9c32fb2 360 return 0;
7a51384c
DB		 361}
362
b9c32fb2
DB		 363static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
364 struct sk_buff *skb)
2e31396f
WB		 365{
366 union tpacket_uhdr h;
367 struct timespec ts;
b9c32fb2 368 __u32 ts_status;
2e31396f 369
b9c32fb2
DB		 370 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
371 return 0;
2e31396f
WB		 372
373 h.raw = frame;
374 switch (po->tp_version) {
375 case TPACKET_V1:
376 h.h1->tp_sec = ts.tv_sec;
377 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
378 break;
379 case TPACKET_V2:
380 h.h2->tp_sec = ts.tv_sec;
381 h.h2->tp_nsec = ts.tv_nsec;
382 break;
383 case TPACKET_V3:
384 default:
385 WARN(1, "TPACKET version not supported.\n");
386 BUG();
387 }
388
389 /* one flush is safe, as both fields always lie on the same cacheline */
390 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
391 smp_wmb();
b9c32fb2
DB		 392
393 return ts_status;
2e31396f
WB		 394}
395
69e3c75f
JB		 396static void *packet_lookup_frame(struct packet_sock *po,
397 struct packet_ring_buffer *rb,
398 unsigned int position,
399 int status)
400{
401 unsigned int pg_vec_pos, frame_offset;
184f489e 402 union tpacket_uhdr h;
69e3c75f
JB		 403
404 pg_vec_pos = position / rb->frames_per_block;
405 frame_offset = position % rb->frames_per_block;
406
0e3125c7
NH		 407 h.raw = rb->pg_vec[pg_vec_pos].buffer +
408 (frame_offset * rb->frame_size);
69e3c75f
JB		 409
410 if (status != __packet_get_status(po, h.raw))
411 return NULL;
412
413 return h.raw;
414}
415
eea49cc9 416static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 417 struct packet_ring_buffer *rb,
418 int status)
419{
420 return packet_lookup_frame(po, rb, rb->head, status);
421}
422
bc59ba39 423static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 424{
425 del_timer_sync(&pkc->retire_blk_timer);
426}
427
428static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
429 int tx_ring,
430 struct sk_buff_head *rb_queue)
431{
bc59ba39 432 struct tpacket_kbdq_core *pkc;
f6fb8f10 433
434 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
435
436 spin_lock(&rb_queue->lock);
437 pkc->delete_blk_timer = 1;
438 spin_unlock(&rb_queue->lock);
439
440 prb_del_retire_blk_timer(pkc);
441}
442
443static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 444 struct tpacket_kbdq_core *pkc,
f6fb8f10 445 void (*func) (unsigned long))
446{
447 init_timer(&pkc->retire_blk_timer);
448 pkc->retire_blk_timer.data = (long)po;
449 pkc->retire_blk_timer.function = func;
450 pkc->retire_blk_timer.expires = jiffies;
451}
452
453static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
454{
bc59ba39 455 struct tpacket_kbdq_core *pkc;
f6fb8f10 456
457 if (tx_ring)
458 BUG();
459
460 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc;
461 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
462}
463
464static int prb_calc_retire_blk_tmo(struct packet_sock *po,
465 int blk_size_in_bytes)
466{
467 struct net_device *dev;
468 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 469 struct ethtool_cmd ecmd;
470 int err;
e440cf2c 471 u32 speed;
f6fb8f10 472
4bc71cb9
JP		 473 rtnl_lock();
474 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
475 if (unlikely(!dev)) {
476 rtnl_unlock();
f6fb8f10 477 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 478 }
479 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 480 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 481 rtnl_unlock();
482 if (!err) {
4bc71cb9
JP		 483 /*
484 * If the link speed is so slow you don't really
485 * need to worry about perf anyways
486 */
e440cf2c 487 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 488 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 489 } else {
490 msec = 1;
491 div = speed / 1000;
f6fb8f10 492 }
493 }
494
495 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
496
497 if (div)
498 mbits /= div;
499
500 tmo = mbits * msec;
501
502 if (div)
503 return tmo+1;
504 return tmo;
505}
506
bc59ba39 507static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 508 union tpacket_req_u *req_u)
509{
510 p1->feature_req_word = req_u->req3.tp_feature_req_word;
511}
512
513static void init_prb_bdqc(struct packet_sock *po,
514 struct packet_ring_buffer *rb,
515 struct pgv *pg_vec,
516 union tpacket_req_u *req_u, int tx_ring)
517{
bc59ba39 518 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc;
519 struct tpacket_block_desc *pbd;
f6fb8f10 520
521 memset(p1, 0x0, sizeof(*p1));
522
523 p1->knxt_seq_num = 1;
524 p1->pkbdq = pg_vec;
bc59ba39 525 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 526 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 527 p1->kblk_size = req_u->req3.tp_block_size;
528 p1->knum_blocks = req_u->req3.tp_block_nr;
529 p1->hdrlen = po->tp_hdrlen;
530 p1->version = po->tp_version;
531 p1->last_kactive_blk_num = 0;
ee80fbf3 532 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 533 if (req_u->req3.tp_retire_blk_tov)
534 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
535 else
536 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
537 req_u->req3.tp_block_size);
538 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
539 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
540
541 prb_init_ft_ops(p1, req_u);
542 prb_setup_retire_blk_timer(po, tx_ring);
543 prb_open_block(p1, pbd);
544}
545
546/* Do NOT update the last_blk_num first.
547 * Assumes sk_buff_head lock is held.
548 */
bc59ba39 549static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 550{
551 mod_timer(&pkc->retire_blk_timer,
552 jiffies + pkc->tov_in_jiffies);
553 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
554}
555
556/*
557 * Timer logic:
558 * 1) We refresh the timer only when we open a block.
559 * By doing this we don't waste cycles refreshing the timer
560 * on packet-by-packet basis.
561 *
562 * With a 1MB block-size, on a 1Gbps line, it will take
563 * i) ~8 ms to fill a block + ii) memcpy etc.
564 * In this cut we are not accounting for the memcpy time.
565 *
566 * So, if the user sets the 'tmo' to 10ms then the timer
567 * will never fire while the block is still getting filled
568 * (which is what we want). However, the user could choose
569 * to close a block early and that's fine.
570 *
571 * But when the timer does fire, we check whether or not to refresh it.
572 * Since the tmo granularity is in msecs, it is not too expensive
573 * to refresh the timer, lets say every '8' msecs.
574 * Either the user can set the 'tmo' or we can derive it based on
575 * a) line-speed and b) block-size.
576 * prb_calc_retire_blk_tmo() calculates the tmo.
577 *
578 */
579static void prb_retire_rx_blk_timer_expired(unsigned long data)
580{
581 struct packet_sock *po = (struct packet_sock *)data;
bc59ba39 582 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc;
f6fb8f10 583 unsigned int frozen;
bc59ba39 584 struct tpacket_block_desc *pbd;
f6fb8f10 585
586 spin_lock(&po->sk.sk_receive_queue.lock);
587
588 frozen = prb_queue_frozen(pkc);
589 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
590
591 if (unlikely(pkc->delete_blk_timer))
592 goto out;
593
594 /* We only need to plug the race when the block is partially filled.
595 * tpacket_rcv:
596 * lock(); increment BLOCK_NUM_PKTS; unlock()
597 * copy_bits() is in progress ...
598 * timer fires on other cpu:
599 * we can't retire the current block because copy_bits
600 * is in progress.
601 *
602 */
603 if (BLOCK_NUM_PKTS(pbd)) {
604 while (atomic_read(&pkc->blk_fill_in_prog)) {
605 /* Waiting for skb_copy_bits to finish... */
606 cpu_relax();
607 }
608 }
609
610 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
611 if (!frozen) {
612 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
613 if (!prb_dispatch_next_block(pkc, po))
614 goto refresh_timer;
615 else
616 goto out;
617 } else {
618 /* Case 1. Queue was frozen because user-space was
619 * lagging behind.
620 */
621 if (prb_curr_blk_in_use(pkc, pbd)) {
622 /*
623 * Ok, user-space is still behind.
624 * So just refresh the timer.
625 */
626 goto refresh_timer;
627 } else {
628 /* Case 2. queue was frozen,user-space caught up,
629 * now the link went idle && the timer fired.
630 * We don't have a block to close.So we open this
631 * block and restart the timer.
632 * opening a block thaws the queue,restarts timer
633 * Thawing/timer-refresh is a side effect.
634 */
635 prb_open_block(pkc, pbd);
636 goto out;
637 }
638 }
639 }
640
641refresh_timer:
642 _prb_refresh_rx_retire_blk_timer(pkc);
643
644out:
645 spin_unlock(&po->sk.sk_receive_queue.lock);
646}
647
eea49cc9 648static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 649 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 650{
651 /* Flush everything minus the block header */
652
653#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
654 u8 *start, *end;
655
656 start = (u8 *)pbd1;
657
658 /* Skip the block header(we know header WILL fit in 4K) */
659 start += PAGE_SIZE;
660
661 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
662 for (; start < end; start += PAGE_SIZE)
663 flush_dcache_page(pgv_to_page(start));
664
665 smp_wmb();
666#endif
667
668 /* Now update the block status. */
669
670 BLOCK_STATUS(pbd1) = status;
671
672 /* Flush the block header */
673
674#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
675 start = (u8 *)pbd1;
676 flush_dcache_page(pgv_to_page(start));
677
678 smp_wmb();
679#endif
680}
681
682/*
683 * Side effect:
684 *
685 * 1) flush the block
686 * 2) Increment active_blk_num
687 *
688 * Note:We DONT refresh the timer on purpose.
689 * Because almost always the next block will be opened.
690 */
bc59ba39 691static void prb_close_block(struct tpacket_kbdq_core *pkc1,
692 struct tpacket_block_desc *pbd1,
f6fb8f10 693 struct packet_sock *po, unsigned int stat)
694{
695 __u32 status = TP_STATUS_USER | stat;
696
697 struct tpacket3_hdr *last_pkt;
bc59ba39 698 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 699
ee80fbf3 700 if (po->stats.stats3.tp_drops)
f6fb8f10 701 status |= TP_STATUS_LOSING;
702
703 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
704 last_pkt->tp_next_offset = 0;
705
706 /* Get the ts of the last pkt */
707 if (BLOCK_NUM_PKTS(pbd1)) {
708 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
709 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
710 } else {
711 /* Ok, we tmo'd - so get the current time */
712 struct timespec ts;
713 getnstimeofday(&ts);
714 h1->ts_last_pkt.ts_sec = ts.tv_sec;
715 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
716 }
717
718 smp_wmb();
719
720 /* Flush the block */
721 prb_flush_block(pkc1, pbd1, status);
722
723 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
724}
725
eea49cc9 726static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 727{
728 pkc->reset_pending_on_curr_blk = 0;
729}
730
731/*
732 * Side effect of opening a block:
733 *
734 * 1) prb_queue is thawed.
735 * 2) retire_blk_timer is refreshed.
736 *
737 */
bc59ba39 738static void prb_open_block(struct tpacket_kbdq_core *pkc1,
739 struct tpacket_block_desc *pbd1)
f6fb8f10 740{
741 struct timespec ts;
bc59ba39 742 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 743
744 smp_rmb();
745
8da3056c
DB		 746 /* We could have just memset this but we will lose the
747 * flexibility of making the priv area sticky
748 */
f6fb8f10 749
8da3056c
DB		 750 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
751 BLOCK_NUM_PKTS(pbd1) = 0;
752 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 753
8da3056c
DB		 754 getnstimeofday(&ts);
755
756 h1->ts_first_pkt.ts_sec = ts.tv_sec;
757 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 758
8da3056c
DB		 759 pkc1->pkblk_start = (char *)pbd1;
760 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
761
762 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
763 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
764
765 pbd1->version = pkc1->version;
766 pkc1->prev = pkc1->nxt_offset;
767 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
768
769 prb_thaw_queue(pkc1);
770 _prb_refresh_rx_retire_blk_timer(pkc1);
771
772 smp_wmb();
f6fb8f10 773}
774
775/*
776 * Queue freeze logic:
777 * 1) Assume tp_block_nr = 8 blocks.
778 * 2) At time 't0', user opens Rx ring.
779 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
780 * 4) user-space is either sleeping or processing block '0'.
781 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
782 * it will close block-7,loop around and try to fill block '0'.
783 * call-flow:
784 * __packet_lookup_frame_in_block
785 * prb_retire_current_block()
786 * prb_dispatch_next_block()
787 * |->(BLOCK_STATUS == USER) evaluates to true
788 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
789 * 6) Now there are two cases:
790 * 6.1) Link goes idle right after the queue is frozen.
791 * But remember, the last open_block() refreshed the timer.
792 * When this timer expires,it will refresh itself so that we can
793 * re-open block-0 in near future.
794 * 6.2) Link is busy and keeps on receiving packets. This is a simple
795 * case and __packet_lookup_frame_in_block will check if block-0
796 * is free and can now be re-used.
797 */
eea49cc9 798static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 799 struct packet_sock *po)
800{
801 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 802 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 803}
804
805#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
806
807/*
808 * If the next block is free then we will dispatch it
809 * and return a good offset.
810 * Else, we will freeze the queue.
811 * So, caller must check the return value.
812 */
bc59ba39 813static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 814 struct packet_sock *po)
815{
bc59ba39 816 struct tpacket_block_desc *pbd;
f6fb8f10 817
818 smp_rmb();
819
820 /* 1. Get current block num */
821 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
822
823 /* 2. If this block is currently in_use then freeze the queue */
824 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
825 prb_freeze_queue(pkc, po);
826 return NULL;
827 }
828
829 /*
830 * 3.
831 * open this block and return the offset where the first packet
832 * needs to get stored.
833 */
834 prb_open_block(pkc, pbd);
835 return (void *)pkc->nxt_offset;
836}
837
bc59ba39 838static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 839 struct packet_sock *po, unsigned int status)
840{
bc59ba39 841 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 842
843 /* retire/close the current block */
844 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
845 /*
846 * Plug the case where copy_bits() is in progress on
847 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
848 * have space to copy the pkt in the current block and
849 * called prb_retire_current_block()
850 *
851 * We don't need to worry about the TMO case because
852 * the timer-handler already handled this case.
853 */
854 if (!(status & TP_STATUS_BLK_TMO)) {
855 while (atomic_read(&pkc->blk_fill_in_prog)) {
856 /* Waiting for skb_copy_bits to finish... */
857 cpu_relax();
858 }
859 }
860 prb_close_block(pkc, pbd, po, status);
861 return;
862 }
f6fb8f10 863}
864
eea49cc9 865static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 866 struct tpacket_block_desc *pbd)
f6fb8f10 867{
868 return TP_STATUS_USER & BLOCK_STATUS(pbd);
869}
870
eea49cc9 871static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 872{
873 return pkc->reset_pending_on_curr_blk;
874}
875
eea49cc9 876static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 877{
bc59ba39 878 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 879 atomic_dec(&pkc->blk_fill_in_prog);
880}
881
eea49cc9 882static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 883 struct tpacket3_hdr *ppd)
884{
885 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb);
886}
887
eea49cc9 888static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 889 struct tpacket3_hdr *ppd)
890{
891 ppd->hv1.tp_rxhash = 0;
892}
893
eea49cc9 894static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 895 struct tpacket3_hdr *ppd)
896{
897 if (vlan_tx_tag_present(pkc->skb)) {
898 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
899 ppd->tp_status = TP_STATUS_VLAN_VALID;
900 } else {
9e67030a 901 ppd->hv1.tp_vlan_tci = 0;
902 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 903 }
904}
905
bc59ba39 906static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 907 struct tpacket3_hdr *ppd)
908{
909 prb_fill_vlan_info(pkc, ppd);
910
911 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
912 prb_fill_rxhash(pkc, ppd);
913 else
914 prb_clear_rxhash(pkc, ppd);
915}
916
eea49cc9 917static void prb_fill_curr_block(char *curr,
bc59ba39 918 struct tpacket_kbdq_core *pkc,
919 struct tpacket_block_desc *pbd,
f6fb8f10 920 unsigned int len)
921{
922 struct tpacket3_hdr *ppd;
923
924 ppd = (struct tpacket3_hdr *)curr;
925 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
926 pkc->prev = curr;
927 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
928 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
929 BLOCK_NUM_PKTS(pbd) += 1;
930 atomic_inc(&pkc->blk_fill_in_prog);
931 prb_run_all_ft_ops(pkc, ppd);
932}
933
934/* Assumes caller has the sk->rx_queue.lock */
935static void *__packet_lookup_frame_in_block(struct packet_sock *po,
936 struct sk_buff *skb,
937 int status,
938 unsigned int len
939 )
940{
bc59ba39 941 struct tpacket_kbdq_core *pkc;
942 struct tpacket_block_desc *pbd;
f6fb8f10 943 char *curr, *end;
944
e3192690 945 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 946 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
947
948 /* Queue is frozen when user space is lagging behind */
949 if (prb_queue_frozen(pkc)) {
950 /*
951 * Check if that last block which caused the queue to freeze,
952 * is still in_use by user-space.
953 */
954 if (prb_curr_blk_in_use(pkc, pbd)) {
955 /* Can't record this packet */
956 return NULL;
957 } else {
958 /*
959 * Ok, the block was released by user-space.
960 * Now let's open that block.
961 * opening a block also thaws the queue.
962 * Thawing is a side effect.
963 */
964 prb_open_block(pkc, pbd);
965 }
966 }
967
968 smp_mb();
969 curr = pkc->nxt_offset;
970 pkc->skb = skb;
e3192690 971 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 972
973 /* first try the current block */
974 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
975 prb_fill_curr_block(curr, pkc, pbd, len);
976 return (void *)curr;
977 }
978
979 /* Ok, close the current block */
980 prb_retire_current_block(pkc, po, 0);
981
982 /* Now, try to dispatch the next block */
983 curr = (char *)prb_dispatch_next_block(pkc, po);
984 if (curr) {
985 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
986 prb_fill_curr_block(curr, pkc, pbd, len);
987 return (void *)curr;
988 }
989
990 /*
991 * No free blocks are available.user_space hasn't caught up yet.
992 * Queue was just frozen and now this packet will get dropped.
993 */
994 return NULL;
995}
996
eea49cc9 997static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 998 struct sk_buff *skb,
999 int status, unsigned int len)
1000{
1001 char *curr = NULL;
1002 switch (po->tp_version) {
1003 case TPACKET_V1:
1004 case TPACKET_V2:
1005 curr = packet_lookup_frame(po, &po->rx_ring,
1006 po->rx_ring.head, status);
1007 return curr;
1008 case TPACKET_V3:
1009 return __packet_lookup_frame_in_block(po, skb, status, len);
1010 default:
1011 WARN(1, "TPACKET version not supported\n");
1012 BUG();
99aa3473 1013 return NULL;
f6fb8f10 1014 }
1015}
1016
eea49cc9 1017static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1018 struct packet_ring_buffer *rb,
77f65ebd 1019 unsigned int idx,
f6fb8f10 1020 int status)
1021{
bc59ba39 1022 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1023 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1024
1025 if (status != BLOCK_STATUS(pbd))
1026 return NULL;
1027 return pbd;
1028}
1029
eea49cc9 1030static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1031{
1032 unsigned int prev;
1033 if (rb->prb_bdqc.kactive_blk_num)
1034 prev = rb->prb_bdqc.kactive_blk_num-1;
1035 else
1036 prev = rb->prb_bdqc.knum_blocks-1;
1037 return prev;
1038}
1039
1040/* Assumes caller has held the rx_queue.lock */
eea49cc9 1041static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1042 struct packet_ring_buffer *rb,
1043 int status)
1044{
1045 unsigned int previous = prb_previous_blk_num(rb);
1046 return prb_lookup_block(po, rb, previous, status);
1047}
1048
eea49cc9 1049static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1050 struct packet_ring_buffer *rb,
1051 int status)
1052{
1053 if (po->tp_version <= TPACKET_V2)
1054 return packet_previous_frame(po, rb, status);
1055
1056 return __prb_previous_block(po, rb, status);
1057}
1058
eea49cc9 1059static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1060 struct packet_ring_buffer *rb)
1061{
1062 switch (po->tp_version) {
1063 case TPACKET_V1:
1064 case TPACKET_V2:
1065 return packet_increment_head(rb);
1066 case TPACKET_V3:
1067 default:
1068 WARN(1, "TPACKET version not supported.\n");
1069 BUG();
1070 return;
1071 }
1072}
1073
eea49cc9 1074static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1075 struct packet_ring_buffer *rb,
1076 int status)
1077{
1078 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1079 return packet_lookup_frame(po, rb, previous, status);
1080}
1081
eea49cc9 1082static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1083{
1084 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1085}
1086
77f65ebd
WB		 1087static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1088{
1089 struct sock *sk = &po->sk;
1090 bool has_room;
1091
1092 if (po->prot_hook.func != tpacket_rcv)
1093 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1094 <= sk->sk_rcvbuf;
1095
1096 spin_lock(&sk->sk_receive_queue.lock);
1097 if (po->tp_version == TPACKET_V3)
1098 has_room = prb_lookup_block(po, &po->rx_ring,
1099 po->rx_ring.prb_bdqc.kactive_blk_num,
1100 TP_STATUS_KERNEL);
1101 else
1102 has_room = packet_lookup_frame(po, &po->rx_ring,
1103 po->rx_ring.head,
1104 TP_STATUS_KERNEL);
1105 spin_unlock(&sk->sk_receive_queue.lock);
1106
1107 return has_room;
1108}
1109
1da177e4
LT		 1110static void packet_sock_destruct(struct sock *sk)
1111{
ed85b565
RC		 1112 skb_queue_purge(&sk->sk_error_queue);
1113
547b792c
IJ		 1114 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1115 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1116
1117 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1118 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1119 return;
1120 }
1121
17ab56a2 1122 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1123}
1124
dc99f600
DM		 1125static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1126{
1127 int x = atomic_read(&f->rr_cur) + 1;
1128
1129 if (x >= num)
1130 x = 0;
1131
1132 return x;
1133}
1134
77f65ebd
WB		 1135static unsigned int fanout_demux_hash(struct packet_fanout *f,
1136 struct sk_buff *skb,
1137 unsigned int num)
dc99f600 1138{
77f65ebd 1139 return (((u64)skb->rxhash) * num) >> 32;
dc99f600
DM		 1140}
1141
77f65ebd
WB		 1142static unsigned int fanout_demux_lb(struct packet_fanout *f,
1143 struct sk_buff *skb,
1144 unsigned int num)
dc99f600
DM		 1145{
1146 int cur, old;
1147
1148 cur = atomic_read(&f->rr_cur);
1149 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1150 fanout_rr_next(f, num))) != cur)
1151 cur = old;
77f65ebd
WB		 1152 return cur;
1153}
1154
1155static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1156 struct sk_buff *skb,
1157 unsigned int num)
1158{
1159 return smp_processor_id() % num;
dc99f600
DM		 1160}
1161
77f65ebd
WB		 1162static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1163 struct sk_buff *skb,
1164 unsigned int idx, unsigned int skip,
1165 unsigned int num)
95ec3eb4 1166{
77f65ebd 1167 unsigned int i, j;
95ec3eb4 1168
77f65ebd
WB		 1169 i = j = min_t(int, f->next[idx], num - 1);
1170 do {
1171 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1172 if (i != j)
1173 f->next[idx] = i;
1174 return i;
1175 }
1176 if (++i == num)
1177 i = 0;
1178 } while (i != j);
1179
1180 return idx;
1181}
1182
1183static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1184{
1185 return f->flags & (flag >> 8);
95ec3eb4
DM		 1186}
1187
95ec3eb4
DM		 1188static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1189 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1190{
1191 struct packet_fanout *f = pt->af_packet_priv;
1192 unsigned int num = f->num_members;
1193 struct packet_sock *po;
77f65ebd 1194 unsigned int idx;
dc99f600
DM		 1195
1196 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1197 !num) {
1198 kfree_skb(skb);
1199 return 0;
1200 }
1201
95ec3eb4
DM		 1202 switch (f->type) {
1203 case PACKET_FANOUT_HASH:
1204 default:
77f65ebd 1205 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1206 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1207 if (!skb)
1208 return 0;
1209 }
1210 skb_get_rxhash(skb);
77f65ebd 1211 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1212 break;
1213 case PACKET_FANOUT_LB:
77f65ebd 1214 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1215 break;
1216 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1217 idx = fanout_demux_cpu(f, skb, num);
1218 break;
1219 case PACKET_FANOUT_ROLLOVER:
1220 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1221 break;
dc99f600
DM		 1222 }
1223
77f65ebd
WB		 1224 po = pkt_sk(f->arr[idx]);
1225 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1226 unlikely(!packet_rcv_has_room(po, skb))) {
1227 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1228 po = pkt_sk(f->arr[idx]);
1229 }
dc99f600
DM		 1230
1231 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1232}
1233
fff3321d
PE		 1234DEFINE_MUTEX(fanout_mutex);
1235EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1236static LIST_HEAD(fanout_list);
1237
1238static void __fanout_link(struct sock *sk, struct packet_sock *po)
1239{
1240 struct packet_fanout *f = po->fanout;
1241
1242 spin_lock(&f->lock);
1243 f->arr[f->num_members] = sk;
1244 smp_wmb();
1245 f->num_members++;
1246 spin_unlock(&f->lock);
1247}
1248
1249static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1250{
1251 struct packet_fanout *f = po->fanout;
1252 int i;
1253
1254 spin_lock(&f->lock);
1255 for (i = 0; i < f->num_members; i++) {
1256 if (f->arr[i] == sk)
1257 break;
1258 }
1259 BUG_ON(i >= f->num_members);
1260 f->arr[i] = f->arr[f->num_members - 1];
1261 f->num_members--;
1262 spin_unlock(&f->lock);
1263}
1264
a0dfb263 1265static bool match_fanout_group(struct packet_type *ptype, struct sock * sk)
c0de08d0
EL		 1266{
1267 if (ptype->af_packet_priv == (void*)((struct packet_sock *)sk)->fanout)
1268 return true;
1269
1270 return false;
1271}
1272
7736d33f 1273static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1274{
1275 struct packet_sock *po = pkt_sk(sk);
1276 struct packet_fanout *f, *match;
7736d33f 1277 u8 type = type_flags & 0xff;
77f65ebd 1278 u8 flags = type_flags >> 8;
dc99f600
DM		 1279 int err;
1280
1281 switch (type) {
77f65ebd
WB		 1282 case PACKET_FANOUT_ROLLOVER:
1283 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1284 return -EINVAL;
dc99f600
DM		 1285 case PACKET_FANOUT_HASH:
1286 case PACKET_FANOUT_LB:
95ec3eb4 1287 case PACKET_FANOUT_CPU:
dc99f600
DM		 1288 break;
1289 default:
1290 return -EINVAL;
1291 }
1292
1293 if (!po->running)
1294 return -EINVAL;
1295
1296 if (po->fanout)
1297 return -EALREADY;
1298
1299 mutex_lock(&fanout_mutex);
1300 match = NULL;
1301 list_for_each_entry(f, &fanout_list, list) {
1302 if (f->id == id &&
1303 read_pnet(&f->net) == sock_net(sk)) {
1304 match = f;
1305 break;
1306 }
1307 }
afe62c68 1308 err = -EINVAL;
77f65ebd 1309 if (match && match->flags != flags)
afe62c68 1310 goto out;
dc99f600 1311 if (!match) {
afe62c68 1312 err = -ENOMEM;
dc99f600 1313 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1314 if (!match)
1315 goto out;
1316 write_pnet(&match->net, sock_net(sk));
1317 match->id = id;
1318 match->type = type;
77f65ebd 1319 match->flags = flags;
afe62c68
ED		 1320 atomic_set(&match->rr_cur, 0);
1321 INIT_LIST_HEAD(&match->list);
1322 spin_lock_init(&match->lock);
1323 atomic_set(&match->sk_ref, 0);
1324 match->prot_hook.type = po->prot_hook.type;
1325 match->prot_hook.dev = po->prot_hook.dev;
1326 match->prot_hook.func = packet_rcv_fanout;
1327 match->prot_hook.af_packet_priv = match;
c0de08d0 1328 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1329 dev_add_pack(&match->prot_hook);
1330 list_add(&match->list, &fanout_list);
dc99f600 1331 }
afe62c68
ED		 1332 err = -EINVAL;
1333 if (match->type == type &&
1334 match->prot_hook.type == po->prot_hook.type &&
1335 match->prot_hook.dev == po->prot_hook.dev) {
1336 err = -ENOSPC;
1337 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1338 __dev_remove_pack(&po->prot_hook);
1339 po->fanout = match;
1340 atomic_inc(&match->sk_ref);
1341 __fanout_link(sk, po);
1342 err = 0;
dc99f600
DM		 1343 }
1344 }
afe62c68 1345out:
dc99f600
DM		 1346 mutex_unlock(&fanout_mutex);
1347 return err;
1348}
1349
1350static void fanout_release(struct sock *sk)
1351{
1352 struct packet_sock *po = pkt_sk(sk);
1353 struct packet_fanout *f;
1354
1355 f = po->fanout;
1356 if (!f)
1357 return;
1358
fff3321d 1359 mutex_lock(&fanout_mutex);
dc99f600
DM		 1360 po->fanout = NULL;
1361
dc99f600
DM		 1362 if (atomic_dec_and_test(&f->sk_ref)) {
1363 list_del(&f->list);
1364 dev_remove_pack(&f->prot_hook);
1365 kfree(f);
1366 }
1367 mutex_unlock(&fanout_mutex);
1368}
1da177e4 1369
90ddc4f0 1370static const struct proto_ops packet_ops;
1da177e4 1371
90ddc4f0 1372static const struct proto_ops packet_ops_spkt;
1da177e4 1373
40d4e3df
ED		 1374static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1375 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1376{
1377 struct sock *sk;
1378 struct sockaddr_pkt *spkt;
1379
1380 /*
1381 * When we registered the protocol we saved the socket in the data
1382 * field for just this event.
1383 */
1384
1385 sk = pt->af_packet_priv;
1ce4f28b 1386
1da177e4
LT		 1387 /*
1388 * Yank back the headers [hope the device set this
1389 * right or kerboom...]
1390 *
1391 * Incoming packets have ll header pulled,
1392 * push it back.
1393 *
98e399f8 1394 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1395 * so that this procedure is noop.
1396 */
1397
1398 if (skb->pkt_type == PACKET_LOOPBACK)
1399 goto out;
1400
09ad9bc7 1401 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1402 goto out;
1403
40d4e3df
ED		 1404 skb = skb_share_check(skb, GFP_ATOMIC);
1405 if (skb == NULL)
1da177e4
LT		 1406 goto oom;
1407
1408 /* drop any routing info */
adf30907 1409 skb_dst_drop(skb);
1da177e4 1410
84531c24
PO		 1411 /* drop conntrack reference */
1412 nf_reset(skb);
1413
ffbc6111 1414 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1415
98e399f8 1416 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1417
1418 /*
1419 * The SOCK_PACKET socket receives _all_ frames.
1420 */
1421
1422 spkt->spkt_family = dev->type;
1423 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1424 spkt->spkt_protocol = skb->protocol;
1425
1426 /*
1427 * Charge the memory to the socket. This is done specifically
1428 * to prevent sockets using all the memory up.
1429 */
1430
40d4e3df 1431 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1432 return 0;
1433
1434out:
1435 kfree_skb(skb);
1436oom:
1437 return 0;
1438}
1439
1440
1441/*
1442 * Output a raw packet to a device layer. This bypasses all the other
1443 * protocol layers and you must therefore supply it with a complete frame
1444 */
1ce4f28b 1445
1da177e4
LT		 1446static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1447 struct msghdr *msg, size_t len)
1448{
1449 struct sock *sk = sock->sk;
40d4e3df 1450 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1a35ca80 1451 struct sk_buff *skb = NULL;
1da177e4 1452 struct net_device *dev;
40d4e3df 1453 __be16 proto = 0;
1da177e4 1454 int err;
3bdc0eba 1455 int extra_len = 0;
1ce4f28b 1456
1da177e4 1457 /*
1ce4f28b 1458 * Get and verify the address.
1da177e4
LT		 1459 */
1460
40d4e3df 1461 if (saddr) {
1da177e4 1462 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1463 return -EINVAL;
1464 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1465 proto = saddr->spkt_protocol;
1466 } else
1467 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1468
1469 /*
1ce4f28b 1470 * Find the device first to size check it
1da177e4
LT		 1471 */
1472
de74e92a 1473 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1474retry:
654d1f8a
ED		 1475 rcu_read_lock();
1476 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1477 err = -ENODEV;
1478 if (dev == NULL)
1479 goto out_unlock;
1ce4f28b 1480
d5e76b0a
DM		 1481 err = -ENETDOWN;
1482 if (!(dev->flags & IFF_UP))
1483 goto out_unlock;
1484
1da177e4 1485 /*
40d4e3df
ED		 1486 * You may not queue a frame bigger than the mtu. This is the lowest level
1487 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1488 */
1ce4f28b 1489
3bdc0eba
BG		 1490 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1491 if (!netif_supports_nofcs(dev)) {
1492 err = -EPROTONOSUPPORT;
1493 goto out_unlock;
1494 }
1495 extra_len = 4; /* We're doing our own CRC */
1496 }
1497
1da177e4 1498 err = -EMSGSIZE;
3bdc0eba 1499 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1500 goto out_unlock;
1501
1a35ca80
ED		 1502 if (!skb) {
1503 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1504 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1505 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1506
1507 rcu_read_unlock();
4ce40912 1508 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1509 if (skb == NULL)
1510 return -ENOBUFS;
1511 /* FIXME: Save some space for broken drivers that write a hard
1512 * header at transmission time by themselves. PPP is the notable
1513 * one here. This should really be fixed at the driver level.
1514 */
1515 skb_reserve(skb, reserved);
1516 skb_reset_network_header(skb);
1517
1518 /* Try to align data part correctly */
1519 if (hhlen) {
1520 skb->data -= hhlen;
1521 skb->tail -= hhlen;
1522 if (len < hhlen)
1523 skb_reset_network_header(skb);
1524 }
1525 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1526 if (err)
1527 goto out_free;
1528 goto retry;
1da177e4
LT		 1529 }
1530
3bdc0eba 1531 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1532 /* Earlier code assumed this would be a VLAN pkt,
1533 * double-check this now that we have the actual
1534 * packet in hand.
1535 */
1536 struct ethhdr *ehdr;
1537 skb_reset_mac_header(skb);
1538 ehdr = eth_hdr(skb);
1539 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1540 err = -EMSGSIZE;
1541 goto out_unlock;
1542 }
1543 }
1a35ca80 1544
1da177e4
LT		 1545 skb->protocol = proto;
1546 skb->dev = dev;
1547 skb->priority = sk->sk_priority;
2d37a186 1548 skb->mark = sk->sk_mark;
bf84a010
DB		 1549
1550 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1551
3bdc0eba
BG		 1552 if (unlikely(extra_len == 4))
1553 skb->no_fcs = 1;
1554
40893fd0 1555 skb_probe_transport_header(skb, 0);
c1aad275 1556
1da177e4 1557 dev_queue_xmit(skb);
654d1f8a 1558 rcu_read_unlock();
40d4e3df 1559 return len;
1da177e4 1560
1da177e4 1561out_unlock:
654d1f8a 1562 rcu_read_unlock();
1a35ca80
ED		 1563out_free:
1564 kfree_skb(skb);
1da177e4
LT		 1565 return err;
1566}
1da177e4 1567
eea49cc9 1568static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1569 const struct sock *sk,
dbcb5855 1570 unsigned int res)
1da177e4
LT		 1571{
1572 struct sk_filter *filter;
fda9ef5d 1573
80f8f102
ED		 1574 rcu_read_lock();
1575 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1576 if (filter != NULL)
0a14842f 1577 res = SK_RUN_FILTER(filter, skb);
80f8f102 1578 rcu_read_unlock();
1da177e4 1579
dbcb5855 1580 return res;
1da177e4
LT		 1581}
1582
1583/*
62ab0812
ED		 1584 * This function makes lazy skb cloning in hope that most of packets
1585 * are discarded by BPF.
1586 *
1587 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1588 * and skb->cb are mangled. It works because (and until) packets
1589 * falling here are owned by current CPU. Output packets are cloned
1590 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1591 * sequencially, so that if we return skb to original state on exit,
1592 * we will not harm anyone.
1da177e4
LT		 1593 */
1594
40d4e3df
ED		 1595static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1596 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1597{
1598 struct sock *sk;
1599 struct sockaddr_ll *sll;
1600 struct packet_sock *po;
40d4e3df 1601 u8 *skb_head = skb->data;
1da177e4 1602 int skb_len = skb->len;
dbcb5855 1603 unsigned int snaplen, res;
1da177e4
LT		 1604
1605 if (skb->pkt_type == PACKET_LOOPBACK)
1606 goto drop;
1607
1608 sk = pt->af_packet_priv;
1609 po = pkt_sk(sk);
1610
09ad9bc7 1611 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1612 goto drop;
1613
1da177e4
LT		 1614 skb->dev = dev;
1615
3b04ddde 1616 if (dev->header_ops) {
1da177e4 1617 /* The device has an explicit notion of ll header,
62ab0812
ED		 1618 * exported to higher levels.
1619 *
1620 * Otherwise, the device hides details of its frame
1621 * structure, so that corresponding packet head is
1622 * never delivered to user.
1da177e4
LT		 1623 */
1624 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1625 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1626 else if (skb->pkt_type == PACKET_OUTGOING) {
1627 /* Special case: outgoing packets have ll header at head */
bbe735e4 1628 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1629 }
1630 }
1631
1632 snaplen = skb->len;
1633
dbcb5855
DM		 1634 res = run_filter(skb, sk, snaplen);
1635 if (!res)
fda9ef5d 1636 goto drop_n_restore;
dbcb5855
DM		 1637 if (snaplen > res)
1638 snaplen = res;
1da177e4 1639
0fd7bac6 1640 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1641 goto drop_n_acct;
1642
1643 if (skb_shared(skb)) {
1644 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1645 if (nskb == NULL)
1646 goto drop_n_acct;
1647
1648 if (skb_head != skb->data) {
1649 skb->data = skb_head;
1650 skb->len = skb_len;
1651 }
abc4e4fa 1652 consume_skb(skb);
1da177e4
LT		 1653 skb = nskb;
1654 }
1655
ffbc6111
HX		 1656 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1657 sizeof(skb->cb));
1658
1659 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1660 sll->sll_family = AF_PACKET;
1661 sll->sll_hatype = dev->type;
1662 sll->sll_protocol = skb->protocol;
1663 sll->sll_pkttype = skb->pkt_type;
8032b464 1664 if (unlikely(po->origdev))
80feaacb
PWJ		 1665 sll->sll_ifindex = orig_dev->ifindex;
1666 else
1667 sll->sll_ifindex = dev->ifindex;
1da177e4 1668
b95cce35 1669 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1670
ffbc6111 1671 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1672
1da177e4
LT		 1673 if (pskb_trim(skb, snaplen))
1674 goto drop_n_acct;
1675
1676 skb_set_owner_r(skb, sk);
1677 skb->dev = NULL;
adf30907 1678 skb_dst_drop(skb);
1da177e4 1679
84531c24
PO		 1680 /* drop conntrack reference */
1681 nf_reset(skb);
1682
1da177e4 1683 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1684 po->stats.stats1.tp_packets++;
3b885787 1685 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1686 __skb_queue_tail(&sk->sk_receive_queue, skb);
1687 spin_unlock(&sk->sk_receive_queue.lock);
1688 sk->sk_data_ready(sk, skb->len);
1689 return 0;
1690
1691drop_n_acct:
7091fbd8 1692 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1693 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1694 atomic_inc(&sk->sk_drops);
1695 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1696
1697drop_n_restore:
1698 if (skb_head != skb->data && skb_shared(skb)) {
1699 skb->data = skb_head;
1700 skb->len = skb_len;
1701 }
1702drop:
ead2ceb0 1703 consume_skb(skb);
1da177e4
LT		 1704 return 0;
1705}
1706
40d4e3df
ED		 1707static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1708 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1709{
1710 struct sock *sk;
1711 struct packet_sock *po;
1712 struct sockaddr_ll *sll;
184f489e 1713 union tpacket_uhdr h;
40d4e3df 1714 u8 *skb_head = skb->data;
1da177e4 1715 int skb_len = skb->len;
dbcb5855 1716 unsigned int snaplen, res;
f6fb8f10 1717 unsigned long status = TP_STATUS_USER;
bbd6ef87 1718 unsigned short macoff, netoff, hdrlen;
1da177e4 1719 struct sk_buff *copy_skb = NULL;
bbd6ef87 1720 struct timespec ts;
b9c32fb2 1721 __u32 ts_status;
1da177e4
LT		 1722
1723 if (skb->pkt_type == PACKET_LOOPBACK)
1724 goto drop;
1725
1726 sk = pt->af_packet_priv;
1727 po = pkt_sk(sk);
1728
09ad9bc7 1729 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1730 goto drop;
1731
3b04ddde 1732 if (dev->header_ops) {
1da177e4 1733 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1734 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1735 else if (skb->pkt_type == PACKET_OUTGOING) {
1736 /* Special case: outgoing packets have ll header at head */
bbe735e4 1737 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1738 }
1739 }
1740
8dc41944
HX		 1741 if (skb->ip_summed == CHECKSUM_PARTIAL)
1742 status |= TP_STATUS_CSUMNOTREADY;
1743
1da177e4
LT		 1744 snaplen = skb->len;
1745
dbcb5855
DM		 1746 res = run_filter(skb, sk, snaplen);
1747 if (!res)
fda9ef5d 1748 goto drop_n_restore;
dbcb5855
DM		 1749 if (snaplen > res)
1750 snaplen = res;
1da177e4
LT		 1751
1752 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1753 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1754 po->tp_reserve;
1da177e4 1755 } else {
95c96174 1756 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1757 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1758 (maclen < 16 ? 16 : maclen)) +
1759 po->tp_reserve;
1da177e4
LT		 1760 macoff = netoff - maclen;
1761 }
f6fb8f10 1762 if (po->tp_version <= TPACKET_V2) {
1763 if (macoff + snaplen > po->rx_ring.frame_size) {
1764 if (po->copy_thresh &&
0fd7bac6 1765 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1766 if (skb_shared(skb)) {
1767 copy_skb = skb_clone(skb, GFP_ATOMIC);
1768 } else {
1769 copy_skb = skb_get(skb);
1770 skb_head = skb->data;
1771 }
1772 if (copy_skb)
1773 skb_set_owner_r(copy_skb, sk);
1da177e4 1774 }
f6fb8f10 1775 snaplen = po->rx_ring.frame_size - macoff;
1776 if ((int)snaplen < 0)
1777 snaplen = 0;
1da177e4 1778 }
1da177e4 1779 }
1da177e4 1780 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1781 h.raw = packet_current_rx_frame(po, skb,
1782 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1783 if (!h.raw)
1da177e4 1784 goto ring_is_full;
f6fb8f10 1785 if (po->tp_version <= TPACKET_V2) {
1786 packet_increment_rx_head(po, &po->rx_ring);
1787 /*
1788 * LOSING will be reported till you read the stats,
1789 * because it's COR - Clear On Read.
1790 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1791 * at packet level.
1792 */
ee80fbf3 1793 if (po->stats.stats1.tp_drops)
f6fb8f10 1794 status |= TP_STATUS_LOSING;
1795 }
ee80fbf3 1796 po->stats.stats1.tp_packets++;
1da177e4
LT		 1797 if (copy_skb) {
1798 status |= TP_STATUS_COPY;
1799 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1800 }
1da177e4
LT		 1801 spin_unlock(&sk->sk_receive_queue.lock);
1802
bbd6ef87 1803 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1804
1805 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1806 getnstimeofday(&ts);
1da177e4 1807
b9c32fb2
DB		 1808 status |= ts_status;
1809
bbd6ef87
PM		 1810 switch (po->tp_version) {
1811 case TPACKET_V1:
1812 h.h1->tp_len = skb->len;
1813 h.h1->tp_snaplen = snaplen;
1814 h.h1->tp_mac = macoff;
1815 h.h1->tp_net = netoff;
4b457bdf
DB		 1816 h.h1->tp_sec = ts.tv_sec;
1817 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1818 hdrlen = sizeof(*h.h1);
1819 break;
1820 case TPACKET_V2:
1821 h.h2->tp_len = skb->len;
1822 h.h2->tp_snaplen = snaplen;
1823 h.h2->tp_mac = macoff;
1824 h.h2->tp_net = netoff;
bbd6ef87
PM		 1825 h.h2->tp_sec = ts.tv_sec;
1826 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1827 if (vlan_tx_tag_present(skb)) {
1828 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1829 status |= TP_STATUS_VLAN_VALID;
1830 } else {
1831 h.h2->tp_vlan_tci = 0;
1832 }
13fcb7bd 1833 h.h2->tp_padding = 0;
bbd6ef87
PM		 1834 hdrlen = sizeof(*h.h2);
1835 break;
f6fb8f10 1836 case TPACKET_V3:
1837 /* tp_nxt_offset,vlan are already populated above.
1838 * So DONT clear those fields here
1839 */
1840 h.h3->tp_status |= status;
1841 h.h3->tp_len = skb->len;
1842 h.h3->tp_snaplen = snaplen;
1843 h.h3->tp_mac = macoff;
1844 h.h3->tp_net = netoff;
f6fb8f10 1845 h.h3->tp_sec = ts.tv_sec;
1846 h.h3->tp_nsec = ts.tv_nsec;
1847 hdrlen = sizeof(*h.h3);
1848 break;
bbd6ef87
PM		 1849 default:
1850 BUG();
1851 }
1da177e4 1852
bbd6ef87 1853 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 1854 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 1855 sll->sll_family = AF_PACKET;
1856 sll->sll_hatype = dev->type;
1857 sll->sll_protocol = skb->protocol;
1858 sll->sll_pkttype = skb->pkt_type;
8032b464 1859 if (unlikely(po->origdev))
80feaacb
PWJ		 1860 sll->sll_ifindex = orig_dev->ifindex;
1861 else
1862 sll->sll_ifindex = dev->ifindex;
1da177e4 1863
e16aa207 1864 smp_mb();
f6dafa95 1865#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1da177e4 1866 {
0af55bb5
CG		 1867 u8 *start, *end;
1868
f6fb8f10 1869 if (po->tp_version <= TPACKET_V2) {
1870 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1871 + macoff + snaplen);
1872 for (start = h.raw; start < end; start += PAGE_SIZE)
1873 flush_dcache_page(pgv_to_page(start));
1874 }
cc9f01b2 1875 smp_wmb();
1da177e4 1876 }
f6dafa95 1877#endif
f6fb8f10 1878 if (po->tp_version <= TPACKET_V2)
1879 __packet_set_status(po, h.raw, status);
1880 else
1881 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 1882
1883 sk->sk_data_ready(sk, 0);
1884
1885drop_n_restore:
1886 if (skb_head != skb->data && skb_shared(skb)) {
1887 skb->data = skb_head;
1888 skb->len = skb_len;
1889 }
1890drop:
1ce4f28b 1891 kfree_skb(skb);
1da177e4
LT		 1892 return 0;
1893
1894ring_is_full:
ee80fbf3 1895 po->stats.stats1.tp_drops++;
1da177e4
LT		 1896 spin_unlock(&sk->sk_receive_queue.lock);
1897
1898 sk->sk_data_ready(sk, 0);
acb5d75b 1899 kfree_skb(copy_skb);
1da177e4
LT		 1900 goto drop_n_restore;
1901}
1902
69e3c75f
JB		 1903static void tpacket_destruct_skb(struct sk_buff *skb)
1904{
1905 struct packet_sock *po = pkt_sk(skb->sk);
40d4e3df 1906 void *ph;
1da177e4 1907
69e3c75f 1908 if (likely(po->tx_ring.pg_vec)) {
b9c32fb2
DB		 1909 __u32 ts;
1910
69e3c75f 1911 ph = skb_shinfo(skb)->destructor_arg;
69e3c75f
JB		 1912 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
1913 atomic_dec(&po->tx_ring.pending);
b9c32fb2
DB		 1914
1915 ts = __packet_set_timestamp(po, ph, skb);
1916 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 1917 }
1918
1919 sock_wfree(skb);
1920}
1921
40d4e3df
ED		 1922static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
1923 void *frame, struct net_device *dev, int size_max,
ae641949 1924 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 1925{
184f489e 1926 union tpacket_uhdr ph;
69e3c75f
JB		 1927 int to_write, offset, len, tp_len, nr_frags, len_max;
1928 struct socket *sock = po->sk.sk_socket;
1929 struct page *page;
1930 void *data;
1931 int err;
1932
1933 ph.raw = frame;
1934
1935 skb->protocol = proto;
1936 skb->dev = dev;
1937 skb->priority = po->sk.sk_priority;
2d37a186 1938 skb->mark = po->sk.sk_mark;
2e31396f 1939 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 1940 skb_shinfo(skb)->destructor_arg = ph.raw;
1941
1942 switch (po->tp_version) {
1943 case TPACKET_V2:
1944 tp_len = ph.h2->tp_len;
1945 break;
1946 default:
1947 tp_len = ph.h1->tp_len;
1948 break;
1949 }
1950 if (unlikely(tp_len > size_max)) {
40d4e3df 1951 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
69e3c75f
JB		 1952 return -EMSGSIZE;
1953 }
1954
ae641949 1955 skb_reserve(skb, hlen);
69e3c75f 1956 skb_reset_network_header(skb);
40893fd0 1957 skb_probe_transport_header(skb, 0);
c1aad275 1958
5920cd3a
PC		 1959 if (po->tp_tx_has_off) {
1960 int off_min, off_max, off;
1961 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
1962 off_max = po->tx_ring.frame_size - tp_len;
1963 if (sock->type == SOCK_DGRAM) {
1964 switch (po->tp_version) {
1965 case TPACKET_V2:
1966 off = ph.h2->tp_net;
1967 break;
1968 default:
1969 off = ph.h1->tp_net;
1970 break;
1971 }
1972 } else {
1973 switch (po->tp_version) {
1974 case TPACKET_V2:
1975 off = ph.h2->tp_mac;
1976 break;
1977 default:
1978 off = ph.h1->tp_mac;
1979 break;
1980 }
1981 }
1982 if (unlikely((off < off_min) || (off_max < off)))
1983 return -EINVAL;
1984 data = ph.raw + off;
1985 } else {
1986 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
1987 }
69e3c75f
JB		 1988 to_write = tp_len;
1989
1990 if (sock->type == SOCK_DGRAM) {
1991 err = dev_hard_header(skb, dev, ntohs(proto), addr,
1992 NULL, tp_len);
1993 if (unlikely(err < 0))
1994 return -EINVAL;
40d4e3df 1995 } else if (dev->hard_header_len) {
69e3c75f
JB		 1996 /* net device doesn't like empty head */
1997 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 1998 pr_err("packet size is too short (%d < %d)\n",
1999 tp_len, dev->hard_header_len);
69e3c75f
JB		 2000 return -EINVAL;
2001 }
2002
2003 skb_push(skb, dev->hard_header_len);
2004 err = skb_store_bits(skb, 0, data,
2005 dev->hard_header_len);
2006 if (unlikely(err))
2007 return err;
2008
0f75b09c
PS		 2009 if (dev->type == ARPHRD_ETHER)
2010 skb->protocol = eth_type_trans(skb, dev);
2011
69e3c75f
JB		 2012 data += dev->hard_header_len;
2013 to_write -= dev->hard_header_len;
2014 }
2015
69e3c75f
JB		 2016 offset = offset_in_page(data);
2017 len_max = PAGE_SIZE - offset;
2018 len = ((to_write > len_max) ? len_max : to_write);
2019
2020 skb->data_len = to_write;
2021 skb->len += to_write;
2022 skb->truesize += to_write;
2023 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2024
2025 while (likely(to_write)) {
2026 nr_frags = skb_shinfo(skb)->nr_frags;
2027
2028 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2029 pr_err("Packet exceed the number of skb frags(%lu)\n",
2030 MAX_SKB_FRAGS);
69e3c75f
JB		 2031 return -EFAULT;
2032 }
2033
0af55bb5
CG		 2034 page = pgv_to_page(data);
2035 data += len;
69e3c75f
JB		 2036 flush_dcache_page(page);
2037 get_page(page);
0af55bb5 2038 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2039 to_write -= len;
2040 offset = 0;
2041 len_max = PAGE_SIZE;
2042 len = ((to_write > len_max) ? len_max : to_write);
2043 }
2044
2045 return tp_len;
2046}
2047
2048static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2049{
69e3c75f
JB		 2050 struct sk_buff *skb;
2051 struct net_device *dev;
2052 __be16 proto;
827d9780
BG		 2053 bool need_rls_dev = false;
2054 int err, reserve = 0;
40d4e3df
ED		 2055 void *ph;
2056 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
69e3c75f
JB		 2057 int tp_len, size_max;
2058 unsigned char *addr;
2059 int len_sum = 0;
9e67030a 2060 int status = TP_STATUS_AVAILABLE;
ae641949 2061 int hlen, tlen;
69e3c75f 2062
69e3c75f
JB		 2063 mutex_lock(&po->pg_vec_lock);
2064
69e3c75f 2065 if (saddr == NULL) {
827d9780 2066 dev = po->prot_hook.dev;
69e3c75f
JB		 2067 proto = po->num;
2068 addr = NULL;
2069 } else {
2070 err = -EINVAL;
2071 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2072 goto out;
2073 if (msg->msg_namelen < (saddr->sll_halen
2074 + offsetof(struct sockaddr_ll,
2075 sll_addr)))
2076 goto out;
69e3c75f
JB		 2077 proto = saddr->sll_protocol;
2078 addr = saddr->sll_addr;
827d9780
BG		 2079 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2080 need_rls_dev = true;
69e3c75f
JB		 2081 }
2082
69e3c75f
JB		 2083 err = -ENXIO;
2084 if (unlikely(dev == NULL))
2085 goto out;
2086
2087 reserve = dev->hard_header_len;
2088
2089 err = -ENETDOWN;
2090 if (unlikely(!(dev->flags & IFF_UP)))
2091 goto out_put;
2092
2093 size_max = po->tx_ring.frame_size
b5dd884e 2094 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f
JB		 2095
2096 if (size_max > dev->mtu + reserve)
2097 size_max = dev->mtu + reserve;
2098
2099 do {
2100 ph = packet_current_frame(po, &po->tx_ring,
2101 TP_STATUS_SEND_REQUEST);
2102
2103 if (unlikely(ph == NULL)) {
2104 schedule();
2105 continue;
2106 }
2107
2108 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2109 hlen = LL_RESERVED_SPACE(dev);
2110 tlen = dev->needed_tailroom;
69e3c75f 2111 skb = sock_alloc_send_skb(&po->sk,
ae641949 2112 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2113 0, &err);
2114
2115 if (unlikely(skb == NULL))
2116 goto out_status;
2117
2118 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2119 addr, hlen);
69e3c75f
JB		 2120
2121 if (unlikely(tp_len < 0)) {
2122 if (po->tp_loss) {
2123 __packet_set_status(po, ph,
2124 TP_STATUS_AVAILABLE);
2125 packet_increment_head(&po->tx_ring);
2126 kfree_skb(skb);
2127 continue;
2128 } else {
2129 status = TP_STATUS_WRONG_FORMAT;
2130 err = tp_len;
2131 goto out_status;
2132 }
2133 }
2134
2135 skb->destructor = tpacket_destruct_skb;
2136 __packet_set_status(po, ph, TP_STATUS_SENDING);
2137 atomic_inc(&po->tx_ring.pending);
2138
2139 status = TP_STATUS_SEND_REQUEST;
2140 err = dev_queue_xmit(skb);
eb70df13
JP		 2141 if (unlikely(err > 0)) {
2142 err = net_xmit_errno(err);
2143 if (err && __packet_get_status(po, ph) ==
2144 TP_STATUS_AVAILABLE) {
2145 /* skb was destructed already */
2146 skb = NULL;
2147 goto out_status;
2148 }
2149 /*
2150 * skb was dropped but not destructed yet;
2151 * let's treat it like congestion or err < 0
2152 */
2153 err = 0;
2154 }
69e3c75f
JB		 2155 packet_increment_head(&po->tx_ring);
2156 len_sum += tp_len;
f64f9e71
JP		 2157 } while (likely((ph != NULL) ||
2158 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
2159 (atomic_read(&po->tx_ring.pending))))
2160 );
69e3c75f
JB		 2161
2162 err = len_sum;
2163 goto out_put;
2164
69e3c75f
JB		 2165out_status:
2166 __packet_set_status(po, ph, status);
2167 kfree_skb(skb);
2168out_put:
827d9780
BG		 2169 if (need_rls_dev)
2170 dev_put(dev);
69e3c75f
JB		 2171out:
2172 mutex_unlock(&po->pg_vec_lock);
2173 return err;
2174}
69e3c75f 2175
eea49cc9
OJ		 2176static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2177 size_t reserve, size_t len,
2178 size_t linear, int noblock,
2179 int *err)
bfd5f4a3
SS		 2180{
2181 struct sk_buff *skb;
2182
2183 /* Under a page? Don't bother with paged skb. */
2184 if (prepad + len < PAGE_SIZE || !linear)
2185 linear = len;
2186
2187 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2188 err);
2189 if (!skb)
2190 return NULL;
2191
2192 skb_reserve(skb, reserve);
2193 skb_put(skb, linear);
2194 skb->data_len = len - linear;
2195 skb->len += len - linear;
2196
2197 return skb;
2198}
2199
69e3c75f 2200static int packet_snd(struct socket *sock,
1da177e4
LT		 2201 struct msghdr *msg, size_t len)
2202{
2203 struct sock *sk = sock->sk;
40d4e3df 2204 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1da177e4
LT		 2205 struct sk_buff *skb;
2206 struct net_device *dev;
0e11c91e 2207 __be16 proto;
827d9780 2208 bool need_rls_dev = false;
1da177e4 2209 unsigned char *addr;
827d9780 2210 int err, reserve = 0;
bfd5f4a3
SS		 2211 struct virtio_net_hdr vnet_hdr = { 0 };
2212 int offset = 0;
2213 int vnet_hdr_len;
2214 struct packet_sock *po = pkt_sk(sk);
2215 unsigned short gso_type = 0;
ae641949 2216 int hlen, tlen;
3bdc0eba 2217 int extra_len = 0;
1da177e4
LT		 2218
2219 /*
1ce4f28b 2220 * Get and verify the address.
1da177e4 2221 */
1ce4f28b 2222
1da177e4 2223 if (saddr == NULL) {
827d9780 2224 dev = po->prot_hook.dev;
1da177e4
LT		 2225 proto = po->num;
2226 addr = NULL;
2227 } else {
2228 err = -EINVAL;
2229 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2230 goto out;
0fb375fb
EB		 2231 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2232 goto out;
1da177e4
LT		 2233 proto = saddr->sll_protocol;
2234 addr = saddr->sll_addr;
827d9780
BG		 2235 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2236 need_rls_dev = true;
1da177e4
LT		 2237 }
2238
1da177e4
LT		 2239 err = -ENXIO;
2240 if (dev == NULL)
2241 goto out_unlock;
2242 if (sock->type == SOCK_RAW)
2243 reserve = dev->hard_header_len;
2244
d5e76b0a
DM		 2245 err = -ENETDOWN;
2246 if (!(dev->flags & IFF_UP))
2247 goto out_unlock;
2248
bfd5f4a3
SS		 2249 if (po->has_vnet_hdr) {
2250 vnet_hdr_len = sizeof(vnet_hdr);
2251
2252 err = -EINVAL;
2253 if (len < vnet_hdr_len)
2254 goto out_unlock;
2255
2256 len -= vnet_hdr_len;
2257
2258 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2259 vnet_hdr_len);
2260 if (err < 0)
2261 goto out_unlock;
2262
2263 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2264 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2265 vnet_hdr.hdr_len))
2266 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2267 vnet_hdr.csum_offset + 2;
2268
2269 err = -EINVAL;
2270 if (vnet_hdr.hdr_len > len)
2271 goto out_unlock;
2272
2273 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2274 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2275 case VIRTIO_NET_HDR_GSO_TCPV4:
2276 gso_type = SKB_GSO_TCPV4;
2277 break;
2278 case VIRTIO_NET_HDR_GSO_TCPV6:
2279 gso_type = SKB_GSO_TCPV6;
2280 break;
2281 case VIRTIO_NET_HDR_GSO_UDP:
2282 gso_type = SKB_GSO_UDP;
2283 break;
2284 default:
2285 goto out_unlock;
2286 }
2287
2288 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2289 gso_type |= SKB_GSO_TCP_ECN;
2290
2291 if (vnet_hdr.gso_size == 0)
2292 goto out_unlock;
2293
2294 }
2295 }
2296
3bdc0eba
BG		 2297 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2298 if (!netif_supports_nofcs(dev)) {
2299 err = -EPROTONOSUPPORT;
2300 goto out_unlock;
2301 }
2302 extra_len = 4; /* We're doing our own CRC */
2303 }
2304
1da177e4 2305 err = -EMSGSIZE;
3bdc0eba 2306 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2307 goto out_unlock;
2308
bfd5f4a3 2309 err = -ENOBUFS;
ae641949
HX		 2310 hlen = LL_RESERVED_SPACE(dev);
2311 tlen = dev->needed_tailroom;
2312 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2313 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2314 if (skb == NULL)
1da177e4
LT		 2315 goto out_unlock;
2316
bfd5f4a3 2317 skb_set_network_header(skb, reserve);
1da177e4 2318
0c4e8581
SH		 2319 err = -EINVAL;
2320 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2321 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2322 goto out_free;
1da177e4
LT		 2323
2324 /* Returns -EFAULT on error */
bfd5f4a3 2325 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2326 if (err)
2327 goto out_free;
bf84a010
DB		 2328
2329 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2330
0f75b09c
PS		 2331 if (dev->type == ARPHRD_ETHER) {
2332 skb->protocol = eth_type_trans(skb, dev);
2333 } else {
2334 skb->protocol = proto;
2335 skb->dev = dev;
2336 }
2337
3bdc0eba 2338 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
57f89bfa
BG		 2339 /* Earlier code assumed this would be a VLAN pkt,
2340 * double-check this now that we have the actual
2341 * packet in hand.
2342 */
2343 struct ethhdr *ehdr;
2344 skb_reset_mac_header(skb);
2345 ehdr = eth_hdr(skb);
2346 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2347 err = -EMSGSIZE;
2348 goto out_free;
2349 }
2350 }
2351
1da177e4 2352 skb->priority = sk->sk_priority;
2d37a186 2353 skb->mark = sk->sk_mark;
1da177e4 2354
bfd5f4a3
SS		 2355 if (po->has_vnet_hdr) {
2356 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2357 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2358 vnet_hdr.csum_offset)) {
2359 err = -EINVAL;
2360 goto out_free;
2361 }
2362 }
2363
2364 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2365 skb_shinfo(skb)->gso_type = gso_type;
2366
2367 /* Header must be checked, and gso_segs computed. */
2368 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2369 skb_shinfo(skb)->gso_segs = 0;
2370
2371 len += vnet_hdr_len;
2372 }
2373
40893fd0 2374 skb_probe_transport_header(skb, reserve);
c1aad275 2375
3bdc0eba
BG		 2376 if (unlikely(extra_len == 4))
2377 skb->no_fcs = 1;
2378
1da177e4
LT		 2379 /*
2380 * Now send it
2381 */
2382
2383 err = dev_queue_xmit(skb);
2384 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2385 goto out_unlock;
2386
827d9780
BG		 2387 if (need_rls_dev)
2388 dev_put(dev);
1da177e4 2389
40d4e3df 2390 return len;
1da177e4
LT		 2391
2392out_free:
2393 kfree_skb(skb);
2394out_unlock:
827d9780 2395 if (dev && need_rls_dev)
1da177e4
LT		 2396 dev_put(dev);
2397out:
2398 return err;
2399}
2400
69e3c75f
JB		 2401static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2402 struct msghdr *msg, size_t len)
2403{
69e3c75f
JB		 2404 struct sock *sk = sock->sk;
2405 struct packet_sock *po = pkt_sk(sk);
2406 if (po->tx_ring.pg_vec)
2407 return tpacket_snd(po, msg);
2408 else
69e3c75f
JB		 2409 return packet_snd(sock, msg, len);
2410}
2411
1da177e4
LT		 2412/*
2413 * Close a PACKET socket. This is fairly simple. We immediately go
2414 * to 'closed' state and remove our protocol entry in the device list.
2415 */
2416
2417static int packet_release(struct socket *sock)
2418{
2419 struct sock *sk = sock->sk;
2420 struct packet_sock *po;
d12d01d6 2421 struct net *net;
f6fb8f10 2422 union tpacket_req_u req_u;
1da177e4
LT		 2423
2424 if (!sk)
2425 return 0;
2426
3b1e0a65 2427 net = sock_net(sk);
1da177e4
LT		 2428 po = pkt_sk(sk);
2429
0fa7fa98 2430 mutex_lock(&net->packet.sklist_lock);
808f5114 2431 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2432 mutex_unlock(&net->packet.sklist_lock);
2433
2434 preempt_disable();
920de804 2435 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2436 preempt_enable();
1da177e4 2437
808f5114 2438 spin_lock(&po->bind_lock);
ce06b03e 2439 unregister_prot_hook(sk, false);
160ff18a
BG		 2440 if (po->prot_hook.dev) {
2441 dev_put(po->prot_hook.dev);
2442 po->prot_hook.dev = NULL;
2443 }
808f5114 2444 spin_unlock(&po->bind_lock);
1da177e4 2445
1da177e4 2446 packet_flush_mclist(sk);
1da177e4 2447
9665d5d6
PS		 2448 if (po->rx_ring.pg_vec) {
2449 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2450 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2451 }
69e3c75f 2452
9665d5d6
PS		 2453 if (po->tx_ring.pg_vec) {
2454 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2455 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2456 }
1da177e4 2457
dc99f600
DM		 2458 fanout_release(sk);
2459
808f5114 2460 synchronize_net();
1da177e4
LT		 2461 /*
2462 * Now the socket is dead. No more input will appear.
2463 */
1da177e4
LT		 2464 sock_orphan(sk);
2465 sock->sk = NULL;
2466
2467 /* Purge queues */
2468
2469 skb_queue_purge(&sk->sk_receive_queue);
17ab56a2 2470 sk_refcnt_debug_release(sk);
1da177e4
LT		 2471
2472 sock_put(sk);
2473 return 0;
2474}
2475
2476/*
2477 * Attach a packet hook.
2478 */
2479
0e11c91e 2480static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1da177e4
LT		 2481{
2482 struct packet_sock *po = pkt_sk(sk);
dc99f600 2483
aef950b4
WY		 2484 if (po->fanout) {
2485 if (dev)
2486 dev_put(dev);
2487
dc99f600 2488 return -EINVAL;
aef950b4 2489 }
1da177e4
LT		 2490
2491 lock_sock(sk);
2492
2493 spin_lock(&po->bind_lock);
ce06b03e 2494 unregister_prot_hook(sk, true);
1da177e4
LT		 2495 po->num = protocol;
2496 po->prot_hook.type = protocol;
160ff18a
BG		 2497 if (po->prot_hook.dev)
2498 dev_put(po->prot_hook.dev);
1da177e4
LT		 2499 po->prot_hook.dev = dev;
2500
2501 po->ifindex = dev ? dev->ifindex : 0;
2502
2503 if (protocol == 0)
2504 goto out_unlock;
2505
be85d4ad 2506 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2507 register_prot_hook(sk);
be85d4ad
UT		 2508 } else {
2509 sk->sk_err = ENETDOWN;
2510 if (!sock_flag(sk, SOCK_DEAD))
2511 sk->sk_error_report(sk);
1da177e4
LT		 2512 }
2513
2514out_unlock:
2515 spin_unlock(&po->bind_lock);
2516 release_sock(sk);
2517 return 0;
2518}
2519
2520/*
2521 * Bind a packet socket to a device
2522 */
2523
40d4e3df
ED		 2524static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2525 int addr_len)
1da177e4 2526{
40d4e3df 2527 struct sock *sk = sock->sk;
1da177e4
LT		 2528 char name[15];
2529 struct net_device *dev;
2530 int err = -ENODEV;
1ce4f28b 2531
1da177e4
LT		 2532 /*
2533 * Check legality
2534 */
1ce4f28b 2535
8ae55f04 2536 if (addr_len != sizeof(struct sockaddr))
1da177e4 2537 return -EINVAL;
40d4e3df 2538 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2539
3b1e0a65 2540 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2541 if (dev)
1da177e4 2542 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2543 return err;
2544}
1da177e4
LT		 2545
2546static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2547{
40d4e3df
ED		 2548 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2549 struct sock *sk = sock->sk;
1da177e4
LT		 2550 struct net_device *dev = NULL;
2551 int err;
2552
2553
2554 /*
2555 * Check legality
2556 */
1ce4f28b 2557
1da177e4
LT		 2558 if (addr_len < sizeof(struct sockaddr_ll))
2559 return -EINVAL;
2560 if (sll->sll_family != AF_PACKET)
2561 return -EINVAL;
2562
2563 if (sll->sll_ifindex) {
2564 err = -ENODEV;
3b1e0a65 2565 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2566 if (dev == NULL)
2567 goto out;
2568 }
2569 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2570
2571out:
2572 return err;
2573}
2574
2575static struct proto packet_proto = {
2576 .name = "PACKET",
2577 .owner = THIS_MODULE,
2578 .obj_size = sizeof(struct packet_sock),
2579};
2580
2581/*
1ce4f28b 2582 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2583 */
2584
3f378b68
EP		 2585static int packet_create(struct net *net, struct socket *sock, int protocol,
2586 int kern)
1da177e4
LT		 2587{
2588 struct sock *sk;
2589 struct packet_sock *po;
0e11c91e 2590 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2591 int err;
2592
df008c91 2593 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2594 return -EPERM;
be02097c
DM		 2595 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2596 sock->type != SOCK_PACKET)
1da177e4
LT		 2597 return -ESOCKTNOSUPPORT;
2598
2599 sock->state = SS_UNCONNECTED;
2600
2601 err = -ENOBUFS;
6257ff21 2602 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2603 if (sk == NULL)
2604 goto out;
2605
2606 sock->ops = &packet_ops;
1da177e4
LT		 2607 if (sock->type == SOCK_PACKET)
2608 sock->ops = &packet_ops_spkt;
be02097c 2609
1da177e4
LT		 2610 sock_init_data(sock, sk);
2611
2612 po = pkt_sk(sk);
2613 sk->sk_family = PF_PACKET;
0e11c91e 2614 po->num = proto;
1da177e4
LT		 2615
2616 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2617 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2618
2619 /*
2620 * Attach a protocol block
2621 */
2622
2623 spin_lock_init(&po->bind_lock);
905db440 2624 mutex_init(&po->pg_vec_lock);
1da177e4 2625 po->prot_hook.func = packet_rcv;
be02097c 2626
1da177e4
LT		 2627 if (sock->type == SOCK_PACKET)
2628 po->prot_hook.func = packet_rcv_spkt;
be02097c 2629
1da177e4
LT		 2630 po->prot_hook.af_packet_priv = sk;
2631
0e11c91e
AV		 2632 if (proto) {
2633 po->prot_hook.type = proto;
ce06b03e 2634 register_prot_hook(sk);
1da177e4
LT		 2635 }
2636
0fa7fa98 2637 mutex_lock(&net->packet.sklist_lock);
808f5114 2638 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2639 mutex_unlock(&net->packet.sklist_lock);
2640
2641 preempt_disable();
3680453c 2642 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2643 preempt_enable();
808f5114 2644
40d4e3df 2645 return 0;
1da177e4
LT		 2646out:
2647 return err;
2648}
2649
2650/*
2651 * Pull a packet from our receive queue and hand it to the user.
2652 * If necessary we block.
2653 */
2654
2655static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2656 struct msghdr *msg, size_t len, int flags)
2657{
2658 struct sock *sk = sock->sk;
2659 struct sk_buff *skb;
2660 int copied, err;
0fb375fb 2661 struct sockaddr_ll *sll;
bfd5f4a3 2662 int vnet_hdr_len = 0;
1da177e4
LT		 2663
2664 err = -EINVAL;
ed85b565 2665 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2666 goto out;
2667
2668#if 0
2669 /* What error should we return now? EUNATTACH? */
2670 if (pkt_sk(sk)->ifindex < 0)
2671 return -ENODEV;
2672#endif
2673
ed85b565 2674 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2675 err = sock_recv_errqueue(sk, msg, len,
2676 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2677 goto out;
2678 }
2679
1da177e4
LT		 2680 /*
2681 * Call the generic datagram receiver. This handles all sorts
2682 * of horrible races and re-entrancy so we can forget about it
2683 * in the protocol layers.
2684 *
2685 * Now it will return ENETDOWN, if device have just gone down,
2686 * but then it will block.
2687 */
2688
40d4e3df 2689 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2690
2691 /*
1ce4f28b 2692 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2693 * handles the blocking we don't see and worry about blocking
2694 * retries.
2695 */
2696
8ae55f04 2697 if (skb == NULL)
1da177e4
LT		 2698 goto out;
2699
bfd5f4a3
SS		 2700 if (pkt_sk(sk)->has_vnet_hdr) {
2701 struct virtio_net_hdr vnet_hdr = { 0 };
2702
2703 err = -EINVAL;
2704 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2705 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2706 goto out_free;
2707
1f18b717
MK		 2708 len -= vnet_hdr_len;
2709
bfd5f4a3
SS		 2710 if (skb_is_gso(skb)) {
2711 struct skb_shared_info *sinfo = skb_shinfo(skb);
2712
2713 /* This is a hint as to how much should be linear. */
2714 vnet_hdr.hdr_len = skb_headlen(skb);
2715 vnet_hdr.gso_size = sinfo->gso_size;
2716 if (sinfo->gso_type & SKB_GSO_TCPV4)
2717 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2718 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2719 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2720 else if (sinfo->gso_type & SKB_GSO_UDP)
2721 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2722 else if (sinfo->gso_type & SKB_GSO_FCOE)
2723 goto out_free;
2724 else
2725 BUG();
2726 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2727 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2728 } else
2729 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2730
2731 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2732 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2733 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2734 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2735 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2736 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2737 } /* else everything is zero */
2738
2739 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2740 vnet_hdr_len);
2741 if (err < 0)
2742 goto out_free;
2743 }
2744
0fb375fb
EB		 2745 /*
2746 * If the address length field is there to be filled in, we fill
2747 * it in now.
2748 */
2749
ffbc6111 2750 sll = &PACKET_SKB_CB(skb)->sa.ll;
0fb375fb
EB		 2751 if (sock->type == SOCK_PACKET)
2752 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2753 else
2754 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
2755
1da177e4
LT		 2756 /*
2757 * You lose any data beyond the buffer you gave. If it worries a
2758 * user program they can ask the device for its MTU anyway.
2759 */
2760
2761 copied = skb->len;
40d4e3df
ED		 2762 if (copied > len) {
2763 copied = len;
2764 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2765 }
2766
2767 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2768 if (err)
2769 goto out_free;
2770
3b885787 2771 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4
LT		 2772
2773 if (msg->msg_name)
ffbc6111
HX		 2774 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2775 msg->msg_namelen);
1da177e4 2776
8dc41944 2777 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2778 struct tpacket_auxdata aux;
2779
2780 aux.tp_status = TP_STATUS_USER;
2781 if (skb->ip_summed == CHECKSUM_PARTIAL)
2782 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2783 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2784 aux.tp_snaplen = skb->len;
2785 aux.tp_mac = 0;
bbe735e4 2786 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2787 if (vlan_tx_tag_present(skb)) {
2788 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2789 aux.tp_status |= TP_STATUS_VLAN_VALID;
2790 } else {
2791 aux.tp_vlan_tci = 0;
2792 }
13fcb7bd 2793 aux.tp_padding = 0;
ffbc6111 2794 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2795 }
2796
1da177e4
LT		 2797 /*
2798 * Free or return the buffer as appropriate. Again this
2799 * hides all the races and re-entrancy issues from us.
2800 */
bfd5f4a3 2801 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2802
2803out_free:
2804 skb_free_datagram(sk, skb);
2805out:
2806 return err;
2807}
2808
1da177e4
LT		 2809static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2810 int *uaddr_len, int peer)
2811{
2812 struct net_device *dev;
2813 struct sock *sk = sock->sk;
2814
2815 if (peer)
2816 return -EOPNOTSUPP;
2817
2818 uaddr->sa_family = AF_PACKET;
2dc85bf3 2819 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 2820 rcu_read_lock();
2821 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2822 if (dev)
2dc85bf3 2823 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 2824 rcu_read_unlock();
1da177e4
LT		 2825 *uaddr_len = sizeof(*uaddr);
2826
2827 return 0;
2828}
1da177e4
LT		 2829
2830static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2831 int *uaddr_len, int peer)
2832{
2833 struct net_device *dev;
2834 struct sock *sk = sock->sk;
2835 struct packet_sock *po = pkt_sk(sk);
13cfa97b 2836 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 2837
2838 if (peer)
2839 return -EOPNOTSUPP;
2840
2841 sll->sll_family = AF_PACKET;
2842 sll->sll_ifindex = po->ifindex;
2843 sll->sll_protocol = po->num;
67286640 2844 sll->sll_pkttype = 0;
654d1f8a
ED		 2845 rcu_read_lock();
2846 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 2847 if (dev) {
2848 sll->sll_hatype = dev->type;
2849 sll->sll_halen = dev->addr_len;
2850 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 2851 } else {
2852 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2853 sll->sll_halen = 0;
2854 }
654d1f8a 2855 rcu_read_unlock();
0fb375fb 2856 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 2857
2858 return 0;
2859}
2860
2aeb0b88
WC		 2861static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2862 int what)
1da177e4
LT		 2863{
2864 switch (i->type) {
2865 case PACKET_MR_MULTICAST:
1162563f
JP		 2866 if (i->alen != dev->addr_len)
2867 return -EINVAL;
1da177e4 2868 if (what > 0)
22bedad3 2869 return dev_mc_add(dev, i->addr);
1da177e4 2870 else
22bedad3 2871 return dev_mc_del(dev, i->addr);
1da177e4
LT		 2872 break;
2873 case PACKET_MR_PROMISC:
2aeb0b88 2874 return dev_set_promiscuity(dev, what);
1da177e4
LT		 2875 break;
2876 case PACKET_MR_ALLMULTI:
2aeb0b88 2877 return dev_set_allmulti(dev, what);
1da177e4 2878 break;
d95ed927 2879 case PACKET_MR_UNICAST:
1162563f
JP		 2880 if (i->alen != dev->addr_len)
2881 return -EINVAL;
d95ed927 2882 if (what > 0)
a748ee24 2883 return dev_uc_add(dev, i->addr);
d95ed927 2884 else
a748ee24 2885 return dev_uc_del(dev, i->addr);
d95ed927 2886 break;
40d4e3df
ED		 2887 default:
2888 break;
1da177e4 2889 }
2aeb0b88 2890 return 0;
1da177e4
LT		 2891}
2892
2893static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
2894{
40d4e3df 2895 for ( ; i; i = i->next) {
1da177e4
LT		 2896 if (i->ifindex == dev->ifindex)
2897 packet_dev_mc(dev, i, what);
2898 }
2899}
2900
0fb375fb 2901static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2902{
2903 struct packet_sock *po = pkt_sk(sk);
2904 struct packet_mclist *ml, *i;
2905 struct net_device *dev;
2906 int err;
2907
2908 rtnl_lock();
2909
2910 err = -ENODEV;
3b1e0a65 2911 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 2912 if (!dev)
2913 goto done;
2914
2915 err = -EINVAL;
1162563f 2916 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 2917 goto done;
2918
2919 err = -ENOBUFS;
8b3a7005 2920 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 2921 if (i == NULL)
2922 goto done;
2923
2924 err = 0;
2925 for (ml = po->mclist; ml; ml = ml->next) {
2926 if (ml->ifindex == mreq->mr_ifindex &&
2927 ml->type == mreq->mr_type &&
2928 ml->alen == mreq->mr_alen &&
2929 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2930 ml->count++;
2931 /* Free the new element ... */
2932 kfree(i);
2933 goto done;
2934 }
2935 }
2936
2937 i->type = mreq->mr_type;
2938 i->ifindex = mreq->mr_ifindex;
2939 i->alen = mreq->mr_alen;
2940 memcpy(i->addr, mreq->mr_address, i->alen);
2941 i->count = 1;
2942 i->next = po->mclist;
2943 po->mclist = i;
2aeb0b88
WC		 2944 err = packet_dev_mc(dev, i, 1);
2945 if (err) {
2946 po->mclist = i->next;
2947 kfree(i);
2948 }
1da177e4
LT		 2949
2950done:
2951 rtnl_unlock();
2952 return err;
2953}
2954
0fb375fb 2955static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 2956{
2957 struct packet_mclist *ml, **mlp;
2958
2959 rtnl_lock();
2960
2961 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
2962 if (ml->ifindex == mreq->mr_ifindex &&
2963 ml->type == mreq->mr_type &&
2964 ml->alen == mreq->mr_alen &&
2965 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
2966 if (--ml->count == 0) {
2967 struct net_device *dev;
2968 *mlp = ml->next;
ad959e76
ED		 2969 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2970 if (dev)
1da177e4 2971 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 2972 kfree(ml);
2973 }
2974 rtnl_unlock();
2975 return 0;
2976 }
2977 }
2978 rtnl_unlock();
2979 return -EADDRNOTAVAIL;
2980}
2981
2982static void packet_flush_mclist(struct sock *sk)
2983{
2984 struct packet_sock *po = pkt_sk(sk);
2985 struct packet_mclist *ml;
2986
2987 if (!po->mclist)
2988 return;
2989
2990 rtnl_lock();
2991 while ((ml = po->mclist) != NULL) {
2992 struct net_device *dev;
2993
2994 po->mclist = ml->next;
ad959e76
ED		 2995 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
2996 if (dev != NULL)
1da177e4 2997 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 2998 kfree(ml);
2999 }
3000 rtnl_unlock();
3001}
1da177e4
LT		 3002
3003static int
b7058842 3004packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3005{
3006 struct sock *sk = sock->sk;
8dc41944 3007 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3008 int ret;
3009
3010 if (level != SOL_PACKET)
3011 return -ENOPROTOOPT;
3012
69e3c75f 3013 switch (optname) {
1ce4f28b 3014 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3015 case PACKET_DROP_MEMBERSHIP:
3016 {
0fb375fb
EB		 3017 struct packet_mreq_max mreq;
3018 int len = optlen;
3019 memset(&mreq, 0, sizeof(mreq));
3020 if (len < sizeof(struct packet_mreq))
1da177e4 3021 return -EINVAL;
0fb375fb
EB		 3022 if (len > sizeof(mreq))
3023 len = sizeof(mreq);
40d4e3df 3024 if (copy_from_user(&mreq, optval, len))
1da177e4 3025 return -EFAULT;
0fb375fb
EB		 3026 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3027 return -EINVAL;
1da177e4
LT		 3028 if (optname == PACKET_ADD_MEMBERSHIP)
3029 ret = packet_mc_add(sk, &mreq);
3030 else
3031 ret = packet_mc_drop(sk, &mreq);
3032 return ret;
3033 }
a2efcfa0 3034
1da177e4 3035 case PACKET_RX_RING:
69e3c75f 3036 case PACKET_TX_RING:
1da177e4 3037 {
f6fb8f10 3038 union tpacket_req_u req_u;
3039 int len;
1da177e4 3040
f6fb8f10 3041 switch (po->tp_version) {
3042 case TPACKET_V1:
3043 case TPACKET_V2:
3044 len = sizeof(req_u.req);
3045 break;
3046 case TPACKET_V3:
3047 default:
3048 len = sizeof(req_u.req3);
3049 break;
3050 }
3051 if (optlen < len)
1da177e4 3052 return -EINVAL;
bfd5f4a3
SS		 3053 if (pkt_sk(sk)->has_vnet_hdr)
3054 return -EINVAL;
f6fb8f10 3055 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3056 return -EFAULT;
f6fb8f10 3057 return packet_set_ring(sk, &req_u, 0,
3058 optname == PACKET_TX_RING);
1da177e4
LT		 3059 }
3060 case PACKET_COPY_THRESH:
3061 {
3062 int val;
3063
40d4e3df 3064 if (optlen != sizeof(val))
1da177e4 3065 return -EINVAL;
40d4e3df 3066 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3067 return -EFAULT;
3068
3069 pkt_sk(sk)->copy_thresh = val;
3070 return 0;
3071 }
bbd6ef87
PM		 3072 case PACKET_VERSION:
3073 {
3074 int val;
3075
3076 if (optlen != sizeof(val))
3077 return -EINVAL;
69e3c75f 3078 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3079 return -EBUSY;
3080 if (copy_from_user(&val, optval, sizeof(val)))
3081 return -EFAULT;
3082 switch (val) {
3083 case TPACKET_V1:
3084 case TPACKET_V2:
f6fb8f10 3085 case TPACKET_V3:
bbd6ef87
PM		 3086 po->tp_version = val;
3087 return 0;
3088 default:
3089 return -EINVAL;
3090 }
3091 }
8913336a
PM		 3092 case PACKET_RESERVE:
3093 {
3094 unsigned int val;
3095
3096 if (optlen != sizeof(val))
3097 return -EINVAL;
69e3c75f 3098 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3099 return -EBUSY;
3100 if (copy_from_user(&val, optval, sizeof(val)))
3101 return -EFAULT;
3102 po->tp_reserve = val;
3103 return 0;
3104 }
69e3c75f
JB		 3105 case PACKET_LOSS:
3106 {
3107 unsigned int val;
3108
3109 if (optlen != sizeof(val))
3110 return -EINVAL;
3111 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3112 return -EBUSY;
3113 if (copy_from_user(&val, optval, sizeof(val)))
3114 return -EFAULT;
3115 po->tp_loss = !!val;
3116 return 0;
3117 }
8dc41944
HX		 3118 case PACKET_AUXDATA:
3119 {
3120 int val;
3121
3122 if (optlen < sizeof(val))
3123 return -EINVAL;
3124 if (copy_from_user(&val, optval, sizeof(val)))
3125 return -EFAULT;
3126
3127 po->auxdata = !!val;
3128 return 0;
3129 }
80feaacb
PWJ		 3130 case PACKET_ORIGDEV:
3131 {
3132 int val;
3133
3134 if (optlen < sizeof(val))
3135 return -EINVAL;
3136 if (copy_from_user(&val, optval, sizeof(val)))
3137 return -EFAULT;
3138
3139 po->origdev = !!val;
3140 return 0;
3141 }
bfd5f4a3
SS		 3142 case PACKET_VNET_HDR:
3143 {
3144 int val;
3145
3146 if (sock->type != SOCK_RAW)
3147 return -EINVAL;
3148 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3149 return -EBUSY;
3150 if (optlen < sizeof(val))
3151 return -EINVAL;
3152 if (copy_from_user(&val, optval, sizeof(val)))
3153 return -EFAULT;
3154
3155 po->has_vnet_hdr = !!val;
3156 return 0;
3157 }
614f60fa
SM		 3158 case PACKET_TIMESTAMP:
3159 {
3160 int val;
3161
3162 if (optlen != sizeof(val))
3163 return -EINVAL;
3164 if (copy_from_user(&val, optval, sizeof(val)))
3165 return -EFAULT;
3166
3167 po->tp_tstamp = val;
3168 return 0;
3169 }
dc99f600
DM		 3170 case PACKET_FANOUT:
3171 {
3172 int val;
3173
3174 if (optlen != sizeof(val))
3175 return -EINVAL;
3176 if (copy_from_user(&val, optval, sizeof(val)))
3177 return -EFAULT;
3178
3179 return fanout_add(sk, val & 0xffff, val >> 16);
3180 }
5920cd3a
PC		 3181 case PACKET_TX_HAS_OFF:
3182 {
3183 unsigned int val;
3184
3185 if (optlen != sizeof(val))
3186 return -EINVAL;
3187 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3188 return -EBUSY;
3189 if (copy_from_user(&val, optval, sizeof(val)))
3190 return -EFAULT;
3191 po->tp_tx_has_off = !!val;
3192 return 0;
3193 }
1da177e4
LT		 3194 default:
3195 return -ENOPROTOOPT;
3196 }
3197}
3198
3199static int packet_getsockopt(struct socket *sock, int level, int optname,
3200 char __user *optval, int __user *optlen)
3201{
3202 int len;
c06fff6e 3203 int val, lv = sizeof(val);
1da177e4
LT		 3204 struct sock *sk = sock->sk;
3205 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3206 void *data = &val;
ee80fbf3 3207 union tpacket_stats_u st;
1da177e4
LT		 3208
3209 if (level != SOL_PACKET)
3210 return -ENOPROTOOPT;
3211
8ae55f04
KK		 3212 if (get_user(len, optlen))
3213 return -EFAULT;
1da177e4
LT		 3214
3215 if (len < 0)
3216 return -EINVAL;
1ce4f28b 3217
69e3c75f 3218 switch (optname) {
1da177e4 3219 case PACKET_STATISTICS:
1da177e4 3220 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3221 memcpy(&st, &po->stats, sizeof(st));
3222 memset(&po->stats, 0, sizeof(po->stats));
3223 spin_unlock_bh(&sk->sk_receive_queue.lock);
3224
f6fb8f10 3225 if (po->tp_version == TPACKET_V3) {
c06fff6e 3226 lv = sizeof(struct tpacket_stats_v3);
ee80fbf3 3227 data = &st.stats3;
f6fb8f10 3228 } else {
c06fff6e 3229 lv = sizeof(struct tpacket_stats);
ee80fbf3 3230 data = &st.stats1;
f6fb8f10 3231 }
ee80fbf3 3232
8dc41944
HX		 3233 break;
3234 case PACKET_AUXDATA:
8dc41944 3235 val = po->auxdata;
80feaacb
PWJ		 3236 break;
3237 case PACKET_ORIGDEV:
80feaacb 3238 val = po->origdev;
bfd5f4a3
SS		 3239 break;
3240 case PACKET_VNET_HDR:
bfd5f4a3 3241 val = po->has_vnet_hdr;
1da177e4 3242 break;
bbd6ef87 3243 case PACKET_VERSION:
bbd6ef87 3244 val = po->tp_version;
bbd6ef87
PM		 3245 break;
3246 case PACKET_HDRLEN:
3247 if (len > sizeof(int))
3248 len = sizeof(int);
3249 if (copy_from_user(&val, optval, len))
3250 return -EFAULT;
3251 switch (val) {
3252 case TPACKET_V1:
3253 val = sizeof(struct tpacket_hdr);
3254 break;
3255 case TPACKET_V2:
3256 val = sizeof(struct tpacket2_hdr);
3257 break;
f6fb8f10 3258 case TPACKET_V3:
3259 val = sizeof(struct tpacket3_hdr);
3260 break;
bbd6ef87
PM		 3261 default:
3262 return -EINVAL;
3263 }
bbd6ef87 3264 break;
8913336a 3265 case PACKET_RESERVE:
8913336a 3266 val = po->tp_reserve;
8913336a 3267 break;
69e3c75f 3268 case PACKET_LOSS:
69e3c75f 3269 val = po->tp_loss;
69e3c75f 3270 break;
614f60fa 3271 case PACKET_TIMESTAMP:
614f60fa 3272 val = po->tp_tstamp;
614f60fa 3273 break;
dc99f600 3274 case PACKET_FANOUT:
dc99f600
DM		 3275 val = (po->fanout ?
3276 ((u32)po->fanout->id |
77f65ebd
WB		 3277 ((u32)po->fanout->type << 16) |
3278 ((u32)po->fanout->flags << 24)) :
dc99f600 3279 0);
dc99f600 3280 break;
5920cd3a
PC		 3281 case PACKET_TX_HAS_OFF:
3282 val = po->tp_tx_has_off;
3283 break;
1da177e4
LT		 3284 default:
3285 return -ENOPROTOOPT;
3286 }
3287
c06fff6e
ED		 3288 if (len > lv)
3289 len = lv;
8ae55f04
KK		 3290 if (put_user(len, optlen))
3291 return -EFAULT;
8dc41944
HX		 3292 if (copy_to_user(optval, data, len))
3293 return -EFAULT;
8ae55f04 3294 return 0;
1da177e4
LT		 3295}
3296
3297
351638e7
JP		 3298static int packet_notifier(struct notifier_block *this,
3299 unsigned long msg, void *ptr)
1da177e4
LT		 3300{
3301 struct sock *sk;
351638e7 3302 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3303 struct net *net = dev_net(dev);
1da177e4 3304
808f5114 3305 rcu_read_lock();
b67bfe0d 3306 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3307 struct packet_sock *po = pkt_sk(sk);
3308
3309 switch (msg) {
3310 case NETDEV_UNREGISTER:
1da177e4
LT		 3311 if (po->mclist)
3312 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3313 /* fallthrough */
3314
1da177e4
LT		 3315 case NETDEV_DOWN:
3316 if (dev->ifindex == po->ifindex) {
3317 spin_lock(&po->bind_lock);
3318 if (po->running) {
ce06b03e 3319 __unregister_prot_hook(sk, false);
1da177e4
LT		 3320 sk->sk_err = ENETDOWN;
3321 if (!sock_flag(sk, SOCK_DEAD))
3322 sk->sk_error_report(sk);
3323 }
3324 if (msg == NETDEV_UNREGISTER) {
3325 po->ifindex = -1;
160ff18a
BG		 3326 if (po->prot_hook.dev)
3327 dev_put(po->prot_hook.dev);
1da177e4
LT		 3328 po->prot_hook.dev = NULL;
3329 }
3330 spin_unlock(&po->bind_lock);
3331 }
3332 break;
3333 case NETDEV_UP:
808f5114 3334 if (dev->ifindex == po->ifindex) {
3335 spin_lock(&po->bind_lock);
ce06b03e
DM		 3336 if (po->num)
3337 register_prot_hook(sk);
808f5114 3338 spin_unlock(&po->bind_lock);
1da177e4 3339 }
1da177e4
LT		 3340 break;
3341 }
3342 }
808f5114 3343 rcu_read_unlock();
1da177e4
LT		 3344 return NOTIFY_DONE;
3345}
3346
3347
3348static int packet_ioctl(struct socket *sock, unsigned int cmd,
3349 unsigned long arg)
3350{
3351 struct sock *sk = sock->sk;
3352
69e3c75f 3353 switch (cmd) {
40d4e3df
ED		 3354 case SIOCOUTQ:
3355 {
3356 int amount = sk_wmem_alloc_get(sk);
31e6d363 3357
40d4e3df
ED		 3358 return put_user(amount, (int __user *)arg);
3359 }
3360 case SIOCINQ:
3361 {
3362 struct sk_buff *skb;
3363 int amount = 0;
3364
3365 spin_lock_bh(&sk->sk_receive_queue.lock);
3366 skb = skb_peek(&sk->sk_receive_queue);
3367 if (skb)
3368 amount = skb->len;
3369 spin_unlock_bh(&sk->sk_receive_queue.lock);
3370 return put_user(amount, (int __user *)arg);
3371 }
3372 case SIOCGSTAMP:
3373 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3374 case SIOCGSTAMPNS:
3375 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3376
1da177e4 3377#ifdef CONFIG_INET
40d4e3df
ED		 3378 case SIOCADDRT:
3379 case SIOCDELRT:
3380 case SIOCDARP:
3381 case SIOCGARP:
3382 case SIOCSARP:
3383 case SIOCGIFADDR:
3384 case SIOCSIFADDR:
3385 case SIOCGIFBRDADDR:
3386 case SIOCSIFBRDADDR:
3387 case SIOCGIFNETMASK:
3388 case SIOCSIFNETMASK:
3389 case SIOCGIFDSTADDR:
3390 case SIOCSIFDSTADDR:
3391 case SIOCSIFFLAGS:
40d4e3df 3392 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3393#endif
3394
40d4e3df
ED		 3395 default:
3396 return -ENOIOCTLCMD;
1da177e4
LT		 3397 }
3398 return 0;
3399}
3400
40d4e3df 3401static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3402 poll_table *wait)
3403{
3404 struct sock *sk = sock->sk;
3405 struct packet_sock *po = pkt_sk(sk);
3406 unsigned int mask = datagram_poll(file, sock, wait);
3407
3408 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3409 if (po->rx_ring.pg_vec) {
f6fb8f10 3410 if (!packet_previous_rx_frame(po, &po->rx_ring,
3411 TP_STATUS_KERNEL))
1da177e4
LT		 3412 mask |= POLLIN | POLLRDNORM;
3413 }
3414 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3415 spin_lock_bh(&sk->sk_write_queue.lock);
3416 if (po->tx_ring.pg_vec) {
3417 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3418 mask |= POLLOUT | POLLWRNORM;
3419 }
3420 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3421 return mask;
3422}
3423
3424
3425/* Dirty? Well, I still did not learn better way to account
3426 * for user mmaps.
3427 */
3428
3429static void packet_mm_open(struct vm_area_struct *vma)
3430{
3431 struct file *file = vma->vm_file;
40d4e3df 3432 struct socket *sock = file->private_data;
1da177e4 3433 struct sock *sk = sock->sk;
1ce4f28b 3434
1da177e4
LT		 3435 if (sk)
3436 atomic_inc(&pkt_sk(sk)->mapped);
3437}
3438
3439static void packet_mm_close(struct vm_area_struct *vma)
3440{
3441 struct file *file = vma->vm_file;
40d4e3df 3442 struct socket *sock = file->private_data;
1da177e4 3443 struct sock *sk = sock->sk;
1ce4f28b 3444
1da177e4
LT		 3445 if (sk)
3446 atomic_dec(&pkt_sk(sk)->mapped);
3447}
3448
f0f37e2f 3449static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3450 .open = packet_mm_open,
3451 .close = packet_mm_close,
1da177e4
LT		 3452};
3453
0e3125c7
NH		 3454static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3455 unsigned int len)
1da177e4
LT		 3456{
3457 int i;
3458
4ebf0ae2 3459 for (i = 0; i < len; i++) {
0e3125c7 3460 if (likely(pg_vec[i].buffer)) {
c56b4d90 3461 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3462 vfree(pg_vec[i].buffer);
3463 else
3464 free_pages((unsigned long)pg_vec[i].buffer,
3465 order);
3466 pg_vec[i].buffer = NULL;
3467 }
1da177e4
LT		 3468 }
3469 kfree(pg_vec);
3470}
3471
eea49cc9 3472static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3473{
0e3125c7
NH		 3474 char *buffer = NULL;
3475 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3476 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3477
3478 buffer = (char *) __get_free_pages(gfp_flags, order);
3479
3480 if (buffer)
3481 return buffer;
3482
3483 /*
3484 * __get_free_pages failed, fall back to vmalloc
3485 */
bbce5a59 3486 buffer = vzalloc((1 << order) * PAGE_SIZE);
719bfeaa 3487
0e3125c7
NH		 3488 if (buffer)
3489 return buffer;
3490
3491 /*
3492 * vmalloc failed, lets dig into swap here
3493 */
0e3125c7
NH		 3494 gfp_flags &= ~__GFP_NORETRY;
3495 buffer = (char *)__get_free_pages(gfp_flags, order);
3496 if (buffer)
3497 return buffer;
3498
3499 /*
3500 * complete and utter failure
3501 */
3502 return NULL;
4ebf0ae2
DM		 3503}
3504
0e3125c7 3505static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3506{
3507 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3508 struct pgv *pg_vec;
4ebf0ae2
DM		 3509 int i;
3510
0e3125c7 3511 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3512 if (unlikely(!pg_vec))
3513 goto out;
3514
3515 for (i = 0; i < block_nr; i++) {
c56b4d90 3516 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3517 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3518 goto out_free_pgvec;
3519 }
3520
3521out:
3522 return pg_vec;
3523
3524out_free_pgvec:
3525 free_pg_vec(pg_vec, order, block_nr);
3526 pg_vec = NULL;
3527 goto out;
3528}
1da177e4 3529
f6fb8f10 3530static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3531 int closing, int tx_ring)
1da177e4 3532{
0e3125c7 3533 struct pgv *pg_vec = NULL;
1da177e4 3534 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3535 int was_running, order = 0;
69e3c75f
JB		 3536 struct packet_ring_buffer *rb;
3537 struct sk_buff_head *rb_queue;
0e11c91e 3538 __be16 num;
f6fb8f10 3539 int err = -EINVAL;
3540 /* Added to avoid minimal code churn */
3541 struct tpacket_req *req = &req_u->req;
3542
3543 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3544 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3545 WARN(1, "Tx-ring is not supported.\n");
3546 goto out;
3547 }
1ce4f28b 3548
69e3c75f
JB		 3549 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3550 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3551
69e3c75f
JB		 3552 err = -EBUSY;
3553 if (!closing) {
3554 if (atomic_read(&po->mapped))
3555 goto out;
3556 if (atomic_read(&rb->pending))
3557 goto out;
3558 }
1da177e4 3559
69e3c75f
JB		 3560 if (req->tp_block_nr) {
3561 /* Sanity tests and some calculations */
3562 err = -EBUSY;
3563 if (unlikely(rb->pg_vec))
3564 goto out;
1da177e4 3565
bbd6ef87
PM		 3566 switch (po->tp_version) {
3567 case TPACKET_V1:
3568 po->tp_hdrlen = TPACKET_HDRLEN;
3569 break;
3570 case TPACKET_V2:
3571 po->tp_hdrlen = TPACKET2_HDRLEN;
3572 break;
f6fb8f10 3573 case TPACKET_V3:
3574 po->tp_hdrlen = TPACKET3_HDRLEN;
3575 break;
bbd6ef87
PM		 3576 }
3577
69e3c75f 3578 err = -EINVAL;
4ebf0ae2 3579 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3580 goto out;
4ebf0ae2 3581 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3582 goto out;
8913336a 3583 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3584 po->tp_reserve))
3585 goto out;
4ebf0ae2 3586 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3587 goto out;
1da177e4 3588
69e3c75f
JB		 3589 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3590 if (unlikely(rb->frames_per_block <= 0))
3591 goto out;
3592 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3593 req->tp_frame_nr))
3594 goto out;
1da177e4
LT		 3595
3596 err = -ENOMEM;
4ebf0ae2
DM		 3597 order = get_order(req->tp_block_size);
3598 pg_vec = alloc_pg_vec(req, order);
3599 if (unlikely(!pg_vec))
1da177e4 3600 goto out;
f6fb8f10 3601 switch (po->tp_version) {
3602 case TPACKET_V3:
3603 /* Transmit path is not supported. We checked
3604 * it above but just being paranoid
3605 */
3606 if (!tx_ring)
3607 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3608 break;
3609 default:
3610 break;
3611 }
69e3c75f
JB		 3612 }
3613 /* Done */
3614 else {
3615 err = -EINVAL;
4ebf0ae2 3616 if (unlikely(req->tp_frame_nr))
69e3c75f 3617 goto out;
1da177e4
LT		 3618 }
3619
3620 lock_sock(sk);
3621
3622 /* Detach socket from network */
3623 spin_lock(&po->bind_lock);
3624 was_running = po->running;
3625 num = po->num;
3626 if (was_running) {
1da177e4 3627 po->num = 0;
ce06b03e 3628 __unregister_prot_hook(sk, false);
1da177e4
LT		 3629 }
3630 spin_unlock(&po->bind_lock);
1ce4f28b 3631
1da177e4
LT		 3632 synchronize_net();
3633
3634 err = -EBUSY;
905db440 3635 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3636 if (closing || atomic_read(&po->mapped) == 0) {
3637 err = 0;
69e3c75f 3638 spin_lock_bh(&rb_queue->lock);
c053fd96 3639 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3640 rb->frame_max = (req->tp_frame_nr - 1);
3641 rb->head = 0;
3642 rb->frame_size = req->tp_frame_size;
3643 spin_unlock_bh(&rb_queue->lock);
3644
c053fd96
CG		 3645 swap(rb->pg_vec_order, order);
3646 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3647
3648 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3649 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3650 tpacket_rcv : packet_rcv;
3651 skb_queue_purge(rb_queue);
1da177e4 3652 if (atomic_read(&po->mapped))
40d4e3df
ED		 3653 pr_err("packet_mmap: vma is busy: %d\n",
3654 atomic_read(&po->mapped));
1da177e4 3655 }
905db440 3656 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3657
3658 spin_lock(&po->bind_lock);
ce06b03e 3659 if (was_running) {
1da177e4 3660 po->num = num;
ce06b03e 3661 register_prot_hook(sk);
1da177e4
LT		 3662 }
3663 spin_unlock(&po->bind_lock);
f6fb8f10 3664 if (closing && (po->tp_version > TPACKET_V2)) {
3665 /* Because we don't support block-based V3 on tx-ring */
3666 if (!tx_ring)
3667 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3668 }
1da177e4
LT		 3669 release_sock(sk);
3670
1da177e4
LT		 3671 if (pg_vec)
3672 free_pg_vec(pg_vec, order, req->tp_block_nr);
3673out:
3674 return err;
3675}
3676
69e3c75f
JB		 3677static int packet_mmap(struct file *file, struct socket *sock,
3678 struct vm_area_struct *vma)
1da177e4
LT		 3679{
3680 struct sock *sk = sock->sk;
3681 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3682 unsigned long size, expected_size;
3683 struct packet_ring_buffer *rb;
1da177e4
LT		 3684 unsigned long start;
3685 int err = -EINVAL;
3686 int i;
3687
3688 if (vma->vm_pgoff)
3689 return -EINVAL;
3690
905db440 3691 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3692
3693 expected_size = 0;
3694 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3695 if (rb->pg_vec) {
3696 expected_size += rb->pg_vec_len
3697 * rb->pg_vec_pages
3698 * PAGE_SIZE;
3699 }
3700 }
3701
3702 if (expected_size == 0)
1da177e4 3703 goto out;
69e3c75f
JB		 3704
3705 size = vma->vm_end - vma->vm_start;
3706 if (size != expected_size)
1da177e4
LT		 3707 goto out;
3708
1da177e4 3709 start = vma->vm_start;
69e3c75f
JB		 3710 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3711 if (rb->pg_vec == NULL)
3712 continue;
3713
3714 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3715 struct page *page;
3716 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3717 int pg_num;
3718
c56b4d90
CG		 3719 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3720 page = pgv_to_page(kaddr);
69e3c75f
JB		 3721 err = vm_insert_page(vma, start, page);
3722 if (unlikely(err))
3723 goto out;
3724 start += PAGE_SIZE;
0e3125c7 3725 kaddr += PAGE_SIZE;
69e3c75f 3726 }
4ebf0ae2 3727 }
1da177e4 3728 }
69e3c75f 3729
4ebf0ae2 3730 atomic_inc(&po->mapped);
1da177e4
LT		 3731 vma->vm_ops = &packet_mmap_ops;
3732 err = 0;
3733
3734out:
905db440 3735 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3736 return err;
3737}
1da177e4 3738
90ddc4f0 3739static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3740 .family = PF_PACKET,
3741 .owner = THIS_MODULE,
3742 .release = packet_release,
3743 .bind = packet_bind_spkt,
3744 .connect = sock_no_connect,
3745 .socketpair = sock_no_socketpair,
3746 .accept = sock_no_accept,
3747 .getname = packet_getname_spkt,
3748 .poll = datagram_poll,
3749 .ioctl = packet_ioctl,
3750 .listen = sock_no_listen,
3751 .shutdown = sock_no_shutdown,
3752 .setsockopt = sock_no_setsockopt,
3753 .getsockopt = sock_no_getsockopt,
3754 .sendmsg = packet_sendmsg_spkt,
3755 .recvmsg = packet_recvmsg,
3756 .mmap = sock_no_mmap,
3757 .sendpage = sock_no_sendpage,
3758};
1da177e4 3759
90ddc4f0 3760static const struct proto_ops packet_ops = {
1da177e4
LT		 3761 .family = PF_PACKET,
3762 .owner = THIS_MODULE,
3763 .release = packet_release,
3764 .bind = packet_bind,
3765 .connect = sock_no_connect,
3766 .socketpair = sock_no_socketpair,
3767 .accept = sock_no_accept,
1ce4f28b 3768 .getname = packet_getname,
1da177e4
LT		 3769 .poll = packet_poll,
3770 .ioctl = packet_ioctl,
3771 .listen = sock_no_listen,
3772 .shutdown = sock_no_shutdown,
3773 .setsockopt = packet_setsockopt,
3774 .getsockopt = packet_getsockopt,
3775 .sendmsg = packet_sendmsg,
3776 .recvmsg = packet_recvmsg,
3777 .mmap = packet_mmap,
3778 .sendpage = sock_no_sendpage,
3779};
3780
ec1b4cf7 3781static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3782 .family = PF_PACKET,
3783 .create = packet_create,
3784 .owner = THIS_MODULE,
3785};
3786
3787static struct notifier_block packet_netdev_notifier = {
40d4e3df 3788 .notifier_call = packet_notifier,
1da177e4
LT		 3789};
3790
3791#ifdef CONFIG_PROC_FS
1da177e4
LT		 3792
3793static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3794 __acquires(RCU)
1da177e4 3795{
e372c414 3796 struct net *net = seq_file_net(seq);
808f5114 3797
3798 rcu_read_lock();
3799 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3800}
3801
3802static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3803{
1bf40954 3804 struct net *net = seq_file_net(seq);
808f5114 3805 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3806}
3807
3808static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3809 __releases(RCU)
1da177e4 3810{
808f5114 3811 rcu_read_unlock();
1da177e4
LT		 3812}
3813
1ce4f28b 3814static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3815{
3816 if (v == SEQ_START_TOKEN)
3817 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3818 else {
b7ceabd9 3819 struct sock *s = sk_entry(v);
1da177e4
LT		 3820 const struct packet_sock *po = pkt_sk(s);
3821
3822 seq_printf(seq,
71338aa7 3823 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 3824 s,
3825 atomic_read(&s->sk_refcnt),
3826 s->sk_type,
3827 ntohs(po->num),
3828 po->ifindex,
3829 po->running,
3830 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 3831 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 3832 sock_i_ino(s));
1da177e4
LT		 3833 }
3834
3835 return 0;
3836}
3837
56b3d975 3838static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 3839 .start = packet_seq_start,
3840 .next = packet_seq_next,
3841 .stop = packet_seq_stop,
3842 .show = packet_seq_show,
3843};
3844
3845static int packet_seq_open(struct inode *inode, struct file *file)
3846{
e372c414
DL		 3847 return seq_open_net(inode, file, &packet_seq_ops,
3848 sizeof(struct seq_net_private));
1da177e4
LT		 3849}
3850
da7071d7 3851static const struct file_operations packet_seq_fops = {
1da177e4
LT		 3852 .owner = THIS_MODULE,
3853 .open = packet_seq_open,
3854 .read = seq_read,
3855 .llseek = seq_lseek,
e372c414 3856 .release = seq_release_net,
1da177e4
LT		 3857};
3858
3859#endif
3860
2c8c1e72 3861static int __net_init packet_net_init(struct net *net)
d12d01d6 3862{
0fa7fa98 3863 mutex_init(&net->packet.sklist_lock);
2aaef4e4 3864 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 3865
d4beaa66 3866 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 3867 return -ENOMEM;
3868
3869 return 0;
3870}
3871
2c8c1e72 3872static void __net_exit packet_net_exit(struct net *net)
d12d01d6 3873{
ece31ffd 3874 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 3875}
3876
3877static struct pernet_operations packet_net_ops = {
3878 .init = packet_net_init,
3879 .exit = packet_net_exit,
3880};
3881
3882
1da177e4
LT		 3883static void __exit packet_exit(void)
3884{
1da177e4 3885 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 3886 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 3887 sock_unregister(PF_PACKET);
3888 proto_unregister(&packet_proto);
3889}
3890
3891static int __init packet_init(void)
3892{
3893 int rc = proto_register(&packet_proto, 0);
3894
3895 if (rc != 0)
3896 goto out;
3897
3898 sock_register(&packet_family_ops);
d12d01d6 3899 register_pernet_subsys(&packet_net_ops);
1da177e4 3900 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 3901out:
3902 return rc;
3903}
3904
3905module_init(packet_init);
3906module_exit(packet_exit);
3907MODULE_LICENSE("GPL");
3908MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux kernel - Deliverables
This page took 1.530487 seconds and 5 git commands to generate.