2 * NET3 Protocol independent device support routines.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
9 * Derived from the non IP parts of dev.c 1.0.19
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
51 * Rudi Cilibrasi : Pass the right thing to
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
75 #include <asm/uaccess.h>
76 #include <asm/system.h>
77 #include <linux/bitops.h>
78 #include <linux/capability.h>
79 #include <linux/cpu.h>
80 #include <linux/types.h>
81 #include <linux/kernel.h>
82 #include <linux/hash.h>
83 #include <linux/slab.h>
84 #include <linux/sched.h>
85 #include <linux/mutex.h>
86 #include <linux/string.h>
88 #include <linux/socket.h>
89 #include <linux/sockios.h>
90 #include <linux/errno.h>
91 #include <linux/interrupt.h>
92 #include <linux/if_ether.h>
93 #include <linux/netdevice.h>
94 #include <linux/etherdevice.h>
95 #include <linux/ethtool.h>
96 #include <linux/notifier.h>
97 #include <linux/skbuff.h>
98 #include <net/net_namespace.h>
100 #include <linux/rtnetlink.h>
101 #include <linux/proc_fs.h>
102 #include <linux/seq_file.h>
103 #include <linux/stat.h>
105 #include <net/pkt_sched.h>
106 #include <net/checksum.h>
107 #include <net/xfrm.h>
108 #include <linux/highmem.h>
109 #include <linux/init.h>
110 #include <linux/kmod.h>
111 #include <linux/module.h>
112 #include <linux/netpoll.h>
113 #include <linux/rcupdate.h>
114 #include <linux/delay.h>
115 #include <net/wext.h>
116 #include <net/iw_handler.h>
117 #include <asm/current.h>
118 #include <linux/audit.h>
119 #include <linux/dmaengine.h>
120 #include <linux/err.h>
121 #include <linux/ctype.h>
122 #include <linux/if_arp.h>
123 #include <linux/if_vlan.h>
124 #include <linux/ip.h>
126 #include <linux/ipv6.h>
127 #include <linux/in.h>
128 #include <linux/jhash.h>
129 #include <linux/random.h>
130 #include <trace/events/napi.h>
131 #include <trace/events/net.h>
132 #include <trace/events/skb.h>
133 #include <linux/pci.h>
134 #include <linux/inetdevice.h>
135 #include <linux/cpu_rmap.h>
136 #include <linux/if_tunnel.h>
137 #include <linux/if_pppox.h>
138 #include <linux/ppp_defs.h>
140 #include "net-sysfs.h"
142 /* Instead of increasing this, you should create a hash table. */
143 #define MAX_GRO_SKBS 8
145 /* This should be increased if a protocol with a bigger head is added. */
146 #define GRO_MAX_HEAD (MAX_HEADER + 128)
149 * The list of packet types we will receive (as opposed to discard)
150 * and the routines to invoke.
152 * Why 16. Because with 16 the only overlap we get on a hash of the
153 * low nibble of the protocol value is RARP/SNAP/X.25.
155 * NOTE: That is no longer true with the addition of VLAN tags. Not
156 * sure which should go first, but I bet it won't make much
157 * difference if we are running VLANs. The good news is that
158 * this protocol won't be in the list unless compiled in, so
159 * the average user (w/out VLANs) will not be adversely affected.
176 #define PTYPE_HASH_SIZE (16)
177 #define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
179 static DEFINE_SPINLOCK(ptype_lock
);
180 static struct list_head ptype_base
[PTYPE_HASH_SIZE
] __read_mostly
;
181 static struct list_head ptype_all __read_mostly
; /* Taps */
184 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
187 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
189 * Writers must hold the rtnl semaphore while they loop through the
190 * dev_base_head list, and hold dev_base_lock for writing when they do the
191 * actual updates. This allows pure readers to access the list even
192 * while a writer is preparing to update it.
194 * To put it another way, dev_base_lock is held for writing only to
195 * protect against pure readers; the rtnl semaphore provides the
196 * protection against other writers.
198 * See, for example usages, register_netdevice() and
199 * unregister_netdevice(), which must be called with the rtnl
202 DEFINE_RWLOCK(dev_base_lock
);
203 EXPORT_SYMBOL(dev_base_lock
);
205 static inline void dev_base_seq_inc(struct net
*net
)
207 while (++net
->dev_base_seq
== 0);
210 static inline struct hlist_head
*dev_name_hash(struct net
*net
, const char *name
)
212 unsigned hash
= full_name_hash(name
, strnlen(name
, IFNAMSIZ
));
213 return &net
->dev_name_head
[hash_32(hash
, NETDEV_HASHBITS
)];
216 static inline struct hlist_head
*dev_index_hash(struct net
*net
, int ifindex
)
218 return &net
->dev_index_head
[ifindex
& (NETDEV_HASHENTRIES
- 1)];
221 static inline void rps_lock(struct softnet_data
*sd
)
224 spin_lock(&sd
->input_pkt_queue
.lock
);
228 static inline void rps_unlock(struct softnet_data
*sd
)
231 spin_unlock(&sd
->input_pkt_queue
.lock
);
235 /* Device list insertion */
236 static int list_netdevice(struct net_device
*dev
)
238 struct net
*net
= dev_net(dev
);
242 write_lock_bh(&dev_base_lock
);
243 list_add_tail_rcu(&dev
->dev_list
, &net
->dev_base_head
);
244 hlist_add_head_rcu(&dev
->name_hlist
, dev_name_hash(net
, dev
->name
));
245 hlist_add_head_rcu(&dev
->index_hlist
,
246 dev_index_hash(net
, dev
->ifindex
));
247 write_unlock_bh(&dev_base_lock
);
249 dev_base_seq_inc(net
);
254 /* Device list removal
255 * caller must respect a RCU grace period before freeing/reusing dev
257 static void unlist_netdevice(struct net_device
*dev
)
261 /* Unlink dev from the device chain */
262 write_lock_bh(&dev_base_lock
);
263 list_del_rcu(&dev
->dev_list
);
264 hlist_del_rcu(&dev
->name_hlist
);
265 hlist_del_rcu(&dev
->index_hlist
);
266 write_unlock_bh(&dev_base_lock
);
268 dev_base_seq_inc(dev_net(dev
));
275 static RAW_NOTIFIER_HEAD(netdev_chain
);
278 * Device drivers call our routines to queue packets here. We empty the
279 * queue in the local softnet handler.
282 DEFINE_PER_CPU_ALIGNED(struct softnet_data
, softnet_data
);
283 EXPORT_PER_CPU_SYMBOL(softnet_data
);
285 #ifdef CONFIG_LOCKDEP
287 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
288 * according to dev->type
290 static const unsigned short netdev_lock_type
[] =
291 {ARPHRD_NETROM
, ARPHRD_ETHER
, ARPHRD_EETHER
, ARPHRD_AX25
,
292 ARPHRD_PRONET
, ARPHRD_CHAOS
, ARPHRD_IEEE802
, ARPHRD_ARCNET
,
293 ARPHRD_APPLETLK
, ARPHRD_DLCI
, ARPHRD_ATM
, ARPHRD_METRICOM
,
294 ARPHRD_IEEE1394
, ARPHRD_EUI64
, ARPHRD_INFINIBAND
, ARPHRD_SLIP
,
295 ARPHRD_CSLIP
, ARPHRD_SLIP6
, ARPHRD_CSLIP6
, ARPHRD_RSRVD
,
296 ARPHRD_ADAPT
, ARPHRD_ROSE
, ARPHRD_X25
, ARPHRD_HWX25
,
297 ARPHRD_PPP
, ARPHRD_CISCO
, ARPHRD_LAPB
, ARPHRD_DDCMP
,
298 ARPHRD_RAWHDLC
, ARPHRD_TUNNEL
, ARPHRD_TUNNEL6
, ARPHRD_FRAD
,
299 ARPHRD_SKIP
, ARPHRD_LOOPBACK
, ARPHRD_LOCALTLK
, ARPHRD_FDDI
,
300 ARPHRD_BIF
, ARPHRD_SIT
, ARPHRD_IPDDP
, ARPHRD_IPGRE
,
301 ARPHRD_PIMREG
, ARPHRD_HIPPI
, ARPHRD_ASH
, ARPHRD_ECONET
,
302 ARPHRD_IRDA
, ARPHRD_FCPP
, ARPHRD_FCAL
, ARPHRD_FCPL
,
303 ARPHRD_FCFABRIC
, ARPHRD_IEEE802_TR
, ARPHRD_IEEE80211
,
304 ARPHRD_IEEE80211_PRISM
, ARPHRD_IEEE80211_RADIOTAP
, ARPHRD_PHONET
,
305 ARPHRD_PHONET_PIPE
, ARPHRD_IEEE802154
,
306 ARPHRD_VOID
, ARPHRD_NONE
};
308 static const char *const netdev_lock_name
[] =
309 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
310 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
311 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
312 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
313 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
314 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
315 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
316 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
317 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
318 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
319 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
320 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
321 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
322 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
323 "_xmit_PHONET_PIPE", "_xmit_IEEE802154",
324 "_xmit_VOID", "_xmit_NONE"};
326 static struct lock_class_key netdev_xmit_lock_key
[ARRAY_SIZE(netdev_lock_type
)];
327 static struct lock_class_key netdev_addr_lock_key
[ARRAY_SIZE(netdev_lock_type
)];
329 static inline unsigned short netdev_lock_pos(unsigned short dev_type
)
333 for (i
= 0; i
< ARRAY_SIZE(netdev_lock_type
); i
++)
334 if (netdev_lock_type
[i
] == dev_type
)
336 /* the last key is used by default */
337 return ARRAY_SIZE(netdev_lock_type
) - 1;
340 static inline void netdev_set_xmit_lockdep_class(spinlock_t
*lock
,
341 unsigned short dev_type
)
345 i
= netdev_lock_pos(dev_type
);
346 lockdep_set_class_and_name(lock
, &netdev_xmit_lock_key
[i
],
347 netdev_lock_name
[i
]);
350 static inline void netdev_set_addr_lockdep_class(struct net_device
*dev
)
354 i
= netdev_lock_pos(dev
->type
);
355 lockdep_set_class_and_name(&dev
->addr_list_lock
,
356 &netdev_addr_lock_key
[i
],
357 netdev_lock_name
[i
]);
360 static inline void netdev_set_xmit_lockdep_class(spinlock_t
*lock
,
361 unsigned short dev_type
)
364 static inline void netdev_set_addr_lockdep_class(struct net_device
*dev
)
369 /*******************************************************************************
371 Protocol management and registration routines
373 *******************************************************************************/
376 * Add a protocol ID to the list. Now that the input handler is
377 * smarter we can dispense with all the messy stuff that used to be
380 * BEWARE!!! Protocol handlers, mangling input packets,
381 * MUST BE last in hash buckets and checking protocol handlers
382 * MUST start from promiscuous ptype_all chain in net_bh.
383 * It is true now, do not change it.
384 * Explanation follows: if protocol handler, mangling packet, will
385 * be the first on list, it is not able to sense, that packet
386 * is cloned and should be copied-on-write, so that it will
387 * change it and subsequent readers will get broken packet.
391 static inline struct list_head
*ptype_head(const struct packet_type
*pt
)
393 if (pt
->type
== htons(ETH_P_ALL
))
396 return &ptype_base
[ntohs(pt
->type
) & PTYPE_HASH_MASK
];
400 * dev_add_pack - add packet handler
401 * @pt: packet type declaration
403 * Add a protocol handler to the networking stack. The passed &packet_type
404 * is linked into kernel lists and may not be freed until it has been
405 * removed from the kernel lists.
407 * This call does not sleep therefore it can not
408 * guarantee all CPU's that are in middle of receiving packets
409 * will see the new packet type (until the next received packet).
412 void dev_add_pack(struct packet_type
*pt
)
414 struct list_head
*head
= ptype_head(pt
);
416 spin_lock(&ptype_lock
);
417 list_add_rcu(&pt
->list
, head
);
418 spin_unlock(&ptype_lock
);
420 EXPORT_SYMBOL(dev_add_pack
);
423 * __dev_remove_pack - remove packet handler
424 * @pt: packet type declaration
426 * Remove a protocol handler that was previously added to the kernel
427 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
428 * from the kernel lists and can be freed or reused once this function
431 * The packet type might still be in use by receivers
432 * and must not be freed until after all the CPU's have gone
433 * through a quiescent state.
435 void __dev_remove_pack(struct packet_type
*pt
)
437 struct list_head
*head
= ptype_head(pt
);
438 struct packet_type
*pt1
;
440 spin_lock(&ptype_lock
);
442 list_for_each_entry(pt1
, head
, list
) {
444 list_del_rcu(&pt
->list
);
449 printk(KERN_WARNING
"dev_remove_pack: %p not found.\n", pt
);
451 spin_unlock(&ptype_lock
);
453 EXPORT_SYMBOL(__dev_remove_pack
);
456 * dev_remove_pack - remove packet handler
457 * @pt: packet type declaration
459 * Remove a protocol handler that was previously added to the kernel
460 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
461 * from the kernel lists and can be freed or reused once this function
464 * This call sleeps to guarantee that no CPU is looking at the packet
467 void dev_remove_pack(struct packet_type
*pt
)
469 __dev_remove_pack(pt
);
473 EXPORT_SYMBOL(dev_remove_pack
);
475 /******************************************************************************
477 Device Boot-time Settings Routines
479 *******************************************************************************/
481 /* Boot time configuration table */
482 static struct netdev_boot_setup dev_boot_setup
[NETDEV_BOOT_SETUP_MAX
];
485 * netdev_boot_setup_add - add new setup entry
486 * @name: name of the device
487 * @map: configured settings for the device
489 * Adds new setup entry to the dev_boot_setup list. The function
490 * returns 0 on error and 1 on success. This is a generic routine to
493 static int netdev_boot_setup_add(char *name
, struct ifmap
*map
)
495 struct netdev_boot_setup
*s
;
499 for (i
= 0; i
< NETDEV_BOOT_SETUP_MAX
; i
++) {
500 if (s
[i
].name
[0] == '\0' || s
[i
].name
[0] == ' ') {
501 memset(s
[i
].name
, 0, sizeof(s
[i
].name
));
502 strlcpy(s
[i
].name
, name
, IFNAMSIZ
);
503 memcpy(&s
[i
].map
, map
, sizeof(s
[i
].map
));
508 return i
>= NETDEV_BOOT_SETUP_MAX
? 0 : 1;
512 * netdev_boot_setup_check - check boot time settings
513 * @dev: the netdevice
515 * Check boot time settings for the device.
516 * The found settings are set for the device to be used
517 * later in the device probing.
518 * Returns 0 if no settings found, 1 if they are.
520 int netdev_boot_setup_check(struct net_device
*dev
)
522 struct netdev_boot_setup
*s
= dev_boot_setup
;
525 for (i
= 0; i
< NETDEV_BOOT_SETUP_MAX
; i
++) {
526 if (s
[i
].name
[0] != '\0' && s
[i
].name
[0] != ' ' &&
527 !strcmp(dev
->name
, s
[i
].name
)) {
528 dev
->irq
= s
[i
].map
.irq
;
529 dev
->base_addr
= s
[i
].map
.base_addr
;
530 dev
->mem_start
= s
[i
].map
.mem_start
;
531 dev
->mem_end
= s
[i
].map
.mem_end
;
537 EXPORT_SYMBOL(netdev_boot_setup_check
);
541 * netdev_boot_base - get address from boot time settings
542 * @prefix: prefix for network device
543 * @unit: id for network device
545 * Check boot time settings for the base address of device.
546 * The found settings are set for the device to be used
547 * later in the device probing.
548 * Returns 0 if no settings found.
550 unsigned long netdev_boot_base(const char *prefix
, int unit
)
552 const struct netdev_boot_setup
*s
= dev_boot_setup
;
556 sprintf(name
, "%s%d", prefix
, unit
);
559 * If device already registered then return base of 1
560 * to indicate not to probe for this interface
562 if (__dev_get_by_name(&init_net
, name
))
565 for (i
= 0; i
< NETDEV_BOOT_SETUP_MAX
; i
++)
566 if (!strcmp(name
, s
[i
].name
))
567 return s
[i
].map
.base_addr
;
572 * Saves at boot time configured settings for any netdevice.
574 int __init
netdev_boot_setup(char *str
)
579 str
= get_options(str
, ARRAY_SIZE(ints
), ints
);
584 memset(&map
, 0, sizeof(map
));
588 map
.base_addr
= ints
[2];
590 map
.mem_start
= ints
[3];
592 map
.mem_end
= ints
[4];
594 /* Add new entry to the list */
595 return netdev_boot_setup_add(str
, &map
);
598 __setup("netdev=", netdev_boot_setup
);
600 /*******************************************************************************
602 Device Interface Subroutines
604 *******************************************************************************/
607 * __dev_get_by_name - find a device by its name
608 * @net: the applicable net namespace
609 * @name: name to find
611 * Find an interface by name. Must be called under RTNL semaphore
612 * or @dev_base_lock. If the name is found a pointer to the device
613 * is returned. If the name is not found then %NULL is returned. The
614 * reference counters are not incremented so the caller must be
615 * careful with locks.
618 struct net_device
*__dev_get_by_name(struct net
*net
, const char *name
)
620 struct hlist_node
*p
;
621 struct net_device
*dev
;
622 struct hlist_head
*head
= dev_name_hash(net
, name
);
624 hlist_for_each_entry(dev
, p
, head
, name_hlist
)
625 if (!strncmp(dev
->name
, name
, IFNAMSIZ
))
630 EXPORT_SYMBOL(__dev_get_by_name
);
633 * dev_get_by_name_rcu - find a device by its name
634 * @net: the applicable net namespace
635 * @name: name to find
637 * Find an interface by name.
638 * If the name is found a pointer to the device is returned.
639 * If the name is not found then %NULL is returned.
640 * The reference counters are not incremented so the caller must be
641 * careful with locks. The caller must hold RCU lock.
644 struct net_device
*dev_get_by_name_rcu(struct net
*net
, const char *name
)
646 struct hlist_node
*p
;
647 struct net_device
*dev
;
648 struct hlist_head
*head
= dev_name_hash(net
, name
);
650 hlist_for_each_entry_rcu(dev
, p
, head
, name_hlist
)
651 if (!strncmp(dev
->name
, name
, IFNAMSIZ
))
656 EXPORT_SYMBOL(dev_get_by_name_rcu
);
659 * dev_get_by_name - find a device by its name
660 * @net: the applicable net namespace
661 * @name: name to find
663 * Find an interface by name. This can be called from any
664 * context and does its own locking. The returned handle has
665 * the usage count incremented and the caller must use dev_put() to
666 * release it when it is no longer needed. %NULL is returned if no
667 * matching device is found.
670 struct net_device
*dev_get_by_name(struct net
*net
, const char *name
)
672 struct net_device
*dev
;
675 dev
= dev_get_by_name_rcu(net
, name
);
681 EXPORT_SYMBOL(dev_get_by_name
);
684 * __dev_get_by_index - find a device by its ifindex
685 * @net: the applicable net namespace
686 * @ifindex: index of device
688 * Search for an interface by index. Returns %NULL if the device
689 * is not found or a pointer to the device. The device has not
690 * had its reference counter increased so the caller must be careful
691 * about locking. The caller must hold either the RTNL semaphore
695 struct net_device
*__dev_get_by_index(struct net
*net
, int ifindex
)
697 struct hlist_node
*p
;
698 struct net_device
*dev
;
699 struct hlist_head
*head
= dev_index_hash(net
, ifindex
);
701 hlist_for_each_entry(dev
, p
, head
, index_hlist
)
702 if (dev
->ifindex
== ifindex
)
707 EXPORT_SYMBOL(__dev_get_by_index
);
710 * dev_get_by_index_rcu - find a device by its ifindex
711 * @net: the applicable net namespace
712 * @ifindex: index of device
714 * Search for an interface by index. Returns %NULL if the device
715 * is not found or a pointer to the device. The device has not
716 * had its reference counter increased so the caller must be careful
717 * about locking. The caller must hold RCU lock.
720 struct net_device
*dev_get_by_index_rcu(struct net
*net
, int ifindex
)
722 struct hlist_node
*p
;
723 struct net_device
*dev
;
724 struct hlist_head
*head
= dev_index_hash(net
, ifindex
);
726 hlist_for_each_entry_rcu(dev
, p
, head
, index_hlist
)
727 if (dev
->ifindex
== ifindex
)
732 EXPORT_SYMBOL(dev_get_by_index_rcu
);
736 * dev_get_by_index - find a device by its ifindex
737 * @net: the applicable net namespace
738 * @ifindex: index of device
740 * Search for an interface by index. Returns NULL if the device
741 * is not found or a pointer to the device. The device returned has
742 * had a reference added and the pointer is safe until the user calls
743 * dev_put to indicate they have finished with it.
746 struct net_device
*dev_get_by_index(struct net
*net
, int ifindex
)
748 struct net_device
*dev
;
751 dev
= dev_get_by_index_rcu(net
, ifindex
);
757 EXPORT_SYMBOL(dev_get_by_index
);
760 * dev_getbyhwaddr_rcu - find a device by its hardware address
761 * @net: the applicable net namespace
762 * @type: media type of device
763 * @ha: hardware address
765 * Search for an interface by MAC address. Returns NULL if the device
766 * is not found or a pointer to the device.
767 * The caller must hold RCU or RTNL.
768 * The returned device has not had its ref count increased
769 * and the caller must therefore be careful about locking
773 struct net_device
*dev_getbyhwaddr_rcu(struct net
*net
, unsigned short type
,
776 struct net_device
*dev
;
778 for_each_netdev_rcu(net
, dev
)
779 if (dev
->type
== type
&&
780 !memcmp(dev
->dev_addr
, ha
, dev
->addr_len
))
785 EXPORT_SYMBOL(dev_getbyhwaddr_rcu
);
787 struct net_device
*__dev_getfirstbyhwtype(struct net
*net
, unsigned short type
)
789 struct net_device
*dev
;
792 for_each_netdev(net
, dev
)
793 if (dev
->type
== type
)
798 EXPORT_SYMBOL(__dev_getfirstbyhwtype
);
800 struct net_device
*dev_getfirstbyhwtype(struct net
*net
, unsigned short type
)
802 struct net_device
*dev
, *ret
= NULL
;
805 for_each_netdev_rcu(net
, dev
)
806 if (dev
->type
== type
) {
814 EXPORT_SYMBOL(dev_getfirstbyhwtype
);
817 * dev_get_by_flags_rcu - find any device with given flags
818 * @net: the applicable net namespace
819 * @if_flags: IFF_* values
820 * @mask: bitmask of bits in if_flags to check
822 * Search for any interface with the given flags. Returns NULL if a device
823 * is not found or a pointer to the device. Must be called inside
824 * rcu_read_lock(), and result refcount is unchanged.
827 struct net_device
*dev_get_by_flags_rcu(struct net
*net
, unsigned short if_flags
,
830 struct net_device
*dev
, *ret
;
833 for_each_netdev_rcu(net
, dev
) {
834 if (((dev
->flags
^ if_flags
) & mask
) == 0) {
841 EXPORT_SYMBOL(dev_get_by_flags_rcu
);
844 * dev_valid_name - check if name is okay for network device
847 * Network device names need to be valid file names to
848 * to allow sysfs to work. We also disallow any kind of
851 int dev_valid_name(const char *name
)
855 if (strlen(name
) >= IFNAMSIZ
)
857 if (!strcmp(name
, ".") || !strcmp(name
, ".."))
861 if (*name
== '/' || isspace(*name
))
867 EXPORT_SYMBOL(dev_valid_name
);
870 * __dev_alloc_name - allocate a name for a device
871 * @net: network namespace to allocate the device name in
872 * @name: name format string
873 * @buf: scratch buffer and result name string
875 * Passed a format string - eg "lt%d" it will try and find a suitable
876 * id. It scans list of devices to build up a free map, then chooses
877 * the first empty slot. The caller must hold the dev_base or rtnl lock
878 * while allocating the name and adding the device in order to avoid
880 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
881 * Returns the number of the unit assigned or a negative errno code.
884 static int __dev_alloc_name(struct net
*net
, const char *name
, char *buf
)
888 const int max_netdevices
= 8*PAGE_SIZE
;
889 unsigned long *inuse
;
890 struct net_device
*d
;
892 p
= strnchr(name
, IFNAMSIZ
-1, '%');
895 * Verify the string as this thing may have come from
896 * the user. There must be either one "%d" and no other "%"
899 if (p
[1] != 'd' || strchr(p
+ 2, '%'))
902 /* Use one page as a bit array of possible slots */
903 inuse
= (unsigned long *) get_zeroed_page(GFP_ATOMIC
);
907 for_each_netdev(net
, d
) {
908 if (!sscanf(d
->name
, name
, &i
))
910 if (i
< 0 || i
>= max_netdevices
)
913 /* avoid cases where sscanf is not exact inverse of printf */
914 snprintf(buf
, IFNAMSIZ
, name
, i
);
915 if (!strncmp(buf
, d
->name
, IFNAMSIZ
))
919 i
= find_first_zero_bit(inuse
, max_netdevices
);
920 free_page((unsigned long) inuse
);
924 snprintf(buf
, IFNAMSIZ
, name
, i
);
925 if (!__dev_get_by_name(net
, buf
))
928 /* It is possible to run out of possible slots
929 * when the name is long and there isn't enough space left
930 * for the digits, or if all bits are used.
936 * dev_alloc_name - allocate a name for a device
938 * @name: name format string
940 * Passed a format string - eg "lt%d" it will try and find a suitable
941 * id. It scans list of devices to build up a free map, then chooses
942 * the first empty slot. The caller must hold the dev_base or rtnl lock
943 * while allocating the name and adding the device in order to avoid
945 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
946 * Returns the number of the unit assigned or a negative errno code.
949 int dev_alloc_name(struct net_device
*dev
, const char *name
)
955 BUG_ON(!dev_net(dev
));
957 ret
= __dev_alloc_name(net
, name
, buf
);
959 strlcpy(dev
->name
, buf
, IFNAMSIZ
);
962 EXPORT_SYMBOL(dev_alloc_name
);
964 static int dev_get_valid_name(struct net_device
*dev
, const char *name
)
968 BUG_ON(!dev_net(dev
));
971 if (!dev_valid_name(name
))
974 if (strchr(name
, '%'))
975 return dev_alloc_name(dev
, name
);
976 else if (__dev_get_by_name(net
, name
))
978 else if (dev
->name
!= name
)
979 strlcpy(dev
->name
, name
, IFNAMSIZ
);
985 * dev_change_name - change name of a device
987 * @newname: name (or format string) must be at least IFNAMSIZ
989 * Change name of a device, can pass format strings "eth%d".
992 int dev_change_name(struct net_device
*dev
, const char *newname
)
994 char oldname
[IFNAMSIZ
];
1000 BUG_ON(!dev_net(dev
));
1003 if (dev
->flags
& IFF_UP
)
1006 if (strncmp(newname
, dev
->name
, IFNAMSIZ
) == 0)
1009 memcpy(oldname
, dev
->name
, IFNAMSIZ
);
1011 err
= dev_get_valid_name(dev
, newname
);
1016 ret
= device_rename(&dev
->dev
, dev
->name
);
1018 memcpy(dev
->name
, oldname
, IFNAMSIZ
);
1022 write_lock_bh(&dev_base_lock
);
1023 hlist_del_rcu(&dev
->name_hlist
);
1024 write_unlock_bh(&dev_base_lock
);
1028 write_lock_bh(&dev_base_lock
);
1029 hlist_add_head_rcu(&dev
->name_hlist
, dev_name_hash(net
, dev
->name
));
1030 write_unlock_bh(&dev_base_lock
);
1032 ret
= call_netdevice_notifiers(NETDEV_CHANGENAME
, dev
);
1033 ret
= notifier_to_errno(ret
);
1036 /* err >= 0 after dev_alloc_name() or stores the first errno */
1039 memcpy(dev
->name
, oldname
, IFNAMSIZ
);
1043 "%s: name change rollback failed: %d.\n",
1052 * dev_set_alias - change ifalias of a device
1054 * @alias: name up to IFALIASZ
1055 * @len: limit of bytes to copy from info
1057 * Set ifalias for a device,
1059 int dev_set_alias(struct net_device
*dev
, const char *alias
, size_t len
)
1063 if (len
>= IFALIASZ
)
1068 kfree(dev
->ifalias
);
1069 dev
->ifalias
= NULL
;
1074 dev
->ifalias
= krealloc(dev
->ifalias
, len
+ 1, GFP_KERNEL
);
1078 strlcpy(dev
->ifalias
, alias
, len
+1);
1084 * netdev_features_change - device changes features
1085 * @dev: device to cause notification
1087 * Called to indicate a device has changed features.
1089 void netdev_features_change(struct net_device
*dev
)
1091 call_netdevice_notifiers(NETDEV_FEAT_CHANGE
, dev
);
1093 EXPORT_SYMBOL(netdev_features_change
);
1096 * netdev_state_change - device changes state
1097 * @dev: device to cause notification
1099 * Called to indicate a device has changed state. This function calls
1100 * the notifier chains for netdev_chain and sends a NEWLINK message
1101 * to the routing socket.
1103 void netdev_state_change(struct net_device
*dev
)
1105 if (dev
->flags
& IFF_UP
) {
1106 call_netdevice_notifiers(NETDEV_CHANGE
, dev
);
1107 rtmsg_ifinfo(RTM_NEWLINK
, dev
, 0);
1110 EXPORT_SYMBOL(netdev_state_change
);
1112 int netdev_bonding_change(struct net_device
*dev
, unsigned long event
)
1114 return call_netdevice_notifiers(event
, dev
);
1116 EXPORT_SYMBOL(netdev_bonding_change
);
1119 * dev_load - load a network module
1120 * @net: the applicable net namespace
1121 * @name: name of interface
1123 * If a network interface is not present and the process has suitable
1124 * privileges this function loads the module. If module loading is not
1125 * available in this kernel then it becomes a nop.
1128 void dev_load(struct net
*net
, const char *name
)
1130 struct net_device
*dev
;
1134 dev
= dev_get_by_name_rcu(net
, name
);
1138 if (no_module
&& capable(CAP_NET_ADMIN
))
1139 no_module
= request_module("netdev-%s", name
);
1140 if (no_module
&& capable(CAP_SYS_MODULE
)) {
1141 if (!request_module("%s", name
))
1142 pr_err("Loading kernel module for a network device "
1143 "with CAP_SYS_MODULE (deprecated). Use CAP_NET_ADMIN and alias netdev-%s "
1147 EXPORT_SYMBOL(dev_load
);
1149 static int __dev_open(struct net_device
*dev
)
1151 const struct net_device_ops
*ops
= dev
->netdev_ops
;
1156 if (!netif_device_present(dev
))
1159 ret
= call_netdevice_notifiers(NETDEV_PRE_UP
, dev
);
1160 ret
= notifier_to_errno(ret
);
1164 set_bit(__LINK_STATE_START
, &dev
->state
);
1166 if (ops
->ndo_validate_addr
)
1167 ret
= ops
->ndo_validate_addr(dev
);
1169 if (!ret
&& ops
->ndo_open
)
1170 ret
= ops
->ndo_open(dev
);
1173 clear_bit(__LINK_STATE_START
, &dev
->state
);
1175 dev
->flags
|= IFF_UP
;
1176 net_dmaengine_get();
1177 dev_set_rx_mode(dev
);
1185 * dev_open - prepare an interface for use.
1186 * @dev: device to open
1188 * Takes a device from down to up state. The device's private open
1189 * function is invoked and then the multicast lists are loaded. Finally
1190 * the device is moved into the up state and a %NETDEV_UP message is
1191 * sent to the netdev notifier chain.
1193 * Calling this function on an active interface is a nop. On a failure
1194 * a negative errno code is returned.
1196 int dev_open(struct net_device
*dev
)
1200 if (dev
->flags
& IFF_UP
)
1203 ret
= __dev_open(dev
);
1207 rtmsg_ifinfo(RTM_NEWLINK
, dev
, IFF_UP
|IFF_RUNNING
);
1208 call_netdevice_notifiers(NETDEV_UP
, dev
);
1212 EXPORT_SYMBOL(dev_open
);
1214 static int __dev_close_many(struct list_head
*head
)
1216 struct net_device
*dev
;
1221 list_for_each_entry(dev
, head
, unreg_list
) {
1222 call_netdevice_notifiers(NETDEV_GOING_DOWN
, dev
);
1224 clear_bit(__LINK_STATE_START
, &dev
->state
);
1226 /* Synchronize to scheduled poll. We cannot touch poll list, it
1227 * can be even on different cpu. So just clear netif_running().
1229 * dev->stop() will invoke napi_disable() on all of it's
1230 * napi_struct instances on this device.
1232 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1235 dev_deactivate_many(head
);
1237 list_for_each_entry(dev
, head
, unreg_list
) {
1238 const struct net_device_ops
*ops
= dev
->netdev_ops
;
1241 * Call the device specific close. This cannot fail.
1242 * Only if device is UP
1244 * We allow it to be called even after a DETACH hot-plug
1250 dev
->flags
&= ~IFF_UP
;
1251 net_dmaengine_put();
1257 static int __dev_close(struct net_device
*dev
)
1262 list_add(&dev
->unreg_list
, &single
);
1263 retval
= __dev_close_many(&single
);
1268 static int dev_close_many(struct list_head
*head
)
1270 struct net_device
*dev
, *tmp
;
1271 LIST_HEAD(tmp_list
);
1273 list_for_each_entry_safe(dev
, tmp
, head
, unreg_list
)
1274 if (!(dev
->flags
& IFF_UP
))
1275 list_move(&dev
->unreg_list
, &tmp_list
);
1277 __dev_close_many(head
);
1279 list_for_each_entry(dev
, head
, unreg_list
) {
1280 rtmsg_ifinfo(RTM_NEWLINK
, dev
, IFF_UP
|IFF_RUNNING
);
1281 call_netdevice_notifiers(NETDEV_DOWN
, dev
);
1284 /* rollback_registered_many needs the complete original list */
1285 list_splice(&tmp_list
, head
);
1290 * dev_close - shutdown an interface.
1291 * @dev: device to shutdown
1293 * This function moves an active device into down state. A
1294 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1295 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1298 int dev_close(struct net_device
*dev
)
1300 if (dev
->flags
& IFF_UP
) {
1303 list_add(&dev
->unreg_list
, &single
);
1304 dev_close_many(&single
);
1309 EXPORT_SYMBOL(dev_close
);
1313 * dev_disable_lro - disable Large Receive Offload on a device
1316 * Disable Large Receive Offload (LRO) on a net device. Must be
1317 * called under RTNL. This is needed if received packets may be
1318 * forwarded to another interface.
1320 void dev_disable_lro(struct net_device
*dev
)
1325 * If we're trying to disable lro on a vlan device
1326 * use the underlying physical device instead
1328 if (is_vlan_dev(dev
))
1329 dev
= vlan_dev_real_dev(dev
);
1331 if (dev
->ethtool_ops
&& dev
->ethtool_ops
->get_flags
)
1332 flags
= dev
->ethtool_ops
->get_flags(dev
);
1334 flags
= ethtool_op_get_flags(dev
);
1336 if (!(flags
& ETH_FLAG_LRO
))
1339 __ethtool_set_flags(dev
, flags
& ~ETH_FLAG_LRO
);
1340 if (unlikely(dev
->features
& NETIF_F_LRO
))
1341 netdev_WARN(dev
, "failed to disable LRO!\n");
1343 EXPORT_SYMBOL(dev_disable_lro
);
1346 static int dev_boot_phase
= 1;
1349 * register_netdevice_notifier - register a network notifier block
1352 * Register a notifier to be called when network device events occur.
1353 * The notifier passed is linked into the kernel structures and must
1354 * not be reused until it has been unregistered. A negative errno code
1355 * is returned on a failure.
1357 * When registered all registration and up events are replayed
1358 * to the new notifier to allow device to have a race free
1359 * view of the network device list.
1362 int register_netdevice_notifier(struct notifier_block
*nb
)
1364 struct net_device
*dev
;
1365 struct net_device
*last
;
1370 err
= raw_notifier_chain_register(&netdev_chain
, nb
);
1376 for_each_netdev(net
, dev
) {
1377 err
= nb
->notifier_call(nb
, NETDEV_REGISTER
, dev
);
1378 err
= notifier_to_errno(err
);
1382 if (!(dev
->flags
& IFF_UP
))
1385 nb
->notifier_call(nb
, NETDEV_UP
, dev
);
1396 for_each_netdev(net
, dev
) {
1400 if (dev
->flags
& IFF_UP
) {
1401 nb
->notifier_call(nb
, NETDEV_GOING_DOWN
, dev
);
1402 nb
->notifier_call(nb
, NETDEV_DOWN
, dev
);
1404 nb
->notifier_call(nb
, NETDEV_UNREGISTER
, dev
);
1405 nb
->notifier_call(nb
, NETDEV_UNREGISTER_BATCH
, dev
);
1409 raw_notifier_chain_unregister(&netdev_chain
, nb
);
1412 EXPORT_SYMBOL(register_netdevice_notifier
);
1415 * unregister_netdevice_notifier - unregister a network notifier block
1418 * Unregister a notifier previously registered by
1419 * register_netdevice_notifier(). The notifier is unlinked into the
1420 * kernel structures and may then be reused. A negative errno code
1421 * is returned on a failure.
1424 int unregister_netdevice_notifier(struct notifier_block
*nb
)
1429 err
= raw_notifier_chain_unregister(&netdev_chain
, nb
);
1433 EXPORT_SYMBOL(unregister_netdevice_notifier
);
1436 * call_netdevice_notifiers - call all network notifier blocks
1437 * @val: value passed unmodified to notifier function
1438 * @dev: net_device pointer passed unmodified to notifier function
1440 * Call all network notifier blocks. Parameters and return value
1441 * are as for raw_notifier_call_chain().
1444 int call_netdevice_notifiers(unsigned long val
, struct net_device
*dev
)
1447 return raw_notifier_call_chain(&netdev_chain
, val
, dev
);
1449 EXPORT_SYMBOL(call_netdevice_notifiers
);
1451 /* When > 0 there are consumers of rx skb time stamps */
1452 static atomic_t netstamp_needed
= ATOMIC_INIT(0);
1454 void net_enable_timestamp(void)
1456 atomic_inc(&netstamp_needed
);
1458 EXPORT_SYMBOL(net_enable_timestamp
);
1460 void net_disable_timestamp(void)
1462 atomic_dec(&netstamp_needed
);
1464 EXPORT_SYMBOL(net_disable_timestamp
);
1466 static inline void net_timestamp_set(struct sk_buff
*skb
)
1468 if (atomic_read(&netstamp_needed
))
1469 __net_timestamp(skb
);
1471 skb
->tstamp
.tv64
= 0;
1474 static inline void net_timestamp_check(struct sk_buff
*skb
)
1476 if (!skb
->tstamp
.tv64
&& atomic_read(&netstamp_needed
))
1477 __net_timestamp(skb
);
1480 static inline bool is_skb_forwardable(struct net_device
*dev
,
1481 struct sk_buff
*skb
)
1485 if (!(dev
->flags
& IFF_UP
))
1488 len
= dev
->mtu
+ dev
->hard_header_len
+ VLAN_HLEN
;
1489 if (skb
->len
<= len
)
1492 /* if TSO is enabled, we don't care about the length as the packet
1493 * could be forwarded without being segmented before
1495 if (skb_is_gso(skb
))
1502 * dev_forward_skb - loopback an skb to another netif
1504 * @dev: destination network device
1505 * @skb: buffer to forward
1508 * NET_RX_SUCCESS (no congestion)
1509 * NET_RX_DROP (packet was dropped, but freed)
1511 * dev_forward_skb can be used for injecting an skb from the
1512 * start_xmit function of one device into the receive queue
1513 * of another device.
1515 * The receiving device may be in another namespace, so
1516 * we have to clear all information in the skb that could
1517 * impact namespace isolation.
1519 int dev_forward_skb(struct net_device
*dev
, struct sk_buff
*skb
)
1521 if (skb_shinfo(skb
)->tx_flags
& SKBTX_DEV_ZEROCOPY
) {
1522 if (skb_copy_ubufs(skb
, GFP_ATOMIC
)) {
1523 atomic_long_inc(&dev
->rx_dropped
);
1532 if (unlikely(!is_skb_forwardable(dev
, skb
))) {
1533 atomic_long_inc(&dev
->rx_dropped
);
1537 skb_set_dev(skb
, dev
);
1538 skb
->tstamp
.tv64
= 0;
1539 skb
->pkt_type
= PACKET_HOST
;
1540 skb
->protocol
= eth_type_trans(skb
, dev
);
1541 return netif_rx(skb
);
1543 EXPORT_SYMBOL_GPL(dev_forward_skb
);
1545 static inline int deliver_skb(struct sk_buff
*skb
,
1546 struct packet_type
*pt_prev
,
1547 struct net_device
*orig_dev
)
1549 atomic_inc(&skb
->users
);
1550 return pt_prev
->func(skb
, skb
->dev
, pt_prev
, orig_dev
);
1554 * Support routine. Sends outgoing frames to any network
1555 * taps currently in use.
1558 static void dev_queue_xmit_nit(struct sk_buff
*skb
, struct net_device
*dev
)
1560 struct packet_type
*ptype
;
1561 struct sk_buff
*skb2
= NULL
;
1562 struct packet_type
*pt_prev
= NULL
;
1565 list_for_each_entry_rcu(ptype
, &ptype_all
, list
) {
1566 /* Never send packets back to the socket
1567 * they originated from - MvS (miquels@drinkel.ow.org)
1569 if ((ptype
->dev
== dev
|| !ptype
->dev
) &&
1570 (ptype
->af_packet_priv
== NULL
||
1571 (struct sock
*)ptype
->af_packet_priv
!= skb
->sk
)) {
1573 deliver_skb(skb2
, pt_prev
, skb
->dev
);
1578 skb2
= skb_clone(skb
, GFP_ATOMIC
);
1582 net_timestamp_set(skb2
);
1584 /* skb->nh should be correctly
1585 set by sender, so that the second statement is
1586 just protection against buggy protocols.
1588 skb_reset_mac_header(skb2
);
1590 if (skb_network_header(skb2
) < skb2
->data
||
1591 skb2
->network_header
> skb2
->tail
) {
1592 if (net_ratelimit())
1593 printk(KERN_CRIT
"protocol %04x is "
1595 ntohs(skb2
->protocol
),
1597 skb_reset_network_header(skb2
);
1600 skb2
->transport_header
= skb2
->network_header
;
1601 skb2
->pkt_type
= PACKET_OUTGOING
;
1606 pt_prev
->func(skb2
, skb
->dev
, pt_prev
, skb
->dev
);
1610 /* netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1611 * @dev: Network device
1612 * @txq: number of queues available
1614 * If real_num_tx_queues is changed the tc mappings may no longer be
1615 * valid. To resolve this verify the tc mapping remains valid and if
1616 * not NULL the mapping. With no priorities mapping to this
1617 * offset/count pair it will no longer be used. In the worst case TC0
1618 * is invalid nothing can be done so disable priority mappings. If is
1619 * expected that drivers will fix this mapping if they can before
1620 * calling netif_set_real_num_tx_queues.
1622 static void netif_setup_tc(struct net_device
*dev
, unsigned int txq
)
1625 struct netdev_tc_txq
*tc
= &dev
->tc_to_txq
[0];
1627 /* If TC0 is invalidated disable TC mapping */
1628 if (tc
->offset
+ tc
->count
> txq
) {
1629 pr_warning("Number of in use tx queues changed "
1630 "invalidating tc mappings. Priority "
1631 "traffic classification disabled!\n");
1636 /* Invalidated prio to tc mappings set to TC0 */
1637 for (i
= 1; i
< TC_BITMASK
+ 1; i
++) {
1638 int q
= netdev_get_prio_tc_map(dev
, i
);
1640 tc
= &dev
->tc_to_txq
[q
];
1641 if (tc
->offset
+ tc
->count
> txq
) {
1642 pr_warning("Number of in use tx queues "
1643 "changed. Priority %i to tc "
1644 "mapping %i is no longer valid "
1645 "setting map to 0\n",
1647 netdev_set_prio_tc_map(dev
, i
, 0);
1653 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1654 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1656 int netif_set_real_num_tx_queues(struct net_device
*dev
, unsigned int txq
)
1660 if (txq
< 1 || txq
> dev
->num_tx_queues
)
1663 if (dev
->reg_state
== NETREG_REGISTERED
||
1664 dev
->reg_state
== NETREG_UNREGISTERING
) {
1667 rc
= netdev_queue_update_kobjects(dev
, dev
->real_num_tx_queues
,
1673 netif_setup_tc(dev
, txq
);
1675 if (txq
< dev
->real_num_tx_queues
)
1676 qdisc_reset_all_tx_gt(dev
, txq
);
1679 dev
->real_num_tx_queues
= txq
;
1682 EXPORT_SYMBOL(netif_set_real_num_tx_queues
);
1686 * netif_set_real_num_rx_queues - set actual number of RX queues used
1687 * @dev: Network device
1688 * @rxq: Actual number of RX queues
1690 * This must be called either with the rtnl_lock held or before
1691 * registration of the net device. Returns 0 on success, or a
1692 * negative error code. If called before registration, it always
1695 int netif_set_real_num_rx_queues(struct net_device
*dev
, unsigned int rxq
)
1699 if (rxq
< 1 || rxq
> dev
->num_rx_queues
)
1702 if (dev
->reg_state
== NETREG_REGISTERED
) {
1705 rc
= net_rx_queue_update_kobjects(dev
, dev
->real_num_rx_queues
,
1711 dev
->real_num_rx_queues
= rxq
;
1714 EXPORT_SYMBOL(netif_set_real_num_rx_queues
);
1717 static inline void __netif_reschedule(struct Qdisc
*q
)
1719 struct softnet_data
*sd
;
1720 unsigned long flags
;
1722 local_irq_save(flags
);
1723 sd
= &__get_cpu_var(softnet_data
);
1724 q
->next_sched
= NULL
;
1725 *sd
->output_queue_tailp
= q
;
1726 sd
->output_queue_tailp
= &q
->next_sched
;
1727 raise_softirq_irqoff(NET_TX_SOFTIRQ
);
1728 local_irq_restore(flags
);
1731 void __netif_schedule(struct Qdisc
*q
)
1733 if (!test_and_set_bit(__QDISC_STATE_SCHED
, &q
->state
))
1734 __netif_reschedule(q
);
1736 EXPORT_SYMBOL(__netif_schedule
);
1738 void dev_kfree_skb_irq(struct sk_buff
*skb
)
1740 if (atomic_dec_and_test(&skb
->users
)) {
1741 struct softnet_data
*sd
;
1742 unsigned long flags
;
1744 local_irq_save(flags
);
1745 sd
= &__get_cpu_var(softnet_data
);
1746 skb
->next
= sd
->completion_queue
;
1747 sd
->completion_queue
= skb
;
1748 raise_softirq_irqoff(NET_TX_SOFTIRQ
);
1749 local_irq_restore(flags
);
1752 EXPORT_SYMBOL(dev_kfree_skb_irq
);
1754 void dev_kfree_skb_any(struct sk_buff
*skb
)
1756 if (in_irq() || irqs_disabled())
1757 dev_kfree_skb_irq(skb
);
1761 EXPORT_SYMBOL(dev_kfree_skb_any
);
1765 * netif_device_detach - mark device as removed
1766 * @dev: network device
1768 * Mark device as removed from system and therefore no longer available.
1770 void netif_device_detach(struct net_device
*dev
)
1772 if (test_and_clear_bit(__LINK_STATE_PRESENT
, &dev
->state
) &&
1773 netif_running(dev
)) {
1774 netif_tx_stop_all_queues(dev
);
1777 EXPORT_SYMBOL(netif_device_detach
);
1780 * netif_device_attach - mark device as attached
1781 * @dev: network device
1783 * Mark device as attached from system and restart if needed.
1785 void netif_device_attach(struct net_device
*dev
)
1787 if (!test_and_set_bit(__LINK_STATE_PRESENT
, &dev
->state
) &&
1788 netif_running(dev
)) {
1789 netif_tx_wake_all_queues(dev
);
1790 __netdev_watchdog_up(dev
);
1793 EXPORT_SYMBOL(netif_device_attach
);
1796 * skb_dev_set -- assign a new device to a buffer
1797 * @skb: buffer for the new device
1798 * @dev: network device
1800 * If an skb is owned by a device already, we have to reset
1801 * all data private to the namespace a device belongs to
1802 * before assigning it a new device.
1804 #ifdef CONFIG_NET_NS
1805 void skb_set_dev(struct sk_buff
*skb
, struct net_device
*dev
)
1808 if (skb
->dev
&& !net_eq(dev_net(skb
->dev
), dev_net(dev
))) {
1811 skb_init_secmark(skb
);
1815 skb
->ipvs_property
= 0;
1816 #ifdef CONFIG_NET_SCHED
1822 EXPORT_SYMBOL(skb_set_dev
);
1823 #endif /* CONFIG_NET_NS */
1826 * Invalidate hardware checksum when packet is to be mangled, and
1827 * complete checksum manually on outgoing path.
1829 int skb_checksum_help(struct sk_buff
*skb
)
1832 int ret
= 0, offset
;
1834 if (skb
->ip_summed
== CHECKSUM_COMPLETE
)
1835 goto out_set_summed
;
1837 if (unlikely(skb_shinfo(skb
)->gso_size
)) {
1838 /* Let GSO fix up the checksum. */
1839 goto out_set_summed
;
1842 offset
= skb_checksum_start_offset(skb
);
1843 BUG_ON(offset
>= skb_headlen(skb
));
1844 csum
= skb_checksum(skb
, offset
, skb
->len
- offset
, 0);
1846 offset
+= skb
->csum_offset
;
1847 BUG_ON(offset
+ sizeof(__sum16
) > skb_headlen(skb
));
1849 if (skb_cloned(skb
) &&
1850 !skb_clone_writable(skb
, offset
+ sizeof(__sum16
))) {
1851 ret
= pskb_expand_head(skb
, 0, 0, GFP_ATOMIC
);
1856 *(__sum16
*)(skb
->data
+ offset
) = csum_fold(csum
);
1858 skb
->ip_summed
= CHECKSUM_NONE
;
1862 EXPORT_SYMBOL(skb_checksum_help
);
1865 * skb_gso_segment - Perform segmentation on skb.
1866 * @skb: buffer to segment
1867 * @features: features for the output path (see dev->features)
1869 * This function segments the given skb and returns a list of segments.
1871 * It may return NULL if the skb requires no segmentation. This is
1872 * only possible when GSO is used for verifying header integrity.
1874 struct sk_buff
*skb_gso_segment(struct sk_buff
*skb
, u32 features
)
1876 struct sk_buff
*segs
= ERR_PTR(-EPROTONOSUPPORT
);
1877 struct packet_type
*ptype
;
1878 __be16 type
= skb
->protocol
;
1879 int vlan_depth
= ETH_HLEN
;
1882 while (type
== htons(ETH_P_8021Q
)) {
1883 struct vlan_hdr
*vh
;
1885 if (unlikely(!pskb_may_pull(skb
, vlan_depth
+ VLAN_HLEN
)))
1886 return ERR_PTR(-EINVAL
);
1888 vh
= (struct vlan_hdr
*)(skb
->data
+ vlan_depth
);
1889 type
= vh
->h_vlan_encapsulated_proto
;
1890 vlan_depth
+= VLAN_HLEN
;
1893 skb_reset_mac_header(skb
);
1894 skb
->mac_len
= skb
->network_header
- skb
->mac_header
;
1895 __skb_pull(skb
, skb
->mac_len
);
1897 if (unlikely(skb
->ip_summed
!= CHECKSUM_PARTIAL
)) {
1898 struct net_device
*dev
= skb
->dev
;
1899 struct ethtool_drvinfo info
= {};
1901 if (dev
&& dev
->ethtool_ops
&& dev
->ethtool_ops
->get_drvinfo
)
1902 dev
->ethtool_ops
->get_drvinfo(dev
, &info
);
1904 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d ip_summed=%d\n",
1905 info
.driver
, dev
? dev
->features
: 0L,
1906 skb
->sk
? skb
->sk
->sk_route_caps
: 0L,
1907 skb
->len
, skb
->data_len
, skb
->ip_summed
);
1909 if (skb_header_cloned(skb
) &&
1910 (err
= pskb_expand_head(skb
, 0, 0, GFP_ATOMIC
)))
1911 return ERR_PTR(err
);
1915 list_for_each_entry_rcu(ptype
,
1916 &ptype_base
[ntohs(type
) & PTYPE_HASH_MASK
], list
) {
1917 if (ptype
->type
== type
&& !ptype
->dev
&& ptype
->gso_segment
) {
1918 if (unlikely(skb
->ip_summed
!= CHECKSUM_PARTIAL
)) {
1919 err
= ptype
->gso_send_check(skb
);
1920 segs
= ERR_PTR(err
);
1921 if (err
|| skb_gso_ok(skb
, features
))
1923 __skb_push(skb
, (skb
->data
-
1924 skb_network_header(skb
)));
1926 segs
= ptype
->gso_segment(skb
, features
);
1932 __skb_push(skb
, skb
->data
- skb_mac_header(skb
));
1936 EXPORT_SYMBOL(skb_gso_segment
);
1938 /* Take action when hardware reception checksum errors are detected. */
1940 void netdev_rx_csum_fault(struct net_device
*dev
)
1942 if (net_ratelimit()) {
1943 printk(KERN_ERR
"%s: hw csum failure.\n",
1944 dev
? dev
->name
: "<unknown>");
1948 EXPORT_SYMBOL(netdev_rx_csum_fault
);
1951 /* Actually, we should eliminate this check as soon as we know, that:
1952 * 1. IOMMU is present and allows to map all the memory.
1953 * 2. No high memory really exists on this machine.
1956 static int illegal_highdma(struct net_device
*dev
, struct sk_buff
*skb
)
1958 #ifdef CONFIG_HIGHMEM
1960 if (!(dev
->features
& NETIF_F_HIGHDMA
)) {
1961 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
1962 skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
];
1963 if (PageHighMem(skb_frag_page(frag
)))
1968 if (PCI_DMA_BUS_IS_PHYS
) {
1969 struct device
*pdev
= dev
->dev
.parent
;
1973 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
1974 skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
];
1975 dma_addr_t addr
= page_to_phys(skb_frag_page(frag
));
1976 if (!pdev
->dma_mask
|| addr
+ PAGE_SIZE
- 1 > *pdev
->dma_mask
)
1985 void (*destructor
)(struct sk_buff
*skb
);
1988 #define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
1990 static void dev_gso_skb_destructor(struct sk_buff
*skb
)
1992 struct dev_gso_cb
*cb
;
1995 struct sk_buff
*nskb
= skb
->next
;
1997 skb
->next
= nskb
->next
;
2000 } while (skb
->next
);
2002 cb
= DEV_GSO_CB(skb
);
2004 cb
->destructor(skb
);
2008 * dev_gso_segment - Perform emulated hardware segmentation on skb.
2009 * @skb: buffer to segment
2010 * @features: device features as applicable to this skb
2012 * This function segments the given skb and stores the list of segments
2015 static int dev_gso_segment(struct sk_buff
*skb
, int features
)
2017 struct sk_buff
*segs
;
2019 segs
= skb_gso_segment(skb
, features
);
2021 /* Verifying header integrity only. */
2026 return PTR_ERR(segs
);
2029 DEV_GSO_CB(skb
)->destructor
= skb
->destructor
;
2030 skb
->destructor
= dev_gso_skb_destructor
;
2036 * Try to orphan skb early, right before transmission by the device.
2037 * We cannot orphan skb if tx timestamp is requested or the sk-reference
2038 * is needed on driver level for other reasons, e.g. see net/can/raw.c
2040 static inline void skb_orphan_try(struct sk_buff
*skb
)
2042 struct sock
*sk
= skb
->sk
;
2044 if (sk
&& !skb_shinfo(skb
)->tx_flags
) {
2045 /* skb_tx_hash() wont be able to get sk.
2046 * We copy sk_hash into skb->rxhash
2049 skb
->rxhash
= sk
->sk_hash
;
2054 static bool can_checksum_protocol(unsigned long features
, __be16 protocol
)
2056 return ((features
& NETIF_F_GEN_CSUM
) ||
2057 ((features
& NETIF_F_V4_CSUM
) &&
2058 protocol
== htons(ETH_P_IP
)) ||
2059 ((features
& NETIF_F_V6_CSUM
) &&
2060 protocol
== htons(ETH_P_IPV6
)) ||
2061 ((features
& NETIF_F_FCOE_CRC
) &&
2062 protocol
== htons(ETH_P_FCOE
)));
2065 static u32
harmonize_features(struct sk_buff
*skb
, __be16 protocol
, u32 features
)
2067 if (!can_checksum_protocol(features
, protocol
)) {
2068 features
&= ~NETIF_F_ALL_CSUM
;
2069 features
&= ~NETIF_F_SG
;
2070 } else if (illegal_highdma(skb
->dev
, skb
)) {
2071 features
&= ~NETIF_F_SG
;
2077 u32
netif_skb_features(struct sk_buff
*skb
)
2079 __be16 protocol
= skb
->protocol
;
2080 u32 features
= skb
->dev
->features
;
2082 if (protocol
== htons(ETH_P_8021Q
)) {
2083 struct vlan_ethhdr
*veh
= (struct vlan_ethhdr
*)skb
->data
;
2084 protocol
= veh
->h_vlan_encapsulated_proto
;
2085 } else if (!vlan_tx_tag_present(skb
)) {
2086 return harmonize_features(skb
, protocol
, features
);
2089 features
&= (skb
->dev
->vlan_features
| NETIF_F_HW_VLAN_TX
);
2091 if (protocol
!= htons(ETH_P_8021Q
)) {
2092 return harmonize_features(skb
, protocol
, features
);
2094 features
&= NETIF_F_SG
| NETIF_F_HIGHDMA
| NETIF_F_FRAGLIST
|
2095 NETIF_F_GEN_CSUM
| NETIF_F_HW_VLAN_TX
;
2096 return harmonize_features(skb
, protocol
, features
);
2099 EXPORT_SYMBOL(netif_skb_features
);
2102 * Returns true if either:
2103 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
2104 * 2. skb is fragmented and the device does not support SG, or if
2105 * at least one of fragments is in highmem and device does not
2106 * support DMA from it.
2108 static inline int skb_needs_linearize(struct sk_buff
*skb
,
2111 return skb_is_nonlinear(skb
) &&
2112 ((skb_has_frag_list(skb
) &&
2113 !(features
& NETIF_F_FRAGLIST
)) ||
2114 (skb_shinfo(skb
)->nr_frags
&&
2115 !(features
& NETIF_F_SG
)));
2118 int dev_hard_start_xmit(struct sk_buff
*skb
, struct net_device
*dev
,
2119 struct netdev_queue
*txq
)
2121 const struct net_device_ops
*ops
= dev
->netdev_ops
;
2122 int rc
= NETDEV_TX_OK
;
2123 unsigned int skb_len
;
2125 if (likely(!skb
->next
)) {
2129 * If device doesn't need skb->dst, release it right now while
2130 * its hot in this cpu cache
2132 if (dev
->priv_flags
& IFF_XMIT_DST_RELEASE
)
2135 if (!list_empty(&ptype_all
))
2136 dev_queue_xmit_nit(skb
, dev
);
2138 skb_orphan_try(skb
);
2140 features
= netif_skb_features(skb
);
2142 if (vlan_tx_tag_present(skb
) &&
2143 !(features
& NETIF_F_HW_VLAN_TX
)) {
2144 skb
= __vlan_put_tag(skb
, vlan_tx_tag_get(skb
));
2151 if (netif_needs_gso(skb
, features
)) {
2152 if (unlikely(dev_gso_segment(skb
, features
)))
2157 if (skb_needs_linearize(skb
, features
) &&
2158 __skb_linearize(skb
))
2161 /* If packet is not checksummed and device does not
2162 * support checksumming for this protocol, complete
2163 * checksumming here.
2165 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
2166 skb_set_transport_header(skb
,
2167 skb_checksum_start_offset(skb
));
2168 if (!(features
& NETIF_F_ALL_CSUM
) &&
2169 skb_checksum_help(skb
))
2175 rc
= ops
->ndo_start_xmit(skb
, dev
);
2176 trace_net_dev_xmit(skb
, rc
, dev
, skb_len
);
2177 if (rc
== NETDEV_TX_OK
)
2178 txq_trans_update(txq
);
2184 struct sk_buff
*nskb
= skb
->next
;
2186 skb
->next
= nskb
->next
;
2190 * If device doesn't need nskb->dst, release it right now while
2191 * its hot in this cpu cache
2193 if (dev
->priv_flags
& IFF_XMIT_DST_RELEASE
)
2196 skb_len
= nskb
->len
;
2197 rc
= ops
->ndo_start_xmit(nskb
, dev
);
2198 trace_net_dev_xmit(nskb
, rc
, dev
, skb_len
);
2199 if (unlikely(rc
!= NETDEV_TX_OK
)) {
2200 if (rc
& ~NETDEV_TX_MASK
)
2201 goto out_kfree_gso_skb
;
2202 nskb
->next
= skb
->next
;
2206 txq_trans_update(txq
);
2207 if (unlikely(netif_tx_queue_stopped(txq
) && skb
->next
))
2208 return NETDEV_TX_BUSY
;
2209 } while (skb
->next
);
2212 if (likely(skb
->next
== NULL
))
2213 skb
->destructor
= DEV_GSO_CB(skb
)->destructor
;
2220 static u32 hashrnd __read_mostly
;
2223 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2224 * to be used as a distribution range.
2226 u16
__skb_tx_hash(const struct net_device
*dev
, const struct sk_buff
*skb
,
2227 unsigned int num_tx_queues
)
2231 u16 qcount
= num_tx_queues
;
2233 if (skb_rx_queue_recorded(skb
)) {
2234 hash
= skb_get_rx_queue(skb
);
2235 while (unlikely(hash
>= num_tx_queues
))
2236 hash
-= num_tx_queues
;
2241 u8 tc
= netdev_get_prio_tc_map(dev
, skb
->priority
);
2242 qoffset
= dev
->tc_to_txq
[tc
].offset
;
2243 qcount
= dev
->tc_to_txq
[tc
].count
;
2246 if (skb
->sk
&& skb
->sk
->sk_hash
)
2247 hash
= skb
->sk
->sk_hash
;
2249 hash
= (__force u16
) skb
->protocol
^ skb
->rxhash
;
2250 hash
= jhash_1word(hash
, hashrnd
);
2252 return (u16
) (((u64
) hash
* qcount
) >> 32) + qoffset
;
2254 EXPORT_SYMBOL(__skb_tx_hash
);
2256 static inline u16
dev_cap_txqueue(struct net_device
*dev
, u16 queue_index
)
2258 if (unlikely(queue_index
>= dev
->real_num_tx_queues
)) {
2259 if (net_ratelimit()) {
2260 pr_warning("%s selects TX queue %d, but "
2261 "real number of TX queues is %d\n",
2262 dev
->name
, queue_index
, dev
->real_num_tx_queues
);
2269 static inline int get_xps_queue(struct net_device
*dev
, struct sk_buff
*skb
)
2272 struct xps_dev_maps
*dev_maps
;
2273 struct xps_map
*map
;
2274 int queue_index
= -1;
2277 dev_maps
= rcu_dereference(dev
->xps_maps
);
2279 map
= rcu_dereference(
2280 dev_maps
->cpu_map
[raw_smp_processor_id()]);
2283 queue_index
= map
->queues
[0];
2286 if (skb
->sk
&& skb
->sk
->sk_hash
)
2287 hash
= skb
->sk
->sk_hash
;
2289 hash
= (__force u16
) skb
->protocol
^
2291 hash
= jhash_1word(hash
, hashrnd
);
2292 queue_index
= map
->queues
[
2293 ((u64
)hash
* map
->len
) >> 32];
2295 if (unlikely(queue_index
>= dev
->real_num_tx_queues
))
2307 static struct netdev_queue
*dev_pick_tx(struct net_device
*dev
,
2308 struct sk_buff
*skb
)
2311 const struct net_device_ops
*ops
= dev
->netdev_ops
;
2313 if (dev
->real_num_tx_queues
== 1)
2315 else if (ops
->ndo_select_queue
) {
2316 queue_index
= ops
->ndo_select_queue(dev
, skb
);
2317 queue_index
= dev_cap_txqueue(dev
, queue_index
);
2319 struct sock
*sk
= skb
->sk
;
2320 queue_index
= sk_tx_queue_get(sk
);
2322 if (queue_index
< 0 || skb
->ooo_okay
||
2323 queue_index
>= dev
->real_num_tx_queues
) {
2324 int old_index
= queue_index
;
2326 queue_index
= get_xps_queue(dev
, skb
);
2327 if (queue_index
< 0)
2328 queue_index
= skb_tx_hash(dev
, skb
);
2330 if (queue_index
!= old_index
&& sk
) {
2331 struct dst_entry
*dst
=
2332 rcu_dereference_check(sk
->sk_dst_cache
, 1);
2334 if (dst
&& skb_dst(skb
) == dst
)
2335 sk_tx_queue_set(sk
, queue_index
);
2340 skb_set_queue_mapping(skb
, queue_index
);
2341 return netdev_get_tx_queue(dev
, queue_index
);
2344 static inline int __dev_xmit_skb(struct sk_buff
*skb
, struct Qdisc
*q
,
2345 struct net_device
*dev
,
2346 struct netdev_queue
*txq
)
2348 spinlock_t
*root_lock
= qdisc_lock(q
);
2352 qdisc_skb_cb(skb
)->pkt_len
= skb
->len
;
2353 qdisc_calculate_pkt_len(skb
, q
);
2355 * Heuristic to force contended enqueues to serialize on a
2356 * separate lock before trying to get qdisc main lock.
2357 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2358 * and dequeue packets faster.
2360 contended
= qdisc_is_running(q
);
2361 if (unlikely(contended
))
2362 spin_lock(&q
->busylock
);
2364 spin_lock(root_lock
);
2365 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED
, &q
->state
))) {
2368 } else if ((q
->flags
& TCQ_F_CAN_BYPASS
) && !qdisc_qlen(q
) &&
2369 qdisc_run_begin(q
)) {
2371 * This is a work-conserving queue; there are no old skbs
2372 * waiting to be sent out; and the qdisc is not running -
2373 * xmit the skb directly.
2375 if (!(dev
->priv_flags
& IFF_XMIT_DST_RELEASE
))
2378 qdisc_bstats_update(q
, skb
);
2380 if (sch_direct_xmit(skb
, q
, dev
, txq
, root_lock
)) {
2381 if (unlikely(contended
)) {
2382 spin_unlock(&q
->busylock
);
2389 rc
= NET_XMIT_SUCCESS
;
2392 rc
= q
->enqueue(skb
, q
) & NET_XMIT_MASK
;
2393 if (qdisc_run_begin(q
)) {
2394 if (unlikely(contended
)) {
2395 spin_unlock(&q
->busylock
);
2401 spin_unlock(root_lock
);
2402 if (unlikely(contended
))
2403 spin_unlock(&q
->busylock
);
2407 static DEFINE_PER_CPU(int, xmit_recursion
);
2408 #define RECURSION_LIMIT 10
2411 * dev_queue_xmit - transmit a buffer
2412 * @skb: buffer to transmit
2414 * Queue a buffer for transmission to a network device. The caller must
2415 * have set the device and priority and built the buffer before calling
2416 * this function. The function can be called from an interrupt.
2418 * A negative errno code is returned on a failure. A success does not
2419 * guarantee the frame will be transmitted as it may be dropped due
2420 * to congestion or traffic shaping.
2422 * -----------------------------------------------------------------------------------
2423 * I notice this method can also return errors from the queue disciplines,
2424 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2427 * Regardless of the return value, the skb is consumed, so it is currently
2428 * difficult to retry a send to this method. (You can bump the ref count
2429 * before sending to hold a reference for retry if you are careful.)
2431 * When calling this method, interrupts MUST be enabled. This is because
2432 * the BH enable code must have IRQs enabled so that it will not deadlock.
2435 int dev_queue_xmit(struct sk_buff
*skb
)
2437 struct net_device
*dev
= skb
->dev
;
2438 struct netdev_queue
*txq
;
2442 /* Disable soft irqs for various locks below. Also
2443 * stops preemption for RCU.
2447 txq
= dev_pick_tx(dev
, skb
);
2448 q
= rcu_dereference_bh(txq
->qdisc
);
2450 #ifdef CONFIG_NET_CLS_ACT
2451 skb
->tc_verd
= SET_TC_AT(skb
->tc_verd
, AT_EGRESS
);
2453 trace_net_dev_queue(skb
);
2455 rc
= __dev_xmit_skb(skb
, q
, dev
, txq
);
2459 /* The device has no queue. Common case for software devices:
2460 loopback, all the sorts of tunnels...
2462 Really, it is unlikely that netif_tx_lock protection is necessary
2463 here. (f.e. loopback and IP tunnels are clean ignoring statistics
2465 However, it is possible, that they rely on protection
2468 Check this and shot the lock. It is not prone from deadlocks.
2469 Either shot noqueue qdisc, it is even simpler 8)
2471 if (dev
->flags
& IFF_UP
) {
2472 int cpu
= smp_processor_id(); /* ok because BHs are off */
2474 if (txq
->xmit_lock_owner
!= cpu
) {
2476 if (__this_cpu_read(xmit_recursion
) > RECURSION_LIMIT
)
2477 goto recursion_alert
;
2479 HARD_TX_LOCK(dev
, txq
, cpu
);
2481 if (!netif_tx_queue_stopped(txq
)) {
2482 __this_cpu_inc(xmit_recursion
);
2483 rc
= dev_hard_start_xmit(skb
, dev
, txq
);
2484 __this_cpu_dec(xmit_recursion
);
2485 if (dev_xmit_complete(rc
)) {
2486 HARD_TX_UNLOCK(dev
, txq
);
2490 HARD_TX_UNLOCK(dev
, txq
);
2491 if (net_ratelimit())
2492 printk(KERN_CRIT
"Virtual device %s asks to "
2493 "queue packet!\n", dev
->name
);
2495 /* Recursion is detected! It is possible,
2499 if (net_ratelimit())
2500 printk(KERN_CRIT
"Dead loop on virtual device "
2501 "%s, fix it urgently!\n", dev
->name
);
2506 rcu_read_unlock_bh();
2511 rcu_read_unlock_bh();
2514 EXPORT_SYMBOL(dev_queue_xmit
);
2517 /*=======================================================================
2519 =======================================================================*/
2521 int netdev_max_backlog __read_mostly
= 1000;
2522 int netdev_tstamp_prequeue __read_mostly
= 1;
2523 int netdev_budget __read_mostly
= 300;
2524 int weight_p __read_mostly
= 64; /* old backlog weight */
2526 /* Called with irq disabled */
2527 static inline void ____napi_schedule(struct softnet_data
*sd
,
2528 struct napi_struct
*napi
)
2530 list_add_tail(&napi
->poll_list
, &sd
->poll_list
);
2531 __raise_softirq_irqoff(NET_RX_SOFTIRQ
);
2535 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
2536 * and src/dst port numbers. Sets rxhash in skb to non-zero hash value
2537 * on success, zero indicates no valid hash. Also, sets l4_rxhash in skb
2538 * if hash is a canonical 4-tuple hash over transport ports.
2540 void __skb_get_rxhash(struct sk_buff
*skb
)
2542 int nhoff
, hash
= 0, poff
;
2543 const struct ipv6hdr
*ip6
;
2544 const struct iphdr
*ip
;
2545 const struct vlan_hdr
*vlan
;
2554 nhoff
= skb_network_offset(skb
);
2555 proto
= skb
->protocol
;
2559 case __constant_htons(ETH_P_IP
):
2561 if (!pskb_may_pull(skb
, sizeof(*ip
) + nhoff
))
2564 ip
= (const struct iphdr
*) (skb
->data
+ nhoff
);
2565 if (ip_is_fragment(ip
))
2568 ip_proto
= ip
->protocol
;
2569 addr1
= (__force u32
) ip
->saddr
;
2570 addr2
= (__force u32
) ip
->daddr
;
2571 nhoff
+= ip
->ihl
* 4;
2573 case __constant_htons(ETH_P_IPV6
):
2575 if (!pskb_may_pull(skb
, sizeof(*ip6
) + nhoff
))
2578 ip6
= (const struct ipv6hdr
*) (skb
->data
+ nhoff
);
2579 ip_proto
= ip6
->nexthdr
;
2580 addr1
= (__force u32
) ip6
->saddr
.s6_addr32
[3];
2581 addr2
= (__force u32
) ip6
->daddr
.s6_addr32
[3];
2584 case __constant_htons(ETH_P_8021Q
):
2585 if (!pskb_may_pull(skb
, sizeof(*vlan
) + nhoff
))
2587 vlan
= (const struct vlan_hdr
*) (skb
->data
+ nhoff
);
2588 proto
= vlan
->h_vlan_encapsulated_proto
;
2589 nhoff
+= sizeof(*vlan
);
2591 case __constant_htons(ETH_P_PPP_SES
):
2592 if (!pskb_may_pull(skb
, PPPOE_SES_HLEN
+ nhoff
))
2594 proto
= *((__be16
*) (skb
->data
+ nhoff
+
2595 sizeof(struct pppoe_hdr
)));
2596 nhoff
+= PPPOE_SES_HLEN
;
2598 case __constant_htons(PPP_IP
):
2600 case __constant_htons(PPP_IPV6
):
2611 if (pskb_may_pull(skb
, nhoff
+ 16)) {
2612 u8
*h
= skb
->data
+ nhoff
;
2613 __be16 flags
= *(__be16
*)h
;
2616 * Only look inside GRE if version zero and no
2619 if (!(flags
& (GRE_VERSION
|GRE_ROUTING
))) {
2620 proto
= *(__be16
*)(h
+ 2);
2622 if (flags
& GRE_CSUM
)
2624 if (flags
& GRE_KEY
)
2626 if (flags
& GRE_SEQ
)
2639 poff
= proto_ports_offset(ip_proto
);
2642 if (pskb_may_pull(skb
, nhoff
+ 4)) {
2643 ports
.v32
= * (__force u32
*) (skb
->data
+ nhoff
);
2644 if (ports
.v16
[1] < ports
.v16
[0])
2645 swap(ports
.v16
[0], ports
.v16
[1]);
2650 /* get a consistent hash (same value on both flow directions) */
2654 hash
= jhash_3words(addr1
, addr2
, ports
.v32
, hashrnd
);
2661 EXPORT_SYMBOL(__skb_get_rxhash
);
2665 /* One global table that all flow-based protocols share. */
2666 struct rps_sock_flow_table __rcu
*rps_sock_flow_table __read_mostly
;
2667 EXPORT_SYMBOL(rps_sock_flow_table
);
2669 static struct rps_dev_flow
*
2670 set_rps_cpu(struct net_device
*dev
, struct sk_buff
*skb
,
2671 struct rps_dev_flow
*rflow
, u16 next_cpu
)
2675 tcpu
= rflow
->cpu
= next_cpu
;
2676 if (tcpu
!= RPS_NO_CPU
) {
2677 #ifdef CONFIG_RFS_ACCEL
2678 struct netdev_rx_queue
*rxqueue
;
2679 struct rps_dev_flow_table
*flow_table
;
2680 struct rps_dev_flow
*old_rflow
;
2685 /* Should we steer this flow to a different hardware queue? */
2686 if (!skb_rx_queue_recorded(skb
) || !dev
->rx_cpu_rmap
||
2687 !(dev
->features
& NETIF_F_NTUPLE
))
2689 rxq_index
= cpu_rmap_lookup_index(dev
->rx_cpu_rmap
, next_cpu
);
2690 if (rxq_index
== skb_get_rx_queue(skb
))
2693 rxqueue
= dev
->_rx
+ rxq_index
;
2694 flow_table
= rcu_dereference(rxqueue
->rps_flow_table
);
2697 flow_id
= skb
->rxhash
& flow_table
->mask
;
2698 rc
= dev
->netdev_ops
->ndo_rx_flow_steer(dev
, skb
,
2699 rxq_index
, flow_id
);
2703 rflow
= &flow_table
->flows
[flow_id
];
2704 rflow
->cpu
= next_cpu
;
2706 if (old_rflow
->filter
== rflow
->filter
)
2707 old_rflow
->filter
= RPS_NO_FILTER
;
2711 per_cpu(softnet_data
, tcpu
).input_queue_head
;
2718 * get_rps_cpu is called from netif_receive_skb and returns the target
2719 * CPU from the RPS map of the receiving queue for a given skb.
2720 * rcu_read_lock must be held on entry.
2722 static int get_rps_cpu(struct net_device
*dev
, struct sk_buff
*skb
,
2723 struct rps_dev_flow
**rflowp
)
2725 struct netdev_rx_queue
*rxqueue
;
2726 struct rps_map
*map
;
2727 struct rps_dev_flow_table
*flow_table
;
2728 struct rps_sock_flow_table
*sock_flow_table
;
2732 if (skb_rx_queue_recorded(skb
)) {
2733 u16 index
= skb_get_rx_queue(skb
);
2734 if (unlikely(index
>= dev
->real_num_rx_queues
)) {
2735 WARN_ONCE(dev
->real_num_rx_queues
> 1,
2736 "%s received packet on queue %u, but number "
2737 "of RX queues is %u\n",
2738 dev
->name
, index
, dev
->real_num_rx_queues
);
2741 rxqueue
= dev
->_rx
+ index
;
2745 map
= rcu_dereference(rxqueue
->rps_map
);
2747 if (map
->len
== 1 &&
2748 !rcu_access_pointer(rxqueue
->rps_flow_table
)) {
2749 tcpu
= map
->cpus
[0];
2750 if (cpu_online(tcpu
))
2754 } else if (!rcu_access_pointer(rxqueue
->rps_flow_table
)) {
2758 skb_reset_network_header(skb
);
2759 if (!skb_get_rxhash(skb
))
2762 flow_table
= rcu_dereference(rxqueue
->rps_flow_table
);
2763 sock_flow_table
= rcu_dereference(rps_sock_flow_table
);
2764 if (flow_table
&& sock_flow_table
) {
2766 struct rps_dev_flow
*rflow
;
2768 rflow
= &flow_table
->flows
[skb
->rxhash
& flow_table
->mask
];
2771 next_cpu
= sock_flow_table
->ents
[skb
->rxhash
&
2772 sock_flow_table
->mask
];
2775 * If the desired CPU (where last recvmsg was done) is
2776 * different from current CPU (one in the rx-queue flow
2777 * table entry), switch if one of the following holds:
2778 * - Current CPU is unset (equal to RPS_NO_CPU).
2779 * - Current CPU is offline.
2780 * - The current CPU's queue tail has advanced beyond the
2781 * last packet that was enqueued using this table entry.
2782 * This guarantees that all previous packets for the flow
2783 * have been dequeued, thus preserving in order delivery.
2785 if (unlikely(tcpu
!= next_cpu
) &&
2786 (tcpu
== RPS_NO_CPU
|| !cpu_online(tcpu
) ||
2787 ((int)(per_cpu(softnet_data
, tcpu
).input_queue_head
-
2788 rflow
->last_qtail
)) >= 0))
2789 rflow
= set_rps_cpu(dev
, skb
, rflow
, next_cpu
);
2791 if (tcpu
!= RPS_NO_CPU
&& cpu_online(tcpu
)) {
2799 tcpu
= map
->cpus
[((u64
) skb
->rxhash
* map
->len
) >> 32];
2801 if (cpu_online(tcpu
)) {
2811 #ifdef CONFIG_RFS_ACCEL
2814 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
2815 * @dev: Device on which the filter was set
2816 * @rxq_index: RX queue index
2817 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
2818 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
2820 * Drivers that implement ndo_rx_flow_steer() should periodically call
2821 * this function for each installed filter and remove the filters for
2822 * which it returns %true.
2824 bool rps_may_expire_flow(struct net_device
*dev
, u16 rxq_index
,
2825 u32 flow_id
, u16 filter_id
)
2827 struct netdev_rx_queue
*rxqueue
= dev
->_rx
+ rxq_index
;
2828 struct rps_dev_flow_table
*flow_table
;
2829 struct rps_dev_flow
*rflow
;
2834 flow_table
= rcu_dereference(rxqueue
->rps_flow_table
);
2835 if (flow_table
&& flow_id
<= flow_table
->mask
) {
2836 rflow
= &flow_table
->flows
[flow_id
];
2837 cpu
= ACCESS_ONCE(rflow
->cpu
);
2838 if (rflow
->filter
== filter_id
&& cpu
!= RPS_NO_CPU
&&
2839 ((int)(per_cpu(softnet_data
, cpu
).input_queue_head
-
2840 rflow
->last_qtail
) <
2841 (int)(10 * flow_table
->mask
)))
2847 EXPORT_SYMBOL(rps_may_expire_flow
);
2849 #endif /* CONFIG_RFS_ACCEL */
2851 /* Called from hardirq (IPI) context */
2852 static void rps_trigger_softirq(void *data
)
2854 struct softnet_data
*sd
= data
;
2856 ____napi_schedule(sd
, &sd
->backlog
);
2860 #endif /* CONFIG_RPS */
2863 * Check if this softnet_data structure is another cpu one
2864 * If yes, queue it to our IPI list and return 1
2867 static int rps_ipi_queued(struct softnet_data
*sd
)
2870 struct softnet_data
*mysd
= &__get_cpu_var(softnet_data
);
2873 sd
->rps_ipi_next
= mysd
->rps_ipi_list
;
2874 mysd
->rps_ipi_list
= sd
;
2876 __raise_softirq_irqoff(NET_RX_SOFTIRQ
);
2879 #endif /* CONFIG_RPS */
2884 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2885 * queue (may be a remote CPU queue).
2887 static int enqueue_to_backlog(struct sk_buff
*skb
, int cpu
,
2888 unsigned int *qtail
)
2890 struct softnet_data
*sd
;
2891 unsigned long flags
;
2893 sd
= &per_cpu(softnet_data
, cpu
);
2895 local_irq_save(flags
);
2898 if (skb_queue_len(&sd
->input_pkt_queue
) <= netdev_max_backlog
) {
2899 if (skb_queue_len(&sd
->input_pkt_queue
)) {
2901 __skb_queue_tail(&sd
->input_pkt_queue
, skb
);
2902 input_queue_tail_incr_save(sd
, qtail
);
2904 local_irq_restore(flags
);
2905 return NET_RX_SUCCESS
;
2908 /* Schedule NAPI for backlog device
2909 * We can use non atomic operation since we own the queue lock
2911 if (!__test_and_set_bit(NAPI_STATE_SCHED
, &sd
->backlog
.state
)) {
2912 if (!rps_ipi_queued(sd
))
2913 ____napi_schedule(sd
, &sd
->backlog
);
2921 local_irq_restore(flags
);
2923 atomic_long_inc(&skb
->dev
->rx_dropped
);
2929 * netif_rx - post buffer to the network code
2930 * @skb: buffer to post
2932 * This function receives a packet from a device driver and queues it for
2933 * the upper (protocol) levels to process. It always succeeds. The buffer
2934 * may be dropped during processing for congestion control or by the
2938 * NET_RX_SUCCESS (no congestion)
2939 * NET_RX_DROP (packet was dropped)
2943 int netif_rx(struct sk_buff
*skb
)
2947 /* if netpoll wants it, pretend we never saw it */
2948 if (netpoll_rx(skb
))
2951 if (netdev_tstamp_prequeue
)
2952 net_timestamp_check(skb
);
2954 trace_netif_rx(skb
);
2957 struct rps_dev_flow voidflow
, *rflow
= &voidflow
;
2963 cpu
= get_rps_cpu(skb
->dev
, skb
, &rflow
);
2965 cpu
= smp_processor_id();
2967 ret
= enqueue_to_backlog(skb
, cpu
, &rflow
->last_qtail
);
2975 ret
= enqueue_to_backlog(skb
, get_cpu(), &qtail
);
2981 EXPORT_SYMBOL(netif_rx
);
2983 int netif_rx_ni(struct sk_buff
*skb
)
2988 err
= netif_rx(skb
);
2989 if (local_softirq_pending())
2995 EXPORT_SYMBOL(netif_rx_ni
);
2997 static void net_tx_action(struct softirq_action
*h
)
2999 struct softnet_data
*sd
= &__get_cpu_var(softnet_data
);
3001 if (sd
->completion_queue
) {
3002 struct sk_buff
*clist
;
3004 local_irq_disable();
3005 clist
= sd
->completion_queue
;
3006 sd
->completion_queue
= NULL
;
3010 struct sk_buff
*skb
= clist
;
3011 clist
= clist
->next
;
3013 WARN_ON(atomic_read(&skb
->users
));
3014 trace_kfree_skb(skb
, net_tx_action
);
3019 if (sd
->output_queue
) {
3022 local_irq_disable();
3023 head
= sd
->output_queue
;
3024 sd
->output_queue
= NULL
;
3025 sd
->output_queue_tailp
= &sd
->output_queue
;
3029 struct Qdisc
*q
= head
;
3030 spinlock_t
*root_lock
;
3032 head
= head
->next_sched
;
3034 root_lock
= qdisc_lock(q
);
3035 if (spin_trylock(root_lock
)) {
3036 smp_mb__before_clear_bit();
3037 clear_bit(__QDISC_STATE_SCHED
,
3040 spin_unlock(root_lock
);
3042 if (!test_bit(__QDISC_STATE_DEACTIVATED
,
3044 __netif_reschedule(q
);
3046 smp_mb__before_clear_bit();
3047 clear_bit(__QDISC_STATE_SCHED
,
3055 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3056 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
3057 /* This hook is defined here for ATM LANE */
3058 int (*br_fdb_test_addr_hook
)(struct net_device
*dev
,
3059 unsigned char *addr
) __read_mostly
;
3060 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook
);
3063 #ifdef CONFIG_NET_CLS_ACT
3064 /* TODO: Maybe we should just force sch_ingress to be compiled in
3065 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
3066 * a compare and 2 stores extra right now if we dont have it on
3067 * but have CONFIG_NET_CLS_ACT
3068 * NOTE: This doesn't stop any functionality; if you dont have
3069 * the ingress scheduler, you just can't add policies on ingress.
3072 static int ing_filter(struct sk_buff
*skb
, struct netdev_queue
*rxq
)
3074 struct net_device
*dev
= skb
->dev
;
3075 u32 ttl
= G_TC_RTTL(skb
->tc_verd
);
3076 int result
= TC_ACT_OK
;
3079 if (unlikely(MAX_RED_LOOP
< ttl
++)) {
3080 if (net_ratelimit())
3081 pr_warning( "Redir loop detected Dropping packet (%d->%d)\n",
3082 skb
->skb_iif
, dev
->ifindex
);
3086 skb
->tc_verd
= SET_TC_RTTL(skb
->tc_verd
, ttl
);
3087 skb
->tc_verd
= SET_TC_AT(skb
->tc_verd
, AT_INGRESS
);
3090 if (q
!= &noop_qdisc
) {
3091 spin_lock(qdisc_lock(q
));
3092 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED
, &q
->state
)))
3093 result
= qdisc_enqueue_root(skb
, q
);
3094 spin_unlock(qdisc_lock(q
));
3100 static inline struct sk_buff
*handle_ing(struct sk_buff
*skb
,
3101 struct packet_type
**pt_prev
,
3102 int *ret
, struct net_device
*orig_dev
)
3104 struct netdev_queue
*rxq
= rcu_dereference(skb
->dev
->ingress_queue
);
3106 if (!rxq
|| rxq
->qdisc
== &noop_qdisc
)
3110 *ret
= deliver_skb(skb
, *pt_prev
, orig_dev
);
3114 switch (ing_filter(skb
, rxq
)) {
3128 * netdev_rx_handler_register - register receive handler
3129 * @dev: device to register a handler for
3130 * @rx_handler: receive handler to register
3131 * @rx_handler_data: data pointer that is used by rx handler
3133 * Register a receive hander for a device. This handler will then be
3134 * called from __netif_receive_skb. A negative errno code is returned
3137 * The caller must hold the rtnl_mutex.
3139 * For a general description of rx_handler, see enum rx_handler_result.
3141 int netdev_rx_handler_register(struct net_device
*dev
,
3142 rx_handler_func_t
*rx_handler
,
3143 void *rx_handler_data
)
3147 if (dev
->rx_handler
)
3150 rcu_assign_pointer(dev
->rx_handler_data
, rx_handler_data
);
3151 rcu_assign_pointer(dev
->rx_handler
, rx_handler
);
3155 EXPORT_SYMBOL_GPL(netdev_rx_handler_register
);
3158 * netdev_rx_handler_unregister - unregister receive handler
3159 * @dev: device to unregister a handler from
3161 * Unregister a receive hander from a device.
3163 * The caller must hold the rtnl_mutex.
3165 void netdev_rx_handler_unregister(struct net_device
*dev
)
3169 RCU_INIT_POINTER(dev
->rx_handler
, NULL
);
3170 RCU_INIT_POINTER(dev
->rx_handler_data
, NULL
);
3172 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister
);
3174 static int __netif_receive_skb(struct sk_buff
*skb
)
3176 struct packet_type
*ptype
, *pt_prev
;
3177 rx_handler_func_t
*rx_handler
;
3178 struct net_device
*orig_dev
;
3179 struct net_device
*null_or_dev
;
3180 bool deliver_exact
= false;
3181 int ret
= NET_RX_DROP
;
3184 if (!netdev_tstamp_prequeue
)
3185 net_timestamp_check(skb
);
3187 trace_netif_receive_skb(skb
);
3189 /* if we've gotten here through NAPI, check netpoll */
3190 if (netpoll_receive_skb(skb
))
3194 skb
->skb_iif
= skb
->dev
->ifindex
;
3195 orig_dev
= skb
->dev
;
3197 skb_reset_network_header(skb
);
3198 skb_reset_transport_header(skb
);
3199 skb_reset_mac_len(skb
);
3207 __this_cpu_inc(softnet_data
.processed
);
3209 if (skb
->protocol
== cpu_to_be16(ETH_P_8021Q
)) {
3210 skb
= vlan_untag(skb
);
3215 #ifdef CONFIG_NET_CLS_ACT
3216 if (skb
->tc_verd
& TC_NCLS
) {
3217 skb
->tc_verd
= CLR_TC_NCLS(skb
->tc_verd
);
3222 list_for_each_entry_rcu(ptype
, &ptype_all
, list
) {
3223 if (!ptype
->dev
|| ptype
->dev
== skb
->dev
) {
3225 ret
= deliver_skb(skb
, pt_prev
, orig_dev
);
3230 #ifdef CONFIG_NET_CLS_ACT
3231 skb
= handle_ing(skb
, &pt_prev
, &ret
, orig_dev
);
3237 rx_handler
= rcu_dereference(skb
->dev
->rx_handler
);
3240 ret
= deliver_skb(skb
, pt_prev
, orig_dev
);
3243 switch (rx_handler(&skb
)) {
3244 case RX_HANDLER_CONSUMED
:
3246 case RX_HANDLER_ANOTHER
:
3248 case RX_HANDLER_EXACT
:
3249 deliver_exact
= true;
3250 case RX_HANDLER_PASS
:
3257 if (vlan_tx_tag_present(skb
)) {
3259 ret
= deliver_skb(skb
, pt_prev
, orig_dev
);
3262 if (vlan_do_receive(&skb
))
3264 else if (unlikely(!skb
))
3268 /* deliver only exact match when indicated */
3269 null_or_dev
= deliver_exact
? skb
->dev
: NULL
;
3271 type
= skb
->protocol
;
3272 list_for_each_entry_rcu(ptype
,
3273 &ptype_base
[ntohs(type
) & PTYPE_HASH_MASK
], list
) {
3274 if (ptype
->type
== type
&&
3275 (ptype
->dev
== null_or_dev
|| ptype
->dev
== skb
->dev
||
3276 ptype
->dev
== orig_dev
)) {
3278 ret
= deliver_skb(skb
, pt_prev
, orig_dev
);
3284 ret
= pt_prev
->func(skb
, skb
->dev
, pt_prev
, orig_dev
);
3286 atomic_long_inc(&skb
->dev
->rx_dropped
);
3288 /* Jamal, now you will not able to escape explaining
3289 * me how you were going to use this. :-)
3300 * netif_receive_skb - process receive buffer from network
3301 * @skb: buffer to process
3303 * netif_receive_skb() is the main receive data processing function.
3304 * It always succeeds. The buffer may be dropped during processing
3305 * for congestion control or by the protocol layers.
3307 * This function may only be called from softirq context and interrupts
3308 * should be enabled.
3310 * Return values (usually ignored):
3311 * NET_RX_SUCCESS: no congestion
3312 * NET_RX_DROP: packet was dropped
3314 int netif_receive_skb(struct sk_buff
*skb
)
3316 if (netdev_tstamp_prequeue
)
3317 net_timestamp_check(skb
);
3319 if (skb_defer_rx_timestamp(skb
))
3320 return NET_RX_SUCCESS
;
3324 struct rps_dev_flow voidflow
, *rflow
= &voidflow
;
3329 cpu
= get_rps_cpu(skb
->dev
, skb
, &rflow
);
3332 ret
= enqueue_to_backlog(skb
, cpu
, &rflow
->last_qtail
);
3336 ret
= __netif_receive_skb(skb
);
3342 return __netif_receive_skb(skb
);
3345 EXPORT_SYMBOL(netif_receive_skb
);
3347 /* Network device is going away, flush any packets still pending
3348 * Called with irqs disabled.
3350 static void flush_backlog(void *arg
)
3352 struct net_device
*dev
= arg
;
3353 struct softnet_data
*sd
= &__get_cpu_var(softnet_data
);
3354 struct sk_buff
*skb
, *tmp
;
3357 skb_queue_walk_safe(&sd
->input_pkt_queue
, skb
, tmp
) {
3358 if (skb
->dev
== dev
) {
3359 __skb_unlink(skb
, &sd
->input_pkt_queue
);
3361 input_queue_head_incr(sd
);
3366 skb_queue_walk_safe(&sd
->process_queue
, skb
, tmp
) {
3367 if (skb
->dev
== dev
) {
3368 __skb_unlink(skb
, &sd
->process_queue
);
3370 input_queue_head_incr(sd
);
3375 static int napi_gro_complete(struct sk_buff
*skb
)
3377 struct packet_type
*ptype
;
3378 __be16 type
= skb
->protocol
;
3379 struct list_head
*head
= &ptype_base
[ntohs(type
) & PTYPE_HASH_MASK
];
3382 if (NAPI_GRO_CB(skb
)->count
== 1) {
3383 skb_shinfo(skb
)->gso_size
= 0;
3388 list_for_each_entry_rcu(ptype
, head
, list
) {
3389 if (ptype
->type
!= type
|| ptype
->dev
|| !ptype
->gro_complete
)
3392 err
= ptype
->gro_complete(skb
);
3398 WARN_ON(&ptype
->list
== head
);
3400 return NET_RX_SUCCESS
;
3404 return netif_receive_skb(skb
);
3407 inline void napi_gro_flush(struct napi_struct
*napi
)
3409 struct sk_buff
*skb
, *next
;
3411 for (skb
= napi
->gro_list
; skb
; skb
= next
) {
3414 napi_gro_complete(skb
);
3417 napi
->gro_count
= 0;
3418 napi
->gro_list
= NULL
;
3420 EXPORT_SYMBOL(napi_gro_flush
);
3422 enum gro_result
dev_gro_receive(struct napi_struct
*napi
, struct sk_buff
*skb
)
3424 struct sk_buff
**pp
= NULL
;
3425 struct packet_type
*ptype
;
3426 __be16 type
= skb
->protocol
;
3427 struct list_head
*head
= &ptype_base
[ntohs(type
) & PTYPE_HASH_MASK
];
3430 enum gro_result ret
;
3432 if (!(skb
->dev
->features
& NETIF_F_GRO
) || netpoll_rx_on(skb
))
3435 if (skb_is_gso(skb
) || skb_has_frag_list(skb
))
3439 list_for_each_entry_rcu(ptype
, head
, list
) {
3440 if (ptype
->type
!= type
|| ptype
->dev
|| !ptype
->gro_receive
)
3443 skb_set_network_header(skb
, skb_gro_offset(skb
));
3444 mac_len
= skb
->network_header
- skb
->mac_header
;
3445 skb
->mac_len
= mac_len
;
3446 NAPI_GRO_CB(skb
)->same_flow
= 0;
3447 NAPI_GRO_CB(skb
)->flush
= 0;
3448 NAPI_GRO_CB(skb
)->free
= 0;
3450 pp
= ptype
->gro_receive(&napi
->gro_list
, skb
);
3455 if (&ptype
->list
== head
)
3458 same_flow
= NAPI_GRO_CB(skb
)->same_flow
;
3459 ret
= NAPI_GRO_CB(skb
)->free
? GRO_MERGED_FREE
: GRO_MERGED
;
3462 struct sk_buff
*nskb
= *pp
;
3466 napi_gro_complete(nskb
);
3473 if (NAPI_GRO_CB(skb
)->flush
|| napi
->gro_count
>= MAX_GRO_SKBS
)
3477 NAPI_GRO_CB(skb
)->count
= 1;
3478 skb_shinfo(skb
)->gso_size
= skb_gro_len(skb
);
3479 skb
->next
= napi
->gro_list
;
3480 napi
->gro_list
= skb
;
3484 if (skb_headlen(skb
) < skb_gro_offset(skb
)) {
3485 int grow
= skb_gro_offset(skb
) - skb_headlen(skb
);
3487 BUG_ON(skb
->end
- skb
->tail
< grow
);
3489 memcpy(skb_tail_pointer(skb
), NAPI_GRO_CB(skb
)->frag0
, grow
);
3492 skb
->data_len
-= grow
;
3494 skb_shinfo(skb
)->frags
[0].page_offset
+= grow
;
3495 skb_shinfo(skb
)->frags
[0].size
-= grow
;
3497 if (unlikely(!skb_shinfo(skb
)->frags
[0].size
)) {
3498 skb_frag_unref(skb
, 0);
3499 memmove(skb_shinfo(skb
)->frags
,
3500 skb_shinfo(skb
)->frags
+ 1,
3501 --skb_shinfo(skb
)->nr_frags
* sizeof(skb_frag_t
));
3512 EXPORT_SYMBOL(dev_gro_receive
);
3514 static inline gro_result_t
3515 __napi_gro_receive(struct napi_struct
*napi
, struct sk_buff
*skb
)
3519 for (p
= napi
->gro_list
; p
; p
= p
->next
) {
3520 unsigned long diffs
;
3522 diffs
= (unsigned long)p
->dev
^ (unsigned long)skb
->dev
;
3523 diffs
|= p
->vlan_tci
^ skb
->vlan_tci
;
3524 diffs
|= compare_ether_header(skb_mac_header(p
),
3525 skb_gro_mac_header(skb
));
3526 NAPI_GRO_CB(p
)->same_flow
= !diffs
;
3527 NAPI_GRO_CB(p
)->flush
= 0;
3530 return dev_gro_receive(napi
, skb
);
3533 gro_result_t
napi_skb_finish(gro_result_t ret
, struct sk_buff
*skb
)
3537 if (netif_receive_skb(skb
))
3542 case GRO_MERGED_FREE
:
3553 EXPORT_SYMBOL(napi_skb_finish
);
3555 void skb_gro_reset_offset(struct sk_buff
*skb
)
3557 NAPI_GRO_CB(skb
)->data_offset
= 0;
3558 NAPI_GRO_CB(skb
)->frag0
= NULL
;
3559 NAPI_GRO_CB(skb
)->frag0_len
= 0;
3561 if (skb
->mac_header
== skb
->tail
&&
3562 !PageHighMem(skb_frag_page(&skb_shinfo(skb
)->frags
[0]))) {
3563 NAPI_GRO_CB(skb
)->frag0
=
3564 skb_frag_address(&skb_shinfo(skb
)->frags
[0]);
3565 NAPI_GRO_CB(skb
)->frag0_len
= skb_shinfo(skb
)->frags
[0].size
;
3568 EXPORT_SYMBOL(skb_gro_reset_offset
);
3570 gro_result_t
napi_gro_receive(struct napi_struct
*napi
, struct sk_buff
*skb
)
3572 skb_gro_reset_offset(skb
);
3574 return napi_skb_finish(__napi_gro_receive(napi
, skb
), skb
);
3576 EXPORT_SYMBOL(napi_gro_receive
);
3578 static void napi_reuse_skb(struct napi_struct
*napi
, struct sk_buff
*skb
)
3580 __skb_pull(skb
, skb_headlen(skb
));
3581 skb_reserve(skb
, NET_IP_ALIGN
- skb_headroom(skb
));
3583 skb
->dev
= napi
->dev
;
3589 struct sk_buff
*napi_get_frags(struct napi_struct
*napi
)
3591 struct sk_buff
*skb
= napi
->skb
;
3594 skb
= netdev_alloc_skb_ip_align(napi
->dev
, GRO_MAX_HEAD
);
3600 EXPORT_SYMBOL(napi_get_frags
);
3602 gro_result_t
napi_frags_finish(struct napi_struct
*napi
, struct sk_buff
*skb
,
3608 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
3610 if (ret
== GRO_HELD
)
3611 skb_gro_pull(skb
, -ETH_HLEN
);
3612 else if (netif_receive_skb(skb
))
3617 case GRO_MERGED_FREE
:
3618 napi_reuse_skb(napi
, skb
);
3627 EXPORT_SYMBOL(napi_frags_finish
);
3629 struct sk_buff
*napi_frags_skb(struct napi_struct
*napi
)
3631 struct sk_buff
*skb
= napi
->skb
;
3638 skb_reset_mac_header(skb
);
3639 skb_gro_reset_offset(skb
);
3641 off
= skb_gro_offset(skb
);
3642 hlen
= off
+ sizeof(*eth
);
3643 eth
= skb_gro_header_fast(skb
, off
);
3644 if (skb_gro_header_hard(skb
, hlen
)) {
3645 eth
= skb_gro_header_slow(skb
, hlen
, off
);
3646 if (unlikely(!eth
)) {
3647 napi_reuse_skb(napi
, skb
);
3653 skb_gro_pull(skb
, sizeof(*eth
));
3656 * This works because the only protocols we care about don't require
3657 * special handling. We'll fix it up properly at the end.
3659 skb
->protocol
= eth
->h_proto
;
3664 EXPORT_SYMBOL(napi_frags_skb
);
3666 gro_result_t
napi_gro_frags(struct napi_struct
*napi
)
3668 struct sk_buff
*skb
= napi_frags_skb(napi
);
3673 return napi_frags_finish(napi
, skb
, __napi_gro_receive(napi
, skb
));
3675 EXPORT_SYMBOL(napi_gro_frags
);
3678 * net_rps_action sends any pending IPI's for rps.
3679 * Note: called with local irq disabled, but exits with local irq enabled.
3681 static void net_rps_action_and_irq_enable(struct softnet_data
*sd
)
3684 struct softnet_data
*remsd
= sd
->rps_ipi_list
;
3687 sd
->rps_ipi_list
= NULL
;
3691 /* Send pending IPI's to kick RPS processing on remote cpus. */
3693 struct softnet_data
*next
= remsd
->rps_ipi_next
;
3695 if (cpu_online(remsd
->cpu
))
3696 __smp_call_function_single(remsd
->cpu
,
3705 static int process_backlog(struct napi_struct
*napi
, int quota
)
3708 struct softnet_data
*sd
= container_of(napi
, struct softnet_data
, backlog
);
3711 /* Check if we have pending ipi, its better to send them now,
3712 * not waiting net_rx_action() end.
3714 if (sd
->rps_ipi_list
) {
3715 local_irq_disable();
3716 net_rps_action_and_irq_enable(sd
);
3719 napi
->weight
= weight_p
;
3720 local_irq_disable();
3721 while (work
< quota
) {
3722 struct sk_buff
*skb
;
3725 while ((skb
= __skb_dequeue(&sd
->process_queue
))) {
3727 __netif_receive_skb(skb
);
3728 local_irq_disable();
3729 input_queue_head_incr(sd
);
3730 if (++work
>= quota
) {
3737 qlen
= skb_queue_len(&sd
->input_pkt_queue
);
3739 skb_queue_splice_tail_init(&sd
->input_pkt_queue
,
3740 &sd
->process_queue
);
3742 if (qlen
< quota
- work
) {
3744 * Inline a custom version of __napi_complete().
3745 * only current cpu owns and manipulates this napi,
3746 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3747 * we can use a plain write instead of clear_bit(),
3748 * and we dont need an smp_mb() memory barrier.
3750 list_del(&napi
->poll_list
);
3753 quota
= work
+ qlen
;
3763 * __napi_schedule - schedule for receive
3764 * @n: entry to schedule
3766 * The entry's receive function will be scheduled to run
3768 void __napi_schedule(struct napi_struct
*n
)
3770 unsigned long flags
;
3772 local_irq_save(flags
);
3773 ____napi_schedule(&__get_cpu_var(softnet_data
), n
);
3774 local_irq_restore(flags
);
3776 EXPORT_SYMBOL(__napi_schedule
);
3778 void __napi_complete(struct napi_struct
*n
)
3780 BUG_ON(!test_bit(NAPI_STATE_SCHED
, &n
->state
));
3781 BUG_ON(n
->gro_list
);
3783 list_del(&n
->poll_list
);
3784 smp_mb__before_clear_bit();
3785 clear_bit(NAPI_STATE_SCHED
, &n
->state
);
3787 EXPORT_SYMBOL(__napi_complete
);
3789 void napi_complete(struct napi_struct
*n
)
3791 unsigned long flags
;
3794 * don't let napi dequeue from the cpu poll list
3795 * just in case its running on a different cpu
3797 if (unlikely(test_bit(NAPI_STATE_NPSVC
, &n
->state
)))
3801 local_irq_save(flags
);
3803 local_irq_restore(flags
);
3805 EXPORT_SYMBOL(napi_complete
);
3807 void netif_napi_add(struct net_device
*dev
, struct napi_struct
*napi
,
3808 int (*poll
)(struct napi_struct
*, int), int weight
)
3810 INIT_LIST_HEAD(&napi
->poll_list
);
3811 napi
->gro_count
= 0;
3812 napi
->gro_list
= NULL
;
3815 napi
->weight
= weight
;
3816 list_add(&napi
->dev_list
, &dev
->napi_list
);
3818 #ifdef CONFIG_NETPOLL
3819 spin_lock_init(&napi
->poll_lock
);
3820 napi
->poll_owner
= -1;
3822 set_bit(NAPI_STATE_SCHED
, &napi
->state
);
3824 EXPORT_SYMBOL(netif_napi_add
);
3826 void netif_napi_del(struct napi_struct
*napi
)
3828 struct sk_buff
*skb
, *next
;
3830 list_del_init(&napi
->dev_list
);
3831 napi_free_frags(napi
);
3833 for (skb
= napi
->gro_list
; skb
; skb
= next
) {
3839 napi
->gro_list
= NULL
;
3840 napi
->gro_count
= 0;
3842 EXPORT_SYMBOL(netif_napi_del
);
3844 static void net_rx_action(struct softirq_action
*h
)
3846 struct softnet_data
*sd
= &__get_cpu_var(softnet_data
);
3847 unsigned long time_limit
= jiffies
+ 2;
3848 int budget
= netdev_budget
;
3851 local_irq_disable();
3853 while (!list_empty(&sd
->poll_list
)) {
3854 struct napi_struct
*n
;
3857 /* If softirq window is exhuasted then punt.
3858 * Allow this to run for 2 jiffies since which will allow
3859 * an average latency of 1.5/HZ.
3861 if (unlikely(budget
<= 0 || time_after(jiffies
, time_limit
)))
3866 /* Even though interrupts have been re-enabled, this
3867 * access is safe because interrupts can only add new
3868 * entries to the tail of this list, and only ->poll()
3869 * calls can remove this head entry from the list.
3871 n
= list_first_entry(&sd
->poll_list
, struct napi_struct
, poll_list
);
3873 have
= netpoll_poll_lock(n
);
3877 /* This NAPI_STATE_SCHED test is for avoiding a race
3878 * with netpoll's poll_napi(). Only the entity which
3879 * obtains the lock and sees NAPI_STATE_SCHED set will
3880 * actually make the ->poll() call. Therefore we avoid
3881 * accidentally calling ->poll() when NAPI is not scheduled.
3884 if (test_bit(NAPI_STATE_SCHED
, &n
->state
)) {
3885 work
= n
->poll(n
, weight
);
3889 WARN_ON_ONCE(work
> weight
);
3893 local_irq_disable();
3895 /* Drivers must not modify the NAPI state if they
3896 * consume the entire weight. In such cases this code
3897 * still "owns" the NAPI instance and therefore can
3898 * move the instance around on the list at-will.
3900 if (unlikely(work
== weight
)) {
3901 if (unlikely(napi_disable_pending(n
))) {
3904 local_irq_disable();
3906 list_move_tail(&n
->poll_list
, &sd
->poll_list
);
3909 netpoll_poll_unlock(have
);
3912 net_rps_action_and_irq_enable(sd
);
3914 #ifdef CONFIG_NET_DMA
3916 * There may not be any more sk_buffs coming right now, so push
3917 * any pending DMA copies to hardware
3919 dma_issue_pending_all();
3926 __raise_softirq_irqoff(NET_RX_SOFTIRQ
);
3930 static gifconf_func_t
*gifconf_list
[NPROTO
];
3933 * register_gifconf - register a SIOCGIF handler
3934 * @family: Address family
3935 * @gifconf: Function handler
3937 * Register protocol dependent address dumping routines. The handler
3938 * that is passed must not be freed or reused until it has been replaced
3939 * by another handler.
3941 int register_gifconf(unsigned int family
, gifconf_func_t
*gifconf
)
3943 if (family
>= NPROTO
)
3945 gifconf_list
[family
] = gifconf
;
3948 EXPORT_SYMBOL(register_gifconf
);
3952 * Map an interface index to its name (SIOCGIFNAME)
3956 * We need this ioctl for efficient implementation of the
3957 * if_indextoname() function required by the IPv6 API. Without
3958 * it, we would have to search all the interfaces to find a
3962 static int dev_ifname(struct net
*net
, struct ifreq __user
*arg
)
3964 struct net_device
*dev
;
3968 * Fetch the caller's info block.
3971 if (copy_from_user(&ifr
, arg
, sizeof(struct ifreq
)))
3975 dev
= dev_get_by_index_rcu(net
, ifr
.ifr_ifindex
);
3981 strcpy(ifr
.ifr_name
, dev
->name
);
3984 if (copy_to_user(arg
, &ifr
, sizeof(struct ifreq
)))
3990 * Perform a SIOCGIFCONF call. This structure will change
3991 * size eventually, and there is nothing I can do about it.
3992 * Thus we will need a 'compatibility mode'.
3995 static int dev_ifconf(struct net
*net
, char __user
*arg
)
3998 struct net_device
*dev
;
4005 * Fetch the caller's info block.
4008 if (copy_from_user(&ifc
, arg
, sizeof(struct ifconf
)))
4015 * Loop over the interfaces, and write an info block for each.
4019 for_each_netdev(net
, dev
) {
4020 for (i
= 0; i
< NPROTO
; i
++) {
4021 if (gifconf_list
[i
]) {
4024 done
= gifconf_list
[i
](dev
, NULL
, 0);
4026 done
= gifconf_list
[i
](dev
, pos
+ total
,
4036 * All done. Write the updated control block back to the caller.
4038 ifc
.ifc_len
= total
;
4041 * Both BSD and Solaris return 0 here, so we do too.
4043 return copy_to_user(arg
, &ifc
, sizeof(struct ifconf
)) ? -EFAULT
: 0;
4046 #ifdef CONFIG_PROC_FS
4048 * This is invoked by the /proc filesystem handler to display a device
4051 void *dev_seq_start(struct seq_file
*seq
, loff_t
*pos
)
4054 struct net
*net
= seq_file_net(seq
);
4056 struct net_device
*dev
;
4060 return SEQ_START_TOKEN
;
4063 for_each_netdev_rcu(net
, dev
)
4070 void *dev_seq_next(struct seq_file
*seq
, void *v
, loff_t
*pos
)
4072 struct net_device
*dev
= v
;
4074 if (v
== SEQ_START_TOKEN
)
4075 dev
= first_net_device_rcu(seq_file_net(seq
));
4077 dev
= next_net_device_rcu(dev
);
4083 void dev_seq_stop(struct seq_file
*seq
, void *v
)
4089 static void dev_seq_printf_stats(struct seq_file
*seq
, struct net_device
*dev
)
4091 struct rtnl_link_stats64 temp
;
4092 const struct rtnl_link_stats64
*stats
= dev_get_stats(dev
, &temp
);
4094 seq_printf(seq
, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
4095 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
4096 dev
->name
, stats
->rx_bytes
, stats
->rx_packets
,
4098 stats
->rx_dropped
+ stats
->rx_missed_errors
,
4099 stats
->rx_fifo_errors
,
4100 stats
->rx_length_errors
+ stats
->rx_over_errors
+
4101 stats
->rx_crc_errors
+ stats
->rx_frame_errors
,
4102 stats
->rx_compressed
, stats
->multicast
,
4103 stats
->tx_bytes
, stats
->tx_packets
,
4104 stats
->tx_errors
, stats
->tx_dropped
,
4105 stats
->tx_fifo_errors
, stats
->collisions
,
4106 stats
->tx_carrier_errors
+
4107 stats
->tx_aborted_errors
+
4108 stats
->tx_window_errors
+
4109 stats
->tx_heartbeat_errors
,
4110 stats
->tx_compressed
);
4114 * Called from the PROCfs module. This now uses the new arbitrary sized
4115 * /proc/net interface to create /proc/net/dev
4117 static int dev_seq_show(struct seq_file
*seq
, void *v
)
4119 if (v
== SEQ_START_TOKEN
)
4120 seq_puts(seq
, "Inter-| Receive "
4122 " face |bytes packets errs drop fifo frame "
4123 "compressed multicast|bytes packets errs "
4124 "drop fifo colls carrier compressed\n");
4126 dev_seq_printf_stats(seq
, v
);
4130 static struct softnet_data
*softnet_get_online(loff_t
*pos
)
4132 struct softnet_data
*sd
= NULL
;
4134 while (*pos
< nr_cpu_ids
)
4135 if (cpu_online(*pos
)) {
4136 sd
= &per_cpu(softnet_data
, *pos
);
4143 static void *softnet_seq_start(struct seq_file
*seq
, loff_t
*pos
)
4145 return softnet_get_online(pos
);
4148 static void *softnet_seq_next(struct seq_file
*seq
, void *v
, loff_t
*pos
)
4151 return softnet_get_online(pos
);
4154 static void softnet_seq_stop(struct seq_file
*seq
, void *v
)
4158 static int softnet_seq_show(struct seq_file
*seq
, void *v
)
4160 struct softnet_data
*sd
= v
;
4162 seq_printf(seq
, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
4163 sd
->processed
, sd
->dropped
, sd
->time_squeeze
, 0,
4164 0, 0, 0, 0, /* was fastroute */
4165 sd
->cpu_collision
, sd
->received_rps
);
4169 static const struct seq_operations dev_seq_ops
= {
4170 .start
= dev_seq_start
,
4171 .next
= dev_seq_next
,
4172 .stop
= dev_seq_stop
,
4173 .show
= dev_seq_show
,
4176 static int dev_seq_open(struct inode
*inode
, struct file
*file
)
4178 return seq_open_net(inode
, file
, &dev_seq_ops
,
4179 sizeof(struct seq_net_private
));
4182 static const struct file_operations dev_seq_fops
= {
4183 .owner
= THIS_MODULE
,
4184 .open
= dev_seq_open
,
4186 .llseek
= seq_lseek
,
4187 .release
= seq_release_net
,
4190 static const struct seq_operations softnet_seq_ops
= {
4191 .start
= softnet_seq_start
,
4192 .next
= softnet_seq_next
,
4193 .stop
= softnet_seq_stop
,
4194 .show
= softnet_seq_show
,
4197 static int softnet_seq_open(struct inode
*inode
, struct file
*file
)
4199 return seq_open(file
, &softnet_seq_ops
);
4202 static const struct file_operations softnet_seq_fops
= {
4203 .owner
= THIS_MODULE
,
4204 .open
= softnet_seq_open
,
4206 .llseek
= seq_lseek
,
4207 .release
= seq_release
,
4210 static void *ptype_get_idx(loff_t pos
)
4212 struct packet_type
*pt
= NULL
;
4216 list_for_each_entry_rcu(pt
, &ptype_all
, list
) {
4222 for (t
= 0; t
< PTYPE_HASH_SIZE
; t
++) {
4223 list_for_each_entry_rcu(pt
, &ptype_base
[t
], list
) {
4232 static void *ptype_seq_start(struct seq_file
*seq
, loff_t
*pos
)
4236 return *pos
? ptype_get_idx(*pos
- 1) : SEQ_START_TOKEN
;
4239 static void *ptype_seq_next(struct seq_file
*seq
, void *v
, loff_t
*pos
)
4241 struct packet_type
*pt
;
4242 struct list_head
*nxt
;
4246 if (v
== SEQ_START_TOKEN
)
4247 return ptype_get_idx(0);
4250 nxt
= pt
->list
.next
;
4251 if (pt
->type
== htons(ETH_P_ALL
)) {
4252 if (nxt
!= &ptype_all
)
4255 nxt
= ptype_base
[0].next
;
4257 hash
= ntohs(pt
->type
) & PTYPE_HASH_MASK
;
4259 while (nxt
== &ptype_base
[hash
]) {
4260 if (++hash
>= PTYPE_HASH_SIZE
)
4262 nxt
= ptype_base
[hash
].next
;
4265 return list_entry(nxt
, struct packet_type
, list
);
4268 static void ptype_seq_stop(struct seq_file
*seq
, void *v
)
4274 static int ptype_seq_show(struct seq_file
*seq
, void *v
)
4276 struct packet_type
*pt
= v
;
4278 if (v
== SEQ_START_TOKEN
)
4279 seq_puts(seq
, "Type Device Function\n");
4280 else if (pt
->dev
== NULL
|| dev_net(pt
->dev
) == seq_file_net(seq
)) {
4281 if (pt
->type
== htons(ETH_P_ALL
))
4282 seq_puts(seq
, "ALL ");
4284 seq_printf(seq
, "%04x", ntohs(pt
->type
));
4286 seq_printf(seq
, " %-8s %pF\n",
4287 pt
->dev
? pt
->dev
->name
: "", pt
->func
);
4293 static const struct seq_operations ptype_seq_ops
= {
4294 .start
= ptype_seq_start
,
4295 .next
= ptype_seq_next
,
4296 .stop
= ptype_seq_stop
,
4297 .show
= ptype_seq_show
,
4300 static int ptype_seq_open(struct inode
*inode
, struct file
*file
)
4302 return seq_open_net(inode
, file
, &ptype_seq_ops
,
4303 sizeof(struct seq_net_private
));
4306 static const struct file_operations ptype_seq_fops
= {
4307 .owner
= THIS_MODULE
,
4308 .open
= ptype_seq_open
,
4310 .llseek
= seq_lseek
,
4311 .release
= seq_release_net
,
4315 static int __net_init
dev_proc_net_init(struct net
*net
)
4319 if (!proc_net_fops_create(net
, "dev", S_IRUGO
, &dev_seq_fops
))
4321 if (!proc_net_fops_create(net
, "softnet_stat", S_IRUGO
, &softnet_seq_fops
))
4323 if (!proc_net_fops_create(net
, "ptype", S_IRUGO
, &ptype_seq_fops
))
4326 if (wext_proc_init(net
))
4332 proc_net_remove(net
, "ptype");
4334 proc_net_remove(net
, "softnet_stat");
4336 proc_net_remove(net
, "dev");
4340 static void __net_exit
dev_proc_net_exit(struct net
*net
)
4342 wext_proc_exit(net
);
4344 proc_net_remove(net
, "ptype");
4345 proc_net_remove(net
, "softnet_stat");
4346 proc_net_remove(net
, "dev");
4349 static struct pernet_operations __net_initdata dev_proc_ops
= {
4350 .init
= dev_proc_net_init
,
4351 .exit
= dev_proc_net_exit
,
4354 static int __init
dev_proc_init(void)
4356 return register_pernet_subsys(&dev_proc_ops
);
4359 #define dev_proc_init() 0
4360 #endif /* CONFIG_PROC_FS */
4364 * netdev_set_master - set up master pointer
4365 * @slave: slave device
4366 * @master: new master device
4368 * Changes the master device of the slave. Pass %NULL to break the
4369 * bonding. The caller must hold the RTNL semaphore. On a failure
4370 * a negative errno code is returned. On success the reference counts
4371 * are adjusted and the function returns zero.
4373 int netdev_set_master(struct net_device
*slave
, struct net_device
*master
)
4375 struct net_device
*old
= slave
->master
;
4385 slave
->master
= master
;
4391 EXPORT_SYMBOL(netdev_set_master
);
4394 * netdev_set_bond_master - set up bonding master/slave pair
4395 * @slave: slave device
4396 * @master: new master device
4398 * Changes the master device of the slave. Pass %NULL to break the
4399 * bonding. The caller must hold the RTNL semaphore. On a failure
4400 * a negative errno code is returned. On success %RTM_NEWLINK is sent
4401 * to the routing socket and the function returns zero.
4403 int netdev_set_bond_master(struct net_device
*slave
, struct net_device
*master
)
4409 err
= netdev_set_master(slave
, master
);
4413 slave
->flags
|= IFF_SLAVE
;
4415 slave
->flags
&= ~IFF_SLAVE
;
4417 rtmsg_ifinfo(RTM_NEWLINK
, slave
, IFF_SLAVE
);
4420 EXPORT_SYMBOL(netdev_set_bond_master
);
4422 static void dev_change_rx_flags(struct net_device
*dev
, int flags
)
4424 const struct net_device_ops
*ops
= dev
->netdev_ops
;
4426 if ((dev
->flags
& IFF_UP
) && ops
->ndo_change_rx_flags
)
4427 ops
->ndo_change_rx_flags(dev
, flags
);
4430 static int __dev_set_promiscuity(struct net_device
*dev
, int inc
)
4432 unsigned short old_flags
= dev
->flags
;
4438 dev
->flags
|= IFF_PROMISC
;
4439 dev
->promiscuity
+= inc
;
4440 if (dev
->promiscuity
== 0) {
4443 * If inc causes overflow, untouch promisc and return error.
4446 dev
->flags
&= ~IFF_PROMISC
;
4448 dev
->promiscuity
-= inc
;
4449 printk(KERN_WARNING
"%s: promiscuity touches roof, "
4450 "set promiscuity failed, promiscuity feature "
4451 "of device might be broken.\n", dev
->name
);
4455 if (dev
->flags
!= old_flags
) {
4456 printk(KERN_INFO
"device %s %s promiscuous mode\n",
4457 dev
->name
, (dev
->flags
& IFF_PROMISC
) ? "entered" :
4459 if (audit_enabled
) {
4460 current_uid_gid(&uid
, &gid
);
4461 audit_log(current
->audit_context
, GFP_ATOMIC
,
4462 AUDIT_ANOM_PROMISCUOUS
,
4463 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4464 dev
->name
, (dev
->flags
& IFF_PROMISC
),
4465 (old_flags
& IFF_PROMISC
),
4466 audit_get_loginuid(current
),
4468 audit_get_sessionid(current
));
4471 dev_change_rx_flags(dev
, IFF_PROMISC
);
4477 * dev_set_promiscuity - update promiscuity count on a device
4481 * Add or remove promiscuity from a device. While the count in the device
4482 * remains above zero the interface remains promiscuous. Once it hits zero
4483 * the device reverts back to normal filtering operation. A negative inc
4484 * value is used to drop promiscuity on the device.
4485 * Return 0 if successful or a negative errno code on error.
4487 int dev_set_promiscuity(struct net_device
*dev
, int inc
)
4489 unsigned short old_flags
= dev
->flags
;
4492 err
= __dev_set_promiscuity(dev
, inc
);
4495 if (dev
->flags
!= old_flags
)
4496 dev_set_rx_mode(dev
);
4499 EXPORT_SYMBOL(dev_set_promiscuity
);
4502 * dev_set_allmulti - update allmulti count on a device
4506 * Add or remove reception of all multicast frames to a device. While the
4507 * count in the device remains above zero the interface remains listening
4508 * to all interfaces. Once it hits zero the device reverts back to normal
4509 * filtering operation. A negative @inc value is used to drop the counter
4510 * when releasing a resource needing all multicasts.
4511 * Return 0 if successful or a negative errno code on error.
4514 int dev_set_allmulti(struct net_device
*dev
, int inc
)
4516 unsigned short old_flags
= dev
->flags
;
4520 dev
->flags
|= IFF_ALLMULTI
;
4521 dev
->allmulti
+= inc
;
4522 if (dev
->allmulti
== 0) {
4525 * If inc causes overflow, untouch allmulti and return error.
4528 dev
->flags
&= ~IFF_ALLMULTI
;
4530 dev
->allmulti
-= inc
;
4531 printk(KERN_WARNING
"%s: allmulti touches roof, "
4532 "set allmulti failed, allmulti feature of "
4533 "device might be broken.\n", dev
->name
);
4537 if (dev
->flags
^ old_flags
) {
4538 dev_change_rx_flags(dev
, IFF_ALLMULTI
);
4539 dev_set_rx_mode(dev
);
4543 EXPORT_SYMBOL(dev_set_allmulti
);
4546 * Upload unicast and multicast address lists to device and
4547 * configure RX filtering. When the device doesn't support unicast
4548 * filtering it is put in promiscuous mode while unicast addresses
4551 void __dev_set_rx_mode(struct net_device
*dev
)
4553 const struct net_device_ops
*ops
= dev
->netdev_ops
;
4555 /* dev_open will call this function so the list will stay sane. */
4556 if (!(dev
->flags
&IFF_UP
))
4559 if (!netif_device_present(dev
))
4562 if (!(dev
->priv_flags
& IFF_UNICAST_FLT
)) {
4563 /* Unicast addresses changes may only happen under the rtnl,
4564 * therefore calling __dev_set_promiscuity here is safe.
4566 if (!netdev_uc_empty(dev
) && !dev
->uc_promisc
) {
4567 __dev_set_promiscuity(dev
, 1);
4568 dev
->uc_promisc
= true;
4569 } else if (netdev_uc_empty(dev
) && dev
->uc_promisc
) {
4570 __dev_set_promiscuity(dev
, -1);
4571 dev
->uc_promisc
= false;
4575 if (ops
->ndo_set_rx_mode
)
4576 ops
->ndo_set_rx_mode(dev
);
4579 void dev_set_rx_mode(struct net_device
*dev
)
4581 netif_addr_lock_bh(dev
);
4582 __dev_set_rx_mode(dev
);
4583 netif_addr_unlock_bh(dev
);
4587 * dev_get_flags - get flags reported to userspace
4590 * Get the combination of flag bits exported through APIs to userspace.
4592 unsigned dev_get_flags(const struct net_device
*dev
)
4596 flags
= (dev
->flags
& ~(IFF_PROMISC
|
4601 (dev
->gflags
& (IFF_PROMISC
|
4604 if (netif_running(dev
)) {
4605 if (netif_oper_up(dev
))
4606 flags
|= IFF_RUNNING
;
4607 if (netif_carrier_ok(dev
))
4608 flags
|= IFF_LOWER_UP
;
4609 if (netif_dormant(dev
))
4610 flags
|= IFF_DORMANT
;
4615 EXPORT_SYMBOL(dev_get_flags
);
4617 int __dev_change_flags(struct net_device
*dev
, unsigned int flags
)
4619 int old_flags
= dev
->flags
;
4625 * Set the flags on our device.
4628 dev
->flags
= (flags
& (IFF_DEBUG
| IFF_NOTRAILERS
| IFF_NOARP
|
4629 IFF_DYNAMIC
| IFF_MULTICAST
| IFF_PORTSEL
|
4631 (dev
->flags
& (IFF_UP
| IFF_VOLATILE
| IFF_PROMISC
|
4635 * Load in the correct multicast list now the flags have changed.
4638 if ((old_flags
^ flags
) & IFF_MULTICAST
)
4639 dev_change_rx_flags(dev
, IFF_MULTICAST
);
4641 dev_set_rx_mode(dev
);
4644 * Have we downed the interface. We handle IFF_UP ourselves
4645 * according to user attempts to set it, rather than blindly
4650 if ((old_flags
^ flags
) & IFF_UP
) { /* Bit is different ? */
4651 ret
= ((old_flags
& IFF_UP
) ? __dev_close
: __dev_open
)(dev
);
4654 dev_set_rx_mode(dev
);
4657 if ((flags
^ dev
->gflags
) & IFF_PROMISC
) {
4658 int inc
= (flags
& IFF_PROMISC
) ? 1 : -1;
4660 dev
->gflags
^= IFF_PROMISC
;
4661 dev_set_promiscuity(dev
, inc
);
4664 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4665 is important. Some (broken) drivers set IFF_PROMISC, when
4666 IFF_ALLMULTI is requested not asking us and not reporting.
4668 if ((flags
^ dev
->gflags
) & IFF_ALLMULTI
) {
4669 int inc
= (flags
& IFF_ALLMULTI
) ? 1 : -1;
4671 dev
->gflags
^= IFF_ALLMULTI
;
4672 dev_set_allmulti(dev
, inc
);
4678 void __dev_notify_flags(struct net_device
*dev
, unsigned int old_flags
)
4680 unsigned int changes
= dev
->flags
^ old_flags
;
4682 if (changes
& IFF_UP
) {
4683 if (dev
->flags
& IFF_UP
)
4684 call_netdevice_notifiers(NETDEV_UP
, dev
);
4686 call_netdevice_notifiers(NETDEV_DOWN
, dev
);
4689 if (dev
->flags
& IFF_UP
&&
4690 (changes
& ~(IFF_UP
| IFF_PROMISC
| IFF_ALLMULTI
| IFF_VOLATILE
)))
4691 call_netdevice_notifiers(NETDEV_CHANGE
, dev
);
4695 * dev_change_flags - change device settings
4697 * @flags: device state flags
4699 * Change settings on device based state flags. The flags are
4700 * in the userspace exported format.
4702 int dev_change_flags(struct net_device
*dev
, unsigned flags
)
4705 int old_flags
= dev
->flags
;
4707 ret
= __dev_change_flags(dev
, flags
);
4711 changes
= old_flags
^ dev
->flags
;
4713 rtmsg_ifinfo(RTM_NEWLINK
, dev
, changes
);
4715 __dev_notify_flags(dev
, old_flags
);
4718 EXPORT_SYMBOL(dev_change_flags
);
4721 * dev_set_mtu - Change maximum transfer unit
4723 * @new_mtu: new transfer unit
4725 * Change the maximum transfer size of the network device.
4727 int dev_set_mtu(struct net_device
*dev
, int new_mtu
)
4729 const struct net_device_ops
*ops
= dev
->netdev_ops
;
4732 if (new_mtu
== dev
->mtu
)
4735 /* MTU must be positive. */
4739 if (!netif_device_present(dev
))
4743 if (ops
->ndo_change_mtu
)
4744 err
= ops
->ndo_change_mtu(dev
, new_mtu
);
4748 if (!err
&& dev
->flags
& IFF_UP
)
4749 call_netdevice_notifiers(NETDEV_CHANGEMTU
, dev
);
4752 EXPORT_SYMBOL(dev_set_mtu
);
4755 * dev_set_group - Change group this device belongs to
4757 * @new_group: group this device should belong to
4759 void dev_set_group(struct net_device
*dev
, int new_group
)
4761 dev
->group
= new_group
;
4763 EXPORT_SYMBOL(dev_set_group
);
4766 * dev_set_mac_address - Change Media Access Control Address
4770 * Change the hardware (MAC) address of the device
4772 int dev_set_mac_address(struct net_device
*dev
, struct sockaddr
*sa
)
4774 const struct net_device_ops
*ops
= dev
->netdev_ops
;
4777 if (!ops
->ndo_set_mac_address
)
4779 if (sa
->sa_family
!= dev
->type
)
4781 if (!netif_device_present(dev
))
4783 err
= ops
->ndo_set_mac_address(dev
, sa
);
4785 call_netdevice_notifiers(NETDEV_CHANGEADDR
, dev
);
4788 EXPORT_SYMBOL(dev_set_mac_address
);
4791 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
4793 static int dev_ifsioc_locked(struct net
*net
, struct ifreq
*ifr
, unsigned int cmd
)
4796 struct net_device
*dev
= dev_get_by_name_rcu(net
, ifr
->ifr_name
);
4802 case SIOCGIFFLAGS
: /* Get interface flags */
4803 ifr
->ifr_flags
= (short) dev_get_flags(dev
);
4806 case SIOCGIFMETRIC
: /* Get the metric on the interface
4807 (currently unused) */
4808 ifr
->ifr_metric
= 0;
4811 case SIOCGIFMTU
: /* Get the MTU of a device */
4812 ifr
->ifr_mtu
= dev
->mtu
;
4817 memset(ifr
->ifr_hwaddr
.sa_data
, 0, sizeof ifr
->ifr_hwaddr
.sa_data
);
4819 memcpy(ifr
->ifr_hwaddr
.sa_data
, dev
->dev_addr
,
4820 min(sizeof ifr
->ifr_hwaddr
.sa_data
, (size_t) dev
->addr_len
));
4821 ifr
->ifr_hwaddr
.sa_family
= dev
->type
;
4829 ifr
->ifr_map
.mem_start
= dev
->mem_start
;
4830 ifr
->ifr_map
.mem_end
= dev
->mem_end
;
4831 ifr
->ifr_map
.base_addr
= dev
->base_addr
;
4832 ifr
->ifr_map
.irq
= dev
->irq
;
4833 ifr
->ifr_map
.dma
= dev
->dma
;
4834 ifr
->ifr_map
.port
= dev
->if_port
;
4838 ifr
->ifr_ifindex
= dev
->ifindex
;
4842 ifr
->ifr_qlen
= dev
->tx_queue_len
;
4846 /* dev_ioctl() should ensure this case
4858 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4860 static int dev_ifsioc(struct net
*net
, struct ifreq
*ifr
, unsigned int cmd
)
4863 struct net_device
*dev
= __dev_get_by_name(net
, ifr
->ifr_name
);
4864 const struct net_device_ops
*ops
;
4869 ops
= dev
->netdev_ops
;
4872 case SIOCSIFFLAGS
: /* Set interface flags */
4873 return dev_change_flags(dev
, ifr
->ifr_flags
);
4875 case SIOCSIFMETRIC
: /* Set the metric on the interface
4876 (currently unused) */
4879 case SIOCSIFMTU
: /* Set the MTU of a device */
4880 return dev_set_mtu(dev
, ifr
->ifr_mtu
);
4883 return dev_set_mac_address(dev
, &ifr
->ifr_hwaddr
);
4885 case SIOCSIFHWBROADCAST
:
4886 if (ifr
->ifr_hwaddr
.sa_family
!= dev
->type
)
4888 memcpy(dev
->broadcast
, ifr
->ifr_hwaddr
.sa_data
,
4889 min(sizeof ifr
->ifr_hwaddr
.sa_data
, (size_t) dev
->addr_len
));
4890 call_netdevice_notifiers(NETDEV_CHANGEADDR
, dev
);
4894 if (ops
->ndo_set_config
) {
4895 if (!netif_device_present(dev
))
4897 return ops
->ndo_set_config(dev
, &ifr
->ifr_map
);
4902 if (!ops
->ndo_set_rx_mode
||
4903 ifr
->ifr_hwaddr
.sa_family
!= AF_UNSPEC
)
4905 if (!netif_device_present(dev
))
4907 return dev_mc_add_global(dev
, ifr
->ifr_hwaddr
.sa_data
);
4910 if (!ops
->ndo_set_rx_mode
||
4911 ifr
->ifr_hwaddr
.sa_family
!= AF_UNSPEC
)
4913 if (!netif_device_present(dev
))
4915 return dev_mc_del_global(dev
, ifr
->ifr_hwaddr
.sa_data
);
4918 if (ifr
->ifr_qlen
< 0)
4920 dev
->tx_queue_len
= ifr
->ifr_qlen
;
4924 ifr
->ifr_newname
[IFNAMSIZ
-1] = '\0';
4925 return dev_change_name(dev
, ifr
->ifr_newname
);
4928 * Unknown or private ioctl
4931 if ((cmd
>= SIOCDEVPRIVATE
&&
4932 cmd
<= SIOCDEVPRIVATE
+ 15) ||
4933 cmd
== SIOCBONDENSLAVE
||
4934 cmd
== SIOCBONDRELEASE
||
4935 cmd
== SIOCBONDSETHWADDR
||
4936 cmd
== SIOCBONDSLAVEINFOQUERY
||
4937 cmd
== SIOCBONDINFOQUERY
||
4938 cmd
== SIOCBONDCHANGEACTIVE
||
4939 cmd
== SIOCGMIIPHY
||
4940 cmd
== SIOCGMIIREG
||
4941 cmd
== SIOCSMIIREG
||
4942 cmd
== SIOCBRADDIF
||
4943 cmd
== SIOCBRDELIF
||
4944 cmd
== SIOCSHWTSTAMP
||
4945 cmd
== SIOCWANDEV
) {
4947 if (ops
->ndo_do_ioctl
) {
4948 if (netif_device_present(dev
))
4949 err
= ops
->ndo_do_ioctl(dev
, ifr
, cmd
);
4961 * This function handles all "interface"-type I/O control requests. The actual
4962 * 'doing' part of this is dev_ifsioc above.
4966 * dev_ioctl - network device ioctl
4967 * @net: the applicable net namespace
4968 * @cmd: command to issue
4969 * @arg: pointer to a struct ifreq in user space
4971 * Issue ioctl functions to devices. This is normally called by the
4972 * user space syscall interfaces but can sometimes be useful for
4973 * other purposes. The return value is the return from the syscall if
4974 * positive or a negative errno code on error.
4977 int dev_ioctl(struct net
*net
, unsigned int cmd
, void __user
*arg
)
4983 /* One special case: SIOCGIFCONF takes ifconf argument
4984 and requires shared lock, because it sleeps writing
4988 if (cmd
== SIOCGIFCONF
) {
4990 ret
= dev_ifconf(net
, (char __user
*) arg
);
4994 if (cmd
== SIOCGIFNAME
)
4995 return dev_ifname(net
, (struct ifreq __user
*)arg
);
4997 if (copy_from_user(&ifr
, arg
, sizeof(struct ifreq
)))
5000 ifr
.ifr_name
[IFNAMSIZ
-1] = 0;
5002 colon
= strchr(ifr
.ifr_name
, ':');
5007 * See which interface the caller is talking about.
5012 * These ioctl calls:
5013 * - can be done by all.
5014 * - atomic and do not require locking.
5025 dev_load(net
, ifr
.ifr_name
);
5027 ret
= dev_ifsioc_locked(net
, &ifr
, cmd
);
5032 if (copy_to_user(arg
, &ifr
,
5033 sizeof(struct ifreq
)))
5039 dev_load(net
, ifr
.ifr_name
);
5041 ret
= dev_ethtool(net
, &ifr
);
5046 if (copy_to_user(arg
, &ifr
,
5047 sizeof(struct ifreq
)))
5053 * These ioctl calls:
5054 * - require superuser power.
5055 * - require strict serialization.
5061 if (!capable(CAP_NET_ADMIN
))
5063 dev_load(net
, ifr
.ifr_name
);
5065 ret
= dev_ifsioc(net
, &ifr
, cmd
);
5070 if (copy_to_user(arg
, &ifr
,
5071 sizeof(struct ifreq
)))
5077 * These ioctl calls:
5078 * - require superuser power.
5079 * - require strict serialization.
5080 * - do not return a value
5090 case SIOCSIFHWBROADCAST
:
5093 case SIOCBONDENSLAVE
:
5094 case SIOCBONDRELEASE
:
5095 case SIOCBONDSETHWADDR
:
5096 case SIOCBONDCHANGEACTIVE
:
5100 if (!capable(CAP_NET_ADMIN
))
5103 case SIOCBONDSLAVEINFOQUERY
:
5104 case SIOCBONDINFOQUERY
:
5105 dev_load(net
, ifr
.ifr_name
);
5107 ret
= dev_ifsioc(net
, &ifr
, cmd
);
5112 /* Get the per device memory space. We can add this but
5113 * currently do not support it */
5115 /* Set the per device memory buffer space.
5116 * Not applicable in our case */
5121 * Unknown or private ioctl.
5124 if (cmd
== SIOCWANDEV
||
5125 (cmd
>= SIOCDEVPRIVATE
&&
5126 cmd
<= SIOCDEVPRIVATE
+ 15)) {
5127 dev_load(net
, ifr
.ifr_name
);
5129 ret
= dev_ifsioc(net
, &ifr
, cmd
);
5131 if (!ret
&& copy_to_user(arg
, &ifr
,
5132 sizeof(struct ifreq
)))
5136 /* Take care of Wireless Extensions */
5137 if (cmd
>= SIOCIWFIRST
&& cmd
<= SIOCIWLAST
)
5138 return wext_handle_ioctl(net
, &ifr
, cmd
, arg
);
5145 * dev_new_index - allocate an ifindex
5146 * @net: the applicable net namespace
5148 * Returns a suitable unique value for a new device interface
5149 * number. The caller must hold the rtnl semaphore or the
5150 * dev_base_lock to be sure it remains unique.
5152 static int dev_new_index(struct net
*net
)
5158 if (!__dev_get_by_index(net
, ifindex
))
5163 /* Delayed registration/unregisteration */
5164 static LIST_HEAD(net_todo_list
);
5166 static void net_set_todo(struct net_device
*dev
)
5168 list_add_tail(&dev
->todo_list
, &net_todo_list
);
5171 static void rollback_registered_many(struct list_head
*head
)
5173 struct net_device
*dev
, *tmp
;
5175 BUG_ON(dev_boot_phase
);
5178 list_for_each_entry_safe(dev
, tmp
, head
, unreg_list
) {
5179 /* Some devices call without registering
5180 * for initialization unwind. Remove those
5181 * devices and proceed with the remaining.
5183 if (dev
->reg_state
== NETREG_UNINITIALIZED
) {
5184 pr_debug("unregister_netdevice: device %s/%p never "
5185 "was registered\n", dev
->name
, dev
);
5188 list_del(&dev
->unreg_list
);
5191 dev
->dismantle
= true;
5192 BUG_ON(dev
->reg_state
!= NETREG_REGISTERED
);
5195 /* If device is running, close it first. */
5196 dev_close_many(head
);
5198 list_for_each_entry(dev
, head
, unreg_list
) {
5199 /* And unlink it from device chain. */
5200 unlist_netdevice(dev
);
5202 dev
->reg_state
= NETREG_UNREGISTERING
;
5207 list_for_each_entry(dev
, head
, unreg_list
) {
5208 /* Shutdown queueing discipline. */
5212 /* Notify protocols, that we are about to destroy
5213 this device. They should clean all the things.
5215 call_netdevice_notifiers(NETDEV_UNREGISTER
, dev
);
5217 if (!dev
->rtnl_link_ops
||
5218 dev
->rtnl_link_state
== RTNL_LINK_INITIALIZED
)
5219 rtmsg_ifinfo(RTM_DELLINK
, dev
, ~0U);
5222 * Flush the unicast and multicast chains
5227 if (dev
->netdev_ops
->ndo_uninit
)
5228 dev
->netdev_ops
->ndo_uninit(dev
);
5230 /* Notifier chain MUST detach us from master device. */
5231 WARN_ON(dev
->master
);
5233 /* Remove entries from kobject tree */
5234 netdev_unregister_kobject(dev
);
5237 /* Process any work delayed until the end of the batch */
5238 dev
= list_first_entry(head
, struct net_device
, unreg_list
);
5239 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH
, dev
);
5243 list_for_each_entry(dev
, head
, unreg_list
)
5247 static void rollback_registered(struct net_device
*dev
)
5251 list_add(&dev
->unreg_list
, &single
);
5252 rollback_registered_many(&single
);
5256 static u32
netdev_fix_features(struct net_device
*dev
, u32 features
)
5258 /* Fix illegal checksum combinations */
5259 if ((features
& NETIF_F_HW_CSUM
) &&
5260 (features
& (NETIF_F_IP_CSUM
|NETIF_F_IPV6_CSUM
))) {
5261 netdev_warn(dev
, "mixed HW and IP checksum settings.\n");
5262 features
&= ~(NETIF_F_IP_CSUM
|NETIF_F_IPV6_CSUM
);
5265 if ((features
& NETIF_F_NO_CSUM
) &&
5266 (features
& (NETIF_F_HW_CSUM
|NETIF_F_IP_CSUM
|NETIF_F_IPV6_CSUM
))) {
5267 netdev_warn(dev
, "mixed no checksumming and other settings.\n");
5268 features
&= ~(NETIF_F_IP_CSUM
|NETIF_F_IPV6_CSUM
|NETIF_F_HW_CSUM
);
5271 /* Fix illegal SG+CSUM combinations. */
5272 if ((features
& NETIF_F_SG
) &&
5273 !(features
& NETIF_F_ALL_CSUM
)) {
5275 "Dropping NETIF_F_SG since no checksum feature.\n");
5276 features
&= ~NETIF_F_SG
;
5279 /* TSO requires that SG is present as well. */
5280 if ((features
& NETIF_F_ALL_TSO
) && !(features
& NETIF_F_SG
)) {
5281 netdev_dbg(dev
, "Dropping TSO features since no SG feature.\n");
5282 features
&= ~NETIF_F_ALL_TSO
;
5285 /* TSO ECN requires that TSO is present as well. */
5286 if ((features
& NETIF_F_ALL_TSO
) == NETIF_F_TSO_ECN
)
5287 features
&= ~NETIF_F_TSO_ECN
;
5289 /* Software GSO depends on SG. */
5290 if ((features
& NETIF_F_GSO
) && !(features
& NETIF_F_SG
)) {
5291 netdev_dbg(dev
, "Dropping NETIF_F_GSO since no SG feature.\n");
5292 features
&= ~NETIF_F_GSO
;
5295 /* UFO needs SG and checksumming */
5296 if (features
& NETIF_F_UFO
) {
5297 /* maybe split UFO into V4 and V6? */
5298 if (!((features
& NETIF_F_GEN_CSUM
) ||
5299 (features
& (NETIF_F_IP_CSUM
|NETIF_F_IPV6_CSUM
))
5300 == (NETIF_F_IP_CSUM
|NETIF_F_IPV6_CSUM
))) {
5302 "Dropping NETIF_F_UFO since no checksum offload features.\n");
5303 features
&= ~NETIF_F_UFO
;
5306 if (!(features
& NETIF_F_SG
)) {
5308 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
5309 features
&= ~NETIF_F_UFO
;
5316 int __netdev_update_features(struct net_device
*dev
)
5323 features
= netdev_get_wanted_features(dev
);
5325 if (dev
->netdev_ops
->ndo_fix_features
)
5326 features
= dev
->netdev_ops
->ndo_fix_features(dev
, features
);
5328 /* driver might be less strict about feature dependencies */
5329 features
= netdev_fix_features(dev
, features
);
5331 if (dev
->features
== features
)
5334 netdev_dbg(dev
, "Features changed: 0x%08x -> 0x%08x\n",
5335 dev
->features
, features
);
5337 if (dev
->netdev_ops
->ndo_set_features
)
5338 err
= dev
->netdev_ops
->ndo_set_features(dev
, features
);
5340 if (unlikely(err
< 0)) {
5342 "set_features() failed (%d); wanted 0x%08x, left 0x%08x\n",
5343 err
, features
, dev
->features
);
5348 dev
->features
= features
;
5354 * netdev_update_features - recalculate device features
5355 * @dev: the device to check
5357 * Recalculate dev->features set and send notifications if it
5358 * has changed. Should be called after driver or hardware dependent
5359 * conditions might have changed that influence the features.
5361 void netdev_update_features(struct net_device
*dev
)
5363 if (__netdev_update_features(dev
))
5364 netdev_features_change(dev
);
5366 EXPORT_SYMBOL(netdev_update_features
);
5369 * netdev_change_features - recalculate device features
5370 * @dev: the device to check
5372 * Recalculate dev->features set and send notifications even
5373 * if they have not changed. Should be called instead of
5374 * netdev_update_features() if also dev->vlan_features might
5375 * have changed to allow the changes to be propagated to stacked
5378 void netdev_change_features(struct net_device
*dev
)
5380 __netdev_update_features(dev
);
5381 netdev_features_change(dev
);
5383 EXPORT_SYMBOL(netdev_change_features
);
5386 * netif_stacked_transfer_operstate - transfer operstate
5387 * @rootdev: the root or lower level device to transfer state from
5388 * @dev: the device to transfer operstate to
5390 * Transfer operational state from root to device. This is normally
5391 * called when a stacking relationship exists between the root
5392 * device and the device(a leaf device).
5394 void netif_stacked_transfer_operstate(const struct net_device
*rootdev
,
5395 struct net_device
*dev
)
5397 if (rootdev
->operstate
== IF_OPER_DORMANT
)
5398 netif_dormant_on(dev
);
5400 netif_dormant_off(dev
);
5402 if (netif_carrier_ok(rootdev
)) {
5403 if (!netif_carrier_ok(dev
))
5404 netif_carrier_on(dev
);
5406 if (netif_carrier_ok(dev
))
5407 netif_carrier_off(dev
);
5410 EXPORT_SYMBOL(netif_stacked_transfer_operstate
);
5413 static int netif_alloc_rx_queues(struct net_device
*dev
)
5415 unsigned int i
, count
= dev
->num_rx_queues
;
5416 struct netdev_rx_queue
*rx
;
5420 rx
= kcalloc(count
, sizeof(struct netdev_rx_queue
), GFP_KERNEL
);
5422 pr_err("netdev: Unable to allocate %u rx queues.\n", count
);
5427 for (i
= 0; i
< count
; i
++)
5433 static void netdev_init_one_queue(struct net_device
*dev
,
5434 struct netdev_queue
*queue
, void *_unused
)
5436 /* Initialize queue lock */
5437 spin_lock_init(&queue
->_xmit_lock
);
5438 netdev_set_xmit_lockdep_class(&queue
->_xmit_lock
, dev
->type
);
5439 queue
->xmit_lock_owner
= -1;
5440 netdev_queue_numa_node_write(queue
, NUMA_NO_NODE
);
5444 static int netif_alloc_netdev_queues(struct net_device
*dev
)
5446 unsigned int count
= dev
->num_tx_queues
;
5447 struct netdev_queue
*tx
;
5451 tx
= kcalloc(count
, sizeof(struct netdev_queue
), GFP_KERNEL
);
5453 pr_err("netdev: Unable to allocate %u tx queues.\n",
5459 netdev_for_each_tx_queue(dev
, netdev_init_one_queue
, NULL
);
5460 spin_lock_init(&dev
->tx_global_lock
);
5466 * register_netdevice - register a network device
5467 * @dev: device to register
5469 * Take a completed network device structure and add it to the kernel
5470 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5471 * chain. 0 is returned on success. A negative errno code is returned
5472 * on a failure to set up the device, or if the name is a duplicate.
5474 * Callers must hold the rtnl semaphore. You may want
5475 * register_netdev() instead of this.
5478 * The locking appears insufficient to guarantee two parallel registers
5479 * will not get the same name.
5482 int register_netdevice(struct net_device
*dev
)
5485 struct net
*net
= dev_net(dev
);
5487 BUG_ON(dev_boot_phase
);
5492 /* When net_device's are persistent, this will be fatal. */
5493 BUG_ON(dev
->reg_state
!= NETREG_UNINITIALIZED
);
5496 spin_lock_init(&dev
->addr_list_lock
);
5497 netdev_set_addr_lockdep_class(dev
);
5501 ret
= dev_get_valid_name(dev
, dev
->name
);
5505 /* Init, if this function is available */
5506 if (dev
->netdev_ops
->ndo_init
) {
5507 ret
= dev
->netdev_ops
->ndo_init(dev
);
5515 dev
->ifindex
= dev_new_index(net
);
5516 if (dev
->iflink
== -1)
5517 dev
->iflink
= dev
->ifindex
;
5519 /* Transfer changeable features to wanted_features and enable
5520 * software offloads (GSO and GRO).
5522 dev
->hw_features
|= NETIF_F_SOFT_FEATURES
;
5523 dev
->features
|= NETIF_F_SOFT_FEATURES
;
5524 dev
->wanted_features
= dev
->features
& dev
->hw_features
;
5526 /* Turn on no cache copy if HW is doing checksum */
5527 dev
->hw_features
|= NETIF_F_NOCACHE_COPY
;
5528 if ((dev
->features
& NETIF_F_ALL_CSUM
) &&
5529 !(dev
->features
& NETIF_F_NO_CSUM
)) {
5530 dev
->wanted_features
|= NETIF_F_NOCACHE_COPY
;
5531 dev
->features
|= NETIF_F_NOCACHE_COPY
;
5534 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
5536 dev
->vlan_features
|= NETIF_F_HIGHDMA
;
5538 ret
= call_netdevice_notifiers(NETDEV_POST_INIT
, dev
);
5539 ret
= notifier_to_errno(ret
);
5543 ret
= netdev_register_kobject(dev
);
5546 dev
->reg_state
= NETREG_REGISTERED
;
5548 __netdev_update_features(dev
);
5551 * Default initial state at registry is that the
5552 * device is present.
5555 set_bit(__LINK_STATE_PRESENT
, &dev
->state
);
5557 dev_init_scheduler(dev
);
5559 list_netdevice(dev
);
5561 /* Notify protocols, that a new device appeared. */
5562 ret
= call_netdevice_notifiers(NETDEV_REGISTER
, dev
);
5563 ret
= notifier_to_errno(ret
);
5565 rollback_registered(dev
);
5566 dev
->reg_state
= NETREG_UNREGISTERED
;
5569 * Prevent userspace races by waiting until the network
5570 * device is fully setup before sending notifications.
5572 if (!dev
->rtnl_link_ops
||
5573 dev
->rtnl_link_state
== RTNL_LINK_INITIALIZED
)
5574 rtmsg_ifinfo(RTM_NEWLINK
, dev
, ~0U);
5580 if (dev
->netdev_ops
->ndo_uninit
)
5581 dev
->netdev_ops
->ndo_uninit(dev
);
5584 EXPORT_SYMBOL(register_netdevice
);
5587 * init_dummy_netdev - init a dummy network device for NAPI
5588 * @dev: device to init
5590 * This takes a network device structure and initialize the minimum
5591 * amount of fields so it can be used to schedule NAPI polls without
5592 * registering a full blown interface. This is to be used by drivers
5593 * that need to tie several hardware interfaces to a single NAPI
5594 * poll scheduler due to HW limitations.
5596 int init_dummy_netdev(struct net_device
*dev
)
5598 /* Clear everything. Note we don't initialize spinlocks
5599 * are they aren't supposed to be taken by any of the
5600 * NAPI code and this dummy netdev is supposed to be
5601 * only ever used for NAPI polls
5603 memset(dev
, 0, sizeof(struct net_device
));
5605 /* make sure we BUG if trying to hit standard
5606 * register/unregister code path
5608 dev
->reg_state
= NETREG_DUMMY
;
5610 /* NAPI wants this */
5611 INIT_LIST_HEAD(&dev
->napi_list
);
5613 /* a dummy interface is started by default */
5614 set_bit(__LINK_STATE_PRESENT
, &dev
->state
);
5615 set_bit(__LINK_STATE_START
, &dev
->state
);
5617 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5618 * because users of this 'device' dont need to change
5624 EXPORT_SYMBOL_GPL(init_dummy_netdev
);
5628 * register_netdev - register a network device
5629 * @dev: device to register
5631 * Take a completed network device structure and add it to the kernel
5632 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5633 * chain. 0 is returned on success. A negative errno code is returned
5634 * on a failure to set up the device, or if the name is a duplicate.
5636 * This is a wrapper around register_netdevice that takes the rtnl semaphore
5637 * and expands the device name if you passed a format string to
5640 int register_netdev(struct net_device
*dev
)
5645 err
= register_netdevice(dev
);
5649 EXPORT_SYMBOL(register_netdev
);
5651 int netdev_refcnt_read(const struct net_device
*dev
)
5655 for_each_possible_cpu(i
)
5656 refcnt
+= *per_cpu_ptr(dev
->pcpu_refcnt
, i
);
5659 EXPORT_SYMBOL(netdev_refcnt_read
);
5662 * netdev_wait_allrefs - wait until all references are gone.
5664 * This is called when unregistering network devices.
5666 * Any protocol or device that holds a reference should register
5667 * for netdevice notification, and cleanup and put back the
5668 * reference if they receive an UNREGISTER event.
5669 * We can get stuck here if buggy protocols don't correctly
5672 static void netdev_wait_allrefs(struct net_device
*dev
)
5674 unsigned long rebroadcast_time
, warning_time
;
5677 linkwatch_forget_dev(dev
);
5679 rebroadcast_time
= warning_time
= jiffies
;
5680 refcnt
= netdev_refcnt_read(dev
);
5682 while (refcnt
!= 0) {
5683 if (time_after(jiffies
, rebroadcast_time
+ 1 * HZ
)) {
5686 /* Rebroadcast unregister notification */
5687 call_netdevice_notifiers(NETDEV_UNREGISTER
, dev
);
5688 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
5689 * should have already handle it the first time */
5691 if (test_bit(__LINK_STATE_LINKWATCH_PENDING
,
5693 /* We must not have linkwatch events
5694 * pending on unregister. If this
5695 * happens, we simply run the queue
5696 * unscheduled, resulting in a noop
5699 linkwatch_run_queue();
5704 rebroadcast_time
= jiffies
;
5709 refcnt
= netdev_refcnt_read(dev
);
5711 if (time_after(jiffies
, warning_time
+ 10 * HZ
)) {
5712 printk(KERN_EMERG
"unregister_netdevice: "
5713 "waiting for %s to become free. Usage "
5716 warning_time
= jiffies
;
5725 * register_netdevice(x1);
5726 * register_netdevice(x2);
5728 * unregister_netdevice(y1);
5729 * unregister_netdevice(y2);
5735 * We are invoked by rtnl_unlock().
5736 * This allows us to deal with problems:
5737 * 1) We can delete sysfs objects which invoke hotplug
5738 * without deadlocking with linkwatch via keventd.
5739 * 2) Since we run with the RTNL semaphore not held, we can sleep
5740 * safely in order to wait for the netdev refcnt to drop to zero.
5742 * We must not return until all unregister events added during
5743 * the interval the lock was held have been completed.
5745 void netdev_run_todo(void)
5747 struct list_head list
;
5749 /* Snapshot list, allow later requests */
5750 list_replace_init(&net_todo_list
, &list
);
5754 while (!list_empty(&list
)) {
5755 struct net_device
*dev
5756 = list_first_entry(&list
, struct net_device
, todo_list
);
5757 list_del(&dev
->todo_list
);
5759 if (unlikely(dev
->reg_state
!= NETREG_UNREGISTERING
)) {
5760 printk(KERN_ERR
"network todo '%s' but state %d\n",
5761 dev
->name
, dev
->reg_state
);
5766 dev
->reg_state
= NETREG_UNREGISTERED
;
5768 on_each_cpu(flush_backlog
, dev
, 1);
5770 netdev_wait_allrefs(dev
);
5773 BUG_ON(netdev_refcnt_read(dev
));
5774 WARN_ON(rcu_access_pointer(dev
->ip_ptr
));
5775 WARN_ON(rcu_access_pointer(dev
->ip6_ptr
));
5776 WARN_ON(dev
->dn_ptr
);
5778 if (dev
->destructor
)
5779 dev
->destructor(dev
);
5781 /* Free network device */
5782 kobject_put(&dev
->dev
.kobj
);
5786 /* Convert net_device_stats to rtnl_link_stats64. They have the same
5787 * fields in the same order, with only the type differing.
5789 static void netdev_stats_to_stats64(struct rtnl_link_stats64
*stats64
,
5790 const struct net_device_stats
*netdev_stats
)
5792 #if BITS_PER_LONG == 64
5793 BUILD_BUG_ON(sizeof(*stats64
) != sizeof(*netdev_stats
));
5794 memcpy(stats64
, netdev_stats
, sizeof(*stats64
));
5796 size_t i
, n
= sizeof(*stats64
) / sizeof(u64
);
5797 const unsigned long *src
= (const unsigned long *)netdev_stats
;
5798 u64
*dst
= (u64
*)stats64
;
5800 BUILD_BUG_ON(sizeof(*netdev_stats
) / sizeof(unsigned long) !=
5801 sizeof(*stats64
) / sizeof(u64
));
5802 for (i
= 0; i
< n
; i
++)
5808 * dev_get_stats - get network device statistics
5809 * @dev: device to get statistics from
5810 * @storage: place to store stats
5812 * Get network statistics from device. Return @storage.
5813 * The device driver may provide its own method by setting
5814 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5815 * otherwise the internal statistics structure is used.
5817 struct rtnl_link_stats64
*dev_get_stats(struct net_device
*dev
,
5818 struct rtnl_link_stats64
*storage
)
5820 const struct net_device_ops
*ops
= dev
->netdev_ops
;
5822 if (ops
->ndo_get_stats64
) {
5823 memset(storage
, 0, sizeof(*storage
));
5824 ops
->ndo_get_stats64(dev
, storage
);
5825 } else if (ops
->ndo_get_stats
) {
5826 netdev_stats_to_stats64(storage
, ops
->ndo_get_stats(dev
));
5828 netdev_stats_to_stats64(storage
, &dev
->stats
);
5830 storage
->rx_dropped
+= atomic_long_read(&dev
->rx_dropped
);
5833 EXPORT_SYMBOL(dev_get_stats
);
5835 struct netdev_queue
*dev_ingress_queue_create(struct net_device
*dev
)
5837 struct netdev_queue
*queue
= dev_ingress_queue(dev
);
5839 #ifdef CONFIG_NET_CLS_ACT
5842 queue
= kzalloc(sizeof(*queue
), GFP_KERNEL
);
5845 netdev_init_one_queue(dev
, queue
, NULL
);
5846 queue
->qdisc
= &noop_qdisc
;
5847 queue
->qdisc_sleeping
= &noop_qdisc
;
5848 rcu_assign_pointer(dev
->ingress_queue
, queue
);
5854 * alloc_netdev_mqs - allocate network device
5855 * @sizeof_priv: size of private data to allocate space for
5856 * @name: device name format string
5857 * @setup: callback to initialize device
5858 * @txqs: the number of TX subqueues to allocate
5859 * @rxqs: the number of RX subqueues to allocate
5861 * Allocates a struct net_device with private data area for driver use
5862 * and performs basic initialization. Also allocates subquue structs
5863 * for each queue on the device.
5865 struct net_device
*alloc_netdev_mqs(int sizeof_priv
, const char *name
,
5866 void (*setup
)(struct net_device
*),
5867 unsigned int txqs
, unsigned int rxqs
)
5869 struct net_device
*dev
;
5871 struct net_device
*p
;
5873 BUG_ON(strlen(name
) >= sizeof(dev
->name
));
5876 pr_err("alloc_netdev: Unable to allocate device "
5877 "with zero queues.\n");
5883 pr_err("alloc_netdev: Unable to allocate device "
5884 "with zero RX queues.\n");
5889 alloc_size
= sizeof(struct net_device
);
5891 /* ensure 32-byte alignment of private area */
5892 alloc_size
= ALIGN(alloc_size
, NETDEV_ALIGN
);
5893 alloc_size
+= sizeof_priv
;
5895 /* ensure 32-byte alignment of whole construct */
5896 alloc_size
+= NETDEV_ALIGN
- 1;
5898 p
= kzalloc(alloc_size
, GFP_KERNEL
);
5900 printk(KERN_ERR
"alloc_netdev: Unable to allocate device.\n");
5904 dev
= PTR_ALIGN(p
, NETDEV_ALIGN
);
5905 dev
->padded
= (char *)dev
- (char *)p
;
5907 dev
->pcpu_refcnt
= alloc_percpu(int);
5908 if (!dev
->pcpu_refcnt
)
5911 if (dev_addr_init(dev
))
5917 dev_net_set(dev
, &init_net
);
5919 dev
->gso_max_size
= GSO_MAX_SIZE
;
5921 INIT_LIST_HEAD(&dev
->napi_list
);
5922 INIT_LIST_HEAD(&dev
->unreg_list
);
5923 INIT_LIST_HEAD(&dev
->link_watch_list
);
5924 dev
->priv_flags
= IFF_XMIT_DST_RELEASE
;
5927 dev
->num_tx_queues
= txqs
;
5928 dev
->real_num_tx_queues
= txqs
;
5929 if (netif_alloc_netdev_queues(dev
))
5933 dev
->num_rx_queues
= rxqs
;
5934 dev
->real_num_rx_queues
= rxqs
;
5935 if (netif_alloc_rx_queues(dev
))
5939 strcpy(dev
->name
, name
);
5940 dev
->group
= INIT_NETDEV_GROUP
;
5948 free_percpu(dev
->pcpu_refcnt
);
5958 EXPORT_SYMBOL(alloc_netdev_mqs
);
5961 * free_netdev - free network device
5964 * This function does the last stage of destroying an allocated device
5965 * interface. The reference to the device object is released.
5966 * If this is the last reference then it will be freed.
5968 void free_netdev(struct net_device
*dev
)
5970 struct napi_struct
*p
, *n
;
5972 release_net(dev_net(dev
));
5979 kfree(rcu_dereference_protected(dev
->ingress_queue
, 1));
5981 /* Flush device addresses */
5982 dev_addr_flush(dev
);
5984 list_for_each_entry_safe(p
, n
, &dev
->napi_list
, dev_list
)
5987 free_percpu(dev
->pcpu_refcnt
);
5988 dev
->pcpu_refcnt
= NULL
;
5990 /* Compatibility with error handling in drivers */
5991 if (dev
->reg_state
== NETREG_UNINITIALIZED
) {
5992 kfree((char *)dev
- dev
->padded
);
5996 BUG_ON(dev
->reg_state
!= NETREG_UNREGISTERED
);
5997 dev
->reg_state
= NETREG_RELEASED
;
5999 /* will free via device release */
6000 put_device(&dev
->dev
);
6002 EXPORT_SYMBOL(free_netdev
);
6005 * synchronize_net - Synchronize with packet receive processing
6007 * Wait for packets currently being received to be done.
6008 * Does not block later packets from starting.
6010 void synchronize_net(void)
6013 if (rtnl_is_locked())
6014 synchronize_rcu_expedited();
6018 EXPORT_SYMBOL(synchronize_net
);
6021 * unregister_netdevice_queue - remove device from the kernel
6025 * This function shuts down a device interface and removes it
6026 * from the kernel tables.
6027 * If head not NULL, device is queued to be unregistered later.
6029 * Callers must hold the rtnl semaphore. You may want
6030 * unregister_netdev() instead of this.
6033 void unregister_netdevice_queue(struct net_device
*dev
, struct list_head
*head
)
6038 list_move_tail(&dev
->unreg_list
, head
);
6040 rollback_registered(dev
);
6041 /* Finish processing unregister after unlock */
6045 EXPORT_SYMBOL(unregister_netdevice_queue
);
6048 * unregister_netdevice_many - unregister many devices
6049 * @head: list of devices
6051 void unregister_netdevice_many(struct list_head
*head
)
6053 struct net_device
*dev
;
6055 if (!list_empty(head
)) {
6056 rollback_registered_many(head
);
6057 list_for_each_entry(dev
, head
, unreg_list
)
6061 EXPORT_SYMBOL(unregister_netdevice_many
);
6064 * unregister_netdev - remove device from the kernel
6067 * This function shuts down a device interface and removes it
6068 * from the kernel tables.
6070 * This is just a wrapper for unregister_netdevice that takes
6071 * the rtnl semaphore. In general you want to use this and not
6072 * unregister_netdevice.
6074 void unregister_netdev(struct net_device
*dev
)
6077 unregister_netdevice(dev
);
6080 EXPORT_SYMBOL(unregister_netdev
);
6083 * dev_change_net_namespace - move device to different nethost namespace
6085 * @net: network namespace
6086 * @pat: If not NULL name pattern to try if the current device name
6087 * is already taken in the destination network namespace.
6089 * This function shuts down a device interface and moves it
6090 * to a new network namespace. On success 0 is returned, on
6091 * a failure a netagive errno code is returned.
6093 * Callers must hold the rtnl semaphore.
6096 int dev_change_net_namespace(struct net_device
*dev
, struct net
*net
, const char *pat
)
6102 /* Don't allow namespace local devices to be moved. */
6104 if (dev
->features
& NETIF_F_NETNS_LOCAL
)
6107 /* Ensure the device has been registrered */
6109 if (dev
->reg_state
!= NETREG_REGISTERED
)
6112 /* Get out if there is nothing todo */
6114 if (net_eq(dev_net(dev
), net
))
6117 /* Pick the destination device name, and ensure
6118 * we can use it in the destination network namespace.
6121 if (__dev_get_by_name(net
, dev
->name
)) {
6122 /* We get here if we can't use the current device name */
6125 if (dev_get_valid_name(dev
, pat
) < 0)
6130 * And now a mini version of register_netdevice unregister_netdevice.
6133 /* If device is running close it first. */
6136 /* And unlink it from device chain */
6138 unlist_netdevice(dev
);
6142 /* Shutdown queueing discipline. */
6145 /* Notify protocols, that we are about to destroy
6146 this device. They should clean all the things.
6148 Note that dev->reg_state stays at NETREG_REGISTERED.
6149 This is wanted because this way 8021q and macvlan know
6150 the device is just moving and can keep their slaves up.
6152 call_netdevice_notifiers(NETDEV_UNREGISTER
, dev
);
6153 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH
, dev
);
6156 * Flush the unicast and multicast chains
6161 /* Actually switch the network namespace */
6162 dev_net_set(dev
, net
);
6164 /* If there is an ifindex conflict assign a new one */
6165 if (__dev_get_by_index(net
, dev
->ifindex
)) {
6166 int iflink
= (dev
->iflink
== dev
->ifindex
);
6167 dev
->ifindex
= dev_new_index(net
);
6169 dev
->iflink
= dev
->ifindex
;
6172 /* Fixup kobjects */
6173 err
= device_rename(&dev
->dev
, dev
->name
);
6176 /* Add the device back in the hashes */
6177 list_netdevice(dev
);
6179 /* Notify protocols, that a new device appeared. */
6180 call_netdevice_notifiers(NETDEV_REGISTER
, dev
);
6183 * Prevent userspace races by waiting until the network
6184 * device is fully setup before sending notifications.
6186 rtmsg_ifinfo(RTM_NEWLINK
, dev
, ~0U);
6193 EXPORT_SYMBOL_GPL(dev_change_net_namespace
);
6195 static int dev_cpu_callback(struct notifier_block
*nfb
,
6196 unsigned long action
,
6199 struct sk_buff
**list_skb
;
6200 struct sk_buff
*skb
;
6201 unsigned int cpu
, oldcpu
= (unsigned long)ocpu
;
6202 struct softnet_data
*sd
, *oldsd
;
6204 if (action
!= CPU_DEAD
&& action
!= CPU_DEAD_FROZEN
)
6207 local_irq_disable();
6208 cpu
= smp_processor_id();
6209 sd
= &per_cpu(softnet_data
, cpu
);
6210 oldsd
= &per_cpu(softnet_data
, oldcpu
);
6212 /* Find end of our completion_queue. */
6213 list_skb
= &sd
->completion_queue
;
6215 list_skb
= &(*list_skb
)->next
;
6216 /* Append completion queue from offline CPU. */
6217 *list_skb
= oldsd
->completion_queue
;
6218 oldsd
->completion_queue
= NULL
;
6220 /* Append output queue from offline CPU. */
6221 if (oldsd
->output_queue
) {
6222 *sd
->output_queue_tailp
= oldsd
->output_queue
;
6223 sd
->output_queue_tailp
= oldsd
->output_queue_tailp
;
6224 oldsd
->output_queue
= NULL
;
6225 oldsd
->output_queue_tailp
= &oldsd
->output_queue
;
6227 /* Append NAPI poll list from offline CPU. */
6228 if (!list_empty(&oldsd
->poll_list
)) {
6229 list_splice_init(&oldsd
->poll_list
, &sd
->poll_list
);
6230 raise_softirq_irqoff(NET_RX_SOFTIRQ
);
6233 raise_softirq_irqoff(NET_TX_SOFTIRQ
);
6236 /* Process offline CPU's input_pkt_queue */
6237 while ((skb
= __skb_dequeue(&oldsd
->process_queue
))) {
6239 input_queue_head_incr(oldsd
);
6241 while ((skb
= __skb_dequeue(&oldsd
->input_pkt_queue
))) {
6243 input_queue_head_incr(oldsd
);
6251 * netdev_increment_features - increment feature set by one
6252 * @all: current feature set
6253 * @one: new feature set
6254 * @mask: mask feature set
6256 * Computes a new feature set after adding a device with feature set
6257 * @one to the master device with current feature set @all. Will not
6258 * enable anything that is off in @mask. Returns the new feature set.
6260 u32
netdev_increment_features(u32 all
, u32 one
, u32 mask
)
6262 if (mask
& NETIF_F_GEN_CSUM
)
6263 mask
|= NETIF_F_ALL_CSUM
;
6264 mask
|= NETIF_F_VLAN_CHALLENGED
;
6266 all
|= one
& (NETIF_F_ONE_FOR_ALL
|NETIF_F_ALL_CSUM
) & mask
;
6267 all
&= one
| ~NETIF_F_ALL_FOR_ALL
;
6269 /* If device needs checksumming, downgrade to it. */
6270 if (all
& (NETIF_F_ALL_CSUM
& ~NETIF_F_NO_CSUM
))
6271 all
&= ~NETIF_F_NO_CSUM
;
6273 /* If one device supports hw checksumming, set for all. */
6274 if (all
& NETIF_F_GEN_CSUM
)
6275 all
&= ~(NETIF_F_ALL_CSUM
& ~NETIF_F_GEN_CSUM
);
6279 EXPORT_SYMBOL(netdev_increment_features
);
6281 static struct hlist_head
*netdev_create_hash(void)
6284 struct hlist_head
*hash
;
6286 hash
= kmalloc(sizeof(*hash
) * NETDEV_HASHENTRIES
, GFP_KERNEL
);
6288 for (i
= 0; i
< NETDEV_HASHENTRIES
; i
++)
6289 INIT_HLIST_HEAD(&hash
[i
]);
6294 /* Initialize per network namespace state */
6295 static int __net_init
netdev_init(struct net
*net
)
6297 INIT_LIST_HEAD(&net
->dev_base_head
);
6299 net
->dev_name_head
= netdev_create_hash();
6300 if (net
->dev_name_head
== NULL
)
6303 net
->dev_index_head
= netdev_create_hash();
6304 if (net
->dev_index_head
== NULL
)
6310 kfree(net
->dev_name_head
);
6316 * netdev_drivername - network driver for the device
6317 * @dev: network device
6319 * Determine network driver for device.
6321 const char *netdev_drivername(const struct net_device
*dev
)
6323 const struct device_driver
*driver
;
6324 const struct device
*parent
;
6325 const char *empty
= "";
6327 parent
= dev
->dev
.parent
;
6331 driver
= parent
->driver
;
6332 if (driver
&& driver
->name
)
6333 return driver
->name
;
6337 static int __netdev_printk(const char *level
, const struct net_device
*dev
,
6338 struct va_format
*vaf
)
6342 if (dev
&& dev
->dev
.parent
)
6343 r
= dev_printk(level
, dev
->dev
.parent
, "%s: %pV",
6344 netdev_name(dev
), vaf
);
6346 r
= printk("%s%s: %pV", level
, netdev_name(dev
), vaf
);
6348 r
= printk("%s(NULL net_device): %pV", level
, vaf
);
6353 int netdev_printk(const char *level
, const struct net_device
*dev
,
6354 const char *format
, ...)
6356 struct va_format vaf
;
6360 va_start(args
, format
);
6365 r
= __netdev_printk(level
, dev
, &vaf
);
6370 EXPORT_SYMBOL(netdev_printk
);
6372 #define define_netdev_printk_level(func, level) \
6373 int func(const struct net_device *dev, const char *fmt, ...) \
6376 struct va_format vaf; \
6379 va_start(args, fmt); \
6384 r = __netdev_printk(level, dev, &vaf); \
6389 EXPORT_SYMBOL(func);
6391 define_netdev_printk_level(netdev_emerg
, KERN_EMERG
);
6392 define_netdev_printk_level(netdev_alert
, KERN_ALERT
);
6393 define_netdev_printk_level(netdev_crit
, KERN_CRIT
);
6394 define_netdev_printk_level(netdev_err
, KERN_ERR
);
6395 define_netdev_printk_level(netdev_warn
, KERN_WARNING
);
6396 define_netdev_printk_level(netdev_notice
, KERN_NOTICE
);
6397 define_netdev_printk_level(netdev_info
, KERN_INFO
);
6399 static void __net_exit
netdev_exit(struct net
*net
)
6401 kfree(net
->dev_name_head
);
6402 kfree(net
->dev_index_head
);
6405 static struct pernet_operations __net_initdata netdev_net_ops
= {
6406 .init
= netdev_init
,
6407 .exit
= netdev_exit
,
6410 static void __net_exit
default_device_exit(struct net
*net
)
6412 struct net_device
*dev
, *aux
;
6414 * Push all migratable network devices back to the
6415 * initial network namespace
6418 for_each_netdev_safe(net
, dev
, aux
) {
6420 char fb_name
[IFNAMSIZ
];
6422 /* Ignore unmoveable devices (i.e. loopback) */
6423 if (dev
->features
& NETIF_F_NETNS_LOCAL
)
6426 /* Leave virtual devices for the generic cleanup */
6427 if (dev
->rtnl_link_ops
)
6430 /* Push remaining network devices to init_net */
6431 snprintf(fb_name
, IFNAMSIZ
, "dev%d", dev
->ifindex
);
6432 err
= dev_change_net_namespace(dev
, &init_net
, fb_name
);
6434 printk(KERN_EMERG
"%s: failed to move %s to init_net: %d\n",
6435 __func__
, dev
->name
, err
);
6442 static void __net_exit
default_device_exit_batch(struct list_head
*net_list
)
6444 /* At exit all network devices most be removed from a network
6445 * namespace. Do this in the reverse order of registration.
6446 * Do this across as many network namespaces as possible to
6447 * improve batching efficiency.
6449 struct net_device
*dev
;
6451 LIST_HEAD(dev_kill_list
);
6454 list_for_each_entry(net
, net_list
, exit_list
) {
6455 for_each_netdev_reverse(net
, dev
) {
6456 if (dev
->rtnl_link_ops
)
6457 dev
->rtnl_link_ops
->dellink(dev
, &dev_kill_list
);
6459 unregister_netdevice_queue(dev
, &dev_kill_list
);
6462 unregister_netdevice_many(&dev_kill_list
);
6463 list_del(&dev_kill_list
);
6467 static struct pernet_operations __net_initdata default_device_ops
= {
6468 .exit
= default_device_exit
,
6469 .exit_batch
= default_device_exit_batch
,
6473 * Initialize the DEV module. At boot time this walks the device list and
6474 * unhooks any devices that fail to initialise (normally hardware not
6475 * present) and leaves us with a valid list of present and active devices.
6480 * This is called single threaded during boot, so no need
6481 * to take the rtnl semaphore.
6483 static int __init
net_dev_init(void)
6485 int i
, rc
= -ENOMEM
;
6487 BUG_ON(!dev_boot_phase
);
6489 if (dev_proc_init())
6492 if (netdev_kobject_init())
6495 INIT_LIST_HEAD(&ptype_all
);
6496 for (i
= 0; i
< PTYPE_HASH_SIZE
; i
++)
6497 INIT_LIST_HEAD(&ptype_base
[i
]);
6499 if (register_pernet_subsys(&netdev_net_ops
))
6503 * Initialise the packet receive queues.
6506 for_each_possible_cpu(i
) {
6507 struct softnet_data
*sd
= &per_cpu(softnet_data
, i
);
6509 memset(sd
, 0, sizeof(*sd
));
6510 skb_queue_head_init(&sd
->input_pkt_queue
);
6511 skb_queue_head_init(&sd
->process_queue
);
6512 sd
->completion_queue
= NULL
;
6513 INIT_LIST_HEAD(&sd
->poll_list
);
6514 sd
->output_queue
= NULL
;
6515 sd
->output_queue_tailp
= &sd
->output_queue
;
6517 sd
->csd
.func
= rps_trigger_softirq
;
6523 sd
->backlog
.poll
= process_backlog
;
6524 sd
->backlog
.weight
= weight_p
;
6525 sd
->backlog
.gro_list
= NULL
;
6526 sd
->backlog
.gro_count
= 0;
6531 /* The loopback device is special if any other network devices
6532 * is present in a network namespace the loopback device must
6533 * be present. Since we now dynamically allocate and free the
6534 * loopback device ensure this invariant is maintained by
6535 * keeping the loopback device as the first device on the
6536 * list of network devices. Ensuring the loopback devices
6537 * is the first device that appears and the last network device
6540 if (register_pernet_device(&loopback_net_ops
))
6543 if (register_pernet_device(&default_device_ops
))
6546 open_softirq(NET_TX_SOFTIRQ
, net_tx_action
);
6547 open_softirq(NET_RX_SOFTIRQ
, net_rx_action
);
6549 hotcpu_notifier(dev_cpu_callback
, 0);
6557 subsys_initcall(net_dev_init
);
6559 static int __init
initialize_hashrnd(void)
6561 get_random_bytes(&hashrnd
, sizeof(hashrnd
));
6565 late_initcall_sync(initialize_hashrnd
);