2 * NET3 Protocol independent device support routines.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
9 * Derived from the non IP parts of dev.c 1.0.19
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
51 * Rudi Cilibrasi : Pass the right thing to
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
75 #include <asm/uaccess.h>
76 #include <linux/bitops.h>
77 #include <linux/capability.h>
78 #include <linux/cpu.h>
79 #include <linux/types.h>
80 #include <linux/kernel.h>
81 #include <linux/hash.h>
82 #include <linux/slab.h>
83 #include <linux/sched.h>
84 #include <linux/mutex.h>
85 #include <linux/string.h>
87 #include <linux/socket.h>
88 #include <linux/sockios.h>
89 #include <linux/errno.h>
90 #include <linux/interrupt.h>
91 #include <linux/if_ether.h>
92 #include <linux/netdevice.h>
93 #include <linux/etherdevice.h>
94 #include <linux/ethtool.h>
95 #include <linux/notifier.h>
96 #include <linux/skbuff.h>
97 #include <net/net_namespace.h>
99 #include <linux/rtnetlink.h>
100 #include <linux/stat.h>
102 #include <net/pkt_sched.h>
103 #include <net/checksum.h>
104 #include <net/xfrm.h>
105 #include <linux/highmem.h>
106 #include <linux/init.h>
107 #include <linux/module.h>
108 #include <linux/netpoll.h>
109 #include <linux/rcupdate.h>
110 #include <linux/delay.h>
111 #include <net/iw_handler.h>
112 #include <asm/current.h>
113 #include <linux/audit.h>
114 #include <linux/dmaengine.h>
115 #include <linux/err.h>
116 #include <linux/ctype.h>
117 #include <linux/if_arp.h>
118 #include <linux/if_vlan.h>
119 #include <linux/ip.h>
121 #include <net/mpls.h>
122 #include <linux/ipv6.h>
123 #include <linux/in.h>
124 #include <linux/jhash.h>
125 #include <linux/random.h>
126 #include <trace/events/napi.h>
127 #include <trace/events/net.h>
128 #include <trace/events/skb.h>
129 #include <linux/pci.h>
130 #include <linux/inetdevice.h>
131 #include <linux/cpu_rmap.h>
132 #include <linux/static_key.h>
133 #include <linux/hashtable.h>
134 #include <linux/vmalloc.h>
135 #include <linux/if_macvlan.h>
136 #include <linux/errqueue.h>
137 #include <linux/hrtimer.h>
138 #include <linux/netfilter_ingress.h>
140 #include "net-sysfs.h"
142 /* Instead of increasing this, you should create a hash table. */
143 #define MAX_GRO_SKBS 8
145 /* This should be increased if a protocol with a bigger head is added. */
146 #define GRO_MAX_HEAD (MAX_HEADER + 128)
148 static DEFINE_SPINLOCK(ptype_lock
);
149 static DEFINE_SPINLOCK(offload_lock
);
150 struct list_head ptype_base
[PTYPE_HASH_SIZE
] __read_mostly
;
151 struct list_head ptype_all __read_mostly
; /* Taps */
152 static struct list_head offload_base __read_mostly
;
154 static int netif_rx_internal(struct sk_buff
*skb
);
155 static int call_netdevice_notifiers_info(unsigned long val
,
156 struct net_device
*dev
,
157 struct netdev_notifier_info
*info
);
160 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
163 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
165 * Writers must hold the rtnl semaphore while they loop through the
166 * dev_base_head list, and hold dev_base_lock for writing when they do the
167 * actual updates. This allows pure readers to access the list even
168 * while a writer is preparing to update it.
170 * To put it another way, dev_base_lock is held for writing only to
171 * protect against pure readers; the rtnl semaphore provides the
172 * protection against other writers.
174 * See, for example usages, register_netdevice() and
175 * unregister_netdevice(), which must be called with the rtnl
178 DEFINE_RWLOCK(dev_base_lock
);
179 EXPORT_SYMBOL(dev_base_lock
);
181 /* protects napi_hash addition/deletion and napi_gen_id */
182 static DEFINE_SPINLOCK(napi_hash_lock
);
184 static unsigned int napi_gen_id
;
185 static DEFINE_HASHTABLE(napi_hash
, 8);
187 static seqcount_t devnet_rename_seq
;
189 static inline void dev_base_seq_inc(struct net
*net
)
191 while (++net
->dev_base_seq
== 0);
194 static inline struct hlist_head
*dev_name_hash(struct net
*net
, const char *name
)
196 unsigned int hash
= full_name_hash(name
, strnlen(name
, IFNAMSIZ
));
198 return &net
->dev_name_head
[hash_32(hash
, NETDEV_HASHBITS
)];
201 static inline struct hlist_head
*dev_index_hash(struct net
*net
, int ifindex
)
203 return &net
->dev_index_head
[ifindex
& (NETDEV_HASHENTRIES
- 1)];
206 static inline void rps_lock(struct softnet_data
*sd
)
209 spin_lock(&sd
->input_pkt_queue
.lock
);
213 static inline void rps_unlock(struct softnet_data
*sd
)
216 spin_unlock(&sd
->input_pkt_queue
.lock
);
220 /* Device list insertion */
221 static void list_netdevice(struct net_device
*dev
)
223 struct net
*net
= dev_net(dev
);
227 write_lock_bh(&dev_base_lock
);
228 list_add_tail_rcu(&dev
->dev_list
, &net
->dev_base_head
);
229 hlist_add_head_rcu(&dev
->name_hlist
, dev_name_hash(net
, dev
->name
));
230 hlist_add_head_rcu(&dev
->index_hlist
,
231 dev_index_hash(net
, dev
->ifindex
));
232 write_unlock_bh(&dev_base_lock
);
234 dev_base_seq_inc(net
);
237 /* Device list removal
238 * caller must respect a RCU grace period before freeing/reusing dev
240 static void unlist_netdevice(struct net_device
*dev
)
244 /* Unlink dev from the device chain */
245 write_lock_bh(&dev_base_lock
);
246 list_del_rcu(&dev
->dev_list
);
247 hlist_del_rcu(&dev
->name_hlist
);
248 hlist_del_rcu(&dev
->index_hlist
);
249 write_unlock_bh(&dev_base_lock
);
251 dev_base_seq_inc(dev_net(dev
));
258 static RAW_NOTIFIER_HEAD(netdev_chain
);
261 * Device drivers call our routines to queue packets here. We empty the
262 * queue in the local softnet handler.
265 DEFINE_PER_CPU_ALIGNED(struct softnet_data
, softnet_data
);
266 EXPORT_PER_CPU_SYMBOL(softnet_data
);
268 #ifdef CONFIG_LOCKDEP
270 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
271 * according to dev->type
273 static const unsigned short netdev_lock_type
[] =
274 {ARPHRD_NETROM
, ARPHRD_ETHER
, ARPHRD_EETHER
, ARPHRD_AX25
,
275 ARPHRD_PRONET
, ARPHRD_CHAOS
, ARPHRD_IEEE802
, ARPHRD_ARCNET
,
276 ARPHRD_APPLETLK
, ARPHRD_DLCI
, ARPHRD_ATM
, ARPHRD_METRICOM
,
277 ARPHRD_IEEE1394
, ARPHRD_EUI64
, ARPHRD_INFINIBAND
, ARPHRD_SLIP
,
278 ARPHRD_CSLIP
, ARPHRD_SLIP6
, ARPHRD_CSLIP6
, ARPHRD_RSRVD
,
279 ARPHRD_ADAPT
, ARPHRD_ROSE
, ARPHRD_X25
, ARPHRD_HWX25
,
280 ARPHRD_PPP
, ARPHRD_CISCO
, ARPHRD_LAPB
, ARPHRD_DDCMP
,
281 ARPHRD_RAWHDLC
, ARPHRD_TUNNEL
, ARPHRD_TUNNEL6
, ARPHRD_FRAD
,
282 ARPHRD_SKIP
, ARPHRD_LOOPBACK
, ARPHRD_LOCALTLK
, ARPHRD_FDDI
,
283 ARPHRD_BIF
, ARPHRD_SIT
, ARPHRD_IPDDP
, ARPHRD_IPGRE
,
284 ARPHRD_PIMREG
, ARPHRD_HIPPI
, ARPHRD_ASH
, ARPHRD_ECONET
,
285 ARPHRD_IRDA
, ARPHRD_FCPP
, ARPHRD_FCAL
, ARPHRD_FCPL
,
286 ARPHRD_FCFABRIC
, ARPHRD_IEEE80211
, ARPHRD_IEEE80211_PRISM
,
287 ARPHRD_IEEE80211_RADIOTAP
, ARPHRD_PHONET
, ARPHRD_PHONET_PIPE
,
288 ARPHRD_IEEE802154
, ARPHRD_VOID
, ARPHRD_NONE
};
290 static const char *const netdev_lock_name
[] =
291 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
292 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
293 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
294 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
295 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
296 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
297 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
298 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
299 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
300 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
301 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
302 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
303 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
304 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
305 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
307 static struct lock_class_key netdev_xmit_lock_key
[ARRAY_SIZE(netdev_lock_type
)];
308 static struct lock_class_key netdev_addr_lock_key
[ARRAY_SIZE(netdev_lock_type
)];
310 static inline unsigned short netdev_lock_pos(unsigned short dev_type
)
314 for (i
= 0; i
< ARRAY_SIZE(netdev_lock_type
); i
++)
315 if (netdev_lock_type
[i
] == dev_type
)
317 /* the last key is used by default */
318 return ARRAY_SIZE(netdev_lock_type
) - 1;
321 static inline void netdev_set_xmit_lockdep_class(spinlock_t
*lock
,
322 unsigned short dev_type
)
326 i
= netdev_lock_pos(dev_type
);
327 lockdep_set_class_and_name(lock
, &netdev_xmit_lock_key
[i
],
328 netdev_lock_name
[i
]);
331 static inline void netdev_set_addr_lockdep_class(struct net_device
*dev
)
335 i
= netdev_lock_pos(dev
->type
);
336 lockdep_set_class_and_name(&dev
->addr_list_lock
,
337 &netdev_addr_lock_key
[i
],
338 netdev_lock_name
[i
]);
341 static inline void netdev_set_xmit_lockdep_class(spinlock_t
*lock
,
342 unsigned short dev_type
)
345 static inline void netdev_set_addr_lockdep_class(struct net_device
*dev
)
350 /*******************************************************************************
352 Protocol management and registration routines
354 *******************************************************************************/
357 * Add a protocol ID to the list. Now that the input handler is
358 * smarter we can dispense with all the messy stuff that used to be
361 * BEWARE!!! Protocol handlers, mangling input packets,
362 * MUST BE last in hash buckets and checking protocol handlers
363 * MUST start from promiscuous ptype_all chain in net_bh.
364 * It is true now, do not change it.
365 * Explanation follows: if protocol handler, mangling packet, will
366 * be the first on list, it is not able to sense, that packet
367 * is cloned and should be copied-on-write, so that it will
368 * change it and subsequent readers will get broken packet.
372 static inline struct list_head
*ptype_head(const struct packet_type
*pt
)
374 if (pt
->type
== htons(ETH_P_ALL
))
375 return pt
->dev
? &pt
->dev
->ptype_all
: &ptype_all
;
377 return pt
->dev
? &pt
->dev
->ptype_specific
:
378 &ptype_base
[ntohs(pt
->type
) & PTYPE_HASH_MASK
];
382 * dev_add_pack - add packet handler
383 * @pt: packet type declaration
385 * Add a protocol handler to the networking stack. The passed &packet_type
386 * is linked into kernel lists and may not be freed until it has been
387 * removed from the kernel lists.
389 * This call does not sleep therefore it can not
390 * guarantee all CPU's that are in middle of receiving packets
391 * will see the new packet type (until the next received packet).
394 void dev_add_pack(struct packet_type
*pt
)
396 struct list_head
*head
= ptype_head(pt
);
398 spin_lock(&ptype_lock
);
399 list_add_rcu(&pt
->list
, head
);
400 spin_unlock(&ptype_lock
);
402 EXPORT_SYMBOL(dev_add_pack
);
405 * __dev_remove_pack - remove packet handler
406 * @pt: packet type declaration
408 * Remove a protocol handler that was previously added to the kernel
409 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
410 * from the kernel lists and can be freed or reused once this function
413 * The packet type might still be in use by receivers
414 * and must not be freed until after all the CPU's have gone
415 * through a quiescent state.
417 void __dev_remove_pack(struct packet_type
*pt
)
419 struct list_head
*head
= ptype_head(pt
);
420 struct packet_type
*pt1
;
422 spin_lock(&ptype_lock
);
424 list_for_each_entry(pt1
, head
, list
) {
426 list_del_rcu(&pt
->list
);
431 pr_warn("dev_remove_pack: %p not found\n", pt
);
433 spin_unlock(&ptype_lock
);
435 EXPORT_SYMBOL(__dev_remove_pack
);
438 * dev_remove_pack - remove packet handler
439 * @pt: packet type declaration
441 * Remove a protocol handler that was previously added to the kernel
442 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
443 * from the kernel lists and can be freed or reused once this function
446 * This call sleeps to guarantee that no CPU is looking at the packet
449 void dev_remove_pack(struct packet_type
*pt
)
451 __dev_remove_pack(pt
);
455 EXPORT_SYMBOL(dev_remove_pack
);
459 * dev_add_offload - register offload handlers
460 * @po: protocol offload declaration
462 * Add protocol offload handlers to the networking stack. The passed
463 * &proto_offload is linked into kernel lists and may not be freed until
464 * it has been removed from the kernel lists.
466 * This call does not sleep therefore it can not
467 * guarantee all CPU's that are in middle of receiving packets
468 * will see the new offload handlers (until the next received packet).
470 void dev_add_offload(struct packet_offload
*po
)
472 struct list_head
*head
= &offload_base
;
474 spin_lock(&offload_lock
);
475 list_add_rcu(&po
->list
, head
);
476 spin_unlock(&offload_lock
);
478 EXPORT_SYMBOL(dev_add_offload
);
481 * __dev_remove_offload - remove offload handler
482 * @po: packet offload declaration
484 * Remove a protocol offload handler that was previously added to the
485 * kernel offload handlers by dev_add_offload(). The passed &offload_type
486 * is removed from the kernel lists and can be freed or reused once this
489 * The packet type might still be in use by receivers
490 * and must not be freed until after all the CPU's have gone
491 * through a quiescent state.
493 static void __dev_remove_offload(struct packet_offload
*po
)
495 struct list_head
*head
= &offload_base
;
496 struct packet_offload
*po1
;
498 spin_lock(&offload_lock
);
500 list_for_each_entry(po1
, head
, list
) {
502 list_del_rcu(&po
->list
);
507 pr_warn("dev_remove_offload: %p not found\n", po
);
509 spin_unlock(&offload_lock
);
513 * dev_remove_offload - remove packet offload handler
514 * @po: packet offload declaration
516 * Remove a packet offload handler that was previously added to the kernel
517 * offload handlers by dev_add_offload(). The passed &offload_type is
518 * removed from the kernel lists and can be freed or reused once this
521 * This call sleeps to guarantee that no CPU is looking at the packet
524 void dev_remove_offload(struct packet_offload
*po
)
526 __dev_remove_offload(po
);
530 EXPORT_SYMBOL(dev_remove_offload
);
532 /******************************************************************************
534 Device Boot-time Settings Routines
536 *******************************************************************************/
538 /* Boot time configuration table */
539 static struct netdev_boot_setup dev_boot_setup
[NETDEV_BOOT_SETUP_MAX
];
542 * netdev_boot_setup_add - add new setup entry
543 * @name: name of the device
544 * @map: configured settings for the device
546 * Adds new setup entry to the dev_boot_setup list. The function
547 * returns 0 on error and 1 on success. This is a generic routine to
550 static int netdev_boot_setup_add(char *name
, struct ifmap
*map
)
552 struct netdev_boot_setup
*s
;
556 for (i
= 0; i
< NETDEV_BOOT_SETUP_MAX
; i
++) {
557 if (s
[i
].name
[0] == '\0' || s
[i
].name
[0] == ' ') {
558 memset(s
[i
].name
, 0, sizeof(s
[i
].name
));
559 strlcpy(s
[i
].name
, name
, IFNAMSIZ
);
560 memcpy(&s
[i
].map
, map
, sizeof(s
[i
].map
));
565 return i
>= NETDEV_BOOT_SETUP_MAX
? 0 : 1;
569 * netdev_boot_setup_check - check boot time settings
570 * @dev: the netdevice
572 * Check boot time settings for the device.
573 * The found settings are set for the device to be used
574 * later in the device probing.
575 * Returns 0 if no settings found, 1 if they are.
577 int netdev_boot_setup_check(struct net_device
*dev
)
579 struct netdev_boot_setup
*s
= dev_boot_setup
;
582 for (i
= 0; i
< NETDEV_BOOT_SETUP_MAX
; i
++) {
583 if (s
[i
].name
[0] != '\0' && s
[i
].name
[0] != ' ' &&
584 !strcmp(dev
->name
, s
[i
].name
)) {
585 dev
->irq
= s
[i
].map
.irq
;
586 dev
->base_addr
= s
[i
].map
.base_addr
;
587 dev
->mem_start
= s
[i
].map
.mem_start
;
588 dev
->mem_end
= s
[i
].map
.mem_end
;
594 EXPORT_SYMBOL(netdev_boot_setup_check
);
598 * netdev_boot_base - get address from boot time settings
599 * @prefix: prefix for network device
600 * @unit: id for network device
602 * Check boot time settings for the base address of device.
603 * The found settings are set for the device to be used
604 * later in the device probing.
605 * Returns 0 if no settings found.
607 unsigned long netdev_boot_base(const char *prefix
, int unit
)
609 const struct netdev_boot_setup
*s
= dev_boot_setup
;
613 sprintf(name
, "%s%d", prefix
, unit
);
616 * If device already registered then return base of 1
617 * to indicate not to probe for this interface
619 if (__dev_get_by_name(&init_net
, name
))
622 for (i
= 0; i
< NETDEV_BOOT_SETUP_MAX
; i
++)
623 if (!strcmp(name
, s
[i
].name
))
624 return s
[i
].map
.base_addr
;
629 * Saves at boot time configured settings for any netdevice.
631 int __init
netdev_boot_setup(char *str
)
636 str
= get_options(str
, ARRAY_SIZE(ints
), ints
);
641 memset(&map
, 0, sizeof(map
));
645 map
.base_addr
= ints
[2];
647 map
.mem_start
= ints
[3];
649 map
.mem_end
= ints
[4];
651 /* Add new entry to the list */
652 return netdev_boot_setup_add(str
, &map
);
655 __setup("netdev=", netdev_boot_setup
);
657 /*******************************************************************************
659 Device Interface Subroutines
661 *******************************************************************************/
664 * dev_get_iflink - get 'iflink' value of a interface
665 * @dev: targeted interface
667 * Indicates the ifindex the interface is linked to.
668 * Physical interfaces have the same 'ifindex' and 'iflink' values.
671 int dev_get_iflink(const struct net_device
*dev
)
673 if (dev
->netdev_ops
&& dev
->netdev_ops
->ndo_get_iflink
)
674 return dev
->netdev_ops
->ndo_get_iflink(dev
);
676 /* If dev->rtnl_link_ops is set, it's a virtual interface. */
677 if (dev
->rtnl_link_ops
)
682 EXPORT_SYMBOL(dev_get_iflink
);
685 * __dev_get_by_name - find a device by its name
686 * @net: the applicable net namespace
687 * @name: name to find
689 * Find an interface by name. Must be called under RTNL semaphore
690 * or @dev_base_lock. If the name is found a pointer to the device
691 * is returned. If the name is not found then %NULL is returned. The
692 * reference counters are not incremented so the caller must be
693 * careful with locks.
696 struct net_device
*__dev_get_by_name(struct net
*net
, const char *name
)
698 struct net_device
*dev
;
699 struct hlist_head
*head
= dev_name_hash(net
, name
);
701 hlist_for_each_entry(dev
, head
, name_hlist
)
702 if (!strncmp(dev
->name
, name
, IFNAMSIZ
))
707 EXPORT_SYMBOL(__dev_get_by_name
);
710 * dev_get_by_name_rcu - find a device by its name
711 * @net: the applicable net namespace
712 * @name: name to find
714 * Find an interface by name.
715 * If the name is found a pointer to the device is returned.
716 * If the name is not found then %NULL is returned.
717 * The reference counters are not incremented so the caller must be
718 * careful with locks. The caller must hold RCU lock.
721 struct net_device
*dev_get_by_name_rcu(struct net
*net
, const char *name
)
723 struct net_device
*dev
;
724 struct hlist_head
*head
= dev_name_hash(net
, name
);
726 hlist_for_each_entry_rcu(dev
, head
, name_hlist
)
727 if (!strncmp(dev
->name
, name
, IFNAMSIZ
))
732 EXPORT_SYMBOL(dev_get_by_name_rcu
);
735 * dev_get_by_name - find a device by its name
736 * @net: the applicable net namespace
737 * @name: name to find
739 * Find an interface by name. This can be called from any
740 * context and does its own locking. The returned handle has
741 * the usage count incremented and the caller must use dev_put() to
742 * release it when it is no longer needed. %NULL is returned if no
743 * matching device is found.
746 struct net_device
*dev_get_by_name(struct net
*net
, const char *name
)
748 struct net_device
*dev
;
751 dev
= dev_get_by_name_rcu(net
, name
);
757 EXPORT_SYMBOL(dev_get_by_name
);
760 * __dev_get_by_index - find a device by its ifindex
761 * @net: the applicable net namespace
762 * @ifindex: index of device
764 * Search for an interface by index. Returns %NULL if the device
765 * is not found or a pointer to the device. The device has not
766 * had its reference counter increased so the caller must be careful
767 * about locking. The caller must hold either the RTNL semaphore
771 struct net_device
*__dev_get_by_index(struct net
*net
, int ifindex
)
773 struct net_device
*dev
;
774 struct hlist_head
*head
= dev_index_hash(net
, ifindex
);
776 hlist_for_each_entry(dev
, head
, index_hlist
)
777 if (dev
->ifindex
== ifindex
)
782 EXPORT_SYMBOL(__dev_get_by_index
);
785 * dev_get_by_index_rcu - find a device by its ifindex
786 * @net: the applicable net namespace
787 * @ifindex: index of device
789 * Search for an interface by index. Returns %NULL if the device
790 * is not found or a pointer to the device. The device has not
791 * had its reference counter increased so the caller must be careful
792 * about locking. The caller must hold RCU lock.
795 struct net_device
*dev_get_by_index_rcu(struct net
*net
, int ifindex
)
797 struct net_device
*dev
;
798 struct hlist_head
*head
= dev_index_hash(net
, ifindex
);
800 hlist_for_each_entry_rcu(dev
, head
, index_hlist
)
801 if (dev
->ifindex
== ifindex
)
806 EXPORT_SYMBOL(dev_get_by_index_rcu
);
810 * dev_get_by_index - find a device by its ifindex
811 * @net: the applicable net namespace
812 * @ifindex: index of device
814 * Search for an interface by index. Returns NULL if the device
815 * is not found or a pointer to the device. The device returned has
816 * had a reference added and the pointer is safe until the user calls
817 * dev_put to indicate they have finished with it.
820 struct net_device
*dev_get_by_index(struct net
*net
, int ifindex
)
822 struct net_device
*dev
;
825 dev
= dev_get_by_index_rcu(net
, ifindex
);
831 EXPORT_SYMBOL(dev_get_by_index
);
834 * netdev_get_name - get a netdevice name, knowing its ifindex.
835 * @net: network namespace
836 * @name: a pointer to the buffer where the name will be stored.
837 * @ifindex: the ifindex of the interface to get the name from.
839 * The use of raw_seqcount_begin() and cond_resched() before
840 * retrying is required as we want to give the writers a chance
841 * to complete when CONFIG_PREEMPT is not set.
843 int netdev_get_name(struct net
*net
, char *name
, int ifindex
)
845 struct net_device
*dev
;
849 seq
= raw_seqcount_begin(&devnet_rename_seq
);
851 dev
= dev_get_by_index_rcu(net
, ifindex
);
857 strcpy(name
, dev
->name
);
859 if (read_seqcount_retry(&devnet_rename_seq
, seq
)) {
868 * dev_getbyhwaddr_rcu - find a device by its hardware address
869 * @net: the applicable net namespace
870 * @type: media type of device
871 * @ha: hardware address
873 * Search for an interface by MAC address. Returns NULL if the device
874 * is not found or a pointer to the device.
875 * The caller must hold RCU or RTNL.
876 * The returned device has not had its ref count increased
877 * and the caller must therefore be careful about locking
881 struct net_device
*dev_getbyhwaddr_rcu(struct net
*net
, unsigned short type
,
884 struct net_device
*dev
;
886 for_each_netdev_rcu(net
, dev
)
887 if (dev
->type
== type
&&
888 !memcmp(dev
->dev_addr
, ha
, dev
->addr_len
))
893 EXPORT_SYMBOL(dev_getbyhwaddr_rcu
);
895 struct net_device
*__dev_getfirstbyhwtype(struct net
*net
, unsigned short type
)
897 struct net_device
*dev
;
900 for_each_netdev(net
, dev
)
901 if (dev
->type
== type
)
906 EXPORT_SYMBOL(__dev_getfirstbyhwtype
);
908 struct net_device
*dev_getfirstbyhwtype(struct net
*net
, unsigned short type
)
910 struct net_device
*dev
, *ret
= NULL
;
913 for_each_netdev_rcu(net
, dev
)
914 if (dev
->type
== type
) {
922 EXPORT_SYMBOL(dev_getfirstbyhwtype
);
925 * __dev_get_by_flags - find any device with given flags
926 * @net: the applicable net namespace
927 * @if_flags: IFF_* values
928 * @mask: bitmask of bits in if_flags to check
930 * Search for any interface with the given flags. Returns NULL if a device
931 * is not found or a pointer to the device. Must be called inside
932 * rtnl_lock(), and result refcount is unchanged.
935 struct net_device
*__dev_get_by_flags(struct net
*net
, unsigned short if_flags
,
938 struct net_device
*dev
, *ret
;
943 for_each_netdev(net
, dev
) {
944 if (((dev
->flags
^ if_flags
) & mask
) == 0) {
951 EXPORT_SYMBOL(__dev_get_by_flags
);
954 * dev_valid_name - check if name is okay for network device
957 * Network device names need to be valid file names to
958 * to allow sysfs to work. We also disallow any kind of
961 bool dev_valid_name(const char *name
)
965 if (strlen(name
) >= IFNAMSIZ
)
967 if (!strcmp(name
, ".") || !strcmp(name
, ".."))
971 if (*name
== '/' || *name
== ':' || isspace(*name
))
977 EXPORT_SYMBOL(dev_valid_name
);
980 * __dev_alloc_name - allocate a name for a device
981 * @net: network namespace to allocate the device name in
982 * @name: name format string
983 * @buf: scratch buffer and result name string
985 * Passed a format string - eg "lt%d" it will try and find a suitable
986 * id. It scans list of devices to build up a free map, then chooses
987 * the first empty slot. The caller must hold the dev_base or rtnl lock
988 * while allocating the name and adding the device in order to avoid
990 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
991 * Returns the number of the unit assigned or a negative errno code.
994 static int __dev_alloc_name(struct net
*net
, const char *name
, char *buf
)
998 const int max_netdevices
= 8*PAGE_SIZE
;
999 unsigned long *inuse
;
1000 struct net_device
*d
;
1002 p
= strnchr(name
, IFNAMSIZ
-1, '%');
1005 * Verify the string as this thing may have come from
1006 * the user. There must be either one "%d" and no other "%"
1009 if (p
[1] != 'd' || strchr(p
+ 2, '%'))
1012 /* Use one page as a bit array of possible slots */
1013 inuse
= (unsigned long *) get_zeroed_page(GFP_ATOMIC
);
1017 for_each_netdev(net
, d
) {
1018 if (!sscanf(d
->name
, name
, &i
))
1020 if (i
< 0 || i
>= max_netdevices
)
1023 /* avoid cases where sscanf is not exact inverse of printf */
1024 snprintf(buf
, IFNAMSIZ
, name
, i
);
1025 if (!strncmp(buf
, d
->name
, IFNAMSIZ
))
1029 i
= find_first_zero_bit(inuse
, max_netdevices
);
1030 free_page((unsigned long) inuse
);
1034 snprintf(buf
, IFNAMSIZ
, name
, i
);
1035 if (!__dev_get_by_name(net
, buf
))
1038 /* It is possible to run out of possible slots
1039 * when the name is long and there isn't enough space left
1040 * for the digits, or if all bits are used.
1046 * dev_alloc_name - allocate a name for a device
1048 * @name: name format string
1050 * Passed a format string - eg "lt%d" it will try and find a suitable
1051 * id. It scans list of devices to build up a free map, then chooses
1052 * the first empty slot. The caller must hold the dev_base or rtnl lock
1053 * while allocating the name and adding the device in order to avoid
1055 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1056 * Returns the number of the unit assigned or a negative errno code.
1059 int dev_alloc_name(struct net_device
*dev
, const char *name
)
1065 BUG_ON(!dev_net(dev
));
1067 ret
= __dev_alloc_name(net
, name
, buf
);
1069 strlcpy(dev
->name
, buf
, IFNAMSIZ
);
1072 EXPORT_SYMBOL(dev_alloc_name
);
1074 static int dev_alloc_name_ns(struct net
*net
,
1075 struct net_device
*dev
,
1081 ret
= __dev_alloc_name(net
, name
, buf
);
1083 strlcpy(dev
->name
, buf
, IFNAMSIZ
);
1087 static int dev_get_valid_name(struct net
*net
,
1088 struct net_device
*dev
,
1093 if (!dev_valid_name(name
))
1096 if (strchr(name
, '%'))
1097 return dev_alloc_name_ns(net
, dev
, name
);
1098 else if (__dev_get_by_name(net
, name
))
1100 else if (dev
->name
!= name
)
1101 strlcpy(dev
->name
, name
, IFNAMSIZ
);
1107 * dev_change_name - change name of a device
1109 * @newname: name (or format string) must be at least IFNAMSIZ
1111 * Change name of a device, can pass format strings "eth%d".
1114 int dev_change_name(struct net_device
*dev
, const char *newname
)
1116 unsigned char old_assign_type
;
1117 char oldname
[IFNAMSIZ
];
1123 BUG_ON(!dev_net(dev
));
1126 if (dev
->flags
& IFF_UP
)
1129 write_seqcount_begin(&devnet_rename_seq
);
1131 if (strncmp(newname
, dev
->name
, IFNAMSIZ
) == 0) {
1132 write_seqcount_end(&devnet_rename_seq
);
1136 memcpy(oldname
, dev
->name
, IFNAMSIZ
);
1138 err
= dev_get_valid_name(net
, dev
, newname
);
1140 write_seqcount_end(&devnet_rename_seq
);
1144 if (oldname
[0] && !strchr(oldname
, '%'))
1145 netdev_info(dev
, "renamed from %s\n", oldname
);
1147 old_assign_type
= dev
->name_assign_type
;
1148 dev
->name_assign_type
= NET_NAME_RENAMED
;
1151 ret
= device_rename(&dev
->dev
, dev
->name
);
1153 memcpy(dev
->name
, oldname
, IFNAMSIZ
);
1154 dev
->name_assign_type
= old_assign_type
;
1155 write_seqcount_end(&devnet_rename_seq
);
1159 write_seqcount_end(&devnet_rename_seq
);
1161 netdev_adjacent_rename_links(dev
, oldname
);
1163 write_lock_bh(&dev_base_lock
);
1164 hlist_del_rcu(&dev
->name_hlist
);
1165 write_unlock_bh(&dev_base_lock
);
1169 write_lock_bh(&dev_base_lock
);
1170 hlist_add_head_rcu(&dev
->name_hlist
, dev_name_hash(net
, dev
->name
));
1171 write_unlock_bh(&dev_base_lock
);
1173 ret
= call_netdevice_notifiers(NETDEV_CHANGENAME
, dev
);
1174 ret
= notifier_to_errno(ret
);
1177 /* err >= 0 after dev_alloc_name() or stores the first errno */
1180 write_seqcount_begin(&devnet_rename_seq
);
1181 memcpy(dev
->name
, oldname
, IFNAMSIZ
);
1182 memcpy(oldname
, newname
, IFNAMSIZ
);
1183 dev
->name_assign_type
= old_assign_type
;
1184 old_assign_type
= NET_NAME_RENAMED
;
1187 pr_err("%s: name change rollback failed: %d\n",
1196 * dev_set_alias - change ifalias of a device
1198 * @alias: name up to IFALIASZ
1199 * @len: limit of bytes to copy from info
1201 * Set ifalias for a device,
1203 int dev_set_alias(struct net_device
*dev
, const char *alias
, size_t len
)
1209 if (len
>= IFALIASZ
)
1213 kfree(dev
->ifalias
);
1214 dev
->ifalias
= NULL
;
1218 new_ifalias
= krealloc(dev
->ifalias
, len
+ 1, GFP_KERNEL
);
1221 dev
->ifalias
= new_ifalias
;
1223 strlcpy(dev
->ifalias
, alias
, len
+1);
1229 * netdev_features_change - device changes features
1230 * @dev: device to cause notification
1232 * Called to indicate a device has changed features.
1234 void netdev_features_change(struct net_device
*dev
)
1236 call_netdevice_notifiers(NETDEV_FEAT_CHANGE
, dev
);
1238 EXPORT_SYMBOL(netdev_features_change
);
1241 * netdev_state_change - device changes state
1242 * @dev: device to cause notification
1244 * Called to indicate a device has changed state. This function calls
1245 * the notifier chains for netdev_chain and sends a NEWLINK message
1246 * to the routing socket.
1248 void netdev_state_change(struct net_device
*dev
)
1250 if (dev
->flags
& IFF_UP
) {
1251 struct netdev_notifier_change_info change_info
;
1253 change_info
.flags_changed
= 0;
1254 call_netdevice_notifiers_info(NETDEV_CHANGE
, dev
,
1256 rtmsg_ifinfo(RTM_NEWLINK
, dev
, 0, GFP_KERNEL
);
1259 EXPORT_SYMBOL(netdev_state_change
);
1262 * netdev_notify_peers - notify network peers about existence of @dev
1263 * @dev: network device
1265 * Generate traffic such that interested network peers are aware of
1266 * @dev, such as by generating a gratuitous ARP. This may be used when
1267 * a device wants to inform the rest of the network about some sort of
1268 * reconfiguration such as a failover event or virtual machine
1271 void netdev_notify_peers(struct net_device
*dev
)
1274 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS
, dev
);
1277 EXPORT_SYMBOL(netdev_notify_peers
);
1279 static int __dev_open(struct net_device
*dev
)
1281 const struct net_device_ops
*ops
= dev
->netdev_ops
;
1286 if (!netif_device_present(dev
))
1289 /* Block netpoll from trying to do any rx path servicing.
1290 * If we don't do this there is a chance ndo_poll_controller
1291 * or ndo_poll may be running while we open the device
1293 netpoll_poll_disable(dev
);
1295 ret
= call_netdevice_notifiers(NETDEV_PRE_UP
, dev
);
1296 ret
= notifier_to_errno(ret
);
1300 set_bit(__LINK_STATE_START
, &dev
->state
);
1302 if (ops
->ndo_validate_addr
)
1303 ret
= ops
->ndo_validate_addr(dev
);
1305 if (!ret
&& ops
->ndo_open
)
1306 ret
= ops
->ndo_open(dev
);
1308 netpoll_poll_enable(dev
);
1311 clear_bit(__LINK_STATE_START
, &dev
->state
);
1313 dev
->flags
|= IFF_UP
;
1314 dev_set_rx_mode(dev
);
1316 add_device_randomness(dev
->dev_addr
, dev
->addr_len
);
1323 * dev_open - prepare an interface for use.
1324 * @dev: device to open
1326 * Takes a device from down to up state. The device's private open
1327 * function is invoked and then the multicast lists are loaded. Finally
1328 * the device is moved into the up state and a %NETDEV_UP message is
1329 * sent to the netdev notifier chain.
1331 * Calling this function on an active interface is a nop. On a failure
1332 * a negative errno code is returned.
1334 int dev_open(struct net_device
*dev
)
1338 if (dev
->flags
& IFF_UP
)
1341 ret
= __dev_open(dev
);
1345 rtmsg_ifinfo(RTM_NEWLINK
, dev
, IFF_UP
|IFF_RUNNING
, GFP_KERNEL
);
1346 call_netdevice_notifiers(NETDEV_UP
, dev
);
1350 EXPORT_SYMBOL(dev_open
);
1352 static int __dev_close_many(struct list_head
*head
)
1354 struct net_device
*dev
;
1359 list_for_each_entry(dev
, head
, close_list
) {
1360 /* Temporarily disable netpoll until the interface is down */
1361 netpoll_poll_disable(dev
);
1363 call_netdevice_notifiers(NETDEV_GOING_DOWN
, dev
);
1365 clear_bit(__LINK_STATE_START
, &dev
->state
);
1367 /* Synchronize to scheduled poll. We cannot touch poll list, it
1368 * can be even on different cpu. So just clear netif_running().
1370 * dev->stop() will invoke napi_disable() on all of it's
1371 * napi_struct instances on this device.
1373 smp_mb__after_atomic(); /* Commit netif_running(). */
1376 dev_deactivate_many(head
);
1378 list_for_each_entry(dev
, head
, close_list
) {
1379 const struct net_device_ops
*ops
= dev
->netdev_ops
;
1382 * Call the device specific close. This cannot fail.
1383 * Only if device is UP
1385 * We allow it to be called even after a DETACH hot-plug
1391 dev
->flags
&= ~IFF_UP
;
1392 netpoll_poll_enable(dev
);
1398 static int __dev_close(struct net_device
*dev
)
1403 list_add(&dev
->close_list
, &single
);
1404 retval
= __dev_close_many(&single
);
1410 int dev_close_many(struct list_head
*head
, bool unlink
)
1412 struct net_device
*dev
, *tmp
;
1414 /* Remove the devices that don't need to be closed */
1415 list_for_each_entry_safe(dev
, tmp
, head
, close_list
)
1416 if (!(dev
->flags
& IFF_UP
))
1417 list_del_init(&dev
->close_list
);
1419 __dev_close_many(head
);
1421 list_for_each_entry_safe(dev
, tmp
, head
, close_list
) {
1422 rtmsg_ifinfo(RTM_NEWLINK
, dev
, IFF_UP
|IFF_RUNNING
, GFP_KERNEL
);
1423 call_netdevice_notifiers(NETDEV_DOWN
, dev
);
1425 list_del_init(&dev
->close_list
);
1430 EXPORT_SYMBOL(dev_close_many
);
1433 * dev_close - shutdown an interface.
1434 * @dev: device to shutdown
1436 * This function moves an active device into down state. A
1437 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1438 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1441 int dev_close(struct net_device
*dev
)
1443 if (dev
->flags
& IFF_UP
) {
1446 list_add(&dev
->close_list
, &single
);
1447 dev_close_many(&single
, true);
1452 EXPORT_SYMBOL(dev_close
);
1456 * dev_disable_lro - disable Large Receive Offload on a device
1459 * Disable Large Receive Offload (LRO) on a net device. Must be
1460 * called under RTNL. This is needed if received packets may be
1461 * forwarded to another interface.
1463 void dev_disable_lro(struct net_device
*dev
)
1465 struct net_device
*lower_dev
;
1466 struct list_head
*iter
;
1468 dev
->wanted_features
&= ~NETIF_F_LRO
;
1469 netdev_update_features(dev
);
1471 if (unlikely(dev
->features
& NETIF_F_LRO
))
1472 netdev_WARN(dev
, "failed to disable LRO!\n");
1474 netdev_for_each_lower_dev(dev
, lower_dev
, iter
)
1475 dev_disable_lro(lower_dev
);
1477 EXPORT_SYMBOL(dev_disable_lro
);
1479 static int call_netdevice_notifier(struct notifier_block
*nb
, unsigned long val
,
1480 struct net_device
*dev
)
1482 struct netdev_notifier_info info
;
1484 netdev_notifier_info_init(&info
, dev
);
1485 return nb
->notifier_call(nb
, val
, &info
);
1488 static int dev_boot_phase
= 1;
1491 * register_netdevice_notifier - register a network notifier block
1494 * Register a notifier to be called when network device events occur.
1495 * The notifier passed is linked into the kernel structures and must
1496 * not be reused until it has been unregistered. A negative errno code
1497 * is returned on a failure.
1499 * When registered all registration and up events are replayed
1500 * to the new notifier to allow device to have a race free
1501 * view of the network device list.
1504 int register_netdevice_notifier(struct notifier_block
*nb
)
1506 struct net_device
*dev
;
1507 struct net_device
*last
;
1512 err
= raw_notifier_chain_register(&netdev_chain
, nb
);
1518 for_each_netdev(net
, dev
) {
1519 err
= call_netdevice_notifier(nb
, NETDEV_REGISTER
, dev
);
1520 err
= notifier_to_errno(err
);
1524 if (!(dev
->flags
& IFF_UP
))
1527 call_netdevice_notifier(nb
, NETDEV_UP
, dev
);
1538 for_each_netdev(net
, dev
) {
1542 if (dev
->flags
& IFF_UP
) {
1543 call_netdevice_notifier(nb
, NETDEV_GOING_DOWN
,
1545 call_netdevice_notifier(nb
, NETDEV_DOWN
, dev
);
1547 call_netdevice_notifier(nb
, NETDEV_UNREGISTER
, dev
);
1552 raw_notifier_chain_unregister(&netdev_chain
, nb
);
1555 EXPORT_SYMBOL(register_netdevice_notifier
);
1558 * unregister_netdevice_notifier - unregister a network notifier block
1561 * Unregister a notifier previously registered by
1562 * register_netdevice_notifier(). The notifier is unlinked into the
1563 * kernel structures and may then be reused. A negative errno code
1564 * is returned on a failure.
1566 * After unregistering unregister and down device events are synthesized
1567 * for all devices on the device list to the removed notifier to remove
1568 * the need for special case cleanup code.
1571 int unregister_netdevice_notifier(struct notifier_block
*nb
)
1573 struct net_device
*dev
;
1578 err
= raw_notifier_chain_unregister(&netdev_chain
, nb
);
1583 for_each_netdev(net
, dev
) {
1584 if (dev
->flags
& IFF_UP
) {
1585 call_netdevice_notifier(nb
, NETDEV_GOING_DOWN
,
1587 call_netdevice_notifier(nb
, NETDEV_DOWN
, dev
);
1589 call_netdevice_notifier(nb
, NETDEV_UNREGISTER
, dev
);
1596 EXPORT_SYMBOL(unregister_netdevice_notifier
);
1599 * call_netdevice_notifiers_info - call all network notifier blocks
1600 * @val: value passed unmodified to notifier function
1601 * @dev: net_device pointer passed unmodified to notifier function
1602 * @info: notifier information data
1604 * Call all network notifier blocks. Parameters and return value
1605 * are as for raw_notifier_call_chain().
1608 static int call_netdevice_notifiers_info(unsigned long val
,
1609 struct net_device
*dev
,
1610 struct netdev_notifier_info
*info
)
1613 netdev_notifier_info_init(info
, dev
);
1614 return raw_notifier_call_chain(&netdev_chain
, val
, info
);
1618 * call_netdevice_notifiers - call all network notifier blocks
1619 * @val: value passed unmodified to notifier function
1620 * @dev: net_device pointer passed unmodified to notifier function
1622 * Call all network notifier blocks. Parameters and return value
1623 * are as for raw_notifier_call_chain().
1626 int call_netdevice_notifiers(unsigned long val
, struct net_device
*dev
)
1628 struct netdev_notifier_info info
;
1630 return call_netdevice_notifiers_info(val
, dev
, &info
);
1632 EXPORT_SYMBOL(call_netdevice_notifiers
);
1634 #ifdef CONFIG_NET_INGRESS
1635 static struct static_key ingress_needed __read_mostly
;
1637 void net_inc_ingress_queue(void)
1639 static_key_slow_inc(&ingress_needed
);
1641 EXPORT_SYMBOL_GPL(net_inc_ingress_queue
);
1643 void net_dec_ingress_queue(void)
1645 static_key_slow_dec(&ingress_needed
);
1647 EXPORT_SYMBOL_GPL(net_dec_ingress_queue
);
1650 static struct static_key netstamp_needed __read_mostly
;
1651 #ifdef HAVE_JUMP_LABEL
1652 /* We are not allowed to call static_key_slow_dec() from irq context
1653 * If net_disable_timestamp() is called from irq context, defer the
1654 * static_key_slow_dec() calls.
1656 static atomic_t netstamp_needed_deferred
;
1659 void net_enable_timestamp(void)
1661 #ifdef HAVE_JUMP_LABEL
1662 int deferred
= atomic_xchg(&netstamp_needed_deferred
, 0);
1666 static_key_slow_dec(&netstamp_needed
);
1670 static_key_slow_inc(&netstamp_needed
);
1672 EXPORT_SYMBOL(net_enable_timestamp
);
1674 void net_disable_timestamp(void)
1676 #ifdef HAVE_JUMP_LABEL
1677 if (in_interrupt()) {
1678 atomic_inc(&netstamp_needed_deferred
);
1682 static_key_slow_dec(&netstamp_needed
);
1684 EXPORT_SYMBOL(net_disable_timestamp
);
1686 static inline void net_timestamp_set(struct sk_buff
*skb
)
1688 skb
->tstamp
.tv64
= 0;
1689 if (static_key_false(&netstamp_needed
))
1690 __net_timestamp(skb
);
1693 #define net_timestamp_check(COND, SKB) \
1694 if (static_key_false(&netstamp_needed)) { \
1695 if ((COND) && !(SKB)->tstamp.tv64) \
1696 __net_timestamp(SKB); \
1699 bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb)
1703 if (!(dev
->flags
& IFF_UP
))
1706 len
= dev
->mtu
+ dev
->hard_header_len
+ VLAN_HLEN
;
1707 if (skb
->len
<= len
)
1710 /* if TSO is enabled, we don't care about the length as the packet
1711 * could be forwarded without being segmented before
1713 if (skb_is_gso(skb
))
1718 EXPORT_SYMBOL_GPL(is_skb_forwardable
);
1720 int __dev_forward_skb(struct net_device
*dev
, struct sk_buff
*skb
)
1722 if (skb_shinfo(skb
)->tx_flags
& SKBTX_DEV_ZEROCOPY
) {
1723 if (skb_copy_ubufs(skb
, GFP_ATOMIC
)) {
1724 atomic_long_inc(&dev
->rx_dropped
);
1730 if (unlikely(!is_skb_forwardable(dev
, skb
))) {
1731 atomic_long_inc(&dev
->rx_dropped
);
1736 skb_scrub_packet(skb
, true);
1738 skb
->protocol
= eth_type_trans(skb
, dev
);
1739 skb_postpull_rcsum(skb
, eth_hdr(skb
), ETH_HLEN
);
1743 EXPORT_SYMBOL_GPL(__dev_forward_skb
);
1746 * dev_forward_skb - loopback an skb to another netif
1748 * @dev: destination network device
1749 * @skb: buffer to forward
1752 * NET_RX_SUCCESS (no congestion)
1753 * NET_RX_DROP (packet was dropped, but freed)
1755 * dev_forward_skb can be used for injecting an skb from the
1756 * start_xmit function of one device into the receive queue
1757 * of another device.
1759 * The receiving device may be in another namespace, so
1760 * we have to clear all information in the skb that could
1761 * impact namespace isolation.
1763 int dev_forward_skb(struct net_device
*dev
, struct sk_buff
*skb
)
1765 return __dev_forward_skb(dev
, skb
) ?: netif_rx_internal(skb
);
1767 EXPORT_SYMBOL_GPL(dev_forward_skb
);
1769 static inline int deliver_skb(struct sk_buff
*skb
,
1770 struct packet_type
*pt_prev
,
1771 struct net_device
*orig_dev
)
1773 if (unlikely(skb_orphan_frags(skb
, GFP_ATOMIC
)))
1775 atomic_inc(&skb
->users
);
1776 return pt_prev
->func(skb
, skb
->dev
, pt_prev
, orig_dev
);
1779 static inline void deliver_ptype_list_skb(struct sk_buff
*skb
,
1780 struct packet_type
**pt
,
1781 struct net_device
*orig_dev
,
1783 struct list_head
*ptype_list
)
1785 struct packet_type
*ptype
, *pt_prev
= *pt
;
1787 list_for_each_entry_rcu(ptype
, ptype_list
, list
) {
1788 if (ptype
->type
!= type
)
1791 deliver_skb(skb
, pt_prev
, orig_dev
);
1797 static inline bool skb_loop_sk(struct packet_type
*ptype
, struct sk_buff
*skb
)
1799 if (!ptype
->af_packet_priv
|| !skb
->sk
)
1802 if (ptype
->id_match
)
1803 return ptype
->id_match(ptype
, skb
->sk
);
1804 else if ((struct sock
*)ptype
->af_packet_priv
== skb
->sk
)
1811 * Support routine. Sends outgoing frames to any network
1812 * taps currently in use.
1815 static void dev_queue_xmit_nit(struct sk_buff
*skb
, struct net_device
*dev
)
1817 struct packet_type
*ptype
;
1818 struct sk_buff
*skb2
= NULL
;
1819 struct packet_type
*pt_prev
= NULL
;
1820 struct list_head
*ptype_list
= &ptype_all
;
1824 list_for_each_entry_rcu(ptype
, ptype_list
, list
) {
1825 /* Never send packets back to the socket
1826 * they originated from - MvS (miquels@drinkel.ow.org)
1828 if (skb_loop_sk(ptype
, skb
))
1832 deliver_skb(skb2
, pt_prev
, skb
->dev
);
1837 /* need to clone skb, done only once */
1838 skb2
= skb_clone(skb
, GFP_ATOMIC
);
1842 net_timestamp_set(skb2
);
1844 /* skb->nh should be correctly
1845 * set by sender, so that the second statement is
1846 * just protection against buggy protocols.
1848 skb_reset_mac_header(skb2
);
1850 if (skb_network_header(skb2
) < skb2
->data
||
1851 skb_network_header(skb2
) > skb_tail_pointer(skb2
)) {
1852 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1853 ntohs(skb2
->protocol
),
1855 skb_reset_network_header(skb2
);
1858 skb2
->transport_header
= skb2
->network_header
;
1859 skb2
->pkt_type
= PACKET_OUTGOING
;
1863 if (ptype_list
== &ptype_all
) {
1864 ptype_list
= &dev
->ptype_all
;
1869 pt_prev
->func(skb2
, skb
->dev
, pt_prev
, skb
->dev
);
1874 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1875 * @dev: Network device
1876 * @txq: number of queues available
1878 * If real_num_tx_queues is changed the tc mappings may no longer be
1879 * valid. To resolve this verify the tc mapping remains valid and if
1880 * not NULL the mapping. With no priorities mapping to this
1881 * offset/count pair it will no longer be used. In the worst case TC0
1882 * is invalid nothing can be done so disable priority mappings. If is
1883 * expected that drivers will fix this mapping if they can before
1884 * calling netif_set_real_num_tx_queues.
1886 static void netif_setup_tc(struct net_device
*dev
, unsigned int txq
)
1889 struct netdev_tc_txq
*tc
= &dev
->tc_to_txq
[0];
1891 /* If TC0 is invalidated disable TC mapping */
1892 if (tc
->offset
+ tc
->count
> txq
) {
1893 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1898 /* Invalidated prio to tc mappings set to TC0 */
1899 for (i
= 1; i
< TC_BITMASK
+ 1; i
++) {
1900 int q
= netdev_get_prio_tc_map(dev
, i
);
1902 tc
= &dev
->tc_to_txq
[q
];
1903 if (tc
->offset
+ tc
->count
> txq
) {
1904 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1906 netdev_set_prio_tc_map(dev
, i
, 0);
1912 static DEFINE_MUTEX(xps_map_mutex
);
1913 #define xmap_dereference(P) \
1914 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1916 static struct xps_map
*remove_xps_queue(struct xps_dev_maps
*dev_maps
,
1919 struct xps_map
*map
= NULL
;
1923 map
= xmap_dereference(dev_maps
->cpu_map
[cpu
]);
1925 for (pos
= 0; map
&& pos
< map
->len
; pos
++) {
1926 if (map
->queues
[pos
] == index
) {
1928 map
->queues
[pos
] = map
->queues
[--map
->len
];
1930 RCU_INIT_POINTER(dev_maps
->cpu_map
[cpu
], NULL
);
1931 kfree_rcu(map
, rcu
);
1941 static void netif_reset_xps_queues_gt(struct net_device
*dev
, u16 index
)
1943 struct xps_dev_maps
*dev_maps
;
1945 bool active
= false;
1947 mutex_lock(&xps_map_mutex
);
1948 dev_maps
= xmap_dereference(dev
->xps_maps
);
1953 for_each_possible_cpu(cpu
) {
1954 for (i
= index
; i
< dev
->num_tx_queues
; i
++) {
1955 if (!remove_xps_queue(dev_maps
, cpu
, i
))
1958 if (i
== dev
->num_tx_queues
)
1963 RCU_INIT_POINTER(dev
->xps_maps
, NULL
);
1964 kfree_rcu(dev_maps
, rcu
);
1967 for (i
= index
; i
< dev
->num_tx_queues
; i
++)
1968 netdev_queue_numa_node_write(netdev_get_tx_queue(dev
, i
),
1972 mutex_unlock(&xps_map_mutex
);
1975 static struct xps_map
*expand_xps_map(struct xps_map
*map
,
1978 struct xps_map
*new_map
;
1979 int alloc_len
= XPS_MIN_MAP_ALLOC
;
1982 for (pos
= 0; map
&& pos
< map
->len
; pos
++) {
1983 if (map
->queues
[pos
] != index
)
1988 /* Need to add queue to this CPU's existing map */
1990 if (pos
< map
->alloc_len
)
1993 alloc_len
= map
->alloc_len
* 2;
1996 /* Need to allocate new map to store queue on this CPU's map */
1997 new_map
= kzalloc_node(XPS_MAP_SIZE(alloc_len
), GFP_KERNEL
,
2002 for (i
= 0; i
< pos
; i
++)
2003 new_map
->queues
[i
] = map
->queues
[i
];
2004 new_map
->alloc_len
= alloc_len
;
2010 int netif_set_xps_queue(struct net_device
*dev
, const struct cpumask
*mask
,
2013 struct xps_dev_maps
*dev_maps
, *new_dev_maps
= NULL
;
2014 struct xps_map
*map
, *new_map
;
2015 int maps_sz
= max_t(unsigned int, XPS_DEV_MAPS_SIZE
, L1_CACHE_BYTES
);
2016 int cpu
, numa_node_id
= -2;
2017 bool active
= false;
2019 mutex_lock(&xps_map_mutex
);
2021 dev_maps
= xmap_dereference(dev
->xps_maps
);
2023 /* allocate memory for queue storage */
2024 for_each_online_cpu(cpu
) {
2025 if (!cpumask_test_cpu(cpu
, mask
))
2029 new_dev_maps
= kzalloc(maps_sz
, GFP_KERNEL
);
2030 if (!new_dev_maps
) {
2031 mutex_unlock(&xps_map_mutex
);
2035 map
= dev_maps
? xmap_dereference(dev_maps
->cpu_map
[cpu
]) :
2038 map
= expand_xps_map(map
, cpu
, index
);
2042 RCU_INIT_POINTER(new_dev_maps
->cpu_map
[cpu
], map
);
2046 goto out_no_new_maps
;
2048 for_each_possible_cpu(cpu
) {
2049 if (cpumask_test_cpu(cpu
, mask
) && cpu_online(cpu
)) {
2050 /* add queue to CPU maps */
2053 map
= xmap_dereference(new_dev_maps
->cpu_map
[cpu
]);
2054 while ((pos
< map
->len
) && (map
->queues
[pos
] != index
))
2057 if (pos
== map
->len
)
2058 map
->queues
[map
->len
++] = index
;
2060 if (numa_node_id
== -2)
2061 numa_node_id
= cpu_to_node(cpu
);
2062 else if (numa_node_id
!= cpu_to_node(cpu
))
2065 } else if (dev_maps
) {
2066 /* fill in the new device map from the old device map */
2067 map
= xmap_dereference(dev_maps
->cpu_map
[cpu
]);
2068 RCU_INIT_POINTER(new_dev_maps
->cpu_map
[cpu
], map
);
2073 rcu_assign_pointer(dev
->xps_maps
, new_dev_maps
);
2075 /* Cleanup old maps */
2077 for_each_possible_cpu(cpu
) {
2078 new_map
= xmap_dereference(new_dev_maps
->cpu_map
[cpu
]);
2079 map
= xmap_dereference(dev_maps
->cpu_map
[cpu
]);
2080 if (map
&& map
!= new_map
)
2081 kfree_rcu(map
, rcu
);
2084 kfree_rcu(dev_maps
, rcu
);
2087 dev_maps
= new_dev_maps
;
2091 /* update Tx queue numa node */
2092 netdev_queue_numa_node_write(netdev_get_tx_queue(dev
, index
),
2093 (numa_node_id
>= 0) ? numa_node_id
:
2099 /* removes queue from unused CPUs */
2100 for_each_possible_cpu(cpu
) {
2101 if (cpumask_test_cpu(cpu
, mask
) && cpu_online(cpu
))
2104 if (remove_xps_queue(dev_maps
, cpu
, index
))
2108 /* free map if not active */
2110 RCU_INIT_POINTER(dev
->xps_maps
, NULL
);
2111 kfree_rcu(dev_maps
, rcu
);
2115 mutex_unlock(&xps_map_mutex
);
2119 /* remove any maps that we added */
2120 for_each_possible_cpu(cpu
) {
2121 new_map
= xmap_dereference(new_dev_maps
->cpu_map
[cpu
]);
2122 map
= dev_maps
? xmap_dereference(dev_maps
->cpu_map
[cpu
]) :
2124 if (new_map
&& new_map
!= map
)
2128 mutex_unlock(&xps_map_mutex
);
2130 kfree(new_dev_maps
);
2133 EXPORT_SYMBOL(netif_set_xps_queue
);
2137 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2138 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2140 int netif_set_real_num_tx_queues(struct net_device
*dev
, unsigned int txq
)
2144 if (txq
< 1 || txq
> dev
->num_tx_queues
)
2147 if (dev
->reg_state
== NETREG_REGISTERED
||
2148 dev
->reg_state
== NETREG_UNREGISTERING
) {
2151 rc
= netdev_queue_update_kobjects(dev
, dev
->real_num_tx_queues
,
2157 netif_setup_tc(dev
, txq
);
2159 if (txq
< dev
->real_num_tx_queues
) {
2160 qdisc_reset_all_tx_gt(dev
, txq
);
2162 netif_reset_xps_queues_gt(dev
, txq
);
2167 dev
->real_num_tx_queues
= txq
;
2170 EXPORT_SYMBOL(netif_set_real_num_tx_queues
);
2174 * netif_set_real_num_rx_queues - set actual number of RX queues used
2175 * @dev: Network device
2176 * @rxq: Actual number of RX queues
2178 * This must be called either with the rtnl_lock held or before
2179 * registration of the net device. Returns 0 on success, or a
2180 * negative error code. If called before registration, it always
2183 int netif_set_real_num_rx_queues(struct net_device
*dev
, unsigned int rxq
)
2187 if (rxq
< 1 || rxq
> dev
->num_rx_queues
)
2190 if (dev
->reg_state
== NETREG_REGISTERED
) {
2193 rc
= net_rx_queue_update_kobjects(dev
, dev
->real_num_rx_queues
,
2199 dev
->real_num_rx_queues
= rxq
;
2202 EXPORT_SYMBOL(netif_set_real_num_rx_queues
);
2206 * netif_get_num_default_rss_queues - default number of RSS queues
2208 * This routine should set an upper limit on the number of RSS queues
2209 * used by default by multiqueue devices.
2211 int netif_get_num_default_rss_queues(void)
2213 return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES
, num_online_cpus());
2215 EXPORT_SYMBOL(netif_get_num_default_rss_queues
);
2217 static inline void __netif_reschedule(struct Qdisc
*q
)
2219 struct softnet_data
*sd
;
2220 unsigned long flags
;
2222 local_irq_save(flags
);
2223 sd
= this_cpu_ptr(&softnet_data
);
2224 q
->next_sched
= NULL
;
2225 *sd
->output_queue_tailp
= q
;
2226 sd
->output_queue_tailp
= &q
->next_sched
;
2227 raise_softirq_irqoff(NET_TX_SOFTIRQ
);
2228 local_irq_restore(flags
);
2231 void __netif_schedule(struct Qdisc
*q
)
2233 if (!test_and_set_bit(__QDISC_STATE_SCHED
, &q
->state
))
2234 __netif_reschedule(q
);
2236 EXPORT_SYMBOL(__netif_schedule
);
2238 struct dev_kfree_skb_cb
{
2239 enum skb_free_reason reason
;
2242 static struct dev_kfree_skb_cb
*get_kfree_skb_cb(const struct sk_buff
*skb
)
2244 return (struct dev_kfree_skb_cb
*)skb
->cb
;
2247 void netif_schedule_queue(struct netdev_queue
*txq
)
2250 if (!(txq
->state
& QUEUE_STATE_ANY_XOFF
)) {
2251 struct Qdisc
*q
= rcu_dereference(txq
->qdisc
);
2253 __netif_schedule(q
);
2257 EXPORT_SYMBOL(netif_schedule_queue
);
2260 * netif_wake_subqueue - allow sending packets on subqueue
2261 * @dev: network device
2262 * @queue_index: sub queue index
2264 * Resume individual transmit queue of a device with multiple transmit queues.
2266 void netif_wake_subqueue(struct net_device
*dev
, u16 queue_index
)
2268 struct netdev_queue
*txq
= netdev_get_tx_queue(dev
, queue_index
);
2270 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF
, &txq
->state
)) {
2274 q
= rcu_dereference(txq
->qdisc
);
2275 __netif_schedule(q
);
2279 EXPORT_SYMBOL(netif_wake_subqueue
);
2281 void netif_tx_wake_queue(struct netdev_queue
*dev_queue
)
2283 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF
, &dev_queue
->state
)) {
2287 q
= rcu_dereference(dev_queue
->qdisc
);
2288 __netif_schedule(q
);
2292 EXPORT_SYMBOL(netif_tx_wake_queue
);
2294 void __dev_kfree_skb_irq(struct sk_buff
*skb
, enum skb_free_reason reason
)
2296 unsigned long flags
;
2298 if (likely(atomic_read(&skb
->users
) == 1)) {
2300 atomic_set(&skb
->users
, 0);
2301 } else if (likely(!atomic_dec_and_test(&skb
->users
))) {
2304 get_kfree_skb_cb(skb
)->reason
= reason
;
2305 local_irq_save(flags
);
2306 skb
->next
= __this_cpu_read(softnet_data
.completion_queue
);
2307 __this_cpu_write(softnet_data
.completion_queue
, skb
);
2308 raise_softirq_irqoff(NET_TX_SOFTIRQ
);
2309 local_irq_restore(flags
);
2311 EXPORT_SYMBOL(__dev_kfree_skb_irq
);
2313 void __dev_kfree_skb_any(struct sk_buff
*skb
, enum skb_free_reason reason
)
2315 if (in_irq() || irqs_disabled())
2316 __dev_kfree_skb_irq(skb
, reason
);
2320 EXPORT_SYMBOL(__dev_kfree_skb_any
);
2324 * netif_device_detach - mark device as removed
2325 * @dev: network device
2327 * Mark device as removed from system and therefore no longer available.
2329 void netif_device_detach(struct net_device
*dev
)
2331 if (test_and_clear_bit(__LINK_STATE_PRESENT
, &dev
->state
) &&
2332 netif_running(dev
)) {
2333 netif_tx_stop_all_queues(dev
);
2336 EXPORT_SYMBOL(netif_device_detach
);
2339 * netif_device_attach - mark device as attached
2340 * @dev: network device
2342 * Mark device as attached from system and restart if needed.
2344 void netif_device_attach(struct net_device
*dev
)
2346 if (!test_and_set_bit(__LINK_STATE_PRESENT
, &dev
->state
) &&
2347 netif_running(dev
)) {
2348 netif_tx_wake_all_queues(dev
);
2349 __netdev_watchdog_up(dev
);
2352 EXPORT_SYMBOL(netif_device_attach
);
2355 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2356 * to be used as a distribution range.
2358 u16
__skb_tx_hash(const struct net_device
*dev
, struct sk_buff
*skb
,
2359 unsigned int num_tx_queues
)
2363 u16 qcount
= num_tx_queues
;
2365 if (skb_rx_queue_recorded(skb
)) {
2366 hash
= skb_get_rx_queue(skb
);
2367 while (unlikely(hash
>= num_tx_queues
))
2368 hash
-= num_tx_queues
;
2373 u8 tc
= netdev_get_prio_tc_map(dev
, skb
->priority
);
2374 qoffset
= dev
->tc_to_txq
[tc
].offset
;
2375 qcount
= dev
->tc_to_txq
[tc
].count
;
2378 return (u16
) reciprocal_scale(skb_get_hash(skb
), qcount
) + qoffset
;
2380 EXPORT_SYMBOL(__skb_tx_hash
);
2382 static void skb_warn_bad_offload(const struct sk_buff
*skb
)
2384 static const netdev_features_t null_features
= 0;
2385 struct net_device
*dev
= skb
->dev
;
2386 const char *driver
= "";
2388 if (!net_ratelimit())
2391 if (dev
&& dev
->dev
.parent
)
2392 driver
= dev_driver_string(dev
->dev
.parent
);
2394 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2395 "gso_type=%d ip_summed=%d\n",
2396 driver
, dev
? &dev
->features
: &null_features
,
2397 skb
->sk
? &skb
->sk
->sk_route_caps
: &null_features
,
2398 skb
->len
, skb
->data_len
, skb_shinfo(skb
)->gso_size
,
2399 skb_shinfo(skb
)->gso_type
, skb
->ip_summed
);
2403 * Invalidate hardware checksum when packet is to be mangled, and
2404 * complete checksum manually on outgoing path.
2406 int skb_checksum_help(struct sk_buff
*skb
)
2409 int ret
= 0, offset
;
2411 if (skb
->ip_summed
== CHECKSUM_COMPLETE
)
2412 goto out_set_summed
;
2414 if (unlikely(skb_shinfo(skb
)->gso_size
)) {
2415 skb_warn_bad_offload(skb
);
2419 /* Before computing a checksum, we should make sure no frag could
2420 * be modified by an external entity : checksum could be wrong.
2422 if (skb_has_shared_frag(skb
)) {
2423 ret
= __skb_linearize(skb
);
2428 offset
= skb_checksum_start_offset(skb
);
2429 BUG_ON(offset
>= skb_headlen(skb
));
2430 csum
= skb_checksum(skb
, offset
, skb
->len
- offset
, 0);
2432 offset
+= skb
->csum_offset
;
2433 BUG_ON(offset
+ sizeof(__sum16
) > skb_headlen(skb
));
2435 if (skb_cloned(skb
) &&
2436 !skb_clone_writable(skb
, offset
+ sizeof(__sum16
))) {
2437 ret
= pskb_expand_head(skb
, 0, 0, GFP_ATOMIC
);
2442 *(__sum16
*)(skb
->data
+ offset
) = csum_fold(csum
);
2444 skb
->ip_summed
= CHECKSUM_NONE
;
2448 EXPORT_SYMBOL(skb_checksum_help
);
2450 __be16
skb_network_protocol(struct sk_buff
*skb
, int *depth
)
2452 __be16 type
= skb
->protocol
;
2454 /* Tunnel gso handlers can set protocol to ethernet. */
2455 if (type
== htons(ETH_P_TEB
)) {
2458 if (unlikely(!pskb_may_pull(skb
, sizeof(struct ethhdr
))))
2461 eth
= (struct ethhdr
*)skb_mac_header(skb
);
2462 type
= eth
->h_proto
;
2465 return __vlan_get_protocol(skb
, type
, depth
);
2469 * skb_mac_gso_segment - mac layer segmentation handler.
2470 * @skb: buffer to segment
2471 * @features: features for the output path (see dev->features)
2473 struct sk_buff
*skb_mac_gso_segment(struct sk_buff
*skb
,
2474 netdev_features_t features
)
2476 struct sk_buff
*segs
= ERR_PTR(-EPROTONOSUPPORT
);
2477 struct packet_offload
*ptype
;
2478 int vlan_depth
= skb
->mac_len
;
2479 __be16 type
= skb_network_protocol(skb
, &vlan_depth
);
2481 if (unlikely(!type
))
2482 return ERR_PTR(-EINVAL
);
2484 __skb_pull(skb
, vlan_depth
);
2487 list_for_each_entry_rcu(ptype
, &offload_base
, list
) {
2488 if (ptype
->type
== type
&& ptype
->callbacks
.gso_segment
) {
2489 segs
= ptype
->callbacks
.gso_segment(skb
, features
);
2495 __skb_push(skb
, skb
->data
- skb_mac_header(skb
));
2499 EXPORT_SYMBOL(skb_mac_gso_segment
);
2502 /* openvswitch calls this on rx path, so we need a different check.
2504 static inline bool skb_needs_check(struct sk_buff
*skb
, bool tx_path
)
2507 return skb
->ip_summed
!= CHECKSUM_PARTIAL
;
2509 return skb
->ip_summed
== CHECKSUM_NONE
;
2513 * __skb_gso_segment - Perform segmentation on skb.
2514 * @skb: buffer to segment
2515 * @features: features for the output path (see dev->features)
2516 * @tx_path: whether it is called in TX path
2518 * This function segments the given skb and returns a list of segments.
2520 * It may return NULL if the skb requires no segmentation. This is
2521 * only possible when GSO is used for verifying header integrity.
2523 struct sk_buff
*__skb_gso_segment(struct sk_buff
*skb
,
2524 netdev_features_t features
, bool tx_path
)
2526 if (unlikely(skb_needs_check(skb
, tx_path
))) {
2529 skb_warn_bad_offload(skb
);
2531 err
= skb_cow_head(skb
, 0);
2533 return ERR_PTR(err
);
2536 SKB_GSO_CB(skb
)->mac_offset
= skb_headroom(skb
);
2537 SKB_GSO_CB(skb
)->encap_level
= 0;
2539 skb_reset_mac_header(skb
);
2540 skb_reset_mac_len(skb
);
2542 return skb_mac_gso_segment(skb
, features
);
2544 EXPORT_SYMBOL(__skb_gso_segment
);
2546 /* Take action when hardware reception checksum errors are detected. */
2548 void netdev_rx_csum_fault(struct net_device
*dev
)
2550 if (net_ratelimit()) {
2551 pr_err("%s: hw csum failure\n", dev
? dev
->name
: "<unknown>");
2555 EXPORT_SYMBOL(netdev_rx_csum_fault
);
2558 /* Actually, we should eliminate this check as soon as we know, that:
2559 * 1. IOMMU is present and allows to map all the memory.
2560 * 2. No high memory really exists on this machine.
2563 static int illegal_highdma(struct net_device
*dev
, struct sk_buff
*skb
)
2565 #ifdef CONFIG_HIGHMEM
2567 if (!(dev
->features
& NETIF_F_HIGHDMA
)) {
2568 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
2569 skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
];
2570 if (PageHighMem(skb_frag_page(frag
)))
2575 if (PCI_DMA_BUS_IS_PHYS
) {
2576 struct device
*pdev
= dev
->dev
.parent
;
2580 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
2581 skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
];
2582 dma_addr_t addr
= page_to_phys(skb_frag_page(frag
));
2583 if (!pdev
->dma_mask
|| addr
+ PAGE_SIZE
- 1 > *pdev
->dma_mask
)
2591 /* If MPLS offload request, verify we are testing hardware MPLS features
2592 * instead of standard features for the netdev.
2594 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2595 static netdev_features_t
net_mpls_features(struct sk_buff
*skb
,
2596 netdev_features_t features
,
2599 if (eth_p_mpls(type
))
2600 features
&= skb
->dev
->mpls_features
;
2605 static netdev_features_t
net_mpls_features(struct sk_buff
*skb
,
2606 netdev_features_t features
,
2613 static netdev_features_t
harmonize_features(struct sk_buff
*skb
,
2614 netdev_features_t features
)
2619 type
= skb_network_protocol(skb
, &tmp
);
2620 features
= net_mpls_features(skb
, features
, type
);
2622 if (skb
->ip_summed
!= CHECKSUM_NONE
&&
2623 !can_checksum_protocol(features
, type
)) {
2624 features
&= ~NETIF_F_ALL_CSUM
;
2625 } else if (illegal_highdma(skb
->dev
, skb
)) {
2626 features
&= ~NETIF_F_SG
;
2632 netdev_features_t
passthru_features_check(struct sk_buff
*skb
,
2633 struct net_device
*dev
,
2634 netdev_features_t features
)
2638 EXPORT_SYMBOL(passthru_features_check
);
2640 static netdev_features_t
dflt_features_check(const struct sk_buff
*skb
,
2641 struct net_device
*dev
,
2642 netdev_features_t features
)
2644 return vlan_features_check(skb
, features
);
2647 netdev_features_t
netif_skb_features(struct sk_buff
*skb
)
2649 struct net_device
*dev
= skb
->dev
;
2650 netdev_features_t features
= dev
->features
;
2651 u16 gso_segs
= skb_shinfo(skb
)->gso_segs
;
2653 if (gso_segs
> dev
->gso_max_segs
|| gso_segs
< dev
->gso_min_segs
)
2654 features
&= ~NETIF_F_GSO_MASK
;
2656 /* If encapsulation offload request, verify we are testing
2657 * hardware encapsulation features instead of standard
2658 * features for the netdev
2660 if (skb
->encapsulation
)
2661 features
&= dev
->hw_enc_features
;
2663 if (skb_vlan_tagged(skb
))
2664 features
= netdev_intersect_features(features
,
2665 dev
->vlan_features
|
2666 NETIF_F_HW_VLAN_CTAG_TX
|
2667 NETIF_F_HW_VLAN_STAG_TX
);
2669 if (dev
->netdev_ops
->ndo_features_check
)
2670 features
&= dev
->netdev_ops
->ndo_features_check(skb
, dev
,
2673 features
&= dflt_features_check(skb
, dev
, features
);
2675 return harmonize_features(skb
, features
);
2677 EXPORT_SYMBOL(netif_skb_features
);
2679 static int xmit_one(struct sk_buff
*skb
, struct net_device
*dev
,
2680 struct netdev_queue
*txq
, bool more
)
2685 if (!list_empty(&ptype_all
) || !list_empty(&dev
->ptype_all
))
2686 dev_queue_xmit_nit(skb
, dev
);
2689 trace_net_dev_start_xmit(skb
, dev
);
2690 rc
= netdev_start_xmit(skb
, dev
, txq
, more
);
2691 trace_net_dev_xmit(skb
, rc
, dev
, len
);
2696 struct sk_buff
*dev_hard_start_xmit(struct sk_buff
*first
, struct net_device
*dev
,
2697 struct netdev_queue
*txq
, int *ret
)
2699 struct sk_buff
*skb
= first
;
2700 int rc
= NETDEV_TX_OK
;
2703 struct sk_buff
*next
= skb
->next
;
2706 rc
= xmit_one(skb
, dev
, txq
, next
!= NULL
);
2707 if (unlikely(!dev_xmit_complete(rc
))) {
2713 if (netif_xmit_stopped(txq
) && skb
) {
2714 rc
= NETDEV_TX_BUSY
;
2724 static struct sk_buff
*validate_xmit_vlan(struct sk_buff
*skb
,
2725 netdev_features_t features
)
2727 if (skb_vlan_tag_present(skb
) &&
2728 !vlan_hw_offload_capable(features
, skb
->vlan_proto
))
2729 skb
= __vlan_hwaccel_push_inside(skb
);
2733 static struct sk_buff
*validate_xmit_skb(struct sk_buff
*skb
, struct net_device
*dev
)
2735 netdev_features_t features
;
2740 features
= netif_skb_features(skb
);
2741 skb
= validate_xmit_vlan(skb
, features
);
2745 if (netif_needs_gso(skb
, features
)) {
2746 struct sk_buff
*segs
;
2748 segs
= skb_gso_segment(skb
, features
);
2756 if (skb_needs_linearize(skb
, features
) &&
2757 __skb_linearize(skb
))
2760 /* If packet is not checksummed and device does not
2761 * support checksumming for this protocol, complete
2762 * checksumming here.
2764 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
2765 if (skb
->encapsulation
)
2766 skb_set_inner_transport_header(skb
,
2767 skb_checksum_start_offset(skb
));
2769 skb_set_transport_header(skb
,
2770 skb_checksum_start_offset(skb
));
2771 if (!(features
& NETIF_F_ALL_CSUM
) &&
2772 skb_checksum_help(skb
))
2785 struct sk_buff
*validate_xmit_skb_list(struct sk_buff
*skb
, struct net_device
*dev
)
2787 struct sk_buff
*next
, *head
= NULL
, *tail
;
2789 for (; skb
!= NULL
; skb
= next
) {
2793 /* in case skb wont be segmented, point to itself */
2796 skb
= validate_xmit_skb(skb
, dev
);
2804 /* If skb was segmented, skb->prev points to
2805 * the last segment. If not, it still contains skb.
2812 static void qdisc_pkt_len_init(struct sk_buff
*skb
)
2814 const struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
2816 qdisc_skb_cb(skb
)->pkt_len
= skb
->len
;
2818 /* To get more precise estimation of bytes sent on wire,
2819 * we add to pkt_len the headers size of all segments
2821 if (shinfo
->gso_size
) {
2822 unsigned int hdr_len
;
2823 u16 gso_segs
= shinfo
->gso_segs
;
2825 /* mac layer + network layer */
2826 hdr_len
= skb_transport_header(skb
) - skb_mac_header(skb
);
2828 /* + transport layer */
2829 if (likely(shinfo
->gso_type
& (SKB_GSO_TCPV4
| SKB_GSO_TCPV6
)))
2830 hdr_len
+= tcp_hdrlen(skb
);
2832 hdr_len
+= sizeof(struct udphdr
);
2834 if (shinfo
->gso_type
& SKB_GSO_DODGY
)
2835 gso_segs
= DIV_ROUND_UP(skb
->len
- hdr_len
,
2838 qdisc_skb_cb(skb
)->pkt_len
+= (gso_segs
- 1) * hdr_len
;
2842 static inline int __dev_xmit_skb(struct sk_buff
*skb
, struct Qdisc
*q
,
2843 struct net_device
*dev
,
2844 struct netdev_queue
*txq
)
2846 spinlock_t
*root_lock
= qdisc_lock(q
);
2850 qdisc_pkt_len_init(skb
);
2851 qdisc_calculate_pkt_len(skb
, q
);
2853 * Heuristic to force contended enqueues to serialize on a
2854 * separate lock before trying to get qdisc main lock.
2855 * This permits __QDISC___STATE_RUNNING owner to get the lock more
2856 * often and dequeue packets faster.
2858 contended
= qdisc_is_running(q
);
2859 if (unlikely(contended
))
2860 spin_lock(&q
->busylock
);
2862 spin_lock(root_lock
);
2863 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED
, &q
->state
))) {
2866 } else if ((q
->flags
& TCQ_F_CAN_BYPASS
) && !qdisc_qlen(q
) &&
2867 qdisc_run_begin(q
)) {
2869 * This is a work-conserving queue; there are no old skbs
2870 * waiting to be sent out; and the qdisc is not running -
2871 * xmit the skb directly.
2874 qdisc_bstats_update(q
, skb
);
2876 if (sch_direct_xmit(skb
, q
, dev
, txq
, root_lock
, true)) {
2877 if (unlikely(contended
)) {
2878 spin_unlock(&q
->busylock
);
2885 rc
= NET_XMIT_SUCCESS
;
2887 rc
= q
->enqueue(skb
, q
) & NET_XMIT_MASK
;
2888 if (qdisc_run_begin(q
)) {
2889 if (unlikely(contended
)) {
2890 spin_unlock(&q
->busylock
);
2896 spin_unlock(root_lock
);
2897 if (unlikely(contended
))
2898 spin_unlock(&q
->busylock
);
2902 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
2903 static void skb_update_prio(struct sk_buff
*skb
)
2905 struct netprio_map
*map
= rcu_dereference_bh(skb
->dev
->priomap
);
2907 if (!skb
->priority
&& skb
->sk
&& map
) {
2908 unsigned int prioidx
= skb
->sk
->sk_cgrp_prioidx
;
2910 if (prioidx
< map
->priomap_len
)
2911 skb
->priority
= map
->priomap
[prioidx
];
2915 #define skb_update_prio(skb)
2918 DEFINE_PER_CPU(int, xmit_recursion
);
2919 EXPORT_SYMBOL(xmit_recursion
);
2921 #define RECURSION_LIMIT 10
2924 * dev_loopback_xmit - loop back @skb
2925 * @skb: buffer to transmit
2927 int dev_loopback_xmit(struct sock
*sk
, struct sk_buff
*skb
)
2929 skb_reset_mac_header(skb
);
2930 __skb_pull(skb
, skb_network_offset(skb
));
2931 skb
->pkt_type
= PACKET_LOOPBACK
;
2932 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
2933 WARN_ON(!skb_dst(skb
));
2938 EXPORT_SYMBOL(dev_loopback_xmit
);
2940 static inline int get_xps_queue(struct net_device
*dev
, struct sk_buff
*skb
)
2943 struct xps_dev_maps
*dev_maps
;
2944 struct xps_map
*map
;
2945 int queue_index
= -1;
2948 dev_maps
= rcu_dereference(dev
->xps_maps
);
2950 map
= rcu_dereference(
2951 dev_maps
->cpu_map
[skb
->sender_cpu
- 1]);
2954 queue_index
= map
->queues
[0];
2956 queue_index
= map
->queues
[reciprocal_scale(skb_get_hash(skb
),
2958 if (unlikely(queue_index
>= dev
->real_num_tx_queues
))
2970 static u16
__netdev_pick_tx(struct net_device
*dev
, struct sk_buff
*skb
)
2972 struct sock
*sk
= skb
->sk
;
2973 int queue_index
= sk_tx_queue_get(sk
);
2975 if (queue_index
< 0 || skb
->ooo_okay
||
2976 queue_index
>= dev
->real_num_tx_queues
) {
2977 int new_index
= get_xps_queue(dev
, skb
);
2979 new_index
= skb_tx_hash(dev
, skb
);
2981 if (queue_index
!= new_index
&& sk
&&
2982 rcu_access_pointer(sk
->sk_dst_cache
))
2983 sk_tx_queue_set(sk
, new_index
);
2985 queue_index
= new_index
;
2991 struct netdev_queue
*netdev_pick_tx(struct net_device
*dev
,
2992 struct sk_buff
*skb
,
2995 int queue_index
= 0;
2998 if (skb
->sender_cpu
== 0)
2999 skb
->sender_cpu
= raw_smp_processor_id() + 1;
3002 if (dev
->real_num_tx_queues
!= 1) {
3003 const struct net_device_ops
*ops
= dev
->netdev_ops
;
3004 if (ops
->ndo_select_queue
)
3005 queue_index
= ops
->ndo_select_queue(dev
, skb
, accel_priv
,
3008 queue_index
= __netdev_pick_tx(dev
, skb
);
3011 queue_index
= netdev_cap_txqueue(dev
, queue_index
);
3014 skb_set_queue_mapping(skb
, queue_index
);
3015 return netdev_get_tx_queue(dev
, queue_index
);
3019 * __dev_queue_xmit - transmit a buffer
3020 * @skb: buffer to transmit
3021 * @accel_priv: private data used for L2 forwarding offload
3023 * Queue a buffer for transmission to a network device. The caller must
3024 * have set the device and priority and built the buffer before calling
3025 * this function. The function can be called from an interrupt.
3027 * A negative errno code is returned on a failure. A success does not
3028 * guarantee the frame will be transmitted as it may be dropped due
3029 * to congestion or traffic shaping.
3031 * -----------------------------------------------------------------------------------
3032 * I notice this method can also return errors from the queue disciplines,
3033 * including NET_XMIT_DROP, which is a positive value. So, errors can also
3036 * Regardless of the return value, the skb is consumed, so it is currently
3037 * difficult to retry a send to this method. (You can bump the ref count
3038 * before sending to hold a reference for retry if you are careful.)
3040 * When calling this method, interrupts MUST be enabled. This is because
3041 * the BH enable code must have IRQs enabled so that it will not deadlock.
3044 static int __dev_queue_xmit(struct sk_buff
*skb
, void *accel_priv
)
3046 struct net_device
*dev
= skb
->dev
;
3047 struct netdev_queue
*txq
;
3051 skb_reset_mac_header(skb
);
3053 if (unlikely(skb_shinfo(skb
)->tx_flags
& SKBTX_SCHED_TSTAMP
))
3054 __skb_tstamp_tx(skb
, NULL
, skb
->sk
, SCM_TSTAMP_SCHED
);
3056 /* Disable soft irqs for various locks below. Also
3057 * stops preemption for RCU.
3061 skb_update_prio(skb
);
3063 /* If device/qdisc don't need skb->dst, release it right now while
3064 * its hot in this cpu cache.
3066 if (dev
->priv_flags
& IFF_XMIT_DST_RELEASE
)
3071 txq
= netdev_pick_tx(dev
, skb
, accel_priv
);
3072 q
= rcu_dereference_bh(txq
->qdisc
);
3074 #ifdef CONFIG_NET_CLS_ACT
3075 skb
->tc_verd
= SET_TC_AT(skb
->tc_verd
, AT_EGRESS
);
3077 trace_net_dev_queue(skb
);
3079 rc
= __dev_xmit_skb(skb
, q
, dev
, txq
);
3083 /* The device has no queue. Common case for software devices:
3084 loopback, all the sorts of tunnels...
3086 Really, it is unlikely that netif_tx_lock protection is necessary
3087 here. (f.e. loopback and IP tunnels are clean ignoring statistics
3089 However, it is possible, that they rely on protection
3092 Check this and shot the lock. It is not prone from deadlocks.
3093 Either shot noqueue qdisc, it is even simpler 8)
3095 if (dev
->flags
& IFF_UP
) {
3096 int cpu
= smp_processor_id(); /* ok because BHs are off */
3098 if (txq
->xmit_lock_owner
!= cpu
) {
3100 if (__this_cpu_read(xmit_recursion
) > RECURSION_LIMIT
)
3101 goto recursion_alert
;
3103 skb
= validate_xmit_skb(skb
, dev
);
3107 HARD_TX_LOCK(dev
, txq
, cpu
);
3109 if (!netif_xmit_stopped(txq
)) {
3110 __this_cpu_inc(xmit_recursion
);
3111 skb
= dev_hard_start_xmit(skb
, dev
, txq
, &rc
);
3112 __this_cpu_dec(xmit_recursion
);
3113 if (dev_xmit_complete(rc
)) {
3114 HARD_TX_UNLOCK(dev
, txq
);
3118 HARD_TX_UNLOCK(dev
, txq
);
3119 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3122 /* Recursion is detected! It is possible,
3126 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3133 rcu_read_unlock_bh();
3135 atomic_long_inc(&dev
->tx_dropped
);
3136 kfree_skb_list(skb
);
3139 rcu_read_unlock_bh();
3143 int dev_queue_xmit_sk(struct sock
*sk
, struct sk_buff
*skb
)
3145 return __dev_queue_xmit(skb
, NULL
);
3147 EXPORT_SYMBOL(dev_queue_xmit_sk
);
3149 int dev_queue_xmit_accel(struct sk_buff
*skb
, void *accel_priv
)
3151 return __dev_queue_xmit(skb
, accel_priv
);
3153 EXPORT_SYMBOL(dev_queue_xmit_accel
);
3156 /*=======================================================================
3158 =======================================================================*/
3160 int netdev_max_backlog __read_mostly
= 1000;
3161 EXPORT_SYMBOL(netdev_max_backlog
);
3163 int netdev_tstamp_prequeue __read_mostly
= 1;
3164 int netdev_budget __read_mostly
= 300;
3165 int weight_p __read_mostly
= 64; /* old backlog weight */
3167 /* Called with irq disabled */
3168 static inline void ____napi_schedule(struct softnet_data
*sd
,
3169 struct napi_struct
*napi
)
3171 list_add_tail(&napi
->poll_list
, &sd
->poll_list
);
3172 __raise_softirq_irqoff(NET_RX_SOFTIRQ
);
3177 /* One global table that all flow-based protocols share. */
3178 struct rps_sock_flow_table __rcu
*rps_sock_flow_table __read_mostly
;
3179 EXPORT_SYMBOL(rps_sock_flow_table
);
3180 u32 rps_cpu_mask __read_mostly
;
3181 EXPORT_SYMBOL(rps_cpu_mask
);
3183 struct static_key rps_needed __read_mostly
;
3185 static struct rps_dev_flow
*
3186 set_rps_cpu(struct net_device
*dev
, struct sk_buff
*skb
,
3187 struct rps_dev_flow
*rflow
, u16 next_cpu
)
3189 if (next_cpu
< nr_cpu_ids
) {
3190 #ifdef CONFIG_RFS_ACCEL
3191 struct netdev_rx_queue
*rxqueue
;
3192 struct rps_dev_flow_table
*flow_table
;
3193 struct rps_dev_flow
*old_rflow
;
3198 /* Should we steer this flow to a different hardware queue? */
3199 if (!skb_rx_queue_recorded(skb
) || !dev
->rx_cpu_rmap
||
3200 !(dev
->features
& NETIF_F_NTUPLE
))
3202 rxq_index
= cpu_rmap_lookup_index(dev
->rx_cpu_rmap
, next_cpu
);
3203 if (rxq_index
== skb_get_rx_queue(skb
))
3206 rxqueue
= dev
->_rx
+ rxq_index
;
3207 flow_table
= rcu_dereference(rxqueue
->rps_flow_table
);
3210 flow_id
= skb_get_hash(skb
) & flow_table
->mask
;
3211 rc
= dev
->netdev_ops
->ndo_rx_flow_steer(dev
, skb
,
3212 rxq_index
, flow_id
);
3216 rflow
= &flow_table
->flows
[flow_id
];
3218 if (old_rflow
->filter
== rflow
->filter
)
3219 old_rflow
->filter
= RPS_NO_FILTER
;
3223 per_cpu(softnet_data
, next_cpu
).input_queue_head
;
3226 rflow
->cpu
= next_cpu
;
3231 * get_rps_cpu is called from netif_receive_skb and returns the target
3232 * CPU from the RPS map of the receiving queue for a given skb.
3233 * rcu_read_lock must be held on entry.
3235 static int get_rps_cpu(struct net_device
*dev
, struct sk_buff
*skb
,
3236 struct rps_dev_flow
**rflowp
)
3238 const struct rps_sock_flow_table
*sock_flow_table
;
3239 struct netdev_rx_queue
*rxqueue
= dev
->_rx
;
3240 struct rps_dev_flow_table
*flow_table
;
3241 struct rps_map
*map
;
3246 if (skb_rx_queue_recorded(skb
)) {
3247 u16 index
= skb_get_rx_queue(skb
);
3249 if (unlikely(index
>= dev
->real_num_rx_queues
)) {
3250 WARN_ONCE(dev
->real_num_rx_queues
> 1,
3251 "%s received packet on queue %u, but number "
3252 "of RX queues is %u\n",
3253 dev
->name
, index
, dev
->real_num_rx_queues
);
3259 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3261 flow_table
= rcu_dereference(rxqueue
->rps_flow_table
);
3262 map
= rcu_dereference(rxqueue
->rps_map
);
3263 if (!flow_table
&& !map
)
3266 skb_reset_network_header(skb
);
3267 hash
= skb_get_hash(skb
);
3271 sock_flow_table
= rcu_dereference(rps_sock_flow_table
);
3272 if (flow_table
&& sock_flow_table
) {
3273 struct rps_dev_flow
*rflow
;
3277 /* First check into global flow table if there is a match */
3278 ident
= sock_flow_table
->ents
[hash
& sock_flow_table
->mask
];
3279 if ((ident
^ hash
) & ~rps_cpu_mask
)
3282 next_cpu
= ident
& rps_cpu_mask
;
3284 /* OK, now we know there is a match,
3285 * we can look at the local (per receive queue) flow table
3287 rflow
= &flow_table
->flows
[hash
& flow_table
->mask
];
3291 * If the desired CPU (where last recvmsg was done) is
3292 * different from current CPU (one in the rx-queue flow
3293 * table entry), switch if one of the following holds:
3294 * - Current CPU is unset (>= nr_cpu_ids).
3295 * - Current CPU is offline.
3296 * - The current CPU's queue tail has advanced beyond the
3297 * last packet that was enqueued using this table entry.
3298 * This guarantees that all previous packets for the flow
3299 * have been dequeued, thus preserving in order delivery.
3301 if (unlikely(tcpu
!= next_cpu
) &&
3302 (tcpu
>= nr_cpu_ids
|| !cpu_online(tcpu
) ||
3303 ((int)(per_cpu(softnet_data
, tcpu
).input_queue_head
-
3304 rflow
->last_qtail
)) >= 0)) {
3306 rflow
= set_rps_cpu(dev
, skb
, rflow
, next_cpu
);
3309 if (tcpu
< nr_cpu_ids
&& cpu_online(tcpu
)) {
3319 tcpu
= map
->cpus
[reciprocal_scale(hash
, map
->len
)];
3320 if (cpu_online(tcpu
)) {
3330 #ifdef CONFIG_RFS_ACCEL
3333 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3334 * @dev: Device on which the filter was set
3335 * @rxq_index: RX queue index
3336 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3337 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3339 * Drivers that implement ndo_rx_flow_steer() should periodically call
3340 * this function for each installed filter and remove the filters for
3341 * which it returns %true.
3343 bool rps_may_expire_flow(struct net_device
*dev
, u16 rxq_index
,
3344 u32 flow_id
, u16 filter_id
)
3346 struct netdev_rx_queue
*rxqueue
= dev
->_rx
+ rxq_index
;
3347 struct rps_dev_flow_table
*flow_table
;
3348 struct rps_dev_flow
*rflow
;
3353 flow_table
= rcu_dereference(rxqueue
->rps_flow_table
);
3354 if (flow_table
&& flow_id
<= flow_table
->mask
) {
3355 rflow
= &flow_table
->flows
[flow_id
];
3356 cpu
= ACCESS_ONCE(rflow
->cpu
);
3357 if (rflow
->filter
== filter_id
&& cpu
< nr_cpu_ids
&&
3358 ((int)(per_cpu(softnet_data
, cpu
).input_queue_head
-
3359 rflow
->last_qtail
) <
3360 (int)(10 * flow_table
->mask
)))
3366 EXPORT_SYMBOL(rps_may_expire_flow
);
3368 #endif /* CONFIG_RFS_ACCEL */
3370 /* Called from hardirq (IPI) context */
3371 static void rps_trigger_softirq(void *data
)
3373 struct softnet_data
*sd
= data
;
3375 ____napi_schedule(sd
, &sd
->backlog
);
3379 #endif /* CONFIG_RPS */
3382 * Check if this softnet_data structure is another cpu one
3383 * If yes, queue it to our IPI list and return 1
3386 static int rps_ipi_queued(struct softnet_data
*sd
)
3389 struct softnet_data
*mysd
= this_cpu_ptr(&softnet_data
);
3392 sd
->rps_ipi_next
= mysd
->rps_ipi_list
;
3393 mysd
->rps_ipi_list
= sd
;
3395 __raise_softirq_irqoff(NET_RX_SOFTIRQ
);
3398 #endif /* CONFIG_RPS */
3402 #ifdef CONFIG_NET_FLOW_LIMIT
3403 int netdev_flow_limit_table_len __read_mostly
= (1 << 12);
3406 static bool skb_flow_limit(struct sk_buff
*skb
, unsigned int qlen
)
3408 #ifdef CONFIG_NET_FLOW_LIMIT
3409 struct sd_flow_limit
*fl
;
3410 struct softnet_data
*sd
;
3411 unsigned int old_flow
, new_flow
;
3413 if (qlen
< (netdev_max_backlog
>> 1))
3416 sd
= this_cpu_ptr(&softnet_data
);
3419 fl
= rcu_dereference(sd
->flow_limit
);
3421 new_flow
= skb_get_hash(skb
) & (fl
->num_buckets
- 1);
3422 old_flow
= fl
->history
[fl
->history_head
];
3423 fl
->history
[fl
->history_head
] = new_flow
;
3426 fl
->history_head
&= FLOW_LIMIT_HISTORY
- 1;
3428 if (likely(fl
->buckets
[old_flow
]))
3429 fl
->buckets
[old_flow
]--;
3431 if (++fl
->buckets
[new_flow
] > (FLOW_LIMIT_HISTORY
>> 1)) {
3443 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3444 * queue (may be a remote CPU queue).
3446 static int enqueue_to_backlog(struct sk_buff
*skb
, int cpu
,
3447 unsigned int *qtail
)
3449 struct softnet_data
*sd
;
3450 unsigned long flags
;
3453 sd
= &per_cpu(softnet_data
, cpu
);
3455 local_irq_save(flags
);
3458 qlen
= skb_queue_len(&sd
->input_pkt_queue
);
3459 if (qlen
<= netdev_max_backlog
&& !skb_flow_limit(skb
, qlen
)) {
3462 __skb_queue_tail(&sd
->input_pkt_queue
, skb
);
3463 input_queue_tail_incr_save(sd
, qtail
);
3465 local_irq_restore(flags
);
3466 return NET_RX_SUCCESS
;
3469 /* Schedule NAPI for backlog device
3470 * We can use non atomic operation since we own the queue lock
3472 if (!__test_and_set_bit(NAPI_STATE_SCHED
, &sd
->backlog
.state
)) {
3473 if (!rps_ipi_queued(sd
))
3474 ____napi_schedule(sd
, &sd
->backlog
);
3482 local_irq_restore(flags
);
3484 atomic_long_inc(&skb
->dev
->rx_dropped
);
3489 static int netif_rx_internal(struct sk_buff
*skb
)
3493 net_timestamp_check(netdev_tstamp_prequeue
, skb
);
3495 trace_netif_rx(skb
);
3497 if (static_key_false(&rps_needed
)) {
3498 struct rps_dev_flow voidflow
, *rflow
= &voidflow
;
3504 cpu
= get_rps_cpu(skb
->dev
, skb
, &rflow
);
3506 cpu
= smp_processor_id();
3508 ret
= enqueue_to_backlog(skb
, cpu
, &rflow
->last_qtail
);
3516 ret
= enqueue_to_backlog(skb
, get_cpu(), &qtail
);
3523 * netif_rx - post buffer to the network code
3524 * @skb: buffer to post
3526 * This function receives a packet from a device driver and queues it for
3527 * the upper (protocol) levels to process. It always succeeds. The buffer
3528 * may be dropped during processing for congestion control or by the
3532 * NET_RX_SUCCESS (no congestion)
3533 * NET_RX_DROP (packet was dropped)
3537 int netif_rx(struct sk_buff
*skb
)
3539 trace_netif_rx_entry(skb
);
3541 return netif_rx_internal(skb
);
3543 EXPORT_SYMBOL(netif_rx
);
3545 int netif_rx_ni(struct sk_buff
*skb
)
3549 trace_netif_rx_ni_entry(skb
);
3552 err
= netif_rx_internal(skb
);
3553 if (local_softirq_pending())
3559 EXPORT_SYMBOL(netif_rx_ni
);
3561 static void net_tx_action(struct softirq_action
*h
)
3563 struct softnet_data
*sd
= this_cpu_ptr(&softnet_data
);
3565 if (sd
->completion_queue
) {
3566 struct sk_buff
*clist
;
3568 local_irq_disable();
3569 clist
= sd
->completion_queue
;
3570 sd
->completion_queue
= NULL
;
3574 struct sk_buff
*skb
= clist
;
3575 clist
= clist
->next
;
3577 WARN_ON(atomic_read(&skb
->users
));
3578 if (likely(get_kfree_skb_cb(skb
)->reason
== SKB_REASON_CONSUMED
))
3579 trace_consume_skb(skb
);
3581 trace_kfree_skb(skb
, net_tx_action
);
3586 if (sd
->output_queue
) {
3589 local_irq_disable();
3590 head
= sd
->output_queue
;
3591 sd
->output_queue
= NULL
;
3592 sd
->output_queue_tailp
= &sd
->output_queue
;
3596 struct Qdisc
*q
= head
;
3597 spinlock_t
*root_lock
;
3599 head
= head
->next_sched
;
3601 root_lock
= qdisc_lock(q
);
3602 if (spin_trylock(root_lock
)) {
3603 smp_mb__before_atomic();
3604 clear_bit(__QDISC_STATE_SCHED
,
3607 spin_unlock(root_lock
);
3609 if (!test_bit(__QDISC_STATE_DEACTIVATED
,
3611 __netif_reschedule(q
);
3613 smp_mb__before_atomic();
3614 clear_bit(__QDISC_STATE_SCHED
,
3622 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3623 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
3624 /* This hook is defined here for ATM LANE */
3625 int (*br_fdb_test_addr_hook
)(struct net_device
*dev
,
3626 unsigned char *addr
) __read_mostly
;
3627 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook
);
3630 #ifdef CONFIG_NET_CLS_ACT
3631 static inline struct sk_buff
*handle_ing(struct sk_buff
*skb
,
3632 struct packet_type
**pt_prev
,
3633 int *ret
, struct net_device
*orig_dev
)
3635 struct tcf_proto
*cl
= rcu_dereference_bh(skb
->dev
->ingress_cl_list
);
3636 struct tcf_result cl_res
;
3638 /* If there's at least one ingress present somewhere (so
3639 * we get here via enabled static key), remaining devices
3640 * that are not configured with an ingress qdisc will bail
3646 *ret
= deliver_skb(skb
, *pt_prev
, orig_dev
);
3650 qdisc_bstats_update_cpu(cl
->q
, skb
);
3651 skb
->tc_verd
= SET_TC_AT(skb
->tc_verd
, AT_INGRESS
);
3653 switch (tc_classify(skb
, cl
, &cl_res
)) {
3655 case TC_ACT_RECLASSIFY
:
3656 skb
->tc_index
= TC_H_MIN(cl_res
.classid
);
3659 qdisc_qstats_drop_cpu(cl
->q
);
3671 static inline struct sk_buff
*handle_ing(struct sk_buff
*skb
,
3672 struct packet_type
**pt_prev
,
3673 int *ret
, struct net_device
*orig_dev
)
3680 * netdev_rx_handler_register - register receive handler
3681 * @dev: device to register a handler for
3682 * @rx_handler: receive handler to register
3683 * @rx_handler_data: data pointer that is used by rx handler
3685 * Register a receive handler for a device. This handler will then be
3686 * called from __netif_receive_skb. A negative errno code is returned
3689 * The caller must hold the rtnl_mutex.
3691 * For a general description of rx_handler, see enum rx_handler_result.
3693 int netdev_rx_handler_register(struct net_device
*dev
,
3694 rx_handler_func_t
*rx_handler
,
3695 void *rx_handler_data
)
3699 if (dev
->rx_handler
)
3702 /* Note: rx_handler_data must be set before rx_handler */
3703 rcu_assign_pointer(dev
->rx_handler_data
, rx_handler_data
);
3704 rcu_assign_pointer(dev
->rx_handler
, rx_handler
);
3708 EXPORT_SYMBOL_GPL(netdev_rx_handler_register
);
3711 * netdev_rx_handler_unregister - unregister receive handler
3712 * @dev: device to unregister a handler from
3714 * Unregister a receive handler from a device.
3716 * The caller must hold the rtnl_mutex.
3718 void netdev_rx_handler_unregister(struct net_device
*dev
)
3722 RCU_INIT_POINTER(dev
->rx_handler
, NULL
);
3723 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3724 * section has a guarantee to see a non NULL rx_handler_data
3728 RCU_INIT_POINTER(dev
->rx_handler_data
, NULL
);
3730 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister
);
3733 * Limit the use of PFMEMALLOC reserves to those protocols that implement
3734 * the special handling of PFMEMALLOC skbs.
3736 static bool skb_pfmemalloc_protocol(struct sk_buff
*skb
)
3738 switch (skb
->protocol
) {
3739 case htons(ETH_P_ARP
):
3740 case htons(ETH_P_IP
):
3741 case htons(ETH_P_IPV6
):
3742 case htons(ETH_P_8021Q
):
3743 case htons(ETH_P_8021AD
):
3750 #ifdef CONFIG_NETFILTER_INGRESS
3751 static inline int nf_ingress(struct sk_buff
*skb
, struct packet_type
**pt_prev
,
3752 int *ret
, struct net_device
*orig_dev
)
3754 if (nf_hook_ingress_active(skb
)) {
3756 *ret
= deliver_skb(skb
, *pt_prev
, orig_dev
);
3760 return nf_hook_ingress(skb
);
3765 static inline int nf_ingress(struct sk_buff
*skb
, struct packet_type
**pt_prev
,
3766 int *ret
, struct net_device
*orig_dev
)
3772 static int __netif_receive_skb_core(struct sk_buff
*skb
, bool pfmemalloc
)
3774 struct packet_type
*ptype
, *pt_prev
;
3775 rx_handler_func_t
*rx_handler
;
3776 struct net_device
*orig_dev
;
3777 bool deliver_exact
= false;
3778 int ret
= NET_RX_DROP
;
3781 net_timestamp_check(!netdev_tstamp_prequeue
, skb
);
3783 trace_netif_receive_skb(skb
);
3785 orig_dev
= skb
->dev
;
3787 skb_reset_network_header(skb
);
3788 if (!skb_transport_header_was_set(skb
))
3789 skb_reset_transport_header(skb
);
3790 skb_reset_mac_len(skb
);
3797 skb
->skb_iif
= skb
->dev
->ifindex
;
3799 __this_cpu_inc(softnet_data
.processed
);
3801 if (skb
->protocol
== cpu_to_be16(ETH_P_8021Q
) ||
3802 skb
->protocol
== cpu_to_be16(ETH_P_8021AD
)) {
3803 skb
= skb_vlan_untag(skb
);
3808 #ifdef CONFIG_NET_CLS_ACT
3809 if (skb
->tc_verd
& TC_NCLS
) {
3810 skb
->tc_verd
= CLR_TC_NCLS(skb
->tc_verd
);
3818 list_for_each_entry_rcu(ptype
, &ptype_all
, list
) {
3820 ret
= deliver_skb(skb
, pt_prev
, orig_dev
);
3824 list_for_each_entry_rcu(ptype
, &skb
->dev
->ptype_all
, list
) {
3826 ret
= deliver_skb(skb
, pt_prev
, orig_dev
);
3831 #ifdef CONFIG_NET_INGRESS
3832 if (static_key_false(&ingress_needed
)) {
3833 skb
= handle_ing(skb
, &pt_prev
, &ret
, orig_dev
);
3837 if (nf_ingress(skb
, &pt_prev
, &ret
, orig_dev
) < 0)
3841 #ifdef CONFIG_NET_CLS_ACT
3845 if (pfmemalloc
&& !skb_pfmemalloc_protocol(skb
))
3848 if (skb_vlan_tag_present(skb
)) {
3850 ret
= deliver_skb(skb
, pt_prev
, orig_dev
);
3853 if (vlan_do_receive(&skb
))
3855 else if (unlikely(!skb
))
3859 rx_handler
= rcu_dereference(skb
->dev
->rx_handler
);
3862 ret
= deliver_skb(skb
, pt_prev
, orig_dev
);
3865 switch (rx_handler(&skb
)) {
3866 case RX_HANDLER_CONSUMED
:
3867 ret
= NET_RX_SUCCESS
;
3869 case RX_HANDLER_ANOTHER
:
3871 case RX_HANDLER_EXACT
:
3872 deliver_exact
= true;
3873 case RX_HANDLER_PASS
:
3880 if (unlikely(skb_vlan_tag_present(skb
))) {
3881 if (skb_vlan_tag_get_id(skb
))
3882 skb
->pkt_type
= PACKET_OTHERHOST
;
3883 /* Note: we might in the future use prio bits
3884 * and set skb->priority like in vlan_do_receive()
3885 * For the time being, just ignore Priority Code Point
3890 type
= skb
->protocol
;
3892 /* deliver only exact match when indicated */
3893 if (likely(!deliver_exact
)) {
3894 deliver_ptype_list_skb(skb
, &pt_prev
, orig_dev
, type
,
3895 &ptype_base
[ntohs(type
) &
3899 deliver_ptype_list_skb(skb
, &pt_prev
, orig_dev
, type
,
3900 &orig_dev
->ptype_specific
);
3902 if (unlikely(skb
->dev
!= orig_dev
)) {
3903 deliver_ptype_list_skb(skb
, &pt_prev
, orig_dev
, type
,
3904 &skb
->dev
->ptype_specific
);
3908 if (unlikely(skb_orphan_frags(skb
, GFP_ATOMIC
)))
3911 ret
= pt_prev
->func(skb
, skb
->dev
, pt_prev
, orig_dev
);
3914 atomic_long_inc(&skb
->dev
->rx_dropped
);
3916 /* Jamal, now you will not able to escape explaining
3917 * me how you were going to use this. :-)
3927 static int __netif_receive_skb(struct sk_buff
*skb
)
3931 if (sk_memalloc_socks() && skb_pfmemalloc(skb
)) {
3932 unsigned long pflags
= current
->flags
;
3935 * PFMEMALLOC skbs are special, they should
3936 * - be delivered to SOCK_MEMALLOC sockets only
3937 * - stay away from userspace
3938 * - have bounded memory usage
3940 * Use PF_MEMALLOC as this saves us from propagating the allocation
3941 * context down to all allocation sites.
3943 current
->flags
|= PF_MEMALLOC
;
3944 ret
= __netif_receive_skb_core(skb
, true);
3945 tsk_restore_flags(current
, pflags
, PF_MEMALLOC
);
3947 ret
= __netif_receive_skb_core(skb
, false);
3952 static int netif_receive_skb_internal(struct sk_buff
*skb
)
3954 net_timestamp_check(netdev_tstamp_prequeue
, skb
);
3956 if (skb_defer_rx_timestamp(skb
))
3957 return NET_RX_SUCCESS
;
3960 if (static_key_false(&rps_needed
)) {
3961 struct rps_dev_flow voidflow
, *rflow
= &voidflow
;
3966 cpu
= get_rps_cpu(skb
->dev
, skb
, &rflow
);
3969 ret
= enqueue_to_backlog(skb
, cpu
, &rflow
->last_qtail
);
3976 return __netif_receive_skb(skb
);
3980 * netif_receive_skb - process receive buffer from network
3981 * @skb: buffer to process
3983 * netif_receive_skb() is the main receive data processing function.
3984 * It always succeeds. The buffer may be dropped during processing
3985 * for congestion control or by the protocol layers.
3987 * This function may only be called from softirq context and interrupts
3988 * should be enabled.
3990 * Return values (usually ignored):
3991 * NET_RX_SUCCESS: no congestion
3992 * NET_RX_DROP: packet was dropped
3994 int netif_receive_skb_sk(struct sock
*sk
, struct sk_buff
*skb
)
3996 trace_netif_receive_skb_entry(skb
);
3998 return netif_receive_skb_internal(skb
);
4000 EXPORT_SYMBOL(netif_receive_skb_sk
);
4002 /* Network device is going away, flush any packets still pending
4003 * Called with irqs disabled.
4005 static void flush_backlog(void *arg
)
4007 struct net_device
*dev
= arg
;
4008 struct softnet_data
*sd
= this_cpu_ptr(&softnet_data
);
4009 struct sk_buff
*skb
, *tmp
;
4012 skb_queue_walk_safe(&sd
->input_pkt_queue
, skb
, tmp
) {
4013 if (skb
->dev
== dev
) {
4014 __skb_unlink(skb
, &sd
->input_pkt_queue
);
4016 input_queue_head_incr(sd
);
4021 skb_queue_walk_safe(&sd
->process_queue
, skb
, tmp
) {
4022 if (skb
->dev
== dev
) {
4023 __skb_unlink(skb
, &sd
->process_queue
);
4025 input_queue_head_incr(sd
);
4030 static int napi_gro_complete(struct sk_buff
*skb
)
4032 struct packet_offload
*ptype
;
4033 __be16 type
= skb
->protocol
;
4034 struct list_head
*head
= &offload_base
;
4037 BUILD_BUG_ON(sizeof(struct napi_gro_cb
) > sizeof(skb
->cb
));
4039 if (NAPI_GRO_CB(skb
)->count
== 1) {
4040 skb_shinfo(skb
)->gso_size
= 0;
4045 list_for_each_entry_rcu(ptype
, head
, list
) {
4046 if (ptype
->type
!= type
|| !ptype
->callbacks
.gro_complete
)
4049 err
= ptype
->callbacks
.gro_complete(skb
, 0);
4055 WARN_ON(&ptype
->list
== head
);
4057 return NET_RX_SUCCESS
;
4061 return netif_receive_skb_internal(skb
);
4064 /* napi->gro_list contains packets ordered by age.
4065 * youngest packets at the head of it.
4066 * Complete skbs in reverse order to reduce latencies.
4068 void napi_gro_flush(struct napi_struct
*napi
, bool flush_old
)
4070 struct sk_buff
*skb
, *prev
= NULL
;
4072 /* scan list and build reverse chain */
4073 for (skb
= napi
->gro_list
; skb
!= NULL
; skb
= skb
->next
) {
4078 for (skb
= prev
; skb
; skb
= prev
) {
4081 if (flush_old
&& NAPI_GRO_CB(skb
)->age
== jiffies
)
4085 napi_gro_complete(skb
);
4089 napi
->gro_list
= NULL
;
4091 EXPORT_SYMBOL(napi_gro_flush
);
4093 static void gro_list_prepare(struct napi_struct
*napi
, struct sk_buff
*skb
)
4096 unsigned int maclen
= skb
->dev
->hard_header_len
;
4097 u32 hash
= skb_get_hash_raw(skb
);
4099 for (p
= napi
->gro_list
; p
; p
= p
->next
) {
4100 unsigned long diffs
;
4102 NAPI_GRO_CB(p
)->flush
= 0;
4104 if (hash
!= skb_get_hash_raw(p
)) {
4105 NAPI_GRO_CB(p
)->same_flow
= 0;
4109 diffs
= (unsigned long)p
->dev
^ (unsigned long)skb
->dev
;
4110 diffs
|= p
->vlan_tci
^ skb
->vlan_tci
;
4111 if (maclen
== ETH_HLEN
)
4112 diffs
|= compare_ether_header(skb_mac_header(p
),
4113 skb_mac_header(skb
));
4115 diffs
= memcmp(skb_mac_header(p
),
4116 skb_mac_header(skb
),
4118 NAPI_GRO_CB(p
)->same_flow
= !diffs
;
4122 static void skb_gro_reset_offset(struct sk_buff
*skb
)
4124 const struct skb_shared_info
*pinfo
= skb_shinfo(skb
);
4125 const skb_frag_t
*frag0
= &pinfo
->frags
[0];
4127 NAPI_GRO_CB(skb
)->data_offset
= 0;
4128 NAPI_GRO_CB(skb
)->frag0
= NULL
;
4129 NAPI_GRO_CB(skb
)->frag0_len
= 0;
4131 if (skb_mac_header(skb
) == skb_tail_pointer(skb
) &&
4133 !PageHighMem(skb_frag_page(frag0
))) {
4134 NAPI_GRO_CB(skb
)->frag0
= skb_frag_address(frag0
);
4135 NAPI_GRO_CB(skb
)->frag0_len
= skb_frag_size(frag0
);
4139 static void gro_pull_from_frag0(struct sk_buff
*skb
, int grow
)
4141 struct skb_shared_info
*pinfo
= skb_shinfo(skb
);
4143 BUG_ON(skb
->end
- skb
->tail
< grow
);
4145 memcpy(skb_tail_pointer(skb
), NAPI_GRO_CB(skb
)->frag0
, grow
);
4147 skb
->data_len
-= grow
;
4150 pinfo
->frags
[0].page_offset
+= grow
;
4151 skb_frag_size_sub(&pinfo
->frags
[0], grow
);
4153 if (unlikely(!skb_frag_size(&pinfo
->frags
[0]))) {
4154 skb_frag_unref(skb
, 0);
4155 memmove(pinfo
->frags
, pinfo
->frags
+ 1,
4156 --pinfo
->nr_frags
* sizeof(pinfo
->frags
[0]));
4160 static enum gro_result
dev_gro_receive(struct napi_struct
*napi
, struct sk_buff
*skb
)
4162 struct sk_buff
**pp
= NULL
;
4163 struct packet_offload
*ptype
;
4164 __be16 type
= skb
->protocol
;
4165 struct list_head
*head
= &offload_base
;
4167 enum gro_result ret
;
4170 if (!(skb
->dev
->features
& NETIF_F_GRO
))
4173 if (skb_is_gso(skb
) || skb_has_frag_list(skb
) || skb
->csum_bad
)
4176 gro_list_prepare(napi
, skb
);
4179 list_for_each_entry_rcu(ptype
, head
, list
) {
4180 if (ptype
->type
!= type
|| !ptype
->callbacks
.gro_receive
)
4183 skb_set_network_header(skb
, skb_gro_offset(skb
));
4184 skb_reset_mac_len(skb
);
4185 NAPI_GRO_CB(skb
)->same_flow
= 0;
4186 NAPI_GRO_CB(skb
)->flush
= 0;
4187 NAPI_GRO_CB(skb
)->free
= 0;
4188 NAPI_GRO_CB(skb
)->udp_mark
= 0;
4189 NAPI_GRO_CB(skb
)->gro_remcsum_start
= 0;
4191 /* Setup for GRO checksum validation */
4192 switch (skb
->ip_summed
) {
4193 case CHECKSUM_COMPLETE
:
4194 NAPI_GRO_CB(skb
)->csum
= skb
->csum
;
4195 NAPI_GRO_CB(skb
)->csum_valid
= 1;
4196 NAPI_GRO_CB(skb
)->csum_cnt
= 0;
4198 case CHECKSUM_UNNECESSARY
:
4199 NAPI_GRO_CB(skb
)->csum_cnt
= skb
->csum_level
+ 1;
4200 NAPI_GRO_CB(skb
)->csum_valid
= 0;
4203 NAPI_GRO_CB(skb
)->csum_cnt
= 0;
4204 NAPI_GRO_CB(skb
)->csum_valid
= 0;
4207 pp
= ptype
->callbacks
.gro_receive(&napi
->gro_list
, skb
);
4212 if (&ptype
->list
== head
)
4215 same_flow
= NAPI_GRO_CB(skb
)->same_flow
;
4216 ret
= NAPI_GRO_CB(skb
)->free
? GRO_MERGED_FREE
: GRO_MERGED
;
4219 struct sk_buff
*nskb
= *pp
;
4223 napi_gro_complete(nskb
);
4230 if (NAPI_GRO_CB(skb
)->flush
)
4233 if (unlikely(napi
->gro_count
>= MAX_GRO_SKBS
)) {
4234 struct sk_buff
*nskb
= napi
->gro_list
;
4236 /* locate the end of the list to select the 'oldest' flow */
4237 while (nskb
->next
) {
4243 napi_gro_complete(nskb
);
4247 NAPI_GRO_CB(skb
)->count
= 1;
4248 NAPI_GRO_CB(skb
)->age
= jiffies
;
4249 NAPI_GRO_CB(skb
)->last
= skb
;
4250 skb_shinfo(skb
)->gso_size
= skb_gro_len(skb
);
4251 skb
->next
= napi
->gro_list
;
4252 napi
->gro_list
= skb
;
4256 grow
= skb_gro_offset(skb
) - skb_headlen(skb
);
4258 gro_pull_from_frag0(skb
, grow
);
4267 struct packet_offload
*gro_find_receive_by_type(__be16 type
)
4269 struct list_head
*offload_head
= &offload_base
;
4270 struct packet_offload
*ptype
;
4272 list_for_each_entry_rcu(ptype
, offload_head
, list
) {
4273 if (ptype
->type
!= type
|| !ptype
->callbacks
.gro_receive
)
4279 EXPORT_SYMBOL(gro_find_receive_by_type
);
4281 struct packet_offload
*gro_find_complete_by_type(__be16 type
)
4283 struct list_head
*offload_head
= &offload_base
;
4284 struct packet_offload
*ptype
;
4286 list_for_each_entry_rcu(ptype
, offload_head
, list
) {
4287 if (ptype
->type
!= type
|| !ptype
->callbacks
.gro_complete
)
4293 EXPORT_SYMBOL(gro_find_complete_by_type
);
4295 static gro_result_t
napi_skb_finish(gro_result_t ret
, struct sk_buff
*skb
)
4299 if (netif_receive_skb_internal(skb
))
4307 case GRO_MERGED_FREE
:
4308 if (NAPI_GRO_CB(skb
)->free
== NAPI_GRO_FREE_STOLEN_HEAD
)
4309 kmem_cache_free(skbuff_head_cache
, skb
);
4322 gro_result_t
napi_gro_receive(struct napi_struct
*napi
, struct sk_buff
*skb
)
4324 trace_napi_gro_receive_entry(skb
);
4326 skb_gro_reset_offset(skb
);
4328 return napi_skb_finish(dev_gro_receive(napi
, skb
), skb
);
4330 EXPORT_SYMBOL(napi_gro_receive
);
4332 static void napi_reuse_skb(struct napi_struct
*napi
, struct sk_buff
*skb
)
4334 if (unlikely(skb
->pfmemalloc
)) {
4338 __skb_pull(skb
, skb_headlen(skb
));
4339 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4340 skb_reserve(skb
, NET_SKB_PAD
+ NET_IP_ALIGN
- skb_headroom(skb
));
4342 skb
->dev
= napi
->dev
;
4344 skb
->encapsulation
= 0;
4345 skb_shinfo(skb
)->gso_type
= 0;
4346 skb
->truesize
= SKB_TRUESIZE(skb_end_offset(skb
));
4351 struct sk_buff
*napi_get_frags(struct napi_struct
*napi
)
4353 struct sk_buff
*skb
= napi
->skb
;
4356 skb
= napi_alloc_skb(napi
, GRO_MAX_HEAD
);
4361 EXPORT_SYMBOL(napi_get_frags
);
4363 static gro_result_t
napi_frags_finish(struct napi_struct
*napi
,
4364 struct sk_buff
*skb
,
4370 __skb_push(skb
, ETH_HLEN
);
4371 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
4372 if (ret
== GRO_NORMAL
&& netif_receive_skb_internal(skb
))
4377 case GRO_MERGED_FREE
:
4378 napi_reuse_skb(napi
, skb
);
4388 /* Upper GRO stack assumes network header starts at gro_offset=0
4389 * Drivers could call both napi_gro_frags() and napi_gro_receive()
4390 * We copy ethernet header into skb->data to have a common layout.
4392 static struct sk_buff
*napi_frags_skb(struct napi_struct
*napi
)
4394 struct sk_buff
*skb
= napi
->skb
;
4395 const struct ethhdr
*eth
;
4396 unsigned int hlen
= sizeof(*eth
);
4400 skb_reset_mac_header(skb
);
4401 skb_gro_reset_offset(skb
);
4403 eth
= skb_gro_header_fast(skb
, 0);
4404 if (unlikely(skb_gro_header_hard(skb
, hlen
))) {
4405 eth
= skb_gro_header_slow(skb
, hlen
, 0);
4406 if (unlikely(!eth
)) {
4407 napi_reuse_skb(napi
, skb
);
4411 gro_pull_from_frag0(skb
, hlen
);
4412 NAPI_GRO_CB(skb
)->frag0
+= hlen
;
4413 NAPI_GRO_CB(skb
)->frag0_len
-= hlen
;
4415 __skb_pull(skb
, hlen
);
4418 * This works because the only protocols we care about don't require
4420 * We'll fix it up properly in napi_frags_finish()
4422 skb
->protocol
= eth
->h_proto
;
4427 gro_result_t
napi_gro_frags(struct napi_struct
*napi
)
4429 struct sk_buff
*skb
= napi_frags_skb(napi
);
4434 trace_napi_gro_frags_entry(skb
);
4436 return napi_frags_finish(napi
, skb
, dev_gro_receive(napi
, skb
));
4438 EXPORT_SYMBOL(napi_gro_frags
);
4440 /* Compute the checksum from gro_offset and return the folded value
4441 * after adding in any pseudo checksum.
4443 __sum16
__skb_gro_checksum_complete(struct sk_buff
*skb
)
4448 wsum
= skb_checksum(skb
, skb_gro_offset(skb
), skb_gro_len(skb
), 0);
4450 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4451 sum
= csum_fold(csum_add(NAPI_GRO_CB(skb
)->csum
, wsum
));
4453 if (unlikely(skb
->ip_summed
== CHECKSUM_COMPLETE
) &&
4454 !skb
->csum_complete_sw
)
4455 netdev_rx_csum_fault(skb
->dev
);
4458 NAPI_GRO_CB(skb
)->csum
= wsum
;
4459 NAPI_GRO_CB(skb
)->csum_valid
= 1;
4463 EXPORT_SYMBOL(__skb_gro_checksum_complete
);
4466 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
4467 * Note: called with local irq disabled, but exits with local irq enabled.
4469 static void net_rps_action_and_irq_enable(struct softnet_data
*sd
)
4472 struct softnet_data
*remsd
= sd
->rps_ipi_list
;
4475 sd
->rps_ipi_list
= NULL
;
4479 /* Send pending IPI's to kick RPS processing on remote cpus. */
4481 struct softnet_data
*next
= remsd
->rps_ipi_next
;
4483 if (cpu_online(remsd
->cpu
))
4484 smp_call_function_single_async(remsd
->cpu
,
4493 static bool sd_has_rps_ipi_waiting(struct softnet_data
*sd
)
4496 return sd
->rps_ipi_list
!= NULL
;
4502 static int process_backlog(struct napi_struct
*napi
, int quota
)
4505 struct softnet_data
*sd
= container_of(napi
, struct softnet_data
, backlog
);
4507 /* Check if we have pending ipi, its better to send them now,
4508 * not waiting net_rx_action() end.
4510 if (sd_has_rps_ipi_waiting(sd
)) {
4511 local_irq_disable();
4512 net_rps_action_and_irq_enable(sd
);
4515 napi
->weight
= weight_p
;
4516 local_irq_disable();
4518 struct sk_buff
*skb
;
4520 while ((skb
= __skb_dequeue(&sd
->process_queue
))) {
4522 __netif_receive_skb(skb
);
4523 local_irq_disable();
4524 input_queue_head_incr(sd
);
4525 if (++work
>= quota
) {
4532 if (skb_queue_empty(&sd
->input_pkt_queue
)) {
4534 * Inline a custom version of __napi_complete().
4535 * only current cpu owns and manipulates this napi,
4536 * and NAPI_STATE_SCHED is the only possible flag set
4538 * We can use a plain write instead of clear_bit(),
4539 * and we dont need an smp_mb() memory barrier.
4547 skb_queue_splice_tail_init(&sd
->input_pkt_queue
,
4548 &sd
->process_queue
);
4557 * __napi_schedule - schedule for receive
4558 * @n: entry to schedule
4560 * The entry's receive function will be scheduled to run.
4561 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
4563 void __napi_schedule(struct napi_struct
*n
)
4565 unsigned long flags
;
4567 local_irq_save(flags
);
4568 ____napi_schedule(this_cpu_ptr(&softnet_data
), n
);
4569 local_irq_restore(flags
);
4571 EXPORT_SYMBOL(__napi_schedule
);
4574 * __napi_schedule_irqoff - schedule for receive
4575 * @n: entry to schedule
4577 * Variant of __napi_schedule() assuming hard irqs are masked
4579 void __napi_schedule_irqoff(struct napi_struct
*n
)
4581 ____napi_schedule(this_cpu_ptr(&softnet_data
), n
);
4583 EXPORT_SYMBOL(__napi_schedule_irqoff
);
4585 void __napi_complete(struct napi_struct
*n
)
4587 BUG_ON(!test_bit(NAPI_STATE_SCHED
, &n
->state
));
4589 list_del_init(&n
->poll_list
);
4590 smp_mb__before_atomic();
4591 clear_bit(NAPI_STATE_SCHED
, &n
->state
);
4593 EXPORT_SYMBOL(__napi_complete
);
4595 void napi_complete_done(struct napi_struct
*n
, int work_done
)
4597 unsigned long flags
;
4600 * don't let napi dequeue from the cpu poll list
4601 * just in case its running on a different cpu
4603 if (unlikely(test_bit(NAPI_STATE_NPSVC
, &n
->state
)))
4607 unsigned long timeout
= 0;
4610 timeout
= n
->dev
->gro_flush_timeout
;
4613 hrtimer_start(&n
->timer
, ns_to_ktime(timeout
),
4614 HRTIMER_MODE_REL_PINNED
);
4616 napi_gro_flush(n
, false);
4618 if (likely(list_empty(&n
->poll_list
))) {
4619 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED
, &n
->state
));
4621 /* If n->poll_list is not empty, we need to mask irqs */
4622 local_irq_save(flags
);
4624 local_irq_restore(flags
);
4627 EXPORT_SYMBOL(napi_complete_done
);
4629 /* must be called under rcu_read_lock(), as we dont take a reference */
4630 struct napi_struct
*napi_by_id(unsigned int napi_id
)
4632 unsigned int hash
= napi_id
% HASH_SIZE(napi_hash
);
4633 struct napi_struct
*napi
;
4635 hlist_for_each_entry_rcu(napi
, &napi_hash
[hash
], napi_hash_node
)
4636 if (napi
->napi_id
== napi_id
)
4641 EXPORT_SYMBOL_GPL(napi_by_id
);
4643 void napi_hash_add(struct napi_struct
*napi
)
4645 if (!test_and_set_bit(NAPI_STATE_HASHED
, &napi
->state
)) {
4647 spin_lock(&napi_hash_lock
);
4649 /* 0 is not a valid id, we also skip an id that is taken
4650 * we expect both events to be extremely rare
4653 while (!napi
->napi_id
) {
4654 napi
->napi_id
= ++napi_gen_id
;
4655 if (napi_by_id(napi
->napi_id
))
4659 hlist_add_head_rcu(&napi
->napi_hash_node
,
4660 &napi_hash
[napi
->napi_id
% HASH_SIZE(napi_hash
)]);
4662 spin_unlock(&napi_hash_lock
);
4665 EXPORT_SYMBOL_GPL(napi_hash_add
);
4667 /* Warning : caller is responsible to make sure rcu grace period
4668 * is respected before freeing memory containing @napi
4670 void napi_hash_del(struct napi_struct
*napi
)
4672 spin_lock(&napi_hash_lock
);
4674 if (test_and_clear_bit(NAPI_STATE_HASHED
, &napi
->state
))
4675 hlist_del_rcu(&napi
->napi_hash_node
);
4677 spin_unlock(&napi_hash_lock
);
4679 EXPORT_SYMBOL_GPL(napi_hash_del
);
4681 static enum hrtimer_restart
napi_watchdog(struct hrtimer
*timer
)
4683 struct napi_struct
*napi
;
4685 napi
= container_of(timer
, struct napi_struct
, timer
);
4687 napi_schedule(napi
);
4689 return HRTIMER_NORESTART
;
4692 void netif_napi_add(struct net_device
*dev
, struct napi_struct
*napi
,
4693 int (*poll
)(struct napi_struct
*, int), int weight
)
4695 INIT_LIST_HEAD(&napi
->poll_list
);
4696 hrtimer_init(&napi
->timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_REL_PINNED
);
4697 napi
->timer
.function
= napi_watchdog
;
4698 napi
->gro_count
= 0;
4699 napi
->gro_list
= NULL
;
4702 if (weight
> NAPI_POLL_WEIGHT
)
4703 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
4705 napi
->weight
= weight
;
4706 list_add(&napi
->dev_list
, &dev
->napi_list
);
4708 #ifdef CONFIG_NETPOLL
4709 spin_lock_init(&napi
->poll_lock
);
4710 napi
->poll_owner
= -1;
4712 set_bit(NAPI_STATE_SCHED
, &napi
->state
);
4714 EXPORT_SYMBOL(netif_napi_add
);
4716 void napi_disable(struct napi_struct
*n
)
4719 set_bit(NAPI_STATE_DISABLE
, &n
->state
);
4721 while (test_and_set_bit(NAPI_STATE_SCHED
, &n
->state
))
4724 hrtimer_cancel(&n
->timer
);
4726 clear_bit(NAPI_STATE_DISABLE
, &n
->state
);
4728 EXPORT_SYMBOL(napi_disable
);
4730 void netif_napi_del(struct napi_struct
*napi
)
4732 list_del_init(&napi
->dev_list
);
4733 napi_free_frags(napi
);
4735 kfree_skb_list(napi
->gro_list
);
4736 napi
->gro_list
= NULL
;
4737 napi
->gro_count
= 0;
4739 EXPORT_SYMBOL(netif_napi_del
);
4741 static int napi_poll(struct napi_struct
*n
, struct list_head
*repoll
)
4746 list_del_init(&n
->poll_list
);
4748 have
= netpoll_poll_lock(n
);
4752 /* This NAPI_STATE_SCHED test is for avoiding a race
4753 * with netpoll's poll_napi(). Only the entity which
4754 * obtains the lock and sees NAPI_STATE_SCHED set will
4755 * actually make the ->poll() call. Therefore we avoid
4756 * accidentally calling ->poll() when NAPI is not scheduled.
4759 if (test_bit(NAPI_STATE_SCHED
, &n
->state
)) {
4760 work
= n
->poll(n
, weight
);
4764 WARN_ON_ONCE(work
> weight
);
4766 if (likely(work
< weight
))
4769 /* Drivers must not modify the NAPI state if they
4770 * consume the entire weight. In such cases this code
4771 * still "owns" the NAPI instance and therefore can
4772 * move the instance around on the list at-will.
4774 if (unlikely(napi_disable_pending(n
))) {
4780 /* flush too old packets
4781 * If HZ < 1000, flush all packets.
4783 napi_gro_flush(n
, HZ
>= 1000);
4786 /* Some drivers may have called napi_schedule
4787 * prior to exhausting their budget.
4789 if (unlikely(!list_empty(&n
->poll_list
))) {
4790 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
4791 n
->dev
? n
->dev
->name
: "backlog");
4795 list_add_tail(&n
->poll_list
, repoll
);
4798 netpoll_poll_unlock(have
);
4803 static void net_rx_action(struct softirq_action
*h
)
4805 struct softnet_data
*sd
= this_cpu_ptr(&softnet_data
);
4806 unsigned long time_limit
= jiffies
+ 2;
4807 int budget
= netdev_budget
;
4811 local_irq_disable();
4812 list_splice_init(&sd
->poll_list
, &list
);
4816 struct napi_struct
*n
;
4818 if (list_empty(&list
)) {
4819 if (!sd_has_rps_ipi_waiting(sd
) && list_empty(&repoll
))
4824 n
= list_first_entry(&list
, struct napi_struct
, poll_list
);
4825 budget
-= napi_poll(n
, &repoll
);
4827 /* If softirq window is exhausted then punt.
4828 * Allow this to run for 2 jiffies since which will allow
4829 * an average latency of 1.5/HZ.
4831 if (unlikely(budget
<= 0 ||
4832 time_after_eq(jiffies
, time_limit
))) {
4838 local_irq_disable();
4840 list_splice_tail_init(&sd
->poll_list
, &list
);
4841 list_splice_tail(&repoll
, &list
);
4842 list_splice(&list
, &sd
->poll_list
);
4843 if (!list_empty(&sd
->poll_list
))
4844 __raise_softirq_irqoff(NET_RX_SOFTIRQ
);
4846 net_rps_action_and_irq_enable(sd
);
4849 struct netdev_adjacent
{
4850 struct net_device
*dev
;
4852 /* upper master flag, there can only be one master device per list */
4855 /* counter for the number of times this device was added to us */
4858 /* private field for the users */
4861 struct list_head list
;
4862 struct rcu_head rcu
;
4865 static struct netdev_adjacent
*__netdev_find_adj(struct net_device
*dev
,
4866 struct net_device
*adj_dev
,
4867 struct list_head
*adj_list
)
4869 struct netdev_adjacent
*adj
;
4871 list_for_each_entry(adj
, adj_list
, list
) {
4872 if (adj
->dev
== adj_dev
)
4879 * netdev_has_upper_dev - Check if device is linked to an upper device
4881 * @upper_dev: upper device to check
4883 * Find out if a device is linked to specified upper device and return true
4884 * in case it is. Note that this checks only immediate upper device,
4885 * not through a complete stack of devices. The caller must hold the RTNL lock.
4887 bool netdev_has_upper_dev(struct net_device
*dev
,
4888 struct net_device
*upper_dev
)
4892 return __netdev_find_adj(dev
, upper_dev
, &dev
->all_adj_list
.upper
);
4894 EXPORT_SYMBOL(netdev_has_upper_dev
);
4897 * netdev_has_any_upper_dev - Check if device is linked to some device
4900 * Find out if a device is linked to an upper device and return true in case
4901 * it is. The caller must hold the RTNL lock.
4903 static bool netdev_has_any_upper_dev(struct net_device
*dev
)
4907 return !list_empty(&dev
->all_adj_list
.upper
);
4911 * netdev_master_upper_dev_get - Get master upper device
4914 * Find a master upper device and return pointer to it or NULL in case
4915 * it's not there. The caller must hold the RTNL lock.
4917 struct net_device
*netdev_master_upper_dev_get(struct net_device
*dev
)
4919 struct netdev_adjacent
*upper
;
4923 if (list_empty(&dev
->adj_list
.upper
))
4926 upper
= list_first_entry(&dev
->adj_list
.upper
,
4927 struct netdev_adjacent
, list
);
4928 if (likely(upper
->master
))
4932 EXPORT_SYMBOL(netdev_master_upper_dev_get
);
4934 void *netdev_adjacent_get_private(struct list_head
*adj_list
)
4936 struct netdev_adjacent
*adj
;
4938 adj
= list_entry(adj_list
, struct netdev_adjacent
, list
);
4940 return adj
->private;
4942 EXPORT_SYMBOL(netdev_adjacent_get_private
);
4945 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
4947 * @iter: list_head ** of the current position
4949 * Gets the next device from the dev's upper list, starting from iter
4950 * position. The caller must hold RCU read lock.
4952 struct net_device
*netdev_upper_get_next_dev_rcu(struct net_device
*dev
,
4953 struct list_head
**iter
)
4955 struct netdev_adjacent
*upper
;
4957 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
4959 upper
= list_entry_rcu((*iter
)->next
, struct netdev_adjacent
, list
);
4961 if (&upper
->list
== &dev
->adj_list
.upper
)
4964 *iter
= &upper
->list
;
4968 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu
);
4971 * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
4973 * @iter: list_head ** of the current position
4975 * Gets the next device from the dev's upper list, starting from iter
4976 * position. The caller must hold RCU read lock.
4978 struct net_device
*netdev_all_upper_get_next_dev_rcu(struct net_device
*dev
,
4979 struct list_head
**iter
)
4981 struct netdev_adjacent
*upper
;
4983 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
4985 upper
= list_entry_rcu((*iter
)->next
, struct netdev_adjacent
, list
);
4987 if (&upper
->list
== &dev
->all_adj_list
.upper
)
4990 *iter
= &upper
->list
;
4994 EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu
);
4997 * netdev_lower_get_next_private - Get the next ->private from the
4998 * lower neighbour list
5000 * @iter: list_head ** of the current position
5002 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5003 * list, starting from iter position. The caller must hold either hold the
5004 * RTNL lock or its own locking that guarantees that the neighbour lower
5005 * list will remain unchainged.
5007 void *netdev_lower_get_next_private(struct net_device
*dev
,
5008 struct list_head
**iter
)
5010 struct netdev_adjacent
*lower
;
5012 lower
= list_entry(*iter
, struct netdev_adjacent
, list
);
5014 if (&lower
->list
== &dev
->adj_list
.lower
)
5017 *iter
= lower
->list
.next
;
5019 return lower
->private;
5021 EXPORT_SYMBOL(netdev_lower_get_next_private
);
5024 * netdev_lower_get_next_private_rcu - Get the next ->private from the
5025 * lower neighbour list, RCU
5028 * @iter: list_head ** of the current position
5030 * Gets the next netdev_adjacent->private from the dev's lower neighbour
5031 * list, starting from iter position. The caller must hold RCU read lock.
5033 void *netdev_lower_get_next_private_rcu(struct net_device
*dev
,
5034 struct list_head
**iter
)
5036 struct netdev_adjacent
*lower
;
5038 WARN_ON_ONCE(!rcu_read_lock_held());
5040 lower
= list_entry_rcu((*iter
)->next
, struct netdev_adjacent
, list
);
5042 if (&lower
->list
== &dev
->adj_list
.lower
)
5045 *iter
= &lower
->list
;
5047 return lower
->private;
5049 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu
);
5052 * netdev_lower_get_next - Get the next device from the lower neighbour
5055 * @iter: list_head ** of the current position
5057 * Gets the next netdev_adjacent from the dev's lower neighbour
5058 * list, starting from iter position. The caller must hold RTNL lock or
5059 * its own locking that guarantees that the neighbour lower
5060 * list will remain unchainged.
5062 void *netdev_lower_get_next(struct net_device
*dev
, struct list_head
**iter
)
5064 struct netdev_adjacent
*lower
;
5066 lower
= list_entry((*iter
)->next
, struct netdev_adjacent
, list
);
5068 if (&lower
->list
== &dev
->adj_list
.lower
)
5071 *iter
= &lower
->list
;
5075 EXPORT_SYMBOL(netdev_lower_get_next
);
5078 * netdev_lower_get_first_private_rcu - Get the first ->private from the
5079 * lower neighbour list, RCU
5083 * Gets the first netdev_adjacent->private from the dev's lower neighbour
5084 * list. The caller must hold RCU read lock.
5086 void *netdev_lower_get_first_private_rcu(struct net_device
*dev
)
5088 struct netdev_adjacent
*lower
;
5090 lower
= list_first_or_null_rcu(&dev
->adj_list
.lower
,
5091 struct netdev_adjacent
, list
);
5093 return lower
->private;
5096 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu
);
5099 * netdev_master_upper_dev_get_rcu - Get master upper device
5102 * Find a master upper device and return pointer to it or NULL in case
5103 * it's not there. The caller must hold the RCU read lock.
5105 struct net_device
*netdev_master_upper_dev_get_rcu(struct net_device
*dev
)
5107 struct netdev_adjacent
*upper
;
5109 upper
= list_first_or_null_rcu(&dev
->adj_list
.upper
,
5110 struct netdev_adjacent
, list
);
5111 if (upper
&& likely(upper
->master
))
5115 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu
);
5117 static int netdev_adjacent_sysfs_add(struct net_device
*dev
,
5118 struct net_device
*adj_dev
,
5119 struct list_head
*dev_list
)
5121 char linkname
[IFNAMSIZ
+7];
5122 sprintf(linkname
, dev_list
== &dev
->adj_list
.upper
?
5123 "upper_%s" : "lower_%s", adj_dev
->name
);
5124 return sysfs_create_link(&(dev
->dev
.kobj
), &(adj_dev
->dev
.kobj
),
5127 static void netdev_adjacent_sysfs_del(struct net_device
*dev
,
5129 struct list_head
*dev_list
)
5131 char linkname
[IFNAMSIZ
+7];
5132 sprintf(linkname
, dev_list
== &dev
->adj_list
.upper
?
5133 "upper_%s" : "lower_%s", name
);
5134 sysfs_remove_link(&(dev
->dev
.kobj
), linkname
);
5137 static inline bool netdev_adjacent_is_neigh_list(struct net_device
*dev
,
5138 struct net_device
*adj_dev
,
5139 struct list_head
*dev_list
)
5141 return (dev_list
== &dev
->adj_list
.upper
||
5142 dev_list
== &dev
->adj_list
.lower
) &&
5143 net_eq(dev_net(dev
), dev_net(adj_dev
));
5146 static int __netdev_adjacent_dev_insert(struct net_device
*dev
,
5147 struct net_device
*adj_dev
,
5148 struct list_head
*dev_list
,
5149 void *private, bool master
)
5151 struct netdev_adjacent
*adj
;
5154 adj
= __netdev_find_adj(dev
, adj_dev
, dev_list
);
5161 adj
= kmalloc(sizeof(*adj
), GFP_KERNEL
);
5166 adj
->master
= master
;
5168 adj
->private = private;
5171 pr_debug("dev_hold for %s, because of link added from %s to %s\n",
5172 adj_dev
->name
, dev
->name
, adj_dev
->name
);
5174 if (netdev_adjacent_is_neigh_list(dev
, adj_dev
, dev_list
)) {
5175 ret
= netdev_adjacent_sysfs_add(dev
, adj_dev
, dev_list
);
5180 /* Ensure that master link is always the first item in list. */
5182 ret
= sysfs_create_link(&(dev
->dev
.kobj
),
5183 &(adj_dev
->dev
.kobj
), "master");
5185 goto remove_symlinks
;
5187 list_add_rcu(&adj
->list
, dev_list
);
5189 list_add_tail_rcu(&adj
->list
, dev_list
);
5195 if (netdev_adjacent_is_neigh_list(dev
, adj_dev
, dev_list
))
5196 netdev_adjacent_sysfs_del(dev
, adj_dev
->name
, dev_list
);
5204 static void __netdev_adjacent_dev_remove(struct net_device
*dev
,
5205 struct net_device
*adj_dev
,
5206 struct list_head
*dev_list
)
5208 struct netdev_adjacent
*adj
;
5210 adj
= __netdev_find_adj(dev
, adj_dev
, dev_list
);
5213 pr_err("tried to remove device %s from %s\n",
5214 dev
->name
, adj_dev
->name
);
5218 if (adj
->ref_nr
> 1) {
5219 pr_debug("%s to %s ref_nr-- = %d\n", dev
->name
, adj_dev
->name
,
5226 sysfs_remove_link(&(dev
->dev
.kobj
), "master");
5228 if (netdev_adjacent_is_neigh_list(dev
, adj_dev
, dev_list
))
5229 netdev_adjacent_sysfs_del(dev
, adj_dev
->name
, dev_list
);
5231 list_del_rcu(&adj
->list
);
5232 pr_debug("dev_put for %s, because link removed from %s to %s\n",
5233 adj_dev
->name
, dev
->name
, adj_dev
->name
);
5235 kfree_rcu(adj
, rcu
);
5238 static int __netdev_adjacent_dev_link_lists(struct net_device
*dev
,
5239 struct net_device
*upper_dev
,
5240 struct list_head
*up_list
,
5241 struct list_head
*down_list
,
5242 void *private, bool master
)
5246 ret
= __netdev_adjacent_dev_insert(dev
, upper_dev
, up_list
, private,
5251 ret
= __netdev_adjacent_dev_insert(upper_dev
, dev
, down_list
, private,
5254 __netdev_adjacent_dev_remove(dev
, upper_dev
, up_list
);
5261 static int __netdev_adjacent_dev_link(struct net_device
*dev
,
5262 struct net_device
*upper_dev
)
5264 return __netdev_adjacent_dev_link_lists(dev
, upper_dev
,
5265 &dev
->all_adj_list
.upper
,
5266 &upper_dev
->all_adj_list
.lower
,
5270 static void __netdev_adjacent_dev_unlink_lists(struct net_device
*dev
,
5271 struct net_device
*upper_dev
,
5272 struct list_head
*up_list
,
5273 struct list_head
*down_list
)
5275 __netdev_adjacent_dev_remove(dev
, upper_dev
, up_list
);
5276 __netdev_adjacent_dev_remove(upper_dev
, dev
, down_list
);
5279 static void __netdev_adjacent_dev_unlink(struct net_device
*dev
,
5280 struct net_device
*upper_dev
)
5282 __netdev_adjacent_dev_unlink_lists(dev
, upper_dev
,
5283 &dev
->all_adj_list
.upper
,
5284 &upper_dev
->all_adj_list
.lower
);
5287 static int __netdev_adjacent_dev_link_neighbour(struct net_device
*dev
,
5288 struct net_device
*upper_dev
,
5289 void *private, bool master
)
5291 int ret
= __netdev_adjacent_dev_link(dev
, upper_dev
);
5296 ret
= __netdev_adjacent_dev_link_lists(dev
, upper_dev
,
5297 &dev
->adj_list
.upper
,
5298 &upper_dev
->adj_list
.lower
,
5301 __netdev_adjacent_dev_unlink(dev
, upper_dev
);
5308 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device
*dev
,
5309 struct net_device
*upper_dev
)
5311 __netdev_adjacent_dev_unlink(dev
, upper_dev
);
5312 __netdev_adjacent_dev_unlink_lists(dev
, upper_dev
,
5313 &dev
->adj_list
.upper
,
5314 &upper_dev
->adj_list
.lower
);
5317 static int __netdev_upper_dev_link(struct net_device
*dev
,
5318 struct net_device
*upper_dev
, bool master
,
5321 struct netdev_adjacent
*i
, *j
, *to_i
, *to_j
;
5326 if (dev
== upper_dev
)
5329 /* To prevent loops, check if dev is not upper device to upper_dev. */
5330 if (__netdev_find_adj(upper_dev
, dev
, &upper_dev
->all_adj_list
.upper
))
5333 if (__netdev_find_adj(dev
, upper_dev
, &dev
->adj_list
.upper
))
5336 if (master
&& netdev_master_upper_dev_get(dev
))
5339 ret
= __netdev_adjacent_dev_link_neighbour(dev
, upper_dev
, private,
5344 /* Now that we linked these devs, make all the upper_dev's
5345 * all_adj_list.upper visible to every dev's all_adj_list.lower an
5346 * versa, and don't forget the devices itself. All of these
5347 * links are non-neighbours.
5349 list_for_each_entry(i
, &dev
->all_adj_list
.lower
, list
) {
5350 list_for_each_entry(j
, &upper_dev
->all_adj_list
.upper
, list
) {
5351 pr_debug("Interlinking %s with %s, non-neighbour\n",
5352 i
->dev
->name
, j
->dev
->name
);
5353 ret
= __netdev_adjacent_dev_link(i
->dev
, j
->dev
);
5359 /* add dev to every upper_dev's upper device */
5360 list_for_each_entry(i
, &upper_dev
->all_adj_list
.upper
, list
) {
5361 pr_debug("linking %s's upper device %s with %s\n",
5362 upper_dev
->name
, i
->dev
->name
, dev
->name
);
5363 ret
= __netdev_adjacent_dev_link(dev
, i
->dev
);
5365 goto rollback_upper_mesh
;
5368 /* add upper_dev to every dev's lower device */
5369 list_for_each_entry(i
, &dev
->all_adj_list
.lower
, list
) {
5370 pr_debug("linking %s's lower device %s with %s\n", dev
->name
,
5371 i
->dev
->name
, upper_dev
->name
);
5372 ret
= __netdev_adjacent_dev_link(i
->dev
, upper_dev
);
5374 goto rollback_lower_mesh
;
5377 call_netdevice_notifiers(NETDEV_CHANGEUPPER
, dev
);
5380 rollback_lower_mesh
:
5382 list_for_each_entry(i
, &dev
->all_adj_list
.lower
, list
) {
5385 __netdev_adjacent_dev_unlink(i
->dev
, upper_dev
);
5390 rollback_upper_mesh
:
5392 list_for_each_entry(i
, &upper_dev
->all_adj_list
.upper
, list
) {
5395 __netdev_adjacent_dev_unlink(dev
, i
->dev
);
5403 list_for_each_entry(i
, &dev
->all_adj_list
.lower
, list
) {
5404 list_for_each_entry(j
, &upper_dev
->all_adj_list
.upper
, list
) {
5405 if (i
== to_i
&& j
== to_j
)
5407 __netdev_adjacent_dev_unlink(i
->dev
, j
->dev
);
5413 __netdev_adjacent_dev_unlink_neighbour(dev
, upper_dev
);
5419 * netdev_upper_dev_link - Add a link to the upper device
5421 * @upper_dev: new upper device
5423 * Adds a link to device which is upper to this one. The caller must hold
5424 * the RTNL lock. On a failure a negative errno code is returned.
5425 * On success the reference counts are adjusted and the function
5428 int netdev_upper_dev_link(struct net_device
*dev
,
5429 struct net_device
*upper_dev
)
5431 return __netdev_upper_dev_link(dev
, upper_dev
, false, NULL
);
5433 EXPORT_SYMBOL(netdev_upper_dev_link
);
5436 * netdev_master_upper_dev_link - Add a master link to the upper device
5438 * @upper_dev: new upper device
5440 * Adds a link to device which is upper to this one. In this case, only
5441 * one master upper device can be linked, although other non-master devices
5442 * might be linked as well. The caller must hold the RTNL lock.
5443 * On a failure a negative errno code is returned. On success the reference
5444 * counts are adjusted and the function returns zero.
5446 int netdev_master_upper_dev_link(struct net_device
*dev
,
5447 struct net_device
*upper_dev
)
5449 return __netdev_upper_dev_link(dev
, upper_dev
, true, NULL
);
5451 EXPORT_SYMBOL(netdev_master_upper_dev_link
);
5453 int netdev_master_upper_dev_link_private(struct net_device
*dev
,
5454 struct net_device
*upper_dev
,
5457 return __netdev_upper_dev_link(dev
, upper_dev
, true, private);
5459 EXPORT_SYMBOL(netdev_master_upper_dev_link_private
);
5462 * netdev_upper_dev_unlink - Removes a link to upper device
5464 * @upper_dev: new upper device
5466 * Removes a link to device which is upper to this one. The caller must hold
5469 void netdev_upper_dev_unlink(struct net_device
*dev
,
5470 struct net_device
*upper_dev
)
5472 struct netdev_adjacent
*i
, *j
;
5475 __netdev_adjacent_dev_unlink_neighbour(dev
, upper_dev
);
5477 /* Here is the tricky part. We must remove all dev's lower
5478 * devices from all upper_dev's upper devices and vice
5479 * versa, to maintain the graph relationship.
5481 list_for_each_entry(i
, &dev
->all_adj_list
.lower
, list
)
5482 list_for_each_entry(j
, &upper_dev
->all_adj_list
.upper
, list
)
5483 __netdev_adjacent_dev_unlink(i
->dev
, j
->dev
);
5485 /* remove also the devices itself from lower/upper device
5488 list_for_each_entry(i
, &dev
->all_adj_list
.lower
, list
)
5489 __netdev_adjacent_dev_unlink(i
->dev
, upper_dev
);
5491 list_for_each_entry(i
, &upper_dev
->all_adj_list
.upper
, list
)
5492 __netdev_adjacent_dev_unlink(dev
, i
->dev
);
5494 call_netdevice_notifiers(NETDEV_CHANGEUPPER
, dev
);
5496 EXPORT_SYMBOL(netdev_upper_dev_unlink
);
5499 * netdev_bonding_info_change - Dispatch event about slave change
5501 * @bonding_info: info to dispatch
5503 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
5504 * The caller must hold the RTNL lock.
5506 void netdev_bonding_info_change(struct net_device
*dev
,
5507 struct netdev_bonding_info
*bonding_info
)
5509 struct netdev_notifier_bonding_info info
;
5511 memcpy(&info
.bonding_info
, bonding_info
,
5512 sizeof(struct netdev_bonding_info
));
5513 call_netdevice_notifiers_info(NETDEV_BONDING_INFO
, dev
,
5516 EXPORT_SYMBOL(netdev_bonding_info_change
);
5518 static void netdev_adjacent_add_links(struct net_device
*dev
)
5520 struct netdev_adjacent
*iter
;
5522 struct net
*net
= dev_net(dev
);
5524 list_for_each_entry(iter
, &dev
->adj_list
.upper
, list
) {
5525 if (!net_eq(net
,dev_net(iter
->dev
)))
5527 netdev_adjacent_sysfs_add(iter
->dev
, dev
,
5528 &iter
->dev
->adj_list
.lower
);
5529 netdev_adjacent_sysfs_add(dev
, iter
->dev
,
5530 &dev
->adj_list
.upper
);
5533 list_for_each_entry(iter
, &dev
->adj_list
.lower
, list
) {
5534 if (!net_eq(net
,dev_net(iter
->dev
)))
5536 netdev_adjacent_sysfs_add(iter
->dev
, dev
,
5537 &iter
->dev
->adj_list
.upper
);
5538 netdev_adjacent_sysfs_add(dev
, iter
->dev
,
5539 &dev
->adj_list
.lower
);
5543 static void netdev_adjacent_del_links(struct net_device
*dev
)
5545 struct netdev_adjacent
*iter
;
5547 struct net
*net
= dev_net(dev
);
5549 list_for_each_entry(iter
, &dev
->adj_list
.upper
, list
) {
5550 if (!net_eq(net
,dev_net(iter
->dev
)))
5552 netdev_adjacent_sysfs_del(iter
->dev
, dev
->name
,
5553 &iter
->dev
->adj_list
.lower
);
5554 netdev_adjacent_sysfs_del(dev
, iter
->dev
->name
,
5555 &dev
->adj_list
.upper
);
5558 list_for_each_entry(iter
, &dev
->adj_list
.lower
, list
) {
5559 if (!net_eq(net
,dev_net(iter
->dev
)))
5561 netdev_adjacent_sysfs_del(iter
->dev
, dev
->name
,
5562 &iter
->dev
->adj_list
.upper
);
5563 netdev_adjacent_sysfs_del(dev
, iter
->dev
->name
,
5564 &dev
->adj_list
.lower
);
5568 void netdev_adjacent_rename_links(struct net_device
*dev
, char *oldname
)
5570 struct netdev_adjacent
*iter
;
5572 struct net
*net
= dev_net(dev
);
5574 list_for_each_entry(iter
, &dev
->adj_list
.upper
, list
) {
5575 if (!net_eq(net
,dev_net(iter
->dev
)))
5577 netdev_adjacent_sysfs_del(iter
->dev
, oldname
,
5578 &iter
->dev
->adj_list
.lower
);
5579 netdev_adjacent_sysfs_add(iter
->dev
, dev
,
5580 &iter
->dev
->adj_list
.lower
);
5583 list_for_each_entry(iter
, &dev
->adj_list
.lower
, list
) {
5584 if (!net_eq(net
,dev_net(iter
->dev
)))
5586 netdev_adjacent_sysfs_del(iter
->dev
, oldname
,
5587 &iter
->dev
->adj_list
.upper
);
5588 netdev_adjacent_sysfs_add(iter
->dev
, dev
,
5589 &iter
->dev
->adj_list
.upper
);
5593 void *netdev_lower_dev_get_private(struct net_device
*dev
,
5594 struct net_device
*lower_dev
)
5596 struct netdev_adjacent
*lower
;
5600 lower
= __netdev_find_adj(dev
, lower_dev
, &dev
->adj_list
.lower
);
5604 return lower
->private;
5606 EXPORT_SYMBOL(netdev_lower_dev_get_private
);
5609 int dev_get_nest_level(struct net_device
*dev
,
5610 bool (*type_check
)(struct net_device
*dev
))
5612 struct net_device
*lower
= NULL
;
5613 struct list_head
*iter
;
5619 netdev_for_each_lower_dev(dev
, lower
, iter
) {
5620 nest
= dev_get_nest_level(lower
, type_check
);
5621 if (max_nest
< nest
)
5625 if (type_check(dev
))
5630 EXPORT_SYMBOL(dev_get_nest_level
);
5632 static void dev_change_rx_flags(struct net_device
*dev
, int flags
)
5634 const struct net_device_ops
*ops
= dev
->netdev_ops
;
5636 if (ops
->ndo_change_rx_flags
)
5637 ops
->ndo_change_rx_flags(dev
, flags
);
5640 static int __dev_set_promiscuity(struct net_device
*dev
, int inc
, bool notify
)
5642 unsigned int old_flags
= dev
->flags
;
5648 dev
->flags
|= IFF_PROMISC
;
5649 dev
->promiscuity
+= inc
;
5650 if (dev
->promiscuity
== 0) {
5653 * If inc causes overflow, untouch promisc and return error.
5656 dev
->flags
&= ~IFF_PROMISC
;
5658 dev
->promiscuity
-= inc
;
5659 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
5664 if (dev
->flags
!= old_flags
) {
5665 pr_info("device %s %s promiscuous mode\n",
5667 dev
->flags
& IFF_PROMISC
? "entered" : "left");
5668 if (audit_enabled
) {
5669 current_uid_gid(&uid
, &gid
);
5670 audit_log(current
->audit_context
, GFP_ATOMIC
,
5671 AUDIT_ANOM_PROMISCUOUS
,
5672 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
5673 dev
->name
, (dev
->flags
& IFF_PROMISC
),
5674 (old_flags
& IFF_PROMISC
),
5675 from_kuid(&init_user_ns
, audit_get_loginuid(current
)),
5676 from_kuid(&init_user_ns
, uid
),
5677 from_kgid(&init_user_ns
, gid
),
5678 audit_get_sessionid(current
));
5681 dev_change_rx_flags(dev
, IFF_PROMISC
);
5684 __dev_notify_flags(dev
, old_flags
, IFF_PROMISC
);
5689 * dev_set_promiscuity - update promiscuity count on a device
5693 * Add or remove promiscuity from a device. While the count in the device
5694 * remains above zero the interface remains promiscuous. Once it hits zero
5695 * the device reverts back to normal filtering operation. A negative inc
5696 * value is used to drop promiscuity on the device.
5697 * Return 0 if successful or a negative errno code on error.
5699 int dev_set_promiscuity(struct net_device
*dev
, int inc
)
5701 unsigned int old_flags
= dev
->flags
;
5704 err
= __dev_set_promiscuity(dev
, inc
, true);
5707 if (dev
->flags
!= old_flags
)
5708 dev_set_rx_mode(dev
);
5711 EXPORT_SYMBOL(dev_set_promiscuity
);
5713 static int __dev_set_allmulti(struct net_device
*dev
, int inc
, bool notify
)
5715 unsigned int old_flags
= dev
->flags
, old_gflags
= dev
->gflags
;
5719 dev
->flags
|= IFF_ALLMULTI
;
5720 dev
->allmulti
+= inc
;
5721 if (dev
->allmulti
== 0) {
5724 * If inc causes overflow, untouch allmulti and return error.
5727 dev
->flags
&= ~IFF_ALLMULTI
;
5729 dev
->allmulti
-= inc
;
5730 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
5735 if (dev
->flags
^ old_flags
) {
5736 dev_change_rx_flags(dev
, IFF_ALLMULTI
);
5737 dev_set_rx_mode(dev
);
5739 __dev_notify_flags(dev
, old_flags
,
5740 dev
->gflags
^ old_gflags
);
5746 * dev_set_allmulti - update allmulti count on a device
5750 * Add or remove reception of all multicast frames to a device. While the
5751 * count in the device remains above zero the interface remains listening
5752 * to all interfaces. Once it hits zero the device reverts back to normal
5753 * filtering operation. A negative @inc value is used to drop the counter
5754 * when releasing a resource needing all multicasts.
5755 * Return 0 if successful or a negative errno code on error.
5758 int dev_set_allmulti(struct net_device
*dev
, int inc
)
5760 return __dev_set_allmulti(dev
, inc
, true);
5762 EXPORT_SYMBOL(dev_set_allmulti
);
5765 * Upload unicast and multicast address lists to device and
5766 * configure RX filtering. When the device doesn't support unicast
5767 * filtering it is put in promiscuous mode while unicast addresses
5770 void __dev_set_rx_mode(struct net_device
*dev
)
5772 const struct net_device_ops
*ops
= dev
->netdev_ops
;
5774 /* dev_open will call this function so the list will stay sane. */
5775 if (!(dev
->flags
&IFF_UP
))
5778 if (!netif_device_present(dev
))
5781 if (!(dev
->priv_flags
& IFF_UNICAST_FLT
)) {
5782 /* Unicast addresses changes may only happen under the rtnl,
5783 * therefore calling __dev_set_promiscuity here is safe.
5785 if (!netdev_uc_empty(dev
) && !dev
->uc_promisc
) {
5786 __dev_set_promiscuity(dev
, 1, false);
5787 dev
->uc_promisc
= true;
5788 } else if (netdev_uc_empty(dev
) && dev
->uc_promisc
) {
5789 __dev_set_promiscuity(dev
, -1, false);
5790 dev
->uc_promisc
= false;
5794 if (ops
->ndo_set_rx_mode
)
5795 ops
->ndo_set_rx_mode(dev
);
5798 void dev_set_rx_mode(struct net_device
*dev
)
5800 netif_addr_lock_bh(dev
);
5801 __dev_set_rx_mode(dev
);
5802 netif_addr_unlock_bh(dev
);
5806 * dev_get_flags - get flags reported to userspace
5809 * Get the combination of flag bits exported through APIs to userspace.
5811 unsigned int dev_get_flags(const struct net_device
*dev
)
5815 flags
= (dev
->flags
& ~(IFF_PROMISC
|
5820 (dev
->gflags
& (IFF_PROMISC
|
5823 if (netif_running(dev
)) {
5824 if (netif_oper_up(dev
))
5825 flags
|= IFF_RUNNING
;
5826 if (netif_carrier_ok(dev
))
5827 flags
|= IFF_LOWER_UP
;
5828 if (netif_dormant(dev
))
5829 flags
|= IFF_DORMANT
;
5834 EXPORT_SYMBOL(dev_get_flags
);
5836 int __dev_change_flags(struct net_device
*dev
, unsigned int flags
)
5838 unsigned int old_flags
= dev
->flags
;
5844 * Set the flags on our device.
5847 dev
->flags
= (flags
& (IFF_DEBUG
| IFF_NOTRAILERS
| IFF_NOARP
|
5848 IFF_DYNAMIC
| IFF_MULTICAST
| IFF_PORTSEL
|
5850 (dev
->flags
& (IFF_UP
| IFF_VOLATILE
| IFF_PROMISC
|
5854 * Load in the correct multicast list now the flags have changed.
5857 if ((old_flags
^ flags
) & IFF_MULTICAST
)
5858 dev_change_rx_flags(dev
, IFF_MULTICAST
);
5860 dev_set_rx_mode(dev
);
5863 * Have we downed the interface. We handle IFF_UP ourselves
5864 * according to user attempts to set it, rather than blindly
5869 if ((old_flags
^ flags
) & IFF_UP
)
5870 ret
= ((old_flags
& IFF_UP
) ? __dev_close
: __dev_open
)(dev
);
5872 if ((flags
^ dev
->gflags
) & IFF_PROMISC
) {
5873 int inc
= (flags
& IFF_PROMISC
) ? 1 : -1;
5874 unsigned int old_flags
= dev
->flags
;
5876 dev
->gflags
^= IFF_PROMISC
;
5878 if (__dev_set_promiscuity(dev
, inc
, false) >= 0)
5879 if (dev
->flags
!= old_flags
)
5880 dev_set_rx_mode(dev
);
5883 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
5884 is important. Some (broken) drivers set IFF_PROMISC, when
5885 IFF_ALLMULTI is requested not asking us and not reporting.
5887 if ((flags
^ dev
->gflags
) & IFF_ALLMULTI
) {
5888 int inc
= (flags
& IFF_ALLMULTI
) ? 1 : -1;
5890 dev
->gflags
^= IFF_ALLMULTI
;
5891 __dev_set_allmulti(dev
, inc
, false);
5897 void __dev_notify_flags(struct net_device
*dev
, unsigned int old_flags
,
5898 unsigned int gchanges
)
5900 unsigned int changes
= dev
->flags
^ old_flags
;
5903 rtmsg_ifinfo(RTM_NEWLINK
, dev
, gchanges
, GFP_ATOMIC
);
5905 if (changes
& IFF_UP
) {
5906 if (dev
->flags
& IFF_UP
)
5907 call_netdevice_notifiers(NETDEV_UP
, dev
);
5909 call_netdevice_notifiers(NETDEV_DOWN
, dev
);
5912 if (dev
->flags
& IFF_UP
&&
5913 (changes
& ~(IFF_UP
| IFF_PROMISC
| IFF_ALLMULTI
| IFF_VOLATILE
))) {
5914 struct netdev_notifier_change_info change_info
;
5916 change_info
.flags_changed
= changes
;
5917 call_netdevice_notifiers_info(NETDEV_CHANGE
, dev
,
5923 * dev_change_flags - change device settings
5925 * @flags: device state flags
5927 * Change settings on device based state flags. The flags are
5928 * in the userspace exported format.
5930 int dev_change_flags(struct net_device
*dev
, unsigned int flags
)
5933 unsigned int changes
, old_flags
= dev
->flags
, old_gflags
= dev
->gflags
;
5935 ret
= __dev_change_flags(dev
, flags
);
5939 changes
= (old_flags
^ dev
->flags
) | (old_gflags
^ dev
->gflags
);
5940 __dev_notify_flags(dev
, old_flags
, changes
);
5943 EXPORT_SYMBOL(dev_change_flags
);
5945 static int __dev_set_mtu(struct net_device
*dev
, int new_mtu
)
5947 const struct net_device_ops
*ops
= dev
->netdev_ops
;
5949 if (ops
->ndo_change_mtu
)
5950 return ops
->ndo_change_mtu(dev
, new_mtu
);
5957 * dev_set_mtu - Change maximum transfer unit
5959 * @new_mtu: new transfer unit
5961 * Change the maximum transfer size of the network device.
5963 int dev_set_mtu(struct net_device
*dev
, int new_mtu
)
5967 if (new_mtu
== dev
->mtu
)
5970 /* MTU must be positive. */
5974 if (!netif_device_present(dev
))
5977 err
= call_netdevice_notifiers(NETDEV_PRECHANGEMTU
, dev
);
5978 err
= notifier_to_errno(err
);
5982 orig_mtu
= dev
->mtu
;
5983 err
= __dev_set_mtu(dev
, new_mtu
);
5986 err
= call_netdevice_notifiers(NETDEV_CHANGEMTU
, dev
);
5987 err
= notifier_to_errno(err
);
5989 /* setting mtu back and notifying everyone again,
5990 * so that they have a chance to revert changes.
5992 __dev_set_mtu(dev
, orig_mtu
);
5993 call_netdevice_notifiers(NETDEV_CHANGEMTU
, dev
);
5998 EXPORT_SYMBOL(dev_set_mtu
);
6001 * dev_set_group - Change group this device belongs to
6003 * @new_group: group this device should belong to
6005 void dev_set_group(struct net_device
*dev
, int new_group
)
6007 dev
->group
= new_group
;
6009 EXPORT_SYMBOL(dev_set_group
);
6012 * dev_set_mac_address - Change Media Access Control Address
6016 * Change the hardware (MAC) address of the device
6018 int dev_set_mac_address(struct net_device
*dev
, struct sockaddr
*sa
)
6020 const struct net_device_ops
*ops
= dev
->netdev_ops
;
6023 if (!ops
->ndo_set_mac_address
)
6025 if (sa
->sa_family
!= dev
->type
)
6027 if (!netif_device_present(dev
))
6029 err
= ops
->ndo_set_mac_address(dev
, sa
);
6032 dev
->addr_assign_type
= NET_ADDR_SET
;
6033 call_netdevice_notifiers(NETDEV_CHANGEADDR
, dev
);
6034 add_device_randomness(dev
->dev_addr
, dev
->addr_len
);
6037 EXPORT_SYMBOL(dev_set_mac_address
);
6040 * dev_change_carrier - Change device carrier
6042 * @new_carrier: new value
6044 * Change device carrier
6046 int dev_change_carrier(struct net_device
*dev
, bool new_carrier
)
6048 const struct net_device_ops
*ops
= dev
->netdev_ops
;
6050 if (!ops
->ndo_change_carrier
)
6052 if (!netif_device_present(dev
))
6054 return ops
->ndo_change_carrier(dev
, new_carrier
);
6056 EXPORT_SYMBOL(dev_change_carrier
);
6059 * dev_get_phys_port_id - Get device physical port ID
6063 * Get device physical port ID
6065 int dev_get_phys_port_id(struct net_device
*dev
,
6066 struct netdev_phys_item_id
*ppid
)
6068 const struct net_device_ops
*ops
= dev
->netdev_ops
;
6070 if (!ops
->ndo_get_phys_port_id
)
6072 return ops
->ndo_get_phys_port_id(dev
, ppid
);
6074 EXPORT_SYMBOL(dev_get_phys_port_id
);
6077 * dev_get_phys_port_name - Get device physical port name
6081 * Get device physical port name
6083 int dev_get_phys_port_name(struct net_device
*dev
,
6084 char *name
, size_t len
)
6086 const struct net_device_ops
*ops
= dev
->netdev_ops
;
6088 if (!ops
->ndo_get_phys_port_name
)
6090 return ops
->ndo_get_phys_port_name(dev
, name
, len
);
6092 EXPORT_SYMBOL(dev_get_phys_port_name
);
6095 * dev_new_index - allocate an ifindex
6096 * @net: the applicable net namespace
6098 * Returns a suitable unique value for a new device interface
6099 * number. The caller must hold the rtnl semaphore or the
6100 * dev_base_lock to be sure it remains unique.
6102 static int dev_new_index(struct net
*net
)
6104 int ifindex
= net
->ifindex
;
6108 if (!__dev_get_by_index(net
, ifindex
))
6109 return net
->ifindex
= ifindex
;
6113 /* Delayed registration/unregisteration */
6114 static LIST_HEAD(net_todo_list
);
6115 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq
);
6117 static void net_set_todo(struct net_device
*dev
)
6119 list_add_tail(&dev
->todo_list
, &net_todo_list
);
6120 dev_net(dev
)->dev_unreg_count
++;
6123 static void rollback_registered_many(struct list_head
*head
)
6125 struct net_device
*dev
, *tmp
;
6126 LIST_HEAD(close_head
);
6128 BUG_ON(dev_boot_phase
);
6131 list_for_each_entry_safe(dev
, tmp
, head
, unreg_list
) {
6132 /* Some devices call without registering
6133 * for initialization unwind. Remove those
6134 * devices and proceed with the remaining.
6136 if (dev
->reg_state
== NETREG_UNINITIALIZED
) {
6137 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6141 list_del(&dev
->unreg_list
);
6144 dev
->dismantle
= true;
6145 BUG_ON(dev
->reg_state
!= NETREG_REGISTERED
);
6148 /* If device is running, close it first. */
6149 list_for_each_entry(dev
, head
, unreg_list
)
6150 list_add_tail(&dev
->close_list
, &close_head
);
6151 dev_close_many(&close_head
, true);
6153 list_for_each_entry(dev
, head
, unreg_list
) {
6154 /* And unlink it from device chain. */
6155 unlist_netdevice(dev
);
6157 dev
->reg_state
= NETREG_UNREGISTERING
;
6162 list_for_each_entry(dev
, head
, unreg_list
) {
6163 struct sk_buff
*skb
= NULL
;
6165 /* Shutdown queueing discipline. */
6169 /* Notify protocols, that we are about to destroy
6170 this device. They should clean all the things.
6172 call_netdevice_notifiers(NETDEV_UNREGISTER
, dev
);
6174 if (!dev
->rtnl_link_ops
||
6175 dev
->rtnl_link_state
== RTNL_LINK_INITIALIZED
)
6176 skb
= rtmsg_ifinfo_build_skb(RTM_DELLINK
, dev
, ~0U,
6180 * Flush the unicast and multicast chains
6185 if (dev
->netdev_ops
->ndo_uninit
)
6186 dev
->netdev_ops
->ndo_uninit(dev
);
6189 rtmsg_ifinfo_send(skb
, dev
, GFP_KERNEL
);
6191 /* Notifier chain MUST detach us all upper devices. */
6192 WARN_ON(netdev_has_any_upper_dev(dev
));
6194 /* Remove entries from kobject tree */
6195 netdev_unregister_kobject(dev
);
6197 /* Remove XPS queueing entries */
6198 netif_reset_xps_queues_gt(dev
, 0);
6204 list_for_each_entry(dev
, head
, unreg_list
)
6208 static void rollback_registered(struct net_device
*dev
)
6212 list_add(&dev
->unreg_list
, &single
);
6213 rollback_registered_many(&single
);
6217 static netdev_features_t
netdev_fix_features(struct net_device
*dev
,
6218 netdev_features_t features
)
6220 /* Fix illegal checksum combinations */
6221 if ((features
& NETIF_F_HW_CSUM
) &&
6222 (features
& (NETIF_F_IP_CSUM
|NETIF_F_IPV6_CSUM
))) {
6223 netdev_warn(dev
, "mixed HW and IP checksum settings.\n");
6224 features
&= ~(NETIF_F_IP_CSUM
|NETIF_F_IPV6_CSUM
);
6227 /* TSO requires that SG is present as well. */
6228 if ((features
& NETIF_F_ALL_TSO
) && !(features
& NETIF_F_SG
)) {
6229 netdev_dbg(dev
, "Dropping TSO features since no SG feature.\n");
6230 features
&= ~NETIF_F_ALL_TSO
;
6233 if ((features
& NETIF_F_TSO
) && !(features
& NETIF_F_HW_CSUM
) &&
6234 !(features
& NETIF_F_IP_CSUM
)) {
6235 netdev_dbg(dev
, "Dropping TSO features since no CSUM feature.\n");
6236 features
&= ~NETIF_F_TSO
;
6237 features
&= ~NETIF_F_TSO_ECN
;
6240 if ((features
& NETIF_F_TSO6
) && !(features
& NETIF_F_HW_CSUM
) &&
6241 !(features
& NETIF_F_IPV6_CSUM
)) {
6242 netdev_dbg(dev
, "Dropping TSO6 features since no CSUM feature.\n");
6243 features
&= ~NETIF_F_TSO6
;
6246 /* TSO ECN requires that TSO is present as well. */
6247 if ((features
& NETIF_F_ALL_TSO
) == NETIF_F_TSO_ECN
)
6248 features
&= ~NETIF_F_TSO_ECN
;
6250 /* Software GSO depends on SG. */
6251 if ((features
& NETIF_F_GSO
) && !(features
& NETIF_F_SG
)) {
6252 netdev_dbg(dev
, "Dropping NETIF_F_GSO since no SG feature.\n");
6253 features
&= ~NETIF_F_GSO
;
6256 /* UFO needs SG and checksumming */
6257 if (features
& NETIF_F_UFO
) {
6258 /* maybe split UFO into V4 and V6? */
6259 if (!((features
& NETIF_F_GEN_CSUM
) ||
6260 (features
& (NETIF_F_IP_CSUM
|NETIF_F_IPV6_CSUM
))
6261 == (NETIF_F_IP_CSUM
|NETIF_F_IPV6_CSUM
))) {
6263 "Dropping NETIF_F_UFO since no checksum offload features.\n");
6264 features
&= ~NETIF_F_UFO
;
6267 if (!(features
& NETIF_F_SG
)) {
6269 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
6270 features
&= ~NETIF_F_UFO
;
6274 #ifdef CONFIG_NET_RX_BUSY_POLL
6275 if (dev
->netdev_ops
->ndo_busy_poll
)
6276 features
|= NETIF_F_BUSY_POLL
;
6279 features
&= ~NETIF_F_BUSY_POLL
;
6284 int __netdev_update_features(struct net_device
*dev
)
6286 netdev_features_t features
;
6291 features
= netdev_get_wanted_features(dev
);
6293 if (dev
->netdev_ops
->ndo_fix_features
)
6294 features
= dev
->netdev_ops
->ndo_fix_features(dev
, features
);
6296 /* driver might be less strict about feature dependencies */
6297 features
= netdev_fix_features(dev
, features
);
6299 if (dev
->features
== features
)
6302 netdev_dbg(dev
, "Features changed: %pNF -> %pNF\n",
6303 &dev
->features
, &features
);
6305 if (dev
->netdev_ops
->ndo_set_features
)
6306 err
= dev
->netdev_ops
->ndo_set_features(dev
, features
);
6308 if (unlikely(err
< 0)) {
6310 "set_features() failed (%d); wanted %pNF, left %pNF\n",
6311 err
, &features
, &dev
->features
);
6316 dev
->features
= features
;
6322 * netdev_update_features - recalculate device features
6323 * @dev: the device to check
6325 * Recalculate dev->features set and send notifications if it
6326 * has changed. Should be called after driver or hardware dependent
6327 * conditions might have changed that influence the features.
6329 void netdev_update_features(struct net_device
*dev
)
6331 if (__netdev_update_features(dev
))
6332 netdev_features_change(dev
);
6334 EXPORT_SYMBOL(netdev_update_features
);
6337 * netdev_change_features - recalculate device features
6338 * @dev: the device to check
6340 * Recalculate dev->features set and send notifications even
6341 * if they have not changed. Should be called instead of
6342 * netdev_update_features() if also dev->vlan_features might
6343 * have changed to allow the changes to be propagated to stacked
6346 void netdev_change_features(struct net_device
*dev
)
6348 __netdev_update_features(dev
);
6349 netdev_features_change(dev
);
6351 EXPORT_SYMBOL(netdev_change_features
);
6354 * netif_stacked_transfer_operstate - transfer operstate
6355 * @rootdev: the root or lower level device to transfer state from
6356 * @dev: the device to transfer operstate to
6358 * Transfer operational state from root to device. This is normally
6359 * called when a stacking relationship exists between the root
6360 * device and the device(a leaf device).
6362 void netif_stacked_transfer_operstate(const struct net_device
*rootdev
,
6363 struct net_device
*dev
)
6365 if (rootdev
->operstate
== IF_OPER_DORMANT
)
6366 netif_dormant_on(dev
);
6368 netif_dormant_off(dev
);
6370 if (netif_carrier_ok(rootdev
)) {
6371 if (!netif_carrier_ok(dev
))
6372 netif_carrier_on(dev
);
6374 if (netif_carrier_ok(dev
))
6375 netif_carrier_off(dev
);
6378 EXPORT_SYMBOL(netif_stacked_transfer_operstate
);
6381 static int netif_alloc_rx_queues(struct net_device
*dev
)
6383 unsigned int i
, count
= dev
->num_rx_queues
;
6384 struct netdev_rx_queue
*rx
;
6385 size_t sz
= count
* sizeof(*rx
);
6389 rx
= kzalloc(sz
, GFP_KERNEL
| __GFP_NOWARN
| __GFP_REPEAT
);
6397 for (i
= 0; i
< count
; i
++)
6403 static void netdev_init_one_queue(struct net_device
*dev
,
6404 struct netdev_queue
*queue
, void *_unused
)
6406 /* Initialize queue lock */
6407 spin_lock_init(&queue
->_xmit_lock
);
6408 netdev_set_xmit_lockdep_class(&queue
->_xmit_lock
, dev
->type
);
6409 queue
->xmit_lock_owner
= -1;
6410 netdev_queue_numa_node_write(queue
, NUMA_NO_NODE
);
6413 dql_init(&queue
->dql
, HZ
);
6417 static void netif_free_tx_queues(struct net_device
*dev
)
6422 static int netif_alloc_netdev_queues(struct net_device
*dev
)
6424 unsigned int count
= dev
->num_tx_queues
;
6425 struct netdev_queue
*tx
;
6426 size_t sz
= count
* sizeof(*tx
);
6428 BUG_ON(count
< 1 || count
> 0xffff);
6430 tx
= kzalloc(sz
, GFP_KERNEL
| __GFP_NOWARN
| __GFP_REPEAT
);
6438 netdev_for_each_tx_queue(dev
, netdev_init_one_queue
, NULL
);
6439 spin_lock_init(&dev
->tx_global_lock
);
6444 void netif_tx_stop_all_queues(struct net_device
*dev
)
6448 for (i
= 0; i
< dev
->num_tx_queues
; i
++) {
6449 struct netdev_queue
*txq
= netdev_get_tx_queue(dev
, i
);
6450 netif_tx_stop_queue(txq
);
6453 EXPORT_SYMBOL(netif_tx_stop_all_queues
);
6456 * register_netdevice - register a network device
6457 * @dev: device to register
6459 * Take a completed network device structure and add it to the kernel
6460 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6461 * chain. 0 is returned on success. A negative errno code is returned
6462 * on a failure to set up the device, or if the name is a duplicate.
6464 * Callers must hold the rtnl semaphore. You may want
6465 * register_netdev() instead of this.
6468 * The locking appears insufficient to guarantee two parallel registers
6469 * will not get the same name.
6472 int register_netdevice(struct net_device
*dev
)
6475 struct net
*net
= dev_net(dev
);
6477 BUG_ON(dev_boot_phase
);
6482 /* When net_device's are persistent, this will be fatal. */
6483 BUG_ON(dev
->reg_state
!= NETREG_UNINITIALIZED
);
6486 spin_lock_init(&dev
->addr_list_lock
);
6487 netdev_set_addr_lockdep_class(dev
);
6489 ret
= dev_get_valid_name(net
, dev
, dev
->name
);
6493 /* Init, if this function is available */
6494 if (dev
->netdev_ops
->ndo_init
) {
6495 ret
= dev
->netdev_ops
->ndo_init(dev
);
6503 if (((dev
->hw_features
| dev
->features
) &
6504 NETIF_F_HW_VLAN_CTAG_FILTER
) &&
6505 (!dev
->netdev_ops
->ndo_vlan_rx_add_vid
||
6506 !dev
->netdev_ops
->ndo_vlan_rx_kill_vid
)) {
6507 netdev_WARN(dev
, "Buggy VLAN acceleration in driver!\n");
6514 dev
->ifindex
= dev_new_index(net
);
6515 else if (__dev_get_by_index(net
, dev
->ifindex
))
6518 /* Transfer changeable features to wanted_features and enable
6519 * software offloads (GSO and GRO).
6521 dev
->hw_features
|= NETIF_F_SOFT_FEATURES
;
6522 dev
->features
|= NETIF_F_SOFT_FEATURES
;
6523 dev
->wanted_features
= dev
->features
& dev
->hw_features
;
6525 if (!(dev
->flags
& IFF_LOOPBACK
)) {
6526 dev
->hw_features
|= NETIF_F_NOCACHE_COPY
;
6529 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
6531 dev
->vlan_features
|= NETIF_F_HIGHDMA
;
6533 /* Make NETIF_F_SG inheritable to tunnel devices.
6535 dev
->hw_enc_features
|= NETIF_F_SG
;
6537 /* Make NETIF_F_SG inheritable to MPLS.
6539 dev
->mpls_features
|= NETIF_F_SG
;
6541 ret
= call_netdevice_notifiers(NETDEV_POST_INIT
, dev
);
6542 ret
= notifier_to_errno(ret
);
6546 ret
= netdev_register_kobject(dev
);
6549 dev
->reg_state
= NETREG_REGISTERED
;
6551 __netdev_update_features(dev
);
6554 * Default initial state at registry is that the
6555 * device is present.
6558 set_bit(__LINK_STATE_PRESENT
, &dev
->state
);
6560 linkwatch_init_dev(dev
);
6562 dev_init_scheduler(dev
);
6564 list_netdevice(dev
);
6565 add_device_randomness(dev
->dev_addr
, dev
->addr_len
);
6567 /* If the device has permanent device address, driver should
6568 * set dev_addr and also addr_assign_type should be set to
6569 * NET_ADDR_PERM (default value).
6571 if (dev
->addr_assign_type
== NET_ADDR_PERM
)
6572 memcpy(dev
->perm_addr
, dev
->dev_addr
, dev
->addr_len
);
6574 /* Notify protocols, that a new device appeared. */
6575 ret
= call_netdevice_notifiers(NETDEV_REGISTER
, dev
);
6576 ret
= notifier_to_errno(ret
);
6578 rollback_registered(dev
);
6579 dev
->reg_state
= NETREG_UNREGISTERED
;
6582 * Prevent userspace races by waiting until the network
6583 * device is fully setup before sending notifications.
6585 if (!dev
->rtnl_link_ops
||
6586 dev
->rtnl_link_state
== RTNL_LINK_INITIALIZED
)
6587 rtmsg_ifinfo(RTM_NEWLINK
, dev
, ~0U, GFP_KERNEL
);
6593 if (dev
->netdev_ops
->ndo_uninit
)
6594 dev
->netdev_ops
->ndo_uninit(dev
);
6597 EXPORT_SYMBOL(register_netdevice
);
6600 * init_dummy_netdev - init a dummy network device for NAPI
6601 * @dev: device to init
6603 * This takes a network device structure and initialize the minimum
6604 * amount of fields so it can be used to schedule NAPI polls without
6605 * registering a full blown interface. This is to be used by drivers
6606 * that need to tie several hardware interfaces to a single NAPI
6607 * poll scheduler due to HW limitations.
6609 int init_dummy_netdev(struct net_device
*dev
)
6611 /* Clear everything. Note we don't initialize spinlocks
6612 * are they aren't supposed to be taken by any of the
6613 * NAPI code and this dummy netdev is supposed to be
6614 * only ever used for NAPI polls
6616 memset(dev
, 0, sizeof(struct net_device
));
6618 /* make sure we BUG if trying to hit standard
6619 * register/unregister code path
6621 dev
->reg_state
= NETREG_DUMMY
;
6623 /* NAPI wants this */
6624 INIT_LIST_HEAD(&dev
->napi_list
);
6626 /* a dummy interface is started by default */
6627 set_bit(__LINK_STATE_PRESENT
, &dev
->state
);
6628 set_bit(__LINK_STATE_START
, &dev
->state
);
6630 /* Note : We dont allocate pcpu_refcnt for dummy devices,
6631 * because users of this 'device' dont need to change
6637 EXPORT_SYMBOL_GPL(init_dummy_netdev
);
6641 * register_netdev - register a network device
6642 * @dev: device to register
6644 * Take a completed network device structure and add it to the kernel
6645 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6646 * chain. 0 is returned on success. A negative errno code is returned
6647 * on a failure to set up the device, or if the name is a duplicate.
6649 * This is a wrapper around register_netdevice that takes the rtnl semaphore
6650 * and expands the device name if you passed a format string to
6653 int register_netdev(struct net_device
*dev
)
6658 err
= register_netdevice(dev
);
6662 EXPORT_SYMBOL(register_netdev
);
6664 int netdev_refcnt_read(const struct net_device
*dev
)
6668 for_each_possible_cpu(i
)
6669 refcnt
+= *per_cpu_ptr(dev
->pcpu_refcnt
, i
);
6672 EXPORT_SYMBOL(netdev_refcnt_read
);
6675 * netdev_wait_allrefs - wait until all references are gone.
6676 * @dev: target net_device
6678 * This is called when unregistering network devices.
6680 * Any protocol or device that holds a reference should register
6681 * for netdevice notification, and cleanup and put back the
6682 * reference if they receive an UNREGISTER event.
6683 * We can get stuck here if buggy protocols don't correctly
6686 static void netdev_wait_allrefs(struct net_device
*dev
)
6688 unsigned long rebroadcast_time
, warning_time
;
6691 linkwatch_forget_dev(dev
);
6693 rebroadcast_time
= warning_time
= jiffies
;
6694 refcnt
= netdev_refcnt_read(dev
);
6696 while (refcnt
!= 0) {
6697 if (time_after(jiffies
, rebroadcast_time
+ 1 * HZ
)) {
6700 /* Rebroadcast unregister notification */
6701 call_netdevice_notifiers(NETDEV_UNREGISTER
, dev
);
6707 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL
, dev
);
6708 if (test_bit(__LINK_STATE_LINKWATCH_PENDING
,
6710 /* We must not have linkwatch events
6711 * pending on unregister. If this
6712 * happens, we simply run the queue
6713 * unscheduled, resulting in a noop
6716 linkwatch_run_queue();
6721 rebroadcast_time
= jiffies
;
6726 refcnt
= netdev_refcnt_read(dev
);
6728 if (time_after(jiffies
, warning_time
+ 10 * HZ
)) {
6729 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
6731 warning_time
= jiffies
;
6740 * register_netdevice(x1);
6741 * register_netdevice(x2);
6743 * unregister_netdevice(y1);
6744 * unregister_netdevice(y2);
6750 * We are invoked by rtnl_unlock().
6751 * This allows us to deal with problems:
6752 * 1) We can delete sysfs objects which invoke hotplug
6753 * without deadlocking with linkwatch via keventd.
6754 * 2) Since we run with the RTNL semaphore not held, we can sleep
6755 * safely in order to wait for the netdev refcnt to drop to zero.
6757 * We must not return until all unregister events added during
6758 * the interval the lock was held have been completed.
6760 void netdev_run_todo(void)
6762 struct list_head list
;
6764 /* Snapshot list, allow later requests */
6765 list_replace_init(&net_todo_list
, &list
);
6770 /* Wait for rcu callbacks to finish before next phase */
6771 if (!list_empty(&list
))
6774 while (!list_empty(&list
)) {
6775 struct net_device
*dev
6776 = list_first_entry(&list
, struct net_device
, todo_list
);
6777 list_del(&dev
->todo_list
);
6780 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL
, dev
);
6783 if (unlikely(dev
->reg_state
!= NETREG_UNREGISTERING
)) {
6784 pr_err("network todo '%s' but state %d\n",
6785 dev
->name
, dev
->reg_state
);
6790 dev
->reg_state
= NETREG_UNREGISTERED
;
6792 on_each_cpu(flush_backlog
, dev
, 1);
6794 netdev_wait_allrefs(dev
);
6797 BUG_ON(netdev_refcnt_read(dev
));
6798 BUG_ON(!list_empty(&dev
->ptype_all
));
6799 BUG_ON(!list_empty(&dev
->ptype_specific
));
6800 WARN_ON(rcu_access_pointer(dev
->ip_ptr
));
6801 WARN_ON(rcu_access_pointer(dev
->ip6_ptr
));
6802 WARN_ON(dev
->dn_ptr
);
6804 if (dev
->destructor
)
6805 dev
->destructor(dev
);
6807 /* Report a network device has been unregistered */
6809 dev_net(dev
)->dev_unreg_count
--;
6811 wake_up(&netdev_unregistering_wq
);
6813 /* Free network device */
6814 kobject_put(&dev
->dev
.kobj
);
6818 /* Convert net_device_stats to rtnl_link_stats64. They have the same
6819 * fields in the same order, with only the type differing.
6821 void netdev_stats_to_stats64(struct rtnl_link_stats64
*stats64
,
6822 const struct net_device_stats
*netdev_stats
)
6824 #if BITS_PER_LONG == 64
6825 BUILD_BUG_ON(sizeof(*stats64
) != sizeof(*netdev_stats
));
6826 memcpy(stats64
, netdev_stats
, sizeof(*stats64
));
6828 size_t i
, n
= sizeof(*stats64
) / sizeof(u64
);
6829 const unsigned long *src
= (const unsigned long *)netdev_stats
;
6830 u64
*dst
= (u64
*)stats64
;
6832 BUILD_BUG_ON(sizeof(*netdev_stats
) / sizeof(unsigned long) !=
6833 sizeof(*stats64
) / sizeof(u64
));
6834 for (i
= 0; i
< n
; i
++)
6838 EXPORT_SYMBOL(netdev_stats_to_stats64
);
6841 * dev_get_stats - get network device statistics
6842 * @dev: device to get statistics from
6843 * @storage: place to store stats
6845 * Get network statistics from device. Return @storage.
6846 * The device driver may provide its own method by setting
6847 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
6848 * otherwise the internal statistics structure is used.
6850 struct rtnl_link_stats64
*dev_get_stats(struct net_device
*dev
,
6851 struct rtnl_link_stats64
*storage
)
6853 const struct net_device_ops
*ops
= dev
->netdev_ops
;
6855 if (ops
->ndo_get_stats64
) {
6856 memset(storage
, 0, sizeof(*storage
));
6857 ops
->ndo_get_stats64(dev
, storage
);
6858 } else if (ops
->ndo_get_stats
) {
6859 netdev_stats_to_stats64(storage
, ops
->ndo_get_stats(dev
));
6861 netdev_stats_to_stats64(storage
, &dev
->stats
);
6863 storage
->rx_dropped
+= atomic_long_read(&dev
->rx_dropped
);
6864 storage
->tx_dropped
+= atomic_long_read(&dev
->tx_dropped
);
6867 EXPORT_SYMBOL(dev_get_stats
);
6869 struct netdev_queue
*dev_ingress_queue_create(struct net_device
*dev
)
6871 struct netdev_queue
*queue
= dev_ingress_queue(dev
);
6873 #ifdef CONFIG_NET_CLS_ACT
6876 queue
= kzalloc(sizeof(*queue
), GFP_KERNEL
);
6879 netdev_init_one_queue(dev
, queue
, NULL
);
6880 RCU_INIT_POINTER(queue
->qdisc
, &noop_qdisc
);
6881 queue
->qdisc_sleeping
= &noop_qdisc
;
6882 rcu_assign_pointer(dev
->ingress_queue
, queue
);
6887 static const struct ethtool_ops default_ethtool_ops
;
6889 void netdev_set_default_ethtool_ops(struct net_device
*dev
,
6890 const struct ethtool_ops
*ops
)
6892 if (dev
->ethtool_ops
== &default_ethtool_ops
)
6893 dev
->ethtool_ops
= ops
;
6895 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops
);
6897 void netdev_freemem(struct net_device
*dev
)
6899 char *addr
= (char *)dev
- dev
->padded
;
6905 * alloc_netdev_mqs - allocate network device
6906 * @sizeof_priv: size of private data to allocate space for
6907 * @name: device name format string
6908 * @name_assign_type: origin of device name
6909 * @setup: callback to initialize device
6910 * @txqs: the number of TX subqueues to allocate
6911 * @rxqs: the number of RX subqueues to allocate
6913 * Allocates a struct net_device with private data area for driver use
6914 * and performs basic initialization. Also allocates subqueue structs
6915 * for each queue on the device.
6917 struct net_device
*alloc_netdev_mqs(int sizeof_priv
, const char *name
,
6918 unsigned char name_assign_type
,
6919 void (*setup
)(struct net_device
*),
6920 unsigned int txqs
, unsigned int rxqs
)
6922 struct net_device
*dev
;
6924 struct net_device
*p
;
6926 BUG_ON(strlen(name
) >= sizeof(dev
->name
));
6929 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
6935 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
6940 alloc_size
= sizeof(struct net_device
);
6942 /* ensure 32-byte alignment of private area */
6943 alloc_size
= ALIGN(alloc_size
, NETDEV_ALIGN
);
6944 alloc_size
+= sizeof_priv
;
6946 /* ensure 32-byte alignment of whole construct */
6947 alloc_size
+= NETDEV_ALIGN
- 1;
6949 p
= kzalloc(alloc_size
, GFP_KERNEL
| __GFP_NOWARN
| __GFP_REPEAT
);
6951 p
= vzalloc(alloc_size
);
6955 dev
= PTR_ALIGN(p
, NETDEV_ALIGN
);
6956 dev
->padded
= (char *)dev
- (char *)p
;
6958 dev
->pcpu_refcnt
= alloc_percpu(int);
6959 if (!dev
->pcpu_refcnt
)
6962 if (dev_addr_init(dev
))
6968 dev_net_set(dev
, &init_net
);
6970 dev
->gso_max_size
= GSO_MAX_SIZE
;
6971 dev
->gso_max_segs
= GSO_MAX_SEGS
;
6972 dev
->gso_min_segs
= 0;
6974 INIT_LIST_HEAD(&dev
->napi_list
);
6975 INIT_LIST_HEAD(&dev
->unreg_list
);
6976 INIT_LIST_HEAD(&dev
->close_list
);
6977 INIT_LIST_HEAD(&dev
->link_watch_list
);
6978 INIT_LIST_HEAD(&dev
->adj_list
.upper
);
6979 INIT_LIST_HEAD(&dev
->adj_list
.lower
);
6980 INIT_LIST_HEAD(&dev
->all_adj_list
.upper
);
6981 INIT_LIST_HEAD(&dev
->all_adj_list
.lower
);
6982 INIT_LIST_HEAD(&dev
->ptype_all
);
6983 INIT_LIST_HEAD(&dev
->ptype_specific
);
6984 dev
->priv_flags
= IFF_XMIT_DST_RELEASE
| IFF_XMIT_DST_RELEASE_PERM
;
6987 dev
->num_tx_queues
= txqs
;
6988 dev
->real_num_tx_queues
= txqs
;
6989 if (netif_alloc_netdev_queues(dev
))
6993 dev
->num_rx_queues
= rxqs
;
6994 dev
->real_num_rx_queues
= rxqs
;
6995 if (netif_alloc_rx_queues(dev
))
6999 strcpy(dev
->name
, name
);
7000 dev
->name_assign_type
= name_assign_type
;
7001 dev
->group
= INIT_NETDEV_GROUP
;
7002 if (!dev
->ethtool_ops
)
7003 dev
->ethtool_ops
= &default_ethtool_ops
;
7005 nf_hook_ingress_init(dev
);
7014 free_percpu(dev
->pcpu_refcnt
);
7016 netdev_freemem(dev
);
7019 EXPORT_SYMBOL(alloc_netdev_mqs
);
7022 * free_netdev - free network device
7025 * This function does the last stage of destroying an allocated device
7026 * interface. The reference to the device object is released.
7027 * If this is the last reference then it will be freed.
7029 void free_netdev(struct net_device
*dev
)
7031 struct napi_struct
*p
, *n
;
7033 netif_free_tx_queues(dev
);
7038 kfree(rcu_dereference_protected(dev
->ingress_queue
, 1));
7040 /* Flush device addresses */
7041 dev_addr_flush(dev
);
7043 list_for_each_entry_safe(p
, n
, &dev
->napi_list
, dev_list
)
7046 free_percpu(dev
->pcpu_refcnt
);
7047 dev
->pcpu_refcnt
= NULL
;
7049 /* Compatibility with error handling in drivers */
7050 if (dev
->reg_state
== NETREG_UNINITIALIZED
) {
7051 netdev_freemem(dev
);
7055 BUG_ON(dev
->reg_state
!= NETREG_UNREGISTERED
);
7056 dev
->reg_state
= NETREG_RELEASED
;
7058 /* will free via device release */
7059 put_device(&dev
->dev
);
7061 EXPORT_SYMBOL(free_netdev
);
7064 * synchronize_net - Synchronize with packet receive processing
7066 * Wait for packets currently being received to be done.
7067 * Does not block later packets from starting.
7069 void synchronize_net(void)
7072 if (rtnl_is_locked())
7073 synchronize_rcu_expedited();
7077 EXPORT_SYMBOL(synchronize_net
);
7080 * unregister_netdevice_queue - remove device from the kernel
7084 * This function shuts down a device interface and removes it
7085 * from the kernel tables.
7086 * If head not NULL, device is queued to be unregistered later.
7088 * Callers must hold the rtnl semaphore. You may want
7089 * unregister_netdev() instead of this.
7092 void unregister_netdevice_queue(struct net_device
*dev
, struct list_head
*head
)
7097 list_move_tail(&dev
->unreg_list
, head
);
7099 rollback_registered(dev
);
7100 /* Finish processing unregister after unlock */
7104 EXPORT_SYMBOL(unregister_netdevice_queue
);
7107 * unregister_netdevice_many - unregister many devices
7108 * @head: list of devices
7110 * Note: As most callers use a stack allocated list_head,
7111 * we force a list_del() to make sure stack wont be corrupted later.
7113 void unregister_netdevice_many(struct list_head
*head
)
7115 struct net_device
*dev
;
7117 if (!list_empty(head
)) {
7118 rollback_registered_many(head
);
7119 list_for_each_entry(dev
, head
, unreg_list
)
7124 EXPORT_SYMBOL(unregister_netdevice_many
);
7127 * unregister_netdev - remove device from the kernel
7130 * This function shuts down a device interface and removes it
7131 * from the kernel tables.
7133 * This is just a wrapper for unregister_netdevice that takes
7134 * the rtnl semaphore. In general you want to use this and not
7135 * unregister_netdevice.
7137 void unregister_netdev(struct net_device
*dev
)
7140 unregister_netdevice(dev
);
7143 EXPORT_SYMBOL(unregister_netdev
);
7146 * dev_change_net_namespace - move device to different nethost namespace
7148 * @net: network namespace
7149 * @pat: If not NULL name pattern to try if the current device name
7150 * is already taken in the destination network namespace.
7152 * This function shuts down a device interface and moves it
7153 * to a new network namespace. On success 0 is returned, on
7154 * a failure a netagive errno code is returned.
7156 * Callers must hold the rtnl semaphore.
7159 int dev_change_net_namespace(struct net_device
*dev
, struct net
*net
, const char *pat
)
7165 /* Don't allow namespace local devices to be moved. */
7167 if (dev
->features
& NETIF_F_NETNS_LOCAL
)
7170 /* Ensure the device has been registrered */
7171 if (dev
->reg_state
!= NETREG_REGISTERED
)
7174 /* Get out if there is nothing todo */
7176 if (net_eq(dev_net(dev
), net
))
7179 /* Pick the destination device name, and ensure
7180 * we can use it in the destination network namespace.
7183 if (__dev_get_by_name(net
, dev
->name
)) {
7184 /* We get here if we can't use the current device name */
7187 if (dev_get_valid_name(net
, dev
, pat
) < 0)
7192 * And now a mini version of register_netdevice unregister_netdevice.
7195 /* If device is running close it first. */
7198 /* And unlink it from device chain */
7200 unlist_netdevice(dev
);
7204 /* Shutdown queueing discipline. */
7207 /* Notify protocols, that we are about to destroy
7208 this device. They should clean all the things.
7210 Note that dev->reg_state stays at NETREG_REGISTERED.
7211 This is wanted because this way 8021q and macvlan know
7212 the device is just moving and can keep their slaves up.
7214 call_netdevice_notifiers(NETDEV_UNREGISTER
, dev
);
7216 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL
, dev
);
7217 rtmsg_ifinfo(RTM_DELLINK
, dev
, ~0U, GFP_KERNEL
);
7220 * Flush the unicast and multicast chains
7225 /* Send a netdev-removed uevent to the old namespace */
7226 kobject_uevent(&dev
->dev
.kobj
, KOBJ_REMOVE
);
7227 netdev_adjacent_del_links(dev
);
7229 /* Actually switch the network namespace */
7230 dev_net_set(dev
, net
);
7232 /* If there is an ifindex conflict assign a new one */
7233 if (__dev_get_by_index(net
, dev
->ifindex
))
7234 dev
->ifindex
= dev_new_index(net
);
7236 /* Send a netdev-add uevent to the new namespace */
7237 kobject_uevent(&dev
->dev
.kobj
, KOBJ_ADD
);
7238 netdev_adjacent_add_links(dev
);
7240 /* Fixup kobjects */
7241 err
= device_rename(&dev
->dev
, dev
->name
);
7244 /* Add the device back in the hashes */
7245 list_netdevice(dev
);
7247 /* Notify protocols, that a new device appeared. */
7248 call_netdevice_notifiers(NETDEV_REGISTER
, dev
);
7251 * Prevent userspace races by waiting until the network
7252 * device is fully setup before sending notifications.
7254 rtmsg_ifinfo(RTM_NEWLINK
, dev
, ~0U, GFP_KERNEL
);
7261 EXPORT_SYMBOL_GPL(dev_change_net_namespace
);
7263 static int dev_cpu_callback(struct notifier_block
*nfb
,
7264 unsigned long action
,
7267 struct sk_buff
**list_skb
;
7268 struct sk_buff
*skb
;
7269 unsigned int cpu
, oldcpu
= (unsigned long)ocpu
;
7270 struct softnet_data
*sd
, *oldsd
;
7272 if (action
!= CPU_DEAD
&& action
!= CPU_DEAD_FROZEN
)
7275 local_irq_disable();
7276 cpu
= smp_processor_id();
7277 sd
= &per_cpu(softnet_data
, cpu
);
7278 oldsd
= &per_cpu(softnet_data
, oldcpu
);
7280 /* Find end of our completion_queue. */
7281 list_skb
= &sd
->completion_queue
;
7283 list_skb
= &(*list_skb
)->next
;
7284 /* Append completion queue from offline CPU. */
7285 *list_skb
= oldsd
->completion_queue
;
7286 oldsd
->completion_queue
= NULL
;
7288 /* Append output queue from offline CPU. */
7289 if (oldsd
->output_queue
) {
7290 *sd
->output_queue_tailp
= oldsd
->output_queue
;
7291 sd
->output_queue_tailp
= oldsd
->output_queue_tailp
;
7292 oldsd
->output_queue
= NULL
;
7293 oldsd
->output_queue_tailp
= &oldsd
->output_queue
;
7295 /* Append NAPI poll list from offline CPU, with one exception :
7296 * process_backlog() must be called by cpu owning percpu backlog.
7297 * We properly handle process_queue & input_pkt_queue later.
7299 while (!list_empty(&oldsd
->poll_list
)) {
7300 struct napi_struct
*napi
= list_first_entry(&oldsd
->poll_list
,
7304 list_del_init(&napi
->poll_list
);
7305 if (napi
->poll
== process_backlog
)
7308 ____napi_schedule(sd
, napi
);
7311 raise_softirq_irqoff(NET_TX_SOFTIRQ
);
7314 /* Process offline CPU's input_pkt_queue */
7315 while ((skb
= __skb_dequeue(&oldsd
->process_queue
))) {
7317 input_queue_head_incr(oldsd
);
7319 while ((skb
= skb_dequeue(&oldsd
->input_pkt_queue
))) {
7321 input_queue_head_incr(oldsd
);
7329 * netdev_increment_features - increment feature set by one
7330 * @all: current feature set
7331 * @one: new feature set
7332 * @mask: mask feature set
7334 * Computes a new feature set after adding a device with feature set
7335 * @one to the master device with current feature set @all. Will not
7336 * enable anything that is off in @mask. Returns the new feature set.
7338 netdev_features_t
netdev_increment_features(netdev_features_t all
,
7339 netdev_features_t one
, netdev_features_t mask
)
7341 if (mask
& NETIF_F_GEN_CSUM
)
7342 mask
|= NETIF_F_ALL_CSUM
;
7343 mask
|= NETIF_F_VLAN_CHALLENGED
;
7345 all
|= one
& (NETIF_F_ONE_FOR_ALL
|NETIF_F_ALL_CSUM
) & mask
;
7346 all
&= one
| ~NETIF_F_ALL_FOR_ALL
;
7348 /* If one device supports hw checksumming, set for all. */
7349 if (all
& NETIF_F_GEN_CSUM
)
7350 all
&= ~(NETIF_F_ALL_CSUM
& ~NETIF_F_GEN_CSUM
);
7354 EXPORT_SYMBOL(netdev_increment_features
);
7356 static struct hlist_head
* __net_init
netdev_create_hash(void)
7359 struct hlist_head
*hash
;
7361 hash
= kmalloc(sizeof(*hash
) * NETDEV_HASHENTRIES
, GFP_KERNEL
);
7363 for (i
= 0; i
< NETDEV_HASHENTRIES
; i
++)
7364 INIT_HLIST_HEAD(&hash
[i
]);
7369 /* Initialize per network namespace state */
7370 static int __net_init
netdev_init(struct net
*net
)
7372 if (net
!= &init_net
)
7373 INIT_LIST_HEAD(&net
->dev_base_head
);
7375 net
->dev_name_head
= netdev_create_hash();
7376 if (net
->dev_name_head
== NULL
)
7379 net
->dev_index_head
= netdev_create_hash();
7380 if (net
->dev_index_head
== NULL
)
7386 kfree(net
->dev_name_head
);
7392 * netdev_drivername - network driver for the device
7393 * @dev: network device
7395 * Determine network driver for device.
7397 const char *netdev_drivername(const struct net_device
*dev
)
7399 const struct device_driver
*driver
;
7400 const struct device
*parent
;
7401 const char *empty
= "";
7403 parent
= dev
->dev
.parent
;
7407 driver
= parent
->driver
;
7408 if (driver
&& driver
->name
)
7409 return driver
->name
;
7413 static void __netdev_printk(const char *level
, const struct net_device
*dev
,
7414 struct va_format
*vaf
)
7416 if (dev
&& dev
->dev
.parent
) {
7417 dev_printk_emit(level
[1] - '0',
7420 dev_driver_string(dev
->dev
.parent
),
7421 dev_name(dev
->dev
.parent
),
7422 netdev_name(dev
), netdev_reg_state(dev
),
7425 printk("%s%s%s: %pV",
7426 level
, netdev_name(dev
), netdev_reg_state(dev
), vaf
);
7428 printk("%s(NULL net_device): %pV", level
, vaf
);
7432 void netdev_printk(const char *level
, const struct net_device
*dev
,
7433 const char *format
, ...)
7435 struct va_format vaf
;
7438 va_start(args
, format
);
7443 __netdev_printk(level
, dev
, &vaf
);
7447 EXPORT_SYMBOL(netdev_printk
);
7449 #define define_netdev_printk_level(func, level) \
7450 void func(const struct net_device *dev, const char *fmt, ...) \
7452 struct va_format vaf; \
7455 va_start(args, fmt); \
7460 __netdev_printk(level, dev, &vaf); \
7464 EXPORT_SYMBOL(func);
7466 define_netdev_printk_level(netdev_emerg
, KERN_EMERG
);
7467 define_netdev_printk_level(netdev_alert
, KERN_ALERT
);
7468 define_netdev_printk_level(netdev_crit
, KERN_CRIT
);
7469 define_netdev_printk_level(netdev_err
, KERN_ERR
);
7470 define_netdev_printk_level(netdev_warn
, KERN_WARNING
);
7471 define_netdev_printk_level(netdev_notice
, KERN_NOTICE
);
7472 define_netdev_printk_level(netdev_info
, KERN_INFO
);
7474 static void __net_exit
netdev_exit(struct net
*net
)
7476 kfree(net
->dev_name_head
);
7477 kfree(net
->dev_index_head
);
7480 static struct pernet_operations __net_initdata netdev_net_ops
= {
7481 .init
= netdev_init
,
7482 .exit
= netdev_exit
,
7485 static void __net_exit
default_device_exit(struct net
*net
)
7487 struct net_device
*dev
, *aux
;
7489 * Push all migratable network devices back to the
7490 * initial network namespace
7493 for_each_netdev_safe(net
, dev
, aux
) {
7495 char fb_name
[IFNAMSIZ
];
7497 /* Ignore unmoveable devices (i.e. loopback) */
7498 if (dev
->features
& NETIF_F_NETNS_LOCAL
)
7501 /* Leave virtual devices for the generic cleanup */
7502 if (dev
->rtnl_link_ops
)
7505 /* Push remaining network devices to init_net */
7506 snprintf(fb_name
, IFNAMSIZ
, "dev%d", dev
->ifindex
);
7507 err
= dev_change_net_namespace(dev
, &init_net
, fb_name
);
7509 pr_emerg("%s: failed to move %s to init_net: %d\n",
7510 __func__
, dev
->name
, err
);
7517 static void __net_exit
rtnl_lock_unregistering(struct list_head
*net_list
)
7519 /* Return with the rtnl_lock held when there are no network
7520 * devices unregistering in any network namespace in net_list.
7524 DEFINE_WAIT_FUNC(wait
, woken_wake_function
);
7526 add_wait_queue(&netdev_unregistering_wq
, &wait
);
7528 unregistering
= false;
7530 list_for_each_entry(net
, net_list
, exit_list
) {
7531 if (net
->dev_unreg_count
> 0) {
7532 unregistering
= true;
7540 wait_woken(&wait
, TASK_UNINTERRUPTIBLE
, MAX_SCHEDULE_TIMEOUT
);
7542 remove_wait_queue(&netdev_unregistering_wq
, &wait
);
7545 static void __net_exit
default_device_exit_batch(struct list_head
*net_list
)
7547 /* At exit all network devices most be removed from a network
7548 * namespace. Do this in the reverse order of registration.
7549 * Do this across as many network namespaces as possible to
7550 * improve batching efficiency.
7552 struct net_device
*dev
;
7554 LIST_HEAD(dev_kill_list
);
7556 /* To prevent network device cleanup code from dereferencing
7557 * loopback devices or network devices that have been freed
7558 * wait here for all pending unregistrations to complete,
7559 * before unregistring the loopback device and allowing the
7560 * network namespace be freed.
7562 * The netdev todo list containing all network devices
7563 * unregistrations that happen in default_device_exit_batch
7564 * will run in the rtnl_unlock() at the end of
7565 * default_device_exit_batch.
7567 rtnl_lock_unregistering(net_list
);
7568 list_for_each_entry(net
, net_list
, exit_list
) {
7569 for_each_netdev_reverse(net
, dev
) {
7570 if (dev
->rtnl_link_ops
&& dev
->rtnl_link_ops
->dellink
)
7571 dev
->rtnl_link_ops
->dellink(dev
, &dev_kill_list
);
7573 unregister_netdevice_queue(dev
, &dev_kill_list
);
7576 unregister_netdevice_many(&dev_kill_list
);
7580 static struct pernet_operations __net_initdata default_device_ops
= {
7581 .exit
= default_device_exit
,
7582 .exit_batch
= default_device_exit_batch
,
7586 * Initialize the DEV module. At boot time this walks the device list and
7587 * unhooks any devices that fail to initialise (normally hardware not
7588 * present) and leaves us with a valid list of present and active devices.
7593 * This is called single threaded during boot, so no need
7594 * to take the rtnl semaphore.
7596 static int __init
net_dev_init(void)
7598 int i
, rc
= -ENOMEM
;
7600 BUG_ON(!dev_boot_phase
);
7602 if (dev_proc_init())
7605 if (netdev_kobject_init())
7608 INIT_LIST_HEAD(&ptype_all
);
7609 for (i
= 0; i
< PTYPE_HASH_SIZE
; i
++)
7610 INIT_LIST_HEAD(&ptype_base
[i
]);
7612 INIT_LIST_HEAD(&offload_base
);
7614 if (register_pernet_subsys(&netdev_net_ops
))
7618 * Initialise the packet receive queues.
7621 for_each_possible_cpu(i
) {
7622 struct softnet_data
*sd
= &per_cpu(softnet_data
, i
);
7624 skb_queue_head_init(&sd
->input_pkt_queue
);
7625 skb_queue_head_init(&sd
->process_queue
);
7626 INIT_LIST_HEAD(&sd
->poll_list
);
7627 sd
->output_queue_tailp
= &sd
->output_queue
;
7629 sd
->csd
.func
= rps_trigger_softirq
;
7634 sd
->backlog
.poll
= process_backlog
;
7635 sd
->backlog
.weight
= weight_p
;
7640 /* The loopback device is special if any other network devices
7641 * is present in a network namespace the loopback device must
7642 * be present. Since we now dynamically allocate and free the
7643 * loopback device ensure this invariant is maintained by
7644 * keeping the loopback device as the first device on the
7645 * list of network devices. Ensuring the loopback devices
7646 * is the first device that appears and the last network device
7649 if (register_pernet_device(&loopback_net_ops
))
7652 if (register_pernet_device(&default_device_ops
))
7655 open_softirq(NET_TX_SOFTIRQ
, net_tx_action
);
7656 open_softirq(NET_RX_SOFTIRQ
, net_rx_action
);
7658 hotcpu_notifier(dev_cpu_callback
, 0);
7665 subsys_initcall(net_dev_init
);