projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ixgbe: fix bug when EITR=0 causing no writebacks
[deliverable/linux.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243 34#include <linux/ipv6.h>
5a0e3ad6 35#include <linux/slab.h>
9d5c8243
AK		 36#include <net/checksum.h>
37#include <net/ip6_checksum.h>
c6cb090b 38#include <linux/net_tstamp.h>
9d5c8243
AK		 39#include <linux/mii.h>
40#include <linux/ethtool.h>
41#include <linux/if_vlan.h>
42#include <linux/pci.h>
c54106bb 43#include <linux/pci-aspm.h>
9d5c8243
AK		 44#include <linux/delay.h>
45#include <linux/interrupt.h>
46#include <linux/if_ether.h>
40a914fa 47#include <linux/aer.h>
421e02f0 48#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 49#include <linux/dca.h>
50#endif
9d5c8243
AK		 51#include "igb.h"
52
55cac248 53#define DRV_VERSION "2.1.0-k2"
9d5c8243
AK		 54char igb_driver_name[] = "igb";
55char igb_driver_version[] = DRV_VERSION;
56static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 58static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 59
9d5c8243
AK		 60static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
62};
63
a3aa1884 64static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
2d064c06 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
85 /* required last entry */
86 {0, }
87};
88
89MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
90
91void igb_reset(struct igb_adapter *);
92static int igb_setup_all_tx_resources(struct igb_adapter *);
93static int igb_setup_all_rx_resources(struct igb_adapter *);
94static void igb_free_all_tx_resources(struct igb_adapter *);
95static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 96static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 97void igb_update_stats(struct igb_adapter *);
98static int igb_probe(struct pci_dev *, const struct pci_device_id *);
99static void __devexit igb_remove(struct pci_dev *pdev);
100static int igb_sw_init(struct igb_adapter *);
101static int igb_open(struct net_device *);
102static int igb_close(struct net_device *);
103static void igb_configure_tx(struct igb_adapter *);
104static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 105static void igb_clean_all_tx_rings(struct igb_adapter *);
106static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 107static void igb_clean_tx_ring(struct igb_ring *);
108static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 109static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 110static void igb_update_phy_info(unsigned long);
111static void igb_watchdog(unsigned long);
112static void igb_watchdog_task(struct work_struct *);
b1a436c3 113static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 114static struct net_device_stats *igb_get_stats(struct net_device *);
115static int igb_change_mtu(struct net_device *, int);
116static int igb_set_mac(struct net_device *, void *);
68d480c4 117static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 118static irqreturn_t igb_intr(int irq, void *);
119static irqreturn_t igb_intr_msi(int irq, void *);
120static irqreturn_t igb_msix_other(int irq, void *);
047e0030 121static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 122#ifdef CONFIG_IGB_DCA
047e0030 123static void igb_update_dca(struct igb_q_vector *);
fe4506b6 124static void igb_setup_dca(struct igb_adapter *);
421e02f0 125#endif /* CONFIG_IGB_DCA */
047e0030 126static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 127static int igb_poll(struct napi_struct *, int);
047e0030 128static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 129static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
130static void igb_tx_timeout(struct net_device *);
131static void igb_reset_task(struct work_struct *);
132static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
133static void igb_vlan_rx_add_vid(struct net_device *, u16);
134static void igb_vlan_rx_kill_vid(struct net_device *, u16);
135static void igb_restore_vlan(struct igb_adapter *);
26ad9178 136static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 137static void igb_ping_all_vfs(struct igb_adapter *);
138static void igb_msg_task(struct igb_adapter *);
4ae196df 139static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 140static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 141static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 142static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
143static int igb_ndo_set_vf_vlan(struct net_device *netdev,
144 int vf, u16 vlan, u8 qos);
145static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
146static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
147 struct ifla_vf_info *ivi);
9d5c8243 148
9d5c8243 149#ifdef CONFIG_PM
3fe7c4c9 150static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 151static int igb_resume(struct pci_dev *);
152#endif
153static void igb_shutdown(struct pci_dev *);
421e02f0 154#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 155static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
156static struct notifier_block dca_notifier = {
157 .notifier_call = igb_notify_dca,
158 .next = NULL,
159 .priority = 0
160};
161#endif
9d5c8243
AK		 162#ifdef CONFIG_NET_POLL_CONTROLLER
163/* for netdump / net console */
164static void igb_netpoll(struct net_device *);
165#endif
37680117 166#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 167static unsigned int max_vfs = 0;
168module_param(max_vfs, uint, 0);
169MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
170 "per physical function");
171#endif /* CONFIG_PCI_IOV */
172
9d5c8243
AK		 173static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
174 pci_channel_state_t);
175static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
176static void igb_io_resume(struct pci_dev *);
177
178static struct pci_error_handlers igb_err_handler = {
179 .error_detected = igb_io_error_detected,
180 .slot_reset = igb_io_slot_reset,
181 .resume = igb_io_resume,
182};
183
184
185static struct pci_driver igb_driver = {
186 .name = igb_driver_name,
187 .id_table = igb_pci_tbl,
188 .probe = igb_probe,
189 .remove = __devexit_p(igb_remove),
190#ifdef CONFIG_PM
191 /* Power Managment Hooks */
192 .suspend = igb_suspend,
193 .resume = igb_resume,
194#endif
195 .shutdown = igb_shutdown,
196 .err_handler = &igb_err_handler
197};
198
199MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
200MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
201MODULE_LICENSE("GPL");
202MODULE_VERSION(DRV_VERSION);
203
38c845c7
PO		 204/**
205 * igb_read_clock - read raw cycle counter (to be used by time counter)
206 */
207static cycle_t igb_read_clock(const struct cyclecounter *tc)
208{
209 struct igb_adapter *adapter =
210 container_of(tc, struct igb_adapter, cycles);
211 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 212 u64 stamp = 0;
213 int shift = 0;
38c845c7 214
55cac248
AD		 215 /*
216 * The timestamp latches on lowest register read. For the 82580
217 * the lowest register is SYSTIMR instead of SYSTIML. However we never
218 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
219 */
220 if (hw->mac.type == e1000_82580) {
221 stamp = rd32(E1000_SYSTIMR) >> 8;
222 shift = IGB_82580_TSYNC_SHIFT;
223 }
224
c5b9bd5e
AD		 225 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
226 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 227 return stamp;
228}
229
9d5c8243 230/**
c041076a 231 * igb_get_hw_dev - return device
9d5c8243
AK		 232 * used by hardware layer to print debugging information
233 **/
c041076a 234struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 235{
236 struct igb_adapter *adapter = hw->back;
c041076a 237 return adapter->netdev;
9d5c8243 238}
38c845c7 239
9d5c8243
AK		 240/**
241 * igb_init_module - Driver Registration Routine
242 *
243 * igb_init_module is the first routine called when the driver is
244 * loaded. All it does is register with the PCI subsystem.
245 **/
246static int __init igb_init_module(void)
247{
248 int ret;
249 printk(KERN_INFO "%s - version %s\n",
250 igb_driver_string, igb_driver_version);
251
252 printk(KERN_INFO "%s\n", igb_copyright);
253
421e02f0 254#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 255 dca_register_notify(&dca_notifier);
256#endif
bbd98fe4 257 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 258 return ret;
259}
260
261module_init(igb_init_module);
262
263/**
264 * igb_exit_module - Driver Exit Cleanup Routine
265 *
266 * igb_exit_module is called just before the driver is removed
267 * from memory.
268 **/
269static void __exit igb_exit_module(void)
270{
421e02f0 271#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 272 dca_unregister_notify(&dca_notifier);
273#endif
9d5c8243
AK		 274 pci_unregister_driver(&igb_driver);
275}
276
277module_exit(igb_exit_module);
278
26bc19ec
AD		 279#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
280/**
281 * igb_cache_ring_register - Descriptor ring to register mapping
282 * @adapter: board private structure to initialize
283 *
284 * Once we know the feature-set enabled for the device, we'll cache
285 * the register offset the descriptor ring is assigned to.
286 **/
287static void igb_cache_ring_register(struct igb_adapter *adapter)
288{
ee1b9f06 289 int i = 0, j = 0;
047e0030 290 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 291
292 switch (adapter->hw.mac.type) {
293 case e1000_82576:
294 /* The queues are allocated for virtualization such that VF 0
295 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
296 * In order to avoid collision we start at the first free queue
297 * and continue consuming queues in the same sequence
298 */
ee1b9f06 299 if (adapter->vfs_allocated_count) {
a99955fc 300 for (; i < adapter->rss_queues; i++)
3025a446
AD		 301 adapter->rx_ring[i]->reg_idx = rbase_offset +
302 Q_IDX_82576(i);
a99955fc 303 for (; j < adapter->rss_queues; j++)
3025a446
AD		 304 adapter->tx_ring[j]->reg_idx = rbase_offset +
305 Q_IDX_82576(j);
ee1b9f06 306 }
26bc19ec 307 case e1000_82575:
55cac248 308 case e1000_82580:
d2ba2ed8 309 case e1000_i350:
26bc19ec 310 default:
ee1b9f06 311 for (; i < adapter->num_rx_queues; i++)
3025a446 312 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 313 for (; j < adapter->num_tx_queues; j++)
3025a446 314 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 315 break;
316 }
317}
318
047e0030
AD		 319static void igb_free_queues(struct igb_adapter *adapter)
320{
3025a446 321 int i;
047e0030 322
3025a446
AD		 323 for (i = 0; i < adapter->num_tx_queues; i++) {
324 kfree(adapter->tx_ring[i]);
325 adapter->tx_ring[i] = NULL;
326 }
327 for (i = 0; i < adapter->num_rx_queues; i++) {
328 kfree(adapter->rx_ring[i]);
329 adapter->rx_ring[i] = NULL;
330 }
047e0030
AD		 331 adapter->num_rx_queues = 0;
332 adapter->num_tx_queues = 0;
333}
334
9d5c8243
AK		 335/**
336 * igb_alloc_queues - Allocate memory for all rings
337 * @adapter: board private structure to initialize
338 *
339 * We allocate one ring per queue at run-time since we don't know the
340 * number of queues at compile-time.
341 **/
342static int igb_alloc_queues(struct igb_adapter *adapter)
343{
3025a446 344 struct igb_ring *ring;
9d5c8243
AK		 345 int i;
346
661086df 347 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 348 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
349 if (!ring)
350 goto err;
68fd9910 351 ring->count = adapter->tx_ring_count;
661086df 352 ring->queue_index = i;
80785298 353 ring->pdev = adapter->pdev;
e694e964 354 ring->netdev = adapter->netdev;
85ad76b2
AD		 355 /* For 82575, context index must be unique per ring. */
356 if (adapter->hw.mac.type == e1000_82575)
357 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 358 adapter->tx_ring[i] = ring;
661086df 359 }
85ad76b2 360
9d5c8243 361 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 362 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
363 if (!ring)
364 goto err;
68fd9910 365 ring->count = adapter->rx_ring_count;
844290e5 366 ring->queue_index = i;
80785298 367 ring->pdev = adapter->pdev;
e694e964 368 ring->netdev = adapter->netdev;
4c844851 369 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 370 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
371 /* set flag indicating ring supports SCTP checksum offload */
372 if (adapter->hw.mac.type >= e1000_82576)
373 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 374 adapter->rx_ring[i] = ring;
9d5c8243 375 }
26bc19ec
AD		 376
377 igb_cache_ring_register(adapter);
9d5c8243 378
047e0030 379 return 0;
a88f10ec 380
047e0030
AD		 381err:
382 igb_free_queues(adapter);
d1a8c9e1 383
047e0030 384 return -ENOMEM;
a88f10ec
AD		 385}
386
9d5c8243 387#define IGB_N0_QUEUE -1
047e0030 388static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 389{
390 u32 msixbm = 0;
047e0030 391 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 392 struct e1000_hw *hw = &adapter->hw;
2d064c06 393 u32 ivar, index;
047e0030
AD		 394 int rx_queue = IGB_N0_QUEUE;
395 int tx_queue = IGB_N0_QUEUE;
396
397 if (q_vector->rx_ring)
398 rx_queue = q_vector->rx_ring->reg_idx;
399 if (q_vector->tx_ring)
400 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 401
402 switch (hw->mac.type) {
403 case e1000_82575:
9d5c8243
AK		 404 /* The 82575 assigns vectors using a bitmask, which matches the
405 bitmask for the EICR/EIMS/EIMC registers. To assign one
406 or more queues to a vector, we write the appropriate bits
407 into the MSIXBM register for that vector. */
047e0030 408 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 409 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 410 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 411 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 412 if (!adapter->msix_entries && msix_vector == 0)
413 msixbm |= E1000_EIMS_OTHER;
9d5c8243 414 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 415 q_vector->eims_value = msixbm;
2d064c06
AD		 416 break;
417 case e1000_82576:
26bc19ec 418 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 419 Each queue has a single entry in the table to which we write
420 a vector number along with a "valid" bit. Sadly, the layout
421 of the table is somewhat counterintuitive. */
422 if (rx_queue > IGB_N0_QUEUE) {
047e0030 423 index = (rx_queue & 0x7);
2d064c06 424 ivar = array_rd32(E1000_IVAR0, index);
047e0030 425 if (rx_queue < 8) {
26bc19ec
AD		 426 /* vector goes into low byte of register */
427 ivar = ivar & 0xFFFFFF00;
428 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 429 } else {
430 /* vector goes into third byte of register */
431 ivar = ivar & 0xFF00FFFF;
432 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 433 }
2d064c06
AD		 434 array_wr32(E1000_IVAR0, index, ivar);
435 }
436 if (tx_queue > IGB_N0_QUEUE) {
047e0030 437 index = (tx_queue & 0x7);
2d064c06 438 ivar = array_rd32(E1000_IVAR0, index);
047e0030 439 if (tx_queue < 8) {
26bc19ec
AD		 440 /* vector goes into second byte of register */
441 ivar = ivar & 0xFFFF00FF;
442 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 443 } else {
444 /* vector goes into high byte of register */
445 ivar = ivar & 0x00FFFFFF;
446 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 447 }
2d064c06
AD		 448 array_wr32(E1000_IVAR0, index, ivar);
449 }
047e0030 450 q_vector->eims_value = 1 << msix_vector;
2d064c06 451 break;
55cac248 452 case e1000_82580:
d2ba2ed8 453 case e1000_i350:
55cac248
AD		 454 /* 82580 uses the same table-based approach as 82576 but has fewer
455 entries as a result we carry over for queues greater than 4. */
456 if (rx_queue > IGB_N0_QUEUE) {
457 index = (rx_queue >> 1);
458 ivar = array_rd32(E1000_IVAR0, index);
459 if (rx_queue & 0x1) {
460 /* vector goes into third byte of register */
461 ivar = ivar & 0xFF00FFFF;
462 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
463 } else {
464 /* vector goes into low byte of register */
465 ivar = ivar & 0xFFFFFF00;
466 ivar |= msix_vector | E1000_IVAR_VALID;
467 }
468 array_wr32(E1000_IVAR0, index, ivar);
469 }
470 if (tx_queue > IGB_N0_QUEUE) {
471 index = (tx_queue >> 1);
472 ivar = array_rd32(E1000_IVAR0, index);
473 if (tx_queue & 0x1) {
474 /* vector goes into high byte of register */
475 ivar = ivar & 0x00FFFFFF;
476 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
477 } else {
478 /* vector goes into second byte of register */
479 ivar = ivar & 0xFFFF00FF;
480 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
481 }
482 array_wr32(E1000_IVAR0, index, ivar);
483 }
484 q_vector->eims_value = 1 << msix_vector;
485 break;
2d064c06
AD		 486 default:
487 BUG();
488 break;
489 }
26b39276
AD		 490
491 /* add q_vector eims value to global eims_enable_mask */
492 adapter->eims_enable_mask |= q_vector->eims_value;
493
494 /* configure q_vector to set itr on first interrupt */
495 q_vector->set_itr = 1;
9d5c8243
AK		 496}
497
498/**
499 * igb_configure_msix - Configure MSI-X hardware
500 *
501 * igb_configure_msix sets up the hardware to properly
502 * generate MSI-X interrupts.
503 **/
504static void igb_configure_msix(struct igb_adapter *adapter)
505{
506 u32 tmp;
507 int i, vector = 0;
508 struct e1000_hw *hw = &adapter->hw;
509
510 adapter->eims_enable_mask = 0;
9d5c8243
AK		 511
512 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 513 switch (hw->mac.type) {
514 case e1000_82575:
9d5c8243
AK		 515 tmp = rd32(E1000_CTRL_EXT);
516 /* enable MSI-X PBA support*/
517 tmp |= E1000_CTRL_EXT_PBA_CLR;
518
519 /* Auto-Mask interrupts upon ICR read. */
520 tmp |= E1000_CTRL_EXT_EIAME;
521 tmp |= E1000_CTRL_EXT_IRCA;
522
523 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 524
525 /* enable msix_other interrupt */
526 array_wr32(E1000_MSIXBM(0), vector++,
527 E1000_EIMS_OTHER);
844290e5 528 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 529
2d064c06
AD		 530 break;
531
532 case e1000_82576:
55cac248 533 case e1000_82580:
d2ba2ed8 534 case e1000_i350:
047e0030
AD		 535 /* Turn on MSI-X capability first, or our settings
536 * won't stick. And it will take days to debug. */
537 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
538 E1000_GPIE_PBA | E1000_GPIE_EIAME |
539 E1000_GPIE_NSICR);
540
541 /* enable msix_other interrupt */
542 adapter->eims_other = 1 << vector;
2d064c06 543 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 544
047e0030 545 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 546 break;
547 default:
548 /* do nothing, since nothing else supports MSI-X */
549 break;
550 } /* switch (hw->mac.type) */
047e0030
AD		 551
552 adapter->eims_enable_mask |= adapter->eims_other;
553
26b39276
AD		 554 for (i = 0; i < adapter->num_q_vectors; i++)
555 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 556
9d5c8243
AK		 557 wrfl();
558}
559
560/**
561 * igb_request_msix - Initialize MSI-X interrupts
562 *
563 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
564 * kernel.
565 **/
566static int igb_request_msix(struct igb_adapter *adapter)
567{
568 struct net_device *netdev = adapter->netdev;
047e0030 569 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 570 int i, err = 0, vector = 0;
571
047e0030 572 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 573 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 574 if (err)
575 goto out;
576 vector++;
577
578 for (i = 0; i < adapter->num_q_vectors; i++) {
579 struct igb_q_vector *q_vector = adapter->q_vector[i];
580
581 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
582
583 if (q_vector->rx_ring && q_vector->tx_ring)
584 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
585 q_vector->rx_ring->queue_index);
586 else if (q_vector->tx_ring)
587 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
588 q_vector->tx_ring->queue_index);
589 else if (q_vector->rx_ring)
590 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
591 q_vector->rx_ring->queue_index);
9d5c8243 592 else
047e0030
AD		 593 sprintf(q_vector->name, "%s-unused", netdev->name);
594
9d5c8243 595 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 596 igb_msix_ring, 0, q_vector->name,
047e0030 597 q_vector);
9d5c8243
AK		 598 if (err)
599 goto out;
9d5c8243
AK		 600 vector++;
601 }
602
9d5c8243
AK		 603 igb_configure_msix(adapter);
604 return 0;
605out:
606 return err;
607}
608
609static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
610{
611 if (adapter->msix_entries) {
612 pci_disable_msix(adapter->pdev);
613 kfree(adapter->msix_entries);
614 adapter->msix_entries = NULL;
047e0030 615 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 616 pci_disable_msi(adapter->pdev);
047e0030 617 }
9d5c8243
AK		 618}
619
047e0030
AD		 620/**
621 * igb_free_q_vectors - Free memory allocated for interrupt vectors
622 * @adapter: board private structure to initialize
623 *
624 * This function frees the memory allocated to the q_vectors. In addition if
625 * NAPI is enabled it will delete any references to the NAPI struct prior
626 * to freeing the q_vector.
627 **/
628static void igb_free_q_vectors(struct igb_adapter *adapter)
629{
630 int v_idx;
631
632 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
633 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
634 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 635 if (!q_vector)
636 continue;
047e0030
AD		 637 netif_napi_del(&q_vector->napi);
638 kfree(q_vector);
639 }
640 adapter->num_q_vectors = 0;
641}
642
643/**
644 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
645 *
646 * This function resets the device so that it has 0 rx queues, tx queues, and
647 * MSI-X interrupts allocated.
648 */
649static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
650{
651 igb_free_queues(adapter);
652 igb_free_q_vectors(adapter);
653 igb_reset_interrupt_capability(adapter);
654}
9d5c8243
AK		 655
656/**
657 * igb_set_interrupt_capability - set MSI or MSI-X if supported
658 *
659 * Attempt to configure interrupts using the best available
660 * capabilities of the hardware and kernel.
661 **/
662static void igb_set_interrupt_capability(struct igb_adapter *adapter)
663{
664 int err;
665 int numvecs, i;
666
83b7180d 667 /* Number of supported queues. */
a99955fc
AD		 668 adapter->num_rx_queues = adapter->rss_queues;
669 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 670
047e0030
AD		 671 /* start with one vector for every rx queue */
672 numvecs = adapter->num_rx_queues;
673
3ad2f3fb 674 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 675 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
676 numvecs += adapter->num_tx_queues;
047e0030
AD		 677
678 /* store the number of vectors reserved for queues */
679 adapter->num_q_vectors = numvecs;
680
681 /* add 1 vector for link status interrupts */
682 numvecs++;
9d5c8243
AK		 683 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
684 GFP_KERNEL);
685 if (!adapter->msix_entries)
686 goto msi_only;
687
688 for (i = 0; i < numvecs; i++)
689 adapter->msix_entries[i].entry = i;
690
691 err = pci_enable_msix(adapter->pdev,
692 adapter->msix_entries,
693 numvecs);
694 if (err == 0)
34a20e89 695 goto out;
9d5c8243
AK		 696
697 igb_reset_interrupt_capability(adapter);
698
699 /* If we can't do MSI-X, try MSI */
700msi_only:
2a3abf6d
AD		 701#ifdef CONFIG_PCI_IOV
702 /* disable SR-IOV for non MSI-X configurations */
703 if (adapter->vf_data) {
704 struct e1000_hw *hw = &adapter->hw;
705 /* disable iov and allow time for transactions to clear */
706 pci_disable_sriov(adapter->pdev);
707 msleep(500);
708
709 kfree(adapter->vf_data);
710 adapter->vf_data = NULL;
711 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
712 msleep(100);
713 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
714 }
715#endif
4fc82adf 716 adapter->vfs_allocated_count = 0;
a99955fc 717 adapter->rss_queues = 1;
4fc82adf 718 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 719 adapter->num_rx_queues = 1;
661086df 720 adapter->num_tx_queues = 1;
047e0030 721 adapter->num_q_vectors = 1;
9d5c8243 722 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 723 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 724out:
661086df 725 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 726 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 727 return;
728}
729
047e0030
AD		 730/**
731 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
732 * @adapter: board private structure to initialize
733 *
734 * We allocate one q_vector per queue interrupt. If allocation fails we
735 * return -ENOMEM.
736 **/
737static int igb_alloc_q_vectors(struct igb_adapter *adapter)
738{
739 struct igb_q_vector *q_vector;
740 struct e1000_hw *hw = &adapter->hw;
741 int v_idx;
742
743 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
744 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
745 if (!q_vector)
746 goto err_out;
747 q_vector->adapter = adapter;
047e0030
AD		 748 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
749 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 750 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
751 adapter->q_vector[v_idx] = q_vector;
752 }
753 return 0;
754
755err_out:
fe0592b4 756 igb_free_q_vectors(adapter);
047e0030
AD		 757 return -ENOMEM;
758}
759
760static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
761 int ring_idx, int v_idx)
762{
3025a446 763 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 764
3025a446 765 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 766 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 767 q_vector->itr_val = adapter->rx_itr_setting;
768 if (q_vector->itr_val && q_vector->itr_val <= 3)
769 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 770}
771
772static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
773 int ring_idx, int v_idx)
774{
3025a446 775 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 776
3025a446 777 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 778 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 779 q_vector->itr_val = adapter->tx_itr_setting;
780 if (q_vector->itr_val && q_vector->itr_val <= 3)
781 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 782}
783
784/**
785 * igb_map_ring_to_vector - maps allocated queues to vectors
786 *
787 * This function maps the recently allocated queues to vectors.
788 **/
789static int igb_map_ring_to_vector(struct igb_adapter *adapter)
790{
791 int i;
792 int v_idx = 0;
793
794 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
795 (adapter->num_q_vectors < adapter->num_tx_queues))
796 return -ENOMEM;
797
798 if (adapter->num_q_vectors >=
799 (adapter->num_rx_queues + adapter->num_tx_queues)) {
800 for (i = 0; i < adapter->num_rx_queues; i++)
801 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
802 for (i = 0; i < adapter->num_tx_queues; i++)
803 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
804 } else {
805 for (i = 0; i < adapter->num_rx_queues; i++) {
806 if (i < adapter->num_tx_queues)
807 igb_map_tx_ring_to_vector(adapter, i, v_idx);
808 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
809 }
810 for (; i < adapter->num_tx_queues; i++)
811 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
812 }
813 return 0;
814}
815
816/**
817 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
818 *
819 * This function initializes the interrupts and allocates all of the queues.
820 **/
821static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
822{
823 struct pci_dev *pdev = adapter->pdev;
824 int err;
825
826 igb_set_interrupt_capability(adapter);
827
828 err = igb_alloc_q_vectors(adapter);
829 if (err) {
830 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
831 goto err_alloc_q_vectors;
832 }
833
834 err = igb_alloc_queues(adapter);
835 if (err) {
836 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
837 goto err_alloc_queues;
838 }
839
840 err = igb_map_ring_to_vector(adapter);
841 if (err) {
842 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
843 goto err_map_queues;
844 }
845
846
847 return 0;
848err_map_queues:
849 igb_free_queues(adapter);
850err_alloc_queues:
851 igb_free_q_vectors(adapter);
852err_alloc_q_vectors:
853 igb_reset_interrupt_capability(adapter);
854 return err;
855}
856
9d5c8243
AK		 857/**
858 * igb_request_irq - initialize interrupts
859 *
860 * Attempts to configure interrupts using the best available
861 * capabilities of the hardware and kernel.
862 **/
863static int igb_request_irq(struct igb_adapter *adapter)
864{
865 struct net_device *netdev = adapter->netdev;
047e0030 866 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 867 int err = 0;
868
869 if (adapter->msix_entries) {
870 err = igb_request_msix(adapter);
844290e5 871 if (!err)
9d5c8243 872 goto request_done;
9d5c8243 873 /* fall back to MSI */
047e0030 874 igb_clear_interrupt_scheme(adapter);
9d5c8243 875 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 876 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 877 igb_free_all_tx_resources(adapter);
878 igb_free_all_rx_resources(adapter);
047e0030 879 adapter->num_tx_queues = 1;
9d5c8243 880 adapter->num_rx_queues = 1;
047e0030
AD		 881 adapter->num_q_vectors = 1;
882 err = igb_alloc_q_vectors(adapter);
883 if (err) {
884 dev_err(&pdev->dev,
885 "Unable to allocate memory for vectors\n");
886 goto request_done;
887 }
888 err = igb_alloc_queues(adapter);
889 if (err) {
890 dev_err(&pdev->dev,
891 "Unable to allocate memory for queues\n");
892 igb_free_q_vectors(adapter);
893 goto request_done;
894 }
895 igb_setup_all_tx_resources(adapter);
896 igb_setup_all_rx_resources(adapter);
844290e5 897 } else {
feeb2721 898 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 899 }
844290e5 900
7dfc16fa 901 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 902 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 903 netdev->name, adapter);
9d5c8243
AK		 904 if (!err)
905 goto request_done;
047e0030 906
9d5c8243
AK		 907 /* fall back to legacy interrupts */
908 igb_reset_interrupt_capability(adapter);
7dfc16fa 909 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 910 }
911
a0607fd3 912 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 913 netdev->name, adapter);
9d5c8243 914
6cb5e577 915 if (err)
9d5c8243
AK		 916 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
917 err);
9d5c8243
AK		 918
919request_done:
920 return err;
921}
922
923static void igb_free_irq(struct igb_adapter *adapter)
924{
9d5c8243
AK		 925 if (adapter->msix_entries) {
926 int vector = 0, i;
927
047e0030 928 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 929
047e0030
AD		 930 for (i = 0; i < adapter->num_q_vectors; i++) {
931 struct igb_q_vector *q_vector = adapter->q_vector[i];
932 free_irq(adapter->msix_entries[vector++].vector,
933 q_vector);
934 }
935 } else {
936 free_irq(adapter->pdev->irq, adapter);
9d5c8243 937 }
9d5c8243
AK		 938}
939
940/**
941 * igb_irq_disable - Mask off interrupt generation on the NIC
942 * @adapter: board private structure
943 **/
944static void igb_irq_disable(struct igb_adapter *adapter)
945{
946 struct e1000_hw *hw = &adapter->hw;
947
25568a53
AD		 948 /*
949 * we need to be careful when disabling interrupts. The VFs are also
950 * mapped into these registers and so clearing the bits can cause
951 * issues on the VF drivers so we only need to clear what we set
952 */
9d5c8243 953 if (adapter->msix_entries) {
2dfd1212
AD		 954 u32 regval = rd32(E1000_EIAM);
955 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
956 wr32(E1000_EIMC, adapter->eims_enable_mask);
957 regval = rd32(E1000_EIAC);
958 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 959 }
844290e5
PW		 960
961 wr32(E1000_IAM, 0);
9d5c8243
AK		 962 wr32(E1000_IMC, ~0);
963 wrfl();
964 synchronize_irq(adapter->pdev->irq);
965}
966
967/**
968 * igb_irq_enable - Enable default interrupt generation settings
969 * @adapter: board private structure
970 **/
971static void igb_irq_enable(struct igb_adapter *adapter)
972{
973 struct e1000_hw *hw = &adapter->hw;
974
975 if (adapter->msix_entries) {
25568a53 976 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 977 u32 regval = rd32(E1000_EIAC);
978 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
979 regval = rd32(E1000_EIAM);
980 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 981 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 982 if (adapter->vfs_allocated_count) {
4ae196df 983 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 984 ims |= E1000_IMS_VMMB;
985 }
55cac248
AD		 986 if (adapter->hw.mac.type == e1000_82580)
987 ims |= E1000_IMS_DRSTA;
988
25568a53 989 wr32(E1000_IMS, ims);
844290e5 990 } else {
55cac248
AD		 991 wr32(E1000_IMS, IMS_ENABLE_MASK |
992 E1000_IMS_DRSTA);
993 wr32(E1000_IAM, IMS_ENABLE_MASK |
994 E1000_IMS_DRSTA);
844290e5 995 }
9d5c8243
AK		 996}
997
998static void igb_update_mng_vlan(struct igb_adapter *adapter)
999{
51466239 1000 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1001 u16 vid = adapter->hw.mng_cookie.vlan_id;
1002 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1003
1004 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1005 /* add VID to filter table */
1006 igb_vfta_set(hw, vid, true);
1007 adapter->mng_vlan_id = vid;
1008 } else {
1009 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1010 }
1011
1012 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1013 (vid != old_vid) &&
1014 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1015 /* remove VID from filter table */
1016 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1017 }
1018}
1019
1020/**
1021 * igb_release_hw_control - release control of the h/w to f/w
1022 * @adapter: address of board private structure
1023 *
1024 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1025 * For ASF and Pass Through versions of f/w this means that the
1026 * driver is no longer loaded.
1027 *
1028 **/
1029static void igb_release_hw_control(struct igb_adapter *adapter)
1030{
1031 struct e1000_hw *hw = &adapter->hw;
1032 u32 ctrl_ext;
1033
1034 /* Let firmware take over control of h/w */
1035 ctrl_ext = rd32(E1000_CTRL_EXT);
1036 wr32(E1000_CTRL_EXT,
1037 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1038}
1039
9d5c8243
AK		 1040/**
1041 * igb_get_hw_control - get control of the h/w from f/w
1042 * @adapter: address of board private structure
1043 *
1044 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1045 * For ASF and Pass Through versions of f/w this means that
1046 * the driver is loaded.
1047 *
1048 **/
1049static void igb_get_hw_control(struct igb_adapter *adapter)
1050{
1051 struct e1000_hw *hw = &adapter->hw;
1052 u32 ctrl_ext;
1053
1054 /* Let firmware know the driver has taken over */
1055 ctrl_ext = rd32(E1000_CTRL_EXT);
1056 wr32(E1000_CTRL_EXT,
1057 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1058}
1059
9d5c8243
AK		 1060/**
1061 * igb_configure - configure the hardware for RX and TX
1062 * @adapter: private board structure
1063 **/
1064static void igb_configure(struct igb_adapter *adapter)
1065{
1066 struct net_device *netdev = adapter->netdev;
1067 int i;
1068
1069 igb_get_hw_control(adapter);
ff41f8dc 1070 igb_set_rx_mode(netdev);
9d5c8243
AK		 1071
1072 igb_restore_vlan(adapter);
9d5c8243 1073
85b430b4 1074 igb_setup_tctl(adapter);
06cf2666 1075 igb_setup_mrqc(adapter);
9d5c8243 1076 igb_setup_rctl(adapter);
85b430b4
AD		 1077
1078 igb_configure_tx(adapter);
9d5c8243 1079 igb_configure_rx(adapter);
662d7205
AD		 1080
1081 igb_rx_fifo_flush_82575(&adapter->hw);
1082
c493ea45 1083 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1084 * at least 1 descriptor unused to make sure
1085 * next_to_use != next_to_clean */
1086 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1087 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1088 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243 1089 }
9d5c8243
AK		 1090}
1091
88a268c1
NN		 1092/**
1093 * igb_power_up_link - Power up the phy/serdes link
1094 * @adapter: address of board private structure
1095 **/
1096void igb_power_up_link(struct igb_adapter *adapter)
1097{
1098 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1099 igb_power_up_phy_copper(&adapter->hw);
1100 else
1101 igb_power_up_serdes_link_82575(&adapter->hw);
1102}
1103
1104/**
1105 * igb_power_down_link - Power down the phy/serdes link
1106 * @adapter: address of board private structure
1107 */
1108static void igb_power_down_link(struct igb_adapter *adapter)
1109{
1110 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1111 igb_power_down_phy_copper_82575(&adapter->hw);
1112 else
1113 igb_shutdown_serdes_link_82575(&adapter->hw);
1114}
9d5c8243
AK		 1115
1116/**
1117 * igb_up - Open the interface and prepare it to handle traffic
1118 * @adapter: board private structure
1119 **/
9d5c8243
AK		 1120int igb_up(struct igb_adapter *adapter)
1121{
1122 struct e1000_hw *hw = &adapter->hw;
1123 int i;
1124
1125 /* hardware has been reset, we need to reload some things */
1126 igb_configure(adapter);
1127
1128 clear_bit(__IGB_DOWN, &adapter->state);
1129
047e0030
AD		 1130 for (i = 0; i < adapter->num_q_vectors; i++) {
1131 struct igb_q_vector *q_vector = adapter->q_vector[i];
1132 napi_enable(&q_vector->napi);
1133 }
844290e5 1134 if (adapter->msix_entries)
9d5c8243 1135 igb_configure_msix(adapter);
feeb2721
AD		 1136 else
1137 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1138
1139 /* Clear any pending interrupts. */
1140 rd32(E1000_ICR);
1141 igb_irq_enable(adapter);
1142
d4960307
AD		 1143 /* notify VFs that reset has been completed */
1144 if (adapter->vfs_allocated_count) {
1145 u32 reg_data = rd32(E1000_CTRL_EXT);
1146 reg_data |= E1000_CTRL_EXT_PFRSTD;
1147 wr32(E1000_CTRL_EXT, reg_data);
1148 }
1149
4cb9be7a
JB		 1150 netif_tx_start_all_queues(adapter->netdev);
1151
25568a53
AD		 1152 /* start the watchdog. */
1153 hw->mac.get_link_status = 1;
1154 schedule_work(&adapter->watchdog_task);
1155
9d5c8243
AK		 1156 return 0;
1157}
1158
1159void igb_down(struct igb_adapter *adapter)
1160{
9d5c8243 1161 struct net_device *netdev = adapter->netdev;
330a6d6a 1162 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1163 u32 tctl, rctl;
1164 int i;
1165
1166 /* signal that we're down so the interrupt handler does not
1167 * reschedule our watchdog timer */
1168 set_bit(__IGB_DOWN, &adapter->state);
1169
1170 /* disable receives in the hardware */
1171 rctl = rd32(E1000_RCTL);
1172 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1173 /* flush and sleep below */
1174
fd2ea0a7 1175 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1176
1177 /* disable transmits in the hardware */
1178 tctl = rd32(E1000_TCTL);
1179 tctl &= ~E1000_TCTL_EN;
1180 wr32(E1000_TCTL, tctl);
1181 /* flush both disables and wait for them to finish */
1182 wrfl();
1183 msleep(10);
1184
047e0030
AD		 1185 for (i = 0; i < adapter->num_q_vectors; i++) {
1186 struct igb_q_vector *q_vector = adapter->q_vector[i];
1187 napi_disable(&q_vector->napi);
1188 }
9d5c8243 1189
9d5c8243
AK		 1190 igb_irq_disable(adapter);
1191
1192 del_timer_sync(&adapter->watchdog_timer);
1193 del_timer_sync(&adapter->phy_info_timer);
1194
9d5c8243 1195 netif_carrier_off(netdev);
04fe6358
AD		 1196
1197 /* record the stats before reset*/
1198 igb_update_stats(adapter);
1199
9d5c8243
AK		 1200 adapter->link_speed = 0;
1201 adapter->link_duplex = 0;
1202
3023682e
JK		 1203 if (!pci_channel_offline(adapter->pdev))
1204 igb_reset(adapter);
9d5c8243
AK		 1205 igb_clean_all_tx_rings(adapter);
1206 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1207#ifdef CONFIG_IGB_DCA
1208
1209 /* since we reset the hardware DCA settings were cleared */
1210 igb_setup_dca(adapter);
1211#endif
9d5c8243
AK		 1212}
1213
1214void igb_reinit_locked(struct igb_adapter *adapter)
1215{
1216 WARN_ON(in_interrupt());
1217 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1218 msleep(1);
1219 igb_down(adapter);
1220 igb_up(adapter);
1221 clear_bit(__IGB_RESETTING, &adapter->state);
1222}
1223
1224void igb_reset(struct igb_adapter *adapter)
1225{
090b1795 1226 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1227 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1228 struct e1000_mac_info *mac = &hw->mac;
1229 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1230 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1231 u16 hwm;
1232
1233 /* Repartition Pba for greater than 9k mtu
1234 * To take effect CTRL.RST is required.
1235 */
fa4dfae0 1236 switch (mac->type) {
d2ba2ed8 1237 case e1000_i350:
55cac248
AD		 1238 case e1000_82580:
1239 pba = rd32(E1000_RXPBS);
1240 pba = igb_rxpbs_adjust_82580(pba);
1241 break;
fa4dfae0 1242 case e1000_82576:
d249be54
AD		 1243 pba = rd32(E1000_RXPBS);
1244 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1245 break;
1246 case e1000_82575:
1247 default:
1248 pba = E1000_PBA_34K;
1249 break;
2d064c06 1250 }
9d5c8243 1251
2d064c06
AD		 1252 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1253 (mac->type < e1000_82576)) {
9d5c8243
AK		 1254 /* adjust PBA for jumbo frames */
1255 wr32(E1000_PBA, pba);
1256
1257 /* To maintain wire speed transmits, the Tx FIFO should be
1258 * large enough to accommodate two full transmit packets,
1259 * rounded up to the next 1KB and expressed in KB. Likewise,
1260 * the Rx FIFO should be large enough to accommodate at least
1261 * one full receive packet and is similarly rounded up and
1262 * expressed in KB. */
1263 pba = rd32(E1000_PBA);
1264 /* upper 16 bits has Tx packet buffer allocation size in KB */
1265 tx_space = pba >> 16;
1266 /* lower 16 bits has Rx packet buffer allocation size in KB */
1267 pba &= 0xffff;
1268 /* the tx fifo also stores 16 bytes of information about the tx
1269 * but don't include ethernet FCS because hardware appends it */
1270 min_tx_space = (adapter->max_frame_size +
85e8d004 1271 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1272 ETH_FCS_LEN) * 2;
1273 min_tx_space = ALIGN(min_tx_space, 1024);
1274 min_tx_space >>= 10;
1275 /* software strips receive CRC, so leave room for it */
1276 min_rx_space = adapter->max_frame_size;
1277 min_rx_space = ALIGN(min_rx_space, 1024);
1278 min_rx_space >>= 10;
1279
1280 /* If current Tx allocation is less than the min Tx FIFO size,
1281 * and the min Tx FIFO size is less than the current Rx FIFO
1282 * allocation, take space away from current Rx allocation */
1283 if (tx_space < min_tx_space &&
1284 ((min_tx_space - tx_space) < pba)) {
1285 pba = pba - (min_tx_space - tx_space);
1286
1287 /* if short on rx space, rx wins and must trump tx
1288 * adjustment */
1289 if (pba < min_rx_space)
1290 pba = min_rx_space;
1291 }
2d064c06 1292 wr32(E1000_PBA, pba);
9d5c8243 1293 }
9d5c8243
AK		 1294
1295 /* flow control settings */
1296 /* The high water mark must be low enough to fit one full frame
1297 * (or the size used for early receive) above it in the Rx FIFO.
1298 * Set it to the lower of:
1299 * - 90% of the Rx FIFO size, or
1300 * - the full Rx FIFO size minus one full frame */
1301 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1302 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1303
d405ea3e
AD		 1304 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1305 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1306 fc->pause_time = 0xFFFF;
1307 fc->send_xon = 1;
0cce119a 1308 fc->current_mode = fc->requested_mode;
9d5c8243 1309
4ae196df
AD		 1310 /* disable receive for all VFs and wait one second */
1311 if (adapter->vfs_allocated_count) {
1312 int i;
1313 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1314 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1315
1316 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1317 igb_ping_all_vfs(adapter);
4ae196df
AD		 1318
1319 /* disable transmits and receives */
1320 wr32(E1000_VFRE, 0);
1321 wr32(E1000_VFTE, 0);
1322 }
1323
9d5c8243 1324 /* Allow time for pending master requests to run */
330a6d6a 1325 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1326 wr32(E1000_WUC, 0);
1327
330a6d6a 1328 if (hw->mac.ops.init_hw(hw))
090b1795 1329 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243 1330
55cac248
AD		 1331 if (hw->mac.type == e1000_82580) {
1332 u32 reg = rd32(E1000_PCIEMISC);
1333 wr32(E1000_PCIEMISC,
1334 reg & ~E1000_PCIEMISC_LX_DECISION);
1335 }
88a268c1
NN		 1336 if (!netif_running(adapter->netdev))
1337 igb_power_down_link(adapter);
1338
9d5c8243
AK		 1339 igb_update_mng_vlan(adapter);
1340
1341 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1342 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1343
330a6d6a 1344 igb_get_phy_info(hw);
9d5c8243
AK		 1345}
1346
2e5c6922 1347static const struct net_device_ops igb_netdev_ops = {
559e9c49 1348 .ndo_open = igb_open,
2e5c6922 1349 .ndo_stop = igb_close,
00829823 1350 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1351 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1352 .ndo_set_rx_mode = igb_set_rx_mode,
1353 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1354 .ndo_set_mac_address = igb_set_mac,
1355 .ndo_change_mtu = igb_change_mtu,
1356 .ndo_do_ioctl = igb_ioctl,
1357 .ndo_tx_timeout = igb_tx_timeout,
1358 .ndo_validate_addr = eth_validate_addr,
1359 .ndo_vlan_rx_register = igb_vlan_rx_register,
1360 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1361 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1362 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1363 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1364 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1365 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1366#ifdef CONFIG_NET_POLL_CONTROLLER
1367 .ndo_poll_controller = igb_netpoll,
1368#endif
1369};
1370
9d5c8243
AK		 1371/**
1372 * igb_probe - Device Initialization Routine
1373 * @pdev: PCI device information struct
1374 * @ent: entry in igb_pci_tbl
1375 *
1376 * Returns 0 on success, negative on failure
1377 *
1378 * igb_probe initializes an adapter identified by a pci_dev structure.
1379 * The OS initialization, configuring of the adapter private structure,
1380 * and a hardware reset occur.
1381 **/
1382static int __devinit igb_probe(struct pci_dev *pdev,
1383 const struct pci_device_id *ent)
1384{
1385 struct net_device *netdev;
1386 struct igb_adapter *adapter;
1387 struct e1000_hw *hw;
4337e993
AD		 1388 u16 eeprom_data = 0;
1389 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1390 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1391 unsigned long mmio_start, mmio_len;
2d6a5e95 1392 int err, pci_using_dac;
9d5c8243
AK		 1393 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1394 u32 part_num;
1395
aed5dec3 1396 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1397 if (err)
1398 return err;
1399
1400 pci_using_dac = 0;
6a35528a 1401 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243 1402 if (!err) {
6a35528a 1403 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243
AK		 1404 if (!err)
1405 pci_using_dac = 1;
1406 } else {
284901a9 1407 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243 1408 if (err) {
284901a9 1409 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243
AK		 1410 if (err) {
1411 dev_err(&pdev->dev, "No usable DMA "
1412 "configuration, aborting\n");
1413 goto err_dma;
1414 }
1415 }
1416 }
1417
aed5dec3
AD		 1418 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1419 IORESOURCE_MEM),
1420 igb_driver_name);
9d5c8243
AK		 1421 if (err)
1422 goto err_pci_reg;
1423
19d5afd4 1424 pci_enable_pcie_error_reporting(pdev);
40a914fa 1425
9d5c8243 1426 pci_set_master(pdev);
c682fc23 1427 pci_save_state(pdev);
9d5c8243
AK		 1428
1429 err = -ENOMEM;
1bfaf07b
AD		 1430 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1431 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1432 if (!netdev)
1433 goto err_alloc_etherdev;
1434
1435 SET_NETDEV_DEV(netdev, &pdev->dev);
1436
1437 pci_set_drvdata(pdev, netdev);
1438 adapter = netdev_priv(netdev);
1439 adapter->netdev = netdev;
1440 adapter->pdev = pdev;
1441 hw = &adapter->hw;
1442 hw->back = adapter;
1443 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1444
1445 mmio_start = pci_resource_start(pdev, 0);
1446 mmio_len = pci_resource_len(pdev, 0);
1447
1448 err = -EIO;
28b0759c
AD		 1449 hw->hw_addr = ioremap(mmio_start, mmio_len);
1450 if (!hw->hw_addr)
9d5c8243
AK		 1451 goto err_ioremap;
1452
2e5c6922 1453 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1454 igb_set_ethtool_ops(netdev);
9d5c8243 1455 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1456
1457 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1458
1459 netdev->mem_start = mmio_start;
1460 netdev->mem_end = mmio_start + mmio_len;
1461
9d5c8243
AK		 1462 /* PCI config space info */
1463 hw->vendor_id = pdev->vendor;
1464 hw->device_id = pdev->device;
1465 hw->revision_id = pdev->revision;
1466 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1467 hw->subsystem_device_id = pdev->subsystem_device;
1468
9d5c8243
AK		 1469 /* Copy the default MAC, PHY and NVM function pointers */
1470 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1471 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1472 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1473 /* Initialize skew-specific constants */
1474 err = ei->get_invariants(hw);
1475 if (err)
450c87c8 1476 goto err_sw_init;
9d5c8243 1477
450c87c8 1478 /* setup the private structure */
9d5c8243
AK		 1479 err = igb_sw_init(adapter);
1480 if (err)
1481 goto err_sw_init;
1482
1483 igb_get_bus_info_pcie(hw);
1484
1485 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1486
1487 /* Copper options */
1488 if (hw->phy.media_type == e1000_media_type_copper) {
1489 hw->phy.mdix = AUTO_ALL_MODES;
1490 hw->phy.disable_polarity_correction = false;
1491 hw->phy.ms_type = e1000_ms_hw_default;
1492 }
1493
1494 if (igb_check_reset_block(hw))
1495 dev_info(&pdev->dev,
1496 "PHY reset is blocked due to SOL/IDER session.\n");
1497
1498 netdev->features = NETIF_F_SG |
7d8eb29e 1499 NETIF_F_IP_CSUM |
9d5c8243
AK		 1500 NETIF_F_HW_VLAN_TX |
1501 NETIF_F_HW_VLAN_RX |
1502 NETIF_F_HW_VLAN_FILTER;
1503
7d8eb29e 1504 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1505 netdev->features |= NETIF_F_TSO;
9d5c8243 1506 netdev->features |= NETIF_F_TSO6;
5c0999b7 1507 netdev->features |= NETIF_F_GRO;
d3352520 1508
48f29ffc
JK		 1509 netdev->vlan_features |= NETIF_F_TSO;
1510 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1511 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1512 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1513 netdev->vlan_features |= NETIF_F_SG;
1514
9d5c8243
AK		 1515 if (pci_using_dac)
1516 netdev->features |= NETIF_F_HIGHDMA;
1517
5b043fb0 1518 if (hw->mac.type >= e1000_82576)
b9473560
JB		 1519 netdev->features |= NETIF_F_SCTP_CSUM;
1520
330a6d6a 1521 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1522
1523 /* before reading the NVM, reset the controller to put the device in a
1524 * known good starting state */
1525 hw->mac.ops.reset_hw(hw);
1526
1527 /* make sure the NVM is good */
1528 if (igb_validate_nvm_checksum(hw) < 0) {
1529 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1530 err = -EIO;
1531 goto err_eeprom;
1532 }
1533
1534 /* copy the MAC address out of the NVM */
1535 if (hw->mac.ops.read_mac_addr(hw))
1536 dev_err(&pdev->dev, "NVM Read Error\n");
1537
1538 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1539 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1540
1541 if (!is_valid_ether_addr(netdev->perm_addr)) {
1542 dev_err(&pdev->dev, "Invalid MAC Address\n");
1543 err = -EIO;
1544 goto err_eeprom;
1545 }
1546
0e340485
AD		 1547 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1548 (unsigned long) adapter);
1549 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1550 (unsigned long) adapter);
9d5c8243
AK		 1551
1552 INIT_WORK(&adapter->reset_task, igb_reset_task);
1553 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1554
450c87c8 1555 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1556 adapter->fc_autoneg = true;
1557 hw->mac.autoneg = true;
1558 hw->phy.autoneg_advertised = 0x2f;
1559
0cce119a
AD		 1560 hw->fc.requested_mode = e1000_fc_default;
1561 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1562
9d5c8243
AK		 1563 igb_validate_mdi_setting(hw);
1564
9d5c8243
AK		 1565 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1566 * enable the ACPI Magic Packet filter
1567 */
1568
a2cf8b6c 1569 if (hw->bus.func == 0)
312c75ae 1570 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
55cac248
AD		 1571 else if (hw->mac.type == e1000_82580)
1572 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1573 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1574 &eeprom_data);
a2cf8b6c
AD		 1575 else if (hw->bus.func == 1)
1576 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1577
1578 if (eeprom_data & eeprom_apme_mask)
1579 adapter->eeprom_wol |= E1000_WUFC_MAG;
1580
1581 /* now that we have the eeprom settings, apply the special cases where
1582 * the eeprom may be wrong or the board simply won't support wake on
1583 * lan on a particular port */
1584 switch (pdev->device) {
1585 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1586 adapter->eeprom_wol = 0;
1587 break;
1588 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1589 case E1000_DEV_ID_82576_FIBER:
1590 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1591 /* Wake events only supported on port A for dual fiber
1592 * regardless of eeprom setting */
1593 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1594 adapter->eeprom_wol = 0;
1595 break;
c8ea5ea9 1596 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 1597 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 1598 /* if quad port adapter, disable WoL on all but port A */
1599 if (global_quad_port_a != 0)
1600 adapter->eeprom_wol = 0;
1601 else
1602 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1603 /* Reset for multiple quad port adapters */
1604 if (++global_quad_port_a == 4)
1605 global_quad_port_a = 0;
1606 break;
9d5c8243
AK		 1607 }
1608
1609 /* initialize the wol settings based on the eeprom settings */
1610 adapter->wol = adapter->eeprom_wol;
e1b86d84 1611 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1612
1613 /* reset the hardware with the new settings */
1614 igb_reset(adapter);
1615
1616 /* let the f/w know that the h/w is now under the control of the
1617 * driver. */
1618 igb_get_hw_control(adapter);
1619
9d5c8243
AK		 1620 strcpy(netdev->name, "eth%d");
1621 err = register_netdev(netdev);
1622 if (err)
1623 goto err_register;
1624
b168dfc5
JB		 1625 /* carrier off reporting is important to ethtool even BEFORE open */
1626 netif_carrier_off(netdev);
1627
421e02f0 1628#ifdef CONFIG_IGB_DCA
bbd98fe4 1629 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1630 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1631 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1632 igb_setup_dca(adapter);
1633 }
fe4506b6 1634
38c845c7 1635#endif
9d5c8243
AK		 1636 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1637 /* print bus type/speed/width info */
7c510e4b 1638 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 1639 netdev->name,
559e9c49
AD		 1640 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1641 "unknown"),
59c3de89
AD		 1642 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1643 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1644 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1645 "unknown"),
7c510e4b 1646 netdev->dev_addr);
9d5c8243
AK		 1647
1648 igb_read_part_num(hw, &part_num);
1649 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1650 (part_num >> 8), (part_num & 0xff));
1651
1652 dev_info(&pdev->dev,
1653 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1654 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1655 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1656 adapter->num_rx_queues, adapter->num_tx_queues);
1657
9d5c8243
AK		 1658 return 0;
1659
1660err_register:
1661 igb_release_hw_control(adapter);
1662err_eeprom:
1663 if (!igb_check_reset_block(hw))
f5f4cf08 1664 igb_reset_phy(hw);
9d5c8243
AK		 1665
1666 if (hw->flash_address)
1667 iounmap(hw->flash_address);
9d5c8243 1668err_sw_init:
047e0030 1669 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 1670 iounmap(hw->hw_addr);
1671err_ioremap:
1672 free_netdev(netdev);
1673err_alloc_etherdev:
559e9c49
AD		 1674 pci_release_selected_regions(pdev,
1675 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 1676err_pci_reg:
1677err_dma:
1678 pci_disable_device(pdev);
1679 return err;
1680}
1681
1682/**
1683 * igb_remove - Device Removal Routine
1684 * @pdev: PCI device information struct
1685 *
1686 * igb_remove is called by the PCI subsystem to alert the driver
1687 * that it should release a PCI device. The could be caused by a
1688 * Hot-Plug event, or because the driver is going to be removed from
1689 * memory.
1690 **/
1691static void __devexit igb_remove(struct pci_dev *pdev)
1692{
1693 struct net_device *netdev = pci_get_drvdata(pdev);
1694 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 1695 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1696
1697 /* flush_scheduled work may reschedule our watchdog task, so
1698 * explicitly disable watchdog tasks from being rescheduled */
1699 set_bit(__IGB_DOWN, &adapter->state);
1700 del_timer_sync(&adapter->watchdog_timer);
1701 del_timer_sync(&adapter->phy_info_timer);
1702
1703 flush_scheduled_work();
1704
421e02f0 1705#ifdef CONFIG_IGB_DCA
7dfc16fa 1706 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 1707 dev_info(&pdev->dev, "DCA disabled\n");
1708 dca_remove_requester(&pdev->dev);
7dfc16fa 1709 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 1710 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 1711 }
1712#endif
1713
9d5c8243
AK		 1714 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1715 * would have already happened in close and is redundant. */
1716 igb_release_hw_control(adapter);
1717
1718 unregister_netdev(netdev);
1719
047e0030 1720 igb_clear_interrupt_scheme(adapter);
9d5c8243 1721
37680117
AD		 1722#ifdef CONFIG_PCI_IOV
1723 /* reclaim resources allocated to VFs */
1724 if (adapter->vf_data) {
1725 /* disable iov and allow time for transactions to clear */
1726 pci_disable_sriov(pdev);
1727 msleep(500);
1728
1729 kfree(adapter->vf_data);
1730 adapter->vf_data = NULL;
1731 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1732 msleep(100);
1733 dev_info(&pdev->dev, "IOV Disabled\n");
1734 }
1735#endif
559e9c49 1736
28b0759c
AD		 1737 iounmap(hw->hw_addr);
1738 if (hw->flash_address)
1739 iounmap(hw->flash_address);
559e9c49
AD		 1740 pci_release_selected_regions(pdev,
1741 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 1742
1743 free_netdev(netdev);
1744
19d5afd4 1745 pci_disable_pcie_error_reporting(pdev);
40a914fa 1746
9d5c8243
AK		 1747 pci_disable_device(pdev);
1748}
1749
a6b623e0
AD		 1750/**
1751 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
1752 * @adapter: board private structure to initialize
1753 *
1754 * This function initializes the vf specific data storage and then attempts to
1755 * allocate the VFs. The reason for ordering it this way is because it is much
1756 * mor expensive time wise to disable SR-IOV than it is to allocate and free
1757 * the memory for the VFs.
1758 **/
1759static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
1760{
1761#ifdef CONFIG_PCI_IOV
1762 struct pci_dev *pdev = adapter->pdev;
1763
1764 if (adapter->vfs_allocated_count > 7)
1765 adapter->vfs_allocated_count = 7;
1766
1767 if (adapter->vfs_allocated_count) {
1768 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1769 sizeof(struct vf_data_storage),
1770 GFP_KERNEL);
1771 /* if allocation failed then we do not support SR-IOV */
1772 if (!adapter->vf_data) {
1773 adapter->vfs_allocated_count = 0;
1774 dev_err(&pdev->dev, "Unable to allocate memory for VF "
1775 "Data Storage\n");
1776 }
1777 }
1778
1779 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
1780 kfree(adapter->vf_data);
1781 adapter->vf_data = NULL;
1782#endif /* CONFIG_PCI_IOV */
1783 adapter->vfs_allocated_count = 0;
1784#ifdef CONFIG_PCI_IOV
1785 } else {
1786 unsigned char mac_addr[ETH_ALEN];
1787 int i;
1788 dev_info(&pdev->dev, "%d vfs allocated\n",
1789 adapter->vfs_allocated_count);
1790 for (i = 0; i < adapter->vfs_allocated_count; i++) {
1791 random_ether_addr(mac_addr);
1792 igb_set_vf_mac(adapter, i, mac_addr);
1793 }
1794 }
1795#endif /* CONFIG_PCI_IOV */
1796}
1797
115f459a
AD		 1798
1799/**
1800 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
1801 * @adapter: board private structure to initialize
1802 *
1803 * igb_init_hw_timer initializes the function pointer and values for the hw
1804 * timer found in hardware.
1805 **/
1806static void igb_init_hw_timer(struct igb_adapter *adapter)
1807{
1808 struct e1000_hw *hw = &adapter->hw;
1809
1810 switch (hw->mac.type) {
d2ba2ed8 1811 case e1000_i350:
55cac248
AD		 1812 case e1000_82580:
1813 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1814 adapter->cycles.read = igb_read_clock;
1815 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1816 adapter->cycles.mult = 1;
1817 /*
1818 * The 82580 timesync updates the system timer every 8ns by 8ns
1819 * and the value cannot be shifted. Instead we need to shift
1820 * the registers to generate a 64bit timer value. As a result
1821 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
1822 * 24 in order to generate a larger value for synchronization.
1823 */
1824 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
1825 /* disable system timer temporarily by setting bit 31 */
1826 wr32(E1000_TSAUXC, 0x80000000);
1827 wrfl();
1828
1829 /* Set registers so that rollover occurs soon to test this. */
1830 wr32(E1000_SYSTIMR, 0x00000000);
1831 wr32(E1000_SYSTIML, 0x80000000);
1832 wr32(E1000_SYSTIMH, 0x000000FF);
1833 wrfl();
1834
1835 /* enable system timer by clearing bit 31 */
1836 wr32(E1000_TSAUXC, 0x0);
1837 wrfl();
1838
1839 timecounter_init(&adapter->clock,
1840 &adapter->cycles,
1841 ktime_to_ns(ktime_get_real()));
1842 /*
1843 * Synchronize our NIC clock against system wall clock. NIC
1844 * time stamp reading requires ~3us per sample, each sample
1845 * was pretty stable even under load => only require 10
1846 * samples for each offset comparison.
1847 */
1848 memset(&adapter->compare, 0, sizeof(adapter->compare));
1849 adapter->compare.source = &adapter->clock;
1850 adapter->compare.target = ktime_get_real;
1851 adapter->compare.num_samples = 10;
1852 timecompare_update(&adapter->compare, 0);
1853 break;
115f459a
AD		 1854 case e1000_82576:
1855 /*
1856 * Initialize hardware timer: we keep it running just in case
1857 * that some program needs it later on.
1858 */
1859 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1860 adapter->cycles.read = igb_read_clock;
1861 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1862 adapter->cycles.mult = 1;
1863 /**
1864 * Scale the NIC clock cycle by a large factor so that
1865 * relatively small clock corrections can be added or
1866 * substracted at each clock tick. The drawbacks of a large
1867 * factor are a) that the clock register overflows more quickly
1868 * (not such a big deal) and b) that the increment per tick has
1869 * to fit into 24 bits. As a result we need to use a shift of
1870 * 19 so we can fit a value of 16 into the TIMINCA register.
1871 */
1872 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1873 wr32(E1000_TIMINCA,
1874 (1 << E1000_TIMINCA_16NS_SHIFT) |
1875 (16 << IGB_82576_TSYNC_SHIFT));
1876
1877 /* Set registers so that rollover occurs soon to test this. */
1878 wr32(E1000_SYSTIML, 0x00000000);
1879 wr32(E1000_SYSTIMH, 0xFF800000);
1880 wrfl();
1881
1882 timecounter_init(&adapter->clock,
1883 &adapter->cycles,
1884 ktime_to_ns(ktime_get_real()));
1885 /*
1886 * Synchronize our NIC clock against system wall clock. NIC
1887 * time stamp reading requires ~3us per sample, each sample
1888 * was pretty stable even under load => only require 10
1889 * samples for each offset comparison.
1890 */
1891 memset(&adapter->compare, 0, sizeof(adapter->compare));
1892 adapter->compare.source = &adapter->clock;
1893 adapter->compare.target = ktime_get_real;
1894 adapter->compare.num_samples = 10;
1895 timecompare_update(&adapter->compare, 0);
1896 break;
1897 case e1000_82575:
1898 /* 82575 does not support timesync */
1899 default:
1900 break;
1901 }
1902
1903}
1904
9d5c8243
AK		 1905/**
1906 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1907 * @adapter: board private structure to initialize
1908 *
1909 * igb_sw_init initializes the Adapter private data structure.
1910 * Fields are initialized based on PCI device information and
1911 * OS network device settings (MTU size).
1912 **/
1913static int __devinit igb_sw_init(struct igb_adapter *adapter)
1914{
1915 struct e1000_hw *hw = &adapter->hw;
1916 struct net_device *netdev = adapter->netdev;
1917 struct pci_dev *pdev = adapter->pdev;
1918
1919 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1920
68fd9910
AD		 1921 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1922 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 1923 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1924 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1925
9d5c8243
AK		 1926 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1927 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1928
a6b623e0
AD		 1929#ifdef CONFIG_PCI_IOV
1930 if (hw->mac.type == e1000_82576)
1931 adapter->vfs_allocated_count = max_vfs;
1932
1933#endif /* CONFIG_PCI_IOV */
a99955fc
AD		 1934 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
1935
1936 /*
1937 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
1938 * then we should combine the queues into a queue pair in order to
1939 * conserve interrupts due to limited supply
1940 */
1941 if ((adapter->rss_queues > 4) ||
1942 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
1943 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1944
a6b623e0 1945 /* This call may decrease the number of queues */
047e0030 1946 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 1947 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1948 return -ENOMEM;
1949 }
1950
115f459a 1951 igb_init_hw_timer(adapter);
a6b623e0
AD		 1952 igb_probe_vfs(adapter);
1953
9d5c8243
AK		 1954 /* Explicitly disable IRQ since the NIC can be in any state. */
1955 igb_irq_disable(adapter);
1956
1957 set_bit(__IGB_DOWN, &adapter->state);
1958 return 0;
1959}
1960
1961/**
1962 * igb_open - Called when a network interface is made active
1963 * @netdev: network interface device structure
1964 *
1965 * Returns 0 on success, negative value on failure
1966 *
1967 * The open entry point is called when a network interface is made
1968 * active by the system (IFF_UP). At this point all resources needed
1969 * for transmit and receive operations are allocated, the interrupt
1970 * handler is registered with the OS, the watchdog timer is started,
1971 * and the stack is notified that the interface is ready.
1972 **/
1973static int igb_open(struct net_device *netdev)
1974{
1975 struct igb_adapter *adapter = netdev_priv(netdev);
1976 struct e1000_hw *hw = &adapter->hw;
1977 int err;
1978 int i;
1979
1980 /* disallow open during test */
1981 if (test_bit(__IGB_TESTING, &adapter->state))
1982 return -EBUSY;
1983
b168dfc5
JB		 1984 netif_carrier_off(netdev);
1985
9d5c8243
AK		 1986 /* allocate transmit descriptors */
1987 err = igb_setup_all_tx_resources(adapter);
1988 if (err)
1989 goto err_setup_tx;
1990
1991 /* allocate receive descriptors */
1992 err = igb_setup_all_rx_resources(adapter);
1993 if (err)
1994 goto err_setup_rx;
1995
88a268c1 1996 igb_power_up_link(adapter);
9d5c8243 1997
9d5c8243
AK		 1998 /* before we allocate an interrupt, we must be ready to handle it.
1999 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2000 * as soon as we call pci_request_irq, so we have to setup our
2001 * clean_rx handler before we do so. */
2002 igb_configure(adapter);
2003
2004 err = igb_request_irq(adapter);
2005 if (err)
2006 goto err_req_irq;
2007
2008 /* From here on the code is the same as igb_up() */
2009 clear_bit(__IGB_DOWN, &adapter->state);
2010
047e0030
AD		 2011 for (i = 0; i < adapter->num_q_vectors; i++) {
2012 struct igb_q_vector *q_vector = adapter->q_vector[i];
2013 napi_enable(&q_vector->napi);
2014 }
9d5c8243
AK		 2015
2016 /* Clear any pending interrupts. */
2017 rd32(E1000_ICR);
844290e5
PW		 2018
2019 igb_irq_enable(adapter);
2020
d4960307
AD		 2021 /* notify VFs that reset has been completed */
2022 if (adapter->vfs_allocated_count) {
2023 u32 reg_data = rd32(E1000_CTRL_EXT);
2024 reg_data |= E1000_CTRL_EXT_PFRSTD;
2025 wr32(E1000_CTRL_EXT, reg_data);
2026 }
2027
d55b53ff
JK		 2028 netif_tx_start_all_queues(netdev);
2029
25568a53
AD		 2030 /* start the watchdog. */
2031 hw->mac.get_link_status = 1;
2032 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2033
2034 return 0;
2035
2036err_req_irq:
2037 igb_release_hw_control(adapter);
88a268c1 2038 igb_power_down_link(adapter);
9d5c8243
AK		 2039 igb_free_all_rx_resources(adapter);
2040err_setup_rx:
2041 igb_free_all_tx_resources(adapter);
2042err_setup_tx:
2043 igb_reset(adapter);
2044
2045 return err;
2046}
2047
2048/**
2049 * igb_close - Disables a network interface
2050 * @netdev: network interface device structure
2051 *
2052 * Returns 0, this is not allowed to fail
2053 *
2054 * The close entry point is called when an interface is de-activated
2055 * by the OS. The hardware is still under the driver's control, but
2056 * needs to be disabled. A global MAC reset is issued to stop the
2057 * hardware, and all transmit and receive resources are freed.
2058 **/
2059static int igb_close(struct net_device *netdev)
2060{
2061 struct igb_adapter *adapter = netdev_priv(netdev);
2062
2063 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2064 igb_down(adapter);
2065
2066 igb_free_irq(adapter);
2067
2068 igb_free_all_tx_resources(adapter);
2069 igb_free_all_rx_resources(adapter);
2070
9d5c8243
AK		 2071 return 0;
2072}
2073
2074/**
2075 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2076 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2077 *
2078 * Return 0 on success, negative on failure
2079 **/
80785298 2080int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2081{
80785298 2082 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 2083 int size;
2084
2085 size = sizeof(struct igb_buffer) * tx_ring->count;
2086 tx_ring->buffer_info = vmalloc(size);
2087 if (!tx_ring->buffer_info)
2088 goto err;
2089 memset(tx_ring->buffer_info, 0, size);
2090
2091 /* round up to nearest 4K */
85e8d004 2092 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2093 tx_ring->size = ALIGN(tx_ring->size, 4096);
2094
439705e1
AD		 2095 tx_ring->desc = pci_alloc_consistent(pdev,
2096 tx_ring->size,
9d5c8243
AK		 2097 &tx_ring->dma);
2098
2099 if (!tx_ring->desc)
2100 goto err;
2101
9d5c8243
AK		 2102 tx_ring->next_to_use = 0;
2103 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2104 return 0;
2105
2106err:
2107 vfree(tx_ring->buffer_info);
047e0030 2108 dev_err(&pdev->dev,
9d5c8243
AK		 2109 "Unable to allocate memory for the transmit descriptor ring\n");
2110 return -ENOMEM;
2111}
2112
2113/**
2114 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2115 * (Descriptors) for all queues
2116 * @adapter: board private structure
2117 *
2118 * Return 0 on success, negative on failure
2119 **/
2120static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2121{
439705e1 2122 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2123 int i, err = 0;
2124
2125 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2126 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2127 if (err) {
439705e1 2128 dev_err(&pdev->dev,
9d5c8243
AK		 2129 "Allocation for Tx Queue %u failed\n", i);
2130 for (i--; i >= 0; i--)
3025a446 2131 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2132 break;
2133 }
2134 }
2135
a99955fc 2136 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2137 int r_idx = i % adapter->num_tx_queues;
3025a446 2138 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2139 }
9d5c8243
AK		 2140 return err;
2141}
2142
2143/**
85b430b4
AD		 2144 * igb_setup_tctl - configure the transmit control registers
2145 * @adapter: Board private structure
9d5c8243 2146 **/
d7ee5b3a 2147void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2148{
9d5c8243
AK		 2149 struct e1000_hw *hw = &adapter->hw;
2150 u32 tctl;
9d5c8243 2151
85b430b4
AD		 2152 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2153 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2154
2155 /* Program the Transmit Control Register */
9d5c8243
AK		 2156 tctl = rd32(E1000_TCTL);
2157 tctl &= ~E1000_TCTL_CT;
2158 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2159 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2160
2161 igb_config_collision_dist(hw);
2162
9d5c8243
AK		 2163 /* Enable transmits */
2164 tctl |= E1000_TCTL_EN;
2165
2166 wr32(E1000_TCTL, tctl);
2167}
2168
85b430b4
AD		 2169/**
2170 * igb_configure_tx_ring - Configure transmit ring after Reset
2171 * @adapter: board private structure
2172 * @ring: tx ring to configure
2173 *
2174 * Configure a transmit ring after a reset.
2175 **/
d7ee5b3a
AD		 2176void igb_configure_tx_ring(struct igb_adapter *adapter,
2177 struct igb_ring *ring)
85b430b4
AD		 2178{
2179 struct e1000_hw *hw = &adapter->hw;
2180 u32 txdctl;
2181 u64 tdba = ring->dma;
2182 int reg_idx = ring->reg_idx;
2183
2184 /* disable the queue */
2185 txdctl = rd32(E1000_TXDCTL(reg_idx));
2186 wr32(E1000_TXDCTL(reg_idx),
2187 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2188 wrfl();
2189 mdelay(10);
2190
2191 wr32(E1000_TDLEN(reg_idx),
2192 ring->count * sizeof(union e1000_adv_tx_desc));
2193 wr32(E1000_TDBAL(reg_idx),
2194 tdba & 0x00000000ffffffffULL);
2195 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2196
fce99e34
AD		 2197 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2198 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2199 writel(0, ring->head);
2200 writel(0, ring->tail);
85b430b4
AD		 2201
2202 txdctl |= IGB_TX_PTHRESH;
2203 txdctl |= IGB_TX_HTHRESH << 8;
2204 txdctl |= IGB_TX_WTHRESH << 16;
2205
2206 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2207 wr32(E1000_TXDCTL(reg_idx), txdctl);
2208}
2209
2210/**
2211 * igb_configure_tx - Configure transmit Unit after Reset
2212 * @adapter: board private structure
2213 *
2214 * Configure the Tx unit of the MAC after a reset.
2215 **/
2216static void igb_configure_tx(struct igb_adapter *adapter)
2217{
2218 int i;
2219
2220 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2221 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2222}
2223
9d5c8243
AK		 2224/**
2225 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2226 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2227 *
2228 * Returns 0 on success, negative on failure
2229 **/
80785298 2230int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2231{
80785298 2232 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 2233 int size, desc_len;
2234
2235 size = sizeof(struct igb_buffer) * rx_ring->count;
2236 rx_ring->buffer_info = vmalloc(size);
2237 if (!rx_ring->buffer_info)
2238 goto err;
2239 memset(rx_ring->buffer_info, 0, size);
2240
2241 desc_len = sizeof(union e1000_adv_rx_desc);
2242
2243 /* Round up to nearest 4K */
2244 rx_ring->size = rx_ring->count * desc_len;
2245 rx_ring->size = ALIGN(rx_ring->size, 4096);
2246
2247 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2248 &rx_ring->dma);
2249
2250 if (!rx_ring->desc)
2251 goto err;
2252
2253 rx_ring->next_to_clean = 0;
2254 rx_ring->next_to_use = 0;
9d5c8243 2255
9d5c8243
AK		 2256 return 0;
2257
2258err:
2259 vfree(rx_ring->buffer_info);
439705e1 2260 rx_ring->buffer_info = NULL;
80785298 2261 dev_err(&pdev->dev, "Unable to allocate memory for "
9d5c8243
AK		 2262 "the receive descriptor ring\n");
2263 return -ENOMEM;
2264}
2265
2266/**
2267 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2268 * (Descriptors) for all queues
2269 * @adapter: board private structure
2270 *
2271 * Return 0 on success, negative on failure
2272 **/
2273static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2274{
439705e1 2275 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2276 int i, err = 0;
2277
2278 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2279 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2280 if (err) {
439705e1 2281 dev_err(&pdev->dev,
9d5c8243
AK		 2282 "Allocation for Rx Queue %u failed\n", i);
2283 for (i--; i >= 0; i--)
3025a446 2284 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2285 break;
2286 }
2287 }
2288
2289 return err;
2290}
2291
06cf2666
AD		 2292/**
2293 * igb_setup_mrqc - configure the multiple receive queue control registers
2294 * @adapter: Board private structure
2295 **/
2296static void igb_setup_mrqc(struct igb_adapter *adapter)
2297{
2298 struct e1000_hw *hw = &adapter->hw;
2299 u32 mrqc, rxcsum;
2300 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2301 union e1000_reta {
2302 u32 dword;
2303 u8 bytes[4];
2304 } reta;
2305 static const u8 rsshash[40] = {
2306 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2307 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2308 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2309 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2310
2311 /* Fill out hash function seeds */
2312 for (j = 0; j < 10; j++) {
2313 u32 rsskey = rsshash[(j * 4)];
2314 rsskey |= rsshash[(j * 4) + 1] << 8;
2315 rsskey |= rsshash[(j * 4) + 2] << 16;
2316 rsskey |= rsshash[(j * 4) + 3] << 24;
2317 array_wr32(E1000_RSSRK(0), j, rsskey);
2318 }
2319
a99955fc 2320 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2321
2322 if (adapter->vfs_allocated_count) {
2323 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2324 switch (hw->mac.type) {
d2ba2ed8 2325 case e1000_i350:
55cac248
AD		 2326 case e1000_82580:
2327 num_rx_queues = 1;
2328 shift = 0;
2329 break;
06cf2666
AD		 2330 case e1000_82576:
2331 shift = 3;
2332 num_rx_queues = 2;
2333 break;
2334 case e1000_82575:
2335 shift = 2;
2336 shift2 = 6;
2337 default:
2338 break;
2339 }
2340 } else {
2341 if (hw->mac.type == e1000_82575)
2342 shift = 6;
2343 }
2344
2345 for (j = 0; j < (32 * 4); j++) {
2346 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2347 if (shift2)
2348 reta.bytes[j & 3] |= num_rx_queues << shift2;
2349 if ((j & 3) == 3)
2350 wr32(E1000_RETA(j >> 2), reta.dword);
2351 }
2352
2353 /*
2354 * Disable raw packet checksumming so that RSS hash is placed in
2355 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2356 * offloads as they are enabled by default
2357 */
2358 rxcsum = rd32(E1000_RXCSUM);
2359 rxcsum |= E1000_RXCSUM_PCSD;
2360
2361 if (adapter->hw.mac.type >= e1000_82576)
2362 /* Enable Receive Checksum Offload for SCTP */
2363 rxcsum |= E1000_RXCSUM_CRCOFL;
2364
2365 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2366 wr32(E1000_RXCSUM, rxcsum);
2367
2368 /* If VMDq is enabled then we set the appropriate mode for that, else
2369 * we default to RSS so that an RSS hash is calculated per packet even
2370 * if we are only using one queue */
2371 if (adapter->vfs_allocated_count) {
2372 if (hw->mac.type > e1000_82575) {
2373 /* Set the default pool for the PF's first queue */
2374 u32 vtctl = rd32(E1000_VT_CTL);
2375 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2376 E1000_VT_CTL_DISABLE_DEF_POOL);
2377 vtctl |= adapter->vfs_allocated_count <<
2378 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2379 wr32(E1000_VT_CTL, vtctl);
2380 }
a99955fc 2381 if (adapter->rss_queues > 1)
06cf2666
AD		 2382 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2383 else
2384 mrqc = E1000_MRQC_ENABLE_VMDQ;
2385 } else {
2386 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2387 }
2388 igb_vmm_control(adapter);
2389
2390 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2391 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2392 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2393 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2394 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2395 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2396 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2397 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2398
2399 wr32(E1000_MRQC, mrqc);
2400}
2401
9d5c8243
AK		 2402/**
2403 * igb_setup_rctl - configure the receive control registers
2404 * @adapter: Board private structure
2405 **/
d7ee5b3a 2406void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2407{
2408 struct e1000_hw *hw = &adapter->hw;
2409 u32 rctl;
9d5c8243
AK		 2410
2411 rctl = rd32(E1000_RCTL);
2412
2413 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2414 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2415
69d728ba 2416 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2417 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2418
87cb7e8c
AK		 2419 /*
2420 * enable stripping of CRC. It's unlikely this will break BMC
2421 * redirection as it did with e1000. Newer features require
2422 * that the HW strips the CRC.
73cd78f1 2423 */
87cb7e8c 2424 rctl |= E1000_RCTL_SECRC;
9d5c8243 2425
559e9c49 2426 /* disable store bad packets and clear size bits. */
ec54d7d6 2427 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2428
6ec43fe6
AD		 2429 /* enable LPE to prevent packets larger than max_frame_size */
2430 rctl |= E1000_RCTL_LPE;
9d5c8243 2431
952f72a8
AD		 2432 /* disable queue 0 to prevent tail write w/o re-config */
2433 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2434
e1739522
AD		 2435 /* Attention!!! For SR-IOV PF driver operations you must enable
2436 * queue drop for all VF and PF queues to prevent head of line blocking
2437 * if an un-trusted VF does not provide descriptors to hardware.
2438 */
2439 if (adapter->vfs_allocated_count) {
e1739522
AD		 2440 /* set all queue drop enable bits */
2441 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2442 }
2443
9d5c8243
AK		 2444 wr32(E1000_RCTL, rctl);
2445}
2446
7d5753f0
AD		 2447static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2448 int vfn)
2449{
2450 struct e1000_hw *hw = &adapter->hw;
2451 u32 vmolr;
2452
2453 /* if it isn't the PF check to see if VFs are enabled and
2454 * increase the size to support vlan tags */
2455 if (vfn < adapter->vfs_allocated_count &&
2456 adapter->vf_data[vfn].vlans_enabled)
2457 size += VLAN_TAG_SIZE;
2458
2459 vmolr = rd32(E1000_VMOLR(vfn));
2460 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2461 vmolr |= size | E1000_VMOLR_LPE;
2462 wr32(E1000_VMOLR(vfn), vmolr);
2463
2464 return 0;
2465}
2466
e1739522
AD		 2467/**
2468 * igb_rlpml_set - set maximum receive packet size
2469 * @adapter: board private structure
2470 *
2471 * Configure maximum receivable packet size.
2472 **/
2473static void igb_rlpml_set(struct igb_adapter *adapter)
2474{
2475 u32 max_frame_size = adapter->max_frame_size;
2476 struct e1000_hw *hw = &adapter->hw;
2477 u16 pf_id = adapter->vfs_allocated_count;
2478
2479 if (adapter->vlgrp)
2480 max_frame_size += VLAN_TAG_SIZE;
2481
2482 /* if vfs are enabled we set RLPML to the largest possible request
2483 * size and set the VMOLR RLPML to the size we need */
2484 if (pf_id) {
2485 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2486 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2487 }
2488
2489 wr32(E1000_RLPML, max_frame_size);
2490}
2491
8151d294
WM		 2492static inline void igb_set_vmolr(struct igb_adapter *adapter,
2493 int vfn, bool aupe)
7d5753f0
AD		 2494{
2495 struct e1000_hw *hw = &adapter->hw;
2496 u32 vmolr;
2497
2498 /*
2499 * This register exists only on 82576 and newer so if we are older then
2500 * we should exit and do nothing
2501 */
2502 if (hw->mac.type < e1000_82576)
2503 return;
2504
2505 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2506 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2507 if (aupe)
2508 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2509 else
2510 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 2511
2512 /* clear all bits that might not be set */
2513 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2514
a99955fc 2515 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 2516 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2517 /*
2518 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2519 * multicast packets
2520 */
2521 if (vfn <= adapter->vfs_allocated_count)
2522 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2523
2524 wr32(E1000_VMOLR(vfn), vmolr);
2525}
2526
85b430b4
AD		 2527/**
2528 * igb_configure_rx_ring - Configure a receive ring after Reset
2529 * @adapter: board private structure
2530 * @ring: receive ring to be configured
2531 *
2532 * Configure the Rx unit of the MAC after a reset.
2533 **/
d7ee5b3a
AD		 2534void igb_configure_rx_ring(struct igb_adapter *adapter,
2535 struct igb_ring *ring)
85b430b4
AD		 2536{
2537 struct e1000_hw *hw = &adapter->hw;
2538 u64 rdba = ring->dma;
2539 int reg_idx = ring->reg_idx;
952f72a8 2540 u32 srrctl, rxdctl;
85b430b4
AD		 2541
2542 /* disable the queue */
2543 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2544 wr32(E1000_RXDCTL(reg_idx),
2545 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2546
2547 /* Set DMA base address registers */
2548 wr32(E1000_RDBAL(reg_idx),
2549 rdba & 0x00000000ffffffffULL);
2550 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2551 wr32(E1000_RDLEN(reg_idx),
2552 ring->count * sizeof(union e1000_adv_rx_desc));
2553
2554 /* initialize head and tail */
fce99e34
AD		 2555 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2556 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2557 writel(0, ring->head);
2558 writel(0, ring->tail);
85b430b4 2559
952f72a8 2560 /* set descriptor configuration */
4c844851
AD		 2561 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2562 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2563 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2564#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2565 srrctl |= IGB_RXBUFFER_16384 >>
2566 E1000_SRRCTL_BSIZEPKT_SHIFT;
2567#else
2568 srrctl |= (PAGE_SIZE / 2) >>
2569 E1000_SRRCTL_BSIZEPKT_SHIFT;
2570#endif
2571 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2572 } else {
4c844851 2573 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2574 E1000_SRRCTL_BSIZEPKT_SHIFT;
2575 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2576 }
757b77e2
NN		 2577 if (hw->mac.type == e1000_82580)
2578 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 2579 /* Only set Drop Enable if we are supporting multiple queues */
2580 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2581 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 2582
2583 wr32(E1000_SRRCTL(reg_idx), srrctl);
2584
7d5753f0 2585 /* set filtering for VMDQ pools */
8151d294 2586 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 2587
85b430b4
AD		 2588 /* enable receive descriptor fetching */
2589 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2590 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2591 rxdctl &= 0xFFF00000;
2592 rxdctl |= IGB_RX_PTHRESH;
2593 rxdctl |= IGB_RX_HTHRESH << 8;
2594 rxdctl |= IGB_RX_WTHRESH << 16;
2595 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2596}
2597
9d5c8243
AK		 2598/**
2599 * igb_configure_rx - Configure receive Unit after Reset
2600 * @adapter: board private structure
2601 *
2602 * Configure the Rx unit of the MAC after a reset.
2603 **/
2604static void igb_configure_rx(struct igb_adapter *adapter)
2605{
9107584e 2606 int i;
9d5c8243 2607
68d480c4
AD		 2608 /* set UTA to appropriate mode */
2609 igb_set_uta(adapter);
2610
26ad9178
AD		 2611 /* set the correct pool for the PF default MAC address in entry 0 */
2612 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2613 adapter->vfs_allocated_count);
2614
06cf2666
AD		 2615 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2616 * the Base and Length of the Rx Descriptor Ring */
2617 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2618 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 2619}
2620
2621/**
2622 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2623 * @tx_ring: Tx descriptor ring for a specific queue
2624 *
2625 * Free all transmit software resources
2626 **/
68fd9910 2627void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2628{
3b644cf6 2629 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2630
2631 vfree(tx_ring->buffer_info);
2632 tx_ring->buffer_info = NULL;
2633
439705e1
AD		 2634 /* if not set, then don't free */
2635 if (!tx_ring->desc)
2636 return;
2637
80785298
AD		 2638 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2639 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2640
2641 tx_ring->desc = NULL;
2642}
2643
2644/**
2645 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2646 * @adapter: board private structure
2647 *
2648 * Free all transmit software resources
2649 **/
2650static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2651{
2652 int i;
2653
2654 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2655 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2656}
2657
b1a436c3
AD		 2658void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2659 struct igb_buffer *buffer_info)
9d5c8243 2660{
6366ad33
AD		 2661 if (buffer_info->dma) {
2662 if (buffer_info->mapped_as_page)
2663 pci_unmap_page(tx_ring->pdev,
2664 buffer_info->dma,
2665 buffer_info->length,
2666 PCI_DMA_TODEVICE);
2667 else
2668 pci_unmap_single(tx_ring->pdev,
2669 buffer_info->dma,
2670 buffer_info->length,
2671 PCI_DMA_TODEVICE);
2672 buffer_info->dma = 0;
2673 }
9d5c8243
AK		 2674 if (buffer_info->skb) {
2675 dev_kfree_skb_any(buffer_info->skb);
2676 buffer_info->skb = NULL;
2677 }
2678 buffer_info->time_stamp = 0;
6366ad33
AD		 2679 buffer_info->length = 0;
2680 buffer_info->next_to_watch = 0;
2681 buffer_info->mapped_as_page = false;
9d5c8243
AK		 2682}
2683
2684/**
2685 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 2686 * @tx_ring: ring to be cleaned
2687 **/
3b644cf6 2688static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 2689{
2690 struct igb_buffer *buffer_info;
2691 unsigned long size;
2692 unsigned int i;
2693
2694 if (!tx_ring->buffer_info)
2695 return;
2696 /* Free all the Tx ring sk_buffs */
2697
2698 for (i = 0; i < tx_ring->count; i++) {
2699 buffer_info = &tx_ring->buffer_info[i];
80785298 2700 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 2701 }
2702
2703 size = sizeof(struct igb_buffer) * tx_ring->count;
2704 memset(tx_ring->buffer_info, 0, size);
2705
2706 /* Zero out the descriptor ring */
9d5c8243
AK		 2707 memset(tx_ring->desc, 0, tx_ring->size);
2708
2709 tx_ring->next_to_use = 0;
2710 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2711}
2712
2713/**
2714 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2715 * @adapter: board private structure
2716 **/
2717static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2718{
2719 int i;
2720
2721 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2722 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 2723}
2724
2725/**
2726 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 2727 * @rx_ring: ring to clean the resources from
2728 *
2729 * Free all receive software resources
2730 **/
68fd9910 2731void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2732{
3b644cf6 2733 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 2734
2735 vfree(rx_ring->buffer_info);
2736 rx_ring->buffer_info = NULL;
2737
439705e1
AD		 2738 /* if not set, then don't free */
2739 if (!rx_ring->desc)
2740 return;
2741
80785298
AD		 2742 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2743 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 2744
2745 rx_ring->desc = NULL;
2746}
2747
2748/**
2749 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2750 * @adapter: board private structure
2751 *
2752 * Free all receive software resources
2753 **/
2754static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2755{
2756 int i;
2757
2758 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2759 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2760}
2761
2762/**
2763 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 2764 * @rx_ring: ring to free buffers from
2765 **/
3b644cf6 2766static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 2767{
2768 struct igb_buffer *buffer_info;
9d5c8243
AK		 2769 unsigned long size;
2770 unsigned int i;
2771
2772 if (!rx_ring->buffer_info)
2773 return;
439705e1 2774
9d5c8243
AK		 2775 /* Free all the Rx ring sk_buffs */
2776 for (i = 0; i < rx_ring->count; i++) {
2777 buffer_info = &rx_ring->buffer_info[i];
2778 if (buffer_info->dma) {
80785298
AD		 2779 pci_unmap_single(rx_ring->pdev,
2780 buffer_info->dma,
4c844851 2781 rx_ring->rx_buffer_len,
6ec43fe6 2782 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 2783 buffer_info->dma = 0;
2784 }
2785
2786 if (buffer_info->skb) {
2787 dev_kfree_skb(buffer_info->skb);
2788 buffer_info->skb = NULL;
2789 }
6ec43fe6 2790 if (buffer_info->page_dma) {
80785298
AD		 2791 pci_unmap_page(rx_ring->pdev,
2792 buffer_info->page_dma,
6ec43fe6
AD		 2793 PAGE_SIZE / 2,
2794 PCI_DMA_FROMDEVICE);
2795 buffer_info->page_dma = 0;
2796 }
9d5c8243 2797 if (buffer_info->page) {
9d5c8243
AK		 2798 put_page(buffer_info->page);
2799 buffer_info->page = NULL;
bf36c1a0 2800 buffer_info->page_offset = 0;
9d5c8243
AK		 2801 }
2802 }
2803
9d5c8243
AK		 2804 size = sizeof(struct igb_buffer) * rx_ring->count;
2805 memset(rx_ring->buffer_info, 0, size);
2806
2807 /* Zero out the descriptor ring */
2808 memset(rx_ring->desc, 0, rx_ring->size);
2809
2810 rx_ring->next_to_clean = 0;
2811 rx_ring->next_to_use = 0;
9d5c8243
AK		 2812}
2813
2814/**
2815 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2816 * @adapter: board private structure
2817 **/
2818static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2819{
2820 int i;
2821
2822 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2823 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 2824}
2825
2826/**
2827 * igb_set_mac - Change the Ethernet Address of the NIC
2828 * @netdev: network interface device structure
2829 * @p: pointer to an address structure
2830 *
2831 * Returns 0 on success, negative on failure
2832 **/
2833static int igb_set_mac(struct net_device *netdev, void *p)
2834{
2835 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 2836 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2837 struct sockaddr *addr = p;
2838
2839 if (!is_valid_ether_addr(addr->sa_data))
2840 return -EADDRNOTAVAIL;
2841
2842 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 2843 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 2844
26ad9178
AD		 2845 /* set the correct pool for the new PF MAC address in entry 0 */
2846 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2847 adapter->vfs_allocated_count);
e1739522 2848
9d5c8243
AK		 2849 return 0;
2850}
2851
2852/**
68d480c4 2853 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 2854 * @netdev: network interface device structure
2855 *
68d480c4
AD		 2856 * Writes multicast address list to the MTA hash table.
2857 * Returns: -ENOMEM on failure
2858 * 0 on no addresses written
2859 * X on writing X addresses to MTA
9d5c8243 2860 **/
68d480c4 2861static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 2862{
2863 struct igb_adapter *adapter = netdev_priv(netdev);
2864 struct e1000_hw *hw = &adapter->hw;
22bedad3 2865 struct netdev_hw_addr *ha;
68d480c4 2866 u8 *mta_list;
9d5c8243
AK		 2867 int i;
2868
4cd24eaf 2869 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 2870 /* nothing to program, so clear mc list */
2871 igb_update_mc_addr_list(hw, NULL, 0);
2872 igb_restore_vf_multicasts(adapter);
2873 return 0;
2874 }
9d5c8243 2875
4cd24eaf 2876 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 2877 if (!mta_list)
2878 return -ENOMEM;
ff41f8dc 2879
68d480c4 2880 /* The shared function expects a packed array of only addresses. */
48e2f183 2881 i = 0;
22bedad3
JP		 2882 netdev_for_each_mc_addr(ha, netdev)
2883 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 2884
68d480c4
AD		 2885 igb_update_mc_addr_list(hw, mta_list, i);
2886 kfree(mta_list);
2887
4cd24eaf 2888 return netdev_mc_count(netdev);
68d480c4
AD		 2889}
2890
2891/**
2892 * igb_write_uc_addr_list - write unicast addresses to RAR table
2893 * @netdev: network interface device structure
2894 *
2895 * Writes unicast address list to the RAR table.
2896 * Returns: -ENOMEM on failure/insufficient address space
2897 * 0 on no addresses written
2898 * X on writing X addresses to the RAR table
2899 **/
2900static int igb_write_uc_addr_list(struct net_device *netdev)
2901{
2902 struct igb_adapter *adapter = netdev_priv(netdev);
2903 struct e1000_hw *hw = &adapter->hw;
2904 unsigned int vfn = adapter->vfs_allocated_count;
2905 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2906 int count = 0;
2907
2908 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 2909 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 2910 return -ENOMEM;
9d5c8243 2911
32e7bfc4 2912 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 2913 struct netdev_hw_addr *ha;
32e7bfc4
JP		 2914
2915 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 2916 if (!rar_entries)
2917 break;
26ad9178
AD		 2918 igb_rar_set_qsel(adapter, ha->addr,
2919 rar_entries--,
68d480c4
AD		 2920 vfn);
2921 count++;
ff41f8dc
AD		 2922 }
2923 }
2924 /* write the addresses in reverse order to avoid write combining */
2925 for (; rar_entries > 0 ; rar_entries--) {
2926 wr32(E1000_RAH(rar_entries), 0);
2927 wr32(E1000_RAL(rar_entries), 0);
2928 }
2929 wrfl();
2930
68d480c4
AD		 2931 return count;
2932}
2933
2934/**
2935 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2936 * @netdev: network interface device structure
2937 *
2938 * The set_rx_mode entry point is called whenever the unicast or multicast
2939 * address lists or the network interface flags are updated. This routine is
2940 * responsible for configuring the hardware for proper unicast, multicast,
2941 * promiscuous mode, and all-multi behavior.
2942 **/
2943static void igb_set_rx_mode(struct net_device *netdev)
2944{
2945 struct igb_adapter *adapter = netdev_priv(netdev);
2946 struct e1000_hw *hw = &adapter->hw;
2947 unsigned int vfn = adapter->vfs_allocated_count;
2948 u32 rctl, vmolr = 0;
2949 int count;
2950
2951 /* Check for Promiscuous and All Multicast modes */
2952 rctl = rd32(E1000_RCTL);
2953
2954 /* clear the effected bits */
2955 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2956
2957 if (netdev->flags & IFF_PROMISC) {
2958 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2959 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2960 } else {
2961 if (netdev->flags & IFF_ALLMULTI) {
2962 rctl |= E1000_RCTL_MPE;
2963 vmolr |= E1000_VMOLR_MPME;
2964 } else {
2965 /*
2966 * Write addresses to the MTA, if the attempt fails
2967 * then we should just turn on promiscous mode so
2968 * that we can at least receive multicast traffic
2969 */
2970 count = igb_write_mc_addr_list(netdev);
2971 if (count < 0) {
2972 rctl |= E1000_RCTL_MPE;
2973 vmolr |= E1000_VMOLR_MPME;
2974 } else if (count) {
2975 vmolr |= E1000_VMOLR_ROMPE;
2976 }
2977 }
2978 /*
2979 * Write addresses to available RAR registers, if there is not
2980 * sufficient space to store all the addresses then enable
2981 * unicast promiscous mode
2982 */
2983 count = igb_write_uc_addr_list(netdev);
2984 if (count < 0) {
2985 rctl |= E1000_RCTL_UPE;
2986 vmolr |= E1000_VMOLR_ROPE;
2987 }
2988 rctl |= E1000_RCTL_VFE;
28fc06f5 2989 }
68d480c4 2990 wr32(E1000_RCTL, rctl);
28fc06f5 2991
68d480c4
AD		 2992 /*
2993 * In order to support SR-IOV and eventually VMDq it is necessary to set
2994 * the VMOLR to enable the appropriate modes. Without this workaround
2995 * we will have issues with VLAN tag stripping not being done for frames
2996 * that are only arriving because we are the default pool
2997 */
2998 if (hw->mac.type < e1000_82576)
28fc06f5 2999 return;
9d5c8243 3000
68d480c4
AD		 3001 vmolr |= rd32(E1000_VMOLR(vfn)) &
3002 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3003 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3004 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3005}
3006
3007/* Need to wait a few seconds after link up to get diagnostic information from
3008 * the phy */
3009static void igb_update_phy_info(unsigned long data)
3010{
3011 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3012 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3013}
3014
4d6b725e
AD		 3015/**
3016 * igb_has_link - check shared code for link and determine up/down
3017 * @adapter: pointer to driver private info
3018 **/
3145535a 3019bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3020{
3021 struct e1000_hw *hw = &adapter->hw;
3022 bool link_active = false;
3023 s32 ret_val = 0;
3024
3025 /* get_link_status is set on LSC (link status) interrupt or
3026 * rx sequence error interrupt. get_link_status will stay
3027 * false until the e1000_check_for_link establishes link
3028 * for copper adapters ONLY
3029 */
3030 switch (hw->phy.media_type) {
3031 case e1000_media_type_copper:
3032 if (hw->mac.get_link_status) {
3033 ret_val = hw->mac.ops.check_for_link(hw);
3034 link_active = !hw->mac.get_link_status;
3035 } else {
3036 link_active = true;
3037 }
3038 break;
4d6b725e
AD		 3039 case e1000_media_type_internal_serdes:
3040 ret_val = hw->mac.ops.check_for_link(hw);
3041 link_active = hw->mac.serdes_has_link;
3042 break;
3043 default:
3044 case e1000_media_type_unknown:
3045 break;
3046 }
3047
3048 return link_active;
3049}
3050
9d5c8243
AK		 3051/**
3052 * igb_watchdog - Timer Call-back
3053 * @data: pointer to adapter cast into an unsigned long
3054 **/
3055static void igb_watchdog(unsigned long data)
3056{
3057 struct igb_adapter *adapter = (struct igb_adapter *)data;
3058 /* Do the rest outside of interrupt context */
3059 schedule_work(&adapter->watchdog_task);
3060}
3061
3062static void igb_watchdog_task(struct work_struct *work)
3063{
3064 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3065 struct igb_adapter,
3066 watchdog_task);
9d5c8243 3067 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3068 struct net_device *netdev = adapter->netdev;
9d5c8243 3069 u32 link;
7a6ea550 3070 int i;
9d5c8243 3071
4d6b725e 3072 link = igb_has_link(adapter);
9d5c8243
AK		 3073 if (link) {
3074 if (!netif_carrier_ok(netdev)) {
3075 u32 ctrl;
330a6d6a
AD		 3076 hw->mac.ops.get_speed_and_duplex(hw,
3077 &adapter->link_speed,
3078 &adapter->link_duplex);
9d5c8243
AK		 3079
3080 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3081 /* Links status message must follow this format */
3082 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3083 "Flow Control: %s\n",
559e9c49
AD		 3084 netdev->name,
3085 adapter->link_speed,
3086 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3087 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3088 ((ctrl & E1000_CTRL_TFCE) &&
3089 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3090 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3091 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3092
d07f3e37 3093 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3094 adapter->tx_timeout_factor = 1;
3095 switch (adapter->link_speed) {
3096 case SPEED_10:
9d5c8243
AK		 3097 adapter->tx_timeout_factor = 14;
3098 break;
3099 case SPEED_100:
9d5c8243
AK		 3100 /* maybe add some timeout factor ? */
3101 break;
3102 }
3103
3104 netif_carrier_on(netdev);
9d5c8243 3105
4ae196df
AD		 3106 igb_ping_all_vfs(adapter);
3107
4b1a9877 3108 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3109 if (!test_bit(__IGB_DOWN, &adapter->state))
3110 mod_timer(&adapter->phy_info_timer,
3111 round_jiffies(jiffies + 2 * HZ));
3112 }
3113 } else {
3114 if (netif_carrier_ok(netdev)) {
3115 adapter->link_speed = 0;
3116 adapter->link_duplex = 0;
527d47c1
AD		 3117 /* Links status message must follow this format */
3118 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3119 netdev->name);
9d5c8243 3120 netif_carrier_off(netdev);
4b1a9877 3121
4ae196df
AD		 3122 igb_ping_all_vfs(adapter);
3123
4b1a9877 3124 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3125 if (!test_bit(__IGB_DOWN, &adapter->state))
3126 mod_timer(&adapter->phy_info_timer,
3127 round_jiffies(jiffies + 2 * HZ));
3128 }
3129 }
3130
9d5c8243 3131 igb_update_stats(adapter);
9d5c8243 3132
dbabb065 3133 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3134 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3135 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3136 /* We've lost link, so the controller stops DMA,
3137 * but we've got queued Tx work that's never going
3138 * to get done, so reset controller to flush Tx.
3139 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3140 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3141 adapter->tx_timeout_count++;
3142 schedule_work(&adapter->reset_task);
3143 /* return immediately since reset is imminent */
3144 return;
3145 }
9d5c8243 3146 }
9d5c8243 3147
dbabb065
AD		 3148 /* Force detection of hung controller every watchdog period */
3149 tx_ring->detect_tx_hung = true;
3150 }
f7ba205e 3151
9d5c8243 3152 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3153 if (adapter->msix_entries) {
047e0030
AD		 3154 u32 eics = 0;
3155 for (i = 0; i < adapter->num_q_vectors; i++) {
3156 struct igb_q_vector *q_vector = adapter->q_vector[i];
3157 eics |= q_vector->eims_value;
3158 }
7a6ea550
AD		 3159 wr32(E1000_EICS, eics);
3160 } else {
3161 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3162 }
9d5c8243 3163
9d5c8243
AK		 3164 /* Reset the timer */
3165 if (!test_bit(__IGB_DOWN, &adapter->state))
3166 mod_timer(&adapter->watchdog_timer,
3167 round_jiffies(jiffies + 2 * HZ));
3168}
3169
3170enum latency_range {
3171 lowest_latency = 0,
3172 low_latency = 1,
3173 bulk_latency = 2,
3174 latency_invalid = 255
3175};
3176
6eb5a7f1
AD		 3177/**
3178 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3179 *
3180 * Stores a new ITR value based on strictly on packet size. This
3181 * algorithm is less sophisticated than that used in igb_update_itr,
3182 * due to the difficulty of synchronizing statistics across multiple
3183 * receive rings. The divisors and thresholds used by this fuction
3184 * were determined based on theoretical maximum wire speed and testing
3185 * data, in order to minimize response time while increasing bulk
3186 * throughput.
3187 * This functionality is controlled by the InterruptThrottleRate module
3188 * parameter (see igb_param.c)
3189 * NOTE: This function is called only when operating in a multiqueue
3190 * receive environment.
047e0030 3191 * @q_vector: pointer to q_vector
6eb5a7f1 3192 **/
047e0030 3193static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3194{
047e0030 3195 int new_val = q_vector->itr_val;
6eb5a7f1 3196 int avg_wire_size = 0;
047e0030 3197 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3198
6eb5a7f1
AD		 3199 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3200 * ints/sec - ITR timer value of 120 ticks.
3201 */
3202 if (adapter->link_speed != SPEED_1000) {
047e0030 3203 new_val = 976;
6eb5a7f1 3204 goto set_itr_val;
9d5c8243 3205 }
047e0030
AD		 3206
3207 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3208 struct igb_ring *ring = q_vector->rx_ring;
3209 avg_wire_size = ring->total_bytes / ring->total_packets;
3210 }
3211
3212 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3213 struct igb_ring *ring = q_vector->tx_ring;
3214 avg_wire_size = max_t(u32, avg_wire_size,
3215 (ring->total_bytes /
3216 ring->total_packets));
3217 }
3218
3219 /* if avg_wire_size isn't set no work was done */
3220 if (!avg_wire_size)
3221 goto clear_counts;
9d5c8243 3222
6eb5a7f1
AD		 3223 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3224 avg_wire_size += 24;
3225
3226 /* Don't starve jumbo frames */
3227 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3228
6eb5a7f1
AD		 3229 /* Give a little boost to mid-size frames */
3230 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3231 new_val = avg_wire_size / 3;
3232 else
3233 new_val = avg_wire_size / 2;
9d5c8243 3234
abe1c363
NN		 3235 /* when in itr mode 3 do not exceed 20K ints/sec */
3236 if (adapter->rx_itr_setting == 3 && new_val < 196)
3237 new_val = 196;
3238
6eb5a7f1 3239set_itr_val:
047e0030
AD		 3240 if (new_val != q_vector->itr_val) {
3241 q_vector->itr_val = new_val;
3242 q_vector->set_itr = 1;
9d5c8243 3243 }
6eb5a7f1 3244clear_counts:
047e0030
AD		 3245 if (q_vector->rx_ring) {
3246 q_vector->rx_ring->total_bytes = 0;
3247 q_vector->rx_ring->total_packets = 0;
3248 }
3249 if (q_vector->tx_ring) {
3250 q_vector->tx_ring->total_bytes = 0;
3251 q_vector->tx_ring->total_packets = 0;
3252 }
9d5c8243
AK		 3253}
3254
3255/**
3256 * igb_update_itr - update the dynamic ITR value based on statistics
3257 * Stores a new ITR value based on packets and byte
3258 * counts during the last interrupt. The advantage of per interrupt
3259 * computation is faster updates and more accurate ITR for the current
3260 * traffic pattern. Constants in this function were computed
3261 * based on theoretical maximum wire speed and thresholds were set based
3262 * on testing data as well as attempting to minimize response time
3263 * while increasing bulk throughput.
3264 * this functionality is controlled by the InterruptThrottleRate module
3265 * parameter (see igb_param.c)
3266 * NOTE: These calculations are only valid when operating in a single-
3267 * queue environment.
3268 * @adapter: pointer to adapter
047e0030 3269 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3270 * @packets: the number of packets during this measurement interval
3271 * @bytes: the number of bytes during this measurement interval
3272 **/
3273static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3274 int packets, int bytes)
3275{
3276 unsigned int retval = itr_setting;
3277
3278 if (packets == 0)
3279 goto update_itr_done;
3280
3281 switch (itr_setting) {
3282 case lowest_latency:
3283 /* handle TSO and jumbo frames */
3284 if (bytes/packets > 8000)
3285 retval = bulk_latency;
3286 else if ((packets < 5) && (bytes > 512))
3287 retval = low_latency;
3288 break;
3289 case low_latency: /* 50 usec aka 20000 ints/s */
3290 if (bytes > 10000) {
3291 /* this if handles the TSO accounting */
3292 if (bytes/packets > 8000) {
3293 retval = bulk_latency;
3294 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3295 retval = bulk_latency;
3296 } else if ((packets > 35)) {
3297 retval = lowest_latency;
3298 }
3299 } else if (bytes/packets > 2000) {
3300 retval = bulk_latency;
3301 } else if (packets <= 2 && bytes < 512) {
3302 retval = lowest_latency;
3303 }
3304 break;
3305 case bulk_latency: /* 250 usec aka 4000 ints/s */
3306 if (bytes > 25000) {
3307 if (packets > 35)
3308 retval = low_latency;
1e5c3d21 3309 } else if (bytes < 1500) {
9d5c8243
AK		 3310 retval = low_latency;
3311 }
3312 break;
3313 }
3314
3315update_itr_done:
3316 return retval;
3317}
3318
6eb5a7f1 3319static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3320{
047e0030 3321 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3322 u16 current_itr;
047e0030 3323 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3324
3325 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3326 if (adapter->link_speed != SPEED_1000) {
3327 current_itr = 0;
3328 new_itr = 4000;
3329 goto set_itr_now;
3330 }
3331
3332 adapter->rx_itr = igb_update_itr(adapter,
3333 adapter->rx_itr,
3025a446
AD		 3334 q_vector->rx_ring->total_packets,
3335 q_vector->rx_ring->total_bytes);
9d5c8243 3336
047e0030
AD		 3337 adapter->tx_itr = igb_update_itr(adapter,
3338 adapter->tx_itr,
3025a446
AD		 3339 q_vector->tx_ring->total_packets,
3340 q_vector->tx_ring->total_bytes);
047e0030 3341 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3342
6eb5a7f1 3343 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3344 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3345 current_itr = low_latency;
3346
9d5c8243
AK		 3347 switch (current_itr) {
3348 /* counts and packets in update_itr are dependent on these numbers */
3349 case lowest_latency:
78b1f607 3350 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3351 break;
3352 case low_latency:
78b1f607 3353 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3354 break;
3355 case bulk_latency:
78b1f607 3356 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3357 break;
3358 default:
3359 break;
3360 }
3361
3362set_itr_now:
3025a446
AD		 3363 q_vector->rx_ring->total_bytes = 0;
3364 q_vector->rx_ring->total_packets = 0;
3365 q_vector->tx_ring->total_bytes = 0;
3366 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3367
047e0030 3368 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3369 /* this attempts to bias the interrupt rate towards Bulk
3370 * by adding intermediate steps when interrupt rate is
3371 * increasing */
047e0030
AD		 3372 new_itr = new_itr > q_vector->itr_val ?
3373 max((new_itr * q_vector->itr_val) /
3374 (new_itr + (q_vector->itr_val >> 2)),
3375 new_itr) :
9d5c8243
AK		 3376 new_itr;
3377 /* Don't write the value here; it resets the adapter's
3378 * internal timer, and causes us to delay far longer than
3379 * we should between interrupts. Instead, we write the ITR
3380 * value at the beginning of the next interrupt so the timing
3381 * ends up being correct.
3382 */
047e0030
AD		 3383 q_vector->itr_val = new_itr;
3384 q_vector->set_itr = 1;
9d5c8243
AK		 3385 }
3386
3387 return;
3388}
3389
9d5c8243
AK		 3390#define IGB_TX_FLAGS_CSUM 0x00000001
3391#define IGB_TX_FLAGS_VLAN 0x00000002
3392#define IGB_TX_FLAGS_TSO 0x00000004
3393#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3394#define IGB_TX_FLAGS_TSTAMP 0x00000010
3395#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3396#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3397
85ad76b2 3398static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3399 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3400{
3401 struct e1000_adv_tx_context_desc *context_desc;
3402 unsigned int i;
3403 int err;
3404 struct igb_buffer *buffer_info;
3405 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3406 u32 mss_l4len_idx;
3407 u8 l4len;
9d5c8243
AK		 3408
3409 if (skb_header_cloned(skb)) {
3410 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3411 if (err)
3412 return err;
3413 }
3414
3415 l4len = tcp_hdrlen(skb);
3416 *hdr_len += l4len;
3417
3418 if (skb->protocol == htons(ETH_P_IP)) {
3419 struct iphdr *iph = ip_hdr(skb);
3420 iph->tot_len = 0;
3421 iph->check = 0;
3422 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3423 iph->daddr, 0,
3424 IPPROTO_TCP,
3425 0);
8e1e8a47 3426 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3427 ipv6_hdr(skb)->payload_len = 0;
3428 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3429 &ipv6_hdr(skb)->daddr,
3430 0, IPPROTO_TCP, 0);
3431 }
3432
3433 i = tx_ring->next_to_use;
3434
3435 buffer_info = &tx_ring->buffer_info[i];
3436 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3437 /* VLAN MACLEN IPLEN */
3438 if (tx_flags & IGB_TX_FLAGS_VLAN)
3439 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3440 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3441 *hdr_len += skb_network_offset(skb);
3442 info |= skb_network_header_len(skb);
3443 *hdr_len += skb_network_header_len(skb);
3444 context_desc->vlan_macip_lens = cpu_to_le32(info);
3445
3446 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3447 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3448
3449 if (skb->protocol == htons(ETH_P_IP))
3450 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3451 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3452
3453 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3454
3455 /* MSS L4LEN IDX */
3456 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3457 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3458
73cd78f1 3459 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3460 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3461 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3462
3463 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3464 context_desc->seqnum_seed = 0;
3465
3466 buffer_info->time_stamp = jiffies;
0e014cb1 3467 buffer_info->next_to_watch = i;
9d5c8243
AK		 3468 buffer_info->dma = 0;
3469 i++;
3470 if (i == tx_ring->count)
3471 i = 0;
3472
3473 tx_ring->next_to_use = i;
3474
3475 return true;
3476}
3477
85ad76b2
AD		 3478static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3479 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3480{
3481 struct e1000_adv_tx_context_desc *context_desc;
80785298 3482 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3483 struct igb_buffer *buffer_info;
3484 u32 info = 0, tu_cmd = 0;
80785298 3485 unsigned int i;
9d5c8243
AK		 3486
3487 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3488 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3489 i = tx_ring->next_to_use;
3490 buffer_info = &tx_ring->buffer_info[i];
3491 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3492
3493 if (tx_flags & IGB_TX_FLAGS_VLAN)
3494 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3495
9d5c8243
AK		 3496 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3497 if (skb->ip_summed == CHECKSUM_PARTIAL)
3498 info |= skb_network_header_len(skb);
3499
3500 context_desc->vlan_macip_lens = cpu_to_le32(info);
3501
3502 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3503
3504 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3505 __be16 protocol;
3506
3507 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3508 const struct vlan_ethhdr *vhdr =
3509 (const struct vlan_ethhdr*)skb->data;
3510
3511 protocol = vhdr->h_vlan_encapsulated_proto;
3512 } else {
3513 protocol = skb->protocol;
3514 }
3515
3516 switch (protocol) {
09640e63 3517 case cpu_to_be16(ETH_P_IP):
9d5c8243 3518 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3519 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3520 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3521 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3522 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3523 break;
09640e63 3524 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3525 /* XXX what about other V6 headers?? */
3526 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3527 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3528 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3529 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3530 break;
3531 default:
3532 if (unlikely(net_ratelimit()))
80785298 3533 dev_warn(&pdev->dev,
44b0cda3
MW		 3534 "partial checksum but proto=%x!\n",
3535 skb->protocol);
3536 break;
3537 }
9d5c8243
AK		 3538 }
3539
3540 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3541 context_desc->seqnum_seed = 0;
85ad76b2 3542 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3543 context_desc->mss_l4len_idx =
85ad76b2 3544 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3545
3546 buffer_info->time_stamp = jiffies;
0e014cb1 3547 buffer_info->next_to_watch = i;
9d5c8243
AK		 3548 buffer_info->dma = 0;
3549
3550 i++;
3551 if (i == tx_ring->count)
3552 i = 0;
3553 tx_ring->next_to_use = i;
3554
3555 return true;
3556 }
9d5c8243
AK		 3557 return false;
3558}
3559
3560#define IGB_MAX_TXD_PWR 16
3561#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3562
80785298 3563static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3564 unsigned int first)
9d5c8243
AK		 3565{
3566 struct igb_buffer *buffer_info;
80785298 3567 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3568 unsigned int len = skb_headlen(skb);
3569 unsigned int count = 0, i;
3570 unsigned int f;
3571
3572 i = tx_ring->next_to_use;
3573
3574 buffer_info = &tx_ring->buffer_info[i];
3575 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3576 buffer_info->length = len;
3577 /* set time_stamp *before* dma to help avoid a possible race */
3578 buffer_info->time_stamp = jiffies;
0e014cb1 3579 buffer_info->next_to_watch = i;
6366ad33
AD		 3580 buffer_info->dma = pci_map_single(pdev, skb->data, len,
3581 PCI_DMA_TODEVICE);
3582 if (pci_dma_mapping_error(pdev, buffer_info->dma))
3583 goto dma_error;
9d5c8243
AK		 3584
3585 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3586 struct skb_frag_struct *frag;
3587
8581145f 3588 count++;
65689fef
AD		 3589 i++;
3590 if (i == tx_ring->count)
3591 i = 0;
3592
9d5c8243
AK		 3593 frag = &skb_shinfo(skb)->frags[f];
3594 len = frag->size;
3595
3596 buffer_info = &tx_ring->buffer_info[i];
3597 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3598 buffer_info->length = len;
3599 buffer_info->time_stamp = jiffies;
0e014cb1 3600 buffer_info->next_to_watch = i;
6366ad33
AD		 3601 buffer_info->mapped_as_page = true;
3602 buffer_info->dma = pci_map_page(pdev,
3603 frag->page,
3604 frag->page_offset,
3605 len,
3606 PCI_DMA_TODEVICE);
3607 if (pci_dma_mapping_error(pdev, buffer_info->dma))
3608 goto dma_error;
3609
9d5c8243
AK		 3610 }
3611
9d5c8243 3612 tx_ring->buffer_info[i].skb = skb;
40e90c26 3613 tx_ring->buffer_info[i].gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
0e014cb1 3614 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3615
cdfd01fc 3616 return ++count;
6366ad33
AD		 3617
3618dma_error:
3619 dev_err(&pdev->dev, "TX DMA map failed\n");
3620
3621 /* clear timestamp and dma mappings for failed buffer_info mapping */
3622 buffer_info->dma = 0;
3623 buffer_info->time_stamp = 0;
3624 buffer_info->length = 0;
3625 buffer_info->next_to_watch = 0;
3626 buffer_info->mapped_as_page = false;
6366ad33
AD		 3627
3628 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 3629 while (count--) {
3630 if (i == 0)
3631 i = tx_ring->count;
6366ad33 3632 i--;
6366ad33
AD		 3633 buffer_info = &tx_ring->buffer_info[i];
3634 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3635 }
3636
3637 return 0;
9d5c8243
AK		 3638}
3639
85ad76b2 3640static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 3641 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 3642 u8 hdr_len)
3643{
cdfd01fc 3644 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 3645 struct igb_buffer *buffer_info;
3646 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 3647 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 3648
3649 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3650 E1000_ADVTXD_DCMD_DEXT);
3651
3652 if (tx_flags & IGB_TX_FLAGS_VLAN)
3653 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3654
33af6bcc
PO		 3655 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3656 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3657
9d5c8243
AK		 3658 if (tx_flags & IGB_TX_FLAGS_TSO) {
3659 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3660
3661 /* insert tcp checksum */
3662 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3663
3664 /* insert ip checksum */
3665 if (tx_flags & IGB_TX_FLAGS_IPV4)
3666 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3667
3668 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3669 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3670 }
3671
85ad76b2
AD		 3672 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3673 (tx_flags & (IGB_TX_FLAGS_CSUM |
3674 IGB_TX_FLAGS_TSO |
7dfc16fa 3675 IGB_TX_FLAGS_VLAN)))
85ad76b2 3676 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3677
3678 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3679
cdfd01fc 3680 do {
9d5c8243
AK		 3681 buffer_info = &tx_ring->buffer_info[i];
3682 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3683 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3684 tx_desc->read.cmd_type_len =
3685 cpu_to_le32(cmd_type_len | buffer_info->length);
3686 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 3687 count--;
9d5c8243
AK		 3688 i++;
3689 if (i == tx_ring->count)
3690 i = 0;
cdfd01fc 3691 } while (count > 0);
9d5c8243 3692
85ad76b2 3693 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 3694 /* Force memory writes to complete before letting h/w
3695 * know there are new descriptors to fetch. (Only
3696 * applicable for weak-ordered memory model archs,
3697 * such as IA-64). */
3698 wmb();
3699
3700 tx_ring->next_to_use = i;
fce99e34 3701 writel(i, tx_ring->tail);
9d5c8243
AK		 3702 /* we need this if more than one processor can write to our tail
3703 * at a time, it syncronizes IO on IA64/Altix systems */
3704 mmiowb();
3705}
3706
e694e964 3707static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3708{
e694e964
AD		 3709 struct net_device *netdev = tx_ring->netdev;
3710
661086df 3711 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 3712
9d5c8243
AK		 3713 /* Herbert's original patch had:
3714 * smp_mb__after_netif_stop_queue();
3715 * but since that doesn't exist yet, just open code it. */
3716 smp_mb();
3717
3718 /* We need to check again in a case another CPU has just
3719 * made room available. */
c493ea45 3720 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 3721 return -EBUSY;
3722
3723 /* A reprieve! */
661086df 3724 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 3725 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 3726 return 0;
3727}
3728
717ba089 3729static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3730{
c493ea45 3731 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 3732 return 0;
e694e964 3733 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 3734}
3735
b1a436c3
AD		 3736netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3737 struct igb_ring *tx_ring)
9d5c8243 3738{
e694e964 3739 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
cdfd01fc 3740 int tso = 0, count;
91d4ee33
NN		 3741 u32 tx_flags = 0;
3742 u16 first;
3743 u8 hdr_len = 0;
c5b9bd5e 3744 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 3745
9d5c8243
AK		 3746 /* need: 1 descriptor per page,
3747 * + 2 desc gap to keep tail from touching head,
3748 * + 1 desc for skb->data,
3749 * + 1 desc for context descriptor,
3750 * otherwise try next time */
e694e964 3751 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 3752 /* this is a hard error */
9d5c8243
AK		 3753 return NETDEV_TX_BUSY;
3754 }
33af6bcc 3755
33af6bcc
PO		 3756 if (unlikely(shtx->hardware)) {
3757 shtx->in_progress = 1;
3758 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 3759 }
9d5c8243 3760
cdfd01fc 3761 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 3762 tx_flags |= IGB_TX_FLAGS_VLAN;
3763 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3764 }
3765
661086df
PWJ		 3766 if (skb->protocol == htons(ETH_P_IP))
3767 tx_flags |= IGB_TX_FLAGS_IPV4;
3768
0e014cb1 3769 first = tx_ring->next_to_use;
85ad76b2
AD		 3770 if (skb_is_gso(skb)) {
3771 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 3772
85ad76b2
AD		 3773 if (tso < 0) {
3774 dev_kfree_skb_any(skb);
3775 return NETDEV_TX_OK;
3776 }
9d5c8243
AK		 3777 }
3778
3779 if (tso)
3780 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 3781 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 3782 (skb->ip_summed == CHECKSUM_PARTIAL))
3783 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 3784
65689fef 3785 /*
cdfd01fc 3786 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 3787 * has occured and we need to rewind the descriptor queue
3788 */
80785298 3789 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 3790 if (!count) {
65689fef
AD		 3791 dev_kfree_skb_any(skb);
3792 tx_ring->buffer_info[first].time_stamp = 0;
3793 tx_ring->next_to_use = first;
85ad76b2 3794 return NETDEV_TX_OK;
65689fef 3795 }
9d5c8243 3796
85ad76b2
AD		 3797 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3798
3799 /* Make sure there is space in the ring for the next send. */
e694e964 3800 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 3801
9d5c8243
AK		 3802 return NETDEV_TX_OK;
3803}
3804
3b29a56d
SH		 3805static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3806 struct net_device *netdev)
9d5c8243
AK		 3807{
3808 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 3809 struct igb_ring *tx_ring;
661086df 3810 int r_idx = 0;
b1a436c3
AD		 3811
3812 if (test_bit(__IGB_DOWN, &adapter->state)) {
3813 dev_kfree_skb_any(skb);
3814 return NETDEV_TX_OK;
3815 }
3816
3817 if (skb->len <= 0) {
3818 dev_kfree_skb_any(skb);
3819 return NETDEV_TX_OK;
3820 }
3821
1bfaf07b 3822 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 3823 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 3824
3825 /* This goes back to the question of how to logically map a tx queue
3826 * to a flow. Right now, performance is impacted slightly negatively
3827 * if using multiple tx queues. If the stack breaks away from a
3828 * single qdisc implementation, we can look at this again. */
e694e964 3829 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 3830}
3831
3832/**
3833 * igb_tx_timeout - Respond to a Tx Hang
3834 * @netdev: network interface device structure
3835 **/
3836static void igb_tx_timeout(struct net_device *netdev)
3837{
3838 struct igb_adapter *adapter = netdev_priv(netdev);
3839 struct e1000_hw *hw = &adapter->hw;
3840
3841 /* Do the reset outside of interrupt context */
3842 adapter->tx_timeout_count++;
f7ba205e 3843
55cac248
AD		 3844 if (hw->mac.type == e1000_82580)
3845 hw->dev_spec._82575.global_device_reset = true;
3846
9d5c8243 3847 schedule_work(&adapter->reset_task);
265de409
AD		 3848 wr32(E1000_EICS,
3849 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 3850}
3851
3852static void igb_reset_task(struct work_struct *work)
3853{
3854 struct igb_adapter *adapter;
3855 adapter = container_of(work, struct igb_adapter, reset_task);
3856
3857 igb_reinit_locked(adapter);
3858}
3859
3860/**
3861 * igb_get_stats - Get System Network Statistics
3862 * @netdev: network interface device structure
3863 *
3864 * Returns the address of the device statistics structure.
3865 * The statistics are actually updated from the timer callback.
3866 **/
73cd78f1 3867static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 3868{
9d5c8243 3869 /* only return the current stats */
8d24e933 3870 return &netdev->stats;
9d5c8243
AK		 3871}
3872
3873/**
3874 * igb_change_mtu - Change the Maximum Transfer Unit
3875 * @netdev: network interface device structure
3876 * @new_mtu: new value for maximum frame size
3877 *
3878 * Returns 0 on success, negative on failure
3879 **/
3880static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3881{
3882 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 3883 struct pci_dev *pdev = adapter->pdev;
9d5c8243 3884 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 3885 u32 rx_buffer_len, i;
9d5c8243 3886
c809d227 3887 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 3888 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 3889 return -EINVAL;
3890 }
3891
9d5c8243 3892 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 3893 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 3894 return -EINVAL;
3895 }
3896
3897 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3898 msleep(1);
73cd78f1 3899
9d5c8243
AK		 3900 /* igb_down has a dependency on max_frame_size */
3901 adapter->max_frame_size = max_frame;
559e9c49 3902
9d5c8243
AK		 3903 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3904 * means we reserve 2 more, this pushes us to allocate from the next
3905 * larger slab size.
3906 * i.e. RXBUFFER_2048 --> size-4096 slab
3907 */
3908
757b77e2
NN		 3909 if (adapter->hw.mac.type == e1000_82580)
3910 max_frame += IGB_TS_HDR_LEN;
3911
7d95b717 3912 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 3913 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 3914 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 3915 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 3916 else
4c844851
AD		 3917 rx_buffer_len = IGB_RXBUFFER_128;
3918
757b77e2
NN		 3919 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
3920 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
3921 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
3922
3923 if ((adapter->hw.mac.type == e1000_82580) &&
3924 (rx_buffer_len == IGB_RXBUFFER_128))
3925 rx_buffer_len += IGB_RXBUFFER_64;
3926
4c844851
AD		 3927 if (netif_running(netdev))
3928 igb_down(adapter);
9d5c8243 3929
090b1795 3930 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 3931 netdev->mtu, new_mtu);
3932 netdev->mtu = new_mtu;
3933
4c844851 3934 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3935 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 3936
9d5c8243
AK		 3937 if (netif_running(netdev))
3938 igb_up(adapter);
3939 else
3940 igb_reset(adapter);
3941
3942 clear_bit(__IGB_RESETTING, &adapter->state);
3943
3944 return 0;
3945}
3946
3947/**
3948 * igb_update_stats - Update the board statistics counters
3949 * @adapter: board private structure
3950 **/
3951
3952void igb_update_stats(struct igb_adapter *adapter)
3953{
128e45eb 3954 struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
9d5c8243
AK		 3955 struct e1000_hw *hw = &adapter->hw;
3956 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 3957 u32 reg, mpc;
9d5c8243 3958 u16 phy_tmp;
3f9c0164
AD		 3959 int i;
3960 u64 bytes, packets;
9d5c8243
AK		 3961
3962#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3963
3964 /*
3965 * Prevent stats update while adapter is being reset, or if the pci
3966 * connection is down.
3967 */
3968 if (adapter->link_speed == 0)
3969 return;
3970 if (pci_channel_offline(pdev))
3971 return;
3972
3f9c0164
AD		 3973 bytes = 0;
3974 packets = 0;
3975 for (i = 0; i < adapter->num_rx_queues; i++) {
3976 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446
AD		 3977 struct igb_ring *ring = adapter->rx_ring[i];
3978 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 3979 net_stats->rx_fifo_errors += rqdpc_tmp;
3025a446
AD		 3980 bytes += ring->rx_stats.bytes;
3981 packets += ring->rx_stats.packets;
3f9c0164
AD		 3982 }
3983
128e45eb
AD		 3984 net_stats->rx_bytes = bytes;
3985 net_stats->rx_packets = packets;
3f9c0164
AD		 3986
3987 bytes = 0;
3988 packets = 0;
3989 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 3990 struct igb_ring *ring = adapter->tx_ring[i];
3991 bytes += ring->tx_stats.bytes;
3992 packets += ring->tx_stats.packets;
3f9c0164 3993 }
128e45eb
AD		 3994 net_stats->tx_bytes = bytes;
3995 net_stats->tx_packets = packets;
3f9c0164
AD		 3996
3997 /* read stats registers */
9d5c8243
AK		 3998 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
3999 adapter->stats.gprc += rd32(E1000_GPRC);
4000 adapter->stats.gorc += rd32(E1000_GORCL);
4001 rd32(E1000_GORCH); /* clear GORCL */
4002 adapter->stats.bprc += rd32(E1000_BPRC);
4003 adapter->stats.mprc += rd32(E1000_MPRC);
4004 adapter->stats.roc += rd32(E1000_ROC);
4005
4006 adapter->stats.prc64 += rd32(E1000_PRC64);
4007 adapter->stats.prc127 += rd32(E1000_PRC127);
4008 adapter->stats.prc255 += rd32(E1000_PRC255);
4009 adapter->stats.prc511 += rd32(E1000_PRC511);
4010 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4011 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4012 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4013 adapter->stats.sec += rd32(E1000_SEC);
4014
fa3d9a6d
MW		 4015 mpc = rd32(E1000_MPC);
4016 adapter->stats.mpc += mpc;
4017 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4018 adapter->stats.scc += rd32(E1000_SCC);
4019 adapter->stats.ecol += rd32(E1000_ECOL);
4020 adapter->stats.mcc += rd32(E1000_MCC);
4021 adapter->stats.latecol += rd32(E1000_LATECOL);
4022 adapter->stats.dc += rd32(E1000_DC);
4023 adapter->stats.rlec += rd32(E1000_RLEC);
4024 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4025 adapter->stats.xontxc += rd32(E1000_XONTXC);
4026 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4027 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4028 adapter->stats.fcruc += rd32(E1000_FCRUC);
4029 adapter->stats.gptc += rd32(E1000_GPTC);
4030 adapter->stats.gotc += rd32(E1000_GOTCL);
4031 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4032 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4033 adapter->stats.ruc += rd32(E1000_RUC);
4034 adapter->stats.rfc += rd32(E1000_RFC);
4035 adapter->stats.rjc += rd32(E1000_RJC);
4036 adapter->stats.tor += rd32(E1000_TORH);
4037 adapter->stats.tot += rd32(E1000_TOTH);
4038 adapter->stats.tpr += rd32(E1000_TPR);
4039
4040 adapter->stats.ptc64 += rd32(E1000_PTC64);
4041 adapter->stats.ptc127 += rd32(E1000_PTC127);
4042 adapter->stats.ptc255 += rd32(E1000_PTC255);
4043 adapter->stats.ptc511 += rd32(E1000_PTC511);
4044 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4045 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4046
4047 adapter->stats.mptc += rd32(E1000_MPTC);
4048 adapter->stats.bptc += rd32(E1000_BPTC);
4049
2d0b0f69
NN		 4050 adapter->stats.tpt += rd32(E1000_TPT);
4051 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4052
4053 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4054 /* read internal phy specific stats */
4055 reg = rd32(E1000_CTRL_EXT);
4056 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4057 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4058 adapter->stats.tncrs += rd32(E1000_TNCRS);
4059 }
4060
9d5c8243
AK		 4061 adapter->stats.tsctc += rd32(E1000_TSCTC);
4062 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4063
4064 adapter->stats.iac += rd32(E1000_IAC);
4065 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4066 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4067 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4068 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4069 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4070 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4071 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4072 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4073
4074 /* Fill out the OS statistics structure */
128e45eb
AD		 4075 net_stats->multicast = adapter->stats.mprc;
4076 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4077
4078 /* Rx Errors */
4079
4080 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4081 * our own version based on RUC and ROC */
128e45eb 4082 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4083 adapter->stats.crcerrs + adapter->stats.algnerrc +
4084 adapter->stats.ruc + adapter->stats.roc +
4085 adapter->stats.cexterr;
128e45eb
AD		 4086 net_stats->rx_length_errors = adapter->stats.ruc +
4087 adapter->stats.roc;
4088 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4089 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4090 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4091
4092 /* Tx Errors */
128e45eb
AD		 4093 net_stats->tx_errors = adapter->stats.ecol +
4094 adapter->stats.latecol;
4095 net_stats->tx_aborted_errors = adapter->stats.ecol;
4096 net_stats->tx_window_errors = adapter->stats.latecol;
4097 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4098
4099 /* Tx Dropped needs to be maintained elsewhere */
4100
4101 /* Phy Stats */
4102 if (hw->phy.media_type == e1000_media_type_copper) {
4103 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4104 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4105 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4106 adapter->phy_stats.idle_errors += phy_tmp;
4107 }
4108 }
4109
4110 /* Management Stats */
4111 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4112 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4113 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4114}
4115
9d5c8243
AK		 4116static irqreturn_t igb_msix_other(int irq, void *data)
4117{
047e0030 4118 struct igb_adapter *adapter = data;
9d5c8243 4119 struct e1000_hw *hw = &adapter->hw;
844290e5 4120 u32 icr = rd32(E1000_ICR);
844290e5 4121 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4122
7f081d40
AD		 4123 if (icr & E1000_ICR_DRSTA)
4124 schedule_work(&adapter->reset_task);
4125
047e0030 4126 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4127 /* HW is reporting DMA is out of sync */
4128 adapter->stats.doosync++;
4129 }
eebbbdba 4130
4ae196df
AD		 4131 /* Check for a mailbox event */
4132 if (icr & E1000_ICR_VMMB)
4133 igb_msg_task(adapter);
4134
4135 if (icr & E1000_ICR_LSC) {
4136 hw->mac.get_link_status = 1;
4137 /* guard against interrupt when we're going down */
4138 if (!test_bit(__IGB_DOWN, &adapter->state))
4139 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4140 }
4141
25568a53
AD		 4142 if (adapter->vfs_allocated_count)
4143 wr32(E1000_IMS, E1000_IMS_LSC |
4144 E1000_IMS_VMMB |
4145 E1000_IMS_DOUTSYNC);
4146 else
4147 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4148 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4149
4150 return IRQ_HANDLED;
4151}
4152
047e0030 4153static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4154{
26b39276 4155 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4156 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4157
047e0030
AD		 4158 if (!q_vector->set_itr)
4159 return;
73cd78f1 4160
047e0030
AD		 4161 if (!itr_val)
4162 itr_val = 0x4;
661086df 4163
26b39276
AD		 4164 if (adapter->hw.mac.type == e1000_82575)
4165 itr_val |= itr_val << 16;
661086df 4166 else
047e0030 4167 itr_val |= 0x8000000;
661086df 4168
047e0030
AD		 4169 writel(itr_val, q_vector->itr_register);
4170 q_vector->set_itr = 0;
6eb5a7f1
AD		 4171}
4172
047e0030 4173static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4174{
047e0030 4175 struct igb_q_vector *q_vector = data;
9d5c8243 4176
047e0030
AD		 4177 /* Write the ITR value calculated from the previous interrupt. */
4178 igb_write_itr(q_vector);
9d5c8243 4179
047e0030 4180 napi_schedule(&q_vector->napi);
844290e5 4181
047e0030 4182 return IRQ_HANDLED;
fe4506b6
JC		 4183}
4184
421e02f0 4185#ifdef CONFIG_IGB_DCA
047e0030 4186static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4187{
047e0030 4188 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4189 struct e1000_hw *hw = &adapter->hw;
4190 int cpu = get_cpu();
fe4506b6 4191
047e0030
AD		 4192 if (q_vector->cpu == cpu)
4193 goto out_no_update;
4194
4195 if (q_vector->tx_ring) {
4196 int q = q_vector->tx_ring->reg_idx;
4197 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4198 if (hw->mac.type == e1000_82575) {
4199 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4200 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4201 } else {
047e0030
AD		 4202 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4203 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4204 E1000_DCA_TXCTRL_CPUID_SHIFT;
4205 }
4206 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4207 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4208 }
4209 if (q_vector->rx_ring) {
4210 int q = q_vector->rx_ring->reg_idx;
4211 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4212 if (hw->mac.type == e1000_82575) {
2d064c06 4213 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4214 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4215 } else {
4216 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4217 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4218 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4219 }
fe4506b6
JC		 4220 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4221 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4222 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4223 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4224 }
047e0030
AD		 4225 q_vector->cpu = cpu;
4226out_no_update:
fe4506b6
JC		 4227 put_cpu();
4228}
4229
4230static void igb_setup_dca(struct igb_adapter *adapter)
4231{
7e0e99ef 4232 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4233 int i;
4234
7dfc16fa 4235 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4236 return;
4237
7e0e99ef
AD		 4238 /* Always use CB2 mode, difference is masked in the CB driver. */
4239 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4240
047e0030 4241 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4242 adapter->q_vector[i]->cpu = -1;
4243 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4244 }
4245}
4246
4247static int __igb_notify_dca(struct device *dev, void *data)
4248{
4249 struct net_device *netdev = dev_get_drvdata(dev);
4250 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4251 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4252 struct e1000_hw *hw = &adapter->hw;
4253 unsigned long event = *(unsigned long *)data;
4254
4255 switch (event) {
4256 case DCA_PROVIDER_ADD:
4257 /* if already enabled, don't do it again */
7dfc16fa 4258 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4259 break;
fe4506b6 4260 if (dca_add_requester(dev) == 0) {
bbd98fe4 4261 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4262 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4263 igb_setup_dca(adapter);
4264 break;
4265 }
4266 /* Fall Through since DCA is disabled. */
4267 case DCA_PROVIDER_REMOVE:
7dfc16fa 4268 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4269 /* without this a class_device is left
047e0030 4270 * hanging around in the sysfs model */
fe4506b6 4271 dca_remove_requester(dev);
090b1795 4272 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4273 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4274 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4275 }
4276 break;
4277 }
bbd98fe4 4278
fe4506b6 4279 return 0;
9d5c8243
AK		 4280}
4281
fe4506b6
JC		 4282static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4283 void *p)
4284{
4285 int ret_val;
4286
4287 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4288 __igb_notify_dca);
4289
4290 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4291}
421e02f0 4292#endif /* CONFIG_IGB_DCA */
9d5c8243 4293
4ae196df
AD		 4294static void igb_ping_all_vfs(struct igb_adapter *adapter)
4295{
4296 struct e1000_hw *hw = &adapter->hw;
4297 u32 ping;
4298 int i;
4299
4300 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4301 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4302 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4303 ping |= E1000_VT_MSGTYPE_CTS;
4304 igb_write_mbx(hw, &ping, 1, i);
4305 }
4306}
4307
7d5753f0
AD		 4308static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4309{
4310 struct e1000_hw *hw = &adapter->hw;
4311 u32 vmolr = rd32(E1000_VMOLR(vf));
4312 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4313
4314 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4315 IGB_VF_FLAG_MULTI_PROMISC);
4316 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4317
4318 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4319 vmolr |= E1000_VMOLR_MPME;
4320 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4321 } else {
4322 /*
4323 * if we have hashes and we are clearing a multicast promisc
4324 * flag we need to write the hashes to the MTA as this step
4325 * was previously skipped
4326 */
4327 if (vf_data->num_vf_mc_hashes > 30) {
4328 vmolr |= E1000_VMOLR_MPME;
4329 } else if (vf_data->num_vf_mc_hashes) {
4330 int j;
4331 vmolr |= E1000_VMOLR_ROMPE;
4332 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4333 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4334 }
4335 }
4336
4337 wr32(E1000_VMOLR(vf), vmolr);
4338
4339 /* there are flags left unprocessed, likely not supported */
4340 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4341 return -EINVAL;
4342
4343 return 0;
4344
4345}
4346
4ae196df
AD		 4347static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4348 u32 *msgbuf, u32 vf)
4349{
4350 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4351 u16 *hash_list = (u16 *)&msgbuf[1];
4352 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4353 int i;
4354
7d5753f0 4355 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4356 * to this VF for later use to restore when the PF multi cast
4357 * list changes
4358 */
4359 vf_data->num_vf_mc_hashes = n;
4360
7d5753f0
AD		 4361 /* only up to 30 hash values supported */
4362 if (n > 30)
4363 n = 30;
4364
4365 /* store the hashes for later use */
4ae196df 4366 for (i = 0; i < n; i++)
a419aef8 4367 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4368
4369 /* Flush and reset the mta with the new values */
ff41f8dc 4370 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4371
4372 return 0;
4373}
4374
4375static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4376{
4377 struct e1000_hw *hw = &adapter->hw;
4378 struct vf_data_storage *vf_data;
4379 int i, j;
4380
4381 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4382 u32 vmolr = rd32(E1000_VMOLR(i));
4383 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4384
4ae196df 4385 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4386
4387 if ((vf_data->num_vf_mc_hashes > 30) ||
4388 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4389 vmolr |= E1000_VMOLR_MPME;
4390 } else if (vf_data->num_vf_mc_hashes) {
4391 vmolr |= E1000_VMOLR_ROMPE;
4392 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4393 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4394 }
4395 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4396 }
4397}
4398
4399static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4400{
4401 struct e1000_hw *hw = &adapter->hw;
4402 u32 pool_mask, reg, vid;
4403 int i;
4404
4405 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4406
4407 /* Find the vlan filter for this id */
4408 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4409 reg = rd32(E1000_VLVF(i));
4410
4411 /* remove the vf from the pool */
4412 reg &= ~pool_mask;
4413
4414 /* if pool is empty then remove entry from vfta */
4415 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4416 (reg & E1000_VLVF_VLANID_ENABLE)) {
4417 reg = 0;
4418 vid = reg & E1000_VLVF_VLANID_MASK;
4419 igb_vfta_set(hw, vid, false);
4420 }
4421
4422 wr32(E1000_VLVF(i), reg);
4423 }
ae641bdc
AD		 4424
4425 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4426}
4427
4428static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4429{
4430 struct e1000_hw *hw = &adapter->hw;
4431 u32 reg, i;
4432
51466239
AD		 4433 /* The vlvf table only exists on 82576 hardware and newer */
4434 if (hw->mac.type < e1000_82576)
4435 return -1;
4436
4437 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4438 if (!adapter->vfs_allocated_count)
4439 return -1;
4440
4441 /* Find the vlan filter for this id */
4442 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4443 reg = rd32(E1000_VLVF(i));
4444 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4445 vid == (reg & E1000_VLVF_VLANID_MASK))
4446 break;
4447 }
4448
4449 if (add) {
4450 if (i == E1000_VLVF_ARRAY_SIZE) {
4451 /* Did not find a matching VLAN ID entry that was
4452 * enabled. Search for a free filter entry, i.e.
4453 * one without the enable bit set
4454 */
4455 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4456 reg = rd32(E1000_VLVF(i));
4457 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4458 break;
4459 }
4460 }
4461 if (i < E1000_VLVF_ARRAY_SIZE) {
4462 /* Found an enabled/available entry */
4463 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4464
4465 /* if !enabled we need to set this up in vfta */
4466 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4467 /* add VID to filter table */
4468 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4469 reg |= E1000_VLVF_VLANID_ENABLE;
4470 }
cad6d05f
AD		 4471 reg &= ~E1000_VLVF_VLANID_MASK;
4472 reg |= vid;
4ae196df 4473 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4474
4475 /* do not modify RLPML for PF devices */
4476 if (vf >= adapter->vfs_allocated_count)
4477 return 0;
4478
4479 if (!adapter->vf_data[vf].vlans_enabled) {
4480 u32 size;
4481 reg = rd32(E1000_VMOLR(vf));
4482 size = reg & E1000_VMOLR_RLPML_MASK;
4483 size += 4;
4484 reg &= ~E1000_VMOLR_RLPML_MASK;
4485 reg |= size;
4486 wr32(E1000_VMOLR(vf), reg);
4487 }
ae641bdc 4488
51466239 4489 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4490 return 0;
4491 }
4492 } else {
4493 if (i < E1000_VLVF_ARRAY_SIZE) {
4494 /* remove vf from the pool */
4495 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4496 /* if pool is empty then remove entry from vfta */
4497 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4498 reg = 0;
4499 igb_vfta_set(hw, vid, false);
4500 }
4501 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4502
4503 /* do not modify RLPML for PF devices */
4504 if (vf >= adapter->vfs_allocated_count)
4505 return 0;
4506
4507 adapter->vf_data[vf].vlans_enabled--;
4508 if (!adapter->vf_data[vf].vlans_enabled) {
4509 u32 size;
4510 reg = rd32(E1000_VMOLR(vf));
4511 size = reg & E1000_VMOLR_RLPML_MASK;
4512 size -= 4;
4513 reg &= ~E1000_VMOLR_RLPML_MASK;
4514 reg |= size;
4515 wr32(E1000_VMOLR(vf), reg);
4516 }
4ae196df
AD		 4517 }
4518 }
8151d294
WM		 4519 return 0;
4520}
4521
4522static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4523{
4524 struct e1000_hw *hw = &adapter->hw;
4525
4526 if (vid)
4527 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4528 else
4529 wr32(E1000_VMVIR(vf), 0);
4530}
4531
4532static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4533 int vf, u16 vlan, u8 qos)
4534{
4535 int err = 0;
4536 struct igb_adapter *adapter = netdev_priv(netdev);
4537
4538 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4539 return -EINVAL;
4540 if (vlan || qos) {
4541 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4542 if (err)
4543 goto out;
4544 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4545 igb_set_vmolr(adapter, vf, !vlan);
4546 adapter->vf_data[vf].pf_vlan = vlan;
4547 adapter->vf_data[vf].pf_qos = qos;
4548 dev_info(&adapter->pdev->dev,
4549 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4550 if (test_bit(__IGB_DOWN, &adapter->state)) {
4551 dev_warn(&adapter->pdev->dev,
4552 "The VF VLAN has been set,"
4553 " but the PF device is not up.\n");
4554 dev_warn(&adapter->pdev->dev,
4555 "Bring the PF device up before"
4556 " attempting to use the VF device.\n");
4557 }
4558 } else {
4559 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4560 false, vf);
4561 igb_set_vmvir(adapter, vlan, vf);
4562 igb_set_vmolr(adapter, vf, true);
4563 adapter->vf_data[vf].pf_vlan = 0;
4564 adapter->vf_data[vf].pf_qos = 0;
4565 }
4566out:
4567 return err;
4ae196df
AD		 4568}
4569
4570static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4571{
4572 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4573 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4574
4575 return igb_vlvf_set(adapter, vid, add, vf);
4576}
4577
f2ca0dbe 4578static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4579{
8151d294
WM		 4580 /* clear flags */
4581 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
f2ca0dbe 4582 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4583
4584 /* reset offloads to defaults */
8151d294 4585 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 4586
4587 /* reset vlans for device */
4588 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 4589 if (adapter->vf_data[vf].pf_vlan)
4590 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4591 adapter->vf_data[vf].pf_vlan,
4592 adapter->vf_data[vf].pf_qos);
4593 else
4594 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 4595
4596 /* reset multicast table array for vf */
4597 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4598
4599 /* Flush and reset the mta with the new values */
ff41f8dc 4600 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4601}
4602
f2ca0dbe
AD		 4603static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4604{
4605 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4606
4607 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 4608 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4609 random_ether_addr(vf_mac);
f2ca0dbe
AD		 4610
4611 /* process remaining reset events */
4612 igb_vf_reset(adapter, vf);
4613}
4614
4615static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4616{
4617 struct e1000_hw *hw = &adapter->hw;
4618 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4619 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4620 u32 reg, msgbuf[3];
4621 u8 *addr = (u8 *)(&msgbuf[1]);
4622
4623 /* process all the same items cleared in a function level reset */
f2ca0dbe 4624 igb_vf_reset(adapter, vf);
4ae196df
AD		 4625
4626 /* set vf mac address */
26ad9178 4627 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4628
4629 /* enable transmit and receive for vf */
4630 reg = rd32(E1000_VFTE);
4631 wr32(E1000_VFTE, reg | (1 << vf));
4632 reg = rd32(E1000_VFRE);
4633 wr32(E1000_VFRE, reg | (1 << vf));
4634
f2ca0dbe 4635 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4636
4637 /* reply to reset with ack and vf mac address */
4638 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4639 memcpy(addr, vf_mac, 6);
4640 igb_write_mbx(hw, msgbuf, 3, vf);
4641}
4642
4643static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4644{
f2ca0dbe
AD		 4645 unsigned char *addr = (char *)&msg[1];
4646 int err = -1;
4ae196df 4647
f2ca0dbe
AD		 4648 if (is_valid_ether_addr(addr))
4649 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4650
f2ca0dbe 4651 return err;
4ae196df
AD		 4652}
4653
4654static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4655{
4656 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4657 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4658 u32 msg = E1000_VT_MSGTYPE_NACK;
4659
4660 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 4661 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4662 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 4663 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 4664 vf_data->last_nack = jiffies;
4ae196df
AD		 4665 }
4666}
4667
f2ca0dbe 4668static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 4669{
f2ca0dbe
AD		 4670 struct pci_dev *pdev = adapter->pdev;
4671 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 4672 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4673 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4674 s32 retval;
4675
f2ca0dbe 4676 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 4677
fef45f4c
AD		 4678 if (retval) {
4679 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 4680 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 4681 vf_data->flags &= ~IGB_VF_FLAG_CTS;
4682 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4683 return;
4684 goto out;
4685 }
4ae196df
AD		 4686
4687 /* this is a message we already processed, do nothing */
4688 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 4689 return;
4ae196df
AD		 4690
4691 /*
4692 * until the vf completes a reset it should not be
4693 * allowed to start any configuration.
4694 */
4695
4696 if (msgbuf[0] == E1000_VF_RESET) {
4697 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 4698 return;
4ae196df
AD		 4699 }
4700
f2ca0dbe 4701 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 4702 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4703 return;
4704 retval = -1;
4705 goto out;
4ae196df
AD		 4706 }
4707
4708 switch ((msgbuf[0] & 0xFFFF)) {
4709 case E1000_VF_SET_MAC_ADDR:
4710 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4711 break;
7d5753f0
AD		 4712 case E1000_VF_SET_PROMISC:
4713 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4714 break;
4ae196df
AD		 4715 case E1000_VF_SET_MULTICAST:
4716 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4717 break;
4718 case E1000_VF_SET_LPE:
4719 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4720 break;
4721 case E1000_VF_SET_VLAN:
8151d294
WM		 4722 if (adapter->vf_data[vf].pf_vlan)
4723 retval = -1;
4724 else
4725 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 4726 break;
4727 default:
090b1795 4728 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 4729 retval = -1;
4730 break;
4731 }
4732
fef45f4c
AD		 4733 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4734out:
4ae196df
AD		 4735 /* notify the VF of the results of what it sent us */
4736 if (retval)
4737 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4738 else
4739 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4740
4ae196df 4741 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 4742}
4ae196df 4743
f2ca0dbe
AD		 4744static void igb_msg_task(struct igb_adapter *adapter)
4745{
4746 struct e1000_hw *hw = &adapter->hw;
4747 u32 vf;
4748
4749 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4750 /* process any reset requests */
4751 if (!igb_check_for_rst(hw, vf))
4752 igb_vf_reset_event(adapter, vf);
4753
4754 /* process any messages pending */
4755 if (!igb_check_for_msg(hw, vf))
4756 igb_rcv_msg_from_vf(adapter, vf);
4757
4758 /* process any acks */
4759 if (!igb_check_for_ack(hw, vf))
4760 igb_rcv_ack_from_vf(adapter, vf);
4761 }
4ae196df
AD		 4762}
4763
68d480c4
AD		 4764/**
4765 * igb_set_uta - Set unicast filter table address
4766 * @adapter: board private structure
4767 *
4768 * The unicast table address is a register array of 32-bit registers.
4769 * The table is meant to be used in a way similar to how the MTA is used
4770 * however due to certain limitations in the hardware it is necessary to
4771 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4772 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4773 **/
4774static void igb_set_uta(struct igb_adapter *adapter)
4775{
4776 struct e1000_hw *hw = &adapter->hw;
4777 int i;
4778
4779 /* The UTA table only exists on 82576 hardware and newer */
4780 if (hw->mac.type < e1000_82576)
4781 return;
4782
4783 /* we only need to do this if VMDq is enabled */
4784 if (!adapter->vfs_allocated_count)
4785 return;
4786
4787 for (i = 0; i < hw->mac.uta_reg_count; i++)
4788 array_wr32(E1000_UTA, i, ~0);
4789}
4790
9d5c8243
AK		 4791/**
4792 * igb_intr_msi - Interrupt Handler
4793 * @irq: interrupt number
4794 * @data: pointer to a network interface device structure
4795 **/
4796static irqreturn_t igb_intr_msi(int irq, void *data)
4797{
047e0030
AD		 4798 struct igb_adapter *adapter = data;
4799 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4800 struct e1000_hw *hw = &adapter->hw;
4801 /* read ICR disables interrupts using IAM */
4802 u32 icr = rd32(E1000_ICR);
4803
047e0030 4804 igb_write_itr(q_vector);
9d5c8243 4805
7f081d40
AD		 4806 if (icr & E1000_ICR_DRSTA)
4807 schedule_work(&adapter->reset_task);
4808
047e0030 4809 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4810 /* HW is reporting DMA is out of sync */
4811 adapter->stats.doosync++;
4812 }
4813
9d5c8243
AK		 4814 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4815 hw->mac.get_link_status = 1;
4816 if (!test_bit(__IGB_DOWN, &adapter->state))
4817 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4818 }
4819
047e0030 4820 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4821
4822 return IRQ_HANDLED;
4823}
4824
4825/**
4a3c6433 4826 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 4827 * @irq: interrupt number
4828 * @data: pointer to a network interface device structure
4829 **/
4830static irqreturn_t igb_intr(int irq, void *data)
4831{
047e0030
AD		 4832 struct igb_adapter *adapter = data;
4833 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4834 struct e1000_hw *hw = &adapter->hw;
4835 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4836 * need for the IMC write */
4837 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 4838 if (!icr)
4839 return IRQ_NONE; /* Not our interrupt */
4840
047e0030 4841 igb_write_itr(q_vector);
9d5c8243
AK		 4842
4843 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4844 * not set, then the adapter didn't send an interrupt */
4845 if (!(icr & E1000_ICR_INT_ASSERTED))
4846 return IRQ_NONE;
4847
7f081d40
AD		 4848 if (icr & E1000_ICR_DRSTA)
4849 schedule_work(&adapter->reset_task);
4850
047e0030 4851 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4852 /* HW is reporting DMA is out of sync */
4853 adapter->stats.doosync++;
4854 }
4855
9d5c8243
AK		 4856 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4857 hw->mac.get_link_status = 1;
4858 /* guard against interrupt when we're going down */
4859 if (!test_bit(__IGB_DOWN, &adapter->state))
4860 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4861 }
4862
047e0030 4863 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4864
4865 return IRQ_HANDLED;
4866}
4867
047e0030 4868static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 4869{
047e0030 4870 struct igb_adapter *adapter = q_vector->adapter;
46544258 4871 struct e1000_hw *hw = &adapter->hw;
9d5c8243 4872
4fc82adf
AD		 4873 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4874 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 4875 if (!adapter->msix_entries)
6eb5a7f1 4876 igb_set_itr(adapter);
46544258 4877 else
047e0030 4878 igb_update_ring_itr(q_vector);
9d5c8243
AK		 4879 }
4880
46544258
AD		 4881 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4882 if (adapter->msix_entries)
047e0030 4883 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 4884 else
4885 igb_irq_enable(adapter);
4886 }
9d5c8243
AK		 4887}
4888
46544258
AD		 4889/**
4890 * igb_poll - NAPI Rx polling callback
4891 * @napi: napi polling structure
4892 * @budget: count of how many packets we should handle
4893 **/
4894static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 4895{
047e0030
AD		 4896 struct igb_q_vector *q_vector = container_of(napi,
4897 struct igb_q_vector,
4898 napi);
4899 int tx_clean_complete = 1, work_done = 0;
9d5c8243 4900
421e02f0 4901#ifdef CONFIG_IGB_DCA
047e0030
AD		 4902 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4903 igb_update_dca(q_vector);
fe4506b6 4904#endif
047e0030
AD		 4905 if (q_vector->tx_ring)
4906 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 4907
047e0030
AD		 4908 if (q_vector->rx_ring)
4909 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4910
4911 if (!tx_clean_complete)
4912 work_done = budget;
46544258 4913
9d5c8243 4914 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 4915 if (work_done < budget) {
288379f0 4916 napi_complete(napi);
047e0030 4917 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 4918 }
4919
46544258 4920 return work_done;
9d5c8243 4921}
6d8126f9 4922
33af6bcc 4923/**
c5b9bd5e 4924 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 4925 * @adapter: board private structure
c5b9bd5e
AD		 4926 * @shhwtstamps: timestamp structure to update
4927 * @regval: unsigned 64bit system time value.
4928 *
4929 * We need to convert the system time value stored in the RX/TXSTMP registers
4930 * into a hwtstamp which can be used by the upper level timestamping functions
4931 */
4932static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4933 struct skb_shared_hwtstamps *shhwtstamps,
4934 u64 regval)
4935{
4936 u64 ns;
4937
55cac248
AD		 4938 /*
4939 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
4940 * 24 to match clock shift we setup earlier.
4941 */
4942 if (adapter->hw.mac.type == e1000_82580)
4943 regval <<= IGB_82580_TSYNC_SHIFT;
4944
c5b9bd5e
AD		 4945 ns = timecounter_cyc2time(&adapter->clock, regval);
4946 timecompare_update(&adapter->compare, ns);
4947 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4948 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4949 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4950}
4951
4952/**
4953 * igb_tx_hwtstamp - utility function which checks for TX time stamp
4954 * @q_vector: pointer to q_vector containing needed info
33af6bcc
PO		 4955 * @skb: packet that was just sent
4956 *
4957 * If we were asked to do hardware stamping and such a time stamp is
4958 * available, then it must have been for this skb here because we only
4959 * allow only one such packet into the queue.
4960 */
c5b9bd5e 4961static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
33af6bcc 4962{
c5b9bd5e 4963 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc
PO		 4964 union skb_shared_tx *shtx = skb_tx(skb);
4965 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 4966 struct skb_shared_hwtstamps shhwtstamps;
4967 u64 regval;
33af6bcc 4968
c5b9bd5e
AD		 4969 /* if skb does not support hw timestamp or TX stamp not valid exit */
4970 if (likely(!shtx->hardware) ||
4971 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4972 return;
4973
4974 regval = rd32(E1000_TXSTMPL);
4975 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4976
4977 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4978 skb_tstamp_tx(skb, &shhwtstamps);
33af6bcc
PO		 4979}
4980
9d5c8243
AK		 4981/**
4982 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 4983 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 4984 * returns true if ring is completely cleaned
4985 **/
047e0030 4986static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 4987{
047e0030
AD		 4988 struct igb_adapter *adapter = q_vector->adapter;
4989 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 4990 struct net_device *netdev = tx_ring->netdev;
0e014cb1 4991 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 4992 struct igb_buffer *buffer_info;
4993 struct sk_buff *skb;
0e014cb1 4994 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 4995 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 4996 unsigned int i, eop, count = 0;
4997 bool cleaned = false;
9d5c8243 4998
9d5c8243 4999 i = tx_ring->next_to_clean;
0e014cb1
AD		 5000 eop = tx_ring->buffer_info[i].next_to_watch;
5001 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5002
5003 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5004 (count < tx_ring->count)) {
5005 for (cleaned = false; !cleaned; count++) {
5006 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5007 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5008 cleaned = (i == eop);
9d5c8243
AK		 5009 skb = buffer_info->skb;
5010
5011 if (skb) {
5012 unsigned int segs, bytecount;
5013 /* gso_segs is currently only valid for tcp */
40e90c26 5014 segs = buffer_info->gso_segs;
9d5c8243
AK		 5015 /* multiply data chunks by size of headers */
5016 bytecount = ((segs - 1) * skb_headlen(skb)) +
5017 skb->len;
5018 total_packets += segs;
5019 total_bytes += bytecount;
33af6bcc 5020
c5b9bd5e 5021 igb_tx_hwtstamp(q_vector, skb);
9d5c8243
AK		 5022 }
5023
80785298 5024 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5025 tx_desc->wb.status = 0;
9d5c8243
AK		 5026
5027 i++;
5028 if (i == tx_ring->count)
5029 i = 0;
9d5c8243 5030 }
0e014cb1
AD		 5031 eop = tx_ring->buffer_info[i].next_to_watch;
5032 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5033 }
5034
9d5c8243
AK		 5035 tx_ring->next_to_clean = i;
5036
fc7d345d 5037 if (unlikely(count &&
9d5c8243 5038 netif_carrier_ok(netdev) &&
c493ea45 5039 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5040 /* Make sure that anybody stopping the queue after this
5041 * sees the new next_to_clean.
5042 */
5043 smp_mb();
661086df
PWJ		 5044 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5045 !(test_bit(__IGB_DOWN, &adapter->state))) {
5046 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 5047 tx_ring->tx_stats.restart_queue++;
661086df 5048 }
9d5c8243
AK		 5049 }
5050
5051 if (tx_ring->detect_tx_hung) {
5052 /* Detect a transmit hang in hardware, this serializes the
5053 * check with the clearing of time_stamp and movement of i */
5054 tx_ring->detect_tx_hung = false;
5055 if (tx_ring->buffer_info[i].time_stamp &&
5056 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5057 (adapter->tx_timeout_factor * HZ)) &&
5058 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5059
9d5c8243 5060 /* detected Tx unit hang */
80785298 5061 dev_err(&tx_ring->pdev->dev,
9d5c8243 5062 "Detected Tx Unit Hang\n"
2d064c06 5063 " Tx Queue <%d>\n"
9d5c8243
AK		 5064 " TDH <%x>\n"
5065 " TDT <%x>\n"
5066 " next_to_use <%x>\n"
5067 " next_to_clean <%x>\n"
9d5c8243
AK		 5068 "buffer_info[next_to_clean]\n"
5069 " time_stamp <%lx>\n"
0e014cb1 5070 " next_to_watch <%x>\n"
9d5c8243
AK		 5071 " jiffies <%lx>\n"
5072 " desc.status <%x>\n",
2d064c06 5073 tx_ring->queue_index,
fce99e34
AD		 5074 readl(tx_ring->head),
5075 readl(tx_ring->tail),
9d5c8243
AK		 5076 tx_ring->next_to_use,
5077 tx_ring->next_to_clean,
f7ba205e 5078 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5079 eop,
9d5c8243 5080 jiffies,
0e014cb1 5081 eop_desc->wb.status);
661086df 5082 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5083 }
5084 }
5085 tx_ring->total_bytes += total_bytes;
5086 tx_ring->total_packets += total_packets;
e21ed353
AD		 5087 tx_ring->tx_stats.bytes += total_bytes;
5088 tx_ring->tx_stats.packets += total_packets;
0e014cb1 5089 return (count < tx_ring->count);
9d5c8243
AK		 5090}
5091
9d5c8243
AK		 5092/**
5093 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 5094 * @q_vector: structure containing interrupt and ring information
5095 * @skb: packet to send up
5096 * @vlan_tag: vlan tag for packet
9d5c8243 5097 **/
047e0030
AD		 5098static void igb_receive_skb(struct igb_q_vector *q_vector,
5099 struct sk_buff *skb,
5100 u16 vlan_tag)
5101{
5102 struct igb_adapter *adapter = q_vector->adapter;
5103
31b24b95 5104 if (vlan_tag && adapter->vlgrp)
047e0030
AD		 5105 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5106 vlan_tag, skb);
182ff8df 5107 else
047e0030 5108 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 5109}
5110
04a5fcaa 5111static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5112 u32 status_err, struct sk_buff *skb)
5113{
5114 skb->ip_summed = CHECKSUM_NONE;
5115
5116 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5117 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5118 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5119 return;
85ad76b2 5120
9d5c8243
AK		 5121 /* TCP/UDP checksum error bit is set */
5122 if (status_err &
5123 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5124 /*
5125 * work around errata with sctp packets where the TCPE aka
5126 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5127 * packets, (aka let the stack check the crc32c)
5128 */
85ad76b2
AD		 5129 if ((skb->len == 60) &&
5130 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 5131 ring->rx_stats.csum_err++;
85ad76b2 5132
9d5c8243 5133 /* let the stack verify checksum errors */
9d5c8243
AK		 5134 return;
5135 }
5136 /* It must be a TCP or UDP packet with a valid checksum */
5137 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5138 skb->ip_summed = CHECKSUM_UNNECESSARY;
5139
85ad76b2 5140 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5141}
5142
757b77e2 5143static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
c5b9bd5e
AD		 5144 struct sk_buff *skb)
5145{
5146 struct igb_adapter *adapter = q_vector->adapter;
5147 struct e1000_hw *hw = &adapter->hw;
5148 u64 regval;
5149
5150 /*
5151 * If this bit is set, then the RX registers contain the time stamp. No
5152 * other packet will be time stamped until we read these registers, so
5153 * read the registers to make them available again. Because only one
5154 * packet can be time stamped at a time, we know that the register
5155 * values must belong to this one here and therefore we don't need to
5156 * compare any of the additional attributes stored for it.
5157 *
5158 * If nothing went wrong, then it should have a skb_shared_tx that we
5159 * can turn into a skb_shared_hwtstamps.
5160 */
757b77e2
NN		 5161 if (staterr & E1000_RXDADV_STAT_TSIP) {
5162 u32 *stamp = (u32 *)skb->data;
5163 regval = le32_to_cpu(*(stamp + 2));
5164 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5165 skb_pull(skb, IGB_TS_HDR_LEN);
5166 } else {
5167 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5168 return;
c5b9bd5e 5169
757b77e2
NN		 5170 regval = rd32(E1000_RXSTMPL);
5171 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5172 }
c5b9bd5e
AD		 5173
5174 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5175}
4c844851 5176static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5177 union e1000_adv_rx_desc *rx_desc)
5178{
5179 /* HW will not DMA in data larger than the given buffer, even if it
5180 * parses the (NFS, of course) header to be larger. In that case, it
5181 * fills the header buffer and spills the rest into the page.
5182 */
5183 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5184 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5185 if (hlen > rx_ring->rx_buffer_len)
5186 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5187 return hlen;
5188}
5189
047e0030
AD		 5190static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5191 int *work_done, int budget)
9d5c8243 5192{
047e0030 5193 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5194 struct net_device *netdev = rx_ring->netdev;
80785298 5195 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 5196 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5197 struct igb_buffer *buffer_info , *next_buffer;
5198 struct sk_buff *skb;
9d5c8243
AK		 5199 bool cleaned = false;
5200 int cleaned_count = 0;
d1eff350 5201 int current_node = numa_node_id();
9d5c8243 5202 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5203 unsigned int i;
2d94d8ab
AD		 5204 u32 staterr;
5205 u16 length;
047e0030 5206 u16 vlan_tag;
9d5c8243
AK		 5207
5208 i = rx_ring->next_to_clean;
69d3ca53 5209 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5210 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5211 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5212
5213 while (staterr & E1000_RXD_STAT_DD) {
5214 if (*work_done >= budget)
5215 break;
5216 (*work_done)++;
9d5c8243 5217
69d3ca53
AD		 5218 skb = buffer_info->skb;
5219 prefetch(skb->data - NET_IP_ALIGN);
5220 buffer_info->skb = NULL;
5221
5222 i++;
5223 if (i == rx_ring->count)
5224 i = 0;
42d0781a 5225
69d3ca53
AD		 5226 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5227 prefetch(next_rxd);
5228 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5229
5230 length = le16_to_cpu(rx_desc->wb.upper.length);
5231 cleaned = true;
5232 cleaned_count++;
5233
2d94d8ab 5234 if (buffer_info->dma) {
bf36c1a0 5235 pci_unmap_single(pdev, buffer_info->dma,
4c844851 5236 rx_ring->rx_buffer_len,
bf36c1a0 5237 PCI_DMA_FROMDEVICE);
91615f76 5238 buffer_info->dma = 0;
4c844851 5239 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5240 skb_put(skb, length);
5241 goto send_up;
5242 }
4c844851 5243 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5244 }
5245
5246 if (length) {
9d5c8243 5247 pci_unmap_page(pdev, buffer_info->page_dma,
bf36c1a0 5248 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
9d5c8243 5249 buffer_info->page_dma = 0;
bf36c1a0 5250
aa913403 5251 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 5252 buffer_info->page,
5253 buffer_info->page_offset,
5254 length);
5255
d1eff350
AD		 5256 if ((page_count(buffer_info->page) != 1) ||
5257 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5258 buffer_info->page = NULL;
5259 else
5260 get_page(buffer_info->page);
9d5c8243
AK		 5261
5262 skb->len += length;
5263 skb->data_len += length;
bf36c1a0 5264 skb->truesize += length;
9d5c8243 5265 }
9d5c8243 5266
bf36c1a0 5267 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5268 buffer_info->skb = next_buffer->skb;
5269 buffer_info->dma = next_buffer->dma;
5270 next_buffer->skb = skb;
5271 next_buffer->dma = 0;
bf36c1a0
AD		 5272 goto next_desc;
5273 }
69d3ca53 5274send_up:
9d5c8243
AK		 5275 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5276 dev_kfree_skb_irq(skb);
5277 goto next_desc;
5278 }
9d5c8243 5279
757b77e2
NN		 5280 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5281 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5282 total_bytes += skb->len;
5283 total_packets++;
5284
85ad76b2 5285 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5286
5287 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5288 skb_record_rx_queue(skb, rx_ring->queue_index);
5289
5290 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5291 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5292
047e0030 5293 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5294
9d5c8243
AK		 5295next_desc:
5296 rx_desc->wb.upper.status_error = 0;
5297
5298 /* return some buffers to hardware, one at a time is too slow */
5299 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5300 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5301 cleaned_count = 0;
5302 }
5303
5304 /* use prefetched values */
5305 rx_desc = next_rxd;
5306 buffer_info = next_buffer;
9d5c8243
AK		 5307 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5308 }
bf36c1a0 5309
9d5c8243 5310 rx_ring->next_to_clean = i;
c493ea45 5311 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5312
5313 if (cleaned_count)
3b644cf6 5314 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5315
5316 rx_ring->total_packets += total_packets;
5317 rx_ring->total_bytes += total_bytes;
5318 rx_ring->rx_stats.packets += total_packets;
5319 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5320 return cleaned;
5321}
5322
9d5c8243
AK		 5323/**
5324 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5325 * @adapter: address of board private structure
5326 **/
d7ee5b3a 5327void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5328{
e694e964 5329 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5330 union e1000_adv_rx_desc *rx_desc;
5331 struct igb_buffer *buffer_info;
5332 struct sk_buff *skb;
5333 unsigned int i;
db761762 5334 int bufsz;
9d5c8243
AK		 5335
5336 i = rx_ring->next_to_use;
5337 buffer_info = &rx_ring->buffer_info[i];
5338
4c844851 5339 bufsz = rx_ring->rx_buffer_len;
db761762 5340
9d5c8243
AK		 5341 while (cleaned_count--) {
5342 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5343
6ec43fe6 5344 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5345 if (!buffer_info->page) {
42d0781a 5346 buffer_info->page = netdev_alloc_page(netdev);
bf36c1a0 5347 if (!buffer_info->page) {
04a5fcaa 5348 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5349 goto no_buffers;
5350 }
5351 buffer_info->page_offset = 0;
5352 } else {
5353 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5354 }
5355 buffer_info->page_dma =
80785298 5356 pci_map_page(rx_ring->pdev, buffer_info->page,
bf36c1a0
AD		 5357 buffer_info->page_offset,
5358 PAGE_SIZE / 2,
9d5c8243 5359 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5360 if (pci_dma_mapping_error(rx_ring->pdev,
5361 buffer_info->page_dma)) {
5362 buffer_info->page_dma = 0;
5363 rx_ring->rx_stats.alloc_failed++;
5364 goto no_buffers;
5365 }
9d5c8243
AK		 5366 }
5367
42d0781a
AD		 5368 skb = buffer_info->skb;
5369 if (!skb) {
89d71a66 5370 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5371 if (!skb) {
04a5fcaa 5372 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5373 goto no_buffers;
5374 }
5375
9d5c8243 5376 buffer_info->skb = skb;
42d0781a
AD		 5377 }
5378 if (!buffer_info->dma) {
80785298
AD		 5379 buffer_info->dma = pci_map_single(rx_ring->pdev,
5380 skb->data,
9d5c8243
AK		 5381 bufsz,
5382 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5383 if (pci_dma_mapping_error(rx_ring->pdev,
5384 buffer_info->dma)) {
5385 buffer_info->dma = 0;
5386 rx_ring->rx_stats.alloc_failed++;
5387 goto no_buffers;
5388 }
9d5c8243
AK		 5389 }
5390 /* Refresh the desc even if buffer_addrs didn't change because
5391 * each write-back erases this info. */
6ec43fe6 5392 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5393 rx_desc->read.pkt_addr =
5394 cpu_to_le64(buffer_info->page_dma);
5395 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5396 } else {
42d0781a 5397 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5398 rx_desc->read.hdr_addr = 0;
5399 }
5400
5401 i++;
5402 if (i == rx_ring->count)
5403 i = 0;
5404 buffer_info = &rx_ring->buffer_info[i];
5405 }
5406
5407no_buffers:
5408 if (rx_ring->next_to_use != i) {
5409 rx_ring->next_to_use = i;
5410 if (i == 0)
5411 i = (rx_ring->count - 1);
5412 else
5413 i--;
5414
5415 /* Force memory writes to complete before letting h/w
5416 * know there are new descriptors to fetch. (Only
5417 * applicable for weak-ordered memory model archs,
5418 * such as IA-64). */
5419 wmb();
fce99e34 5420 writel(i, rx_ring->tail);
9d5c8243
AK		 5421 }
5422}
5423
5424/**
5425 * igb_mii_ioctl -
5426 * @netdev:
5427 * @ifreq:
5428 * @cmd:
5429 **/
5430static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5431{
5432 struct igb_adapter *adapter = netdev_priv(netdev);
5433 struct mii_ioctl_data *data = if_mii(ifr);
5434
5435 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5436 return -EOPNOTSUPP;
5437
5438 switch (cmd) {
5439 case SIOCGMIIPHY:
5440 data->phy_id = adapter->hw.phy.addr;
5441 break;
5442 case SIOCGMIIREG:
f5f4cf08
AD		 5443 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5444 &data->val_out))
9d5c8243
AK		 5445 return -EIO;
5446 break;
5447 case SIOCSMIIREG:
5448 default:
5449 return -EOPNOTSUPP;
5450 }
5451 return 0;
5452}
5453
c6cb090b
PO		 5454/**
5455 * igb_hwtstamp_ioctl - control hardware time stamping
5456 * @netdev:
5457 * @ifreq:
5458 * @cmd:
5459 *
33af6bcc
PO		 5460 * Outgoing time stamping can be enabled and disabled. Play nice and
5461 * disable it when requested, although it shouldn't case any overhead
5462 * when no packet needs it. At most one packet in the queue may be
5463 * marked for time stamping, otherwise it would be impossible to tell
5464 * for sure to which packet the hardware time stamp belongs.
5465 *
5466 * Incoming time stamping has to be configured via the hardware
5467 * filters. Not all combinations are supported, in particular event
5468 * type has to be specified. Matching the kind of event packet is
5469 * not supported, with the exception of "all V2 events regardless of
5470 * level 2 or 4".
5471 *
c6cb090b
PO		 5472 **/
5473static int igb_hwtstamp_ioctl(struct net_device *netdev,
5474 struct ifreq *ifr, int cmd)
5475{
33af6bcc
PO		 5476 struct igb_adapter *adapter = netdev_priv(netdev);
5477 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5478 struct hwtstamp_config config;
c5b9bd5e
AD		 5479 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5480 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5481 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5482 bool is_l4 = false;
5483 bool is_l2 = false;
33af6bcc 5484 u32 regval;
c6cb090b
PO		 5485
5486 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5487 return -EFAULT;
5488
5489 /* reserved for future extensions */
5490 if (config.flags)
5491 return -EINVAL;
5492
33af6bcc
PO		 5493 switch (config.tx_type) {
5494 case HWTSTAMP_TX_OFF:
c5b9bd5e 5495 tsync_tx_ctl = 0;
33af6bcc 5496 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5497 break;
5498 default:
5499 return -ERANGE;
5500 }
5501
5502 switch (config.rx_filter) {
5503 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5504 tsync_rx_ctl = 0;
33af6bcc
PO		 5505 break;
5506 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5507 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5508 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5509 case HWTSTAMP_FILTER_ALL:
5510 /*
5511 * register TSYNCRXCFG must be set, therefore it is not
5512 * possible to time stamp both Sync and Delay_Req messages
5513 * => fall back to time stamping all packets
5514 */
c5b9bd5e 5515 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5516 config.rx_filter = HWTSTAMP_FILTER_ALL;
5517 break;
5518 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5519 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5520 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5521 is_l4 = true;
33af6bcc
PO		 5522 break;
5523 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5524 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5525 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5526 is_l4 = true;
33af6bcc
PO		 5527 break;
5528 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5529 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5530 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5531 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5532 is_l2 = true;
5533 is_l4 = true;
33af6bcc
PO		 5534 config.rx_filter = HWTSTAMP_FILTER_SOME;
5535 break;
5536 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5537 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5538 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5539 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5540 is_l2 = true;
5541 is_l4 = true;
33af6bcc
PO		 5542 config.rx_filter = HWTSTAMP_FILTER_SOME;
5543 break;
5544 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5545 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5546 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5547 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5548 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5549 is_l2 = true;
33af6bcc
PO		 5550 break;
5551 default:
5552 return -ERANGE;
5553 }
5554
c5b9bd5e
AD		 5555 if (hw->mac.type == e1000_82575) {
5556 if (tsync_rx_ctl | tsync_tx_ctl)
5557 return -EINVAL;
5558 return 0;
5559 }
5560
757b77e2
NN		 5561 /*
5562 * Per-packet timestamping only works if all packets are
5563 * timestamped, so enable timestamping in all packets as
5564 * long as one rx filter was configured.
5565 */
5566 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5567 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5568 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5569 }
5570
33af6bcc
PO		 5571 /* enable/disable TX */
5572 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5573 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5574 regval |= tsync_tx_ctl;
33af6bcc
PO		 5575 wr32(E1000_TSYNCTXCTL, regval);
5576
c5b9bd5e 5577 /* enable/disable RX */
33af6bcc 5578 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5579 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5580 regval |= tsync_rx_ctl;
33af6bcc 5581 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5582
c5b9bd5e
AD		 5583 /* define which PTP packets are time stamped */
5584 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5585
c5b9bd5e
AD		 5586 /* define ethertype filter for timestamped packets */
5587 if (is_l2)
5588 wr32(E1000_ETQF(3),
5589 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5590 E1000_ETQF_1588 | /* enable timestamping */
5591 ETH_P_1588)); /* 1588 eth protocol type */
5592 else
5593 wr32(E1000_ETQF(3), 0);
5594
5595#define PTP_PORT 319
5596 /* L4 Queue Filter[3]: filter by destination port and protocol */
5597 if (is_l4) {
5598 u32 ftqf = (IPPROTO_UDP /* UDP */
5599 | E1000_FTQF_VF_BP /* VF not compared */
5600 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5601 | E1000_FTQF_MASK); /* mask all inputs */
5602 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5603
5604 wr32(E1000_IMIR(3), htons(PTP_PORT));
5605 wr32(E1000_IMIREXT(3),
5606 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5607 if (hw->mac.type == e1000_82576) {
5608 /* enable source port check */
5609 wr32(E1000_SPQF(3), htons(PTP_PORT));
5610 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5611 }
5612 wr32(E1000_FTQF(3), ftqf);
5613 } else {
5614 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5615 }
33af6bcc
PO		 5616 wrfl();
5617
5618 adapter->hwtstamp_config = config;
5619
5620 /* clear TX/RX time stamp registers, just to be sure */
5621 regval = rd32(E1000_TXSTMPH);
5622 regval = rd32(E1000_RXSTMPH);
c6cb090b 5623
33af6bcc
PO		 5624 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5625 -EFAULT : 0;
c6cb090b
PO		 5626}
5627
9d5c8243
AK		 5628/**
5629 * igb_ioctl -
5630 * @netdev:
5631 * @ifreq:
5632 * @cmd:
5633 **/
5634static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5635{
5636 switch (cmd) {
5637 case SIOCGMIIPHY:
5638 case SIOCGMIIREG:
5639 case SIOCSMIIREG:
5640 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5641 case SIOCSHWTSTAMP:
5642 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5643 default:
5644 return -EOPNOTSUPP;
5645 }
5646}
5647
009bc06e
AD		 5648s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5649{
5650 struct igb_adapter *adapter = hw->back;
5651 u16 cap_offset;
5652
5653 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5654 if (!cap_offset)
5655 return -E1000_ERR_CONFIG;
5656
5657 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5658
5659 return 0;
5660}
5661
5662s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5663{
5664 struct igb_adapter *adapter = hw->back;
5665 u16 cap_offset;
5666
5667 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5668 if (!cap_offset)
5669 return -E1000_ERR_CONFIG;
5670
5671 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5672
5673 return 0;
5674}
5675
9d5c8243
AK		 5676static void igb_vlan_rx_register(struct net_device *netdev,
5677 struct vlan_group *grp)
5678{
5679 struct igb_adapter *adapter = netdev_priv(netdev);
5680 struct e1000_hw *hw = &adapter->hw;
5681 u32 ctrl, rctl;
5682
5683 igb_irq_disable(adapter);
5684 adapter->vlgrp = grp;
5685
5686 if (grp) {
5687 /* enable VLAN tag insert/strip */
5688 ctrl = rd32(E1000_CTRL);
5689 ctrl |= E1000_CTRL_VME;
5690 wr32(E1000_CTRL, ctrl);
5691
51466239 5692 /* Disable CFI check */
9d5c8243 5693 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 5694 rctl &= ~E1000_RCTL_CFIEN;
5695 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 5696 } else {
5697 /* disable VLAN tag insert/strip */
5698 ctrl = rd32(E1000_CTRL);
5699 ctrl &= ~E1000_CTRL_VME;
5700 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 5701 }
5702
e1739522
AD		 5703 igb_rlpml_set(adapter);
5704
9d5c8243
AK		 5705 if (!test_bit(__IGB_DOWN, &adapter->state))
5706 igb_irq_enable(adapter);
5707}
5708
5709static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5710{
5711 struct igb_adapter *adapter = netdev_priv(netdev);
5712 struct e1000_hw *hw = &adapter->hw;
4ae196df 5713 int pf_id = adapter->vfs_allocated_count;
9d5c8243 5714
51466239
AD		 5715 /* attempt to add filter to vlvf array */
5716 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 5717
51466239
AD		 5718 /* add the filter since PF can receive vlans w/o entry in vlvf */
5719 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 5720}
5721
5722static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5723{
5724 struct igb_adapter *adapter = netdev_priv(netdev);
5725 struct e1000_hw *hw = &adapter->hw;
4ae196df 5726 int pf_id = adapter->vfs_allocated_count;
51466239 5727 s32 err;
9d5c8243
AK		 5728
5729 igb_irq_disable(adapter);
5730 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5731
5732 if (!test_bit(__IGB_DOWN, &adapter->state))
5733 igb_irq_enable(adapter);
5734
51466239
AD		 5735 /* remove vlan from VLVF table array */
5736 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 5737
51466239
AD		 5738 /* if vid was not present in VLVF just remove it from table */
5739 if (err)
4ae196df 5740 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 5741}
5742
5743static void igb_restore_vlan(struct igb_adapter *adapter)
5744{
5745 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5746
5747 if (adapter->vlgrp) {
5748 u16 vid;
5749 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5750 if (!vlan_group_get_device(adapter->vlgrp, vid))
5751 continue;
5752 igb_vlan_rx_add_vid(adapter->netdev, vid);
5753 }
5754 }
5755}
5756
5757int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5758{
090b1795 5759 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 5760 struct e1000_mac_info *mac = &adapter->hw.mac;
5761
5762 mac->autoneg = 0;
5763
9d5c8243
AK		 5764 switch (spddplx) {
5765 case SPEED_10 + DUPLEX_HALF:
5766 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5767 break;
5768 case SPEED_10 + DUPLEX_FULL:
5769 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5770 break;
5771 case SPEED_100 + DUPLEX_HALF:
5772 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5773 break;
5774 case SPEED_100 + DUPLEX_FULL:
5775 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5776 break;
5777 case SPEED_1000 + DUPLEX_FULL:
5778 mac->autoneg = 1;
5779 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5780 break;
5781 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5782 default:
090b1795 5783 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 5784 return -EINVAL;
5785 }
5786 return 0;
5787}
5788
3fe7c4c9 5789static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 5790{
5791 struct net_device *netdev = pci_get_drvdata(pdev);
5792 struct igb_adapter *adapter = netdev_priv(netdev);
5793 struct e1000_hw *hw = &adapter->hw;
2d064c06 5794 u32 ctrl, rctl, status;
9d5c8243
AK		 5795 u32 wufc = adapter->wol;
5796#ifdef CONFIG_PM
5797 int retval = 0;
5798#endif
5799
5800 netif_device_detach(netdev);
5801
a88f10ec
AD		 5802 if (netif_running(netdev))
5803 igb_close(netdev);
5804
047e0030 5805 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 5806
5807#ifdef CONFIG_PM
5808 retval = pci_save_state(pdev);
5809 if (retval)
5810 return retval;
5811#endif
5812
5813 status = rd32(E1000_STATUS);
5814 if (status & E1000_STATUS_LU)
5815 wufc &= ~E1000_WUFC_LNKC;
5816
5817 if (wufc) {
5818 igb_setup_rctl(adapter);
ff41f8dc 5819 igb_set_rx_mode(netdev);
9d5c8243
AK		 5820
5821 /* turn on all-multi mode if wake on multicast is enabled */
5822 if (wufc & E1000_WUFC_MC) {
5823 rctl = rd32(E1000_RCTL);
5824 rctl |= E1000_RCTL_MPE;
5825 wr32(E1000_RCTL, rctl);
5826 }
5827
5828 ctrl = rd32(E1000_CTRL);
5829 /* advertise wake from D3Cold */
5830 #define E1000_CTRL_ADVD3WUC 0x00100000
5831 /* phy power management enable */
5832 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5833 ctrl |= E1000_CTRL_ADVD3WUC;
5834 wr32(E1000_CTRL, ctrl);
5835
9d5c8243 5836 /* Allow time for pending master requests to run */
330a6d6a 5837 igb_disable_pcie_master(hw);
9d5c8243
AK		 5838
5839 wr32(E1000_WUC, E1000_WUC_PME_EN);
5840 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 5841 } else {
5842 wr32(E1000_WUC, 0);
5843 wr32(E1000_WUFC, 0);
9d5c8243
AK		 5844 }
5845
3fe7c4c9
RW		 5846 *enable_wake = wufc || adapter->en_mng_pt;
5847 if (!*enable_wake)
88a268c1
NN		 5848 igb_power_down_link(adapter);
5849 else
5850 igb_power_up_link(adapter);
9d5c8243
AK		 5851
5852 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5853 * would have already happened in close and is redundant. */
5854 igb_release_hw_control(adapter);
5855
5856 pci_disable_device(pdev);
5857
9d5c8243
AK		 5858 return 0;
5859}
5860
5861#ifdef CONFIG_PM
3fe7c4c9
RW		 5862static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5863{
5864 int retval;
5865 bool wake;
5866
5867 retval = __igb_shutdown(pdev, &wake);
5868 if (retval)
5869 return retval;
5870
5871 if (wake) {
5872 pci_prepare_to_sleep(pdev);
5873 } else {
5874 pci_wake_from_d3(pdev, false);
5875 pci_set_power_state(pdev, PCI_D3hot);
5876 }
5877
5878 return 0;
5879}
5880
9d5c8243
AK		 5881static int igb_resume(struct pci_dev *pdev)
5882{
5883 struct net_device *netdev = pci_get_drvdata(pdev);
5884 struct igb_adapter *adapter = netdev_priv(netdev);
5885 struct e1000_hw *hw = &adapter->hw;
5886 u32 err;
5887
5888 pci_set_power_state(pdev, PCI_D0);
5889 pci_restore_state(pdev);
b94f2d77 5890 pci_save_state(pdev);
42bfd33a 5891
aed5dec3 5892 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 5893 if (err) {
5894 dev_err(&pdev->dev,
5895 "igb: Cannot enable PCI device from suspend\n");
5896 return err;
5897 }
5898 pci_set_master(pdev);
5899
5900 pci_enable_wake(pdev, PCI_D3hot, 0);
5901 pci_enable_wake(pdev, PCI_D3cold, 0);
5902
047e0030 5903 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 5904 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5905 return -ENOMEM;
9d5c8243
AK		 5906 }
5907
9d5c8243 5908 igb_reset(adapter);
a8564f03
AD		 5909
5910 /* let the f/w know that the h/w is now under the control of the
5911 * driver. */
5912 igb_get_hw_control(adapter);
5913
9d5c8243
AK		 5914 wr32(E1000_WUS, ~0);
5915
a88f10ec
AD		 5916 if (netif_running(netdev)) {
5917 err = igb_open(netdev);
5918 if (err)
5919 return err;
5920 }
9d5c8243
AK		 5921
5922 netif_device_attach(netdev);
5923
9d5c8243
AK		 5924 return 0;
5925}
5926#endif
5927
5928static void igb_shutdown(struct pci_dev *pdev)
5929{
3fe7c4c9
RW		 5930 bool wake;
5931
5932 __igb_shutdown(pdev, &wake);
5933
5934 if (system_state == SYSTEM_POWER_OFF) {
5935 pci_wake_from_d3(pdev, wake);
5936 pci_set_power_state(pdev, PCI_D3hot);
5937 }
9d5c8243
AK		 5938}
5939
5940#ifdef CONFIG_NET_POLL_CONTROLLER
5941/*
5942 * Polling 'interrupt' - used by things like netconsole to send skbs
5943 * without having to re-enable interrupts. It's not called while
5944 * the interrupt routine is executing.
5945 */
5946static void igb_netpoll(struct net_device *netdev)
5947{
5948 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 5949 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5950 int i;
9d5c8243 5951
eebbbdba 5952 if (!adapter->msix_entries) {
047e0030 5953 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 5954 igb_irq_disable(adapter);
047e0030 5955 napi_schedule(&q_vector->napi);
eebbbdba
AD		 5956 return;
5957 }
9d5c8243 5958
047e0030
AD		 5959 for (i = 0; i < adapter->num_q_vectors; i++) {
5960 struct igb_q_vector *q_vector = adapter->q_vector[i];
5961 wr32(E1000_EIMC, q_vector->eims_value);
5962 napi_schedule(&q_vector->napi);
eebbbdba 5963 }
9d5c8243
AK		 5964}
5965#endif /* CONFIG_NET_POLL_CONTROLLER */
5966
5967/**
5968 * igb_io_error_detected - called when PCI error is detected
5969 * @pdev: Pointer to PCI device
5970 * @state: The current pci connection state
5971 *
5972 * This function is called after a PCI bus error affecting
5973 * this device has been detected.
5974 */
5975static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5976 pci_channel_state_t state)
5977{
5978 struct net_device *netdev = pci_get_drvdata(pdev);
5979 struct igb_adapter *adapter = netdev_priv(netdev);
5980
5981 netif_device_detach(netdev);
5982
59ed6eec
AD		 5983 if (state == pci_channel_io_perm_failure)
5984 return PCI_ERS_RESULT_DISCONNECT;
5985
9d5c8243
AK		 5986 if (netif_running(netdev))
5987 igb_down(adapter);
5988 pci_disable_device(pdev);
5989
5990 /* Request a slot slot reset. */
5991 return PCI_ERS_RESULT_NEED_RESET;
5992}
5993
5994/**
5995 * igb_io_slot_reset - called after the pci bus has been reset.
5996 * @pdev: Pointer to PCI device
5997 *
5998 * Restart the card from scratch, as if from a cold-boot. Implementation
5999 * resembles the first-half of the igb_resume routine.
6000 */
6001static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6002{
6003 struct net_device *netdev = pci_get_drvdata(pdev);
6004 struct igb_adapter *adapter = netdev_priv(netdev);
6005 struct e1000_hw *hw = &adapter->hw;
40a914fa 6006 pci_ers_result_t result;
42bfd33a 6007 int err;
9d5c8243 6008
aed5dec3 6009 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6010 dev_err(&pdev->dev,
6011 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6012 result = PCI_ERS_RESULT_DISCONNECT;
6013 } else {
6014 pci_set_master(pdev);
6015 pci_restore_state(pdev);
b94f2d77 6016 pci_save_state(pdev);
9d5c8243 6017
40a914fa
AD		 6018 pci_enable_wake(pdev, PCI_D3hot, 0);
6019 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6020
40a914fa
AD		 6021 igb_reset(adapter);
6022 wr32(E1000_WUS, ~0);
6023 result = PCI_ERS_RESULT_RECOVERED;
6024 }
9d5c8243 6025
ea943d41
JK		 6026 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6027 if (err) {
6028 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6029 "failed 0x%0x\n", err);
6030 /* non-fatal, continue */
6031 }
40a914fa
AD		 6032
6033 return result;
9d5c8243
AK		 6034}
6035
6036/**
6037 * igb_io_resume - called when traffic can start flowing again.
6038 * @pdev: Pointer to PCI device
6039 *
6040 * This callback is called when the error recovery driver tells us that
6041 * its OK to resume normal operation. Implementation resembles the
6042 * second-half of the igb_resume routine.
6043 */
6044static void igb_io_resume(struct pci_dev *pdev)
6045{
6046 struct net_device *netdev = pci_get_drvdata(pdev);
6047 struct igb_adapter *adapter = netdev_priv(netdev);
6048
9d5c8243
AK		 6049 if (netif_running(netdev)) {
6050 if (igb_up(adapter)) {
6051 dev_err(&pdev->dev, "igb_up failed after reset\n");
6052 return;
6053 }
6054 }
6055
6056 netif_device_attach(netdev);
6057
6058 /* let the f/w know that the h/w is now under the control of the
6059 * driver. */
6060 igb_get_hw_control(adapter);
9d5c8243
AK		 6061}
6062
26ad9178
AD		 6063static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6064 u8 qsel)
6065{
6066 u32 rar_low, rar_high;
6067 struct e1000_hw *hw = &adapter->hw;
6068
6069 /* HW expects these in little endian so we reverse the byte order
6070 * from network order (big endian) to little endian
6071 */
6072 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6073 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6074 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6075
6076 /* Indicate to hardware the Address is Valid. */
6077 rar_high |= E1000_RAH_AV;
6078
6079 if (hw->mac.type == e1000_82575)
6080 rar_high |= E1000_RAH_POOL_1 * qsel;
6081 else
6082 rar_high |= E1000_RAH_POOL_1 << qsel;
6083
6084 wr32(E1000_RAL(index), rar_low);
6085 wrfl();
6086 wr32(E1000_RAH(index), rar_high);
6087 wrfl();
6088}
6089
4ae196df
AD		 6090static int igb_set_vf_mac(struct igb_adapter *adapter,
6091 int vf, unsigned char *mac_addr)
6092{
6093 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6094 /* VF MAC addresses start at end of receive addresses and moves
6095 * torwards the first, as a result a collision should not be possible */
6096 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6097
37680117 6098 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6099
26ad9178 6100 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6101
6102 return 0;
6103}
6104
8151d294
WM		 6105static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6106{
6107 struct igb_adapter *adapter = netdev_priv(netdev);
6108 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6109 return -EINVAL;
6110 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6111 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6112 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6113 " change effective.");
6114 if (test_bit(__IGB_DOWN, &adapter->state)) {
6115 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6116 " but the PF device is not up.\n");
6117 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6118 " attempting to use the VF device.\n");
6119 }
6120 return igb_set_vf_mac(adapter, vf, mac);
6121}
6122
6123static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6124{
6125 return -EOPNOTSUPP;
6126}
6127
6128static int igb_ndo_get_vf_config(struct net_device *netdev,
6129 int vf, struct ifla_vf_info *ivi)
6130{
6131 struct igb_adapter *adapter = netdev_priv(netdev);
6132 if (vf >= adapter->vfs_allocated_count)
6133 return -EINVAL;
6134 ivi->vf = vf;
6135 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6136 ivi->tx_rate = 0;
6137 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6138 ivi->qos = adapter->vf_data[vf].pf_qos;
6139 return 0;
6140}
6141
4ae196df
AD		 6142static void igb_vmm_control(struct igb_adapter *adapter)
6143{
6144 struct e1000_hw *hw = &adapter->hw;
10d8e907 6145 u32 reg;
4ae196df 6146
52a1dd4d
AD		 6147 switch (hw->mac.type) {
6148 case e1000_82575:
6149 default:
6150 /* replication is not supported for 82575 */
4ae196df 6151 return;
52a1dd4d
AD		 6152 case e1000_82576:
6153 /* notify HW that the MAC is adding vlan tags */
6154 reg = rd32(E1000_DTXCTL);
6155 reg |= E1000_DTXCTL_VLAN_ADDED;
6156 wr32(E1000_DTXCTL, reg);
6157 case e1000_82580:
6158 /* enable replication vlan tag stripping */
6159 reg = rd32(E1000_RPLOLR);
6160 reg |= E1000_RPLOLR_STRVLAN;
6161 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6162 case e1000_i350:
6163 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6164 break;
6165 }
10d8e907 6166
d4960307
AD		 6167 if (adapter->vfs_allocated_count) {
6168 igb_vmdq_set_loopback_pf(hw, true);
6169 igb_vmdq_set_replication_pf(hw, true);
6170 } else {
6171 igb_vmdq_set_loopback_pf(hw, false);
6172 igb_vmdq_set_replication_pf(hw, false);
6173 }
4ae196df
AD		 6174}
6175
9d5c8243 6176/* igb_main.c */
Linux kernel - Deliverables
This page took 0.930398 seconds and 5 git commands to generate.