projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: add pci_dev in few spots to clean up use of dev_err/info/warn
[deliverable/linux.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243
AK		 34#include <linux/ipv6.h>
35#include <net/checksum.h>
36#include <net/ip6_checksum.h>
c6cb090b 37#include <linux/net_tstamp.h>
9d5c8243
AK		 38#include <linux/mii.h>
39#include <linux/ethtool.h>
40#include <linux/if_vlan.h>
41#include <linux/pci.h>
c54106bb 42#include <linux/pci-aspm.h>
9d5c8243
AK		 43#include <linux/delay.h>
44#include <linux/interrupt.h>
45#include <linux/if_ether.h>
40a914fa 46#include <linux/aer.h>
421e02f0 47#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 48#include <linux/dca.h>
49#endif
9d5c8243
AK		 50#include "igb.h"
51
86d5d38f 52#define DRV_VERSION "1.3.16-k2"
9d5c8243
AK		 53char igb_driver_name[] = "igb";
54char igb_driver_version[] = DRV_VERSION;
55static const char igb_driver_string[] =
56 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 57static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 58
9d5c8243
AK		 59static const struct e1000_info *igb_info_tbl[] = {
60 [board_82575] = &e1000_82575_info,
61};
62
63static struct pci_device_id igb_pci_tbl[] = {
2d064c06 64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
c8ea5ea9 70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
74 /* required last entry */
75 {0, }
76};
77
78MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
79
80void igb_reset(struct igb_adapter *);
81static int igb_setup_all_tx_resources(struct igb_adapter *);
82static int igb_setup_all_rx_resources(struct igb_adapter *);
83static void igb_free_all_tx_resources(struct igb_adapter *);
84static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 85static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 86void igb_update_stats(struct igb_adapter *);
87static int igb_probe(struct pci_dev *, const struct pci_device_id *);
88static void __devexit igb_remove(struct pci_dev *pdev);
89static int igb_sw_init(struct igb_adapter *);
90static int igb_open(struct net_device *);
91static int igb_close(struct net_device *);
92static void igb_configure_tx(struct igb_adapter *);
93static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 94static void igb_clean_all_tx_rings(struct igb_adapter *);
95static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 96static void igb_clean_tx_ring(struct igb_ring *);
97static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 98static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 99static void igb_update_phy_info(unsigned long);
100static void igb_watchdog(unsigned long);
101static void igb_watchdog_task(struct work_struct *);
b1a436c3 102static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 103static struct net_device_stats *igb_get_stats(struct net_device *);
104static int igb_change_mtu(struct net_device *, int);
105static int igb_set_mac(struct net_device *, void *);
68d480c4 106static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 107static irqreturn_t igb_intr(int irq, void *);
108static irqreturn_t igb_intr_msi(int irq, void *);
109static irqreturn_t igb_msix_other(int irq, void *);
047e0030 110static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 111#ifdef CONFIG_IGB_DCA
047e0030 112static void igb_update_dca(struct igb_q_vector *);
fe4506b6 113static void igb_setup_dca(struct igb_adapter *);
421e02f0 114#endif /* CONFIG_IGB_DCA */
047e0030 115static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 116static int igb_poll(struct napi_struct *, int);
047e0030 117static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 118static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
119static void igb_tx_timeout(struct net_device *);
120static void igb_reset_task(struct work_struct *);
121static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
122static void igb_vlan_rx_add_vid(struct net_device *, u16);
123static void igb_vlan_rx_kill_vid(struct net_device *, u16);
124static void igb_restore_vlan(struct igb_adapter *);
26ad9178 125static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 126static void igb_ping_all_vfs(struct igb_adapter *);
127static void igb_msg_task(struct igb_adapter *);
4ae196df 128static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 129static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 130static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
9d5c8243 131
9d5c8243 132#ifdef CONFIG_PM
3fe7c4c9 133static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 134static int igb_resume(struct pci_dev *);
135#endif
136static void igb_shutdown(struct pci_dev *);
421e02f0 137#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 138static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
139static struct notifier_block dca_notifier = {
140 .notifier_call = igb_notify_dca,
141 .next = NULL,
142 .priority = 0
143};
144#endif
9d5c8243
AK		 145#ifdef CONFIG_NET_POLL_CONTROLLER
146/* for netdump / net console */
147static void igb_netpoll(struct net_device *);
148#endif
37680117 149#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 150static unsigned int max_vfs = 0;
151module_param(max_vfs, uint, 0);
152MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
153 "per physical function");
154#endif /* CONFIG_PCI_IOV */
155
9d5c8243
AK		 156static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
157 pci_channel_state_t);
158static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
159static void igb_io_resume(struct pci_dev *);
160
161static struct pci_error_handlers igb_err_handler = {
162 .error_detected = igb_io_error_detected,
163 .slot_reset = igb_io_slot_reset,
164 .resume = igb_io_resume,
165};
166
167
168static struct pci_driver igb_driver = {
169 .name = igb_driver_name,
170 .id_table = igb_pci_tbl,
171 .probe = igb_probe,
172 .remove = __devexit_p(igb_remove),
173#ifdef CONFIG_PM
174 /* Power Managment Hooks */
175 .suspend = igb_suspend,
176 .resume = igb_resume,
177#endif
178 .shutdown = igb_shutdown,
179 .err_handler = &igb_err_handler
180};
181
182MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
183MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
184MODULE_LICENSE("GPL");
185MODULE_VERSION(DRV_VERSION);
186
38c845c7
PO		 187/**
188 * igb_read_clock - read raw cycle counter (to be used by time counter)
189 */
190static cycle_t igb_read_clock(const struct cyclecounter *tc)
191{
192 struct igb_adapter *adapter =
193 container_of(tc, struct igb_adapter, cycles);
194 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 195 u64 stamp = 0;
196 int shift = 0;
38c845c7 197
c5b9bd5e
AD		 198 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
199 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 200 return stamp;
201}
202
9d5c8243
AK		 203#ifdef DEBUG
204/**
205 * igb_get_hw_dev_name - return device name string
206 * used by hardware layer to print debugging information
207 **/
208char *igb_get_hw_dev_name(struct e1000_hw *hw)
209{
210 struct igb_adapter *adapter = hw->back;
211 return adapter->netdev->name;
212}
38c845c7
PO		 213
214/**
215 * igb_get_time_str - format current NIC and system time as string
216 */
217static char *igb_get_time_str(struct igb_adapter *adapter,
218 char buffer[160])
219{
220 cycle_t hw = adapter->cycles.read(&adapter->cycles);
221 struct timespec nic = ns_to_timespec(timecounter_read(&adapter->clock));
222 struct timespec sys;
223 struct timespec delta;
224 getnstimeofday(&sys);
225
226 delta = timespec_sub(nic, sys);
227
228 sprintf(buffer,
33af6bcc
PO		 229 "HW %llu, NIC %ld.%09lus, SYS %ld.%09lus, NIC-SYS %lds + %09luns",
230 hw,
38c845c7
PO		 231 (long)nic.tv_sec, nic.tv_nsec,
232 (long)sys.tv_sec, sys.tv_nsec,
233 (long)delta.tv_sec, delta.tv_nsec);
234
235 return buffer;
236}
9d5c8243
AK		 237#endif
238
239/**
240 * igb_init_module - Driver Registration Routine
241 *
242 * igb_init_module is the first routine called when the driver is
243 * loaded. All it does is register with the PCI subsystem.
244 **/
245static int __init igb_init_module(void)
246{
247 int ret;
248 printk(KERN_INFO "%s - version %s\n",
249 igb_driver_string, igb_driver_version);
250
251 printk(KERN_INFO "%s\n", igb_copyright);
252
421e02f0 253#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 254 dca_register_notify(&dca_notifier);
255#endif
bbd98fe4 256 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 257 return ret;
258}
259
260module_init(igb_init_module);
261
262/**
263 * igb_exit_module - Driver Exit Cleanup Routine
264 *
265 * igb_exit_module is called just before the driver is removed
266 * from memory.
267 **/
268static void __exit igb_exit_module(void)
269{
421e02f0 270#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 271 dca_unregister_notify(&dca_notifier);
272#endif
9d5c8243
AK		 273 pci_unregister_driver(&igb_driver);
274}
275
276module_exit(igb_exit_module);
277
26bc19ec
AD		 278#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
279/**
280 * igb_cache_ring_register - Descriptor ring to register mapping
281 * @adapter: board private structure to initialize
282 *
283 * Once we know the feature-set enabled for the device, we'll cache
284 * the register offset the descriptor ring is assigned to.
285 **/
286static void igb_cache_ring_register(struct igb_adapter *adapter)
287{
ee1b9f06 288 int i = 0, j = 0;
047e0030 289 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 290
291 switch (adapter->hw.mac.type) {
292 case e1000_82576:
293 /* The queues are allocated for virtualization such that VF 0
294 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
295 * In order to avoid collision we start at the first free queue
296 * and continue consuming queues in the same sequence
297 */
ee1b9f06
AD		 298 if (adapter->vfs_allocated_count) {
299 for (; i < adapter->num_rx_queues; i++)
300 adapter->rx_ring[i].reg_idx = rbase_offset +
301 Q_IDX_82576(i);
302 for (; j < adapter->num_tx_queues; j++)
303 adapter->tx_ring[j].reg_idx = rbase_offset +
304 Q_IDX_82576(j);
305 }
26bc19ec
AD		 306 case e1000_82575:
307 default:
ee1b9f06
AD		 308 for (; i < adapter->num_rx_queues; i++)
309 adapter->rx_ring[i].reg_idx = rbase_offset + i;
310 for (; j < adapter->num_tx_queues; j++)
311 adapter->tx_ring[j].reg_idx = rbase_offset + j;
26bc19ec
AD		 312 break;
313 }
314}
315
047e0030
AD		 316static void igb_free_queues(struct igb_adapter *adapter)
317{
318 kfree(adapter->tx_ring);
319 kfree(adapter->rx_ring);
320
321 adapter->tx_ring = NULL;
322 adapter->rx_ring = NULL;
323
324 adapter->num_rx_queues = 0;
325 adapter->num_tx_queues = 0;
326}
327
9d5c8243
AK		 328/**
329 * igb_alloc_queues - Allocate memory for all rings
330 * @adapter: board private structure to initialize
331 *
332 * We allocate one ring per queue at run-time since we don't know the
333 * number of queues at compile-time.
334 **/
335static int igb_alloc_queues(struct igb_adapter *adapter)
336{
337 int i;
338
339 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
340 sizeof(struct igb_ring), GFP_KERNEL);
341 if (!adapter->tx_ring)
047e0030 342 goto err;
9d5c8243
AK		 343
344 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
345 sizeof(struct igb_ring), GFP_KERNEL);
047e0030
AD		 346 if (!adapter->rx_ring)
347 goto err;
6eb5a7f1 348
661086df
PWJ		 349 for (i = 0; i < adapter->num_tx_queues; i++) {
350 struct igb_ring *ring = &(adapter->tx_ring[i]);
68fd9910 351 ring->count = adapter->tx_ring_count;
661086df 352 ring->queue_index = i;
80785298 353 ring->pdev = adapter->pdev;
e694e964 354 ring->netdev = adapter->netdev;
85ad76b2
AD		 355 /* For 82575, context index must be unique per ring. */
356 if (adapter->hw.mac.type == e1000_82575)
357 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
661086df 358 }
85ad76b2 359
9d5c8243
AK		 360 for (i = 0; i < adapter->num_rx_queues; i++) {
361 struct igb_ring *ring = &(adapter->rx_ring[i]);
68fd9910 362 ring->count = adapter->rx_ring_count;
844290e5 363 ring->queue_index = i;
80785298 364 ring->pdev = adapter->pdev;
e694e964 365 ring->netdev = adapter->netdev;
4c844851 366 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 367 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
368 /* set flag indicating ring supports SCTP checksum offload */
369 if (adapter->hw.mac.type >= e1000_82576)
370 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
9d5c8243 371 }
26bc19ec
AD		 372
373 igb_cache_ring_register(adapter);
9d5c8243 374
047e0030 375 return 0;
a88f10ec 376
047e0030
AD		 377err:
378 igb_free_queues(adapter);
d1a8c9e1 379
047e0030 380 return -ENOMEM;
a88f10ec
AD		 381}
382
9d5c8243 383#define IGB_N0_QUEUE -1
047e0030 384static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 385{
386 u32 msixbm = 0;
047e0030 387 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 388 struct e1000_hw *hw = &adapter->hw;
2d064c06 389 u32 ivar, index;
047e0030
AD		 390 int rx_queue = IGB_N0_QUEUE;
391 int tx_queue = IGB_N0_QUEUE;
392
393 if (q_vector->rx_ring)
394 rx_queue = q_vector->rx_ring->reg_idx;
395 if (q_vector->tx_ring)
396 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 397
398 switch (hw->mac.type) {
399 case e1000_82575:
9d5c8243
AK		 400 /* The 82575 assigns vectors using a bitmask, which matches the
401 bitmask for the EICR/EIMS/EIMC registers. To assign one
402 or more queues to a vector, we write the appropriate bits
403 into the MSIXBM register for that vector. */
047e0030 404 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 405 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 406 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 407 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
9d5c8243 408 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 409 q_vector->eims_value = msixbm;
2d064c06
AD		 410 break;
411 case e1000_82576:
26bc19ec 412 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 413 Each queue has a single entry in the table to which we write
414 a vector number along with a "valid" bit. Sadly, the layout
415 of the table is somewhat counterintuitive. */
416 if (rx_queue > IGB_N0_QUEUE) {
047e0030 417 index = (rx_queue & 0x7);
2d064c06 418 ivar = array_rd32(E1000_IVAR0, index);
047e0030 419 if (rx_queue < 8) {
26bc19ec
AD		 420 /* vector goes into low byte of register */
421 ivar = ivar & 0xFFFFFF00;
422 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 423 } else {
424 /* vector goes into third byte of register */
425 ivar = ivar & 0xFF00FFFF;
426 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 427 }
2d064c06
AD		 428 array_wr32(E1000_IVAR0, index, ivar);
429 }
430 if (tx_queue > IGB_N0_QUEUE) {
047e0030 431 index = (tx_queue & 0x7);
2d064c06 432 ivar = array_rd32(E1000_IVAR0, index);
047e0030 433 if (tx_queue < 8) {
26bc19ec
AD		 434 /* vector goes into second byte of register */
435 ivar = ivar & 0xFFFF00FF;
436 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 437 } else {
438 /* vector goes into high byte of register */
439 ivar = ivar & 0x00FFFFFF;
440 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 441 }
2d064c06
AD		 442 array_wr32(E1000_IVAR0, index, ivar);
443 }
047e0030 444 q_vector->eims_value = 1 << msix_vector;
2d064c06
AD		 445 break;
446 default:
447 BUG();
448 break;
449 }
9d5c8243
AK		 450}
451
452/**
453 * igb_configure_msix - Configure MSI-X hardware
454 *
455 * igb_configure_msix sets up the hardware to properly
456 * generate MSI-X interrupts.
457 **/
458static void igb_configure_msix(struct igb_adapter *adapter)
459{
460 u32 tmp;
461 int i, vector = 0;
462 struct e1000_hw *hw = &adapter->hw;
463
464 adapter->eims_enable_mask = 0;
9d5c8243
AK		 465
466 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 467 switch (hw->mac.type) {
468 case e1000_82575:
9d5c8243
AK		 469 tmp = rd32(E1000_CTRL_EXT);
470 /* enable MSI-X PBA support*/
471 tmp |= E1000_CTRL_EXT_PBA_CLR;
472
473 /* Auto-Mask interrupts upon ICR read. */
474 tmp |= E1000_CTRL_EXT_EIAME;
475 tmp |= E1000_CTRL_EXT_IRCA;
476
477 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 478
479 /* enable msix_other interrupt */
480 array_wr32(E1000_MSIXBM(0), vector++,
481 E1000_EIMS_OTHER);
844290e5 482 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 483
2d064c06
AD		 484 break;
485
486 case e1000_82576:
047e0030
AD		 487 /* Turn on MSI-X capability first, or our settings
488 * won't stick. And it will take days to debug. */
489 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
490 E1000_GPIE_PBA | E1000_GPIE_EIAME |
491 E1000_GPIE_NSICR);
492
493 /* enable msix_other interrupt */
494 adapter->eims_other = 1 << vector;
2d064c06 495 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 496
047e0030 497 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 498 break;
499 default:
500 /* do nothing, since nothing else supports MSI-X */
501 break;
502 } /* switch (hw->mac.type) */
047e0030
AD		 503
504 adapter->eims_enable_mask |= adapter->eims_other;
505
506 for (i = 0; i < adapter->num_q_vectors; i++) {
507 struct igb_q_vector *q_vector = adapter->q_vector[i];
508 igb_assign_vector(q_vector, vector++);
509 adapter->eims_enable_mask |= q_vector->eims_value;
510 }
511
9d5c8243
AK		 512 wrfl();
513}
514
515/**
516 * igb_request_msix - Initialize MSI-X interrupts
517 *
518 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
519 * kernel.
520 **/
521static int igb_request_msix(struct igb_adapter *adapter)
522{
523 struct net_device *netdev = adapter->netdev;
047e0030 524 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 525 int i, err = 0, vector = 0;
526
047e0030
AD		 527 err = request_irq(adapter->msix_entries[vector].vector,
528 &igb_msix_other, 0, netdev->name, adapter);
529 if (err)
530 goto out;
531 vector++;
532
533 for (i = 0; i < adapter->num_q_vectors; i++) {
534 struct igb_q_vector *q_vector = adapter->q_vector[i];
535
536 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
537
538 if (q_vector->rx_ring && q_vector->tx_ring)
539 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
540 q_vector->rx_ring->queue_index);
541 else if (q_vector->tx_ring)
542 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
543 q_vector->tx_ring->queue_index);
544 else if (q_vector->rx_ring)
545 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
546 q_vector->rx_ring->queue_index);
9d5c8243 547 else
047e0030
AD		 548 sprintf(q_vector->name, "%s-unused", netdev->name);
549
9d5c8243 550 err = request_irq(adapter->msix_entries[vector].vector,
047e0030
AD		 551 &igb_msix_ring, 0, q_vector->name,
552 q_vector);
9d5c8243
AK		 553 if (err)
554 goto out;
9d5c8243
AK		 555 vector++;
556 }
557
9d5c8243
AK		 558 igb_configure_msix(adapter);
559 return 0;
560out:
561 return err;
562}
563
564static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
565{
566 if (adapter->msix_entries) {
567 pci_disable_msix(adapter->pdev);
568 kfree(adapter->msix_entries);
569 adapter->msix_entries = NULL;
047e0030 570 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 571 pci_disable_msi(adapter->pdev);
047e0030 572 }
9d5c8243
AK		 573}
574
047e0030
AD		 575/**
576 * igb_free_q_vectors - Free memory allocated for interrupt vectors
577 * @adapter: board private structure to initialize
578 *
579 * This function frees the memory allocated to the q_vectors. In addition if
580 * NAPI is enabled it will delete any references to the NAPI struct prior
581 * to freeing the q_vector.
582 **/
583static void igb_free_q_vectors(struct igb_adapter *adapter)
584{
585 int v_idx;
586
587 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
588 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
589 adapter->q_vector[v_idx] = NULL;
590 netif_napi_del(&q_vector->napi);
591 kfree(q_vector);
592 }
593 adapter->num_q_vectors = 0;
594}
595
596/**
597 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
598 *
599 * This function resets the device so that it has 0 rx queues, tx queues, and
600 * MSI-X interrupts allocated.
601 */
602static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
603{
604 igb_free_queues(adapter);
605 igb_free_q_vectors(adapter);
606 igb_reset_interrupt_capability(adapter);
607}
9d5c8243
AK		 608
609/**
610 * igb_set_interrupt_capability - set MSI or MSI-X if supported
611 *
612 * Attempt to configure interrupts using the best available
613 * capabilities of the hardware and kernel.
614 **/
615static void igb_set_interrupt_capability(struct igb_adapter *adapter)
616{
617 int err;
618 int numvecs, i;
619
83b7180d 620 /* Number of supported queues. */
83b7180d
AD		 621 adapter->num_rx_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
622 adapter->num_tx_queues = min_t(u32, IGB_MAX_TX_QUEUES, num_online_cpus());
623
047e0030
AD		 624 /* start with one vector for every rx queue */
625 numvecs = adapter->num_rx_queues;
626
627 /* if tx handler is seperate add 1 for every tx queue */
628 numvecs += adapter->num_tx_queues;
629
630 /* store the number of vectors reserved for queues */
631 adapter->num_q_vectors = numvecs;
632
633 /* add 1 vector for link status interrupts */
634 numvecs++;
9d5c8243
AK		 635 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
636 GFP_KERNEL);
637 if (!adapter->msix_entries)
638 goto msi_only;
639
640 for (i = 0; i < numvecs; i++)
641 adapter->msix_entries[i].entry = i;
642
643 err = pci_enable_msix(adapter->pdev,
644 adapter->msix_entries,
645 numvecs);
646 if (err == 0)
34a20e89 647 goto out;
9d5c8243
AK		 648
649 igb_reset_interrupt_capability(adapter);
650
651 /* If we can't do MSI-X, try MSI */
652msi_only:
2a3abf6d
AD		 653#ifdef CONFIG_PCI_IOV
654 /* disable SR-IOV for non MSI-X configurations */
655 if (adapter->vf_data) {
656 struct e1000_hw *hw = &adapter->hw;
657 /* disable iov and allow time for transactions to clear */
658 pci_disable_sriov(adapter->pdev);
659 msleep(500);
660
661 kfree(adapter->vf_data);
662 adapter->vf_data = NULL;
663 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
664 msleep(100);
665 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
666 }
667#endif
4fc82adf
AD		 668 adapter->vfs_allocated_count = 0;
669 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 670 adapter->num_rx_queues = 1;
661086df 671 adapter->num_tx_queues = 1;
047e0030 672 adapter->num_q_vectors = 1;
9d5c8243 673 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 674 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 675out:
661086df 676 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 677 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 678 return;
679}
680
047e0030
AD		 681/**
682 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
683 * @adapter: board private structure to initialize
684 *
685 * We allocate one q_vector per queue interrupt. If allocation fails we
686 * return -ENOMEM.
687 **/
688static int igb_alloc_q_vectors(struct igb_adapter *adapter)
689{
690 struct igb_q_vector *q_vector;
691 struct e1000_hw *hw = &adapter->hw;
692 int v_idx;
693
694 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
695 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
696 if (!q_vector)
697 goto err_out;
698 q_vector->adapter = adapter;
699 q_vector->itr_shift = (hw->mac.type == e1000_82575) ? 16 : 0;
700 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
701 q_vector->itr_val = IGB_START_ITR;
702 q_vector->set_itr = 1;
703 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
704 adapter->q_vector[v_idx] = q_vector;
705 }
706 return 0;
707
708err_out:
709 while (v_idx) {
710 v_idx--;
711 q_vector = adapter->q_vector[v_idx];
712 netif_napi_del(&q_vector->napi);
713 kfree(q_vector);
714 adapter->q_vector[v_idx] = NULL;
715 }
716 return -ENOMEM;
717}
718
719static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
720 int ring_idx, int v_idx)
721{
722 struct igb_q_vector *q_vector;
723
724 q_vector = adapter->q_vector[v_idx];
725 q_vector->rx_ring = &adapter->rx_ring[ring_idx];
726 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 727 q_vector->itr_val = adapter->rx_itr_setting;
728 if (q_vector->itr_val && q_vector->itr_val <= 3)
729 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 730}
731
732static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
733 int ring_idx, int v_idx)
734{
735 struct igb_q_vector *q_vector;
736
737 q_vector = adapter->q_vector[v_idx];
738 q_vector->tx_ring = &adapter->tx_ring[ring_idx];
739 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 740 q_vector->itr_val = adapter->tx_itr_setting;
741 if (q_vector->itr_val && q_vector->itr_val <= 3)
742 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 743}
744
745/**
746 * igb_map_ring_to_vector - maps allocated queues to vectors
747 *
748 * This function maps the recently allocated queues to vectors.
749 **/
750static int igb_map_ring_to_vector(struct igb_adapter *adapter)
751{
752 int i;
753 int v_idx = 0;
754
755 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
756 (adapter->num_q_vectors < adapter->num_tx_queues))
757 return -ENOMEM;
758
759 if (adapter->num_q_vectors >=
760 (adapter->num_rx_queues + adapter->num_tx_queues)) {
761 for (i = 0; i < adapter->num_rx_queues; i++)
762 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
763 for (i = 0; i < adapter->num_tx_queues; i++)
764 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
765 } else {
766 for (i = 0; i < adapter->num_rx_queues; i++) {
767 if (i < adapter->num_tx_queues)
768 igb_map_tx_ring_to_vector(adapter, i, v_idx);
769 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
770 }
771 for (; i < adapter->num_tx_queues; i++)
772 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
773 }
774 return 0;
775}
776
777/**
778 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
779 *
780 * This function initializes the interrupts and allocates all of the queues.
781 **/
782static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
783{
784 struct pci_dev *pdev = adapter->pdev;
785 int err;
786
787 igb_set_interrupt_capability(adapter);
788
789 err = igb_alloc_q_vectors(adapter);
790 if (err) {
791 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
792 goto err_alloc_q_vectors;
793 }
794
795 err = igb_alloc_queues(adapter);
796 if (err) {
797 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
798 goto err_alloc_queues;
799 }
800
801 err = igb_map_ring_to_vector(adapter);
802 if (err) {
803 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
804 goto err_map_queues;
805 }
806
807
808 return 0;
809err_map_queues:
810 igb_free_queues(adapter);
811err_alloc_queues:
812 igb_free_q_vectors(adapter);
813err_alloc_q_vectors:
814 igb_reset_interrupt_capability(adapter);
815 return err;
816}
817
9d5c8243
AK		 818/**
819 * igb_request_irq - initialize interrupts
820 *
821 * Attempts to configure interrupts using the best available
822 * capabilities of the hardware and kernel.
823 **/
824static int igb_request_irq(struct igb_adapter *adapter)
825{
826 struct net_device *netdev = adapter->netdev;
047e0030 827 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 828 struct e1000_hw *hw = &adapter->hw;
829 int err = 0;
830
831 if (adapter->msix_entries) {
832 err = igb_request_msix(adapter);
844290e5 833 if (!err)
9d5c8243 834 goto request_done;
9d5c8243 835 /* fall back to MSI */
047e0030 836 igb_clear_interrupt_scheme(adapter);
9d5c8243 837 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 838 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 839 igb_free_all_tx_resources(adapter);
840 igb_free_all_rx_resources(adapter);
047e0030 841 adapter->num_tx_queues = 1;
9d5c8243 842 adapter->num_rx_queues = 1;
047e0030
AD		 843 adapter->num_q_vectors = 1;
844 err = igb_alloc_q_vectors(adapter);
845 if (err) {
846 dev_err(&pdev->dev,
847 "Unable to allocate memory for vectors\n");
848 goto request_done;
849 }
850 err = igb_alloc_queues(adapter);
851 if (err) {
852 dev_err(&pdev->dev,
853 "Unable to allocate memory for queues\n");
854 igb_free_q_vectors(adapter);
855 goto request_done;
856 }
857 igb_setup_all_tx_resources(adapter);
858 igb_setup_all_rx_resources(adapter);
844290e5 859 } else {
2d064c06
AD		 860 switch (hw->mac.type) {
861 case e1000_82575:
862 wr32(E1000_MSIXBM(0),
047e0030
AD		 863 (E1000_EICR_RX_QUEUE0 |
864 E1000_EICR_TX_QUEUE0 |
865 E1000_EIMS_OTHER));
2d064c06
AD		 866 break;
867 case e1000_82576:
868 wr32(E1000_IVAR0, E1000_IVAR_VALID);
869 break;
870 default:
871 break;
872 }
9d5c8243 873 }
844290e5 874
7dfc16fa 875 if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 876 err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0,
047e0030 877 netdev->name, adapter);
9d5c8243
AK		 878 if (!err)
879 goto request_done;
047e0030 880
9d5c8243
AK		 881 /* fall back to legacy interrupts */
882 igb_reset_interrupt_capability(adapter);
7dfc16fa 883 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 884 }
885
886 err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED,
047e0030 887 netdev->name, adapter);
9d5c8243 888
6cb5e577 889 if (err)
9d5c8243
AK		 890 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
891 err);
9d5c8243
AK		 892
893request_done:
894 return err;
895}
896
897static void igb_free_irq(struct igb_adapter *adapter)
898{
9d5c8243
AK		 899 if (adapter->msix_entries) {
900 int vector = 0, i;
901
047e0030 902 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 903
047e0030
AD		 904 for (i = 0; i < adapter->num_q_vectors; i++) {
905 struct igb_q_vector *q_vector = adapter->q_vector[i];
906 free_irq(adapter->msix_entries[vector++].vector,
907 q_vector);
908 }
909 } else {
910 free_irq(adapter->pdev->irq, adapter);
9d5c8243 911 }
9d5c8243
AK		 912}
913
914/**
915 * igb_irq_disable - Mask off interrupt generation on the NIC
916 * @adapter: board private structure
917 **/
918static void igb_irq_disable(struct igb_adapter *adapter)
919{
920 struct e1000_hw *hw = &adapter->hw;
921
25568a53
AD		 922 /*
923 * we need to be careful when disabling interrupts. The VFs are also
924 * mapped into these registers and so clearing the bits can cause
925 * issues on the VF drivers so we only need to clear what we set
926 */
9d5c8243 927 if (adapter->msix_entries) {
2dfd1212
AD		 928 u32 regval = rd32(E1000_EIAM);
929 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
930 wr32(E1000_EIMC, adapter->eims_enable_mask);
931 regval = rd32(E1000_EIAC);
932 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 933 }
844290e5
PW		 934
935 wr32(E1000_IAM, 0);
9d5c8243
AK		 936 wr32(E1000_IMC, ~0);
937 wrfl();
938 synchronize_irq(adapter->pdev->irq);
939}
940
941/**
942 * igb_irq_enable - Enable default interrupt generation settings
943 * @adapter: board private structure
944 **/
945static void igb_irq_enable(struct igb_adapter *adapter)
946{
947 struct e1000_hw *hw = &adapter->hw;
948
949 if (adapter->msix_entries) {
25568a53 950 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 951 u32 regval = rd32(E1000_EIAC);
952 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
953 regval = rd32(E1000_EIAM);
954 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 955 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 956 if (adapter->vfs_allocated_count) {
4ae196df 957 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 958 ims |= E1000_IMS_VMMB;
959 }
960 wr32(E1000_IMS, ims);
844290e5
PW		 961 } else {
962 wr32(E1000_IMS, IMS_ENABLE_MASK);
963 wr32(E1000_IAM, IMS_ENABLE_MASK);
964 }
9d5c8243
AK		 965}
966
967static void igb_update_mng_vlan(struct igb_adapter *adapter)
968{
51466239 969 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 970 u16 vid = adapter->hw.mng_cookie.vlan_id;
971 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 972
973 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
974 /* add VID to filter table */
975 igb_vfta_set(hw, vid, true);
976 adapter->mng_vlan_id = vid;
977 } else {
978 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
979 }
980
981 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
982 (vid != old_vid) &&
983 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
984 /* remove VID from filter table */
985 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 986 }
987}
988
989/**
990 * igb_release_hw_control - release control of the h/w to f/w
991 * @adapter: address of board private structure
992 *
993 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
994 * For ASF and Pass Through versions of f/w this means that the
995 * driver is no longer loaded.
996 *
997 **/
998static void igb_release_hw_control(struct igb_adapter *adapter)
999{
1000 struct e1000_hw *hw = &adapter->hw;
1001 u32 ctrl_ext;
1002
1003 /* Let firmware take over control of h/w */
1004 ctrl_ext = rd32(E1000_CTRL_EXT);
1005 wr32(E1000_CTRL_EXT,
1006 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1007}
1008
1009
1010/**
1011 * igb_get_hw_control - get control of the h/w from f/w
1012 * @adapter: address of board private structure
1013 *
1014 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1015 * For ASF and Pass Through versions of f/w this means that
1016 * the driver is loaded.
1017 *
1018 **/
1019static void igb_get_hw_control(struct igb_adapter *adapter)
1020{
1021 struct e1000_hw *hw = &adapter->hw;
1022 u32 ctrl_ext;
1023
1024 /* Let firmware know the driver has taken over */
1025 ctrl_ext = rd32(E1000_CTRL_EXT);
1026 wr32(E1000_CTRL_EXT,
1027 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1028}
1029
9d5c8243
AK		 1030/**
1031 * igb_configure - configure the hardware for RX and TX
1032 * @adapter: private board structure
1033 **/
1034static void igb_configure(struct igb_adapter *adapter)
1035{
1036 struct net_device *netdev = adapter->netdev;
1037 int i;
1038
1039 igb_get_hw_control(adapter);
ff41f8dc 1040 igb_set_rx_mode(netdev);
9d5c8243
AK		 1041
1042 igb_restore_vlan(adapter);
9d5c8243 1043
85b430b4 1044 igb_setup_tctl(adapter);
06cf2666 1045 igb_setup_mrqc(adapter);
9d5c8243 1046 igb_setup_rctl(adapter);
85b430b4
AD		 1047
1048 igb_configure_tx(adapter);
9d5c8243 1049 igb_configure_rx(adapter);
662d7205
AD		 1050
1051 igb_rx_fifo_flush_82575(&adapter->hw);
1052
c493ea45 1053 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1054 * at least 1 descriptor unused to make sure
1055 * next_to_use != next_to_clean */
1056 for (i = 0; i < adapter->num_rx_queues; i++) {
1057 struct igb_ring *ring = &adapter->rx_ring[i];
c493ea45 1058 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243
AK		 1059 }
1060
1061
1062 adapter->tx_queue_len = netdev->tx_queue_len;
1063}
1064
1065
1066/**
1067 * igb_up - Open the interface and prepare it to handle traffic
1068 * @adapter: board private structure
1069 **/
1070
1071int igb_up(struct igb_adapter *adapter)
1072{
1073 struct e1000_hw *hw = &adapter->hw;
1074 int i;
1075
1076 /* hardware has been reset, we need to reload some things */
1077 igb_configure(adapter);
1078
1079 clear_bit(__IGB_DOWN, &adapter->state);
1080
047e0030
AD		 1081 for (i = 0; i < adapter->num_q_vectors; i++) {
1082 struct igb_q_vector *q_vector = adapter->q_vector[i];
1083 napi_enable(&q_vector->napi);
1084 }
844290e5 1085 if (adapter->msix_entries)
9d5c8243 1086 igb_configure_msix(adapter);
9d5c8243
AK		 1087
1088 /* Clear any pending interrupts. */
1089 rd32(E1000_ICR);
1090 igb_irq_enable(adapter);
1091
d4960307
AD		 1092 /* notify VFs that reset has been completed */
1093 if (adapter->vfs_allocated_count) {
1094 u32 reg_data = rd32(E1000_CTRL_EXT);
1095 reg_data |= E1000_CTRL_EXT_PFRSTD;
1096 wr32(E1000_CTRL_EXT, reg_data);
1097 }
1098
4cb9be7a
JB		 1099 netif_tx_start_all_queues(adapter->netdev);
1100
25568a53
AD		 1101 /* start the watchdog. */
1102 hw->mac.get_link_status = 1;
1103 schedule_work(&adapter->watchdog_task);
1104
9d5c8243
AK		 1105 return 0;
1106}
1107
1108void igb_down(struct igb_adapter *adapter)
1109{
9d5c8243 1110 struct net_device *netdev = adapter->netdev;
330a6d6a 1111 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1112 u32 tctl, rctl;
1113 int i;
1114
1115 /* signal that we're down so the interrupt handler does not
1116 * reschedule our watchdog timer */
1117 set_bit(__IGB_DOWN, &adapter->state);
1118
1119 /* disable receives in the hardware */
1120 rctl = rd32(E1000_RCTL);
1121 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1122 /* flush and sleep below */
1123
fd2ea0a7 1124 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1125
1126 /* disable transmits in the hardware */
1127 tctl = rd32(E1000_TCTL);
1128 tctl &= ~E1000_TCTL_EN;
1129 wr32(E1000_TCTL, tctl);
1130 /* flush both disables and wait for them to finish */
1131 wrfl();
1132 msleep(10);
1133
047e0030
AD		 1134 for (i = 0; i < adapter->num_q_vectors; i++) {
1135 struct igb_q_vector *q_vector = adapter->q_vector[i];
1136 napi_disable(&q_vector->napi);
1137 }
9d5c8243 1138
9d5c8243
AK		 1139 igb_irq_disable(adapter);
1140
1141 del_timer_sync(&adapter->watchdog_timer);
1142 del_timer_sync(&adapter->phy_info_timer);
1143
1144 netdev->tx_queue_len = adapter->tx_queue_len;
1145 netif_carrier_off(netdev);
04fe6358
AD		 1146
1147 /* record the stats before reset*/
1148 igb_update_stats(adapter);
1149
9d5c8243
AK		 1150 adapter->link_speed = 0;
1151 adapter->link_duplex = 0;
1152
3023682e
JK		 1153 if (!pci_channel_offline(adapter->pdev))
1154 igb_reset(adapter);
9d5c8243
AK		 1155 igb_clean_all_tx_rings(adapter);
1156 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1157#ifdef CONFIG_IGB_DCA
1158
1159 /* since we reset the hardware DCA settings were cleared */
1160 igb_setup_dca(adapter);
1161#endif
9d5c8243
AK		 1162}
1163
1164void igb_reinit_locked(struct igb_adapter *adapter)
1165{
1166 WARN_ON(in_interrupt());
1167 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1168 msleep(1);
1169 igb_down(adapter);
1170 igb_up(adapter);
1171 clear_bit(__IGB_RESETTING, &adapter->state);
1172}
1173
1174void igb_reset(struct igb_adapter *adapter)
1175{
090b1795 1176 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1177 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1178 struct e1000_mac_info *mac = &hw->mac;
1179 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1180 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1181 u16 hwm;
1182
1183 /* Repartition Pba for greater than 9k mtu
1184 * To take effect CTRL.RST is required.
1185 */
fa4dfae0
AD		 1186 switch (mac->type) {
1187 case e1000_82576:
d249be54
AD		 1188 pba = rd32(E1000_RXPBS);
1189 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1190 break;
1191 case e1000_82575:
1192 default:
1193 pba = E1000_PBA_34K;
1194 break;
2d064c06 1195 }
9d5c8243 1196
2d064c06
AD		 1197 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1198 (mac->type < e1000_82576)) {
9d5c8243
AK		 1199 /* adjust PBA for jumbo frames */
1200 wr32(E1000_PBA, pba);
1201
1202 /* To maintain wire speed transmits, the Tx FIFO should be
1203 * large enough to accommodate two full transmit packets,
1204 * rounded up to the next 1KB and expressed in KB. Likewise,
1205 * the Rx FIFO should be large enough to accommodate at least
1206 * one full receive packet and is similarly rounded up and
1207 * expressed in KB. */
1208 pba = rd32(E1000_PBA);
1209 /* upper 16 bits has Tx packet buffer allocation size in KB */
1210 tx_space = pba >> 16;
1211 /* lower 16 bits has Rx packet buffer allocation size in KB */
1212 pba &= 0xffff;
1213 /* the tx fifo also stores 16 bytes of information about the tx
1214 * but don't include ethernet FCS because hardware appends it */
1215 min_tx_space = (adapter->max_frame_size +
85e8d004 1216 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1217 ETH_FCS_LEN) * 2;
1218 min_tx_space = ALIGN(min_tx_space, 1024);
1219 min_tx_space >>= 10;
1220 /* software strips receive CRC, so leave room for it */
1221 min_rx_space = adapter->max_frame_size;
1222 min_rx_space = ALIGN(min_rx_space, 1024);
1223 min_rx_space >>= 10;
1224
1225 /* If current Tx allocation is less than the min Tx FIFO size,
1226 * and the min Tx FIFO size is less than the current Rx FIFO
1227 * allocation, take space away from current Rx allocation */
1228 if (tx_space < min_tx_space &&
1229 ((min_tx_space - tx_space) < pba)) {
1230 pba = pba - (min_tx_space - tx_space);
1231
1232 /* if short on rx space, rx wins and must trump tx
1233 * adjustment */
1234 if (pba < min_rx_space)
1235 pba = min_rx_space;
1236 }
2d064c06 1237 wr32(E1000_PBA, pba);
9d5c8243 1238 }
9d5c8243
AK		 1239
1240 /* flow control settings */
1241 /* The high water mark must be low enough to fit one full frame
1242 * (or the size used for early receive) above it in the Rx FIFO.
1243 * Set it to the lower of:
1244 * - 90% of the Rx FIFO size, or
1245 * - the full Rx FIFO size minus one full frame */
1246 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1247 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1248
2d064c06
AD		 1249 if (mac->type < e1000_82576) {
1250 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */
1251 fc->low_water = fc->high_water - 8;
1252 } else {
1253 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1254 fc->low_water = fc->high_water - 16;
1255 }
9d5c8243
AK		 1256 fc->pause_time = 0xFFFF;
1257 fc->send_xon = 1;
0cce119a 1258 fc->current_mode = fc->requested_mode;
9d5c8243 1259
4ae196df
AD		 1260 /* disable receive for all VFs and wait one second */
1261 if (adapter->vfs_allocated_count) {
1262 int i;
1263 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1264 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1265
1266 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1267 igb_ping_all_vfs(adapter);
4ae196df
AD		 1268
1269 /* disable transmits and receives */
1270 wr32(E1000_VFRE, 0);
1271 wr32(E1000_VFTE, 0);
1272 }
1273
9d5c8243 1274 /* Allow time for pending master requests to run */
330a6d6a 1275 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1276 wr32(E1000_WUC, 0);
1277
330a6d6a 1278 if (hw->mac.ops.init_hw(hw))
090b1795 1279 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243
AK		 1280
1281 igb_update_mng_vlan(adapter);
1282
1283 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1284 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1285
330a6d6a
AD		 1286 igb_reset_adaptive(hw);
1287 igb_get_phy_info(hw);
9d5c8243
AK		 1288}
1289
2e5c6922
SH		 1290static const struct net_device_ops igb_netdev_ops = {
1291 .ndo_open = igb_open,
1292 .ndo_stop = igb_close,
00829823 1293 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1294 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1295 .ndo_set_rx_mode = igb_set_rx_mode,
1296 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1297 .ndo_set_mac_address = igb_set_mac,
1298 .ndo_change_mtu = igb_change_mtu,
1299 .ndo_do_ioctl = igb_ioctl,
1300 .ndo_tx_timeout = igb_tx_timeout,
1301 .ndo_validate_addr = eth_validate_addr,
1302 .ndo_vlan_rx_register = igb_vlan_rx_register,
1303 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1304 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1305#ifdef CONFIG_NET_POLL_CONTROLLER
1306 .ndo_poll_controller = igb_netpoll,
1307#endif
1308};
1309
9d5c8243
AK		 1310/**
1311 * igb_probe - Device Initialization Routine
1312 * @pdev: PCI device information struct
1313 * @ent: entry in igb_pci_tbl
1314 *
1315 * Returns 0 on success, negative on failure
1316 *
1317 * igb_probe initializes an adapter identified by a pci_dev structure.
1318 * The OS initialization, configuring of the adapter private structure,
1319 * and a hardware reset occur.
1320 **/
1321static int __devinit igb_probe(struct pci_dev *pdev,
1322 const struct pci_device_id *ent)
1323{
1324 struct net_device *netdev;
1325 struct igb_adapter *adapter;
1326 struct e1000_hw *hw;
4337e993
AD		 1327 u16 eeprom_data = 0;
1328 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1329 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1330 unsigned long mmio_start, mmio_len;
2d6a5e95 1331 int err, pci_using_dac;
9d5c8243
AK		 1332 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1333 u32 part_num;
1334
aed5dec3 1335 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1336 if (err)
1337 return err;
1338
1339 pci_using_dac = 0;
6a35528a 1340 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243 1341 if (!err) {
6a35528a 1342 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
9d5c8243
AK		 1343 if (!err)
1344 pci_using_dac = 1;
1345 } else {
284901a9 1346 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243 1347 if (err) {
284901a9 1348 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
9d5c8243
AK		 1349 if (err) {
1350 dev_err(&pdev->dev, "No usable DMA "
1351 "configuration, aborting\n");
1352 goto err_dma;
1353 }
1354 }
1355 }
1356
aed5dec3
AD		 1357 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1358 IORESOURCE_MEM),
1359 igb_driver_name);
9d5c8243
AK		 1360 if (err)
1361 goto err_pci_reg;
1362
19d5afd4 1363 pci_enable_pcie_error_reporting(pdev);
40a914fa 1364
9d5c8243 1365 pci_set_master(pdev);
c682fc23 1366 pci_save_state(pdev);
9d5c8243
AK		 1367
1368 err = -ENOMEM;
1bfaf07b
AD		 1369 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1370 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1371 if (!netdev)
1372 goto err_alloc_etherdev;
1373
1374 SET_NETDEV_DEV(netdev, &pdev->dev);
1375
1376 pci_set_drvdata(pdev, netdev);
1377 adapter = netdev_priv(netdev);
1378 adapter->netdev = netdev;
1379 adapter->pdev = pdev;
1380 hw = &adapter->hw;
1381 hw->back = adapter;
1382 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1383
1384 mmio_start = pci_resource_start(pdev, 0);
1385 mmio_len = pci_resource_len(pdev, 0);
1386
1387 err = -EIO;
28b0759c
AD		 1388 hw->hw_addr = ioremap(mmio_start, mmio_len);
1389 if (!hw->hw_addr)
9d5c8243
AK		 1390 goto err_ioremap;
1391
2e5c6922 1392 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1393 igb_set_ethtool_ops(netdev);
9d5c8243 1394 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1395
1396 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1397
1398 netdev->mem_start = mmio_start;
1399 netdev->mem_end = mmio_start + mmio_len;
1400
9d5c8243
AK		 1401 /* PCI config space info */
1402 hw->vendor_id = pdev->vendor;
1403 hw->device_id = pdev->device;
1404 hw->revision_id = pdev->revision;
1405 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1406 hw->subsystem_device_id = pdev->subsystem_device;
1407
9d5c8243
AK		 1408 /* Copy the default MAC, PHY and NVM function pointers */
1409 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1410 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1411 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1412 /* Initialize skew-specific constants */
1413 err = ei->get_invariants(hw);
1414 if (err)
450c87c8 1415 goto err_sw_init;
9d5c8243 1416
450c87c8 1417 /* setup the private structure */
9d5c8243
AK		 1418 err = igb_sw_init(adapter);
1419 if (err)
1420 goto err_sw_init;
1421
1422 igb_get_bus_info_pcie(hw);
1423
1424 hw->phy.autoneg_wait_to_complete = false;
1425 hw->mac.adaptive_ifs = true;
1426
1427 /* Copper options */
1428 if (hw->phy.media_type == e1000_media_type_copper) {
1429 hw->phy.mdix = AUTO_ALL_MODES;
1430 hw->phy.disable_polarity_correction = false;
1431 hw->phy.ms_type = e1000_ms_hw_default;
1432 }
1433
1434 if (igb_check_reset_block(hw))
1435 dev_info(&pdev->dev,
1436 "PHY reset is blocked due to SOL/IDER session.\n");
1437
1438 netdev->features = NETIF_F_SG |
7d8eb29e 1439 NETIF_F_IP_CSUM |
9d5c8243
AK		 1440 NETIF_F_HW_VLAN_TX |
1441 NETIF_F_HW_VLAN_RX |
1442 NETIF_F_HW_VLAN_FILTER;
1443
7d8eb29e 1444 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1445 netdev->features |= NETIF_F_TSO;
9d5c8243 1446 netdev->features |= NETIF_F_TSO6;
48f29ffc 1447
5c0999b7 1448 netdev->features |= NETIF_F_GRO;
d3352520 1449
48f29ffc
JK		 1450 netdev->vlan_features |= NETIF_F_TSO;
1451 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1452 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1453 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1454 netdev->vlan_features |= NETIF_F_SG;
1455
9d5c8243
AK		 1456 if (pci_using_dac)
1457 netdev->features |= NETIF_F_HIGHDMA;
1458
b9473560
JB		 1459 if (adapter->hw.mac.type == e1000_82576)
1460 netdev->features |= NETIF_F_SCTP_CSUM;
1461
330a6d6a 1462 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1463
1464 /* before reading the NVM, reset the controller to put the device in a
1465 * known good starting state */
1466 hw->mac.ops.reset_hw(hw);
1467
1468 /* make sure the NVM is good */
1469 if (igb_validate_nvm_checksum(hw) < 0) {
1470 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1471 err = -EIO;
1472 goto err_eeprom;
1473 }
1474
1475 /* copy the MAC address out of the NVM */
1476 if (hw->mac.ops.read_mac_addr(hw))
1477 dev_err(&pdev->dev, "NVM Read Error\n");
1478
1479 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1480 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1481
1482 if (!is_valid_ether_addr(netdev->perm_addr)) {
1483 dev_err(&pdev->dev, "Invalid MAC Address\n");
1484 err = -EIO;
1485 goto err_eeprom;
1486 }
1487
0e340485
AD		 1488 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1489 (unsigned long) adapter);
1490 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1491 (unsigned long) adapter);
9d5c8243
AK		 1492
1493 INIT_WORK(&adapter->reset_task, igb_reset_task);
1494 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1495
450c87c8 1496 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1497 adapter->fc_autoneg = true;
1498 hw->mac.autoneg = true;
1499 hw->phy.autoneg_advertised = 0x2f;
1500
0cce119a
AD		 1501 hw->fc.requested_mode = e1000_fc_default;
1502 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1503
9d5c8243
AK		 1504 igb_validate_mdi_setting(hw);
1505
9d5c8243
AK		 1506 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1507 * enable the ACPI Magic Packet filter
1508 */
1509
a2cf8b6c 1510 if (hw->bus.func == 0)
312c75ae 1511 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
a2cf8b6c
AD		 1512 else if (hw->bus.func == 1)
1513 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1514
1515 if (eeprom_data & eeprom_apme_mask)
1516 adapter->eeprom_wol |= E1000_WUFC_MAG;
1517
1518 /* now that we have the eeprom settings, apply the special cases where
1519 * the eeprom may be wrong or the board simply won't support wake on
1520 * lan on a particular port */
1521 switch (pdev->device) {
1522 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1523 adapter->eeprom_wol = 0;
1524 break;
1525 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1526 case E1000_DEV_ID_82576_FIBER:
1527 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1528 /* Wake events only supported on port A for dual fiber
1529 * regardless of eeprom setting */
1530 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1531 adapter->eeprom_wol = 0;
1532 break;
c8ea5ea9
AD		 1533 case E1000_DEV_ID_82576_QUAD_COPPER:
1534 /* if quad port adapter, disable WoL on all but port A */
1535 if (global_quad_port_a != 0)
1536 adapter->eeprom_wol = 0;
1537 else
1538 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1539 /* Reset for multiple quad port adapters */
1540 if (++global_quad_port_a == 4)
1541 global_quad_port_a = 0;
1542 break;
9d5c8243
AK		 1543 }
1544
1545 /* initialize the wol settings based on the eeprom settings */
1546 adapter->wol = adapter->eeprom_wol;
e1b86d84 1547 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1548
1549 /* reset the hardware with the new settings */
1550 igb_reset(adapter);
1551
1552 /* let the f/w know that the h/w is now under the control of the
1553 * driver. */
1554 igb_get_hw_control(adapter);
1555
9d5c8243
AK		 1556 strcpy(netdev->name, "eth%d");
1557 err = register_netdev(netdev);
1558 if (err)
1559 goto err_register;
1560
b168dfc5
JB		 1561 /* carrier off reporting is important to ethtool even BEFORE open */
1562 netif_carrier_off(netdev);
1563
421e02f0 1564#ifdef CONFIG_IGB_DCA
bbd98fe4 1565 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1566 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1567 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1568 igb_setup_dca(adapter);
1569 }
fe4506b6 1570
38c845c7 1571#endif
38c845c7 1572
c5b9bd5e
AD		 1573 switch (hw->mac.type) {
1574 case e1000_82576:
1575 /*
1576 * Initialize hardware timer: we keep it running just in case
1577 * that some program needs it later on.
1578 */
1579 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1580 adapter->cycles.read = igb_read_clock;
1581 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1582 adapter->cycles.mult = 1;
1583 /**
1584 * Scale the NIC clock cycle by a large factor so that
1585 * relatively small clock corrections can be added or
1586 * substracted at each clock tick. The drawbacks of a large
1587 * factor are a) that the clock register overflows more quickly
1588 * (not such a big deal) and b) that the increment per tick has
1589 * to fit into 24 bits. As a result we need to use a shift of
1590 * 19 so we can fit a value of 16 into the TIMINCA register.
1591 */
1592 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1593 wr32(E1000_TIMINCA,
1594 (1 << E1000_TIMINCA_16NS_SHIFT) |
1595 (16 << IGB_82576_TSYNC_SHIFT));
1596
1597 /* Set registers so that rollover occurs soon to test this. */
1598 wr32(E1000_SYSTIML, 0x00000000);
1599 wr32(E1000_SYSTIMH, 0xFF800000);
1600 wrfl();
1601
1602 timecounter_init(&adapter->clock,
1603 &adapter->cycles,
1604 ktime_to_ns(ktime_get_real()));
1605 /*
1606 * Synchronize our NIC clock against system wall clock. NIC
1607 * time stamp reading requires ~3us per sample, each sample
1608 * was pretty stable even under load => only require 10
1609 * samples for each offset comparison.
1610 */
1611 memset(&adapter->compare, 0, sizeof(adapter->compare));
1612 adapter->compare.source = &adapter->clock;
1613 adapter->compare.target = ktime_get_real;
1614 adapter->compare.num_samples = 10;
1615 timecompare_update(&adapter->compare, 0);
1616 break;
1617 case e1000_82575:
1618 /* 82575 does not support timesync */
1619 default:
1620 break;
38c845c7 1621 }
38c845c7 1622
9d5c8243
AK		 1623 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1624 /* print bus type/speed/width info */
7c510e4b 1625 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243
AK		 1626 netdev->name,
1627 ((hw->bus.speed == e1000_bus_speed_2500)
1628 ? "2.5Gb/s" : "unknown"),
59c3de89
AD		 1629 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1630 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1631 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1632 "unknown"),
7c510e4b 1633 netdev->dev_addr);
9d5c8243
AK		 1634
1635 igb_read_part_num(hw, &part_num);
1636 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1637 (part_num >> 8), (part_num & 0xff));
1638
1639 dev_info(&pdev->dev,
1640 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1641 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1642 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1643 adapter->num_rx_queues, adapter->num_tx_queues);
1644
9d5c8243
AK		 1645 return 0;
1646
1647err_register:
1648 igb_release_hw_control(adapter);
1649err_eeprom:
1650 if (!igb_check_reset_block(hw))
f5f4cf08 1651 igb_reset_phy(hw);
9d5c8243
AK		 1652
1653 if (hw->flash_address)
1654 iounmap(hw->flash_address);
9d5c8243 1655err_sw_init:
047e0030 1656 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 1657 iounmap(hw->hw_addr);
1658err_ioremap:
1659 free_netdev(netdev);
1660err_alloc_etherdev:
aed5dec3
AD		 1661 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1662 IORESOURCE_MEM));
9d5c8243
AK		 1663err_pci_reg:
1664err_dma:
1665 pci_disable_device(pdev);
1666 return err;
1667}
1668
1669/**
1670 * igb_remove - Device Removal Routine
1671 * @pdev: PCI device information struct
1672 *
1673 * igb_remove is called by the PCI subsystem to alert the driver
1674 * that it should release a PCI device. The could be caused by a
1675 * Hot-Plug event, or because the driver is going to be removed from
1676 * memory.
1677 **/
1678static void __devexit igb_remove(struct pci_dev *pdev)
1679{
1680 struct net_device *netdev = pci_get_drvdata(pdev);
1681 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 1682 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1683
1684 /* flush_scheduled work may reschedule our watchdog task, so
1685 * explicitly disable watchdog tasks from being rescheduled */
1686 set_bit(__IGB_DOWN, &adapter->state);
1687 del_timer_sync(&adapter->watchdog_timer);
1688 del_timer_sync(&adapter->phy_info_timer);
1689
1690 flush_scheduled_work();
1691
421e02f0 1692#ifdef CONFIG_IGB_DCA
7dfc16fa 1693 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 1694 dev_info(&pdev->dev, "DCA disabled\n");
1695 dca_remove_requester(&pdev->dev);
7dfc16fa 1696 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 1697 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 1698 }
1699#endif
1700
9d5c8243
AK		 1701 /* Release control of h/w to f/w. If f/w is AMT enabled, this
1702 * would have already happened in close and is redundant. */
1703 igb_release_hw_control(adapter);
1704
1705 unregister_netdev(netdev);
1706
330a6d6a
AD		 1707 if (!igb_check_reset_block(hw))
1708 igb_reset_phy(hw);
9d5c8243 1709
047e0030 1710 igb_clear_interrupt_scheme(adapter);
9d5c8243 1711
37680117
AD		 1712#ifdef CONFIG_PCI_IOV
1713 /* reclaim resources allocated to VFs */
1714 if (adapter->vf_data) {
1715 /* disable iov and allow time for transactions to clear */
1716 pci_disable_sriov(pdev);
1717 msleep(500);
1718
1719 kfree(adapter->vf_data);
1720 adapter->vf_data = NULL;
1721 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1722 msleep(100);
1723 dev_info(&pdev->dev, "IOV Disabled\n");
1724 }
1725#endif
28b0759c
AD		 1726 iounmap(hw->hw_addr);
1727 if (hw->flash_address)
1728 iounmap(hw->flash_address);
aed5dec3
AD		 1729 pci_release_selected_regions(pdev, pci_select_bars(pdev,
1730 IORESOURCE_MEM));
9d5c8243
AK		 1731
1732 free_netdev(netdev);
1733
19d5afd4 1734 pci_disable_pcie_error_reporting(pdev);
40a914fa 1735
9d5c8243
AK		 1736 pci_disable_device(pdev);
1737}
1738
a6b623e0
AD		 1739/**
1740 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
1741 * @adapter: board private structure to initialize
1742 *
1743 * This function initializes the vf specific data storage and then attempts to
1744 * allocate the VFs. The reason for ordering it this way is because it is much
1745 * mor expensive time wise to disable SR-IOV than it is to allocate and free
1746 * the memory for the VFs.
1747 **/
1748static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
1749{
1750#ifdef CONFIG_PCI_IOV
1751 struct pci_dev *pdev = adapter->pdev;
1752
1753 if (adapter->vfs_allocated_count > 7)
1754 adapter->vfs_allocated_count = 7;
1755
1756 if (adapter->vfs_allocated_count) {
1757 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1758 sizeof(struct vf_data_storage),
1759 GFP_KERNEL);
1760 /* if allocation failed then we do not support SR-IOV */
1761 if (!adapter->vf_data) {
1762 adapter->vfs_allocated_count = 0;
1763 dev_err(&pdev->dev, "Unable to allocate memory for VF "
1764 "Data Storage\n");
1765 }
1766 }
1767
1768 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
1769 kfree(adapter->vf_data);
1770 adapter->vf_data = NULL;
1771#endif /* CONFIG_PCI_IOV */
1772 adapter->vfs_allocated_count = 0;
1773#ifdef CONFIG_PCI_IOV
1774 } else {
1775 unsigned char mac_addr[ETH_ALEN];
1776 int i;
1777 dev_info(&pdev->dev, "%d vfs allocated\n",
1778 adapter->vfs_allocated_count);
1779 for (i = 0; i < adapter->vfs_allocated_count; i++) {
1780 random_ether_addr(mac_addr);
1781 igb_set_vf_mac(adapter, i, mac_addr);
1782 }
1783 }
1784#endif /* CONFIG_PCI_IOV */
1785}
1786
9d5c8243
AK		 1787/**
1788 * igb_sw_init - Initialize general software structures (struct igb_adapter)
1789 * @adapter: board private structure to initialize
1790 *
1791 * igb_sw_init initializes the Adapter private data structure.
1792 * Fields are initialized based on PCI device information and
1793 * OS network device settings (MTU size).
1794 **/
1795static int __devinit igb_sw_init(struct igb_adapter *adapter)
1796{
1797 struct e1000_hw *hw = &adapter->hw;
1798 struct net_device *netdev = adapter->netdev;
1799 struct pci_dev *pdev = adapter->pdev;
1800
1801 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1802
68fd9910
AD		 1803 adapter->tx_ring_count = IGB_DEFAULT_TXD;
1804 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 1805 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1806 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1807
9d5c8243
AK		 1808 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1809 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1810
a6b623e0
AD		 1811#ifdef CONFIG_PCI_IOV
1812 if (hw->mac.type == e1000_82576)
1813 adapter->vfs_allocated_count = max_vfs;
1814
1815#endif /* CONFIG_PCI_IOV */
1816 /* This call may decrease the number of queues */
047e0030 1817 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 1818 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1819 return -ENOMEM;
1820 }
1821
a6b623e0
AD		 1822 igb_probe_vfs(adapter);
1823
9d5c8243
AK		 1824 /* Explicitly disable IRQ since the NIC can be in any state. */
1825 igb_irq_disable(adapter);
1826
1827 set_bit(__IGB_DOWN, &adapter->state);
1828 return 0;
1829}
1830
1831/**
1832 * igb_open - Called when a network interface is made active
1833 * @netdev: network interface device structure
1834 *
1835 * Returns 0 on success, negative value on failure
1836 *
1837 * The open entry point is called when a network interface is made
1838 * active by the system (IFF_UP). At this point all resources needed
1839 * for transmit and receive operations are allocated, the interrupt
1840 * handler is registered with the OS, the watchdog timer is started,
1841 * and the stack is notified that the interface is ready.
1842 **/
1843static int igb_open(struct net_device *netdev)
1844{
1845 struct igb_adapter *adapter = netdev_priv(netdev);
1846 struct e1000_hw *hw = &adapter->hw;
1847 int err;
1848 int i;
1849
1850 /* disallow open during test */
1851 if (test_bit(__IGB_TESTING, &adapter->state))
1852 return -EBUSY;
1853
b168dfc5
JB		 1854 netif_carrier_off(netdev);
1855
9d5c8243
AK		 1856 /* allocate transmit descriptors */
1857 err = igb_setup_all_tx_resources(adapter);
1858 if (err)
1859 goto err_setup_tx;
1860
1861 /* allocate receive descriptors */
1862 err = igb_setup_all_rx_resources(adapter);
1863 if (err)
1864 goto err_setup_rx;
1865
1866 /* e1000_power_up_phy(adapter); */
1867
9d5c8243
AK		 1868 /* before we allocate an interrupt, we must be ready to handle it.
1869 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1870 * as soon as we call pci_request_irq, so we have to setup our
1871 * clean_rx handler before we do so. */
1872 igb_configure(adapter);
1873
1874 err = igb_request_irq(adapter);
1875 if (err)
1876 goto err_req_irq;
1877
1878 /* From here on the code is the same as igb_up() */
1879 clear_bit(__IGB_DOWN, &adapter->state);
1880
047e0030
AD		 1881 for (i = 0; i < adapter->num_q_vectors; i++) {
1882 struct igb_q_vector *q_vector = adapter->q_vector[i];
1883 napi_enable(&q_vector->napi);
1884 }
9d5c8243
AK		 1885
1886 /* Clear any pending interrupts. */
1887 rd32(E1000_ICR);
844290e5
PW		 1888
1889 igb_irq_enable(adapter);
1890
d4960307
AD		 1891 /* notify VFs that reset has been completed */
1892 if (adapter->vfs_allocated_count) {
1893 u32 reg_data = rd32(E1000_CTRL_EXT);
1894 reg_data |= E1000_CTRL_EXT_PFRSTD;
1895 wr32(E1000_CTRL_EXT, reg_data);
1896 }
1897
d55b53ff
JK		 1898 netif_tx_start_all_queues(netdev);
1899
25568a53
AD		 1900 /* start the watchdog. */
1901 hw->mac.get_link_status = 1;
1902 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 1903
1904 return 0;
1905
1906err_req_irq:
1907 igb_release_hw_control(adapter);
1908 /* e1000_power_down_phy(adapter); */
1909 igb_free_all_rx_resources(adapter);
1910err_setup_rx:
1911 igb_free_all_tx_resources(adapter);
1912err_setup_tx:
1913 igb_reset(adapter);
1914
1915 return err;
1916}
1917
1918/**
1919 * igb_close - Disables a network interface
1920 * @netdev: network interface device structure
1921 *
1922 * Returns 0, this is not allowed to fail
1923 *
1924 * The close entry point is called when an interface is de-activated
1925 * by the OS. The hardware is still under the driver's control, but
1926 * needs to be disabled. A global MAC reset is issued to stop the
1927 * hardware, and all transmit and receive resources are freed.
1928 **/
1929static int igb_close(struct net_device *netdev)
1930{
1931 struct igb_adapter *adapter = netdev_priv(netdev);
1932
1933 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
1934 igb_down(adapter);
1935
1936 igb_free_irq(adapter);
1937
1938 igb_free_all_tx_resources(adapter);
1939 igb_free_all_rx_resources(adapter);
1940
9d5c8243
AK		 1941 return 0;
1942}
1943
1944/**
1945 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 1946 * @tx_ring: tx descriptor ring (for a specific queue) to setup
1947 *
1948 * Return 0 on success, negative on failure
1949 **/
80785298 1950int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 1951{
80785298 1952 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 1953 int size;
1954
1955 size = sizeof(struct igb_buffer) * tx_ring->count;
1956 tx_ring->buffer_info = vmalloc(size);
1957 if (!tx_ring->buffer_info)
1958 goto err;
1959 memset(tx_ring->buffer_info, 0, size);
1960
1961 /* round up to nearest 4K */
85e8d004 1962 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 1963 tx_ring->size = ALIGN(tx_ring->size, 4096);
1964
439705e1
AD		 1965 tx_ring->desc = pci_alloc_consistent(pdev,
1966 tx_ring->size,
9d5c8243
AK		 1967 &tx_ring->dma);
1968
1969 if (!tx_ring->desc)
1970 goto err;
1971
9d5c8243
AK		 1972 tx_ring->next_to_use = 0;
1973 tx_ring->next_to_clean = 0;
9d5c8243
AK		 1974 return 0;
1975
1976err:
1977 vfree(tx_ring->buffer_info);
047e0030 1978 dev_err(&pdev->dev,
9d5c8243
AK		 1979 "Unable to allocate memory for the transmit descriptor ring\n");
1980 return -ENOMEM;
1981}
1982
1983/**
1984 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
1985 * (Descriptors) for all queues
1986 * @adapter: board private structure
1987 *
1988 * Return 0 on success, negative on failure
1989 **/
1990static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
1991{
439705e1 1992 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1993 int i, err = 0;
1994
1995 for (i = 0; i < adapter->num_tx_queues; i++) {
80785298 1996 err = igb_setup_tx_resources(&adapter->tx_ring[i]);
9d5c8243 1997 if (err) {
439705e1 1998 dev_err(&pdev->dev,
9d5c8243
AK		 1999 "Allocation for Tx Queue %u failed\n", i);
2000 for (i--; i >= 0; i--)
3b644cf6 2001 igb_free_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 2002 break;
2003 }
2004 }
2005
661086df 2006 for (i = 0; i < IGB_MAX_TX_QUEUES; i++) {
439705e1 2007 int r_idx = i % adapter->num_tx_queues;
661086df 2008 adapter->multi_tx_table[i] = &adapter->tx_ring[r_idx];
eebbbdba 2009 }
9d5c8243
AK		 2010 return err;
2011}
2012
2013/**
85b430b4
AD		 2014 * igb_setup_tctl - configure the transmit control registers
2015 * @adapter: Board private structure
9d5c8243 2016 **/
d7ee5b3a 2017void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2018{
9d5c8243
AK		 2019 struct e1000_hw *hw = &adapter->hw;
2020 u32 tctl;
9d5c8243 2021
85b430b4
AD		 2022 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2023 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2024
2025 /* Program the Transmit Control Register */
9d5c8243
AK		 2026 tctl = rd32(E1000_TCTL);
2027 tctl &= ~E1000_TCTL_CT;
2028 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2029 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2030
2031 igb_config_collision_dist(hw);
2032
9d5c8243
AK		 2033 /* Enable transmits */
2034 tctl |= E1000_TCTL_EN;
2035
2036 wr32(E1000_TCTL, tctl);
2037}
2038
85b430b4
AD		 2039/**
2040 * igb_configure_tx_ring - Configure transmit ring after Reset
2041 * @adapter: board private structure
2042 * @ring: tx ring to configure
2043 *
2044 * Configure a transmit ring after a reset.
2045 **/
d7ee5b3a
AD		 2046void igb_configure_tx_ring(struct igb_adapter *adapter,
2047 struct igb_ring *ring)
85b430b4
AD		 2048{
2049 struct e1000_hw *hw = &adapter->hw;
2050 u32 txdctl;
2051 u64 tdba = ring->dma;
2052 int reg_idx = ring->reg_idx;
2053
2054 /* disable the queue */
2055 txdctl = rd32(E1000_TXDCTL(reg_idx));
2056 wr32(E1000_TXDCTL(reg_idx),
2057 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2058 wrfl();
2059 mdelay(10);
2060
2061 wr32(E1000_TDLEN(reg_idx),
2062 ring->count * sizeof(union e1000_adv_tx_desc));
2063 wr32(E1000_TDBAL(reg_idx),
2064 tdba & 0x00000000ffffffffULL);
2065 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2066
fce99e34
AD		 2067 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2068 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2069 writel(0, ring->head);
2070 writel(0, ring->tail);
85b430b4
AD		 2071
2072 txdctl |= IGB_TX_PTHRESH;
2073 txdctl |= IGB_TX_HTHRESH << 8;
2074 txdctl |= IGB_TX_WTHRESH << 16;
2075
2076 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2077 wr32(E1000_TXDCTL(reg_idx), txdctl);
2078}
2079
2080/**
2081 * igb_configure_tx - Configure transmit Unit after Reset
2082 * @adapter: board private structure
2083 *
2084 * Configure the Tx unit of the MAC after a reset.
2085 **/
2086static void igb_configure_tx(struct igb_adapter *adapter)
2087{
2088 int i;
2089
2090 for (i = 0; i < adapter->num_tx_queues; i++)
2091 igb_configure_tx_ring(adapter, &adapter->tx_ring[i]);
85b430b4
AD		 2092}
2093
9d5c8243
AK		 2094/**
2095 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2096 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2097 *
2098 * Returns 0 on success, negative on failure
2099 **/
80785298 2100int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2101{
80785298 2102 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 2103 int size, desc_len;
2104
2105 size = sizeof(struct igb_buffer) * rx_ring->count;
2106 rx_ring->buffer_info = vmalloc(size);
2107 if (!rx_ring->buffer_info)
2108 goto err;
2109 memset(rx_ring->buffer_info, 0, size);
2110
2111 desc_len = sizeof(union e1000_adv_rx_desc);
2112
2113 /* Round up to nearest 4K */
2114 rx_ring->size = rx_ring->count * desc_len;
2115 rx_ring->size = ALIGN(rx_ring->size, 4096);
2116
2117 rx_ring->desc = pci_alloc_consistent(pdev, rx_ring->size,
2118 &rx_ring->dma);
2119
2120 if (!rx_ring->desc)
2121 goto err;
2122
2123 rx_ring->next_to_clean = 0;
2124 rx_ring->next_to_use = 0;
9d5c8243 2125
9d5c8243
AK		 2126 return 0;
2127
2128err:
2129 vfree(rx_ring->buffer_info);
439705e1 2130 rx_ring->buffer_info = NULL;
80785298 2131 dev_err(&pdev->dev, "Unable to allocate memory for "
9d5c8243
AK		 2132 "the receive descriptor ring\n");
2133 return -ENOMEM;
2134}
2135
2136/**
2137 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2138 * (Descriptors) for all queues
2139 * @adapter: board private structure
2140 *
2141 * Return 0 on success, negative on failure
2142 **/
2143static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2144{
439705e1 2145 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2146 int i, err = 0;
2147
2148 for (i = 0; i < adapter->num_rx_queues; i++) {
80785298 2149 err = igb_setup_rx_resources(&adapter->rx_ring[i]);
9d5c8243 2150 if (err) {
439705e1 2151 dev_err(&pdev->dev,
9d5c8243
AK		 2152 "Allocation for Rx Queue %u failed\n", i);
2153 for (i--; i >= 0; i--)
3b644cf6 2154 igb_free_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2155 break;
2156 }
2157 }
2158
2159 return err;
2160}
2161
06cf2666
AD		 2162/**
2163 * igb_setup_mrqc - configure the multiple receive queue control registers
2164 * @adapter: Board private structure
2165 **/
2166static void igb_setup_mrqc(struct igb_adapter *adapter)
2167{
2168 struct e1000_hw *hw = &adapter->hw;
2169 u32 mrqc, rxcsum;
2170 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2171 union e1000_reta {
2172 u32 dword;
2173 u8 bytes[4];
2174 } reta;
2175 static const u8 rsshash[40] = {
2176 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2177 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2178 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2179 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2180
2181 /* Fill out hash function seeds */
2182 for (j = 0; j < 10; j++) {
2183 u32 rsskey = rsshash[(j * 4)];
2184 rsskey |= rsshash[(j * 4) + 1] << 8;
2185 rsskey |= rsshash[(j * 4) + 2] << 16;
2186 rsskey |= rsshash[(j * 4) + 3] << 24;
2187 array_wr32(E1000_RSSRK(0), j, rsskey);
2188 }
2189
2190 num_rx_queues = adapter->num_rx_queues;
2191
2192 if (adapter->vfs_allocated_count) {
2193 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2194 switch (hw->mac.type) {
2195 case e1000_82576:
2196 shift = 3;
2197 num_rx_queues = 2;
2198 break;
2199 case e1000_82575:
2200 shift = 2;
2201 shift2 = 6;
2202 default:
2203 break;
2204 }
2205 } else {
2206 if (hw->mac.type == e1000_82575)
2207 shift = 6;
2208 }
2209
2210 for (j = 0; j < (32 * 4); j++) {
2211 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2212 if (shift2)
2213 reta.bytes[j & 3] |= num_rx_queues << shift2;
2214 if ((j & 3) == 3)
2215 wr32(E1000_RETA(j >> 2), reta.dword);
2216 }
2217
2218 /*
2219 * Disable raw packet checksumming so that RSS hash is placed in
2220 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2221 * offloads as they are enabled by default
2222 */
2223 rxcsum = rd32(E1000_RXCSUM);
2224 rxcsum |= E1000_RXCSUM_PCSD;
2225
2226 if (adapter->hw.mac.type >= e1000_82576)
2227 /* Enable Receive Checksum Offload for SCTP */
2228 rxcsum |= E1000_RXCSUM_CRCOFL;
2229
2230 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2231 wr32(E1000_RXCSUM, rxcsum);
2232
2233 /* If VMDq is enabled then we set the appropriate mode for that, else
2234 * we default to RSS so that an RSS hash is calculated per packet even
2235 * if we are only using one queue */
2236 if (adapter->vfs_allocated_count) {
2237 if (hw->mac.type > e1000_82575) {
2238 /* Set the default pool for the PF's first queue */
2239 u32 vtctl = rd32(E1000_VT_CTL);
2240 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2241 E1000_VT_CTL_DISABLE_DEF_POOL);
2242 vtctl |= adapter->vfs_allocated_count <<
2243 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2244 wr32(E1000_VT_CTL, vtctl);
2245 }
2246 if (adapter->num_rx_queues > 1)
2247 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2248 else
2249 mrqc = E1000_MRQC_ENABLE_VMDQ;
2250 } else {
2251 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2252 }
2253 igb_vmm_control(adapter);
2254
2255 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2256 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2257 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2258 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2259 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2260 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2261 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2262 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2263
2264 wr32(E1000_MRQC, mrqc);
2265}
2266
9d5c8243
AK		 2267/**
2268 * igb_setup_rctl - configure the receive control registers
2269 * @adapter: Board private structure
2270 **/
d7ee5b3a 2271void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2272{
2273 struct e1000_hw *hw = &adapter->hw;
2274 u32 rctl;
9d5c8243
AK		 2275
2276 rctl = rd32(E1000_RCTL);
2277
2278 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2279 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2280
69d728ba 2281 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2282 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2283
87cb7e8c
AK		 2284 /*
2285 * enable stripping of CRC. It's unlikely this will break BMC
2286 * redirection as it did with e1000. Newer features require
2287 * that the HW strips the CRC.
73cd78f1 2288 */
87cb7e8c 2289 rctl |= E1000_RCTL_SECRC;
9d5c8243 2290
9b07f3d3 2291 /*
ec54d7d6 2292 * disable store bad packets and clear size bits.
9b07f3d3 2293 */
ec54d7d6 2294 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2295
6ec43fe6
AD		 2296 /* enable LPE to prevent packets larger than max_frame_size */
2297 rctl |= E1000_RCTL_LPE;
9d5c8243 2298
952f72a8
AD		 2299 /* disable queue 0 to prevent tail write w/o re-config */
2300 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2301
e1739522
AD		 2302 /* Attention!!! For SR-IOV PF driver operations you must enable
2303 * queue drop for all VF and PF queues to prevent head of line blocking
2304 * if an un-trusted VF does not provide descriptors to hardware.
2305 */
2306 if (adapter->vfs_allocated_count) {
e1739522
AD		 2307 /* set all queue drop enable bits */
2308 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2309 }
2310
9d5c8243
AK		 2311 wr32(E1000_RCTL, rctl);
2312}
2313
7d5753f0
AD		 2314static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2315 int vfn)
2316{
2317 struct e1000_hw *hw = &adapter->hw;
2318 u32 vmolr;
2319
2320 /* if it isn't the PF check to see if VFs are enabled and
2321 * increase the size to support vlan tags */
2322 if (vfn < adapter->vfs_allocated_count &&
2323 adapter->vf_data[vfn].vlans_enabled)
2324 size += VLAN_TAG_SIZE;
2325
2326 vmolr = rd32(E1000_VMOLR(vfn));
2327 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2328 vmolr |= size | E1000_VMOLR_LPE;
2329 wr32(E1000_VMOLR(vfn), vmolr);
2330
2331 return 0;
2332}
2333
e1739522
AD		 2334/**
2335 * igb_rlpml_set - set maximum receive packet size
2336 * @adapter: board private structure
2337 *
2338 * Configure maximum receivable packet size.
2339 **/
2340static void igb_rlpml_set(struct igb_adapter *adapter)
2341{
2342 u32 max_frame_size = adapter->max_frame_size;
2343 struct e1000_hw *hw = &adapter->hw;
2344 u16 pf_id = adapter->vfs_allocated_count;
2345
2346 if (adapter->vlgrp)
2347 max_frame_size += VLAN_TAG_SIZE;
2348
2349 /* if vfs are enabled we set RLPML to the largest possible request
2350 * size and set the VMOLR RLPML to the size we need */
2351 if (pf_id) {
2352 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2353 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2354 }
2355
2356 wr32(E1000_RLPML, max_frame_size);
2357}
2358
7d5753f0
AD		 2359static inline void igb_set_vmolr(struct igb_adapter *adapter, int vfn)
2360{
2361 struct e1000_hw *hw = &adapter->hw;
2362 u32 vmolr;
2363
2364 /*
2365 * This register exists only on 82576 and newer so if we are older then
2366 * we should exit and do nothing
2367 */
2368 if (hw->mac.type < e1000_82576)
2369 return;
2370
2371 vmolr = rd32(E1000_VMOLR(vfn));
2372 vmolr |= E1000_VMOLR_AUPE | /* Accept untagged packets */
2373 E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2374
2375 /* clear all bits that might not be set */
2376 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2377
2378 if (adapter->num_rx_queues > 1 && vfn == adapter->vfs_allocated_count)
2379 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2380 /*
2381 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2382 * multicast packets
2383 */
2384 if (vfn <= adapter->vfs_allocated_count)
2385 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2386
2387 wr32(E1000_VMOLR(vfn), vmolr);
2388}
2389
85b430b4
AD		 2390/**
2391 * igb_configure_rx_ring - Configure a receive ring after Reset
2392 * @adapter: board private structure
2393 * @ring: receive ring to be configured
2394 *
2395 * Configure the Rx unit of the MAC after a reset.
2396 **/
d7ee5b3a
AD		 2397void igb_configure_rx_ring(struct igb_adapter *adapter,
2398 struct igb_ring *ring)
85b430b4
AD		 2399{
2400 struct e1000_hw *hw = &adapter->hw;
2401 u64 rdba = ring->dma;
2402 int reg_idx = ring->reg_idx;
952f72a8 2403 u32 srrctl, rxdctl;
85b430b4
AD		 2404
2405 /* disable the queue */
2406 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2407 wr32(E1000_RXDCTL(reg_idx),
2408 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2409
2410 /* Set DMA base address registers */
2411 wr32(E1000_RDBAL(reg_idx),
2412 rdba & 0x00000000ffffffffULL);
2413 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2414 wr32(E1000_RDLEN(reg_idx),
2415 ring->count * sizeof(union e1000_adv_rx_desc));
2416
2417 /* initialize head and tail */
fce99e34
AD		 2418 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2419 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2420 writel(0, ring->head);
2421 writel(0, ring->tail);
85b430b4 2422
952f72a8 2423 /* set descriptor configuration */
4c844851
AD		 2424 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2425 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2426 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2427#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2428 srrctl |= IGB_RXBUFFER_16384 >>
2429 E1000_SRRCTL_BSIZEPKT_SHIFT;
2430#else
2431 srrctl |= (PAGE_SIZE / 2) >>
2432 E1000_SRRCTL_BSIZEPKT_SHIFT;
2433#endif
2434 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2435 } else {
4c844851 2436 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2437 E1000_SRRCTL_BSIZEPKT_SHIFT;
2438 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2439 }
2440
2441 wr32(E1000_SRRCTL(reg_idx), srrctl);
2442
7d5753f0
AD		 2443 /* set filtering for VMDQ pools */
2444 igb_set_vmolr(adapter, reg_idx & 0x7);
2445
85b430b4
AD		 2446 /* enable receive descriptor fetching */
2447 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2448 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2449 rxdctl &= 0xFFF00000;
2450 rxdctl |= IGB_RX_PTHRESH;
2451 rxdctl |= IGB_RX_HTHRESH << 8;
2452 rxdctl |= IGB_RX_WTHRESH << 16;
2453 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2454}
2455
9d5c8243
AK		 2456/**
2457 * igb_configure_rx - Configure receive Unit after Reset
2458 * @adapter: board private structure
2459 *
2460 * Configure the Rx unit of the MAC after a reset.
2461 **/
2462static void igb_configure_rx(struct igb_adapter *adapter)
2463{
9107584e 2464 int i;
9d5c8243 2465
68d480c4
AD		 2466 /* set UTA to appropriate mode */
2467 igb_set_uta(adapter);
2468
26ad9178
AD		 2469 /* set the correct pool for the PF default MAC address in entry 0 */
2470 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2471 adapter->vfs_allocated_count);
2472
06cf2666
AD		 2473 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2474 * the Base and Length of the Rx Descriptor Ring */
2475 for (i = 0; i < adapter->num_rx_queues; i++)
2476 igb_configure_rx_ring(adapter, &adapter->rx_ring[i]);
9d5c8243
AK		 2477}
2478
2479/**
2480 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2481 * @tx_ring: Tx descriptor ring for a specific queue
2482 *
2483 * Free all transmit software resources
2484 **/
68fd9910 2485void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2486{
3b644cf6 2487 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2488
2489 vfree(tx_ring->buffer_info);
2490 tx_ring->buffer_info = NULL;
2491
439705e1
AD		 2492 /* if not set, then don't free */
2493 if (!tx_ring->desc)
2494 return;
2495
80785298
AD		 2496 pci_free_consistent(tx_ring->pdev, tx_ring->size,
2497 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2498
2499 tx_ring->desc = NULL;
2500}
2501
2502/**
2503 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2504 * @adapter: board private structure
2505 *
2506 * Free all transmit software resources
2507 **/
2508static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2509{
2510 int i;
2511
2512 for (i = 0; i < adapter->num_tx_queues; i++)
3b644cf6 2513 igb_free_tx_resources(&adapter->tx_ring[i]);
9d5c8243
AK		 2514}
2515
b1a436c3
AD		 2516void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2517 struct igb_buffer *buffer_info)
9d5c8243 2518{
65689fef 2519 buffer_info->dma = 0;
9d5c8243 2520 if (buffer_info->skb) {
80785298
AD		 2521 skb_dma_unmap(&tx_ring->pdev->dev,
2522 buffer_info->skb,
65689fef 2523 DMA_TO_DEVICE);
9d5c8243
AK		 2524 dev_kfree_skb_any(buffer_info->skb);
2525 buffer_info->skb = NULL;
2526 }
2527 buffer_info->time_stamp = 0;
2528 /* buffer_info must be completely set up in the transmit path */
2529}
2530
2531/**
2532 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 2533 * @tx_ring: ring to be cleaned
2534 **/
3b644cf6 2535static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 2536{
2537 struct igb_buffer *buffer_info;
2538 unsigned long size;
2539 unsigned int i;
2540
2541 if (!tx_ring->buffer_info)
2542 return;
2543 /* Free all the Tx ring sk_buffs */
2544
2545 for (i = 0; i < tx_ring->count; i++) {
2546 buffer_info = &tx_ring->buffer_info[i];
80785298 2547 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 2548 }
2549
2550 size = sizeof(struct igb_buffer) * tx_ring->count;
2551 memset(tx_ring->buffer_info, 0, size);
2552
2553 /* Zero out the descriptor ring */
9d5c8243
AK		 2554 memset(tx_ring->desc, 0, tx_ring->size);
2555
2556 tx_ring->next_to_use = 0;
2557 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2558}
2559
2560/**
2561 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2562 * @adapter: board private structure
2563 **/
2564static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2565{
2566 int i;
2567
2568 for (i = 0; i < adapter->num_tx_queues; i++)
3b644cf6 2569 igb_clean_tx_ring(&adapter->tx_ring[i]);
9d5c8243
AK		 2570}
2571
2572/**
2573 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 2574 * @rx_ring: ring to clean the resources from
2575 *
2576 * Free all receive software resources
2577 **/
68fd9910 2578void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2579{
3b644cf6 2580 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 2581
2582 vfree(rx_ring->buffer_info);
2583 rx_ring->buffer_info = NULL;
2584
439705e1
AD		 2585 /* if not set, then don't free */
2586 if (!rx_ring->desc)
2587 return;
2588
80785298
AD		 2589 pci_free_consistent(rx_ring->pdev, rx_ring->size,
2590 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 2591
2592 rx_ring->desc = NULL;
2593}
2594
2595/**
2596 * igb_free_all_rx_resources - Free Rx Resources for All Queues
2597 * @adapter: board private structure
2598 *
2599 * Free all receive software resources
2600 **/
2601static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2602{
2603 int i;
2604
2605 for (i = 0; i < adapter->num_rx_queues; i++)
3b644cf6 2606 igb_free_rx_resources(&adapter->rx_ring[i]);
9d5c8243
AK		 2607}
2608
2609/**
2610 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 2611 * @rx_ring: ring to free buffers from
2612 **/
3b644cf6 2613static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 2614{
2615 struct igb_buffer *buffer_info;
9d5c8243
AK		 2616 unsigned long size;
2617 unsigned int i;
2618
2619 if (!rx_ring->buffer_info)
2620 return;
439705e1 2621
9d5c8243
AK		 2622 /* Free all the Rx ring sk_buffs */
2623 for (i = 0; i < rx_ring->count; i++) {
2624 buffer_info = &rx_ring->buffer_info[i];
2625 if (buffer_info->dma) {
80785298
AD		 2626 pci_unmap_single(rx_ring->pdev,
2627 buffer_info->dma,
4c844851 2628 rx_ring->rx_buffer_len,
6ec43fe6 2629 PCI_DMA_FROMDEVICE);
9d5c8243
AK		 2630 buffer_info->dma = 0;
2631 }
2632
2633 if (buffer_info->skb) {
2634 dev_kfree_skb(buffer_info->skb);
2635 buffer_info->skb = NULL;
2636 }
6ec43fe6 2637 if (buffer_info->page_dma) {
80785298
AD		 2638 pci_unmap_page(rx_ring->pdev,
2639 buffer_info->page_dma,
6ec43fe6
AD		 2640 PAGE_SIZE / 2,
2641 PCI_DMA_FROMDEVICE);
2642 buffer_info->page_dma = 0;
2643 }
9d5c8243 2644 if (buffer_info->page) {
9d5c8243
AK		 2645 put_page(buffer_info->page);
2646 buffer_info->page = NULL;
bf36c1a0 2647 buffer_info->page_offset = 0;
9d5c8243
AK		 2648 }
2649 }
2650
9d5c8243
AK		 2651 size = sizeof(struct igb_buffer) * rx_ring->count;
2652 memset(rx_ring->buffer_info, 0, size);
2653
2654 /* Zero out the descriptor ring */
2655 memset(rx_ring->desc, 0, rx_ring->size);
2656
2657 rx_ring->next_to_clean = 0;
2658 rx_ring->next_to_use = 0;
9d5c8243
AK		 2659}
2660
2661/**
2662 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2663 * @adapter: board private structure
2664 **/
2665static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2666{
2667 int i;
2668
2669 for (i = 0; i < adapter->num_rx_queues; i++)
3b644cf6 2670 igb_clean_rx_ring(&adapter->rx_ring[i]);
9d5c8243
AK		 2671}
2672
2673/**
2674 * igb_set_mac - Change the Ethernet Address of the NIC
2675 * @netdev: network interface device structure
2676 * @p: pointer to an address structure
2677 *
2678 * Returns 0 on success, negative on failure
2679 **/
2680static int igb_set_mac(struct net_device *netdev, void *p)
2681{
2682 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 2683 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2684 struct sockaddr *addr = p;
2685
2686 if (!is_valid_ether_addr(addr->sa_data))
2687 return -EADDRNOTAVAIL;
2688
2689 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 2690 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 2691
26ad9178
AD		 2692 /* set the correct pool for the new PF MAC address in entry 0 */
2693 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2694 adapter->vfs_allocated_count);
e1739522 2695
9d5c8243
AK		 2696 return 0;
2697}
2698
2699/**
68d480c4 2700 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 2701 * @netdev: network interface device structure
2702 *
68d480c4
AD		 2703 * Writes multicast address list to the MTA hash table.
2704 * Returns: -ENOMEM on failure
2705 * 0 on no addresses written
2706 * X on writing X addresses to MTA
9d5c8243 2707 **/
68d480c4 2708static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 2709{
2710 struct igb_adapter *adapter = netdev_priv(netdev);
2711 struct e1000_hw *hw = &adapter->hw;
ff41f8dc 2712 struct dev_mc_list *mc_ptr = netdev->mc_list;
68d480c4
AD		 2713 u8 *mta_list;
2714 u32 vmolr = 0;
9d5c8243
AK		 2715 int i;
2716
68d480c4
AD		 2717 if (!netdev->mc_count) {
2718 /* nothing to program, so clear mc list */
2719 igb_update_mc_addr_list(hw, NULL, 0);
2720 igb_restore_vf_multicasts(adapter);
2721 return 0;
2722 }
9d5c8243 2723
68d480c4
AD		 2724 mta_list = kzalloc(netdev->mc_count * 6, GFP_ATOMIC);
2725 if (!mta_list)
2726 return -ENOMEM;
ff41f8dc 2727
68d480c4
AD		 2728 /* set vmolr receive overflow multicast bit */
2729 vmolr |= E1000_VMOLR_ROMPE;
2730
2731 /* The shared function expects a packed array of only addresses. */
2732 mc_ptr = netdev->mc_list;
2733
2734 for (i = 0; i < netdev->mc_count; i++) {
2735 if (!mc_ptr)
2736 break;
2737 memcpy(mta_list + (i*ETH_ALEN), mc_ptr->dmi_addr, ETH_ALEN);
2738 mc_ptr = mc_ptr->next;
746b9f02 2739 }
68d480c4
AD		 2740 igb_update_mc_addr_list(hw, mta_list, i);
2741 kfree(mta_list);
2742
2743 return netdev->mc_count;
2744}
2745
2746/**
2747 * igb_write_uc_addr_list - write unicast addresses to RAR table
2748 * @netdev: network interface device structure
2749 *
2750 * Writes unicast address list to the RAR table.
2751 * Returns: -ENOMEM on failure/insufficient address space
2752 * 0 on no addresses written
2753 * X on writing X addresses to the RAR table
2754 **/
2755static int igb_write_uc_addr_list(struct net_device *netdev)
2756{
2757 struct igb_adapter *adapter = netdev_priv(netdev);
2758 struct e1000_hw *hw = &adapter->hw;
2759 unsigned int vfn = adapter->vfs_allocated_count;
2760 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2761 int count = 0;
2762
2763 /* return ENOMEM indicating insufficient memory for addresses */
2764 if (netdev->uc.count > rar_entries)
2765 return -ENOMEM;
9d5c8243 2766
ff41f8dc
AD		 2767 if (netdev->uc.count && rar_entries) {
2768 struct netdev_hw_addr *ha;
2769 list_for_each_entry(ha, &netdev->uc.list, list) {
2770 if (!rar_entries)
2771 break;
26ad9178
AD		 2772 igb_rar_set_qsel(adapter, ha->addr,
2773 rar_entries--,
68d480c4
AD		 2774 vfn);
2775 count++;
ff41f8dc
AD		 2776 }
2777 }
2778 /* write the addresses in reverse order to avoid write combining */
2779 for (; rar_entries > 0 ; rar_entries--) {
2780 wr32(E1000_RAH(rar_entries), 0);
2781 wr32(E1000_RAL(rar_entries), 0);
2782 }
2783 wrfl();
2784
68d480c4
AD		 2785 return count;
2786}
2787
2788/**
2789 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2790 * @netdev: network interface device structure
2791 *
2792 * The set_rx_mode entry point is called whenever the unicast or multicast
2793 * address lists or the network interface flags are updated. This routine is
2794 * responsible for configuring the hardware for proper unicast, multicast,
2795 * promiscuous mode, and all-multi behavior.
2796 **/
2797static void igb_set_rx_mode(struct net_device *netdev)
2798{
2799 struct igb_adapter *adapter = netdev_priv(netdev);
2800 struct e1000_hw *hw = &adapter->hw;
2801 unsigned int vfn = adapter->vfs_allocated_count;
2802 u32 rctl, vmolr = 0;
2803 int count;
2804
2805 /* Check for Promiscuous and All Multicast modes */
2806 rctl = rd32(E1000_RCTL);
2807
2808 /* clear the effected bits */
2809 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2810
2811 if (netdev->flags & IFF_PROMISC) {
2812 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2813 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2814 } else {
2815 if (netdev->flags & IFF_ALLMULTI) {
2816 rctl |= E1000_RCTL_MPE;
2817 vmolr |= E1000_VMOLR_MPME;
2818 } else {
2819 /*
2820 * Write addresses to the MTA, if the attempt fails
2821 * then we should just turn on promiscous mode so
2822 * that we can at least receive multicast traffic
2823 */
2824 count = igb_write_mc_addr_list(netdev);
2825 if (count < 0) {
2826 rctl |= E1000_RCTL_MPE;
2827 vmolr |= E1000_VMOLR_MPME;
2828 } else if (count) {
2829 vmolr |= E1000_VMOLR_ROMPE;
2830 }
2831 }
2832 /*
2833 * Write addresses to available RAR registers, if there is not
2834 * sufficient space to store all the addresses then enable
2835 * unicast promiscous mode
2836 */
2837 count = igb_write_uc_addr_list(netdev);
2838 if (count < 0) {
2839 rctl |= E1000_RCTL_UPE;
2840 vmolr |= E1000_VMOLR_ROPE;
2841 }
2842 rctl |= E1000_RCTL_VFE;
28fc06f5 2843 }
68d480c4 2844 wr32(E1000_RCTL, rctl);
28fc06f5 2845
68d480c4
AD		 2846 /*
2847 * In order to support SR-IOV and eventually VMDq it is necessary to set
2848 * the VMOLR to enable the appropriate modes. Without this workaround
2849 * we will have issues with VLAN tag stripping not being done for frames
2850 * that are only arriving because we are the default pool
2851 */
2852 if (hw->mac.type < e1000_82576)
28fc06f5 2853 return;
9d5c8243 2854
68d480c4
AD		 2855 vmolr |= rd32(E1000_VMOLR(vfn)) &
2856 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
2857 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 2858 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 2859}
2860
2861/* Need to wait a few seconds after link up to get diagnostic information from
2862 * the phy */
2863static void igb_update_phy_info(unsigned long data)
2864{
2865 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 2866 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 2867}
2868
4d6b725e
AD		 2869/**
2870 * igb_has_link - check shared code for link and determine up/down
2871 * @adapter: pointer to driver private info
2872 **/
2873static bool igb_has_link(struct igb_adapter *adapter)
2874{
2875 struct e1000_hw *hw = &adapter->hw;
2876 bool link_active = false;
2877 s32 ret_val = 0;
2878
2879 /* get_link_status is set on LSC (link status) interrupt or
2880 * rx sequence error interrupt. get_link_status will stay
2881 * false until the e1000_check_for_link establishes link
2882 * for copper adapters ONLY
2883 */
2884 switch (hw->phy.media_type) {
2885 case e1000_media_type_copper:
2886 if (hw->mac.get_link_status) {
2887 ret_val = hw->mac.ops.check_for_link(hw);
2888 link_active = !hw->mac.get_link_status;
2889 } else {
2890 link_active = true;
2891 }
2892 break;
4d6b725e
AD		 2893 case e1000_media_type_internal_serdes:
2894 ret_val = hw->mac.ops.check_for_link(hw);
2895 link_active = hw->mac.serdes_has_link;
2896 break;
2897 default:
2898 case e1000_media_type_unknown:
2899 break;
2900 }
2901
2902 return link_active;
2903}
2904
9d5c8243
AK		 2905/**
2906 * igb_watchdog - Timer Call-back
2907 * @data: pointer to adapter cast into an unsigned long
2908 **/
2909static void igb_watchdog(unsigned long data)
2910{
2911 struct igb_adapter *adapter = (struct igb_adapter *)data;
2912 /* Do the rest outside of interrupt context */
2913 schedule_work(&adapter->watchdog_task);
2914}
2915
2916static void igb_watchdog_task(struct work_struct *work)
2917{
2918 struct igb_adapter *adapter = container_of(work,
2919 struct igb_adapter, watchdog_task);
2920 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2921 struct net_device *netdev = adapter->netdev;
2922 struct igb_ring *tx_ring = adapter->tx_ring;
9d5c8243 2923 u32 link;
7a6ea550 2924 int i;
9d5c8243 2925
4d6b725e 2926 link = igb_has_link(adapter);
9d5c8243
AK		 2927 if (link) {
2928 if (!netif_carrier_ok(netdev)) {
2929 u32 ctrl;
330a6d6a
AD		 2930 hw->mac.ops.get_speed_and_duplex(hw,
2931 &adapter->link_speed,
2932 &adapter->link_duplex);
9d5c8243
AK		 2933
2934 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 2935 /* Links status message must follow this format */
2936 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 2937 "Flow Control: %s\n",
527d47c1 2938 netdev->name,
9d5c8243
AK		 2939 adapter->link_speed,
2940 adapter->link_duplex == FULL_DUPLEX ?
2941 "Full Duplex" : "Half Duplex",
2942 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2943 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2944 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2945 E1000_CTRL_TFCE) ? "TX" : "None")));
2946
2947 /* tweak tx_queue_len according to speed/duplex and
2948 * adjust the timeout factor */
2949 netdev->tx_queue_len = adapter->tx_queue_len;
2950 adapter->tx_timeout_factor = 1;
2951 switch (adapter->link_speed) {
2952 case SPEED_10:
2953 netdev->tx_queue_len = 10;
2954 adapter->tx_timeout_factor = 14;
2955 break;
2956 case SPEED_100:
2957 netdev->tx_queue_len = 100;
2958 /* maybe add some timeout factor ? */
2959 break;
2960 }
2961
2962 netif_carrier_on(netdev);
9d5c8243 2963
4ae196df
AD		 2964 igb_ping_all_vfs(adapter);
2965
4b1a9877 2966 /* link state has changed, schedule phy info update */
9d5c8243
AK		 2967 if (!test_bit(__IGB_DOWN, &adapter->state))
2968 mod_timer(&adapter->phy_info_timer,
2969 round_jiffies(jiffies + 2 * HZ));
2970 }
2971 } else {
2972 if (netif_carrier_ok(netdev)) {
2973 adapter->link_speed = 0;
2974 adapter->link_duplex = 0;
527d47c1
AD		 2975 /* Links status message must follow this format */
2976 printk(KERN_INFO "igb: %s NIC Link is Down\n",
2977 netdev->name);
9d5c8243 2978 netif_carrier_off(netdev);
4b1a9877 2979
4ae196df
AD		 2980 igb_ping_all_vfs(adapter);
2981
4b1a9877 2982 /* link state has changed, schedule phy info update */
9d5c8243
AK		 2983 if (!test_bit(__IGB_DOWN, &adapter->state))
2984 mod_timer(&adapter->phy_info_timer,
2985 round_jiffies(jiffies + 2 * HZ));
2986 }
2987 }
2988
9d5c8243 2989 igb_update_stats(adapter);
645a3abd 2990 igb_update_adaptive(hw);
9d5c8243
AK		 2991
2992 if (!netif_carrier_ok(netdev)) {
c493ea45 2993 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
9d5c8243
AK		 2994 /* We've lost link, so the controller stops DMA,
2995 * but we've got queued Tx work that's never going
2996 * to get done, so reset controller to flush Tx.
2997 * (Do the reset outside of interrupt context). */
2998 adapter->tx_timeout_count++;
2999 schedule_work(&adapter->reset_task);
c2d5ab49
JB		 3000 /* return immediately since reset is imminent */
3001 return;
9d5c8243
AK		 3002 }
3003 }
3004
f7ba205e
AD		 3005 /* Force detection of hung controller every watchdog period */
3006 for (i = 0; i < adapter->num_tx_queues; i++)
3007 adapter->tx_ring[i].detect_tx_hung = true;
3008
9d5c8243 3009 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3010 if (adapter->msix_entries) {
047e0030
AD		 3011 u32 eics = 0;
3012 for (i = 0; i < adapter->num_q_vectors; i++) {
3013 struct igb_q_vector *q_vector = adapter->q_vector[i];
3014 eics |= q_vector->eims_value;
3015 }
7a6ea550
AD		 3016 wr32(E1000_EICS, eics);
3017 } else {
3018 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3019 }
9d5c8243 3020
9d5c8243
AK		 3021 /* Reset the timer */
3022 if (!test_bit(__IGB_DOWN, &adapter->state))
3023 mod_timer(&adapter->watchdog_timer,
3024 round_jiffies(jiffies + 2 * HZ));
3025}
3026
3027enum latency_range {
3028 lowest_latency = 0,
3029 low_latency = 1,
3030 bulk_latency = 2,
3031 latency_invalid = 255
3032};
3033
6eb5a7f1
AD		 3034/**
3035 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3036 *
3037 * Stores a new ITR value based on strictly on packet size. This
3038 * algorithm is less sophisticated than that used in igb_update_itr,
3039 * due to the difficulty of synchronizing statistics across multiple
3040 * receive rings. The divisors and thresholds used by this fuction
3041 * were determined based on theoretical maximum wire speed and testing
3042 * data, in order to minimize response time while increasing bulk
3043 * throughput.
3044 * This functionality is controlled by the InterruptThrottleRate module
3045 * parameter (see igb_param.c)
3046 * NOTE: This function is called only when operating in a multiqueue
3047 * receive environment.
047e0030 3048 * @q_vector: pointer to q_vector
6eb5a7f1 3049 **/
047e0030 3050static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3051{
047e0030 3052 int new_val = q_vector->itr_val;
6eb5a7f1 3053 int avg_wire_size = 0;
047e0030 3054 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3055
6eb5a7f1
AD		 3056 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3057 * ints/sec - ITR timer value of 120 ticks.
3058 */
3059 if (adapter->link_speed != SPEED_1000) {
047e0030 3060 new_val = 976;
6eb5a7f1 3061 goto set_itr_val;
9d5c8243 3062 }
047e0030
AD		 3063
3064 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3065 struct igb_ring *ring = q_vector->rx_ring;
3066 avg_wire_size = ring->total_bytes / ring->total_packets;
3067 }
3068
3069 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3070 struct igb_ring *ring = q_vector->tx_ring;
3071 avg_wire_size = max_t(u32, avg_wire_size,
3072 (ring->total_bytes /
3073 ring->total_packets));
3074 }
3075
3076 /* if avg_wire_size isn't set no work was done */
3077 if (!avg_wire_size)
3078 goto clear_counts;
9d5c8243 3079
6eb5a7f1
AD		 3080 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3081 avg_wire_size += 24;
3082
3083 /* Don't starve jumbo frames */
3084 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3085
6eb5a7f1
AD		 3086 /* Give a little boost to mid-size frames */
3087 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3088 new_val = avg_wire_size / 3;
3089 else
3090 new_val = avg_wire_size / 2;
9d5c8243 3091
6eb5a7f1 3092set_itr_val:
047e0030
AD		 3093 if (new_val != q_vector->itr_val) {
3094 q_vector->itr_val = new_val;
3095 q_vector->set_itr = 1;
9d5c8243 3096 }
6eb5a7f1 3097clear_counts:
047e0030
AD		 3098 if (q_vector->rx_ring) {
3099 q_vector->rx_ring->total_bytes = 0;
3100 q_vector->rx_ring->total_packets = 0;
3101 }
3102 if (q_vector->tx_ring) {
3103 q_vector->tx_ring->total_bytes = 0;
3104 q_vector->tx_ring->total_packets = 0;
3105 }
9d5c8243
AK		 3106}
3107
3108/**
3109 * igb_update_itr - update the dynamic ITR value based on statistics
3110 * Stores a new ITR value based on packets and byte
3111 * counts during the last interrupt. The advantage of per interrupt
3112 * computation is faster updates and more accurate ITR for the current
3113 * traffic pattern. Constants in this function were computed
3114 * based on theoretical maximum wire speed and thresholds were set based
3115 * on testing data as well as attempting to minimize response time
3116 * while increasing bulk throughput.
3117 * this functionality is controlled by the InterruptThrottleRate module
3118 * parameter (see igb_param.c)
3119 * NOTE: These calculations are only valid when operating in a single-
3120 * queue environment.
3121 * @adapter: pointer to adapter
047e0030 3122 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3123 * @packets: the number of packets during this measurement interval
3124 * @bytes: the number of bytes during this measurement interval
3125 **/
3126static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3127 int packets, int bytes)
3128{
3129 unsigned int retval = itr_setting;
3130
3131 if (packets == 0)
3132 goto update_itr_done;
3133
3134 switch (itr_setting) {
3135 case lowest_latency:
3136 /* handle TSO and jumbo frames */
3137 if (bytes/packets > 8000)
3138 retval = bulk_latency;
3139 else if ((packets < 5) && (bytes > 512))
3140 retval = low_latency;
3141 break;
3142 case low_latency: /* 50 usec aka 20000 ints/s */
3143 if (bytes > 10000) {
3144 /* this if handles the TSO accounting */
3145 if (bytes/packets > 8000) {
3146 retval = bulk_latency;
3147 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3148 retval = bulk_latency;
3149 } else if ((packets > 35)) {
3150 retval = lowest_latency;
3151 }
3152 } else if (bytes/packets > 2000) {
3153 retval = bulk_latency;
3154 } else if (packets <= 2 && bytes < 512) {
3155 retval = lowest_latency;
3156 }
3157 break;
3158 case bulk_latency: /* 250 usec aka 4000 ints/s */
3159 if (bytes > 25000) {
3160 if (packets > 35)
3161 retval = low_latency;
1e5c3d21 3162 } else if (bytes < 1500) {
9d5c8243
AK		 3163 retval = low_latency;
3164 }
3165 break;
3166 }
3167
3168update_itr_done:
3169 return retval;
3170}
3171
6eb5a7f1 3172static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3173{
047e0030 3174 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3175 u16 current_itr;
047e0030 3176 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3177
3178 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3179 if (adapter->link_speed != SPEED_1000) {
3180 current_itr = 0;
3181 new_itr = 4000;
3182 goto set_itr_now;
3183 }
3184
3185 adapter->rx_itr = igb_update_itr(adapter,
3186 adapter->rx_itr,
3187 adapter->rx_ring->total_packets,
3188 adapter->rx_ring->total_bytes);
9d5c8243 3189
047e0030
AD		 3190 adapter->tx_itr = igb_update_itr(adapter,
3191 adapter->tx_itr,
3192 adapter->tx_ring->total_packets,
3193 adapter->tx_ring->total_bytes);
3194 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3195
6eb5a7f1 3196 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3197 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3198 current_itr = low_latency;
3199
9d5c8243
AK		 3200 switch (current_itr) {
3201 /* counts and packets in update_itr are dependent on these numbers */
3202 case lowest_latency:
78b1f607 3203 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3204 break;
3205 case low_latency:
78b1f607 3206 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3207 break;
3208 case bulk_latency:
78b1f607 3209 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3210 break;
3211 default:
3212 break;
3213 }
3214
3215set_itr_now:
6eb5a7f1
AD		 3216 adapter->rx_ring->total_bytes = 0;
3217 adapter->rx_ring->total_packets = 0;
047e0030
AD		 3218 adapter->tx_ring->total_bytes = 0;
3219 adapter->tx_ring->total_packets = 0;
6eb5a7f1 3220
047e0030 3221 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3222 /* this attempts to bias the interrupt rate towards Bulk
3223 * by adding intermediate steps when interrupt rate is
3224 * increasing */
047e0030
AD		 3225 new_itr = new_itr > q_vector->itr_val ?
3226 max((new_itr * q_vector->itr_val) /
3227 (new_itr + (q_vector->itr_val >> 2)),
3228 new_itr) :
9d5c8243
AK		 3229 new_itr;
3230 /* Don't write the value here; it resets the adapter's
3231 * internal timer, and causes us to delay far longer than
3232 * we should between interrupts. Instead, we write the ITR
3233 * value at the beginning of the next interrupt so the timing
3234 * ends up being correct.
3235 */
047e0030
AD		 3236 q_vector->itr_val = new_itr;
3237 q_vector->set_itr = 1;
9d5c8243
AK		 3238 }
3239
3240 return;
3241}
3242
9d5c8243
AK		 3243#define IGB_TX_FLAGS_CSUM 0x00000001
3244#define IGB_TX_FLAGS_VLAN 0x00000002
3245#define IGB_TX_FLAGS_TSO 0x00000004
3246#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3247#define IGB_TX_FLAGS_TSTAMP 0x00000010
3248#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3249#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3250
85ad76b2 3251static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3252 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3253{
3254 struct e1000_adv_tx_context_desc *context_desc;
3255 unsigned int i;
3256 int err;
3257 struct igb_buffer *buffer_info;
3258 u32 info = 0, tu_cmd = 0;
3259 u32 mss_l4len_idx, l4len;
3260 *hdr_len = 0;
3261
3262 if (skb_header_cloned(skb)) {
3263 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3264 if (err)
3265 return err;
3266 }
3267
3268 l4len = tcp_hdrlen(skb);
3269 *hdr_len += l4len;
3270
3271 if (skb->protocol == htons(ETH_P_IP)) {
3272 struct iphdr *iph = ip_hdr(skb);
3273 iph->tot_len = 0;
3274 iph->check = 0;
3275 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3276 iph->daddr, 0,
3277 IPPROTO_TCP,
3278 0);
3279 } else if (skb_shinfo(skb)->gso_type == SKB_GSO_TCPV6) {
3280 ipv6_hdr(skb)->payload_len = 0;
3281 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3282 &ipv6_hdr(skb)->daddr,
3283 0, IPPROTO_TCP, 0);
3284 }
3285
3286 i = tx_ring->next_to_use;
3287
3288 buffer_info = &tx_ring->buffer_info[i];
3289 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3290 /* VLAN MACLEN IPLEN */
3291 if (tx_flags & IGB_TX_FLAGS_VLAN)
3292 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3293 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3294 *hdr_len += skb_network_offset(skb);
3295 info |= skb_network_header_len(skb);
3296 *hdr_len += skb_network_header_len(skb);
3297 context_desc->vlan_macip_lens = cpu_to_le32(info);
3298
3299 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3300 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3301
3302 if (skb->protocol == htons(ETH_P_IP))
3303 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3304 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3305
3306 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3307
3308 /* MSS L4LEN IDX */
3309 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3310 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3311
73cd78f1 3312 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3313 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3314 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3315
3316 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3317 context_desc->seqnum_seed = 0;
3318
3319 buffer_info->time_stamp = jiffies;
0e014cb1 3320 buffer_info->next_to_watch = i;
9d5c8243
AK		 3321 buffer_info->dma = 0;
3322 i++;
3323 if (i == tx_ring->count)
3324 i = 0;
3325
3326 tx_ring->next_to_use = i;
3327
3328 return true;
3329}
3330
85ad76b2
AD		 3331static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3332 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3333{
3334 struct e1000_adv_tx_context_desc *context_desc;
80785298 3335 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3336 struct igb_buffer *buffer_info;
3337 u32 info = 0, tu_cmd = 0;
80785298 3338 unsigned int i;
9d5c8243
AK		 3339
3340 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3341 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3342 i = tx_ring->next_to_use;
3343 buffer_info = &tx_ring->buffer_info[i];
3344 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3345
3346 if (tx_flags & IGB_TX_FLAGS_VLAN)
3347 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3348
9d5c8243
AK		 3349 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3350 if (skb->ip_summed == CHECKSUM_PARTIAL)
3351 info |= skb_network_header_len(skb);
3352
3353 context_desc->vlan_macip_lens = cpu_to_le32(info);
3354
3355 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3356
3357 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3358 __be16 protocol;
3359
3360 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3361 const struct vlan_ethhdr *vhdr =
3362 (const struct vlan_ethhdr*)skb->data;
3363
3364 protocol = vhdr->h_vlan_encapsulated_proto;
3365 } else {
3366 protocol = skb->protocol;
3367 }
3368
3369 switch (protocol) {
09640e63 3370 case cpu_to_be16(ETH_P_IP):
9d5c8243 3371 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3372 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3373 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3374 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3375 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3376 break;
09640e63 3377 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3378 /* XXX what about other V6 headers?? */
3379 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3380 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3381 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3382 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3383 break;
3384 default:
3385 if (unlikely(net_ratelimit()))
80785298 3386 dev_warn(&pdev->dev,
44b0cda3
MW		 3387 "partial checksum but proto=%x!\n",
3388 skb->protocol);
3389 break;
3390 }
9d5c8243
AK		 3391 }
3392
3393 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3394 context_desc->seqnum_seed = 0;
85ad76b2 3395 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3396 context_desc->mss_l4len_idx =
85ad76b2 3397 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3398
3399 buffer_info->time_stamp = jiffies;
0e014cb1 3400 buffer_info->next_to_watch = i;
9d5c8243
AK		 3401 buffer_info->dma = 0;
3402
3403 i++;
3404 if (i == tx_ring->count)
3405 i = 0;
3406 tx_ring->next_to_use = i;
3407
3408 return true;
3409 }
9d5c8243
AK		 3410 return false;
3411}
3412
3413#define IGB_MAX_TXD_PWR 16
3414#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3415
80785298 3416static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3417 unsigned int first)
9d5c8243
AK		 3418{
3419 struct igb_buffer *buffer_info;
80785298 3420 struct pci_dev *pdev = tx_ring->pdev;
9d5c8243
AK		 3421 unsigned int len = skb_headlen(skb);
3422 unsigned int count = 0, i;
3423 unsigned int f;
65689fef 3424 dma_addr_t *map;
9d5c8243
AK		 3425
3426 i = tx_ring->next_to_use;
3427
80785298
AD		 3428 if (skb_dma_map(&pdev->dev, skb, DMA_TO_DEVICE)) {
3429 dev_err(&pdev->dev, "TX DMA map failed\n");
65689fef
AD		 3430 return 0;
3431 }
3432
3433 map = skb_shinfo(skb)->dma_maps;
3434
9d5c8243
AK		 3435 buffer_info = &tx_ring->buffer_info[i];
3436 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3437 buffer_info->length = len;
3438 /* set time_stamp *before* dma to help avoid a possible race */
3439 buffer_info->time_stamp = jiffies;
0e014cb1 3440 buffer_info->next_to_watch = i;
042a53a9 3441 buffer_info->dma = skb_shinfo(skb)->dma_head;
9d5c8243
AK		 3442
3443 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3444 struct skb_frag_struct *frag;
3445
65689fef
AD		 3446 i++;
3447 if (i == tx_ring->count)
3448 i = 0;
3449
9d5c8243
AK		 3450 frag = &skb_shinfo(skb)->frags[f];
3451 len = frag->size;
3452
3453 buffer_info = &tx_ring->buffer_info[i];
3454 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3455 buffer_info->length = len;
3456 buffer_info->time_stamp = jiffies;
0e014cb1 3457 buffer_info->next_to_watch = i;
65689fef 3458 buffer_info->dma = map[count];
9d5c8243 3459 count++;
9d5c8243
AK		 3460 }
3461
9d5c8243 3462 tx_ring->buffer_info[i].skb = skb;
0e014cb1 3463 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3464
cdfd01fc 3465 return ++count;
9d5c8243
AK		 3466}
3467
85ad76b2 3468static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3469 int tx_flags, int count, u32 paylen,
3470 u8 hdr_len)
3471{
cdfd01fc 3472 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 3473 struct igb_buffer *buffer_info;
3474 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 3475 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 3476
3477 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3478 E1000_ADVTXD_DCMD_DEXT);
3479
3480 if (tx_flags & IGB_TX_FLAGS_VLAN)
3481 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3482
33af6bcc
PO		 3483 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3484 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3485
9d5c8243
AK		 3486 if (tx_flags & IGB_TX_FLAGS_TSO) {
3487 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3488
3489 /* insert tcp checksum */
3490 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3491
3492 /* insert ip checksum */
3493 if (tx_flags & IGB_TX_FLAGS_IPV4)
3494 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3495
3496 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3497 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3498 }
3499
85ad76b2
AD		 3500 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3501 (tx_flags & (IGB_TX_FLAGS_CSUM |
3502 IGB_TX_FLAGS_TSO |
7dfc16fa 3503 IGB_TX_FLAGS_VLAN)))
85ad76b2 3504 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3505
3506 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3507
cdfd01fc 3508 do {
9d5c8243
AK		 3509 buffer_info = &tx_ring->buffer_info[i];
3510 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3511 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3512 tx_desc->read.cmd_type_len =
3513 cpu_to_le32(cmd_type_len | buffer_info->length);
3514 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 3515 count--;
9d5c8243
AK		 3516 i++;
3517 if (i == tx_ring->count)
3518 i = 0;
cdfd01fc 3519 } while (count > 0);
9d5c8243 3520
85ad76b2 3521 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 3522 /* Force memory writes to complete before letting h/w
3523 * know there are new descriptors to fetch. (Only
3524 * applicable for weak-ordered memory model archs,
3525 * such as IA-64). */
3526 wmb();
3527
3528 tx_ring->next_to_use = i;
fce99e34 3529 writel(i, tx_ring->tail);
9d5c8243
AK		 3530 /* we need this if more than one processor can write to our tail
3531 * at a time, it syncronizes IO on IA64/Altix systems */
3532 mmiowb();
3533}
3534
e694e964 3535static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3536{
e694e964
AD		 3537 struct net_device *netdev = tx_ring->netdev;
3538
661086df 3539 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 3540
9d5c8243
AK		 3541 /* Herbert's original patch had:
3542 * smp_mb__after_netif_stop_queue();
3543 * but since that doesn't exist yet, just open code it. */
3544 smp_mb();
3545
3546 /* We need to check again in a case another CPU has just
3547 * made room available. */
c493ea45 3548 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 3549 return -EBUSY;
3550
3551 /* A reprieve! */
661086df 3552 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 3553 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 3554 return 0;
3555}
3556
e694e964 3557static int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 3558{
c493ea45 3559 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 3560 return 0;
e694e964 3561 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 3562}
3563
b1a436c3
AD		 3564netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3565 struct igb_ring *tx_ring)
9d5c8243 3566{
e694e964 3567 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
0e014cb1 3568 unsigned int first;
9d5c8243 3569 unsigned int tx_flags = 0;
9d5c8243 3570 u8 hdr_len = 0;
cdfd01fc 3571 int tso = 0, count;
c5b9bd5e 3572 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 3573
9d5c8243
AK		 3574 /* need: 1 descriptor per page,
3575 * + 2 desc gap to keep tail from touching head,
3576 * + 1 desc for skb->data,
3577 * + 1 desc for context descriptor,
3578 * otherwise try next time */
e694e964 3579 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 3580 /* this is a hard error */
9d5c8243
AK		 3581 return NETDEV_TX_BUSY;
3582 }
33af6bcc 3583
33af6bcc
PO		 3584 if (unlikely(shtx->hardware)) {
3585 shtx->in_progress = 1;
3586 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 3587 }
9d5c8243 3588
cdfd01fc 3589 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 3590 tx_flags |= IGB_TX_FLAGS_VLAN;
3591 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3592 }
3593
661086df
PWJ		 3594 if (skb->protocol == htons(ETH_P_IP))
3595 tx_flags |= IGB_TX_FLAGS_IPV4;
3596
0e014cb1 3597 first = tx_ring->next_to_use;
85ad76b2
AD		 3598 if (skb_is_gso(skb)) {
3599 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 3600
85ad76b2
AD		 3601 if (tso < 0) {
3602 dev_kfree_skb_any(skb);
3603 return NETDEV_TX_OK;
3604 }
9d5c8243
AK		 3605 }
3606
3607 if (tso)
3608 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 3609 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 3610 (skb->ip_summed == CHECKSUM_PARTIAL))
3611 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 3612
65689fef 3613 /*
cdfd01fc 3614 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 3615 * has occured and we need to rewind the descriptor queue
3616 */
80785298 3617 count = igb_tx_map_adv(tx_ring, skb, first);
cdfd01fc 3618 if (count <= 0) {
65689fef
AD		 3619 dev_kfree_skb_any(skb);
3620 tx_ring->buffer_info[first].time_stamp = 0;
3621 tx_ring->next_to_use = first;
85ad76b2 3622 return NETDEV_TX_OK;
65689fef 3623 }
9d5c8243 3624
85ad76b2
AD		 3625 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3626
3627 /* Make sure there is space in the ring for the next send. */
e694e964 3628 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 3629
9d5c8243
AK		 3630 return NETDEV_TX_OK;
3631}
3632
3b29a56d
SH		 3633static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3634 struct net_device *netdev)
9d5c8243
AK		 3635{
3636 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 3637 struct igb_ring *tx_ring;
661086df 3638 int r_idx = 0;
b1a436c3
AD		 3639
3640 if (test_bit(__IGB_DOWN, &adapter->state)) {
3641 dev_kfree_skb_any(skb);
3642 return NETDEV_TX_OK;
3643 }
3644
3645 if (skb->len <= 0) {
3646 dev_kfree_skb_any(skb);
3647 return NETDEV_TX_OK;
3648 }
3649
1bfaf07b 3650 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 3651 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 3652
3653 /* This goes back to the question of how to logically map a tx queue
3654 * to a flow. Right now, performance is impacted slightly negatively
3655 * if using multiple tx queues. If the stack breaks away from a
3656 * single qdisc implementation, we can look at this again. */
e694e964 3657 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 3658}
3659
3660/**
3661 * igb_tx_timeout - Respond to a Tx Hang
3662 * @netdev: network interface device structure
3663 **/
3664static void igb_tx_timeout(struct net_device *netdev)
3665{
3666 struct igb_adapter *adapter = netdev_priv(netdev);
3667 struct e1000_hw *hw = &adapter->hw;
3668
3669 /* Do the reset outside of interrupt context */
3670 adapter->tx_timeout_count++;
f7ba205e 3671
9d5c8243 3672 schedule_work(&adapter->reset_task);
265de409
AD		 3673 wr32(E1000_EICS,
3674 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 3675}
3676
3677static void igb_reset_task(struct work_struct *work)
3678{
3679 struct igb_adapter *adapter;
3680 adapter = container_of(work, struct igb_adapter, reset_task);
3681
3682 igb_reinit_locked(adapter);
3683}
3684
3685/**
3686 * igb_get_stats - Get System Network Statistics
3687 * @netdev: network interface device structure
3688 *
3689 * Returns the address of the device statistics structure.
3690 * The statistics are actually updated from the timer callback.
3691 **/
73cd78f1 3692static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 3693{
9d5c8243 3694 /* only return the current stats */
8d24e933 3695 return &netdev->stats;
9d5c8243
AK		 3696}
3697
3698/**
3699 * igb_change_mtu - Change the Maximum Transfer Unit
3700 * @netdev: network interface device structure
3701 * @new_mtu: new value for maximum frame size
3702 *
3703 * Returns 0 on success, negative on failure
3704 **/
3705static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3706{
3707 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 3708 struct pci_dev *pdev = adapter->pdev;
9d5c8243 3709 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 3710 u32 rx_buffer_len, i;
9d5c8243
AK		 3711
3712 if ((max_frame < ETH_ZLEN + ETH_FCS_LEN) ||
3713 (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 3714 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 3715 return -EINVAL;
3716 }
3717
9d5c8243 3718 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 3719 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 3720 return -EINVAL;
3721 }
3722
3723 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3724 msleep(1);
73cd78f1 3725
9d5c8243
AK		 3726 /* igb_down has a dependency on max_frame_size */
3727 adapter->max_frame_size = max_frame;
9d5c8243
AK		 3728 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3729 * means we reserve 2 more, this pushes us to allocate from the next
3730 * larger slab size.
3731 * i.e. RXBUFFER_2048 --> size-4096 slab
3732 */
3733
7d95b717 3734 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 3735 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 3736 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 3737 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 3738 else
4c844851
AD		 3739 rx_buffer_len = IGB_RXBUFFER_128;
3740
3741 if (netif_running(netdev))
3742 igb_down(adapter);
9d5c8243 3743
090b1795 3744 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 3745 netdev->mtu, new_mtu);
3746 netdev->mtu = new_mtu;
3747
4c844851
AD		 3748 for (i = 0; i < adapter->num_rx_queues; i++)
3749 adapter->rx_ring[i].rx_buffer_len = rx_buffer_len;
3750
9d5c8243
AK		 3751 if (netif_running(netdev))
3752 igb_up(adapter);
3753 else
3754 igb_reset(adapter);
3755
3756 clear_bit(__IGB_RESETTING, &adapter->state);
3757
3758 return 0;
3759}
3760
3761/**
3762 * igb_update_stats - Update the board statistics counters
3763 * @adapter: board private structure
3764 **/
3765
3766void igb_update_stats(struct igb_adapter *adapter)
3767{
8d24e933 3768 struct net_device *netdev = adapter->netdev;
9d5c8243
AK		 3769 struct e1000_hw *hw = &adapter->hw;
3770 struct pci_dev *pdev = adapter->pdev;
3f9c0164 3771 u32 rnbc;
9d5c8243 3772 u16 phy_tmp;
3f9c0164
AD		 3773 int i;
3774 u64 bytes, packets;
9d5c8243
AK		 3775
3776#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3777
3778 /*
3779 * Prevent stats update while adapter is being reset, or if the pci
3780 * connection is down.
3781 */
3782 if (adapter->link_speed == 0)
3783 return;
3784 if (pci_channel_offline(pdev))
3785 return;
3786
3f9c0164
AD		 3787 bytes = 0;
3788 packets = 0;
3789 for (i = 0; i < adapter->num_rx_queues; i++) {
3790 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3791 adapter->rx_ring[i].rx_stats.drops += rqdpc_tmp;
3792 netdev->stats.rx_fifo_errors += rqdpc_tmp;
3793 bytes += adapter->rx_ring[i].rx_stats.bytes;
3794 packets += adapter->rx_ring[i].rx_stats.packets;
3795 }
3796
3797 netdev->stats.rx_bytes = bytes;
3798 netdev->stats.rx_packets = packets;
3799
3800 bytes = 0;
3801 packets = 0;
3802 for (i = 0; i < adapter->num_tx_queues; i++) {
3803 bytes += adapter->tx_ring[i].tx_stats.bytes;
3804 packets += adapter->tx_ring[i].tx_stats.packets;
3805 }
3806 netdev->stats.tx_bytes = bytes;
3807 netdev->stats.tx_packets = packets;
3808
3809 /* read stats registers */
9d5c8243
AK		 3810 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
3811 adapter->stats.gprc += rd32(E1000_GPRC);
3812 adapter->stats.gorc += rd32(E1000_GORCL);
3813 rd32(E1000_GORCH); /* clear GORCL */
3814 adapter->stats.bprc += rd32(E1000_BPRC);
3815 adapter->stats.mprc += rd32(E1000_MPRC);
3816 adapter->stats.roc += rd32(E1000_ROC);
3817
3818 adapter->stats.prc64 += rd32(E1000_PRC64);
3819 adapter->stats.prc127 += rd32(E1000_PRC127);
3820 adapter->stats.prc255 += rd32(E1000_PRC255);
3821 adapter->stats.prc511 += rd32(E1000_PRC511);
3822 adapter->stats.prc1023 += rd32(E1000_PRC1023);
3823 adapter->stats.prc1522 += rd32(E1000_PRC1522);
3824 adapter->stats.symerrs += rd32(E1000_SYMERRS);
3825 adapter->stats.sec += rd32(E1000_SEC);
3826
3827 adapter->stats.mpc += rd32(E1000_MPC);
3828 adapter->stats.scc += rd32(E1000_SCC);
3829 adapter->stats.ecol += rd32(E1000_ECOL);
3830 adapter->stats.mcc += rd32(E1000_MCC);
3831 adapter->stats.latecol += rd32(E1000_LATECOL);
3832 adapter->stats.dc += rd32(E1000_DC);
3833 adapter->stats.rlec += rd32(E1000_RLEC);
3834 adapter->stats.xonrxc += rd32(E1000_XONRXC);
3835 adapter->stats.xontxc += rd32(E1000_XONTXC);
3836 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
3837 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
3838 adapter->stats.fcruc += rd32(E1000_FCRUC);
3839 adapter->stats.gptc += rd32(E1000_GPTC);
3840 adapter->stats.gotc += rd32(E1000_GOTCL);
3841 rd32(E1000_GOTCH); /* clear GOTCL */
3f9c0164
AD		 3842 rnbc = rd32(E1000_RNBC);
3843 adapter->stats.rnbc += rnbc;
3844 netdev->stats.rx_fifo_errors += rnbc;
9d5c8243
AK		 3845 adapter->stats.ruc += rd32(E1000_RUC);
3846 adapter->stats.rfc += rd32(E1000_RFC);
3847 adapter->stats.rjc += rd32(E1000_RJC);
3848 adapter->stats.tor += rd32(E1000_TORH);
3849 adapter->stats.tot += rd32(E1000_TOTH);
3850 adapter->stats.tpr += rd32(E1000_TPR);
3851
3852 adapter->stats.ptc64 += rd32(E1000_PTC64);
3853 adapter->stats.ptc127 += rd32(E1000_PTC127);
3854 adapter->stats.ptc255 += rd32(E1000_PTC255);
3855 adapter->stats.ptc511 += rd32(E1000_PTC511);
3856 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
3857 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
3858
3859 adapter->stats.mptc += rd32(E1000_MPTC);
3860 adapter->stats.bptc += rd32(E1000_BPTC);
3861
3862 /* used for adaptive IFS */
9d5c8243
AK		 3863 hw->mac.tx_packet_delta = rd32(E1000_TPT);
3864 adapter->stats.tpt += hw->mac.tx_packet_delta;
3865 hw->mac.collision_delta = rd32(E1000_COLC);
3866 adapter->stats.colc += hw->mac.collision_delta;
3867
3868 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
3869 adapter->stats.rxerrc += rd32(E1000_RXERRC);
3870 adapter->stats.tncrs += rd32(E1000_TNCRS);
3871 adapter->stats.tsctc += rd32(E1000_TSCTC);
3872 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
3873
3874 adapter->stats.iac += rd32(E1000_IAC);
3875 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
3876 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
3877 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
3878 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
3879 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
3880 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
3881 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
3882 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
3883
3884 /* Fill out the OS statistics structure */
8d24e933
AK		 3885 netdev->stats.multicast = adapter->stats.mprc;
3886 netdev->stats.collisions = adapter->stats.colc;
9d5c8243
AK		 3887
3888 /* Rx Errors */
3889
3890 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 3891 * our own version based on RUC and ROC */
8d24e933 3892 netdev->stats.rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 3893 adapter->stats.crcerrs + adapter->stats.algnerrc +
3894 adapter->stats.ruc + adapter->stats.roc +
3895 adapter->stats.cexterr;
8d24e933 3896 netdev->stats.rx_length_errors = adapter->stats.ruc +
9d5c8243 3897 adapter->stats.roc;
8d24e933
AK		 3898 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3899 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3900 netdev->stats.rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 3901
3902 /* Tx Errors */
8d24e933 3903 netdev->stats.tx_errors = adapter->stats.ecol +
9d5c8243 3904 adapter->stats.latecol;
8d24e933
AK		 3905 netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3906 netdev->stats.tx_window_errors = adapter->stats.latecol;
3907 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 3908
3909 /* Tx Dropped needs to be maintained elsewhere */
3910
3911 /* Phy Stats */
3912 if (hw->phy.media_type == e1000_media_type_copper) {
3913 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 3914 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 3915 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3916 adapter->phy_stats.idle_errors += phy_tmp;
3917 }
3918 }
3919
3920 /* Management Stats */
3921 adapter->stats.mgptc += rd32(E1000_MGTPTC);
3922 adapter->stats.mgprc += rd32(E1000_MGTPRC);
3923 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
3924}
3925
9d5c8243
AK		 3926static irqreturn_t igb_msix_other(int irq, void *data)
3927{
047e0030 3928 struct igb_adapter *adapter = data;
9d5c8243 3929 struct e1000_hw *hw = &adapter->hw;
844290e5 3930 u32 icr = rd32(E1000_ICR);
844290e5 3931 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 3932
047e0030 3933 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 3934 /* HW is reporting DMA is out of sync */
3935 adapter->stats.doosync++;
3936 }
eebbbdba 3937
4ae196df
AD		 3938 /* Check for a mailbox event */
3939 if (icr & E1000_ICR_VMMB)
3940 igb_msg_task(adapter);
3941
3942 if (icr & E1000_ICR_LSC) {
3943 hw->mac.get_link_status = 1;
3944 /* guard against interrupt when we're going down */
3945 if (!test_bit(__IGB_DOWN, &adapter->state))
3946 mod_timer(&adapter->watchdog_timer, jiffies + 1);
3947 }
3948
25568a53
AD		 3949 if (adapter->vfs_allocated_count)
3950 wr32(E1000_IMS, E1000_IMS_LSC |
3951 E1000_IMS_VMMB |
3952 E1000_IMS_DOUTSYNC);
3953 else
3954 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 3955 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 3956
3957 return IRQ_HANDLED;
3958}
3959
047e0030 3960static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 3961{
047e0030 3962 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 3963
047e0030
AD		 3964 if (!q_vector->set_itr)
3965 return;
73cd78f1 3966
047e0030
AD		 3967 if (!itr_val)
3968 itr_val = 0x4;
661086df 3969
047e0030
AD		 3970 if (q_vector->itr_shift)
3971 itr_val |= itr_val << q_vector->itr_shift;
661086df 3972 else
047e0030 3973 itr_val |= 0x8000000;
661086df 3974
047e0030
AD		 3975 writel(itr_val, q_vector->itr_register);
3976 q_vector->set_itr = 0;
6eb5a7f1
AD		 3977}
3978
047e0030 3979static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 3980{
047e0030 3981 struct igb_q_vector *q_vector = data;
9d5c8243 3982
047e0030
AD		 3983 /* Write the ITR value calculated from the previous interrupt. */
3984 igb_write_itr(q_vector);
9d5c8243 3985
047e0030 3986 napi_schedule(&q_vector->napi);
844290e5 3987
047e0030 3988 return IRQ_HANDLED;
fe4506b6
JC		 3989}
3990
421e02f0 3991#ifdef CONFIG_IGB_DCA
047e0030 3992static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 3993{
047e0030 3994 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 3995 struct e1000_hw *hw = &adapter->hw;
3996 int cpu = get_cpu();
fe4506b6 3997
047e0030
AD		 3998 if (q_vector->cpu == cpu)
3999 goto out_no_update;
4000
4001 if (q_vector->tx_ring) {
4002 int q = q_vector->tx_ring->reg_idx;
4003 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4004 if (hw->mac.type == e1000_82575) {
4005 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4006 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4007 } else {
047e0030
AD		 4008 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4009 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4010 E1000_DCA_TXCTRL_CPUID_SHIFT;
4011 }
4012 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4013 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4014 }
4015 if (q_vector->rx_ring) {
4016 int q = q_vector->rx_ring->reg_idx;
4017 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4018 if (hw->mac.type == e1000_82575) {
2d064c06 4019 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4020 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4021 } else {
4022 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4023 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4024 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4025 }
fe4506b6
JC		 4026 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4027 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4028 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4029 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4030 }
047e0030
AD		 4031 q_vector->cpu = cpu;
4032out_no_update:
fe4506b6
JC		 4033 put_cpu();
4034}
4035
4036static void igb_setup_dca(struct igb_adapter *adapter)
4037{
7e0e99ef 4038 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4039 int i;
4040
7dfc16fa 4041 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4042 return;
4043
7e0e99ef
AD		 4044 /* Always use CB2 mode, difference is masked in the CB driver. */
4045 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4046
047e0030
AD		 4047 for (i = 0; i < adapter->num_q_vectors; i++) {
4048 struct igb_q_vector *q_vector = adapter->q_vector[i];
4049 q_vector->cpu = -1;
4050 igb_update_dca(q_vector);
fe4506b6
JC		 4051 }
4052}
4053
4054static int __igb_notify_dca(struct device *dev, void *data)
4055{
4056 struct net_device *netdev = dev_get_drvdata(dev);
4057 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4058 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4059 struct e1000_hw *hw = &adapter->hw;
4060 unsigned long event = *(unsigned long *)data;
4061
4062 switch (event) {
4063 case DCA_PROVIDER_ADD:
4064 /* if already enabled, don't do it again */
7dfc16fa 4065 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4066 break;
fe4506b6 4067 if (dca_add_requester(dev) == 0) {
bbd98fe4 4068 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4069 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4070 igb_setup_dca(adapter);
4071 break;
4072 }
4073 /* Fall Through since DCA is disabled. */
4074 case DCA_PROVIDER_REMOVE:
7dfc16fa 4075 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4076 /* without this a class_device is left
047e0030 4077 * hanging around in the sysfs model */
fe4506b6 4078 dca_remove_requester(dev);
090b1795 4079 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4080 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4081 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4082 }
4083 break;
4084 }
bbd98fe4 4085
fe4506b6 4086 return 0;
9d5c8243
AK		 4087}
4088
fe4506b6
JC		 4089static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4090 void *p)
4091{
4092 int ret_val;
4093
4094 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4095 __igb_notify_dca);
4096
4097 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4098}
421e02f0 4099#endif /* CONFIG_IGB_DCA */
9d5c8243 4100
4ae196df
AD		 4101static void igb_ping_all_vfs(struct igb_adapter *adapter)
4102{
4103 struct e1000_hw *hw = &adapter->hw;
4104 u32 ping;
4105 int i;
4106
4107 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4108 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4109 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4110 ping |= E1000_VT_MSGTYPE_CTS;
4111 igb_write_mbx(hw, &ping, 1, i);
4112 }
4113}
4114
7d5753f0
AD		 4115static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4116{
4117 struct e1000_hw *hw = &adapter->hw;
4118 u32 vmolr = rd32(E1000_VMOLR(vf));
4119 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4120
4121 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4122 IGB_VF_FLAG_MULTI_PROMISC);
4123 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4124
4125 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4126 vmolr |= E1000_VMOLR_MPME;
4127 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4128 } else {
4129 /*
4130 * if we have hashes and we are clearing a multicast promisc
4131 * flag we need to write the hashes to the MTA as this step
4132 * was previously skipped
4133 */
4134 if (vf_data->num_vf_mc_hashes > 30) {
4135 vmolr |= E1000_VMOLR_MPME;
4136 } else if (vf_data->num_vf_mc_hashes) {
4137 int j;
4138 vmolr |= E1000_VMOLR_ROMPE;
4139 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4140 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4141 }
4142 }
4143
4144 wr32(E1000_VMOLR(vf), vmolr);
4145
4146 /* there are flags left unprocessed, likely not supported */
4147 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4148 return -EINVAL;
4149
4150 return 0;
4151
4152}
4153
4ae196df
AD		 4154static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4155 u32 *msgbuf, u32 vf)
4156{
4157 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4158 u16 *hash_list = (u16 *)&msgbuf[1];
4159 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4160 int i;
4161
7d5753f0 4162 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4163 * to this VF for later use to restore when the PF multi cast
4164 * list changes
4165 */
4166 vf_data->num_vf_mc_hashes = n;
4167
7d5753f0
AD		 4168 /* only up to 30 hash values supported */
4169 if (n > 30)
4170 n = 30;
4171
4172 /* store the hashes for later use */
4ae196df 4173 for (i = 0; i < n; i++)
a419aef8 4174 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4175
4176 /* Flush and reset the mta with the new values */
ff41f8dc 4177 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4178
4179 return 0;
4180}
4181
4182static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4183{
4184 struct e1000_hw *hw = &adapter->hw;
4185 struct vf_data_storage *vf_data;
4186 int i, j;
4187
4188 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4189 u32 vmolr = rd32(E1000_VMOLR(i));
4190 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4191
4ae196df 4192 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4193
4194 if ((vf_data->num_vf_mc_hashes > 30) ||
4195 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4196 vmolr |= E1000_VMOLR_MPME;
4197 } else if (vf_data->num_vf_mc_hashes) {
4198 vmolr |= E1000_VMOLR_ROMPE;
4199 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4200 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4201 }
4202 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4203 }
4204}
4205
4206static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4207{
4208 struct e1000_hw *hw = &adapter->hw;
4209 u32 pool_mask, reg, vid;
4210 int i;
4211
4212 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4213
4214 /* Find the vlan filter for this id */
4215 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4216 reg = rd32(E1000_VLVF(i));
4217
4218 /* remove the vf from the pool */
4219 reg &= ~pool_mask;
4220
4221 /* if pool is empty then remove entry from vfta */
4222 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4223 (reg & E1000_VLVF_VLANID_ENABLE)) {
4224 reg = 0;
4225 vid = reg & E1000_VLVF_VLANID_MASK;
4226 igb_vfta_set(hw, vid, false);
4227 }
4228
4229 wr32(E1000_VLVF(i), reg);
4230 }
ae641bdc
AD		 4231
4232 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4233}
4234
4235static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4236{
4237 struct e1000_hw *hw = &adapter->hw;
4238 u32 reg, i;
4239
51466239
AD		 4240 /* The vlvf table only exists on 82576 hardware and newer */
4241 if (hw->mac.type < e1000_82576)
4242 return -1;
4243
4244 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4245 if (!adapter->vfs_allocated_count)
4246 return -1;
4247
4248 /* Find the vlan filter for this id */
4249 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4250 reg = rd32(E1000_VLVF(i));
4251 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4252 vid == (reg & E1000_VLVF_VLANID_MASK))
4253 break;
4254 }
4255
4256 if (add) {
4257 if (i == E1000_VLVF_ARRAY_SIZE) {
4258 /* Did not find a matching VLAN ID entry that was
4259 * enabled. Search for a free filter entry, i.e.
4260 * one without the enable bit set
4261 */
4262 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4263 reg = rd32(E1000_VLVF(i));
4264 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4265 break;
4266 }
4267 }
4268 if (i < E1000_VLVF_ARRAY_SIZE) {
4269 /* Found an enabled/available entry */
4270 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4271
4272 /* if !enabled we need to set this up in vfta */
4273 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4274 /* add VID to filter table */
4275 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4276 reg |= E1000_VLVF_VLANID_ENABLE;
4277 }
cad6d05f
AD		 4278 reg &= ~E1000_VLVF_VLANID_MASK;
4279 reg |= vid;
4ae196df 4280 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4281
4282 /* do not modify RLPML for PF devices */
4283 if (vf >= adapter->vfs_allocated_count)
4284 return 0;
4285
4286 if (!adapter->vf_data[vf].vlans_enabled) {
4287 u32 size;
4288 reg = rd32(E1000_VMOLR(vf));
4289 size = reg & E1000_VMOLR_RLPML_MASK;
4290 size += 4;
4291 reg &= ~E1000_VMOLR_RLPML_MASK;
4292 reg |= size;
4293 wr32(E1000_VMOLR(vf), reg);
4294 }
ae641bdc 4295
51466239 4296 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4297 return 0;
4298 }
4299 } else {
4300 if (i < E1000_VLVF_ARRAY_SIZE) {
4301 /* remove vf from the pool */
4302 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4303 /* if pool is empty then remove entry from vfta */
4304 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4305 reg = 0;
4306 igb_vfta_set(hw, vid, false);
4307 }
4308 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4309
4310 /* do not modify RLPML for PF devices */
4311 if (vf >= adapter->vfs_allocated_count)
4312 return 0;
4313
4314 adapter->vf_data[vf].vlans_enabled--;
4315 if (!adapter->vf_data[vf].vlans_enabled) {
4316 u32 size;
4317 reg = rd32(E1000_VMOLR(vf));
4318 size = reg & E1000_VMOLR_RLPML_MASK;
4319 size -= 4;
4320 reg &= ~E1000_VMOLR_RLPML_MASK;
4321 reg |= size;
4322 wr32(E1000_VMOLR(vf), reg);
4323 }
4ae196df
AD		 4324 return 0;
4325 }
4326 }
4327 return -1;
4328}
4329
4330static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4331{
4332 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4333 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4334
4335 return igb_vlvf_set(adapter, vid, add, vf);
4336}
4337
f2ca0dbe 4338static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4339{
f2ca0dbe
AD		 4340 /* clear all flags */
4341 adapter->vf_data[vf].flags = 0;
4342 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4343
4344 /* reset offloads to defaults */
7d5753f0 4345 igb_set_vmolr(adapter, vf);
4ae196df
AD		 4346
4347 /* reset vlans for device */
4348 igb_clear_vf_vfta(adapter, vf);
4349
4350 /* reset multicast table array for vf */
4351 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4352
4353 /* Flush and reset the mta with the new values */
ff41f8dc 4354 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4355}
4356
f2ca0dbe
AD		 4357static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4358{
4359 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4360
4361 /* generate a new mac address as we were hotplug removed/added */
4362 random_ether_addr(vf_mac);
4363
4364 /* process remaining reset events */
4365 igb_vf_reset(adapter, vf);
4366}
4367
4368static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4369{
4370 struct e1000_hw *hw = &adapter->hw;
4371 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4372 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4373 u32 reg, msgbuf[3];
4374 u8 *addr = (u8 *)(&msgbuf[1]);
4375
4376 /* process all the same items cleared in a function level reset */
f2ca0dbe 4377 igb_vf_reset(adapter, vf);
4ae196df
AD		 4378
4379 /* set vf mac address */
26ad9178 4380 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4381
4382 /* enable transmit and receive for vf */
4383 reg = rd32(E1000_VFTE);
4384 wr32(E1000_VFTE, reg | (1 << vf));
4385 reg = rd32(E1000_VFRE);
4386 wr32(E1000_VFRE, reg | (1 << vf));
4387
f2ca0dbe 4388 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4389
4390 /* reply to reset with ack and vf mac address */
4391 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4392 memcpy(addr, vf_mac, 6);
4393 igb_write_mbx(hw, msgbuf, 3, vf);
4394}
4395
4396static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4397{
f2ca0dbe
AD		 4398 unsigned char *addr = (char *)&msg[1];
4399 int err = -1;
4ae196df 4400
f2ca0dbe
AD		 4401 if (is_valid_ether_addr(addr))
4402 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4403
f2ca0dbe 4404 return err;
4ae196df
AD		 4405}
4406
4407static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4408{
4409 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4410 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4411 u32 msg = E1000_VT_MSGTYPE_NACK;
4412
4413 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 4414 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4415 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 4416 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 4417 vf_data->last_nack = jiffies;
4ae196df
AD		 4418 }
4419}
4420
f2ca0dbe 4421static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 4422{
f2ca0dbe
AD		 4423 struct pci_dev *pdev = adapter->pdev;
4424 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 4425 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4426 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4427 s32 retval;
4428
f2ca0dbe 4429 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df
AD		 4430
4431 if (retval)
f2ca0dbe 4432 dev_err(&pdev->dev, "Error receiving message from VF\n");
4ae196df
AD		 4433
4434 /* this is a message we already processed, do nothing */
4435 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 4436 return;
4ae196df
AD		 4437
4438 /*
4439 * until the vf completes a reset it should not be
4440 * allowed to start any configuration.
4441 */
4442
4443 if (msgbuf[0] == E1000_VF_RESET) {
4444 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 4445 return;
4ae196df
AD		 4446 }
4447
f2ca0dbe
AD		 4448 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
4449 msgbuf[0] = E1000_VT_MSGTYPE_NACK;
4450 if (time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4451 igb_write_mbx(hw, msgbuf, 1, vf);
4452 vf_data->last_nack = jiffies;
4453 }
4454 return;
4ae196df
AD		 4455 }
4456
4457 switch ((msgbuf[0] & 0xFFFF)) {
4458 case E1000_VF_SET_MAC_ADDR:
4459 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4460 break;
7d5753f0
AD		 4461 case E1000_VF_SET_PROMISC:
4462 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4463 break;
4ae196df
AD		 4464 case E1000_VF_SET_MULTICAST:
4465 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4466 break;
4467 case E1000_VF_SET_LPE:
4468 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4469 break;
4470 case E1000_VF_SET_VLAN:
4471 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4472 break;
4473 default:
090b1795 4474 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 4475 retval = -1;
4476 break;
4477 }
4478
4479 /* notify the VF of the results of what it sent us */
4480 if (retval)
4481 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4482 else
4483 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4484
4485 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4486
4487 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 4488}
4ae196df 4489
f2ca0dbe
AD		 4490static void igb_msg_task(struct igb_adapter *adapter)
4491{
4492 struct e1000_hw *hw = &adapter->hw;
4493 u32 vf;
4494
4495 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4496 /* process any reset requests */
4497 if (!igb_check_for_rst(hw, vf))
4498 igb_vf_reset_event(adapter, vf);
4499
4500 /* process any messages pending */
4501 if (!igb_check_for_msg(hw, vf))
4502 igb_rcv_msg_from_vf(adapter, vf);
4503
4504 /* process any acks */
4505 if (!igb_check_for_ack(hw, vf))
4506 igb_rcv_ack_from_vf(adapter, vf);
4507 }
4ae196df
AD		 4508}
4509
68d480c4
AD		 4510/**
4511 * igb_set_uta - Set unicast filter table address
4512 * @adapter: board private structure
4513 *
4514 * The unicast table address is a register array of 32-bit registers.
4515 * The table is meant to be used in a way similar to how the MTA is used
4516 * however due to certain limitations in the hardware it is necessary to
4517 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4518 * enable bit to allow vlan tag stripping when promiscous mode is enabled
4519 **/
4520static void igb_set_uta(struct igb_adapter *adapter)
4521{
4522 struct e1000_hw *hw = &adapter->hw;
4523 int i;
4524
4525 /* The UTA table only exists on 82576 hardware and newer */
4526 if (hw->mac.type < e1000_82576)
4527 return;
4528
4529 /* we only need to do this if VMDq is enabled */
4530 if (!adapter->vfs_allocated_count)
4531 return;
4532
4533 for (i = 0; i < hw->mac.uta_reg_count; i++)
4534 array_wr32(E1000_UTA, i, ~0);
4535}
4536
9d5c8243
AK		 4537/**
4538 * igb_intr_msi - Interrupt Handler
4539 * @irq: interrupt number
4540 * @data: pointer to a network interface device structure
4541 **/
4542static irqreturn_t igb_intr_msi(int irq, void *data)
4543{
047e0030
AD		 4544 struct igb_adapter *adapter = data;
4545 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4546 struct e1000_hw *hw = &adapter->hw;
4547 /* read ICR disables interrupts using IAM */
4548 u32 icr = rd32(E1000_ICR);
4549
047e0030 4550 igb_write_itr(q_vector);
9d5c8243 4551
047e0030 4552 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4553 /* HW is reporting DMA is out of sync */
4554 adapter->stats.doosync++;
4555 }
4556
9d5c8243
AK		 4557 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4558 hw->mac.get_link_status = 1;
4559 if (!test_bit(__IGB_DOWN, &adapter->state))
4560 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4561 }
4562
047e0030 4563 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4564
4565 return IRQ_HANDLED;
4566}
4567
4568/**
4a3c6433 4569 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 4570 * @irq: interrupt number
4571 * @data: pointer to a network interface device structure
4572 **/
4573static irqreturn_t igb_intr(int irq, void *data)
4574{
047e0030
AD		 4575 struct igb_adapter *adapter = data;
4576 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 4577 struct e1000_hw *hw = &adapter->hw;
4578 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
4579 * need for the IMC write */
4580 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 4581 if (!icr)
4582 return IRQ_NONE; /* Not our interrupt */
4583
047e0030 4584 igb_write_itr(q_vector);
9d5c8243
AK		 4585
4586 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4587 * not set, then the adapter didn't send an interrupt */
4588 if (!(icr & E1000_ICR_INT_ASSERTED))
4589 return IRQ_NONE;
4590
047e0030 4591 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4592 /* HW is reporting DMA is out of sync */
4593 adapter->stats.doosync++;
4594 }
4595
9d5c8243
AK		 4596 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4597 hw->mac.get_link_status = 1;
4598 /* guard against interrupt when we're going down */
4599 if (!test_bit(__IGB_DOWN, &adapter->state))
4600 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4601 }
4602
047e0030 4603 napi_schedule(&q_vector->napi);
9d5c8243
AK		 4604
4605 return IRQ_HANDLED;
4606}
4607
047e0030 4608static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 4609{
047e0030 4610 struct igb_adapter *adapter = q_vector->adapter;
46544258 4611 struct e1000_hw *hw = &adapter->hw;
9d5c8243 4612
4fc82adf
AD		 4613 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4614 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 4615 if (!adapter->msix_entries)
6eb5a7f1 4616 igb_set_itr(adapter);
46544258 4617 else
047e0030 4618 igb_update_ring_itr(q_vector);
9d5c8243
AK		 4619 }
4620
46544258
AD		 4621 if (!test_bit(__IGB_DOWN, &adapter->state)) {
4622 if (adapter->msix_entries)
047e0030 4623 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 4624 else
4625 igb_irq_enable(adapter);
4626 }
9d5c8243
AK		 4627}
4628
46544258
AD		 4629/**
4630 * igb_poll - NAPI Rx polling callback
4631 * @napi: napi polling structure
4632 * @budget: count of how many packets we should handle
4633 **/
4634static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 4635{
047e0030
AD		 4636 struct igb_q_vector *q_vector = container_of(napi,
4637 struct igb_q_vector,
4638 napi);
4639 int tx_clean_complete = 1, work_done = 0;
9d5c8243 4640
421e02f0 4641#ifdef CONFIG_IGB_DCA
047e0030
AD		 4642 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4643 igb_update_dca(q_vector);
fe4506b6 4644#endif
047e0030
AD		 4645 if (q_vector->tx_ring)
4646 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 4647
047e0030
AD		 4648 if (q_vector->rx_ring)
4649 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4650
4651 if (!tx_clean_complete)
4652 work_done = budget;
46544258 4653
9d5c8243 4654 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 4655 if (work_done < budget) {
288379f0 4656 napi_complete(napi);
047e0030 4657 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 4658 }
4659
46544258 4660 return work_done;
9d5c8243 4661}
6d8126f9 4662
33af6bcc 4663/**
c5b9bd5e 4664 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 4665 * @adapter: board private structure
c5b9bd5e
AD		 4666 * @shhwtstamps: timestamp structure to update
4667 * @regval: unsigned 64bit system time value.
4668 *
4669 * We need to convert the system time value stored in the RX/TXSTMP registers
4670 * into a hwtstamp which can be used by the upper level timestamping functions
4671 */
4672static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4673 struct skb_shared_hwtstamps *shhwtstamps,
4674 u64 regval)
4675{
4676 u64 ns;
4677
4678 ns = timecounter_cyc2time(&adapter->clock, regval);
4679 timecompare_update(&adapter->compare, ns);
4680 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4681 shhwtstamps->hwtstamp = ns_to_ktime(ns);
4682 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4683}
4684
4685/**
4686 * igb_tx_hwtstamp - utility function which checks for TX time stamp
4687 * @q_vector: pointer to q_vector containing needed info
33af6bcc
PO		 4688 * @skb: packet that was just sent
4689 *
4690 * If we were asked to do hardware stamping and such a time stamp is
4691 * available, then it must have been for this skb here because we only
4692 * allow only one such packet into the queue.
4693 */
c5b9bd5e 4694static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
33af6bcc 4695{
c5b9bd5e 4696 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc
PO		 4697 union skb_shared_tx *shtx = skb_tx(skb);
4698 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 4699 struct skb_shared_hwtstamps shhwtstamps;
4700 u64 regval;
33af6bcc 4701
c5b9bd5e
AD		 4702 /* if skb does not support hw timestamp or TX stamp not valid exit */
4703 if (likely(!shtx->hardware) ||
4704 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4705 return;
4706
4707 regval = rd32(E1000_TXSTMPL);
4708 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4709
4710 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4711 skb_tstamp_tx(skb, &shhwtstamps);
33af6bcc
PO		 4712}
4713
9d5c8243
AK		 4714/**
4715 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 4716 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 4717 * returns true if ring is completely cleaned
4718 **/
047e0030 4719static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 4720{
047e0030
AD		 4721 struct igb_adapter *adapter = q_vector->adapter;
4722 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 4723 struct net_device *netdev = tx_ring->netdev;
0e014cb1 4724 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 4725 struct igb_buffer *buffer_info;
4726 struct sk_buff *skb;
0e014cb1 4727 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 4728 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 4729 unsigned int i, eop, count = 0;
4730 bool cleaned = false;
9d5c8243 4731
9d5c8243 4732 i = tx_ring->next_to_clean;
0e014cb1
AD		 4733 eop = tx_ring->buffer_info[i].next_to_watch;
4734 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4735
4736 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
4737 (count < tx_ring->count)) {
4738 for (cleaned = false; !cleaned; count++) {
4739 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 4740 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 4741 cleaned = (i == eop);
9d5c8243
AK		 4742 skb = buffer_info->skb;
4743
4744 if (skb) {
4745 unsigned int segs, bytecount;
4746 /* gso_segs is currently only valid for tcp */
4747 segs = skb_shinfo(skb)->gso_segs ?: 1;
4748 /* multiply data chunks by size of headers */
4749 bytecount = ((segs - 1) * skb_headlen(skb)) +
4750 skb->len;
4751 total_packets += segs;
4752 total_bytes += bytecount;
33af6bcc 4753
c5b9bd5e 4754 igb_tx_hwtstamp(q_vector, skb);
9d5c8243
AK		 4755 }
4756
80785298 4757 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 4758 tx_desc->wb.status = 0;
9d5c8243
AK		 4759
4760 i++;
4761 if (i == tx_ring->count)
4762 i = 0;
9d5c8243 4763 }
0e014cb1
AD		 4764 eop = tx_ring->buffer_info[i].next_to_watch;
4765 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
4766 }
4767
9d5c8243
AK		 4768 tx_ring->next_to_clean = i;
4769
fc7d345d 4770 if (unlikely(count &&
9d5c8243 4771 netif_carrier_ok(netdev) &&
c493ea45 4772 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 4773 /* Make sure that anybody stopping the queue after this
4774 * sees the new next_to_clean.
4775 */
4776 smp_mb();
661086df
PWJ		 4777 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
4778 !(test_bit(__IGB_DOWN, &adapter->state))) {
4779 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 4780 tx_ring->tx_stats.restart_queue++;
661086df 4781 }
9d5c8243
AK		 4782 }
4783
4784 if (tx_ring->detect_tx_hung) {
4785 /* Detect a transmit hang in hardware, this serializes the
4786 * check with the clearing of time_stamp and movement of i */
4787 tx_ring->detect_tx_hung = false;
4788 if (tx_ring->buffer_info[i].time_stamp &&
4789 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
4790 (adapter->tx_timeout_factor * HZ))
4791 && !(rd32(E1000_STATUS) &
4792 E1000_STATUS_TXOFF)) {
4793
9d5c8243 4794 /* detected Tx unit hang */
80785298 4795 dev_err(&tx_ring->pdev->dev,
9d5c8243 4796 "Detected Tx Unit Hang\n"
2d064c06 4797 " Tx Queue <%d>\n"
9d5c8243
AK		 4798 " TDH <%x>\n"
4799 " TDT <%x>\n"
4800 " next_to_use <%x>\n"
4801 " next_to_clean <%x>\n"
9d5c8243
AK		 4802 "buffer_info[next_to_clean]\n"
4803 " time_stamp <%lx>\n"
0e014cb1 4804 " next_to_watch <%x>\n"
9d5c8243
AK		 4805 " jiffies <%lx>\n"
4806 " desc.status <%x>\n",
2d064c06 4807 tx_ring->queue_index,
fce99e34
AD		 4808 readl(tx_ring->head),
4809 readl(tx_ring->tail),
9d5c8243
AK		 4810 tx_ring->next_to_use,
4811 tx_ring->next_to_clean,
f7ba205e 4812 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 4813 eop,
9d5c8243 4814 jiffies,
0e014cb1 4815 eop_desc->wb.status);
661086df 4816 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 4817 }
4818 }
4819 tx_ring->total_bytes += total_bytes;
4820 tx_ring->total_packets += total_packets;
e21ed353
AD		 4821 tx_ring->tx_stats.bytes += total_bytes;
4822 tx_ring->tx_stats.packets += total_packets;
0e014cb1 4823 return (count < tx_ring->count);
9d5c8243
AK		 4824}
4825
9d5c8243
AK		 4826/**
4827 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 4828 * @q_vector: structure containing interrupt and ring information
4829 * @skb: packet to send up
4830 * @vlan_tag: vlan tag for packet
9d5c8243 4831 **/
047e0030
AD		 4832static void igb_receive_skb(struct igb_q_vector *q_vector,
4833 struct sk_buff *skb,
4834 u16 vlan_tag)
4835{
4836 struct igb_adapter *adapter = q_vector->adapter;
4837
4838 if (vlan_tag)
4839 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
4840 vlan_tag, skb);
182ff8df 4841 else
047e0030 4842 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 4843}
4844
04a5fcaa 4845static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 4846 u32 status_err, struct sk_buff *skb)
4847{
4848 skb->ip_summed = CHECKSUM_NONE;
4849
4850 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 4851 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
4852 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 4853 return;
85ad76b2 4854
9d5c8243
AK		 4855 /* TCP/UDP checksum error bit is set */
4856 if (status_err &
4857 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 4858 /*
4859 * work around errata with sctp packets where the TCPE aka
4860 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
4861 * packets, (aka let the stack check the crc32c)
4862 */
85ad76b2
AD		 4863 if ((skb->len == 60) &&
4864 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 4865 ring->rx_stats.csum_err++;
85ad76b2 4866
9d5c8243 4867 /* let the stack verify checksum errors */
9d5c8243
AK		 4868 return;
4869 }
4870 /* It must be a TCP or UDP packet with a valid checksum */
4871 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
4872 skb->ip_summed = CHECKSUM_UNNECESSARY;
4873
85ad76b2 4874 dev_dbg(&ring->pdev->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 4875}
4876
c5b9bd5e
AD		 4877static inline void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
4878 struct sk_buff *skb)
4879{
4880 struct igb_adapter *adapter = q_vector->adapter;
4881 struct e1000_hw *hw = &adapter->hw;
4882 u64 regval;
4883
4884 /*
4885 * If this bit is set, then the RX registers contain the time stamp. No
4886 * other packet will be time stamped until we read these registers, so
4887 * read the registers to make them available again. Because only one
4888 * packet can be time stamped at a time, we know that the register
4889 * values must belong to this one here and therefore we don't need to
4890 * compare any of the additional attributes stored for it.
4891 *
4892 * If nothing went wrong, then it should have a skb_shared_tx that we
4893 * can turn into a skb_shared_hwtstamps.
4894 */
4895 if (likely(!(staterr & E1000_RXDADV_STAT_TS)))
4896 return;
4897 if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
4898 return;
4899
4900 regval = rd32(E1000_RXSTMPL);
4901 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
4902
4903 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
4904}
4c844851 4905static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 4906 union e1000_adv_rx_desc *rx_desc)
4907{
4908 /* HW will not DMA in data larger than the given buffer, even if it
4909 * parses the (NFS, of course) header to be larger. In that case, it
4910 * fills the header buffer and spills the rest into the page.
4911 */
4912 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
4913 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 4914 if (hlen > rx_ring->rx_buffer_len)
4915 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 4916 return hlen;
4917}
4918
047e0030
AD		 4919static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
4920 int *work_done, int budget)
9d5c8243 4921{
047e0030 4922 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 4923 struct net_device *netdev = rx_ring->netdev;
80785298 4924 struct pci_dev *pdev = rx_ring->pdev;
9d5c8243
AK		 4925 union e1000_adv_rx_desc *rx_desc , *next_rxd;
4926 struct igb_buffer *buffer_info , *next_buffer;
4927 struct sk_buff *skb;
9d5c8243
AK		 4928 bool cleaned = false;
4929 int cleaned_count = 0;
4930 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 4931 unsigned int i;
2d94d8ab
AD		 4932 u32 staterr;
4933 u16 length;
047e0030 4934 u16 vlan_tag;
9d5c8243
AK		 4935
4936 i = rx_ring->next_to_clean;
69d3ca53 4937 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 4938 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
4939 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
4940
4941 while (staterr & E1000_RXD_STAT_DD) {
4942 if (*work_done >= budget)
4943 break;
4944 (*work_done)++;
9d5c8243 4945
69d3ca53
AD		 4946 skb = buffer_info->skb;
4947 prefetch(skb->data - NET_IP_ALIGN);
4948 buffer_info->skb = NULL;
4949
4950 i++;
4951 if (i == rx_ring->count)
4952 i = 0;
42d0781a 4953
69d3ca53
AD		 4954 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
4955 prefetch(next_rxd);
4956 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 4957
4958 length = le16_to_cpu(rx_desc->wb.upper.length);
4959 cleaned = true;
4960 cleaned_count++;
4961
2d94d8ab 4962 if (buffer_info->dma) {
bf36c1a0 4963 pci_unmap_single(pdev, buffer_info->dma,
4c844851 4964 rx_ring->rx_buffer_len,
bf36c1a0 4965 PCI_DMA_FROMDEVICE);
91615f76 4966 buffer_info->dma = 0;
4c844851 4967 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 4968 skb_put(skb, length);
4969 goto send_up;
4970 }
4c844851 4971 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 4972 }
4973
4974 if (length) {
9d5c8243 4975 pci_unmap_page(pdev, buffer_info->page_dma,
bf36c1a0 4976 PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);
9d5c8243 4977 buffer_info->page_dma = 0;
bf36c1a0
AD		 4978
4979 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,
4980 buffer_info->page,
4981 buffer_info->page_offset,
4982 length);
4983
6ec43fe6 4984 if (page_count(buffer_info->page) != 1)
bf36c1a0
AD		 4985 buffer_info->page = NULL;
4986 else
4987 get_page(buffer_info->page);
9d5c8243
AK		 4988
4989 skb->len += length;
4990 skb->data_len += length;
bf36c1a0 4991 skb->truesize += length;
9d5c8243 4992 }
9d5c8243 4993
bf36c1a0 4994 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 4995 buffer_info->skb = next_buffer->skb;
4996 buffer_info->dma = next_buffer->dma;
4997 next_buffer->skb = skb;
4998 next_buffer->dma = 0;
bf36c1a0
AD		 4999 goto next_desc;
5000 }
69d3ca53 5001send_up:
9d5c8243
AK		 5002 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5003 dev_kfree_skb_irq(skb);
5004 goto next_desc;
5005 }
9d5c8243 5006
c5b9bd5e 5007 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5008 total_bytes += skb->len;
5009 total_packets++;
5010
85ad76b2 5011 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5012
5013 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5014 skb_record_rx_queue(skb, rx_ring->queue_index);
5015
5016 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5017 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5018
047e0030 5019 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5020
9d5c8243
AK		 5021next_desc:
5022 rx_desc->wb.upper.status_error = 0;
5023
5024 /* return some buffers to hardware, one at a time is too slow */
5025 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5026 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5027 cleaned_count = 0;
5028 }
5029
5030 /* use prefetched values */
5031 rx_desc = next_rxd;
5032 buffer_info = next_buffer;
9d5c8243
AK		 5033 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5034 }
bf36c1a0 5035
9d5c8243 5036 rx_ring->next_to_clean = i;
c493ea45 5037 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5038
5039 if (cleaned_count)
3b644cf6 5040 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5041
5042 rx_ring->total_packets += total_packets;
5043 rx_ring->total_bytes += total_bytes;
5044 rx_ring->rx_stats.packets += total_packets;
5045 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5046 return cleaned;
5047}
5048
9d5c8243
AK		 5049/**
5050 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5051 * @adapter: address of board private structure
5052 **/
d7ee5b3a 5053void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5054{
e694e964 5055 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5056 union e1000_adv_rx_desc *rx_desc;
5057 struct igb_buffer *buffer_info;
5058 struct sk_buff *skb;
5059 unsigned int i;
db761762 5060 int bufsz;
9d5c8243
AK		 5061
5062 i = rx_ring->next_to_use;
5063 buffer_info = &rx_ring->buffer_info[i];
5064
4c844851 5065 bufsz = rx_ring->rx_buffer_len;
db761762 5066
9d5c8243
AK		 5067 while (cleaned_count--) {
5068 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5069
6ec43fe6 5070 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5071 if (!buffer_info->page) {
42d0781a 5072 buffer_info->page = netdev_alloc_page(netdev);
bf36c1a0 5073 if (!buffer_info->page) {
04a5fcaa 5074 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5075 goto no_buffers;
5076 }
5077 buffer_info->page_offset = 0;
5078 } else {
5079 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5080 }
5081 buffer_info->page_dma =
80785298 5082 pci_map_page(rx_ring->pdev, buffer_info->page,
bf36c1a0
AD		 5083 buffer_info->page_offset,
5084 PAGE_SIZE / 2,
9d5c8243 5085 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5086 if (pci_dma_mapping_error(rx_ring->pdev,
5087 buffer_info->page_dma)) {
5088 buffer_info->page_dma = 0;
5089 rx_ring->rx_stats.alloc_failed++;
5090 goto no_buffers;
5091 }
9d5c8243
AK		 5092 }
5093
42d0781a
AD		 5094 skb = buffer_info->skb;
5095 if (!skb) {
89d71a66 5096 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5097 if (!skb) {
04a5fcaa 5098 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5099 goto no_buffers;
5100 }
5101
9d5c8243 5102 buffer_info->skb = skb;
42d0781a
AD		 5103 }
5104 if (!buffer_info->dma) {
80785298
AD		 5105 buffer_info->dma = pci_map_single(rx_ring->pdev,
5106 skb->data,
9d5c8243
AK		 5107 bufsz,
5108 PCI_DMA_FROMDEVICE);
42d0781a
AD		 5109 if (pci_dma_mapping_error(rx_ring->pdev,
5110 buffer_info->dma)) {
5111 buffer_info->dma = 0;
5112 rx_ring->rx_stats.alloc_failed++;
5113 goto no_buffers;
5114 }
9d5c8243
AK		 5115 }
5116 /* Refresh the desc even if buffer_addrs didn't change because
5117 * each write-back erases this info. */
6ec43fe6 5118 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5119 rx_desc->read.pkt_addr =
5120 cpu_to_le64(buffer_info->page_dma);
5121 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5122 } else {
42d0781a 5123 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5124 rx_desc->read.hdr_addr = 0;
5125 }
5126
5127 i++;
5128 if (i == rx_ring->count)
5129 i = 0;
5130 buffer_info = &rx_ring->buffer_info[i];
5131 }
5132
5133no_buffers:
5134 if (rx_ring->next_to_use != i) {
5135 rx_ring->next_to_use = i;
5136 if (i == 0)
5137 i = (rx_ring->count - 1);
5138 else
5139 i--;
5140
5141 /* Force memory writes to complete before letting h/w
5142 * know there are new descriptors to fetch. (Only
5143 * applicable for weak-ordered memory model archs,
5144 * such as IA-64). */
5145 wmb();
fce99e34 5146 writel(i, rx_ring->tail);
9d5c8243
AK		 5147 }
5148}
5149
5150/**
5151 * igb_mii_ioctl -
5152 * @netdev:
5153 * @ifreq:
5154 * @cmd:
5155 **/
5156static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5157{
5158 struct igb_adapter *adapter = netdev_priv(netdev);
5159 struct mii_ioctl_data *data = if_mii(ifr);
5160
5161 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5162 return -EOPNOTSUPP;
5163
5164 switch (cmd) {
5165 case SIOCGMIIPHY:
5166 data->phy_id = adapter->hw.phy.addr;
5167 break;
5168 case SIOCGMIIREG:
f5f4cf08
AD		 5169 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5170 &data->val_out))
9d5c8243
AK		 5171 return -EIO;
5172 break;
5173 case SIOCSMIIREG:
5174 default:
5175 return -EOPNOTSUPP;
5176 }
5177 return 0;
5178}
5179
c6cb090b
PO		 5180/**
5181 * igb_hwtstamp_ioctl - control hardware time stamping
5182 * @netdev:
5183 * @ifreq:
5184 * @cmd:
5185 *
33af6bcc
PO		 5186 * Outgoing time stamping can be enabled and disabled. Play nice and
5187 * disable it when requested, although it shouldn't case any overhead
5188 * when no packet needs it. At most one packet in the queue may be
5189 * marked for time stamping, otherwise it would be impossible to tell
5190 * for sure to which packet the hardware time stamp belongs.
5191 *
5192 * Incoming time stamping has to be configured via the hardware
5193 * filters. Not all combinations are supported, in particular event
5194 * type has to be specified. Matching the kind of event packet is
5195 * not supported, with the exception of "all V2 events regardless of
5196 * level 2 or 4".
5197 *
c6cb090b
PO		 5198 **/
5199static int igb_hwtstamp_ioctl(struct net_device *netdev,
5200 struct ifreq *ifr, int cmd)
5201{
33af6bcc
PO		 5202 struct igb_adapter *adapter = netdev_priv(netdev);
5203 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5204 struct hwtstamp_config config;
c5b9bd5e
AD		 5205 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5206 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5207 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5208 bool is_l4 = false;
5209 bool is_l2 = false;
33af6bcc 5210 u32 regval;
c6cb090b
PO		 5211
5212 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5213 return -EFAULT;
5214
5215 /* reserved for future extensions */
5216 if (config.flags)
5217 return -EINVAL;
5218
33af6bcc
PO		 5219 switch (config.tx_type) {
5220 case HWTSTAMP_TX_OFF:
c5b9bd5e 5221 tsync_tx_ctl = 0;
33af6bcc 5222 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5223 break;
5224 default:
5225 return -ERANGE;
5226 }
5227
5228 switch (config.rx_filter) {
5229 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5230 tsync_rx_ctl = 0;
33af6bcc
PO		 5231 break;
5232 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5233 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5234 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5235 case HWTSTAMP_FILTER_ALL:
5236 /*
5237 * register TSYNCRXCFG must be set, therefore it is not
5238 * possible to time stamp both Sync and Delay_Req messages
5239 * => fall back to time stamping all packets
5240 */
c5b9bd5e 5241 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5242 config.rx_filter = HWTSTAMP_FILTER_ALL;
5243 break;
5244 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5245 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5246 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5247 is_l4 = true;
33af6bcc
PO		 5248 break;
5249 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5250 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5251 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5252 is_l4 = true;
33af6bcc
PO		 5253 break;
5254 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5255 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5256 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5257 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5258 is_l2 = true;
5259 is_l4 = true;
33af6bcc
PO		 5260 config.rx_filter = HWTSTAMP_FILTER_SOME;
5261 break;
5262 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5263 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5264 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5265 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5266 is_l2 = true;
5267 is_l4 = true;
33af6bcc
PO		 5268 config.rx_filter = HWTSTAMP_FILTER_SOME;
5269 break;
5270 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5271 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5272 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5273 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5274 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5275 is_l2 = true;
33af6bcc
PO		 5276 break;
5277 default:
5278 return -ERANGE;
5279 }
5280
c5b9bd5e
AD		 5281 if (hw->mac.type == e1000_82575) {
5282 if (tsync_rx_ctl | tsync_tx_ctl)
5283 return -EINVAL;
5284 return 0;
5285 }
5286
33af6bcc
PO		 5287 /* enable/disable TX */
5288 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5289 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5290 regval |= tsync_tx_ctl;
33af6bcc
PO		 5291 wr32(E1000_TSYNCTXCTL, regval);
5292
c5b9bd5e 5293 /* enable/disable RX */
33af6bcc 5294 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5295 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5296 regval |= tsync_rx_ctl;
33af6bcc 5297 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5298
c5b9bd5e
AD		 5299 /* define which PTP packets are time stamped */
5300 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5301
c5b9bd5e
AD		 5302 /* define ethertype filter for timestamped packets */
5303 if (is_l2)
5304 wr32(E1000_ETQF(3),
5305 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5306 E1000_ETQF_1588 | /* enable timestamping */
5307 ETH_P_1588)); /* 1588 eth protocol type */
5308 else
5309 wr32(E1000_ETQF(3), 0);
5310
5311#define PTP_PORT 319
5312 /* L4 Queue Filter[3]: filter by destination port and protocol */
5313 if (is_l4) {
5314 u32 ftqf = (IPPROTO_UDP /* UDP */
5315 | E1000_FTQF_VF_BP /* VF not compared */
5316 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5317 | E1000_FTQF_MASK); /* mask all inputs */
5318 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5319
5320 wr32(E1000_IMIR(3), htons(PTP_PORT));
5321 wr32(E1000_IMIREXT(3),
5322 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5323 if (hw->mac.type == e1000_82576) {
5324 /* enable source port check */
5325 wr32(E1000_SPQF(3), htons(PTP_PORT));
5326 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5327 }
5328 wr32(E1000_FTQF(3), ftqf);
5329 } else {
5330 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5331 }
33af6bcc
PO		 5332 wrfl();
5333
5334 adapter->hwtstamp_config = config;
5335
5336 /* clear TX/RX time stamp registers, just to be sure */
5337 regval = rd32(E1000_TXSTMPH);
5338 regval = rd32(E1000_RXSTMPH);
c6cb090b 5339
33af6bcc
PO		 5340 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5341 -EFAULT : 0;
c6cb090b
PO		 5342}
5343
9d5c8243
AK		 5344/**
5345 * igb_ioctl -
5346 * @netdev:
5347 * @ifreq:
5348 * @cmd:
5349 **/
5350static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5351{
5352 switch (cmd) {
5353 case SIOCGMIIPHY:
5354 case SIOCGMIIREG:
5355 case SIOCSMIIREG:
5356 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5357 case SIOCSHWTSTAMP:
5358 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5359 default:
5360 return -EOPNOTSUPP;
5361 }
5362}
5363
009bc06e
AD		 5364s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5365{
5366 struct igb_adapter *adapter = hw->back;
5367 u16 cap_offset;
5368
5369 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5370 if (!cap_offset)
5371 return -E1000_ERR_CONFIG;
5372
5373 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5374
5375 return 0;
5376}
5377
5378s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5379{
5380 struct igb_adapter *adapter = hw->back;
5381 u16 cap_offset;
5382
5383 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5384 if (!cap_offset)
5385 return -E1000_ERR_CONFIG;
5386
5387 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5388
5389 return 0;
5390}
5391
9d5c8243
AK		 5392static void igb_vlan_rx_register(struct net_device *netdev,
5393 struct vlan_group *grp)
5394{
5395 struct igb_adapter *adapter = netdev_priv(netdev);
5396 struct e1000_hw *hw = &adapter->hw;
5397 u32 ctrl, rctl;
5398
5399 igb_irq_disable(adapter);
5400 adapter->vlgrp = grp;
5401
5402 if (grp) {
5403 /* enable VLAN tag insert/strip */
5404 ctrl = rd32(E1000_CTRL);
5405 ctrl |= E1000_CTRL_VME;
5406 wr32(E1000_CTRL, ctrl);
5407
51466239 5408 /* Disable CFI check */
9d5c8243 5409 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 5410 rctl &= ~E1000_RCTL_CFIEN;
5411 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 5412 } else {
5413 /* disable VLAN tag insert/strip */
5414 ctrl = rd32(E1000_CTRL);
5415 ctrl &= ~E1000_CTRL_VME;
5416 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 5417 }
5418
e1739522
AD		 5419 igb_rlpml_set(adapter);
5420
9d5c8243
AK		 5421 if (!test_bit(__IGB_DOWN, &adapter->state))
5422 igb_irq_enable(adapter);
5423}
5424
5425static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5426{
5427 struct igb_adapter *adapter = netdev_priv(netdev);
5428 struct e1000_hw *hw = &adapter->hw;
4ae196df 5429 int pf_id = adapter->vfs_allocated_count;
9d5c8243 5430
51466239
AD		 5431 /* attempt to add filter to vlvf array */
5432 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 5433
51466239
AD		 5434 /* add the filter since PF can receive vlans w/o entry in vlvf */
5435 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 5436}
5437
5438static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5439{
5440 struct igb_adapter *adapter = netdev_priv(netdev);
5441 struct e1000_hw *hw = &adapter->hw;
4ae196df 5442 int pf_id = adapter->vfs_allocated_count;
51466239 5443 s32 err;
9d5c8243
AK		 5444
5445 igb_irq_disable(adapter);
5446 vlan_group_set_device(adapter->vlgrp, vid, NULL);
5447
5448 if (!test_bit(__IGB_DOWN, &adapter->state))
5449 igb_irq_enable(adapter);
5450
51466239
AD		 5451 /* remove vlan from VLVF table array */
5452 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 5453
51466239
AD		 5454 /* if vid was not present in VLVF just remove it from table */
5455 if (err)
4ae196df 5456 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 5457}
5458
5459static void igb_restore_vlan(struct igb_adapter *adapter)
5460{
5461 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5462
5463 if (adapter->vlgrp) {
5464 u16 vid;
5465 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5466 if (!vlan_group_get_device(adapter->vlgrp, vid))
5467 continue;
5468 igb_vlan_rx_add_vid(adapter->netdev, vid);
5469 }
5470 }
5471}
5472
5473int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5474{
090b1795 5475 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 5476 struct e1000_mac_info *mac = &adapter->hw.mac;
5477
5478 mac->autoneg = 0;
5479
9d5c8243
AK		 5480 switch (spddplx) {
5481 case SPEED_10 + DUPLEX_HALF:
5482 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5483 break;
5484 case SPEED_10 + DUPLEX_FULL:
5485 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5486 break;
5487 case SPEED_100 + DUPLEX_HALF:
5488 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5489 break;
5490 case SPEED_100 + DUPLEX_FULL:
5491 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5492 break;
5493 case SPEED_1000 + DUPLEX_FULL:
5494 mac->autoneg = 1;
5495 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5496 break;
5497 case SPEED_1000 + DUPLEX_HALF: /* not supported */
5498 default:
090b1795 5499 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 5500 return -EINVAL;
5501 }
5502 return 0;
5503}
5504
3fe7c4c9 5505static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 5506{
5507 struct net_device *netdev = pci_get_drvdata(pdev);
5508 struct igb_adapter *adapter = netdev_priv(netdev);
5509 struct e1000_hw *hw = &adapter->hw;
2d064c06 5510 u32 ctrl, rctl, status;
9d5c8243
AK		 5511 u32 wufc = adapter->wol;
5512#ifdef CONFIG_PM
5513 int retval = 0;
5514#endif
5515
5516 netif_device_detach(netdev);
5517
a88f10ec
AD		 5518 if (netif_running(netdev))
5519 igb_close(netdev);
5520
047e0030 5521 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 5522
5523#ifdef CONFIG_PM
5524 retval = pci_save_state(pdev);
5525 if (retval)
5526 return retval;
5527#endif
5528
5529 status = rd32(E1000_STATUS);
5530 if (status & E1000_STATUS_LU)
5531 wufc &= ~E1000_WUFC_LNKC;
5532
5533 if (wufc) {
5534 igb_setup_rctl(adapter);
ff41f8dc 5535 igb_set_rx_mode(netdev);
9d5c8243
AK		 5536
5537 /* turn on all-multi mode if wake on multicast is enabled */
5538 if (wufc & E1000_WUFC_MC) {
5539 rctl = rd32(E1000_RCTL);
5540 rctl |= E1000_RCTL_MPE;
5541 wr32(E1000_RCTL, rctl);
5542 }
5543
5544 ctrl = rd32(E1000_CTRL);
5545 /* advertise wake from D3Cold */
5546 #define E1000_CTRL_ADVD3WUC 0x00100000
5547 /* phy power management enable */
5548 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5549 ctrl |= E1000_CTRL_ADVD3WUC;
5550 wr32(E1000_CTRL, ctrl);
5551
9d5c8243 5552 /* Allow time for pending master requests to run */
330a6d6a 5553 igb_disable_pcie_master(hw);
9d5c8243
AK		 5554
5555 wr32(E1000_WUC, E1000_WUC_PME_EN);
5556 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 5557 } else {
5558 wr32(E1000_WUC, 0);
5559 wr32(E1000_WUFC, 0);
9d5c8243
AK		 5560 }
5561
3fe7c4c9
RW		 5562 *enable_wake = wufc || adapter->en_mng_pt;
5563 if (!*enable_wake)
2fb02a26 5564 igb_shutdown_serdes_link_82575(hw);
9d5c8243
AK		 5565
5566 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5567 * would have already happened in close and is redundant. */
5568 igb_release_hw_control(adapter);
5569
5570 pci_disable_device(pdev);
5571
9d5c8243
AK		 5572 return 0;
5573}
5574
5575#ifdef CONFIG_PM
3fe7c4c9
RW		 5576static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5577{
5578 int retval;
5579 bool wake;
5580
5581 retval = __igb_shutdown(pdev, &wake);
5582 if (retval)
5583 return retval;
5584
5585 if (wake) {
5586 pci_prepare_to_sleep(pdev);
5587 } else {
5588 pci_wake_from_d3(pdev, false);
5589 pci_set_power_state(pdev, PCI_D3hot);
5590 }
5591
5592 return 0;
5593}
5594
9d5c8243
AK		 5595static int igb_resume(struct pci_dev *pdev)
5596{
5597 struct net_device *netdev = pci_get_drvdata(pdev);
5598 struct igb_adapter *adapter = netdev_priv(netdev);
5599 struct e1000_hw *hw = &adapter->hw;
5600 u32 err;
5601
5602 pci_set_power_state(pdev, PCI_D0);
5603 pci_restore_state(pdev);
42bfd33a 5604
aed5dec3 5605 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 5606 if (err) {
5607 dev_err(&pdev->dev,
5608 "igb: Cannot enable PCI device from suspend\n");
5609 return err;
5610 }
5611 pci_set_master(pdev);
5612
5613 pci_enable_wake(pdev, PCI_D3hot, 0);
5614 pci_enable_wake(pdev, PCI_D3cold, 0);
5615
047e0030 5616 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 5617 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5618 return -ENOMEM;
9d5c8243
AK		 5619 }
5620
5621 /* e1000_power_up_phy(adapter); */
5622
5623 igb_reset(adapter);
a8564f03
AD		 5624
5625 /* let the f/w know that the h/w is now under the control of the
5626 * driver. */
5627 igb_get_hw_control(adapter);
5628
9d5c8243
AK		 5629 wr32(E1000_WUS, ~0);
5630
a88f10ec
AD		 5631 if (netif_running(netdev)) {
5632 err = igb_open(netdev);
5633 if (err)
5634 return err;
5635 }
9d5c8243
AK		 5636
5637 netif_device_attach(netdev);
5638
9d5c8243
AK		 5639 return 0;
5640}
5641#endif
5642
5643static void igb_shutdown(struct pci_dev *pdev)
5644{
3fe7c4c9
RW		 5645 bool wake;
5646
5647 __igb_shutdown(pdev, &wake);
5648
5649 if (system_state == SYSTEM_POWER_OFF) {
5650 pci_wake_from_d3(pdev, wake);
5651 pci_set_power_state(pdev, PCI_D3hot);
5652 }
9d5c8243
AK		 5653}
5654
5655#ifdef CONFIG_NET_POLL_CONTROLLER
5656/*
5657 * Polling 'interrupt' - used by things like netconsole to send skbs
5658 * without having to re-enable interrupts. It's not called while
5659 * the interrupt routine is executing.
5660 */
5661static void igb_netpoll(struct net_device *netdev)
5662{
5663 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 5664 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5665 int i;
9d5c8243 5666
eebbbdba 5667 if (!adapter->msix_entries) {
047e0030 5668 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 5669 igb_irq_disable(adapter);
047e0030 5670 napi_schedule(&q_vector->napi);
eebbbdba
AD		 5671 return;
5672 }
9d5c8243 5673
047e0030
AD		 5674 for (i = 0; i < adapter->num_q_vectors; i++) {
5675 struct igb_q_vector *q_vector = adapter->q_vector[i];
5676 wr32(E1000_EIMC, q_vector->eims_value);
5677 napi_schedule(&q_vector->napi);
eebbbdba 5678 }
9d5c8243
AK		 5679}
5680#endif /* CONFIG_NET_POLL_CONTROLLER */
5681
5682/**
5683 * igb_io_error_detected - called when PCI error is detected
5684 * @pdev: Pointer to PCI device
5685 * @state: The current pci connection state
5686 *
5687 * This function is called after a PCI bus error affecting
5688 * this device has been detected.
5689 */
5690static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5691 pci_channel_state_t state)
5692{
5693 struct net_device *netdev = pci_get_drvdata(pdev);
5694 struct igb_adapter *adapter = netdev_priv(netdev);
5695
5696 netif_device_detach(netdev);
5697
59ed6eec
AD		 5698 if (state == pci_channel_io_perm_failure)
5699 return PCI_ERS_RESULT_DISCONNECT;
5700
9d5c8243
AK		 5701 if (netif_running(netdev))
5702 igb_down(adapter);
5703 pci_disable_device(pdev);
5704
5705 /* Request a slot slot reset. */
5706 return PCI_ERS_RESULT_NEED_RESET;
5707}
5708
5709/**
5710 * igb_io_slot_reset - called after the pci bus has been reset.
5711 * @pdev: Pointer to PCI device
5712 *
5713 * Restart the card from scratch, as if from a cold-boot. Implementation
5714 * resembles the first-half of the igb_resume routine.
5715 */
5716static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
5717{
5718 struct net_device *netdev = pci_get_drvdata(pdev);
5719 struct igb_adapter *adapter = netdev_priv(netdev);
5720 struct e1000_hw *hw = &adapter->hw;
40a914fa 5721 pci_ers_result_t result;
42bfd33a 5722 int err;
9d5c8243 5723
aed5dec3 5724 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 5725 dev_err(&pdev->dev,
5726 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 5727 result = PCI_ERS_RESULT_DISCONNECT;
5728 } else {
5729 pci_set_master(pdev);
5730 pci_restore_state(pdev);
9d5c8243 5731
40a914fa
AD		 5732 pci_enable_wake(pdev, PCI_D3hot, 0);
5733 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 5734
40a914fa
AD		 5735 igb_reset(adapter);
5736 wr32(E1000_WUS, ~0);
5737 result = PCI_ERS_RESULT_RECOVERED;
5738 }
9d5c8243 5739
ea943d41
JK		 5740 err = pci_cleanup_aer_uncorrect_error_status(pdev);
5741 if (err) {
5742 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
5743 "failed 0x%0x\n", err);
5744 /* non-fatal, continue */
5745 }
40a914fa
AD		 5746
5747 return result;
9d5c8243
AK		 5748}
5749
5750/**
5751 * igb_io_resume - called when traffic can start flowing again.
5752 * @pdev: Pointer to PCI device
5753 *
5754 * This callback is called when the error recovery driver tells us that
5755 * its OK to resume normal operation. Implementation resembles the
5756 * second-half of the igb_resume routine.
5757 */
5758static void igb_io_resume(struct pci_dev *pdev)
5759{
5760 struct net_device *netdev = pci_get_drvdata(pdev);
5761 struct igb_adapter *adapter = netdev_priv(netdev);
5762
9d5c8243
AK		 5763 if (netif_running(netdev)) {
5764 if (igb_up(adapter)) {
5765 dev_err(&pdev->dev, "igb_up failed after reset\n");
5766 return;
5767 }
5768 }
5769
5770 netif_device_attach(netdev);
5771
5772 /* let the f/w know that the h/w is now under the control of the
5773 * driver. */
5774 igb_get_hw_control(adapter);
9d5c8243
AK		 5775}
5776
26ad9178
AD		 5777static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
5778 u8 qsel)
5779{
5780 u32 rar_low, rar_high;
5781 struct e1000_hw *hw = &adapter->hw;
5782
5783 /* HW expects these in little endian so we reverse the byte order
5784 * from network order (big endian) to little endian
5785 */
5786 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
5787 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
5788 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
5789
5790 /* Indicate to hardware the Address is Valid. */
5791 rar_high |= E1000_RAH_AV;
5792
5793 if (hw->mac.type == e1000_82575)
5794 rar_high |= E1000_RAH_POOL_1 * qsel;
5795 else
5796 rar_high |= E1000_RAH_POOL_1 << qsel;
5797
5798 wr32(E1000_RAL(index), rar_low);
5799 wrfl();
5800 wr32(E1000_RAH(index), rar_high);
5801 wrfl();
5802}
5803
4ae196df
AD		 5804static int igb_set_vf_mac(struct igb_adapter *adapter,
5805 int vf, unsigned char *mac_addr)
5806{
5807 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 5808 /* VF MAC addresses start at end of receive addresses and moves
5809 * torwards the first, as a result a collision should not be possible */
5810 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 5811
37680117 5812 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 5813
26ad9178 5814 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 5815
5816 return 0;
5817}
5818
5819static void igb_vmm_control(struct igb_adapter *adapter)
5820{
5821 struct e1000_hw *hw = &adapter->hw;
10d8e907 5822 u32 reg;
4ae196df 5823
d4960307
AD		 5824 /* replication is not supported for 82575 */
5825 if (hw->mac.type == e1000_82575)
4ae196df
AD		 5826 return;
5827
10d8e907
AD		 5828 /* enable replication vlan tag stripping */
5829 reg = rd32(E1000_RPLOLR);
5830 reg |= E1000_RPLOLR_STRVLAN;
5831 wr32(E1000_RPLOLR, reg);
5832
5833 /* notify HW that the MAC is adding vlan tags */
5834 reg = rd32(E1000_DTXCTL);
5835 reg |= E1000_DTXCTL_VLAN_ADDED;
5836 wr32(E1000_DTXCTL, reg);
5837
d4960307
AD		 5838 if (adapter->vfs_allocated_count) {
5839 igb_vmdq_set_loopback_pf(hw, true);
5840 igb_vmdq_set_replication_pf(hw, true);
5841 } else {
5842 igb_vmdq_set_loopback_pf(hw, false);
5843 igb_vmdq_set_replication_pf(hw, false);
5844 }
4ae196df
AD		 5845}
5846
9d5c8243 5847/* igb_main.c */
Linux kernel - Deliverables
This page took 0.615616 seconds and 5 git commands to generate.