projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: add support for SGMII-based MDIO PHYs
[deliverable/linux.git] / drivers / net / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
86d5d38f 4 Copyright(c) 2007-2009 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/vmalloc.h>
32#include <linux/pagemap.h>
33#include <linux/netdevice.h>
9d5c8243 34#include <linux/ipv6.h>
5a0e3ad6 35#include <linux/slab.h>
9d5c8243
AK		 36#include <net/checksum.h>
37#include <net/ip6_checksum.h>
c6cb090b 38#include <linux/net_tstamp.h>
9d5c8243
AK		 39#include <linux/mii.h>
40#include <linux/ethtool.h>
41#include <linux/if_vlan.h>
42#include <linux/pci.h>
c54106bb 43#include <linux/pci-aspm.h>
9d5c8243
AK		 44#include <linux/delay.h>
45#include <linux/interrupt.h>
46#include <linux/if_ether.h>
40a914fa 47#include <linux/aer.h>
421e02f0 48#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 49#include <linux/dca.h>
50#endif
9d5c8243
AK		 51#include "igb.h"
52
55cac248 53#define DRV_VERSION "2.1.0-k2"
9d5c8243
AK		 54char igb_driver_name[] = "igb";
55char igb_driver_version[] = DRV_VERSION;
56static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
86d5d38f 58static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
9d5c8243 59
9d5c8243
AK		 60static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
62};
63
a3aa1884 64static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
2d064c06 74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
85 /* required last entry */
86 {0, }
87};
88
89MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
90
91void igb_reset(struct igb_adapter *);
92static int igb_setup_all_tx_resources(struct igb_adapter *);
93static int igb_setup_all_rx_resources(struct igb_adapter *);
94static void igb_free_all_tx_resources(struct igb_adapter *);
95static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 96static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 97void igb_update_stats(struct igb_adapter *);
98static int igb_probe(struct pci_dev *, const struct pci_device_id *);
99static void __devexit igb_remove(struct pci_dev *pdev);
100static int igb_sw_init(struct igb_adapter *);
101static int igb_open(struct net_device *);
102static int igb_close(struct net_device *);
103static void igb_configure_tx(struct igb_adapter *);
104static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 105static void igb_clean_all_tx_rings(struct igb_adapter *);
106static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 107static void igb_clean_tx_ring(struct igb_ring *);
108static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 109static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 110static void igb_update_phy_info(unsigned long);
111static void igb_watchdog(unsigned long);
112static void igb_watchdog_task(struct work_struct *);
b1a436c3 113static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
9d5c8243
AK		 114static struct net_device_stats *igb_get_stats(struct net_device *);
115static int igb_change_mtu(struct net_device *, int);
116static int igb_set_mac(struct net_device *, void *);
68d480c4 117static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 118static irqreturn_t igb_intr(int irq, void *);
119static irqreturn_t igb_intr_msi(int irq, void *);
120static irqreturn_t igb_msix_other(int irq, void *);
047e0030 121static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 122#ifdef CONFIG_IGB_DCA
047e0030 123static void igb_update_dca(struct igb_q_vector *);
fe4506b6 124static void igb_setup_dca(struct igb_adapter *);
421e02f0 125#endif /* CONFIG_IGB_DCA */
047e0030 126static bool igb_clean_tx_irq(struct igb_q_vector *);
661086df 127static int igb_poll(struct napi_struct *, int);
047e0030 128static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
9d5c8243
AK		 129static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
130static void igb_tx_timeout(struct net_device *);
131static void igb_reset_task(struct work_struct *);
132static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
133static void igb_vlan_rx_add_vid(struct net_device *, u16);
134static void igb_vlan_rx_kill_vid(struct net_device *, u16);
135static void igb_restore_vlan(struct igb_adapter *);
26ad9178 136static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 137static void igb_ping_all_vfs(struct igb_adapter *);
138static void igb_msg_task(struct igb_adapter *);
4ae196df 139static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 140static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 141static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 142static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
143static int igb_ndo_set_vf_vlan(struct net_device *netdev,
144 int vf, u16 vlan, u8 qos);
145static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
146static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
147 struct ifla_vf_info *ivi);
9d5c8243 148
9d5c8243 149#ifdef CONFIG_PM
3fe7c4c9 150static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 151static int igb_resume(struct pci_dev *);
152#endif
153static void igb_shutdown(struct pci_dev *);
421e02f0 154#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 155static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
156static struct notifier_block dca_notifier = {
157 .notifier_call = igb_notify_dca,
158 .next = NULL,
159 .priority = 0
160};
161#endif
9d5c8243
AK		 162#ifdef CONFIG_NET_POLL_CONTROLLER
163/* for netdump / net console */
164static void igb_netpoll(struct net_device *);
165#endif
37680117 166#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 167static unsigned int max_vfs = 0;
168module_param(max_vfs, uint, 0);
169MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
170 "per physical function");
171#endif /* CONFIG_PCI_IOV */
172
9d5c8243
AK		 173static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
174 pci_channel_state_t);
175static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
176static void igb_io_resume(struct pci_dev *);
177
178static struct pci_error_handlers igb_err_handler = {
179 .error_detected = igb_io_error_detected,
180 .slot_reset = igb_io_slot_reset,
181 .resume = igb_io_resume,
182};
183
184
185static struct pci_driver igb_driver = {
186 .name = igb_driver_name,
187 .id_table = igb_pci_tbl,
188 .probe = igb_probe,
189 .remove = __devexit_p(igb_remove),
190#ifdef CONFIG_PM
191 /* Power Managment Hooks */
192 .suspend = igb_suspend,
193 .resume = igb_resume,
194#endif
195 .shutdown = igb_shutdown,
196 .err_handler = &igb_err_handler
197};
198
199MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
200MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
201MODULE_LICENSE("GPL");
202MODULE_VERSION(DRV_VERSION);
203
c97ec42a
TI		 204struct igb_reg_info {
205 u32 ofs;
206 char *name;
207};
208
209static const struct igb_reg_info igb_reg_info_tbl[] = {
210
211 /* General Registers */
212 {E1000_CTRL, "CTRL"},
213 {E1000_STATUS, "STATUS"},
214 {E1000_CTRL_EXT, "CTRL_EXT"},
215
216 /* Interrupt Registers */
217 {E1000_ICR, "ICR"},
218
219 /* RX Registers */
220 {E1000_RCTL, "RCTL"},
221 {E1000_RDLEN(0), "RDLEN"},
222 {E1000_RDH(0), "RDH"},
223 {E1000_RDT(0), "RDT"},
224 {E1000_RXDCTL(0), "RXDCTL"},
225 {E1000_RDBAL(0), "RDBAL"},
226 {E1000_RDBAH(0), "RDBAH"},
227
228 /* TX Registers */
229 {E1000_TCTL, "TCTL"},
230 {E1000_TDBAL(0), "TDBAL"},
231 {E1000_TDBAH(0), "TDBAH"},
232 {E1000_TDLEN(0), "TDLEN"},
233 {E1000_TDH(0), "TDH"},
234 {E1000_TDT(0), "TDT"},
235 {E1000_TXDCTL(0), "TXDCTL"},
236 {E1000_TDFH, "TDFH"},
237 {E1000_TDFT, "TDFT"},
238 {E1000_TDFHS, "TDFHS"},
239 {E1000_TDFPC, "TDFPC"},
240
241 /* List Terminator */
242 {}
243};
244
245/*
246 * igb_regdump - register printout routine
247 */
248static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
249{
250 int n = 0;
251 char rname[16];
252 u32 regs[8];
253
254 switch (reginfo->ofs) {
255 case E1000_RDLEN(0):
256 for (n = 0; n < 4; n++)
257 regs[n] = rd32(E1000_RDLEN(n));
258 break;
259 case E1000_RDH(0):
260 for (n = 0; n < 4; n++)
261 regs[n] = rd32(E1000_RDH(n));
262 break;
263 case E1000_RDT(0):
264 for (n = 0; n < 4; n++)
265 regs[n] = rd32(E1000_RDT(n));
266 break;
267 case E1000_RXDCTL(0):
268 for (n = 0; n < 4; n++)
269 regs[n] = rd32(E1000_RXDCTL(n));
270 break;
271 case E1000_RDBAL(0):
272 for (n = 0; n < 4; n++)
273 regs[n] = rd32(E1000_RDBAL(n));
274 break;
275 case E1000_RDBAH(0):
276 for (n = 0; n < 4; n++)
277 regs[n] = rd32(E1000_RDBAH(n));
278 break;
279 case E1000_TDBAL(0):
280 for (n = 0; n < 4; n++)
281 regs[n] = rd32(E1000_RDBAL(n));
282 break;
283 case E1000_TDBAH(0):
284 for (n = 0; n < 4; n++)
285 regs[n] = rd32(E1000_TDBAH(n));
286 break;
287 case E1000_TDLEN(0):
288 for (n = 0; n < 4; n++)
289 regs[n] = rd32(E1000_TDLEN(n));
290 break;
291 case E1000_TDH(0):
292 for (n = 0; n < 4; n++)
293 regs[n] = rd32(E1000_TDH(n));
294 break;
295 case E1000_TDT(0):
296 for (n = 0; n < 4; n++)
297 regs[n] = rd32(E1000_TDT(n));
298 break;
299 case E1000_TXDCTL(0):
300 for (n = 0; n < 4; n++)
301 regs[n] = rd32(E1000_TXDCTL(n));
302 break;
303 default:
304 printk(KERN_INFO "%-15s %08x\n",
305 reginfo->name, rd32(reginfo->ofs));
306 return;
307 }
308
309 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
310 printk(KERN_INFO "%-15s ", rname);
311 for (n = 0; n < 4; n++)
312 printk(KERN_CONT "%08x ", regs[n]);
313 printk(KERN_CONT "\n");
314}
315
316/*
317 * igb_dump - Print registers, tx-rings and rx-rings
318 */
319static void igb_dump(struct igb_adapter *adapter)
320{
321 struct net_device *netdev = adapter->netdev;
322 struct e1000_hw *hw = &adapter->hw;
323 struct igb_reg_info *reginfo;
324 int n = 0;
325 struct igb_ring *tx_ring;
326 union e1000_adv_tx_desc *tx_desc;
327 struct my_u0 { u64 a; u64 b; } *u0;
328 struct igb_buffer *buffer_info;
329 struct igb_ring *rx_ring;
330 union e1000_adv_rx_desc *rx_desc;
331 u32 staterr;
332 int i = 0;
333
334 if (!netif_msg_hw(adapter))
335 return;
336
337 /* Print netdevice Info */
338 if (netdev) {
339 dev_info(&adapter->pdev->dev, "Net device Info\n");
340 printk(KERN_INFO "Device Name state "
341 "trans_start last_rx\n");
342 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
343 netdev->name,
344 netdev->state,
345 netdev->trans_start,
346 netdev->last_rx);
347 }
348
349 /* Print Registers */
350 dev_info(&adapter->pdev->dev, "Register Dump\n");
351 printk(KERN_INFO " Register Name Value\n");
352 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
353 reginfo->name; reginfo++) {
354 igb_regdump(hw, reginfo);
355 }
356
357 /* Print TX Ring Summary */
358 if (!netdev || !netif_running(netdev))
359 goto exit;
360
361 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
362 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
363 " leng ntw timestamp\n");
364 for (n = 0; n < adapter->num_tx_queues; n++) {
365 tx_ring = adapter->tx_ring[n];
366 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
367 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
368 n, tx_ring->next_to_use, tx_ring->next_to_clean,
369 (u64)buffer_info->dma,
370 buffer_info->length,
371 buffer_info->next_to_watch,
372 (u64)buffer_info->time_stamp);
373 }
374
375 /* Print TX Rings */
376 if (!netif_msg_tx_done(adapter))
377 goto rx_ring_summary;
378
379 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
380
381 /* Transmit Descriptor Formats
382 *
383 * Advanced Transmit Descriptor
384 * +--------------------------------------------------------------+
385 * 0 | Buffer Address [63:0] |
386 * +--------------------------------------------------------------+
387 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
388 * +--------------------------------------------------------------+
389 * 63 46 45 40 39 38 36 35 32 31 24 15 0
390 */
391
392 for (n = 0; n < adapter->num_tx_queues; n++) {
393 tx_ring = adapter->tx_ring[n];
394 printk(KERN_INFO "------------------------------------\n");
395 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
396 printk(KERN_INFO "------------------------------------\n");
397 printk(KERN_INFO "T [desc] [address 63:0 ] "
398 "[PlPOCIStDDM Ln] [bi->dma ] "
399 "leng ntw timestamp bi->skb\n");
400
401 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
402 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
403 buffer_info = &tx_ring->buffer_info[i];
404 u0 = (struct my_u0 *)tx_desc;
405 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
406 " %04X %3X %016llX %p", i,
407 le64_to_cpu(u0->a),
408 le64_to_cpu(u0->b),
409 (u64)buffer_info->dma,
410 buffer_info->length,
411 buffer_info->next_to_watch,
412 (u64)buffer_info->time_stamp,
413 buffer_info->skb);
414 if (i == tx_ring->next_to_use &&
415 i == tx_ring->next_to_clean)
416 printk(KERN_CONT " NTC/U\n");
417 else if (i == tx_ring->next_to_use)
418 printk(KERN_CONT " NTU\n");
419 else if (i == tx_ring->next_to_clean)
420 printk(KERN_CONT " NTC\n");
421 else
422 printk(KERN_CONT "\n");
423
424 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
425 print_hex_dump(KERN_INFO, "",
426 DUMP_PREFIX_ADDRESS,
427 16, 1, phys_to_virt(buffer_info->dma),
428 buffer_info->length, true);
429 }
430 }
431
432 /* Print RX Rings Summary */
433rx_ring_summary:
434 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
435 printk(KERN_INFO "Queue [NTU] [NTC]\n");
436 for (n = 0; n < adapter->num_rx_queues; n++) {
437 rx_ring = adapter->rx_ring[n];
438 printk(KERN_INFO " %5d %5X %5X\n", n,
439 rx_ring->next_to_use, rx_ring->next_to_clean);
440 }
441
442 /* Print RX Rings */
443 if (!netif_msg_rx_status(adapter))
444 goto exit;
445
446 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
447
448 /* Advanced Receive Descriptor (Read) Format
449 * 63 1 0
450 * +-----------------------------------------------------+
451 * 0 | Packet Buffer Address [63:1] |A0/NSE|
452 * +----------------------------------------------+------+
453 * 8 | Header Buffer Address [63:1] | DD |
454 * +-----------------------------------------------------+
455 *
456 *
457 * Advanced Receive Descriptor (Write-Back) Format
458 *
459 * 63 48 47 32 31 30 21 20 17 16 4 3 0
460 * +------------------------------------------------------+
461 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
462 * | Checksum Ident | | | | Type | Type |
463 * +------------------------------------------------------+
464 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
465 * +------------------------------------------------------+
466 * 63 48 47 32 31 20 19 0
467 */
468
469 for (n = 0; n < adapter->num_rx_queues; n++) {
470 rx_ring = adapter->rx_ring[n];
471 printk(KERN_INFO "------------------------------------\n");
472 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
473 printk(KERN_INFO "------------------------------------\n");
474 printk(KERN_INFO "R [desc] [ PktBuf A0] "
475 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
476 "<-- Adv Rx Read format\n");
477 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
478 "[vl er S cks ln] ---------------- [bi->skb] "
479 "<-- Adv Rx Write-Back format\n");
480
481 for (i = 0; i < rx_ring->count; i++) {
482 buffer_info = &rx_ring->buffer_info[i];
483 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
484 u0 = (struct my_u0 *)rx_desc;
485 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
486 if (staterr & E1000_RXD_STAT_DD) {
487 /* Descriptor Done */
488 printk(KERN_INFO "RWB[0x%03X] %016llX "
489 "%016llX ---------------- %p", i,
490 le64_to_cpu(u0->a),
491 le64_to_cpu(u0->b),
492 buffer_info->skb);
493 } else {
494 printk(KERN_INFO "R [0x%03X] %016llX "
495 "%016llX %016llX %p", i,
496 le64_to_cpu(u0->a),
497 le64_to_cpu(u0->b),
498 (u64)buffer_info->dma,
499 buffer_info->skb);
500
501 if (netif_msg_pktdata(adapter)) {
502 print_hex_dump(KERN_INFO, "",
503 DUMP_PREFIX_ADDRESS,
504 16, 1,
505 phys_to_virt(buffer_info->dma),
506 rx_ring->rx_buffer_len, true);
507 if (rx_ring->rx_buffer_len
508 < IGB_RXBUFFER_1024)
509 print_hex_dump(KERN_INFO, "",
510 DUMP_PREFIX_ADDRESS,
511 16, 1,
512 phys_to_virt(
513 buffer_info->page_dma +
514 buffer_info->page_offset),
515 PAGE_SIZE/2, true);
516 }
517 }
518
519 if (i == rx_ring->next_to_use)
520 printk(KERN_CONT " NTU\n");
521 else if (i == rx_ring->next_to_clean)
522 printk(KERN_CONT " NTC\n");
523 else
524 printk(KERN_CONT "\n");
525
526 }
527 }
528
529exit:
530 return;
531}
532
533
38c845c7
PO		 534/**
535 * igb_read_clock - read raw cycle counter (to be used by time counter)
536 */
537static cycle_t igb_read_clock(const struct cyclecounter *tc)
538{
539 struct igb_adapter *adapter =
540 container_of(tc, struct igb_adapter, cycles);
541 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 542 u64 stamp = 0;
543 int shift = 0;
38c845c7 544
55cac248
AD		 545 /*
546 * The timestamp latches on lowest register read. For the 82580
547 * the lowest register is SYSTIMR instead of SYSTIML. However we never
548 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
549 */
550 if (hw->mac.type == e1000_82580) {
551 stamp = rd32(E1000_SYSTIMR) >> 8;
552 shift = IGB_82580_TSYNC_SHIFT;
553 }
554
c5b9bd5e
AD		 555 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
556 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 557 return stamp;
558}
559
9d5c8243 560/**
c041076a 561 * igb_get_hw_dev - return device
9d5c8243
AK		 562 * used by hardware layer to print debugging information
563 **/
c041076a 564struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 565{
566 struct igb_adapter *adapter = hw->back;
c041076a 567 return adapter->netdev;
9d5c8243 568}
38c845c7 569
9d5c8243
AK		 570/**
571 * igb_init_module - Driver Registration Routine
572 *
573 * igb_init_module is the first routine called when the driver is
574 * loaded. All it does is register with the PCI subsystem.
575 **/
576static int __init igb_init_module(void)
577{
578 int ret;
579 printk(KERN_INFO "%s - version %s\n",
580 igb_driver_string, igb_driver_version);
581
582 printk(KERN_INFO "%s\n", igb_copyright);
583
421e02f0 584#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 585 dca_register_notify(&dca_notifier);
586#endif
bbd98fe4 587 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 588 return ret;
589}
590
591module_init(igb_init_module);
592
593/**
594 * igb_exit_module - Driver Exit Cleanup Routine
595 *
596 * igb_exit_module is called just before the driver is removed
597 * from memory.
598 **/
599static void __exit igb_exit_module(void)
600{
421e02f0 601#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 602 dca_unregister_notify(&dca_notifier);
603#endif
9d5c8243
AK		 604 pci_unregister_driver(&igb_driver);
605}
606
607module_exit(igb_exit_module);
608
26bc19ec
AD		 609#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
610/**
611 * igb_cache_ring_register - Descriptor ring to register mapping
612 * @adapter: board private structure to initialize
613 *
614 * Once we know the feature-set enabled for the device, we'll cache
615 * the register offset the descriptor ring is assigned to.
616 **/
617static void igb_cache_ring_register(struct igb_adapter *adapter)
618{
ee1b9f06 619 int i = 0, j = 0;
047e0030 620 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 621
622 switch (adapter->hw.mac.type) {
623 case e1000_82576:
624 /* The queues are allocated for virtualization such that VF 0
625 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
626 * In order to avoid collision we start at the first free queue
627 * and continue consuming queues in the same sequence
628 */
ee1b9f06 629 if (adapter->vfs_allocated_count) {
a99955fc 630 for (; i < adapter->rss_queues; i++)
3025a446
AD		 631 adapter->rx_ring[i]->reg_idx = rbase_offset +
632 Q_IDX_82576(i);
ee1b9f06 633 }
26bc19ec 634 case e1000_82575:
55cac248 635 case e1000_82580:
d2ba2ed8 636 case e1000_i350:
26bc19ec 637 default:
ee1b9f06 638 for (; i < adapter->num_rx_queues; i++)
3025a446 639 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 640 for (; j < adapter->num_tx_queues; j++)
3025a446 641 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 642 break;
643 }
644}
645
047e0030
AD		 646static void igb_free_queues(struct igb_adapter *adapter)
647{
3025a446 648 int i;
047e0030 649
3025a446
AD		 650 for (i = 0; i < adapter->num_tx_queues; i++) {
651 kfree(adapter->tx_ring[i]);
652 adapter->tx_ring[i] = NULL;
653 }
654 for (i = 0; i < adapter->num_rx_queues; i++) {
655 kfree(adapter->rx_ring[i]);
656 adapter->rx_ring[i] = NULL;
657 }
047e0030
AD		 658 adapter->num_rx_queues = 0;
659 adapter->num_tx_queues = 0;
660}
661
9d5c8243
AK		 662/**
663 * igb_alloc_queues - Allocate memory for all rings
664 * @adapter: board private structure to initialize
665 *
666 * We allocate one ring per queue at run-time since we don't know the
667 * number of queues at compile-time.
668 **/
669static int igb_alloc_queues(struct igb_adapter *adapter)
670{
3025a446 671 struct igb_ring *ring;
9d5c8243
AK		 672 int i;
673
661086df 674 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 675 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
676 if (!ring)
677 goto err;
68fd9910 678 ring->count = adapter->tx_ring_count;
661086df 679 ring->queue_index = i;
59d71989 680 ring->dev = &adapter->pdev->dev;
e694e964 681 ring->netdev = adapter->netdev;
85ad76b2
AD		 682 /* For 82575, context index must be unique per ring. */
683 if (adapter->hw.mac.type == e1000_82575)
684 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 685 adapter->tx_ring[i] = ring;
661086df 686 }
85ad76b2 687
9d5c8243 688 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446
AD		 689 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
690 if (!ring)
691 goto err;
68fd9910 692 ring->count = adapter->rx_ring_count;
844290e5 693 ring->queue_index = i;
59d71989 694 ring->dev = &adapter->pdev->dev;
e694e964 695 ring->netdev = adapter->netdev;
4c844851 696 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85ad76b2
AD		 697 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
698 /* set flag indicating ring supports SCTP checksum offload */
699 if (adapter->hw.mac.type >= e1000_82576)
700 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 701 adapter->rx_ring[i] = ring;
9d5c8243 702 }
26bc19ec
AD		 703
704 igb_cache_ring_register(adapter);
9d5c8243 705
047e0030 706 return 0;
a88f10ec 707
047e0030
AD		 708err:
709 igb_free_queues(adapter);
d1a8c9e1 710
047e0030 711 return -ENOMEM;
a88f10ec
AD		 712}
713
9d5c8243 714#define IGB_N0_QUEUE -1
047e0030 715static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 716{
717 u32 msixbm = 0;
047e0030 718 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 719 struct e1000_hw *hw = &adapter->hw;
2d064c06 720 u32 ivar, index;
047e0030
AD		 721 int rx_queue = IGB_N0_QUEUE;
722 int tx_queue = IGB_N0_QUEUE;
723
724 if (q_vector->rx_ring)
725 rx_queue = q_vector->rx_ring->reg_idx;
726 if (q_vector->tx_ring)
727 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 728
729 switch (hw->mac.type) {
730 case e1000_82575:
9d5c8243
AK		 731 /* The 82575 assigns vectors using a bitmask, which matches the
732 bitmask for the EICR/EIMS/EIMC registers. To assign one
733 or more queues to a vector, we write the appropriate bits
734 into the MSIXBM register for that vector. */
047e0030 735 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 736 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 737 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 738 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 739 if (!adapter->msix_entries && msix_vector == 0)
740 msixbm |= E1000_EIMS_OTHER;
9d5c8243 741 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 742 q_vector->eims_value = msixbm;
2d064c06
AD		 743 break;
744 case e1000_82576:
26bc19ec 745 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 746 Each queue has a single entry in the table to which we write
747 a vector number along with a "valid" bit. Sadly, the layout
748 of the table is somewhat counterintuitive. */
749 if (rx_queue > IGB_N0_QUEUE) {
047e0030 750 index = (rx_queue & 0x7);
2d064c06 751 ivar = array_rd32(E1000_IVAR0, index);
047e0030 752 if (rx_queue < 8) {
26bc19ec
AD		 753 /* vector goes into low byte of register */
754 ivar = ivar & 0xFFFFFF00;
755 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 756 } else {
757 /* vector goes into third byte of register */
758 ivar = ivar & 0xFF00FFFF;
759 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 760 }
2d064c06
AD		 761 array_wr32(E1000_IVAR0, index, ivar);
762 }
763 if (tx_queue > IGB_N0_QUEUE) {
047e0030 764 index = (tx_queue & 0x7);
2d064c06 765 ivar = array_rd32(E1000_IVAR0, index);
047e0030 766 if (tx_queue < 8) {
26bc19ec
AD		 767 /* vector goes into second byte of register */
768 ivar = ivar & 0xFFFF00FF;
769 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 770 } else {
771 /* vector goes into high byte of register */
772 ivar = ivar & 0x00FFFFFF;
773 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 774 }
2d064c06
AD		 775 array_wr32(E1000_IVAR0, index, ivar);
776 }
047e0030 777 q_vector->eims_value = 1 << msix_vector;
2d064c06 778 break;
55cac248 779 case e1000_82580:
d2ba2ed8 780 case e1000_i350:
55cac248
AD		 781 /* 82580 uses the same table-based approach as 82576 but has fewer
782 entries as a result we carry over for queues greater than 4. */
783 if (rx_queue > IGB_N0_QUEUE) {
784 index = (rx_queue >> 1);
785 ivar = array_rd32(E1000_IVAR0, index);
786 if (rx_queue & 0x1) {
787 /* vector goes into third byte of register */
788 ivar = ivar & 0xFF00FFFF;
789 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
790 } else {
791 /* vector goes into low byte of register */
792 ivar = ivar & 0xFFFFFF00;
793 ivar |= msix_vector | E1000_IVAR_VALID;
794 }
795 array_wr32(E1000_IVAR0, index, ivar);
796 }
797 if (tx_queue > IGB_N0_QUEUE) {
798 index = (tx_queue >> 1);
799 ivar = array_rd32(E1000_IVAR0, index);
800 if (tx_queue & 0x1) {
801 /* vector goes into high byte of register */
802 ivar = ivar & 0x00FFFFFF;
803 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
804 } else {
805 /* vector goes into second byte of register */
806 ivar = ivar & 0xFFFF00FF;
807 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
808 }
809 array_wr32(E1000_IVAR0, index, ivar);
810 }
811 q_vector->eims_value = 1 << msix_vector;
812 break;
2d064c06
AD		 813 default:
814 BUG();
815 break;
816 }
26b39276
AD		 817
818 /* add q_vector eims value to global eims_enable_mask */
819 adapter->eims_enable_mask |= q_vector->eims_value;
820
821 /* configure q_vector to set itr on first interrupt */
822 q_vector->set_itr = 1;
9d5c8243
AK		 823}
824
825/**
826 * igb_configure_msix - Configure MSI-X hardware
827 *
828 * igb_configure_msix sets up the hardware to properly
829 * generate MSI-X interrupts.
830 **/
831static void igb_configure_msix(struct igb_adapter *adapter)
832{
833 u32 tmp;
834 int i, vector = 0;
835 struct e1000_hw *hw = &adapter->hw;
836
837 adapter->eims_enable_mask = 0;
9d5c8243
AK		 838
839 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 840 switch (hw->mac.type) {
841 case e1000_82575:
9d5c8243
AK		 842 tmp = rd32(E1000_CTRL_EXT);
843 /* enable MSI-X PBA support*/
844 tmp |= E1000_CTRL_EXT_PBA_CLR;
845
846 /* Auto-Mask interrupts upon ICR read. */
847 tmp |= E1000_CTRL_EXT_EIAME;
848 tmp |= E1000_CTRL_EXT_IRCA;
849
850 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 851
852 /* enable msix_other interrupt */
853 array_wr32(E1000_MSIXBM(0), vector++,
854 E1000_EIMS_OTHER);
844290e5 855 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 856
2d064c06
AD		 857 break;
858
859 case e1000_82576:
55cac248 860 case e1000_82580:
d2ba2ed8 861 case e1000_i350:
047e0030
AD		 862 /* Turn on MSI-X capability first, or our settings
863 * won't stick. And it will take days to debug. */
864 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
865 E1000_GPIE_PBA | E1000_GPIE_EIAME |
866 E1000_GPIE_NSICR);
867
868 /* enable msix_other interrupt */
869 adapter->eims_other = 1 << vector;
2d064c06 870 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 871
047e0030 872 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 873 break;
874 default:
875 /* do nothing, since nothing else supports MSI-X */
876 break;
877 } /* switch (hw->mac.type) */
047e0030
AD		 878
879 adapter->eims_enable_mask |= adapter->eims_other;
880
26b39276
AD		 881 for (i = 0; i < adapter->num_q_vectors; i++)
882 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 883
9d5c8243
AK		 884 wrfl();
885}
886
887/**
888 * igb_request_msix - Initialize MSI-X interrupts
889 *
890 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
891 * kernel.
892 **/
893static int igb_request_msix(struct igb_adapter *adapter)
894{
895 struct net_device *netdev = adapter->netdev;
047e0030 896 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 897 int i, err = 0, vector = 0;
898
047e0030 899 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 900 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 901 if (err)
902 goto out;
903 vector++;
904
905 for (i = 0; i < adapter->num_q_vectors; i++) {
906 struct igb_q_vector *q_vector = adapter->q_vector[i];
907
908 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
909
910 if (q_vector->rx_ring && q_vector->tx_ring)
911 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
912 q_vector->rx_ring->queue_index);
913 else if (q_vector->tx_ring)
914 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
915 q_vector->tx_ring->queue_index);
916 else if (q_vector->rx_ring)
917 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
918 q_vector->rx_ring->queue_index);
9d5c8243 919 else
047e0030
AD		 920 sprintf(q_vector->name, "%s-unused", netdev->name);
921
9d5c8243 922 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 923 igb_msix_ring, 0, q_vector->name,
047e0030 924 q_vector);
9d5c8243
AK		 925 if (err)
926 goto out;
9d5c8243
AK		 927 vector++;
928 }
929
9d5c8243
AK		 930 igb_configure_msix(adapter);
931 return 0;
932out:
933 return err;
934}
935
936static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
937{
938 if (adapter->msix_entries) {
939 pci_disable_msix(adapter->pdev);
940 kfree(adapter->msix_entries);
941 adapter->msix_entries = NULL;
047e0030 942 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 943 pci_disable_msi(adapter->pdev);
047e0030 944 }
9d5c8243
AK		 945}
946
047e0030
AD		 947/**
948 * igb_free_q_vectors - Free memory allocated for interrupt vectors
949 * @adapter: board private structure to initialize
950 *
951 * This function frees the memory allocated to the q_vectors. In addition if
952 * NAPI is enabled it will delete any references to the NAPI struct prior
953 * to freeing the q_vector.
954 **/
955static void igb_free_q_vectors(struct igb_adapter *adapter)
956{
957 int v_idx;
958
959 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
960 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
961 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 962 if (!q_vector)
963 continue;
047e0030
AD		 964 netif_napi_del(&q_vector->napi);
965 kfree(q_vector);
966 }
967 adapter->num_q_vectors = 0;
968}
969
970/**
971 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
972 *
973 * This function resets the device so that it has 0 rx queues, tx queues, and
974 * MSI-X interrupts allocated.
975 */
976static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
977{
978 igb_free_queues(adapter);
979 igb_free_q_vectors(adapter);
980 igb_reset_interrupt_capability(adapter);
981}
9d5c8243
AK		 982
983/**
984 * igb_set_interrupt_capability - set MSI or MSI-X if supported
985 *
986 * Attempt to configure interrupts using the best available
987 * capabilities of the hardware and kernel.
988 **/
989static void igb_set_interrupt_capability(struct igb_adapter *adapter)
990{
991 int err;
992 int numvecs, i;
993
83b7180d 994 /* Number of supported queues. */
a99955fc 995 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 996 if (adapter->vfs_allocated_count)
997 adapter->num_tx_queues = 1;
998 else
999 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1000
047e0030
AD		 1001 /* start with one vector for every rx queue */
1002 numvecs = adapter->num_rx_queues;
1003
3ad2f3fb 1004 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1005 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1006 numvecs += adapter->num_tx_queues;
047e0030
AD		 1007
1008 /* store the number of vectors reserved for queues */
1009 adapter->num_q_vectors = numvecs;
1010
1011 /* add 1 vector for link status interrupts */
1012 numvecs++;
9d5c8243
AK		 1013 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1014 GFP_KERNEL);
1015 if (!adapter->msix_entries)
1016 goto msi_only;
1017
1018 for (i = 0; i < numvecs; i++)
1019 adapter->msix_entries[i].entry = i;
1020
1021 err = pci_enable_msix(adapter->pdev,
1022 adapter->msix_entries,
1023 numvecs);
1024 if (err == 0)
34a20e89 1025 goto out;
9d5c8243
AK		 1026
1027 igb_reset_interrupt_capability(adapter);
1028
1029 /* If we can't do MSI-X, try MSI */
1030msi_only:
2a3abf6d
AD		 1031#ifdef CONFIG_PCI_IOV
1032 /* disable SR-IOV for non MSI-X configurations */
1033 if (adapter->vf_data) {
1034 struct e1000_hw *hw = &adapter->hw;
1035 /* disable iov and allow time for transactions to clear */
1036 pci_disable_sriov(adapter->pdev);
1037 msleep(500);
1038
1039 kfree(adapter->vf_data);
1040 adapter->vf_data = NULL;
1041 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1042 msleep(100);
1043 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1044 }
1045#endif
4fc82adf 1046 adapter->vfs_allocated_count = 0;
a99955fc 1047 adapter->rss_queues = 1;
4fc82adf 1048 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1049 adapter->num_rx_queues = 1;
661086df 1050 adapter->num_tx_queues = 1;
047e0030 1051 adapter->num_q_vectors = 1;
9d5c8243 1052 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1053 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1054out:
661086df 1055 /* Notify the stack of the (possibly) reduced Tx Queue count. */
fd2ea0a7 1056 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
9d5c8243
AK		 1057}
1058
047e0030
AD		 1059/**
1060 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1061 * @adapter: board private structure to initialize
1062 *
1063 * We allocate one q_vector per queue interrupt. If allocation fails we
1064 * return -ENOMEM.
1065 **/
1066static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1067{
1068 struct igb_q_vector *q_vector;
1069 struct e1000_hw *hw = &adapter->hw;
1070 int v_idx;
1071
1072 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1073 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1074 if (!q_vector)
1075 goto err_out;
1076 q_vector->adapter = adapter;
047e0030
AD		 1077 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1078 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1079 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1080 adapter->q_vector[v_idx] = q_vector;
1081 }
1082 return 0;
1083
1084err_out:
fe0592b4 1085 igb_free_q_vectors(adapter);
047e0030
AD		 1086 return -ENOMEM;
1087}
1088
1089static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1090 int ring_idx, int v_idx)
1091{
3025a446 1092 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1093
3025a446 1094 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 1095 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 1096 q_vector->itr_val = adapter->rx_itr_setting;
1097 if (q_vector->itr_val && q_vector->itr_val <= 3)
1098 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1099}
1100
1101static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1102 int ring_idx, int v_idx)
1103{
3025a446 1104 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1105
3025a446 1106 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 1107 q_vector->tx_ring->q_vector = q_vector;
4fc82adf
AD		 1108 q_vector->itr_val = adapter->tx_itr_setting;
1109 if (q_vector->itr_val && q_vector->itr_val <= 3)
1110 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1111}
1112
1113/**
1114 * igb_map_ring_to_vector - maps allocated queues to vectors
1115 *
1116 * This function maps the recently allocated queues to vectors.
1117 **/
1118static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1119{
1120 int i;
1121 int v_idx = 0;
1122
1123 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1124 (adapter->num_q_vectors < adapter->num_tx_queues))
1125 return -ENOMEM;
1126
1127 if (adapter->num_q_vectors >=
1128 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1129 for (i = 0; i < adapter->num_rx_queues; i++)
1130 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1131 for (i = 0; i < adapter->num_tx_queues; i++)
1132 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1133 } else {
1134 for (i = 0; i < adapter->num_rx_queues; i++) {
1135 if (i < adapter->num_tx_queues)
1136 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1137 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1138 }
1139 for (; i < adapter->num_tx_queues; i++)
1140 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1141 }
1142 return 0;
1143}
1144
1145/**
1146 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1147 *
1148 * This function initializes the interrupts and allocates all of the queues.
1149 **/
1150static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1151{
1152 struct pci_dev *pdev = adapter->pdev;
1153 int err;
1154
1155 igb_set_interrupt_capability(adapter);
1156
1157 err = igb_alloc_q_vectors(adapter);
1158 if (err) {
1159 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1160 goto err_alloc_q_vectors;
1161 }
1162
1163 err = igb_alloc_queues(adapter);
1164 if (err) {
1165 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1166 goto err_alloc_queues;
1167 }
1168
1169 err = igb_map_ring_to_vector(adapter);
1170 if (err) {
1171 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1172 goto err_map_queues;
1173 }
1174
1175
1176 return 0;
1177err_map_queues:
1178 igb_free_queues(adapter);
1179err_alloc_queues:
1180 igb_free_q_vectors(adapter);
1181err_alloc_q_vectors:
1182 igb_reset_interrupt_capability(adapter);
1183 return err;
1184}
1185
9d5c8243
AK		 1186/**
1187 * igb_request_irq - initialize interrupts
1188 *
1189 * Attempts to configure interrupts using the best available
1190 * capabilities of the hardware and kernel.
1191 **/
1192static int igb_request_irq(struct igb_adapter *adapter)
1193{
1194 struct net_device *netdev = adapter->netdev;
047e0030 1195 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1196 int err = 0;
1197
1198 if (adapter->msix_entries) {
1199 err = igb_request_msix(adapter);
844290e5 1200 if (!err)
9d5c8243 1201 goto request_done;
9d5c8243 1202 /* fall back to MSI */
047e0030 1203 igb_clear_interrupt_scheme(adapter);
9d5c8243 1204 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1205 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1206 igb_free_all_tx_resources(adapter);
1207 igb_free_all_rx_resources(adapter);
047e0030 1208 adapter->num_tx_queues = 1;
9d5c8243 1209 adapter->num_rx_queues = 1;
047e0030
AD		 1210 adapter->num_q_vectors = 1;
1211 err = igb_alloc_q_vectors(adapter);
1212 if (err) {
1213 dev_err(&pdev->dev,
1214 "Unable to allocate memory for vectors\n");
1215 goto request_done;
1216 }
1217 err = igb_alloc_queues(adapter);
1218 if (err) {
1219 dev_err(&pdev->dev,
1220 "Unable to allocate memory for queues\n");
1221 igb_free_q_vectors(adapter);
1222 goto request_done;
1223 }
1224 igb_setup_all_tx_resources(adapter);
1225 igb_setup_all_rx_resources(adapter);
844290e5 1226 } else {
feeb2721 1227 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 1228 }
844290e5 1229
7dfc16fa 1230 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 1231 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 1232 netdev->name, adapter);
9d5c8243
AK		 1233 if (!err)
1234 goto request_done;
047e0030 1235
9d5c8243
AK		 1236 /* fall back to legacy interrupts */
1237 igb_reset_interrupt_capability(adapter);
7dfc16fa 1238 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1239 }
1240
a0607fd3 1241 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1242 netdev->name, adapter);
9d5c8243 1243
6cb5e577 1244 if (err)
9d5c8243
AK		 1245 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1246 err);
9d5c8243
AK		 1247
1248request_done:
1249 return err;
1250}
1251
1252static void igb_free_irq(struct igb_adapter *adapter)
1253{
9d5c8243
AK		 1254 if (adapter->msix_entries) {
1255 int vector = 0, i;
1256
047e0030 1257 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1258
047e0030
AD		 1259 for (i = 0; i < adapter->num_q_vectors; i++) {
1260 struct igb_q_vector *q_vector = adapter->q_vector[i];
1261 free_irq(adapter->msix_entries[vector++].vector,
1262 q_vector);
1263 }
1264 } else {
1265 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1266 }
9d5c8243
AK		 1267}
1268
1269/**
1270 * igb_irq_disable - Mask off interrupt generation on the NIC
1271 * @adapter: board private structure
1272 **/
1273static void igb_irq_disable(struct igb_adapter *adapter)
1274{
1275 struct e1000_hw *hw = &adapter->hw;
1276
25568a53
AD		 1277 /*
1278 * we need to be careful when disabling interrupts. The VFs are also
1279 * mapped into these registers and so clearing the bits can cause
1280 * issues on the VF drivers so we only need to clear what we set
1281 */
9d5c8243 1282 if (adapter->msix_entries) {
2dfd1212
AD		 1283 u32 regval = rd32(E1000_EIAM);
1284 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1285 wr32(E1000_EIMC, adapter->eims_enable_mask);
1286 regval = rd32(E1000_EIAC);
1287 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1288 }
844290e5
PW		 1289
1290 wr32(E1000_IAM, 0);
9d5c8243
AK		 1291 wr32(E1000_IMC, ~0);
1292 wrfl();
1293 synchronize_irq(adapter->pdev->irq);
1294}
1295
1296/**
1297 * igb_irq_enable - Enable default interrupt generation settings
1298 * @adapter: board private structure
1299 **/
1300static void igb_irq_enable(struct igb_adapter *adapter)
1301{
1302 struct e1000_hw *hw = &adapter->hw;
1303
1304 if (adapter->msix_entries) {
25568a53 1305 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 1306 u32 regval = rd32(E1000_EIAC);
1307 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1308 regval = rd32(E1000_EIAM);
1309 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1310 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1311 if (adapter->vfs_allocated_count) {
4ae196df 1312 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1313 ims |= E1000_IMS_VMMB;
1314 }
55cac248
AD		 1315 if (adapter->hw.mac.type == e1000_82580)
1316 ims |= E1000_IMS_DRSTA;
1317
25568a53 1318 wr32(E1000_IMS, ims);
844290e5 1319 } else {
55cac248
AD		 1320 wr32(E1000_IMS, IMS_ENABLE_MASK |
1321 E1000_IMS_DRSTA);
1322 wr32(E1000_IAM, IMS_ENABLE_MASK |
1323 E1000_IMS_DRSTA);
844290e5 1324 }
9d5c8243
AK		 1325}
1326
1327static void igb_update_mng_vlan(struct igb_adapter *adapter)
1328{
51466239 1329 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1330 u16 vid = adapter->hw.mng_cookie.vlan_id;
1331 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1332
1333 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1334 /* add VID to filter table */
1335 igb_vfta_set(hw, vid, true);
1336 adapter->mng_vlan_id = vid;
1337 } else {
1338 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1339 }
1340
1341 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1342 (vid != old_vid) &&
1343 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1344 /* remove VID from filter table */
1345 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1346 }
1347}
1348
1349/**
1350 * igb_release_hw_control - release control of the h/w to f/w
1351 * @adapter: address of board private structure
1352 *
1353 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1354 * For ASF and Pass Through versions of f/w this means that the
1355 * driver is no longer loaded.
1356 *
1357 **/
1358static void igb_release_hw_control(struct igb_adapter *adapter)
1359{
1360 struct e1000_hw *hw = &adapter->hw;
1361 u32 ctrl_ext;
1362
1363 /* Let firmware take over control of h/w */
1364 ctrl_ext = rd32(E1000_CTRL_EXT);
1365 wr32(E1000_CTRL_EXT,
1366 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1367}
1368
9d5c8243
AK		 1369/**
1370 * igb_get_hw_control - get control of the h/w from f/w
1371 * @adapter: address of board private structure
1372 *
1373 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1374 * For ASF and Pass Through versions of f/w this means that
1375 * the driver is loaded.
1376 *
1377 **/
1378static void igb_get_hw_control(struct igb_adapter *adapter)
1379{
1380 struct e1000_hw *hw = &adapter->hw;
1381 u32 ctrl_ext;
1382
1383 /* Let firmware know the driver has taken over */
1384 ctrl_ext = rd32(E1000_CTRL_EXT);
1385 wr32(E1000_CTRL_EXT,
1386 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1387}
1388
9d5c8243
AK		 1389/**
1390 * igb_configure - configure the hardware for RX and TX
1391 * @adapter: private board structure
1392 **/
1393static void igb_configure(struct igb_adapter *adapter)
1394{
1395 struct net_device *netdev = adapter->netdev;
1396 int i;
1397
1398 igb_get_hw_control(adapter);
ff41f8dc 1399 igb_set_rx_mode(netdev);
9d5c8243
AK		 1400
1401 igb_restore_vlan(adapter);
9d5c8243 1402
85b430b4 1403 igb_setup_tctl(adapter);
06cf2666 1404 igb_setup_mrqc(adapter);
9d5c8243 1405 igb_setup_rctl(adapter);
85b430b4
AD		 1406
1407 igb_configure_tx(adapter);
9d5c8243 1408 igb_configure_rx(adapter);
662d7205
AD		 1409
1410 igb_rx_fifo_flush_82575(&adapter->hw);
1411
c493ea45 1412 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1413 * at least 1 descriptor unused to make sure
1414 * next_to_use != next_to_clean */
1415 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1416 struct igb_ring *ring = adapter->rx_ring[i];
c493ea45 1417 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
9d5c8243 1418 }
9d5c8243
AK		 1419}
1420
88a268c1
NN		 1421/**
1422 * igb_power_up_link - Power up the phy/serdes link
1423 * @adapter: address of board private structure
1424 **/
1425void igb_power_up_link(struct igb_adapter *adapter)
1426{
1427 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1428 igb_power_up_phy_copper(&adapter->hw);
1429 else
1430 igb_power_up_serdes_link_82575(&adapter->hw);
1431}
1432
1433/**
1434 * igb_power_down_link - Power down the phy/serdes link
1435 * @adapter: address of board private structure
1436 */
1437static void igb_power_down_link(struct igb_adapter *adapter)
1438{
1439 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1440 igb_power_down_phy_copper_82575(&adapter->hw);
1441 else
1442 igb_shutdown_serdes_link_82575(&adapter->hw);
1443}
9d5c8243
AK		 1444
1445/**
1446 * igb_up - Open the interface and prepare it to handle traffic
1447 * @adapter: board private structure
1448 **/
9d5c8243
AK		 1449int igb_up(struct igb_adapter *adapter)
1450{
1451 struct e1000_hw *hw = &adapter->hw;
1452 int i;
1453
1454 /* hardware has been reset, we need to reload some things */
1455 igb_configure(adapter);
1456
1457 clear_bit(__IGB_DOWN, &adapter->state);
1458
047e0030
AD		 1459 for (i = 0; i < adapter->num_q_vectors; i++) {
1460 struct igb_q_vector *q_vector = adapter->q_vector[i];
1461 napi_enable(&q_vector->napi);
1462 }
844290e5 1463 if (adapter->msix_entries)
9d5c8243 1464 igb_configure_msix(adapter);
feeb2721
AD		 1465 else
1466 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1467
1468 /* Clear any pending interrupts. */
1469 rd32(E1000_ICR);
1470 igb_irq_enable(adapter);
1471
d4960307
AD		 1472 /* notify VFs that reset has been completed */
1473 if (adapter->vfs_allocated_count) {
1474 u32 reg_data = rd32(E1000_CTRL_EXT);
1475 reg_data |= E1000_CTRL_EXT_PFRSTD;
1476 wr32(E1000_CTRL_EXT, reg_data);
1477 }
1478
4cb9be7a
JB		 1479 netif_tx_start_all_queues(adapter->netdev);
1480
25568a53
AD		 1481 /* start the watchdog. */
1482 hw->mac.get_link_status = 1;
1483 schedule_work(&adapter->watchdog_task);
1484
9d5c8243
AK		 1485 return 0;
1486}
1487
1488void igb_down(struct igb_adapter *adapter)
1489{
9d5c8243 1490 struct net_device *netdev = adapter->netdev;
330a6d6a 1491 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1492 u32 tctl, rctl;
1493 int i;
1494
1495 /* signal that we're down so the interrupt handler does not
1496 * reschedule our watchdog timer */
1497 set_bit(__IGB_DOWN, &adapter->state);
1498
1499 /* disable receives in the hardware */
1500 rctl = rd32(E1000_RCTL);
1501 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1502 /* flush and sleep below */
1503
fd2ea0a7 1504 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1505
1506 /* disable transmits in the hardware */
1507 tctl = rd32(E1000_TCTL);
1508 tctl &= ~E1000_TCTL_EN;
1509 wr32(E1000_TCTL, tctl);
1510 /* flush both disables and wait for them to finish */
1511 wrfl();
1512 msleep(10);
1513
047e0030
AD		 1514 for (i = 0; i < adapter->num_q_vectors; i++) {
1515 struct igb_q_vector *q_vector = adapter->q_vector[i];
1516 napi_disable(&q_vector->napi);
1517 }
9d5c8243 1518
9d5c8243
AK		 1519 igb_irq_disable(adapter);
1520
1521 del_timer_sync(&adapter->watchdog_timer);
1522 del_timer_sync(&adapter->phy_info_timer);
1523
9d5c8243 1524 netif_carrier_off(netdev);
04fe6358
AD		 1525
1526 /* record the stats before reset*/
1527 igb_update_stats(adapter);
1528
9d5c8243
AK		 1529 adapter->link_speed = 0;
1530 adapter->link_duplex = 0;
1531
3023682e
JK		 1532 if (!pci_channel_offline(adapter->pdev))
1533 igb_reset(adapter);
9d5c8243
AK		 1534 igb_clean_all_tx_rings(adapter);
1535 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1536#ifdef CONFIG_IGB_DCA
1537
1538 /* since we reset the hardware DCA settings were cleared */
1539 igb_setup_dca(adapter);
1540#endif
9d5c8243
AK		 1541}
1542
1543void igb_reinit_locked(struct igb_adapter *adapter)
1544{
1545 WARN_ON(in_interrupt());
1546 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1547 msleep(1);
1548 igb_down(adapter);
1549 igb_up(adapter);
1550 clear_bit(__IGB_RESETTING, &adapter->state);
1551}
1552
1553void igb_reset(struct igb_adapter *adapter)
1554{
090b1795 1555 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1556 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1557 struct e1000_mac_info *mac = &hw->mac;
1558 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1559 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1560 u16 hwm;
1561
1562 /* Repartition Pba for greater than 9k mtu
1563 * To take effect CTRL.RST is required.
1564 */
fa4dfae0 1565 switch (mac->type) {
d2ba2ed8 1566 case e1000_i350:
55cac248
AD		 1567 case e1000_82580:
1568 pba = rd32(E1000_RXPBS);
1569 pba = igb_rxpbs_adjust_82580(pba);
1570 break;
fa4dfae0 1571 case e1000_82576:
d249be54
AD		 1572 pba = rd32(E1000_RXPBS);
1573 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1574 break;
1575 case e1000_82575:
1576 default:
1577 pba = E1000_PBA_34K;
1578 break;
2d064c06 1579 }
9d5c8243 1580
2d064c06
AD		 1581 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1582 (mac->type < e1000_82576)) {
9d5c8243
AK		 1583 /* adjust PBA for jumbo frames */
1584 wr32(E1000_PBA, pba);
1585
1586 /* To maintain wire speed transmits, the Tx FIFO should be
1587 * large enough to accommodate two full transmit packets,
1588 * rounded up to the next 1KB and expressed in KB. Likewise,
1589 * the Rx FIFO should be large enough to accommodate at least
1590 * one full receive packet and is similarly rounded up and
1591 * expressed in KB. */
1592 pba = rd32(E1000_PBA);
1593 /* upper 16 bits has Tx packet buffer allocation size in KB */
1594 tx_space = pba >> 16;
1595 /* lower 16 bits has Rx packet buffer allocation size in KB */
1596 pba &= 0xffff;
1597 /* the tx fifo also stores 16 bytes of information about the tx
1598 * but don't include ethernet FCS because hardware appends it */
1599 min_tx_space = (adapter->max_frame_size +
85e8d004 1600 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1601 ETH_FCS_LEN) * 2;
1602 min_tx_space = ALIGN(min_tx_space, 1024);
1603 min_tx_space >>= 10;
1604 /* software strips receive CRC, so leave room for it */
1605 min_rx_space = adapter->max_frame_size;
1606 min_rx_space = ALIGN(min_rx_space, 1024);
1607 min_rx_space >>= 10;
1608
1609 /* If current Tx allocation is less than the min Tx FIFO size,
1610 * and the min Tx FIFO size is less than the current Rx FIFO
1611 * allocation, take space away from current Rx allocation */
1612 if (tx_space < min_tx_space &&
1613 ((min_tx_space - tx_space) < pba)) {
1614 pba = pba - (min_tx_space - tx_space);
1615
1616 /* if short on rx space, rx wins and must trump tx
1617 * adjustment */
1618 if (pba < min_rx_space)
1619 pba = min_rx_space;
1620 }
2d064c06 1621 wr32(E1000_PBA, pba);
9d5c8243 1622 }
9d5c8243
AK		 1623
1624 /* flow control settings */
1625 /* The high water mark must be low enough to fit one full frame
1626 * (or the size used for early receive) above it in the Rx FIFO.
1627 * Set it to the lower of:
1628 * - 90% of the Rx FIFO size, or
1629 * - the full Rx FIFO size minus one full frame */
1630 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1631 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1632
d405ea3e
AD		 1633 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1634 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1635 fc->pause_time = 0xFFFF;
1636 fc->send_xon = 1;
0cce119a 1637 fc->current_mode = fc->requested_mode;
9d5c8243 1638
4ae196df
AD		 1639 /* disable receive for all VFs and wait one second */
1640 if (adapter->vfs_allocated_count) {
1641 int i;
1642 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
f2ca0dbe 1643 adapter->vf_data[i].flags = 0;
4ae196df
AD		 1644
1645 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1646 igb_ping_all_vfs(adapter);
4ae196df
AD		 1647
1648 /* disable transmits and receives */
1649 wr32(E1000_VFRE, 0);
1650 wr32(E1000_VFTE, 0);
1651 }
1652
9d5c8243 1653 /* Allow time for pending master requests to run */
330a6d6a 1654 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1655 wr32(E1000_WUC, 0);
1656
330a6d6a 1657 if (hw->mac.ops.init_hw(hw))
090b1795 1658 dev_err(&pdev->dev, "Hardware Error\n");
9d5c8243 1659
55cac248
AD		 1660 if (hw->mac.type == e1000_82580) {
1661 u32 reg = rd32(E1000_PCIEMISC);
1662 wr32(E1000_PCIEMISC,
1663 reg & ~E1000_PCIEMISC_LX_DECISION);
1664 }
88a268c1
NN		 1665 if (!netif_running(adapter->netdev))
1666 igb_power_down_link(adapter);
1667
9d5c8243
AK		 1668 igb_update_mng_vlan(adapter);
1669
1670 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1671 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1672
330a6d6a 1673 igb_get_phy_info(hw);
9d5c8243
AK		 1674}
1675
2e5c6922 1676static const struct net_device_ops igb_netdev_ops = {
559e9c49 1677 .ndo_open = igb_open,
2e5c6922 1678 .ndo_stop = igb_close,
00829823 1679 .ndo_start_xmit = igb_xmit_frame_adv,
2e5c6922 1680 .ndo_get_stats = igb_get_stats,
ff41f8dc
AD		 1681 .ndo_set_rx_mode = igb_set_rx_mode,
1682 .ndo_set_multicast_list = igb_set_rx_mode,
2e5c6922
SH		 1683 .ndo_set_mac_address = igb_set_mac,
1684 .ndo_change_mtu = igb_change_mtu,
1685 .ndo_do_ioctl = igb_ioctl,
1686 .ndo_tx_timeout = igb_tx_timeout,
1687 .ndo_validate_addr = eth_validate_addr,
1688 .ndo_vlan_rx_register = igb_vlan_rx_register,
1689 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1690 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1691 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1692 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1693 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1694 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1695#ifdef CONFIG_NET_POLL_CONTROLLER
1696 .ndo_poll_controller = igb_netpoll,
1697#endif
1698};
1699
9d5c8243
AK		 1700/**
1701 * igb_probe - Device Initialization Routine
1702 * @pdev: PCI device information struct
1703 * @ent: entry in igb_pci_tbl
1704 *
1705 * Returns 0 on success, negative on failure
1706 *
1707 * igb_probe initializes an adapter identified by a pci_dev structure.
1708 * The OS initialization, configuring of the adapter private structure,
1709 * and a hardware reset occur.
1710 **/
1711static int __devinit igb_probe(struct pci_dev *pdev,
1712 const struct pci_device_id *ent)
1713{
1714 struct net_device *netdev;
1715 struct igb_adapter *adapter;
1716 struct e1000_hw *hw;
4337e993
AD		 1717 u16 eeprom_data = 0;
1718 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1719 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1720 unsigned long mmio_start, mmio_len;
2d6a5e95 1721 int err, pci_using_dac;
9d5c8243
AK		 1722 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1723 u32 part_num;
1724
aed5dec3 1725 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1726 if (err)
1727 return err;
1728
1729 pci_using_dac = 0;
59d71989 1730 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1731 if (!err) {
59d71989 1732 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1733 if (!err)
1734 pci_using_dac = 1;
1735 } else {
59d71989 1736 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1737 if (err) {
59d71989 1738 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1739 if (err) {
1740 dev_err(&pdev->dev, "No usable DMA "
1741 "configuration, aborting\n");
1742 goto err_dma;
1743 }
1744 }
1745 }
1746
aed5dec3
AD		 1747 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1748 IORESOURCE_MEM),
1749 igb_driver_name);
9d5c8243
AK		 1750 if (err)
1751 goto err_pci_reg;
1752
19d5afd4 1753 pci_enable_pcie_error_reporting(pdev);
40a914fa 1754
9d5c8243 1755 pci_set_master(pdev);
c682fc23 1756 pci_save_state(pdev);
9d5c8243
AK		 1757
1758 err = -ENOMEM;
1bfaf07b
AD		 1759 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1760 IGB_ABS_MAX_TX_QUEUES);
9d5c8243
AK		 1761 if (!netdev)
1762 goto err_alloc_etherdev;
1763
1764 SET_NETDEV_DEV(netdev, &pdev->dev);
1765
1766 pci_set_drvdata(pdev, netdev);
1767 adapter = netdev_priv(netdev);
1768 adapter->netdev = netdev;
1769 adapter->pdev = pdev;
1770 hw = &adapter->hw;
1771 hw->back = adapter;
1772 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1773
1774 mmio_start = pci_resource_start(pdev, 0);
1775 mmio_len = pci_resource_len(pdev, 0);
1776
1777 err = -EIO;
28b0759c
AD		 1778 hw->hw_addr = ioremap(mmio_start, mmio_len);
1779 if (!hw->hw_addr)
9d5c8243
AK		 1780 goto err_ioremap;
1781
2e5c6922 1782 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1783 igb_set_ethtool_ops(netdev);
9d5c8243 1784 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1785
1786 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1787
1788 netdev->mem_start = mmio_start;
1789 netdev->mem_end = mmio_start + mmio_len;
1790
9d5c8243
AK		 1791 /* PCI config space info */
1792 hw->vendor_id = pdev->vendor;
1793 hw->device_id = pdev->device;
1794 hw->revision_id = pdev->revision;
1795 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1796 hw->subsystem_device_id = pdev->subsystem_device;
1797
9d5c8243
AK		 1798 /* Copy the default MAC, PHY and NVM function pointers */
1799 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1800 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1801 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1802 /* Initialize skew-specific constants */
1803 err = ei->get_invariants(hw);
1804 if (err)
450c87c8 1805 goto err_sw_init;
9d5c8243 1806
450c87c8 1807 /* setup the private structure */
9d5c8243
AK		 1808 err = igb_sw_init(adapter);
1809 if (err)
1810 goto err_sw_init;
1811
1812 igb_get_bus_info_pcie(hw);
1813
1814 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1815
1816 /* Copper options */
1817 if (hw->phy.media_type == e1000_media_type_copper) {
1818 hw->phy.mdix = AUTO_ALL_MODES;
1819 hw->phy.disable_polarity_correction = false;
1820 hw->phy.ms_type = e1000_ms_hw_default;
1821 }
1822
1823 if (igb_check_reset_block(hw))
1824 dev_info(&pdev->dev,
1825 "PHY reset is blocked due to SOL/IDER session.\n");
1826
1827 netdev->features = NETIF_F_SG |
7d8eb29e 1828 NETIF_F_IP_CSUM |
9d5c8243
AK		 1829 NETIF_F_HW_VLAN_TX |
1830 NETIF_F_HW_VLAN_RX |
1831 NETIF_F_HW_VLAN_FILTER;
1832
7d8eb29e 1833 netdev->features |= NETIF_F_IPV6_CSUM;
9d5c8243 1834 netdev->features |= NETIF_F_TSO;
9d5c8243 1835 netdev->features |= NETIF_F_TSO6;
5c0999b7 1836 netdev->features |= NETIF_F_GRO;
d3352520 1837
48f29ffc
JK		 1838 netdev->vlan_features |= NETIF_F_TSO;
1839 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 1840 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 1841 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 1842 netdev->vlan_features |= NETIF_F_SG;
1843
9d5c8243
AK		 1844 if (pci_using_dac)
1845 netdev->features |= NETIF_F_HIGHDMA;
1846
5b043fb0 1847 if (hw->mac.type >= e1000_82576)
b9473560
JB		 1848 netdev->features |= NETIF_F_SCTP_CSUM;
1849
330a6d6a 1850 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1851
1852 /* before reading the NVM, reset the controller to put the device in a
1853 * known good starting state */
1854 hw->mac.ops.reset_hw(hw);
1855
1856 /* make sure the NVM is good */
1857 if (igb_validate_nvm_checksum(hw) < 0) {
1858 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1859 err = -EIO;
1860 goto err_eeprom;
1861 }
1862
1863 /* copy the MAC address out of the NVM */
1864 if (hw->mac.ops.read_mac_addr(hw))
1865 dev_err(&pdev->dev, "NVM Read Error\n");
1866
1867 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1868 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1869
1870 if (!is_valid_ether_addr(netdev->perm_addr)) {
1871 dev_err(&pdev->dev, "Invalid MAC Address\n");
1872 err = -EIO;
1873 goto err_eeprom;
1874 }
1875
0e340485
AD		 1876 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1877 (unsigned long) adapter);
1878 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1879 (unsigned long) adapter);
9d5c8243
AK		 1880
1881 INIT_WORK(&adapter->reset_task, igb_reset_task);
1882 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1883
450c87c8 1884 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 1885 adapter->fc_autoneg = true;
1886 hw->mac.autoneg = true;
1887 hw->phy.autoneg_advertised = 0x2f;
1888
0cce119a
AD		 1889 hw->fc.requested_mode = e1000_fc_default;
1890 hw->fc.current_mode = e1000_fc_default;
9d5c8243 1891
9d5c8243
AK		 1892 igb_validate_mdi_setting(hw);
1893
9d5c8243
AK		 1894 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1895 * enable the ACPI Magic Packet filter
1896 */
1897
a2cf8b6c 1898 if (hw->bus.func == 0)
312c75ae 1899 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
55cac248
AD		 1900 else if (hw->mac.type == e1000_82580)
1901 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1902 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1903 &eeprom_data);
a2cf8b6c
AD		 1904 else if (hw->bus.func == 1)
1905 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 1906
1907 if (eeprom_data & eeprom_apme_mask)
1908 adapter->eeprom_wol |= E1000_WUFC_MAG;
1909
1910 /* now that we have the eeprom settings, apply the special cases where
1911 * the eeprom may be wrong or the board simply won't support wake on
1912 * lan on a particular port */
1913 switch (pdev->device) {
1914 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1915 adapter->eeprom_wol = 0;
1916 break;
1917 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 1918 case E1000_DEV_ID_82576_FIBER:
1919 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 1920 /* Wake events only supported on port A for dual fiber
1921 * regardless of eeprom setting */
1922 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1923 adapter->eeprom_wol = 0;
1924 break;
c8ea5ea9 1925 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 1926 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 1927 /* if quad port adapter, disable WoL on all but port A */
1928 if (global_quad_port_a != 0)
1929 adapter->eeprom_wol = 0;
1930 else
1931 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1932 /* Reset for multiple quad port adapters */
1933 if (++global_quad_port_a == 4)
1934 global_quad_port_a = 0;
1935 break;
9d5c8243
AK		 1936 }
1937
1938 /* initialize the wol settings based on the eeprom settings */
1939 adapter->wol = adapter->eeprom_wol;
e1b86d84 1940 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 1941
1942 /* reset the hardware with the new settings */
1943 igb_reset(adapter);
1944
1945 /* let the f/w know that the h/w is now under the control of the
1946 * driver. */
1947 igb_get_hw_control(adapter);
1948
9d5c8243
AK		 1949 strcpy(netdev->name, "eth%d");
1950 err = register_netdev(netdev);
1951 if (err)
1952 goto err_register;
1953
b168dfc5
JB		 1954 /* carrier off reporting is important to ethtool even BEFORE open */
1955 netif_carrier_off(netdev);
1956
421e02f0 1957#ifdef CONFIG_IGB_DCA
bbd98fe4 1958 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 1959 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 1960 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 1961 igb_setup_dca(adapter);
1962 }
fe4506b6 1963
38c845c7 1964#endif
9d5c8243
AK		 1965 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1966 /* print bus type/speed/width info */
7c510e4b 1967 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 1968 netdev->name,
559e9c49 1969 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 1970 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 1971 "unknown"),
59c3de89
AD		 1972 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1973 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1974 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1975 "unknown"),
7c510e4b 1976 netdev->dev_addr);
9d5c8243
AK		 1977
1978 igb_read_part_num(hw, &part_num);
1979 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1980 (part_num >> 8), (part_num & 0xff));
1981
1982 dev_info(&pdev->dev,
1983 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1984 adapter->msix_entries ? "MSI-X" :
7dfc16fa 1985 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243
AK		 1986 adapter->num_rx_queues, adapter->num_tx_queues);
1987
9d5c8243
AK		 1988 return 0;
1989
1990err_register:
1991 igb_release_hw_control(adapter);
1992err_eeprom:
1993 if (!igb_check_reset_block(hw))
f5f4cf08 1994 igb_reset_phy(hw);
9d5c8243
AK		 1995
1996 if (hw->flash_address)
1997 iounmap(hw->flash_address);
9d5c8243 1998err_sw_init:
047e0030 1999 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2000 iounmap(hw->hw_addr);
2001err_ioremap:
2002 free_netdev(netdev);
2003err_alloc_etherdev:
559e9c49
AD		 2004 pci_release_selected_regions(pdev,
2005 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2006err_pci_reg:
2007err_dma:
2008 pci_disable_device(pdev);
2009 return err;
2010}
2011
2012/**
2013 * igb_remove - Device Removal Routine
2014 * @pdev: PCI device information struct
2015 *
2016 * igb_remove is called by the PCI subsystem to alert the driver
2017 * that it should release a PCI device. The could be caused by a
2018 * Hot-Plug event, or because the driver is going to be removed from
2019 * memory.
2020 **/
2021static void __devexit igb_remove(struct pci_dev *pdev)
2022{
2023 struct net_device *netdev = pci_get_drvdata(pdev);
2024 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2025 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 2026
2027 /* flush_scheduled work may reschedule our watchdog task, so
2028 * explicitly disable watchdog tasks from being rescheduled */
2029 set_bit(__IGB_DOWN, &adapter->state);
2030 del_timer_sync(&adapter->watchdog_timer);
2031 del_timer_sync(&adapter->phy_info_timer);
2032
2033 flush_scheduled_work();
2034
421e02f0 2035#ifdef CONFIG_IGB_DCA
7dfc16fa 2036 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2037 dev_info(&pdev->dev, "DCA disabled\n");
2038 dca_remove_requester(&pdev->dev);
7dfc16fa 2039 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2040 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2041 }
2042#endif
2043
9d5c8243
AK		 2044 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2045 * would have already happened in close and is redundant. */
2046 igb_release_hw_control(adapter);
2047
2048 unregister_netdev(netdev);
2049
047e0030 2050 igb_clear_interrupt_scheme(adapter);
9d5c8243 2051
37680117
AD		 2052#ifdef CONFIG_PCI_IOV
2053 /* reclaim resources allocated to VFs */
2054 if (adapter->vf_data) {
2055 /* disable iov and allow time for transactions to clear */
2056 pci_disable_sriov(pdev);
2057 msleep(500);
2058
2059 kfree(adapter->vf_data);
2060 adapter->vf_data = NULL;
2061 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2062 msleep(100);
2063 dev_info(&pdev->dev, "IOV Disabled\n");
2064 }
2065#endif
559e9c49 2066
28b0759c
AD		 2067 iounmap(hw->hw_addr);
2068 if (hw->flash_address)
2069 iounmap(hw->flash_address);
559e9c49
AD		 2070 pci_release_selected_regions(pdev,
2071 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2072
2073 free_netdev(netdev);
2074
19d5afd4 2075 pci_disable_pcie_error_reporting(pdev);
40a914fa 2076
9d5c8243
AK		 2077 pci_disable_device(pdev);
2078}
2079
a6b623e0
AD		 2080/**
2081 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2082 * @adapter: board private structure to initialize
2083 *
2084 * This function initializes the vf specific data storage and then attempts to
2085 * allocate the VFs. The reason for ordering it this way is because it is much
2086 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2087 * the memory for the VFs.
2088 **/
2089static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2090{
2091#ifdef CONFIG_PCI_IOV
2092 struct pci_dev *pdev = adapter->pdev;
2093
a6b623e0
AD		 2094 if (adapter->vfs_allocated_count) {
2095 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2096 sizeof(struct vf_data_storage),
2097 GFP_KERNEL);
2098 /* if allocation failed then we do not support SR-IOV */
2099 if (!adapter->vf_data) {
2100 adapter->vfs_allocated_count = 0;
2101 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2102 "Data Storage\n");
2103 }
2104 }
2105
2106 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2107 kfree(adapter->vf_data);
2108 adapter->vf_data = NULL;
2109#endif /* CONFIG_PCI_IOV */
2110 adapter->vfs_allocated_count = 0;
2111#ifdef CONFIG_PCI_IOV
2112 } else {
2113 unsigned char mac_addr[ETH_ALEN];
2114 int i;
2115 dev_info(&pdev->dev, "%d vfs allocated\n",
2116 adapter->vfs_allocated_count);
2117 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2118 random_ether_addr(mac_addr);
2119 igb_set_vf_mac(adapter, i, mac_addr);
2120 }
2121 }
2122#endif /* CONFIG_PCI_IOV */
2123}
2124
115f459a
AD		 2125
2126/**
2127 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2128 * @adapter: board private structure to initialize
2129 *
2130 * igb_init_hw_timer initializes the function pointer and values for the hw
2131 * timer found in hardware.
2132 **/
2133static void igb_init_hw_timer(struct igb_adapter *adapter)
2134{
2135 struct e1000_hw *hw = &adapter->hw;
2136
2137 switch (hw->mac.type) {
d2ba2ed8 2138 case e1000_i350:
55cac248
AD		 2139 case e1000_82580:
2140 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2141 adapter->cycles.read = igb_read_clock;
2142 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2143 adapter->cycles.mult = 1;
2144 /*
2145 * The 82580 timesync updates the system timer every 8ns by 8ns
2146 * and the value cannot be shifted. Instead we need to shift
2147 * the registers to generate a 64bit timer value. As a result
2148 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2149 * 24 in order to generate a larger value for synchronization.
2150 */
2151 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2152 /* disable system timer temporarily by setting bit 31 */
2153 wr32(E1000_TSAUXC, 0x80000000);
2154 wrfl();
2155
2156 /* Set registers so that rollover occurs soon to test this. */
2157 wr32(E1000_SYSTIMR, 0x00000000);
2158 wr32(E1000_SYSTIML, 0x80000000);
2159 wr32(E1000_SYSTIMH, 0x000000FF);
2160 wrfl();
2161
2162 /* enable system timer by clearing bit 31 */
2163 wr32(E1000_TSAUXC, 0x0);
2164 wrfl();
2165
2166 timecounter_init(&adapter->clock,
2167 &adapter->cycles,
2168 ktime_to_ns(ktime_get_real()));
2169 /*
2170 * Synchronize our NIC clock against system wall clock. NIC
2171 * time stamp reading requires ~3us per sample, each sample
2172 * was pretty stable even under load => only require 10
2173 * samples for each offset comparison.
2174 */
2175 memset(&adapter->compare, 0, sizeof(adapter->compare));
2176 adapter->compare.source = &adapter->clock;
2177 adapter->compare.target = ktime_get_real;
2178 adapter->compare.num_samples = 10;
2179 timecompare_update(&adapter->compare, 0);
2180 break;
115f459a
AD		 2181 case e1000_82576:
2182 /*
2183 * Initialize hardware timer: we keep it running just in case
2184 * that some program needs it later on.
2185 */
2186 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2187 adapter->cycles.read = igb_read_clock;
2188 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2189 adapter->cycles.mult = 1;
2190 /**
2191 * Scale the NIC clock cycle by a large factor so that
2192 * relatively small clock corrections can be added or
2193 * substracted at each clock tick. The drawbacks of a large
2194 * factor are a) that the clock register overflows more quickly
2195 * (not such a big deal) and b) that the increment per tick has
2196 * to fit into 24 bits. As a result we need to use a shift of
2197 * 19 so we can fit a value of 16 into the TIMINCA register.
2198 */
2199 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2200 wr32(E1000_TIMINCA,
2201 (1 << E1000_TIMINCA_16NS_SHIFT) |
2202 (16 << IGB_82576_TSYNC_SHIFT));
2203
2204 /* Set registers so that rollover occurs soon to test this. */
2205 wr32(E1000_SYSTIML, 0x00000000);
2206 wr32(E1000_SYSTIMH, 0xFF800000);
2207 wrfl();
2208
2209 timecounter_init(&adapter->clock,
2210 &adapter->cycles,
2211 ktime_to_ns(ktime_get_real()));
2212 /*
2213 * Synchronize our NIC clock against system wall clock. NIC
2214 * time stamp reading requires ~3us per sample, each sample
2215 * was pretty stable even under load => only require 10
2216 * samples for each offset comparison.
2217 */
2218 memset(&adapter->compare, 0, sizeof(adapter->compare));
2219 adapter->compare.source = &adapter->clock;
2220 adapter->compare.target = ktime_get_real;
2221 adapter->compare.num_samples = 10;
2222 timecompare_update(&adapter->compare, 0);
2223 break;
2224 case e1000_82575:
2225 /* 82575 does not support timesync */
2226 default:
2227 break;
2228 }
2229
2230}
2231
9d5c8243
AK		 2232/**
2233 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2234 * @adapter: board private structure to initialize
2235 *
2236 * igb_sw_init initializes the Adapter private data structure.
2237 * Fields are initialized based on PCI device information and
2238 * OS network device settings (MTU size).
2239 **/
2240static int __devinit igb_sw_init(struct igb_adapter *adapter)
2241{
2242 struct e1000_hw *hw = &adapter->hw;
2243 struct net_device *netdev = adapter->netdev;
2244 struct pci_dev *pdev = adapter->pdev;
2245
2246 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2247
68fd9910
AD		 2248 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2249 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4fc82adf
AD		 2250 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2251 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2252
9d5c8243
AK		 2253 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2254 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2255
a6b623e0
AD		 2256#ifdef CONFIG_PCI_IOV
2257 if (hw->mac.type == e1000_82576)
c0f2276f 2258 adapter->vfs_allocated_count = (max_vfs > 7) ? 7 : max_vfs;
a6b623e0
AD		 2259
2260#endif /* CONFIG_PCI_IOV */
a99955fc
AD		 2261 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2262
2263 /*
2264 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2265 * then we should combine the queues into a queue pair in order to
2266 * conserve interrupts due to limited supply
2267 */
2268 if ((adapter->rss_queues > 4) ||
2269 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2270 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2271
a6b623e0 2272 /* This call may decrease the number of queues */
047e0030 2273 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2274 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2275 return -ENOMEM;
2276 }
2277
115f459a 2278 igb_init_hw_timer(adapter);
a6b623e0
AD		 2279 igb_probe_vfs(adapter);
2280
9d5c8243
AK		 2281 /* Explicitly disable IRQ since the NIC can be in any state. */
2282 igb_irq_disable(adapter);
2283
2284 set_bit(__IGB_DOWN, &adapter->state);
2285 return 0;
2286}
2287
2288/**
2289 * igb_open - Called when a network interface is made active
2290 * @netdev: network interface device structure
2291 *
2292 * Returns 0 on success, negative value on failure
2293 *
2294 * The open entry point is called when a network interface is made
2295 * active by the system (IFF_UP). At this point all resources needed
2296 * for transmit and receive operations are allocated, the interrupt
2297 * handler is registered with the OS, the watchdog timer is started,
2298 * and the stack is notified that the interface is ready.
2299 **/
2300static int igb_open(struct net_device *netdev)
2301{
2302 struct igb_adapter *adapter = netdev_priv(netdev);
2303 struct e1000_hw *hw = &adapter->hw;
2304 int err;
2305 int i;
2306
2307 /* disallow open during test */
2308 if (test_bit(__IGB_TESTING, &adapter->state))
2309 return -EBUSY;
2310
b168dfc5
JB		 2311 netif_carrier_off(netdev);
2312
9d5c8243
AK		 2313 /* allocate transmit descriptors */
2314 err = igb_setup_all_tx_resources(adapter);
2315 if (err)
2316 goto err_setup_tx;
2317
2318 /* allocate receive descriptors */
2319 err = igb_setup_all_rx_resources(adapter);
2320 if (err)
2321 goto err_setup_rx;
2322
88a268c1 2323 igb_power_up_link(adapter);
9d5c8243 2324
9d5c8243
AK		 2325 /* before we allocate an interrupt, we must be ready to handle it.
2326 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2327 * as soon as we call pci_request_irq, so we have to setup our
2328 * clean_rx handler before we do so. */
2329 igb_configure(adapter);
2330
2331 err = igb_request_irq(adapter);
2332 if (err)
2333 goto err_req_irq;
2334
2335 /* From here on the code is the same as igb_up() */
2336 clear_bit(__IGB_DOWN, &adapter->state);
2337
047e0030
AD		 2338 for (i = 0; i < adapter->num_q_vectors; i++) {
2339 struct igb_q_vector *q_vector = adapter->q_vector[i];
2340 napi_enable(&q_vector->napi);
2341 }
9d5c8243
AK		 2342
2343 /* Clear any pending interrupts. */
2344 rd32(E1000_ICR);
844290e5
PW		 2345
2346 igb_irq_enable(adapter);
2347
d4960307
AD		 2348 /* notify VFs that reset has been completed */
2349 if (adapter->vfs_allocated_count) {
2350 u32 reg_data = rd32(E1000_CTRL_EXT);
2351 reg_data |= E1000_CTRL_EXT_PFRSTD;
2352 wr32(E1000_CTRL_EXT, reg_data);
2353 }
2354
d55b53ff
JK		 2355 netif_tx_start_all_queues(netdev);
2356
25568a53
AD		 2357 /* start the watchdog. */
2358 hw->mac.get_link_status = 1;
2359 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2360
2361 return 0;
2362
2363err_req_irq:
2364 igb_release_hw_control(adapter);
88a268c1 2365 igb_power_down_link(adapter);
9d5c8243
AK		 2366 igb_free_all_rx_resources(adapter);
2367err_setup_rx:
2368 igb_free_all_tx_resources(adapter);
2369err_setup_tx:
2370 igb_reset(adapter);
2371
2372 return err;
2373}
2374
2375/**
2376 * igb_close - Disables a network interface
2377 * @netdev: network interface device structure
2378 *
2379 * Returns 0, this is not allowed to fail
2380 *
2381 * The close entry point is called when an interface is de-activated
2382 * by the OS. The hardware is still under the driver's control, but
2383 * needs to be disabled. A global MAC reset is issued to stop the
2384 * hardware, and all transmit and receive resources are freed.
2385 **/
2386static int igb_close(struct net_device *netdev)
2387{
2388 struct igb_adapter *adapter = netdev_priv(netdev);
2389
2390 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2391 igb_down(adapter);
2392
2393 igb_free_irq(adapter);
2394
2395 igb_free_all_tx_resources(adapter);
2396 igb_free_all_rx_resources(adapter);
2397
9d5c8243
AK		 2398 return 0;
2399}
2400
2401/**
2402 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2403 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2404 *
2405 * Return 0 on success, negative on failure
2406 **/
80785298 2407int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2408{
59d71989 2409 struct device *dev = tx_ring->dev;
9d5c8243
AK		 2410 int size;
2411
2412 size = sizeof(struct igb_buffer) * tx_ring->count;
2413 tx_ring->buffer_info = vmalloc(size);
2414 if (!tx_ring->buffer_info)
2415 goto err;
2416 memset(tx_ring->buffer_info, 0, size);
2417
2418 /* round up to nearest 4K */
85e8d004 2419 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2420 tx_ring->size = ALIGN(tx_ring->size, 4096);
2421
59d71989
AD		 2422 tx_ring->desc = dma_alloc_coherent(dev,
2423 tx_ring->size,
2424 &tx_ring->dma,
2425 GFP_KERNEL);
9d5c8243
AK		 2426
2427 if (!tx_ring->desc)
2428 goto err;
2429
9d5c8243
AK		 2430 tx_ring->next_to_use = 0;
2431 tx_ring->next_to_clean = 0;
9d5c8243
AK		 2432 return 0;
2433
2434err:
2435 vfree(tx_ring->buffer_info);
59d71989 2436 dev_err(dev,
9d5c8243
AK		 2437 "Unable to allocate memory for the transmit descriptor ring\n");
2438 return -ENOMEM;
2439}
2440
2441/**
2442 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2443 * (Descriptors) for all queues
2444 * @adapter: board private structure
2445 *
2446 * Return 0 on success, negative on failure
2447 **/
2448static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2449{
439705e1 2450 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2451 int i, err = 0;
2452
2453 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2454 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2455 if (err) {
439705e1 2456 dev_err(&pdev->dev,
9d5c8243
AK		 2457 "Allocation for Tx Queue %u failed\n", i);
2458 for (i--; i >= 0; i--)
3025a446 2459 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2460 break;
2461 }
2462 }
2463
a99955fc 2464 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
439705e1 2465 int r_idx = i % adapter->num_tx_queues;
3025a446 2466 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
eebbbdba 2467 }
9d5c8243
AK		 2468 return err;
2469}
2470
2471/**
85b430b4
AD		 2472 * igb_setup_tctl - configure the transmit control registers
2473 * @adapter: Board private structure
9d5c8243 2474 **/
d7ee5b3a 2475void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2476{
9d5c8243
AK		 2477 struct e1000_hw *hw = &adapter->hw;
2478 u32 tctl;
9d5c8243 2479
85b430b4
AD		 2480 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2481 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2482
2483 /* Program the Transmit Control Register */
9d5c8243
AK		 2484 tctl = rd32(E1000_TCTL);
2485 tctl &= ~E1000_TCTL_CT;
2486 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2487 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2488
2489 igb_config_collision_dist(hw);
2490
9d5c8243
AK		 2491 /* Enable transmits */
2492 tctl |= E1000_TCTL_EN;
2493
2494 wr32(E1000_TCTL, tctl);
2495}
2496
85b430b4
AD		 2497/**
2498 * igb_configure_tx_ring - Configure transmit ring after Reset
2499 * @adapter: board private structure
2500 * @ring: tx ring to configure
2501 *
2502 * Configure a transmit ring after a reset.
2503 **/
d7ee5b3a
AD		 2504void igb_configure_tx_ring(struct igb_adapter *adapter,
2505 struct igb_ring *ring)
85b430b4
AD		 2506{
2507 struct e1000_hw *hw = &adapter->hw;
2508 u32 txdctl;
2509 u64 tdba = ring->dma;
2510 int reg_idx = ring->reg_idx;
2511
2512 /* disable the queue */
2513 txdctl = rd32(E1000_TXDCTL(reg_idx));
2514 wr32(E1000_TXDCTL(reg_idx),
2515 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2516 wrfl();
2517 mdelay(10);
2518
2519 wr32(E1000_TDLEN(reg_idx),
2520 ring->count * sizeof(union e1000_adv_tx_desc));
2521 wr32(E1000_TDBAL(reg_idx),
2522 tdba & 0x00000000ffffffffULL);
2523 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2524
fce99e34
AD		 2525 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2526 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2527 writel(0, ring->head);
2528 writel(0, ring->tail);
85b430b4
AD		 2529
2530 txdctl |= IGB_TX_PTHRESH;
2531 txdctl |= IGB_TX_HTHRESH << 8;
2532 txdctl |= IGB_TX_WTHRESH << 16;
2533
2534 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2535 wr32(E1000_TXDCTL(reg_idx), txdctl);
2536}
2537
2538/**
2539 * igb_configure_tx - Configure transmit Unit after Reset
2540 * @adapter: board private structure
2541 *
2542 * Configure the Tx unit of the MAC after a reset.
2543 **/
2544static void igb_configure_tx(struct igb_adapter *adapter)
2545{
2546 int i;
2547
2548 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2549 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2550}
2551
9d5c8243
AK		 2552/**
2553 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2554 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2555 *
2556 * Returns 0 on success, negative on failure
2557 **/
80785298 2558int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2559{
59d71989 2560 struct device *dev = rx_ring->dev;
9d5c8243
AK		 2561 int size, desc_len;
2562
2563 size = sizeof(struct igb_buffer) * rx_ring->count;
2564 rx_ring->buffer_info = vmalloc(size);
2565 if (!rx_ring->buffer_info)
2566 goto err;
2567 memset(rx_ring->buffer_info, 0, size);
2568
2569 desc_len = sizeof(union e1000_adv_rx_desc);
2570
2571 /* Round up to nearest 4K */
2572 rx_ring->size = rx_ring->count * desc_len;
2573 rx_ring->size = ALIGN(rx_ring->size, 4096);
2574
59d71989
AD		 2575 rx_ring->desc = dma_alloc_coherent(dev,
2576 rx_ring->size,
2577 &rx_ring->dma,
2578 GFP_KERNEL);
9d5c8243
AK		 2579
2580 if (!rx_ring->desc)
2581 goto err;
2582
2583 rx_ring->next_to_clean = 0;
2584 rx_ring->next_to_use = 0;
9d5c8243 2585
9d5c8243
AK		 2586 return 0;
2587
2588err:
2589 vfree(rx_ring->buffer_info);
439705e1 2590 rx_ring->buffer_info = NULL;
59d71989
AD		 2591 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2592 " ring\n");
9d5c8243
AK		 2593 return -ENOMEM;
2594}
2595
2596/**
2597 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2598 * (Descriptors) for all queues
2599 * @adapter: board private structure
2600 *
2601 * Return 0 on success, negative on failure
2602 **/
2603static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2604{
439705e1 2605 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2606 int i, err = 0;
2607
2608 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2609 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2610 if (err) {
439705e1 2611 dev_err(&pdev->dev,
9d5c8243
AK		 2612 "Allocation for Rx Queue %u failed\n", i);
2613 for (i--; i >= 0; i--)
3025a446 2614 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2615 break;
2616 }
2617 }
2618
2619 return err;
2620}
2621
06cf2666
AD		 2622/**
2623 * igb_setup_mrqc - configure the multiple receive queue control registers
2624 * @adapter: Board private structure
2625 **/
2626static void igb_setup_mrqc(struct igb_adapter *adapter)
2627{
2628 struct e1000_hw *hw = &adapter->hw;
2629 u32 mrqc, rxcsum;
2630 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2631 union e1000_reta {
2632 u32 dword;
2633 u8 bytes[4];
2634 } reta;
2635 static const u8 rsshash[40] = {
2636 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2637 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2638 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2639 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2640
2641 /* Fill out hash function seeds */
2642 for (j = 0; j < 10; j++) {
2643 u32 rsskey = rsshash[(j * 4)];
2644 rsskey |= rsshash[(j * 4) + 1] << 8;
2645 rsskey |= rsshash[(j * 4) + 2] << 16;
2646 rsskey |= rsshash[(j * 4) + 3] << 24;
2647 array_wr32(E1000_RSSRK(0), j, rsskey);
2648 }
2649
a99955fc 2650 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2651
2652 if (adapter->vfs_allocated_count) {
2653 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2654 switch (hw->mac.type) {
d2ba2ed8 2655 case e1000_i350:
55cac248
AD		 2656 case e1000_82580:
2657 num_rx_queues = 1;
2658 shift = 0;
2659 break;
06cf2666
AD		 2660 case e1000_82576:
2661 shift = 3;
2662 num_rx_queues = 2;
2663 break;
2664 case e1000_82575:
2665 shift = 2;
2666 shift2 = 6;
2667 default:
2668 break;
2669 }
2670 } else {
2671 if (hw->mac.type == e1000_82575)
2672 shift = 6;
2673 }
2674
2675 for (j = 0; j < (32 * 4); j++) {
2676 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2677 if (shift2)
2678 reta.bytes[j & 3] |= num_rx_queues << shift2;
2679 if ((j & 3) == 3)
2680 wr32(E1000_RETA(j >> 2), reta.dword);
2681 }
2682
2683 /*
2684 * Disable raw packet checksumming so that RSS hash is placed in
2685 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2686 * offloads as they are enabled by default
2687 */
2688 rxcsum = rd32(E1000_RXCSUM);
2689 rxcsum |= E1000_RXCSUM_PCSD;
2690
2691 if (adapter->hw.mac.type >= e1000_82576)
2692 /* Enable Receive Checksum Offload for SCTP */
2693 rxcsum |= E1000_RXCSUM_CRCOFL;
2694
2695 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2696 wr32(E1000_RXCSUM, rxcsum);
2697
2698 /* If VMDq is enabled then we set the appropriate mode for that, else
2699 * we default to RSS so that an RSS hash is calculated per packet even
2700 * if we are only using one queue */
2701 if (adapter->vfs_allocated_count) {
2702 if (hw->mac.type > e1000_82575) {
2703 /* Set the default pool for the PF's first queue */
2704 u32 vtctl = rd32(E1000_VT_CTL);
2705 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2706 E1000_VT_CTL_DISABLE_DEF_POOL);
2707 vtctl |= adapter->vfs_allocated_count <<
2708 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2709 wr32(E1000_VT_CTL, vtctl);
2710 }
a99955fc 2711 if (adapter->rss_queues > 1)
06cf2666
AD		 2712 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2713 else
2714 mrqc = E1000_MRQC_ENABLE_VMDQ;
2715 } else {
2716 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2717 }
2718 igb_vmm_control(adapter);
2719
4478a9cd
AD		 2720 /*
2721 * Generate RSS hash based on TCP port numbers and/or
2722 * IPv4/v6 src and dst addresses since UDP cannot be
2723 * hashed reliably due to IP fragmentation
2724 */
2725 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2726 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2727 E1000_MRQC_RSS_FIELD_IPV6 |
2728 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2729 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2730
2731 wr32(E1000_MRQC, mrqc);
2732}
2733
9d5c8243
AK		 2734/**
2735 * igb_setup_rctl - configure the receive control registers
2736 * @adapter: Board private structure
2737 **/
d7ee5b3a 2738void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2739{
2740 struct e1000_hw *hw = &adapter->hw;
2741 u32 rctl;
9d5c8243
AK		 2742
2743 rctl = rd32(E1000_RCTL);
2744
2745 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2746 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2747
69d728ba 2748 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2749 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2750
87cb7e8c
AK		 2751 /*
2752 * enable stripping of CRC. It's unlikely this will break BMC
2753 * redirection as it did with e1000. Newer features require
2754 * that the HW strips the CRC.
73cd78f1 2755 */
87cb7e8c 2756 rctl |= E1000_RCTL_SECRC;
9d5c8243 2757
559e9c49 2758 /* disable store bad packets and clear size bits. */
ec54d7d6 2759 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2760
6ec43fe6
AD		 2761 /* enable LPE to prevent packets larger than max_frame_size */
2762 rctl |= E1000_RCTL_LPE;
9d5c8243 2763
952f72a8
AD		 2764 /* disable queue 0 to prevent tail write w/o re-config */
2765 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2766
e1739522
AD		 2767 /* Attention!!! For SR-IOV PF driver operations you must enable
2768 * queue drop for all VF and PF queues to prevent head of line blocking
2769 * if an un-trusted VF does not provide descriptors to hardware.
2770 */
2771 if (adapter->vfs_allocated_count) {
e1739522
AD		 2772 /* set all queue drop enable bits */
2773 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 2774 }
2775
9d5c8243
AK		 2776 wr32(E1000_RCTL, rctl);
2777}
2778
7d5753f0
AD		 2779static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2780 int vfn)
2781{
2782 struct e1000_hw *hw = &adapter->hw;
2783 u32 vmolr;
2784
2785 /* if it isn't the PF check to see if VFs are enabled and
2786 * increase the size to support vlan tags */
2787 if (vfn < adapter->vfs_allocated_count &&
2788 adapter->vf_data[vfn].vlans_enabled)
2789 size += VLAN_TAG_SIZE;
2790
2791 vmolr = rd32(E1000_VMOLR(vfn));
2792 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2793 vmolr |= size | E1000_VMOLR_LPE;
2794 wr32(E1000_VMOLR(vfn), vmolr);
2795
2796 return 0;
2797}
2798
e1739522
AD		 2799/**
2800 * igb_rlpml_set - set maximum receive packet size
2801 * @adapter: board private structure
2802 *
2803 * Configure maximum receivable packet size.
2804 **/
2805static void igb_rlpml_set(struct igb_adapter *adapter)
2806{
2807 u32 max_frame_size = adapter->max_frame_size;
2808 struct e1000_hw *hw = &adapter->hw;
2809 u16 pf_id = adapter->vfs_allocated_count;
2810
2811 if (adapter->vlgrp)
2812 max_frame_size += VLAN_TAG_SIZE;
2813
2814 /* if vfs are enabled we set RLPML to the largest possible request
2815 * size and set the VMOLR RLPML to the size we need */
2816 if (pf_id) {
2817 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
7d5753f0 2818 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 2819 }
2820
2821 wr32(E1000_RLPML, max_frame_size);
2822}
2823
8151d294
WM		 2824static inline void igb_set_vmolr(struct igb_adapter *adapter,
2825 int vfn, bool aupe)
7d5753f0
AD		 2826{
2827 struct e1000_hw *hw = &adapter->hw;
2828 u32 vmolr;
2829
2830 /*
2831 * This register exists only on 82576 and newer so if we are older then
2832 * we should exit and do nothing
2833 */
2834 if (hw->mac.type < e1000_82576)
2835 return;
2836
2837 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 2838 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2839 if (aupe)
2840 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2841 else
2842 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 2843
2844 /* clear all bits that might not be set */
2845 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2846
a99955fc 2847 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 2848 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2849 /*
2850 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2851 * multicast packets
2852 */
2853 if (vfn <= adapter->vfs_allocated_count)
2854 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2855
2856 wr32(E1000_VMOLR(vfn), vmolr);
2857}
2858
85b430b4
AD		 2859/**
2860 * igb_configure_rx_ring - Configure a receive ring after Reset
2861 * @adapter: board private structure
2862 * @ring: receive ring to be configured
2863 *
2864 * Configure the Rx unit of the MAC after a reset.
2865 **/
d7ee5b3a
AD		 2866void igb_configure_rx_ring(struct igb_adapter *adapter,
2867 struct igb_ring *ring)
85b430b4
AD		 2868{
2869 struct e1000_hw *hw = &adapter->hw;
2870 u64 rdba = ring->dma;
2871 int reg_idx = ring->reg_idx;
952f72a8 2872 u32 srrctl, rxdctl;
85b430b4
AD		 2873
2874 /* disable the queue */
2875 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2876 wr32(E1000_RXDCTL(reg_idx),
2877 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2878
2879 /* Set DMA base address registers */
2880 wr32(E1000_RDBAL(reg_idx),
2881 rdba & 0x00000000ffffffffULL);
2882 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2883 wr32(E1000_RDLEN(reg_idx),
2884 ring->count * sizeof(union e1000_adv_rx_desc));
2885
2886 /* initialize head and tail */
fce99e34
AD		 2887 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2888 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2889 writel(0, ring->head);
2890 writel(0, ring->tail);
85b430b4 2891
952f72a8 2892 /* set descriptor configuration */
4c844851
AD		 2893 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2894 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
952f72a8
AD		 2895 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2896#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2897 srrctl |= IGB_RXBUFFER_16384 >>
2898 E1000_SRRCTL_BSIZEPKT_SHIFT;
2899#else
2900 srrctl |= (PAGE_SIZE / 2) >>
2901 E1000_SRRCTL_BSIZEPKT_SHIFT;
2902#endif
2903 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2904 } else {
4c844851 2905 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
952f72a8
AD		 2906 E1000_SRRCTL_BSIZEPKT_SHIFT;
2907 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2908 }
757b77e2
NN		 2909 if (hw->mac.type == e1000_82580)
2910 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 2911 /* Only set Drop Enable if we are supporting multiple queues */
2912 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2913 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 2914
2915 wr32(E1000_SRRCTL(reg_idx), srrctl);
2916
7d5753f0 2917 /* set filtering for VMDQ pools */
8151d294 2918 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 2919
85b430b4
AD		 2920 /* enable receive descriptor fetching */
2921 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2922 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2923 rxdctl &= 0xFFF00000;
2924 rxdctl |= IGB_RX_PTHRESH;
2925 rxdctl |= IGB_RX_HTHRESH << 8;
2926 rxdctl |= IGB_RX_WTHRESH << 16;
2927 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2928}
2929
9d5c8243
AK		 2930/**
2931 * igb_configure_rx - Configure receive Unit after Reset
2932 * @adapter: board private structure
2933 *
2934 * Configure the Rx unit of the MAC after a reset.
2935 **/
2936static void igb_configure_rx(struct igb_adapter *adapter)
2937{
9107584e 2938 int i;
9d5c8243 2939
68d480c4
AD		 2940 /* set UTA to appropriate mode */
2941 igb_set_uta(adapter);
2942
26ad9178
AD		 2943 /* set the correct pool for the PF default MAC address in entry 0 */
2944 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2945 adapter->vfs_allocated_count);
2946
06cf2666
AD		 2947 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2948 * the Base and Length of the Rx Descriptor Ring */
2949 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 2950 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 2951}
2952
2953/**
2954 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 2955 * @tx_ring: Tx descriptor ring for a specific queue
2956 *
2957 * Free all transmit software resources
2958 **/
68fd9910 2959void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2960{
3b644cf6 2961 igb_clean_tx_ring(tx_ring);
9d5c8243
AK		 2962
2963 vfree(tx_ring->buffer_info);
2964 tx_ring->buffer_info = NULL;
2965
439705e1
AD		 2966 /* if not set, then don't free */
2967 if (!tx_ring->desc)
2968 return;
2969
59d71989
AD		 2970 dma_free_coherent(tx_ring->dev, tx_ring->size,
2971 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 2972
2973 tx_ring->desc = NULL;
2974}
2975
2976/**
2977 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2978 * @adapter: board private structure
2979 *
2980 * Free all transmit software resources
2981 **/
2982static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2983{
2984 int i;
2985
2986 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2987 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2988}
2989
b1a436c3
AD		 2990void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2991 struct igb_buffer *buffer_info)
9d5c8243 2992{
6366ad33
AD		 2993 if (buffer_info->dma) {
2994 if (buffer_info->mapped_as_page)
59d71989 2995 dma_unmap_page(tx_ring->dev,
6366ad33
AD		 2996 buffer_info->dma,
2997 buffer_info->length,
59d71989 2998 DMA_TO_DEVICE);
6366ad33 2999 else
59d71989 3000 dma_unmap_single(tx_ring->dev,
6366ad33
AD		 3001 buffer_info->dma,
3002 buffer_info->length,
59d71989 3003 DMA_TO_DEVICE);
6366ad33
AD		 3004 buffer_info->dma = 0;
3005 }
9d5c8243
AK		 3006 if (buffer_info->skb) {
3007 dev_kfree_skb_any(buffer_info->skb);
3008 buffer_info->skb = NULL;
3009 }
3010 buffer_info->time_stamp = 0;
6366ad33
AD		 3011 buffer_info->length = 0;
3012 buffer_info->next_to_watch = 0;
3013 buffer_info->mapped_as_page = false;
9d5c8243
AK		 3014}
3015
3016/**
3017 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3018 * @tx_ring: ring to be cleaned
3019 **/
3b644cf6 3020static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243
AK		 3021{
3022 struct igb_buffer *buffer_info;
3023 unsigned long size;
3024 unsigned int i;
3025
3026 if (!tx_ring->buffer_info)
3027 return;
3028 /* Free all the Tx ring sk_buffs */
3029
3030 for (i = 0; i < tx_ring->count; i++) {
3031 buffer_info = &tx_ring->buffer_info[i];
80785298 3032 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3033 }
3034
3035 size = sizeof(struct igb_buffer) * tx_ring->count;
3036 memset(tx_ring->buffer_info, 0, size);
3037
3038 /* Zero out the descriptor ring */
9d5c8243
AK		 3039 memset(tx_ring->desc, 0, tx_ring->size);
3040
3041 tx_ring->next_to_use = 0;
3042 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3043}
3044
3045/**
3046 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3047 * @adapter: board private structure
3048 **/
3049static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3050{
3051 int i;
3052
3053 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3054 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3055}
3056
3057/**
3058 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3059 * @rx_ring: ring to clean the resources from
3060 *
3061 * Free all receive software resources
3062 **/
68fd9910 3063void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3064{
3b644cf6 3065 igb_clean_rx_ring(rx_ring);
9d5c8243
AK		 3066
3067 vfree(rx_ring->buffer_info);
3068 rx_ring->buffer_info = NULL;
3069
439705e1
AD		 3070 /* if not set, then don't free */
3071 if (!rx_ring->desc)
3072 return;
3073
59d71989
AD		 3074 dma_free_coherent(rx_ring->dev, rx_ring->size,
3075 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3076
3077 rx_ring->desc = NULL;
3078}
3079
3080/**
3081 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3082 * @adapter: board private structure
3083 *
3084 * Free all receive software resources
3085 **/
3086static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3087{
3088 int i;
3089
3090 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3091 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3092}
3093
3094/**
3095 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3096 * @rx_ring: ring to free buffers from
3097 **/
3b644cf6 3098static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243
AK		 3099{
3100 struct igb_buffer *buffer_info;
9d5c8243
AK		 3101 unsigned long size;
3102 unsigned int i;
3103
3104 if (!rx_ring->buffer_info)
3105 return;
439705e1 3106
9d5c8243
AK		 3107 /* Free all the Rx ring sk_buffs */
3108 for (i = 0; i < rx_ring->count; i++) {
3109 buffer_info = &rx_ring->buffer_info[i];
3110 if (buffer_info->dma) {
59d71989 3111 dma_unmap_single(rx_ring->dev,
80785298 3112 buffer_info->dma,
4c844851 3113 rx_ring->rx_buffer_len,
59d71989 3114 DMA_FROM_DEVICE);
9d5c8243
AK		 3115 buffer_info->dma = 0;
3116 }
3117
3118 if (buffer_info->skb) {
3119 dev_kfree_skb(buffer_info->skb);
3120 buffer_info->skb = NULL;
3121 }
6ec43fe6 3122 if (buffer_info->page_dma) {
59d71989 3123 dma_unmap_page(rx_ring->dev,
80785298 3124 buffer_info->page_dma,
6ec43fe6 3125 PAGE_SIZE / 2,
59d71989 3126 DMA_FROM_DEVICE);
6ec43fe6
AD		 3127 buffer_info->page_dma = 0;
3128 }
9d5c8243 3129 if (buffer_info->page) {
9d5c8243
AK		 3130 put_page(buffer_info->page);
3131 buffer_info->page = NULL;
bf36c1a0 3132 buffer_info->page_offset = 0;
9d5c8243
AK		 3133 }
3134 }
3135
9d5c8243
AK		 3136 size = sizeof(struct igb_buffer) * rx_ring->count;
3137 memset(rx_ring->buffer_info, 0, size);
3138
3139 /* Zero out the descriptor ring */
3140 memset(rx_ring->desc, 0, rx_ring->size);
3141
3142 rx_ring->next_to_clean = 0;
3143 rx_ring->next_to_use = 0;
9d5c8243
AK		 3144}
3145
3146/**
3147 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3148 * @adapter: board private structure
3149 **/
3150static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3151{
3152 int i;
3153
3154 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3155 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3156}
3157
3158/**
3159 * igb_set_mac - Change the Ethernet Address of the NIC
3160 * @netdev: network interface device structure
3161 * @p: pointer to an address structure
3162 *
3163 * Returns 0 on success, negative on failure
3164 **/
3165static int igb_set_mac(struct net_device *netdev, void *p)
3166{
3167 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3168 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3169 struct sockaddr *addr = p;
3170
3171 if (!is_valid_ether_addr(addr->sa_data))
3172 return -EADDRNOTAVAIL;
3173
3174 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3175 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3176
26ad9178
AD		 3177 /* set the correct pool for the new PF MAC address in entry 0 */
3178 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3179 adapter->vfs_allocated_count);
e1739522 3180
9d5c8243
AK		 3181 return 0;
3182}
3183
3184/**
68d480c4 3185 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3186 * @netdev: network interface device structure
3187 *
68d480c4
AD		 3188 * Writes multicast address list to the MTA hash table.
3189 * Returns: -ENOMEM on failure
3190 * 0 on no addresses written
3191 * X on writing X addresses to MTA
9d5c8243 3192 **/
68d480c4 3193static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3194{
3195 struct igb_adapter *adapter = netdev_priv(netdev);
3196 struct e1000_hw *hw = &adapter->hw;
22bedad3 3197 struct netdev_hw_addr *ha;
68d480c4 3198 u8 *mta_list;
9d5c8243
AK		 3199 int i;
3200
4cd24eaf 3201 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3202 /* nothing to program, so clear mc list */
3203 igb_update_mc_addr_list(hw, NULL, 0);
3204 igb_restore_vf_multicasts(adapter);
3205 return 0;
3206 }
9d5c8243 3207
4cd24eaf 3208 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3209 if (!mta_list)
3210 return -ENOMEM;
ff41f8dc 3211
68d480c4 3212 /* The shared function expects a packed array of only addresses. */
48e2f183 3213 i = 0;
22bedad3
JP		 3214 netdev_for_each_mc_addr(ha, netdev)
3215 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3216
68d480c4
AD		 3217 igb_update_mc_addr_list(hw, mta_list, i);
3218 kfree(mta_list);
3219
4cd24eaf 3220 return netdev_mc_count(netdev);
68d480c4
AD		 3221}
3222
3223/**
3224 * igb_write_uc_addr_list - write unicast addresses to RAR table
3225 * @netdev: network interface device structure
3226 *
3227 * Writes unicast address list to the RAR table.
3228 * Returns: -ENOMEM on failure/insufficient address space
3229 * 0 on no addresses written
3230 * X on writing X addresses to the RAR table
3231 **/
3232static int igb_write_uc_addr_list(struct net_device *netdev)
3233{
3234 struct igb_adapter *adapter = netdev_priv(netdev);
3235 struct e1000_hw *hw = &adapter->hw;
3236 unsigned int vfn = adapter->vfs_allocated_count;
3237 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3238 int count = 0;
3239
3240 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3241 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3242 return -ENOMEM;
9d5c8243 3243
32e7bfc4 3244 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3245 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3246
3247 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3248 if (!rar_entries)
3249 break;
26ad9178
AD		 3250 igb_rar_set_qsel(adapter, ha->addr,
3251 rar_entries--,
68d480c4
AD		 3252 vfn);
3253 count++;
ff41f8dc
AD		 3254 }
3255 }
3256 /* write the addresses in reverse order to avoid write combining */
3257 for (; rar_entries > 0 ; rar_entries--) {
3258 wr32(E1000_RAH(rar_entries), 0);
3259 wr32(E1000_RAL(rar_entries), 0);
3260 }
3261 wrfl();
3262
68d480c4
AD		 3263 return count;
3264}
3265
3266/**
3267 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3268 * @netdev: network interface device structure
3269 *
3270 * The set_rx_mode entry point is called whenever the unicast or multicast
3271 * address lists or the network interface flags are updated. This routine is
3272 * responsible for configuring the hardware for proper unicast, multicast,
3273 * promiscuous mode, and all-multi behavior.
3274 **/
3275static void igb_set_rx_mode(struct net_device *netdev)
3276{
3277 struct igb_adapter *adapter = netdev_priv(netdev);
3278 struct e1000_hw *hw = &adapter->hw;
3279 unsigned int vfn = adapter->vfs_allocated_count;
3280 u32 rctl, vmolr = 0;
3281 int count;
3282
3283 /* Check for Promiscuous and All Multicast modes */
3284 rctl = rd32(E1000_RCTL);
3285
3286 /* clear the effected bits */
3287 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3288
3289 if (netdev->flags & IFF_PROMISC) {
3290 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3291 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3292 } else {
3293 if (netdev->flags & IFF_ALLMULTI) {
3294 rctl |= E1000_RCTL_MPE;
3295 vmolr |= E1000_VMOLR_MPME;
3296 } else {
3297 /*
3298 * Write addresses to the MTA, if the attempt fails
3299 * then we should just turn on promiscous mode so
3300 * that we can at least receive multicast traffic
3301 */
3302 count = igb_write_mc_addr_list(netdev);
3303 if (count < 0) {
3304 rctl |= E1000_RCTL_MPE;
3305 vmolr |= E1000_VMOLR_MPME;
3306 } else if (count) {
3307 vmolr |= E1000_VMOLR_ROMPE;
3308 }
3309 }
3310 /*
3311 * Write addresses to available RAR registers, if there is not
3312 * sufficient space to store all the addresses then enable
3313 * unicast promiscous mode
3314 */
3315 count = igb_write_uc_addr_list(netdev);
3316 if (count < 0) {
3317 rctl |= E1000_RCTL_UPE;
3318 vmolr |= E1000_VMOLR_ROPE;
3319 }
3320 rctl |= E1000_RCTL_VFE;
28fc06f5 3321 }
68d480c4 3322 wr32(E1000_RCTL, rctl);
28fc06f5 3323
68d480c4
AD		 3324 /*
3325 * In order to support SR-IOV and eventually VMDq it is necessary to set
3326 * the VMOLR to enable the appropriate modes. Without this workaround
3327 * we will have issues with VLAN tag stripping not being done for frames
3328 * that are only arriving because we are the default pool
3329 */
3330 if (hw->mac.type < e1000_82576)
28fc06f5 3331 return;
9d5c8243 3332
68d480c4
AD		 3333 vmolr |= rd32(E1000_VMOLR(vfn)) &
3334 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3335 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3336 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3337}
3338
3339/* Need to wait a few seconds after link up to get diagnostic information from
3340 * the phy */
3341static void igb_update_phy_info(unsigned long data)
3342{
3343 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3344 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3345}
3346
4d6b725e
AD		 3347/**
3348 * igb_has_link - check shared code for link and determine up/down
3349 * @adapter: pointer to driver private info
3350 **/
3145535a 3351bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3352{
3353 struct e1000_hw *hw = &adapter->hw;
3354 bool link_active = false;
3355 s32 ret_val = 0;
3356
3357 /* get_link_status is set on LSC (link status) interrupt or
3358 * rx sequence error interrupt. get_link_status will stay
3359 * false until the e1000_check_for_link establishes link
3360 * for copper adapters ONLY
3361 */
3362 switch (hw->phy.media_type) {
3363 case e1000_media_type_copper:
3364 if (hw->mac.get_link_status) {
3365 ret_val = hw->mac.ops.check_for_link(hw);
3366 link_active = !hw->mac.get_link_status;
3367 } else {
3368 link_active = true;
3369 }
3370 break;
4d6b725e
AD		 3371 case e1000_media_type_internal_serdes:
3372 ret_val = hw->mac.ops.check_for_link(hw);
3373 link_active = hw->mac.serdes_has_link;
3374 break;
3375 default:
3376 case e1000_media_type_unknown:
3377 break;
3378 }
3379
3380 return link_active;
3381}
3382
9d5c8243
AK		 3383/**
3384 * igb_watchdog - Timer Call-back
3385 * @data: pointer to adapter cast into an unsigned long
3386 **/
3387static void igb_watchdog(unsigned long data)
3388{
3389 struct igb_adapter *adapter = (struct igb_adapter *)data;
3390 /* Do the rest outside of interrupt context */
3391 schedule_work(&adapter->watchdog_task);
3392}
3393
3394static void igb_watchdog_task(struct work_struct *work)
3395{
3396 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3397 struct igb_adapter,
3398 watchdog_task);
9d5c8243 3399 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3400 struct net_device *netdev = adapter->netdev;
9d5c8243 3401 u32 link;
7a6ea550 3402 int i;
9d5c8243 3403
4d6b725e 3404 link = igb_has_link(adapter);
9d5c8243
AK		 3405 if (link) {
3406 if (!netif_carrier_ok(netdev)) {
3407 u32 ctrl;
330a6d6a
AD		 3408 hw->mac.ops.get_speed_and_duplex(hw,
3409 &adapter->link_speed,
3410 &adapter->link_duplex);
9d5c8243
AK		 3411
3412 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3413 /* Links status message must follow this format */
3414 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3415 "Flow Control: %s\n",
559e9c49
AD		 3416 netdev->name,
3417 adapter->link_speed,
3418 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3419 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3420 ((ctrl & E1000_CTRL_TFCE) &&
3421 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3422 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3423 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3424
d07f3e37 3425 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3426 adapter->tx_timeout_factor = 1;
3427 switch (adapter->link_speed) {
3428 case SPEED_10:
9d5c8243
AK		 3429 adapter->tx_timeout_factor = 14;
3430 break;
3431 case SPEED_100:
9d5c8243
AK		 3432 /* maybe add some timeout factor ? */
3433 break;
3434 }
3435
3436 netif_carrier_on(netdev);
9d5c8243 3437
4ae196df
AD		 3438 igb_ping_all_vfs(adapter);
3439
4b1a9877 3440 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3441 if (!test_bit(__IGB_DOWN, &adapter->state))
3442 mod_timer(&adapter->phy_info_timer,
3443 round_jiffies(jiffies + 2 * HZ));
3444 }
3445 } else {
3446 if (netif_carrier_ok(netdev)) {
3447 adapter->link_speed = 0;
3448 adapter->link_duplex = 0;
527d47c1
AD		 3449 /* Links status message must follow this format */
3450 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3451 netdev->name);
9d5c8243 3452 netif_carrier_off(netdev);
4b1a9877 3453
4ae196df
AD		 3454 igb_ping_all_vfs(adapter);
3455
4b1a9877 3456 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3457 if (!test_bit(__IGB_DOWN, &adapter->state))
3458 mod_timer(&adapter->phy_info_timer,
3459 round_jiffies(jiffies + 2 * HZ));
3460 }
3461 }
3462
9d5c8243 3463 igb_update_stats(adapter);
9d5c8243 3464
dbabb065 3465 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3466 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3467 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3468 /* We've lost link, so the controller stops DMA,
3469 * but we've got queued Tx work that's never going
3470 * to get done, so reset controller to flush Tx.
3471 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3472 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3473 adapter->tx_timeout_count++;
3474 schedule_work(&adapter->reset_task);
3475 /* return immediately since reset is imminent */
3476 return;
3477 }
9d5c8243 3478 }
9d5c8243 3479
dbabb065
AD		 3480 /* Force detection of hung controller every watchdog period */
3481 tx_ring->detect_tx_hung = true;
3482 }
f7ba205e 3483
9d5c8243 3484 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3485 if (adapter->msix_entries) {
047e0030
AD		 3486 u32 eics = 0;
3487 for (i = 0; i < adapter->num_q_vectors; i++) {
3488 struct igb_q_vector *q_vector = adapter->q_vector[i];
3489 eics |= q_vector->eims_value;
3490 }
7a6ea550
AD		 3491 wr32(E1000_EICS, eics);
3492 } else {
3493 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3494 }
9d5c8243 3495
9d5c8243
AK		 3496 /* Reset the timer */
3497 if (!test_bit(__IGB_DOWN, &adapter->state))
3498 mod_timer(&adapter->watchdog_timer,
3499 round_jiffies(jiffies + 2 * HZ));
3500}
3501
3502enum latency_range {
3503 lowest_latency = 0,
3504 low_latency = 1,
3505 bulk_latency = 2,
3506 latency_invalid = 255
3507};
3508
6eb5a7f1
AD		 3509/**
3510 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3511 *
3512 * Stores a new ITR value based on strictly on packet size. This
3513 * algorithm is less sophisticated than that used in igb_update_itr,
3514 * due to the difficulty of synchronizing statistics across multiple
3515 * receive rings. The divisors and thresholds used by this fuction
3516 * were determined based on theoretical maximum wire speed and testing
3517 * data, in order to minimize response time while increasing bulk
3518 * throughput.
3519 * This functionality is controlled by the InterruptThrottleRate module
3520 * parameter (see igb_param.c)
3521 * NOTE: This function is called only when operating in a multiqueue
3522 * receive environment.
047e0030 3523 * @q_vector: pointer to q_vector
6eb5a7f1 3524 **/
047e0030 3525static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3526{
047e0030 3527 int new_val = q_vector->itr_val;
6eb5a7f1 3528 int avg_wire_size = 0;
047e0030 3529 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 3530
6eb5a7f1
AD		 3531 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3532 * ints/sec - ITR timer value of 120 ticks.
3533 */
3534 if (adapter->link_speed != SPEED_1000) {
047e0030 3535 new_val = 976;
6eb5a7f1 3536 goto set_itr_val;
9d5c8243 3537 }
047e0030
AD		 3538
3539 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3540 struct igb_ring *ring = q_vector->rx_ring;
3541 avg_wire_size = ring->total_bytes / ring->total_packets;
3542 }
3543
3544 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3545 struct igb_ring *ring = q_vector->tx_ring;
3546 avg_wire_size = max_t(u32, avg_wire_size,
3547 (ring->total_bytes /
3548 ring->total_packets));
3549 }
3550
3551 /* if avg_wire_size isn't set no work was done */
3552 if (!avg_wire_size)
3553 goto clear_counts;
9d5c8243 3554
6eb5a7f1
AD		 3555 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3556 avg_wire_size += 24;
3557
3558 /* Don't starve jumbo frames */
3559 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3560
6eb5a7f1
AD		 3561 /* Give a little boost to mid-size frames */
3562 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3563 new_val = avg_wire_size / 3;
3564 else
3565 new_val = avg_wire_size / 2;
9d5c8243 3566
abe1c363
NN		 3567 /* when in itr mode 3 do not exceed 20K ints/sec */
3568 if (adapter->rx_itr_setting == 3 && new_val < 196)
3569 new_val = 196;
3570
6eb5a7f1 3571set_itr_val:
047e0030
AD		 3572 if (new_val != q_vector->itr_val) {
3573 q_vector->itr_val = new_val;
3574 q_vector->set_itr = 1;
9d5c8243 3575 }
6eb5a7f1 3576clear_counts:
047e0030
AD		 3577 if (q_vector->rx_ring) {
3578 q_vector->rx_ring->total_bytes = 0;
3579 q_vector->rx_ring->total_packets = 0;
3580 }
3581 if (q_vector->tx_ring) {
3582 q_vector->tx_ring->total_bytes = 0;
3583 q_vector->tx_ring->total_packets = 0;
3584 }
9d5c8243
AK		 3585}
3586
3587/**
3588 * igb_update_itr - update the dynamic ITR value based on statistics
3589 * Stores a new ITR value based on packets and byte
3590 * counts during the last interrupt. The advantage of per interrupt
3591 * computation is faster updates and more accurate ITR for the current
3592 * traffic pattern. Constants in this function were computed
3593 * based on theoretical maximum wire speed and thresholds were set based
3594 * on testing data as well as attempting to minimize response time
3595 * while increasing bulk throughput.
3596 * this functionality is controlled by the InterruptThrottleRate module
3597 * parameter (see igb_param.c)
3598 * NOTE: These calculations are only valid when operating in a single-
3599 * queue environment.
3600 * @adapter: pointer to adapter
047e0030 3601 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3602 * @packets: the number of packets during this measurement interval
3603 * @bytes: the number of bytes during this measurement interval
3604 **/
3605static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3606 int packets, int bytes)
3607{
3608 unsigned int retval = itr_setting;
3609
3610 if (packets == 0)
3611 goto update_itr_done;
3612
3613 switch (itr_setting) {
3614 case lowest_latency:
3615 /* handle TSO and jumbo frames */
3616 if (bytes/packets > 8000)
3617 retval = bulk_latency;
3618 else if ((packets < 5) && (bytes > 512))
3619 retval = low_latency;
3620 break;
3621 case low_latency: /* 50 usec aka 20000 ints/s */
3622 if (bytes > 10000) {
3623 /* this if handles the TSO accounting */
3624 if (bytes/packets > 8000) {
3625 retval = bulk_latency;
3626 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3627 retval = bulk_latency;
3628 } else if ((packets > 35)) {
3629 retval = lowest_latency;
3630 }
3631 } else if (bytes/packets > 2000) {
3632 retval = bulk_latency;
3633 } else if (packets <= 2 && bytes < 512) {
3634 retval = lowest_latency;
3635 }
3636 break;
3637 case bulk_latency: /* 250 usec aka 4000 ints/s */
3638 if (bytes > 25000) {
3639 if (packets > 35)
3640 retval = low_latency;
1e5c3d21 3641 } else if (bytes < 1500) {
9d5c8243
AK		 3642 retval = low_latency;
3643 }
3644 break;
3645 }
3646
3647update_itr_done:
3648 return retval;
3649}
3650
6eb5a7f1 3651static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3652{
047e0030 3653 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3654 u16 current_itr;
047e0030 3655 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3656
3657 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3658 if (adapter->link_speed != SPEED_1000) {
3659 current_itr = 0;
3660 new_itr = 4000;
3661 goto set_itr_now;
3662 }
3663
3664 adapter->rx_itr = igb_update_itr(adapter,
3665 adapter->rx_itr,
3025a446
AD		 3666 q_vector->rx_ring->total_packets,
3667 q_vector->rx_ring->total_bytes);
9d5c8243 3668
047e0030
AD		 3669 adapter->tx_itr = igb_update_itr(adapter,
3670 adapter->tx_itr,
3025a446
AD		 3671 q_vector->tx_ring->total_packets,
3672 q_vector->tx_ring->total_bytes);
047e0030 3673 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3674
6eb5a7f1 3675 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3676 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3677 current_itr = low_latency;
3678
9d5c8243
AK		 3679 switch (current_itr) {
3680 /* counts and packets in update_itr are dependent on these numbers */
3681 case lowest_latency:
78b1f607 3682 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3683 break;
3684 case low_latency:
78b1f607 3685 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3686 break;
3687 case bulk_latency:
78b1f607 3688 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3689 break;
3690 default:
3691 break;
3692 }
3693
3694set_itr_now:
3025a446
AD		 3695 q_vector->rx_ring->total_bytes = 0;
3696 q_vector->rx_ring->total_packets = 0;
3697 q_vector->tx_ring->total_bytes = 0;
3698 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 3699
047e0030 3700 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3701 /* this attempts to bias the interrupt rate towards Bulk
3702 * by adding intermediate steps when interrupt rate is
3703 * increasing */
047e0030
AD		 3704 new_itr = new_itr > q_vector->itr_val ?
3705 max((new_itr * q_vector->itr_val) /
3706 (new_itr + (q_vector->itr_val >> 2)),
3707 new_itr) :
9d5c8243
AK		 3708 new_itr;
3709 /* Don't write the value here; it resets the adapter's
3710 * internal timer, and causes us to delay far longer than
3711 * we should between interrupts. Instead, we write the ITR
3712 * value at the beginning of the next interrupt so the timing
3713 * ends up being correct.
3714 */
047e0030
AD		 3715 q_vector->itr_val = new_itr;
3716 q_vector->set_itr = 1;
9d5c8243 3717 }
9d5c8243
AK		 3718}
3719
9d5c8243
AK		 3720#define IGB_TX_FLAGS_CSUM 0x00000001
3721#define IGB_TX_FLAGS_VLAN 0x00000002
3722#define IGB_TX_FLAGS_TSO 0x00000004
3723#define IGB_TX_FLAGS_IPV4 0x00000008
cdfd01fc
AD		 3724#define IGB_TX_FLAGS_TSTAMP 0x00000010
3725#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3726#define IGB_TX_FLAGS_VLAN_SHIFT 16
9d5c8243 3727
85ad76b2 3728static inline int igb_tso_adv(struct igb_ring *tx_ring,
9d5c8243
AK		 3729 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3730{
3731 struct e1000_adv_tx_context_desc *context_desc;
3732 unsigned int i;
3733 int err;
3734 struct igb_buffer *buffer_info;
3735 u32 info = 0, tu_cmd = 0;
91d4ee33
NN		 3736 u32 mss_l4len_idx;
3737 u8 l4len;
9d5c8243
AK		 3738
3739 if (skb_header_cloned(skb)) {
3740 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3741 if (err)
3742 return err;
3743 }
3744
3745 l4len = tcp_hdrlen(skb);
3746 *hdr_len += l4len;
3747
3748 if (skb->protocol == htons(ETH_P_IP)) {
3749 struct iphdr *iph = ip_hdr(skb);
3750 iph->tot_len = 0;
3751 iph->check = 0;
3752 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3753 iph->daddr, 0,
3754 IPPROTO_TCP,
3755 0);
8e1e8a47 3756 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 3757 ipv6_hdr(skb)->payload_len = 0;
3758 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3759 &ipv6_hdr(skb)->daddr,
3760 0, IPPROTO_TCP, 0);
3761 }
3762
3763 i = tx_ring->next_to_use;
3764
3765 buffer_info = &tx_ring->buffer_info[i];
3766 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3767 /* VLAN MACLEN IPLEN */
3768 if (tx_flags & IGB_TX_FLAGS_VLAN)
3769 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3770 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3771 *hdr_len += skb_network_offset(skb);
3772 info |= skb_network_header_len(skb);
3773 *hdr_len += skb_network_header_len(skb);
3774 context_desc->vlan_macip_lens = cpu_to_le32(info);
3775
3776 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3777 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3778
3779 if (skb->protocol == htons(ETH_P_IP))
3780 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3781 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3782
3783 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3784
3785 /* MSS L4LEN IDX */
3786 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3787 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3788
73cd78f1 3789 /* For 82575, context index must be unique per ring. */
85ad76b2
AD		 3790 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3791 mss_l4len_idx |= tx_ring->reg_idx << 4;
9d5c8243
AK		 3792
3793 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3794 context_desc->seqnum_seed = 0;
3795
3796 buffer_info->time_stamp = jiffies;
0e014cb1 3797 buffer_info->next_to_watch = i;
9d5c8243
AK		 3798 buffer_info->dma = 0;
3799 i++;
3800 if (i == tx_ring->count)
3801 i = 0;
3802
3803 tx_ring->next_to_use = i;
3804
3805 return true;
3806}
3807
85ad76b2
AD		 3808static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3809 struct sk_buff *skb, u32 tx_flags)
9d5c8243
AK		 3810{
3811 struct e1000_adv_tx_context_desc *context_desc;
59d71989 3812 struct device *dev = tx_ring->dev;
9d5c8243
AK		 3813 struct igb_buffer *buffer_info;
3814 u32 info = 0, tu_cmd = 0;
80785298 3815 unsigned int i;
9d5c8243
AK		 3816
3817 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3818 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3819 i = tx_ring->next_to_use;
3820 buffer_info = &tx_ring->buffer_info[i];
3821 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3822
3823 if (tx_flags & IGB_TX_FLAGS_VLAN)
3824 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
cdfd01fc 3825
9d5c8243
AK		 3826 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3827 if (skb->ip_summed == CHECKSUM_PARTIAL)
3828 info |= skb_network_header_len(skb);
3829
3830 context_desc->vlan_macip_lens = cpu_to_le32(info);
3831
3832 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3833
3834 if (skb->ip_summed == CHECKSUM_PARTIAL) {
fa4a7ef3
AJ		 3835 __be16 protocol;
3836
3837 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3838 const struct vlan_ethhdr *vhdr =
3839 (const struct vlan_ethhdr*)skb->data;
3840
3841 protocol = vhdr->h_vlan_encapsulated_proto;
3842 } else {
3843 protocol = skb->protocol;
3844 }
3845
3846 switch (protocol) {
09640e63 3847 case cpu_to_be16(ETH_P_IP):
9d5c8243 3848 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
44b0cda3
MW		 3849 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3850 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3851 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3852 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3 3853 break;
09640e63 3854 case cpu_to_be16(ETH_P_IPV6):
44b0cda3
MW		 3855 /* XXX what about other V6 headers?? */
3856 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3857 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
b9473560
JB		 3858 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3859 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
44b0cda3
MW		 3860 break;
3861 default:
3862 if (unlikely(net_ratelimit()))
59d71989 3863 dev_warn(dev,
44b0cda3
MW		 3864 "partial checksum but proto=%x!\n",
3865 skb->protocol);
3866 break;
3867 }
9d5c8243
AK		 3868 }
3869
3870 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3871 context_desc->seqnum_seed = 0;
85ad76b2 3872 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
7dfc16fa 3873 context_desc->mss_l4len_idx =
85ad76b2 3874 cpu_to_le32(tx_ring->reg_idx << 4);
9d5c8243
AK		 3875
3876 buffer_info->time_stamp = jiffies;
0e014cb1 3877 buffer_info->next_to_watch = i;
9d5c8243
AK		 3878 buffer_info->dma = 0;
3879
3880 i++;
3881 if (i == tx_ring->count)
3882 i = 0;
3883 tx_ring->next_to_use = i;
3884
3885 return true;
3886 }
9d5c8243
AK		 3887 return false;
3888}
3889
3890#define IGB_MAX_TXD_PWR 16
3891#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3892
80785298 3893static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
0e014cb1 3894 unsigned int first)
9d5c8243
AK		 3895{
3896 struct igb_buffer *buffer_info;
59d71989 3897 struct device *dev = tx_ring->dev;
2873957d 3898 unsigned int hlen = skb_headlen(skb);
9d5c8243
AK		 3899 unsigned int count = 0, i;
3900 unsigned int f;
2873957d 3901 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
9d5c8243
AK		 3902
3903 i = tx_ring->next_to_use;
3904
3905 buffer_info = &tx_ring->buffer_info[i];
2873957d
NN		 3906 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3907 buffer_info->length = hlen;
9d5c8243
AK		 3908 /* set time_stamp *before* dma to help avoid a possible race */
3909 buffer_info->time_stamp = jiffies;
0e014cb1 3910 buffer_info->next_to_watch = i;
2873957d 3911 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
59d71989
AD		 3912 DMA_TO_DEVICE);
3913 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33 3914 goto dma_error;
9d5c8243
AK		 3915
3916 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2873957d
NN		 3917 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3918 unsigned int len = frag->size;
9d5c8243 3919
8581145f 3920 count++;
65689fef
AD		 3921 i++;
3922 if (i == tx_ring->count)
3923 i = 0;
3924
9d5c8243
AK		 3925 buffer_info = &tx_ring->buffer_info[i];
3926 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3927 buffer_info->length = len;
3928 buffer_info->time_stamp = jiffies;
0e014cb1 3929 buffer_info->next_to_watch = i;
6366ad33 3930 buffer_info->mapped_as_page = true;
59d71989 3931 buffer_info->dma = dma_map_page(dev,
6366ad33
AD		 3932 frag->page,
3933 frag->page_offset,
3934 len,
59d71989
AD		 3935 DMA_TO_DEVICE);
3936 if (dma_mapping_error(dev, buffer_info->dma))
6366ad33
AD		 3937 goto dma_error;
3938
9d5c8243
AK		 3939 }
3940
9d5c8243 3941 tx_ring->buffer_info[i].skb = skb;
2873957d
NN		 3942 tx_ring->buffer_info[i].shtx = skb_shinfo(skb)->tx_flags;
3943 /* multiply data chunks by size of headers */
3944 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3945 tx_ring->buffer_info[i].gso_segs = gso_segs;
0e014cb1 3946 tx_ring->buffer_info[first].next_to_watch = i;
9d5c8243 3947
cdfd01fc 3948 return ++count;
6366ad33
AD		 3949
3950dma_error:
59d71989 3951 dev_err(dev, "TX DMA map failed\n");
6366ad33
AD		 3952
3953 /* clear timestamp and dma mappings for failed buffer_info mapping */
3954 buffer_info->dma = 0;
3955 buffer_info->time_stamp = 0;
3956 buffer_info->length = 0;
3957 buffer_info->next_to_watch = 0;
3958 buffer_info->mapped_as_page = false;
6366ad33
AD		 3959
3960 /* clear timestamp and dma mappings for remaining portion of packet */
a77ff709
NN		 3961 while (count--) {
3962 if (i == 0)
3963 i = tx_ring->count;
6366ad33 3964 i--;
6366ad33
AD		 3965 buffer_info = &tx_ring->buffer_info[i];
3966 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3967 }
3968
3969 return 0;
9d5c8243
AK		 3970}
3971
85ad76b2 3972static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
91d4ee33 3973 u32 tx_flags, int count, u32 paylen,
9d5c8243
AK		 3974 u8 hdr_len)
3975{
cdfd01fc 3976 union e1000_adv_tx_desc *tx_desc;
9d5c8243
AK		 3977 struct igb_buffer *buffer_info;
3978 u32 olinfo_status = 0, cmd_type_len;
cdfd01fc 3979 unsigned int i = tx_ring->next_to_use;
9d5c8243
AK		 3980
3981 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3982 E1000_ADVTXD_DCMD_DEXT);
3983
3984 if (tx_flags & IGB_TX_FLAGS_VLAN)
3985 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3986
33af6bcc
PO		 3987 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3988 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3989
9d5c8243
AK		 3990 if (tx_flags & IGB_TX_FLAGS_TSO) {
3991 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3992
3993 /* insert tcp checksum */
3994 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3995
3996 /* insert ip checksum */
3997 if (tx_flags & IGB_TX_FLAGS_IPV4)
3998 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3999
4000 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4001 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4002 }
4003
85ad76b2
AD		 4004 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4005 (tx_flags & (IGB_TX_FLAGS_CSUM |
4006 IGB_TX_FLAGS_TSO |
7dfc16fa 4007 IGB_TX_FLAGS_VLAN)))
85ad76b2 4008 olinfo_status |= tx_ring->reg_idx << 4;
9d5c8243
AK		 4009
4010 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4011
cdfd01fc 4012 do {
9d5c8243
AK		 4013 buffer_info = &tx_ring->buffer_info[i];
4014 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4015 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4016 tx_desc->read.cmd_type_len =
4017 cpu_to_le32(cmd_type_len | buffer_info->length);
4018 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
cdfd01fc 4019 count--;
9d5c8243
AK		 4020 i++;
4021 if (i == tx_ring->count)
4022 i = 0;
cdfd01fc 4023 } while (count > 0);
9d5c8243 4024
85ad76b2 4025 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
9d5c8243
AK		 4026 /* Force memory writes to complete before letting h/w
4027 * know there are new descriptors to fetch. (Only
4028 * applicable for weak-ordered memory model archs,
4029 * such as IA-64). */
4030 wmb();
4031
4032 tx_ring->next_to_use = i;
fce99e34 4033 writel(i, tx_ring->tail);
9d5c8243
AK		 4034 /* we need this if more than one processor can write to our tail
4035 * at a time, it syncronizes IO on IA64/Altix systems */
4036 mmiowb();
4037}
4038
e694e964 4039static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4040{
e694e964
AD		 4041 struct net_device *netdev = tx_ring->netdev;
4042
661086df 4043 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4044
9d5c8243
AK		 4045 /* Herbert's original patch had:
4046 * smp_mb__after_netif_stop_queue();
4047 * but since that doesn't exist yet, just open code it. */
4048 smp_mb();
4049
4050 /* We need to check again in a case another CPU has just
4051 * made room available. */
c493ea45 4052 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4053 return -EBUSY;
4054
4055 /* A reprieve! */
661086df 4056 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 4057 tx_ring->tx_stats.restart_queue++;
9d5c8243
AK		 4058 return 0;
4059}
4060
717ba089 4061static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4062{
c493ea45 4063 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4064 return 0;
e694e964 4065 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4066}
4067
b1a436c3
AD		 4068netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4069 struct igb_ring *tx_ring)
9d5c8243 4070{
e694e964 4071 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
cdfd01fc 4072 int tso = 0, count;
91d4ee33
NN		 4073 u32 tx_flags = 0;
4074 u16 first;
4075 u8 hdr_len = 0;
c5b9bd5e 4076 union skb_shared_tx *shtx = skb_tx(skb);
9d5c8243 4077
9d5c8243
AK		 4078 /* need: 1 descriptor per page,
4079 * + 2 desc gap to keep tail from touching head,
4080 * + 1 desc for skb->data,
4081 * + 1 desc for context descriptor,
4082 * otherwise try next time */
e694e964 4083 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4084 /* this is a hard error */
9d5c8243
AK		 4085 return NETDEV_TX_BUSY;
4086 }
33af6bcc 4087
33af6bcc
PO		 4088 if (unlikely(shtx->hardware)) {
4089 shtx->in_progress = 1;
4090 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4091 }
9d5c8243 4092
cdfd01fc 4093 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
9d5c8243
AK		 4094 tx_flags |= IGB_TX_FLAGS_VLAN;
4095 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4096 }
4097
661086df
PWJ		 4098 if (skb->protocol == htons(ETH_P_IP))
4099 tx_flags |= IGB_TX_FLAGS_IPV4;
4100
0e014cb1 4101 first = tx_ring->next_to_use;
85ad76b2
AD		 4102 if (skb_is_gso(skb)) {
4103 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
cdfd01fc 4104
85ad76b2
AD		 4105 if (tso < 0) {
4106 dev_kfree_skb_any(skb);
4107 return NETDEV_TX_OK;
4108 }
9d5c8243
AK		 4109 }
4110
4111 if (tso)
4112 tx_flags |= IGB_TX_FLAGS_TSO;
85ad76b2 4113 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
bc1cbd34
AD		 4114 (skb->ip_summed == CHECKSUM_PARTIAL))
4115 tx_flags |= IGB_TX_FLAGS_CSUM;
9d5c8243 4116
65689fef 4117 /*
cdfd01fc 4118 * count reflects descriptors mapped, if 0 or less then mapping error
65689fef
AD		 4119 * has occured and we need to rewind the descriptor queue
4120 */
80785298 4121 count = igb_tx_map_adv(tx_ring, skb, first);
6366ad33 4122 if (!count) {
65689fef
AD		 4123 dev_kfree_skb_any(skb);
4124 tx_ring->buffer_info[first].time_stamp = 0;
4125 tx_ring->next_to_use = first;
85ad76b2 4126 return NETDEV_TX_OK;
65689fef 4127 }
9d5c8243 4128
85ad76b2
AD		 4129 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4130
4131 /* Make sure there is space in the ring for the next send. */
e694e964 4132 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4133
9d5c8243
AK		 4134 return NETDEV_TX_OK;
4135}
4136
3b29a56d
SH		 4137static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4138 struct net_device *netdev)
9d5c8243
AK		 4139{
4140 struct igb_adapter *adapter = netdev_priv(netdev);
661086df 4141 struct igb_ring *tx_ring;
661086df 4142 int r_idx = 0;
b1a436c3
AD		 4143
4144 if (test_bit(__IGB_DOWN, &adapter->state)) {
4145 dev_kfree_skb_any(skb);
4146 return NETDEV_TX_OK;
4147 }
4148
4149 if (skb->len <= 0) {
4150 dev_kfree_skb_any(skb);
4151 return NETDEV_TX_OK;
4152 }
4153
1bfaf07b 4154 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
661086df 4155 tx_ring = adapter->multi_tx_table[r_idx];
9d5c8243
AK		 4156
4157 /* This goes back to the question of how to logically map a tx queue
4158 * to a flow. Right now, performance is impacted slightly negatively
4159 * if using multiple tx queues. If the stack breaks away from a
4160 * single qdisc implementation, we can look at this again. */
e694e964 4161 return igb_xmit_frame_ring_adv(skb, tx_ring);
9d5c8243
AK		 4162}
4163
4164/**
4165 * igb_tx_timeout - Respond to a Tx Hang
4166 * @netdev: network interface device structure
4167 **/
4168static void igb_tx_timeout(struct net_device *netdev)
4169{
4170 struct igb_adapter *adapter = netdev_priv(netdev);
4171 struct e1000_hw *hw = &adapter->hw;
4172
4173 /* Do the reset outside of interrupt context */
4174 adapter->tx_timeout_count++;
f7ba205e 4175
55cac248
AD		 4176 if (hw->mac.type == e1000_82580)
4177 hw->dev_spec._82575.global_device_reset = true;
4178
9d5c8243 4179 schedule_work(&adapter->reset_task);
265de409
AD		 4180 wr32(E1000_EICS,
4181 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4182}
4183
4184static void igb_reset_task(struct work_struct *work)
4185{
4186 struct igb_adapter *adapter;
4187 adapter = container_of(work, struct igb_adapter, reset_task);
4188
c97ec42a
TI		 4189 igb_dump(adapter);
4190 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4191 igb_reinit_locked(adapter);
4192}
4193
4194/**
4195 * igb_get_stats - Get System Network Statistics
4196 * @netdev: network interface device structure
4197 *
4198 * Returns the address of the device statistics structure.
4199 * The statistics are actually updated from the timer callback.
4200 **/
73cd78f1 4201static struct net_device_stats *igb_get_stats(struct net_device *netdev)
9d5c8243 4202{
9d5c8243 4203 /* only return the current stats */
8d24e933 4204 return &netdev->stats;
9d5c8243
AK		 4205}
4206
4207/**
4208 * igb_change_mtu - Change the Maximum Transfer Unit
4209 * @netdev: network interface device structure
4210 * @new_mtu: new value for maximum frame size
4211 *
4212 * Returns 0 on success, negative on failure
4213 **/
4214static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4215{
4216 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4217 struct pci_dev *pdev = adapter->pdev;
9d5c8243 4218 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4c844851 4219 u32 rx_buffer_len, i;
9d5c8243 4220
c809d227 4221 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4222 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4223 return -EINVAL;
4224 }
4225
9d5c8243 4226 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4227 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4228 return -EINVAL;
4229 }
4230
4231 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4232 msleep(1);
73cd78f1 4233
9d5c8243
AK		 4234 /* igb_down has a dependency on max_frame_size */
4235 adapter->max_frame_size = max_frame;
559e9c49 4236
9d5c8243
AK		 4237 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4238 * means we reserve 2 more, this pushes us to allocate from the next
4239 * larger slab size.
4240 * i.e. RXBUFFER_2048 --> size-4096 slab
4241 */
4242
757b77e2
NN		 4243 if (adapter->hw.mac.type == e1000_82580)
4244 max_frame += IGB_TS_HDR_LEN;
4245
7d95b717 4246 if (max_frame <= IGB_RXBUFFER_1024)
4c844851 4247 rx_buffer_len = IGB_RXBUFFER_1024;
6ec43fe6 4248 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4c844851 4249 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
6ec43fe6 4250 else
4c844851
AD		 4251 rx_buffer_len = IGB_RXBUFFER_128;
4252
757b77e2
NN		 4253 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4254 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4255 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4256
4257 if ((adapter->hw.mac.type == e1000_82580) &&
4258 (rx_buffer_len == IGB_RXBUFFER_128))
4259 rx_buffer_len += IGB_RXBUFFER_64;
4260
4c844851
AD		 4261 if (netif_running(netdev))
4262 igb_down(adapter);
9d5c8243 4263
090b1795 4264 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4265 netdev->mtu, new_mtu);
4266 netdev->mtu = new_mtu;
4267
4c844851 4268 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 4269 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4c844851 4270
9d5c8243
AK		 4271 if (netif_running(netdev))
4272 igb_up(adapter);
4273 else
4274 igb_reset(adapter);
4275
4276 clear_bit(__IGB_RESETTING, &adapter->state);
4277
4278 return 0;
4279}
4280
4281/**
4282 * igb_update_stats - Update the board statistics counters
4283 * @adapter: board private structure
4284 **/
4285
4286void igb_update_stats(struct igb_adapter *adapter)
4287{
128e45eb 4288 struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
9d5c8243
AK		 4289 struct e1000_hw *hw = &adapter->hw;
4290 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4291 u32 reg, mpc;
9d5c8243 4292 u16 phy_tmp;
3f9c0164
AD		 4293 int i;
4294 u64 bytes, packets;
9d5c8243
AK		 4295
4296#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4297
4298 /*
4299 * Prevent stats update while adapter is being reset, or if the pci
4300 * connection is down.
4301 */
4302 if (adapter->link_speed == 0)
4303 return;
4304 if (pci_channel_offline(pdev))
4305 return;
4306
3f9c0164
AD		 4307 bytes = 0;
4308 packets = 0;
4309 for (i = 0; i < adapter->num_rx_queues; i++) {
4310 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446
AD		 4311 struct igb_ring *ring = adapter->rx_ring[i];
4312 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4313 net_stats->rx_fifo_errors += rqdpc_tmp;
3025a446
AD		 4314 bytes += ring->rx_stats.bytes;
4315 packets += ring->rx_stats.packets;
3f9c0164
AD		 4316 }
4317
128e45eb
AD		 4318 net_stats->rx_bytes = bytes;
4319 net_stats->rx_packets = packets;
3f9c0164
AD		 4320
4321 bytes = 0;
4322 packets = 0;
4323 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446
AD		 4324 struct igb_ring *ring = adapter->tx_ring[i];
4325 bytes += ring->tx_stats.bytes;
4326 packets += ring->tx_stats.packets;
3f9c0164 4327 }
128e45eb
AD		 4328 net_stats->tx_bytes = bytes;
4329 net_stats->tx_packets = packets;
3f9c0164
AD		 4330
4331 /* read stats registers */
9d5c8243
AK		 4332 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4333 adapter->stats.gprc += rd32(E1000_GPRC);
4334 adapter->stats.gorc += rd32(E1000_GORCL);
4335 rd32(E1000_GORCH); /* clear GORCL */
4336 adapter->stats.bprc += rd32(E1000_BPRC);
4337 adapter->stats.mprc += rd32(E1000_MPRC);
4338 adapter->stats.roc += rd32(E1000_ROC);
4339
4340 adapter->stats.prc64 += rd32(E1000_PRC64);
4341 adapter->stats.prc127 += rd32(E1000_PRC127);
4342 adapter->stats.prc255 += rd32(E1000_PRC255);
4343 adapter->stats.prc511 += rd32(E1000_PRC511);
4344 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4345 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4346 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4347 adapter->stats.sec += rd32(E1000_SEC);
4348
fa3d9a6d
MW		 4349 mpc = rd32(E1000_MPC);
4350 adapter->stats.mpc += mpc;
4351 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4352 adapter->stats.scc += rd32(E1000_SCC);
4353 adapter->stats.ecol += rd32(E1000_ECOL);
4354 adapter->stats.mcc += rd32(E1000_MCC);
4355 adapter->stats.latecol += rd32(E1000_LATECOL);
4356 adapter->stats.dc += rd32(E1000_DC);
4357 adapter->stats.rlec += rd32(E1000_RLEC);
4358 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4359 adapter->stats.xontxc += rd32(E1000_XONTXC);
4360 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4361 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4362 adapter->stats.fcruc += rd32(E1000_FCRUC);
4363 adapter->stats.gptc += rd32(E1000_GPTC);
4364 adapter->stats.gotc += rd32(E1000_GOTCL);
4365 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4366 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4367 adapter->stats.ruc += rd32(E1000_RUC);
4368 adapter->stats.rfc += rd32(E1000_RFC);
4369 adapter->stats.rjc += rd32(E1000_RJC);
4370 adapter->stats.tor += rd32(E1000_TORH);
4371 adapter->stats.tot += rd32(E1000_TOTH);
4372 adapter->stats.tpr += rd32(E1000_TPR);
4373
4374 adapter->stats.ptc64 += rd32(E1000_PTC64);
4375 adapter->stats.ptc127 += rd32(E1000_PTC127);
4376 adapter->stats.ptc255 += rd32(E1000_PTC255);
4377 adapter->stats.ptc511 += rd32(E1000_PTC511);
4378 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4379 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4380
4381 adapter->stats.mptc += rd32(E1000_MPTC);
4382 adapter->stats.bptc += rd32(E1000_BPTC);
4383
2d0b0f69
NN		 4384 adapter->stats.tpt += rd32(E1000_TPT);
4385 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4386
4387 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4388 /* read internal phy specific stats */
4389 reg = rd32(E1000_CTRL_EXT);
4390 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4391 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4392 adapter->stats.tncrs += rd32(E1000_TNCRS);
4393 }
4394
9d5c8243
AK		 4395 adapter->stats.tsctc += rd32(E1000_TSCTC);
4396 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4397
4398 adapter->stats.iac += rd32(E1000_IAC);
4399 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4400 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4401 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4402 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4403 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4404 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4405 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4406 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4407
4408 /* Fill out the OS statistics structure */
128e45eb
AD		 4409 net_stats->multicast = adapter->stats.mprc;
4410 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4411
4412 /* Rx Errors */
4413
4414 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4415 * our own version based on RUC and ROC */
128e45eb 4416 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4417 adapter->stats.crcerrs + adapter->stats.algnerrc +
4418 adapter->stats.ruc + adapter->stats.roc +
4419 adapter->stats.cexterr;
128e45eb
AD		 4420 net_stats->rx_length_errors = adapter->stats.ruc +
4421 adapter->stats.roc;
4422 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4423 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4424 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4425
4426 /* Tx Errors */
128e45eb
AD		 4427 net_stats->tx_errors = adapter->stats.ecol +
4428 adapter->stats.latecol;
4429 net_stats->tx_aborted_errors = adapter->stats.ecol;
4430 net_stats->tx_window_errors = adapter->stats.latecol;
4431 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4432
4433 /* Tx Dropped needs to be maintained elsewhere */
4434
4435 /* Phy Stats */
4436 if (hw->phy.media_type == e1000_media_type_copper) {
4437 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4438 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4439 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4440 adapter->phy_stats.idle_errors += phy_tmp;
4441 }
4442 }
4443
4444 /* Management Stats */
4445 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4446 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4447 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4448}
4449
9d5c8243
AK		 4450static irqreturn_t igb_msix_other(int irq, void *data)
4451{
047e0030 4452 struct igb_adapter *adapter = data;
9d5c8243 4453 struct e1000_hw *hw = &adapter->hw;
844290e5 4454 u32 icr = rd32(E1000_ICR);
844290e5 4455 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4456
7f081d40
AD		 4457 if (icr & E1000_ICR_DRSTA)
4458 schedule_work(&adapter->reset_task);
4459
047e0030 4460 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4461 /* HW is reporting DMA is out of sync */
4462 adapter->stats.doosync++;
4463 }
eebbbdba 4464
4ae196df
AD		 4465 /* Check for a mailbox event */
4466 if (icr & E1000_ICR_VMMB)
4467 igb_msg_task(adapter);
4468
4469 if (icr & E1000_ICR_LSC) {
4470 hw->mac.get_link_status = 1;
4471 /* guard against interrupt when we're going down */
4472 if (!test_bit(__IGB_DOWN, &adapter->state))
4473 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4474 }
4475
25568a53
AD		 4476 if (adapter->vfs_allocated_count)
4477 wr32(E1000_IMS, E1000_IMS_LSC |
4478 E1000_IMS_VMMB |
4479 E1000_IMS_DOUTSYNC);
4480 else
4481 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4482 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4483
4484 return IRQ_HANDLED;
4485}
4486
047e0030 4487static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4488{
26b39276 4489 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4490 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4491
047e0030
AD		 4492 if (!q_vector->set_itr)
4493 return;
73cd78f1 4494
047e0030
AD		 4495 if (!itr_val)
4496 itr_val = 0x4;
661086df 4497
26b39276
AD		 4498 if (adapter->hw.mac.type == e1000_82575)
4499 itr_val |= itr_val << 16;
661086df 4500 else
047e0030 4501 itr_val |= 0x8000000;
661086df 4502
047e0030
AD		 4503 writel(itr_val, q_vector->itr_register);
4504 q_vector->set_itr = 0;
6eb5a7f1
AD		 4505}
4506
047e0030 4507static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4508{
047e0030 4509 struct igb_q_vector *q_vector = data;
9d5c8243 4510
047e0030
AD		 4511 /* Write the ITR value calculated from the previous interrupt. */
4512 igb_write_itr(q_vector);
9d5c8243 4513
047e0030 4514 napi_schedule(&q_vector->napi);
844290e5 4515
047e0030 4516 return IRQ_HANDLED;
fe4506b6
JC		 4517}
4518
421e02f0 4519#ifdef CONFIG_IGB_DCA
047e0030 4520static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4521{
047e0030 4522 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4523 struct e1000_hw *hw = &adapter->hw;
4524 int cpu = get_cpu();
fe4506b6 4525
047e0030
AD		 4526 if (q_vector->cpu == cpu)
4527 goto out_no_update;
4528
4529 if (q_vector->tx_ring) {
4530 int q = q_vector->tx_ring->reg_idx;
4531 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4532 if (hw->mac.type == e1000_82575) {
4533 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4534 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4535 } else {
047e0030
AD		 4536 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4537 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4538 E1000_DCA_TXCTRL_CPUID_SHIFT;
4539 }
4540 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4541 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4542 }
4543 if (q_vector->rx_ring) {
4544 int q = q_vector->rx_ring->reg_idx;
4545 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4546 if (hw->mac.type == e1000_82575) {
2d064c06 4547 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4548 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4549 } else {
4550 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4551 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4552 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4553 }
fe4506b6
JC		 4554 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4555 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4556 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4557 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4558 }
047e0030
AD		 4559 q_vector->cpu = cpu;
4560out_no_update:
fe4506b6
JC		 4561 put_cpu();
4562}
4563
4564static void igb_setup_dca(struct igb_adapter *adapter)
4565{
7e0e99ef 4566 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4567 int i;
4568
7dfc16fa 4569 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4570 return;
4571
7e0e99ef
AD		 4572 /* Always use CB2 mode, difference is masked in the CB driver. */
4573 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4574
047e0030 4575 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4576 adapter->q_vector[i]->cpu = -1;
4577 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4578 }
4579}
4580
4581static int __igb_notify_dca(struct device *dev, void *data)
4582{
4583 struct net_device *netdev = dev_get_drvdata(dev);
4584 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4585 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4586 struct e1000_hw *hw = &adapter->hw;
4587 unsigned long event = *(unsigned long *)data;
4588
4589 switch (event) {
4590 case DCA_PROVIDER_ADD:
4591 /* if already enabled, don't do it again */
7dfc16fa 4592 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4593 break;
fe4506b6 4594 if (dca_add_requester(dev) == 0) {
bbd98fe4 4595 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4596 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4597 igb_setup_dca(adapter);
4598 break;
4599 }
4600 /* Fall Through since DCA is disabled. */
4601 case DCA_PROVIDER_REMOVE:
7dfc16fa 4602 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4603 /* without this a class_device is left
047e0030 4604 * hanging around in the sysfs model */
fe4506b6 4605 dca_remove_requester(dev);
090b1795 4606 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4607 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4608 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4609 }
4610 break;
4611 }
bbd98fe4 4612
fe4506b6 4613 return 0;
9d5c8243
AK		 4614}
4615
fe4506b6
JC		 4616static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4617 void *p)
4618{
4619 int ret_val;
4620
4621 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4622 __igb_notify_dca);
4623
4624 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4625}
421e02f0 4626#endif /* CONFIG_IGB_DCA */
9d5c8243 4627
4ae196df
AD		 4628static void igb_ping_all_vfs(struct igb_adapter *adapter)
4629{
4630 struct e1000_hw *hw = &adapter->hw;
4631 u32 ping;
4632 int i;
4633
4634 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4635 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4636 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4637 ping |= E1000_VT_MSGTYPE_CTS;
4638 igb_write_mbx(hw, &ping, 1, i);
4639 }
4640}
4641
7d5753f0
AD		 4642static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4643{
4644 struct e1000_hw *hw = &adapter->hw;
4645 u32 vmolr = rd32(E1000_VMOLR(vf));
4646 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4647
4648 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4649 IGB_VF_FLAG_MULTI_PROMISC);
4650 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4651
4652 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4653 vmolr |= E1000_VMOLR_MPME;
4654 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4655 } else {
4656 /*
4657 * if we have hashes and we are clearing a multicast promisc
4658 * flag we need to write the hashes to the MTA as this step
4659 * was previously skipped
4660 */
4661 if (vf_data->num_vf_mc_hashes > 30) {
4662 vmolr |= E1000_VMOLR_MPME;
4663 } else if (vf_data->num_vf_mc_hashes) {
4664 int j;
4665 vmolr |= E1000_VMOLR_ROMPE;
4666 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4667 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4668 }
4669 }
4670
4671 wr32(E1000_VMOLR(vf), vmolr);
4672
4673 /* there are flags left unprocessed, likely not supported */
4674 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4675 return -EINVAL;
4676
4677 return 0;
4678
4679}
4680
4ae196df
AD		 4681static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4682 u32 *msgbuf, u32 vf)
4683{
4684 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4685 u16 *hash_list = (u16 *)&msgbuf[1];
4686 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4687 int i;
4688
7d5753f0 4689 /* salt away the number of multicast addresses assigned
4ae196df
AD		 4690 * to this VF for later use to restore when the PF multi cast
4691 * list changes
4692 */
4693 vf_data->num_vf_mc_hashes = n;
4694
7d5753f0
AD		 4695 /* only up to 30 hash values supported */
4696 if (n > 30)
4697 n = 30;
4698
4699 /* store the hashes for later use */
4ae196df 4700 for (i = 0; i < n; i++)
a419aef8 4701 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 4702
4703 /* Flush and reset the mta with the new values */
ff41f8dc 4704 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4705
4706 return 0;
4707}
4708
4709static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4710{
4711 struct e1000_hw *hw = &adapter->hw;
4712 struct vf_data_storage *vf_data;
4713 int i, j;
4714
4715 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 4716 u32 vmolr = rd32(E1000_VMOLR(i));
4717 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4718
4ae196df 4719 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 4720
4721 if ((vf_data->num_vf_mc_hashes > 30) ||
4722 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4723 vmolr |= E1000_VMOLR_MPME;
4724 } else if (vf_data->num_vf_mc_hashes) {
4725 vmolr |= E1000_VMOLR_ROMPE;
4726 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4727 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4728 }
4729 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 4730 }
4731}
4732
4733static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4734{
4735 struct e1000_hw *hw = &adapter->hw;
4736 u32 pool_mask, reg, vid;
4737 int i;
4738
4739 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4740
4741 /* Find the vlan filter for this id */
4742 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4743 reg = rd32(E1000_VLVF(i));
4744
4745 /* remove the vf from the pool */
4746 reg &= ~pool_mask;
4747
4748 /* if pool is empty then remove entry from vfta */
4749 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4750 (reg & E1000_VLVF_VLANID_ENABLE)) {
4751 reg = 0;
4752 vid = reg & E1000_VLVF_VLANID_MASK;
4753 igb_vfta_set(hw, vid, false);
4754 }
4755
4756 wr32(E1000_VLVF(i), reg);
4757 }
ae641bdc
AD		 4758
4759 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 4760}
4761
4762static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4763{
4764 struct e1000_hw *hw = &adapter->hw;
4765 u32 reg, i;
4766
51466239
AD		 4767 /* The vlvf table only exists on 82576 hardware and newer */
4768 if (hw->mac.type < e1000_82576)
4769 return -1;
4770
4771 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 4772 if (!adapter->vfs_allocated_count)
4773 return -1;
4774
4775 /* Find the vlan filter for this id */
4776 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4777 reg = rd32(E1000_VLVF(i));
4778 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4779 vid == (reg & E1000_VLVF_VLANID_MASK))
4780 break;
4781 }
4782
4783 if (add) {
4784 if (i == E1000_VLVF_ARRAY_SIZE) {
4785 /* Did not find a matching VLAN ID entry that was
4786 * enabled. Search for a free filter entry, i.e.
4787 * one without the enable bit set
4788 */
4789 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4790 reg = rd32(E1000_VLVF(i));
4791 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4792 break;
4793 }
4794 }
4795 if (i < E1000_VLVF_ARRAY_SIZE) {
4796 /* Found an enabled/available entry */
4797 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4798
4799 /* if !enabled we need to set this up in vfta */
4800 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 4801 /* add VID to filter table */
4802 igb_vfta_set(hw, vid, true);
4ae196df
AD		 4803 reg |= E1000_VLVF_VLANID_ENABLE;
4804 }
cad6d05f
AD		 4805 reg &= ~E1000_VLVF_VLANID_MASK;
4806 reg |= vid;
4ae196df 4807 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4808
4809 /* do not modify RLPML for PF devices */
4810 if (vf >= adapter->vfs_allocated_count)
4811 return 0;
4812
4813 if (!adapter->vf_data[vf].vlans_enabled) {
4814 u32 size;
4815 reg = rd32(E1000_VMOLR(vf));
4816 size = reg & E1000_VMOLR_RLPML_MASK;
4817 size += 4;
4818 reg &= ~E1000_VMOLR_RLPML_MASK;
4819 reg |= size;
4820 wr32(E1000_VMOLR(vf), reg);
4821 }
ae641bdc 4822
51466239 4823 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 4824 return 0;
4825 }
4826 } else {
4827 if (i < E1000_VLVF_ARRAY_SIZE) {
4828 /* remove vf from the pool */
4829 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4830 /* if pool is empty then remove entry from vfta */
4831 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4832 reg = 0;
4833 igb_vfta_set(hw, vid, false);
4834 }
4835 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 4836
4837 /* do not modify RLPML for PF devices */
4838 if (vf >= adapter->vfs_allocated_count)
4839 return 0;
4840
4841 adapter->vf_data[vf].vlans_enabled--;
4842 if (!adapter->vf_data[vf].vlans_enabled) {
4843 u32 size;
4844 reg = rd32(E1000_VMOLR(vf));
4845 size = reg & E1000_VMOLR_RLPML_MASK;
4846 size -= 4;
4847 reg &= ~E1000_VMOLR_RLPML_MASK;
4848 reg |= size;
4849 wr32(E1000_VMOLR(vf), reg);
4850 }
4ae196df
AD		 4851 }
4852 }
8151d294
WM		 4853 return 0;
4854}
4855
4856static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4857{
4858 struct e1000_hw *hw = &adapter->hw;
4859
4860 if (vid)
4861 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4862 else
4863 wr32(E1000_VMVIR(vf), 0);
4864}
4865
4866static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4867 int vf, u16 vlan, u8 qos)
4868{
4869 int err = 0;
4870 struct igb_adapter *adapter = netdev_priv(netdev);
4871
4872 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4873 return -EINVAL;
4874 if (vlan || qos) {
4875 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4876 if (err)
4877 goto out;
4878 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4879 igb_set_vmolr(adapter, vf, !vlan);
4880 adapter->vf_data[vf].pf_vlan = vlan;
4881 adapter->vf_data[vf].pf_qos = qos;
4882 dev_info(&adapter->pdev->dev,
4883 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4884 if (test_bit(__IGB_DOWN, &adapter->state)) {
4885 dev_warn(&adapter->pdev->dev,
4886 "The VF VLAN has been set,"
4887 " but the PF device is not up.\n");
4888 dev_warn(&adapter->pdev->dev,
4889 "Bring the PF device up before"
4890 " attempting to use the VF device.\n");
4891 }
4892 } else {
4893 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4894 false, vf);
4895 igb_set_vmvir(adapter, vlan, vf);
4896 igb_set_vmolr(adapter, vf, true);
4897 adapter->vf_data[vf].pf_vlan = 0;
4898 adapter->vf_data[vf].pf_qos = 0;
4899 }
4900out:
4901 return err;
4ae196df
AD		 4902}
4903
4904static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4905{
4906 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4907 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4908
4909 return igb_vlvf_set(adapter, vid, add, vf);
4910}
4911
f2ca0dbe 4912static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 4913{
8151d294
WM		 4914 /* clear flags */
4915 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
f2ca0dbe 4916 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 4917
4918 /* reset offloads to defaults */
8151d294 4919 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 4920
4921 /* reset vlans for device */
4922 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 4923 if (adapter->vf_data[vf].pf_vlan)
4924 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4925 adapter->vf_data[vf].pf_vlan,
4926 adapter->vf_data[vf].pf_qos);
4927 else
4928 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 4929
4930 /* reset multicast table array for vf */
4931 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4932
4933 /* Flush and reset the mta with the new values */
ff41f8dc 4934 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 4935}
4936
f2ca0dbe
AD		 4937static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4938{
4939 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4940
4941 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 4942 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4943 random_ether_addr(vf_mac);
f2ca0dbe
AD		 4944
4945 /* process remaining reset events */
4946 igb_vf_reset(adapter, vf);
4947}
4948
4949static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 4950{
4951 struct e1000_hw *hw = &adapter->hw;
4952 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 4953 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 4954 u32 reg, msgbuf[3];
4955 u8 *addr = (u8 *)(&msgbuf[1]);
4956
4957 /* process all the same items cleared in a function level reset */
f2ca0dbe 4958 igb_vf_reset(adapter, vf);
4ae196df
AD		 4959
4960 /* set vf mac address */
26ad9178 4961 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 4962
4963 /* enable transmit and receive for vf */
4964 reg = rd32(E1000_VFTE);
4965 wr32(E1000_VFTE, reg | (1 << vf));
4966 reg = rd32(E1000_VFRE);
4967 wr32(E1000_VFRE, reg | (1 << vf));
4968
f2ca0dbe 4969 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4ae196df
AD		 4970
4971 /* reply to reset with ack and vf mac address */
4972 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4973 memcpy(addr, vf_mac, 6);
4974 igb_write_mbx(hw, msgbuf, 3, vf);
4975}
4976
4977static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4978{
de42edde
GR		 4979 /*
4980 * The VF MAC Address is stored in a packed array of bytes
4981 * starting at the second 32 bit word of the msg array
4982 */
f2ca0dbe
AD		 4983 unsigned char *addr = (char *)&msg[1];
4984 int err = -1;
4ae196df 4985
f2ca0dbe
AD		 4986 if (is_valid_ether_addr(addr))
4987 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 4988
f2ca0dbe 4989 return err;
4ae196df
AD		 4990}
4991
4992static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4993{
4994 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 4995 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 4996 u32 msg = E1000_VT_MSGTYPE_NACK;
4997
4998 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 4999 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5000 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5001 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5002 vf_data->last_nack = jiffies;
4ae196df
AD		 5003 }
5004}
5005
f2ca0dbe 5006static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5007{
f2ca0dbe
AD		 5008 struct pci_dev *pdev = adapter->pdev;
5009 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5010 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5011 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5012 s32 retval;
5013
f2ca0dbe 5014 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5015
fef45f4c
AD		 5016 if (retval) {
5017 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5018 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5019 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5020 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5021 return;
5022 goto out;
5023 }
4ae196df
AD		 5024
5025 /* this is a message we already processed, do nothing */
5026 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5027 return;
4ae196df
AD		 5028
5029 /*
5030 * until the vf completes a reset it should not be
5031 * allowed to start any configuration.
5032 */
5033
5034 if (msgbuf[0] == E1000_VF_RESET) {
5035 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5036 return;
4ae196df
AD		 5037 }
5038
f2ca0dbe 5039 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5040 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5041 return;
5042 retval = -1;
5043 goto out;
4ae196df
AD		 5044 }
5045
5046 switch ((msgbuf[0] & 0xFFFF)) {
5047 case E1000_VF_SET_MAC_ADDR:
5048 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5049 break;
7d5753f0
AD		 5050 case E1000_VF_SET_PROMISC:
5051 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5052 break;
4ae196df
AD		 5053 case E1000_VF_SET_MULTICAST:
5054 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5055 break;
5056 case E1000_VF_SET_LPE:
5057 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5058 break;
5059 case E1000_VF_SET_VLAN:
8151d294
WM		 5060 if (adapter->vf_data[vf].pf_vlan)
5061 retval = -1;
5062 else
5063 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5064 break;
5065 default:
090b1795 5066 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5067 retval = -1;
5068 break;
5069 }
5070
fef45f4c
AD		 5071 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5072out:
4ae196df
AD		 5073 /* notify the VF of the results of what it sent us */
5074 if (retval)
5075 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5076 else
5077 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5078
4ae196df 5079 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5080}
4ae196df 5081
f2ca0dbe
AD		 5082static void igb_msg_task(struct igb_adapter *adapter)
5083{
5084 struct e1000_hw *hw = &adapter->hw;
5085 u32 vf;
5086
5087 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5088 /* process any reset requests */
5089 if (!igb_check_for_rst(hw, vf))
5090 igb_vf_reset_event(adapter, vf);
5091
5092 /* process any messages pending */
5093 if (!igb_check_for_msg(hw, vf))
5094 igb_rcv_msg_from_vf(adapter, vf);
5095
5096 /* process any acks */
5097 if (!igb_check_for_ack(hw, vf))
5098 igb_rcv_ack_from_vf(adapter, vf);
5099 }
4ae196df
AD		 5100}
5101
68d480c4
AD		 5102/**
5103 * igb_set_uta - Set unicast filter table address
5104 * @adapter: board private structure
5105 *
5106 * The unicast table address is a register array of 32-bit registers.
5107 * The table is meant to be used in a way similar to how the MTA is used
5108 * however due to certain limitations in the hardware it is necessary to
5109 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5110 * enable bit to allow vlan tag stripping when promiscous mode is enabled
5111 **/
5112static void igb_set_uta(struct igb_adapter *adapter)
5113{
5114 struct e1000_hw *hw = &adapter->hw;
5115 int i;
5116
5117 /* The UTA table only exists on 82576 hardware and newer */
5118 if (hw->mac.type < e1000_82576)
5119 return;
5120
5121 /* we only need to do this if VMDq is enabled */
5122 if (!adapter->vfs_allocated_count)
5123 return;
5124
5125 for (i = 0; i < hw->mac.uta_reg_count; i++)
5126 array_wr32(E1000_UTA, i, ~0);
5127}
5128
9d5c8243
AK		 5129/**
5130 * igb_intr_msi - Interrupt Handler
5131 * @irq: interrupt number
5132 * @data: pointer to a network interface device structure
5133 **/
5134static irqreturn_t igb_intr_msi(int irq, void *data)
5135{
047e0030
AD		 5136 struct igb_adapter *adapter = data;
5137 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5138 struct e1000_hw *hw = &adapter->hw;
5139 /* read ICR disables interrupts using IAM */
5140 u32 icr = rd32(E1000_ICR);
5141
047e0030 5142 igb_write_itr(q_vector);
9d5c8243 5143
7f081d40
AD		 5144 if (icr & E1000_ICR_DRSTA)
5145 schedule_work(&adapter->reset_task);
5146
047e0030 5147 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5148 /* HW is reporting DMA is out of sync */
5149 adapter->stats.doosync++;
5150 }
5151
9d5c8243
AK		 5152 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5153 hw->mac.get_link_status = 1;
5154 if (!test_bit(__IGB_DOWN, &adapter->state))
5155 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5156 }
5157
047e0030 5158 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5159
5160 return IRQ_HANDLED;
5161}
5162
5163/**
4a3c6433 5164 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5165 * @irq: interrupt number
5166 * @data: pointer to a network interface device structure
5167 **/
5168static irqreturn_t igb_intr(int irq, void *data)
5169{
047e0030
AD		 5170 struct igb_adapter *adapter = data;
5171 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5172 struct e1000_hw *hw = &adapter->hw;
5173 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5174 * need for the IMC write */
5175 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5176 if (!icr)
5177 return IRQ_NONE; /* Not our interrupt */
5178
047e0030 5179 igb_write_itr(q_vector);
9d5c8243
AK		 5180
5181 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5182 * not set, then the adapter didn't send an interrupt */
5183 if (!(icr & E1000_ICR_INT_ASSERTED))
5184 return IRQ_NONE;
5185
7f081d40
AD		 5186 if (icr & E1000_ICR_DRSTA)
5187 schedule_work(&adapter->reset_task);
5188
047e0030 5189 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5190 /* HW is reporting DMA is out of sync */
5191 adapter->stats.doosync++;
5192 }
5193
9d5c8243
AK		 5194 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5195 hw->mac.get_link_status = 1;
5196 /* guard against interrupt when we're going down */
5197 if (!test_bit(__IGB_DOWN, &adapter->state))
5198 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5199 }
5200
047e0030 5201 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5202
5203 return IRQ_HANDLED;
5204}
5205
047e0030 5206static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5207{
047e0030 5208 struct igb_adapter *adapter = q_vector->adapter;
46544258 5209 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5210
4fc82adf
AD		 5211 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5212 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 5213 if (!adapter->msix_entries)
6eb5a7f1 5214 igb_set_itr(adapter);
46544258 5215 else
047e0030 5216 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5217 }
5218
46544258
AD		 5219 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5220 if (adapter->msix_entries)
047e0030 5221 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5222 else
5223 igb_irq_enable(adapter);
5224 }
9d5c8243
AK		 5225}
5226
46544258
AD		 5227/**
5228 * igb_poll - NAPI Rx polling callback
5229 * @napi: napi polling structure
5230 * @budget: count of how many packets we should handle
5231 **/
5232static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5233{
047e0030
AD		 5234 struct igb_q_vector *q_vector = container_of(napi,
5235 struct igb_q_vector,
5236 napi);
5237 int tx_clean_complete = 1, work_done = 0;
9d5c8243 5238
421e02f0 5239#ifdef CONFIG_IGB_DCA
047e0030
AD		 5240 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5241 igb_update_dca(q_vector);
fe4506b6 5242#endif
047e0030
AD		 5243 if (q_vector->tx_ring)
5244 tx_clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5245
047e0030
AD		 5246 if (q_vector->rx_ring)
5247 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5248
5249 if (!tx_clean_complete)
5250 work_done = budget;
46544258 5251
9d5c8243 5252 /* If not enough Rx work done, exit the polling mode */
5e6d5b17 5253 if (work_done < budget) {
288379f0 5254 napi_complete(napi);
047e0030 5255 igb_ring_irq_enable(q_vector);
9d5c8243
AK		 5256 }
5257
46544258 5258 return work_done;
9d5c8243 5259}
6d8126f9 5260
33af6bcc 5261/**
c5b9bd5e 5262 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5263 * @adapter: board private structure
c5b9bd5e
AD		 5264 * @shhwtstamps: timestamp structure to update
5265 * @regval: unsigned 64bit system time value.
5266 *
5267 * We need to convert the system time value stored in the RX/TXSTMP registers
5268 * into a hwtstamp which can be used by the upper level timestamping functions
5269 */
5270static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5271 struct skb_shared_hwtstamps *shhwtstamps,
5272 u64 regval)
5273{
5274 u64 ns;
5275
55cac248
AD		 5276 /*
5277 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5278 * 24 to match clock shift we setup earlier.
5279 */
5280 if (adapter->hw.mac.type == e1000_82580)
5281 regval <<= IGB_82580_TSYNC_SHIFT;
5282
c5b9bd5e
AD		 5283 ns = timecounter_cyc2time(&adapter->clock, regval);
5284 timecompare_update(&adapter->compare, ns);
5285 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5286 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5287 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5288}
5289
5290/**
5291 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5292 * @q_vector: pointer to q_vector containing needed info
2873957d 5293 * @buffer: pointer to igb_buffer structure
33af6bcc
PO		 5294 *
5295 * If we were asked to do hardware stamping and such a time stamp is
5296 * available, then it must have been for this skb here because we only
5297 * allow only one such packet into the queue.
5298 */
2873957d 5299static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
33af6bcc 5300{
c5b9bd5e 5301 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5302 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5303 struct skb_shared_hwtstamps shhwtstamps;
5304 u64 regval;
33af6bcc 5305
c5b9bd5e 5306 /* if skb does not support hw timestamp or TX stamp not valid exit */
2873957d 5307 if (likely(!buffer_info->shtx.hardware) ||
c5b9bd5e
AD		 5308 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5309 return;
5310
5311 regval = rd32(E1000_TXSTMPL);
5312 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5313
5314 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5315 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5316}
5317
9d5c8243
AK		 5318/**
5319 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5320 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5321 * returns true if ring is completely cleaned
5322 **/
047e0030 5323static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5324{
047e0030
AD		 5325 struct igb_adapter *adapter = q_vector->adapter;
5326 struct igb_ring *tx_ring = q_vector->tx_ring;
e694e964 5327 struct net_device *netdev = tx_ring->netdev;
0e014cb1 5328 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5329 struct igb_buffer *buffer_info;
0e014cb1 5330 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5331 unsigned int total_bytes = 0, total_packets = 0;
0e014cb1
AD		 5332 unsigned int i, eop, count = 0;
5333 bool cleaned = false;
9d5c8243 5334
9d5c8243 5335 i = tx_ring->next_to_clean;
0e014cb1
AD		 5336 eop = tx_ring->buffer_info[i].next_to_watch;
5337 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5338
5339 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5340 (count < tx_ring->count)) {
5341 for (cleaned = false; !cleaned; count++) {
5342 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
9d5c8243 5343 buffer_info = &tx_ring->buffer_info[i];
0e014cb1 5344 cleaned = (i == eop);
9d5c8243 5345
2873957d
NN		 5346 if (buffer_info->skb) {
5347 total_bytes += buffer_info->bytecount;
9d5c8243 5348 /* gso_segs is currently only valid for tcp */
2873957d
NN		 5349 total_packets += buffer_info->gso_segs;
5350 igb_tx_hwtstamp(q_vector, buffer_info);
9d5c8243
AK		 5351 }
5352
80785298 5353 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
0e014cb1 5354 tx_desc->wb.status = 0;
9d5c8243
AK		 5355
5356 i++;
5357 if (i == tx_ring->count)
5358 i = 0;
9d5c8243 5359 }
0e014cb1
AD		 5360 eop = tx_ring->buffer_info[i].next_to_watch;
5361 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5362 }
5363
9d5c8243
AK		 5364 tx_ring->next_to_clean = i;
5365
fc7d345d 5366 if (unlikely(count &&
9d5c8243 5367 netif_carrier_ok(netdev) &&
c493ea45 5368 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
9d5c8243
AK		 5369 /* Make sure that anybody stopping the queue after this
5370 * sees the new next_to_clean.
5371 */
5372 smp_mb();
661086df
PWJ		 5373 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5374 !(test_bit(__IGB_DOWN, &adapter->state))) {
5375 netif_wake_subqueue(netdev, tx_ring->queue_index);
04a5fcaa 5376 tx_ring->tx_stats.restart_queue++;
661086df 5377 }
9d5c8243
AK		 5378 }
5379
5380 if (tx_ring->detect_tx_hung) {
5381 /* Detect a transmit hang in hardware, this serializes the
5382 * check with the clearing of time_stamp and movement of i */
5383 tx_ring->detect_tx_hung = false;
5384 if (tx_ring->buffer_info[i].time_stamp &&
5385 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
8e95a202
JP		 5386 (adapter->tx_timeout_factor * HZ)) &&
5387 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5388
9d5c8243 5389 /* detected Tx unit hang */
59d71989 5390 dev_err(tx_ring->dev,
9d5c8243 5391 "Detected Tx Unit Hang\n"
2d064c06 5392 " Tx Queue <%d>\n"
9d5c8243
AK		 5393 " TDH <%x>\n"
5394 " TDT <%x>\n"
5395 " next_to_use <%x>\n"
5396 " next_to_clean <%x>\n"
9d5c8243
AK		 5397 "buffer_info[next_to_clean]\n"
5398 " time_stamp <%lx>\n"
0e014cb1 5399 " next_to_watch <%x>\n"
9d5c8243
AK		 5400 " jiffies <%lx>\n"
5401 " desc.status <%x>\n",
2d064c06 5402 tx_ring->queue_index,
fce99e34
AD		 5403 readl(tx_ring->head),
5404 readl(tx_ring->tail),
9d5c8243
AK		 5405 tx_ring->next_to_use,
5406 tx_ring->next_to_clean,
f7ba205e 5407 tx_ring->buffer_info[eop].time_stamp,
0e014cb1 5408 eop,
9d5c8243 5409 jiffies,
0e014cb1 5410 eop_desc->wb.status);
661086df 5411 netif_stop_subqueue(netdev, tx_ring->queue_index);
9d5c8243
AK		 5412 }
5413 }
5414 tx_ring->total_bytes += total_bytes;
5415 tx_ring->total_packets += total_packets;
e21ed353
AD		 5416 tx_ring->tx_stats.bytes += total_bytes;
5417 tx_ring->tx_stats.packets += total_packets;
0e014cb1 5418 return (count < tx_ring->count);
9d5c8243
AK		 5419}
5420
9d5c8243
AK		 5421/**
5422 * igb_receive_skb - helper function to handle rx indications
047e0030
AD		 5423 * @q_vector: structure containing interrupt and ring information
5424 * @skb: packet to send up
5425 * @vlan_tag: vlan tag for packet
9d5c8243 5426 **/
047e0030
AD		 5427static void igb_receive_skb(struct igb_q_vector *q_vector,
5428 struct sk_buff *skb,
5429 u16 vlan_tag)
5430{
5431 struct igb_adapter *adapter = q_vector->adapter;
5432
31b24b95 5433 if (vlan_tag && adapter->vlgrp)
047e0030
AD		 5434 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5435 vlan_tag, skb);
182ff8df 5436 else
047e0030 5437 napi_gro_receive(&q_vector->napi, skb);
9d5c8243
AK		 5438}
5439
04a5fcaa 5440static inline void igb_rx_checksum_adv(struct igb_ring *ring,
9d5c8243
AK		 5441 u32 status_err, struct sk_buff *skb)
5442{
5443 skb->ip_summed = CHECKSUM_NONE;
5444
5445 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5446 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5447 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5448 return;
85ad76b2 5449
9d5c8243
AK		 5450 /* TCP/UDP checksum error bit is set */
5451 if (status_err &
5452 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5453 /*
5454 * work around errata with sctp packets where the TCPE aka
5455 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5456 * packets, (aka let the stack check the crc32c)
5457 */
85ad76b2
AD		 5458 if ((skb->len == 60) &&
5459 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
04a5fcaa 5460 ring->rx_stats.csum_err++;
85ad76b2 5461
9d5c8243 5462 /* let the stack verify checksum errors */
9d5c8243
AK		 5463 return;
5464 }
5465 /* It must be a TCP or UDP packet with a valid checksum */
5466 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5467 skb->ip_summed = CHECKSUM_UNNECESSARY;
5468
59d71989 5469 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5470}
5471
757b77e2 5472static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
c5b9bd5e
AD		 5473 struct sk_buff *skb)
5474{
5475 struct igb_adapter *adapter = q_vector->adapter;
5476 struct e1000_hw *hw = &adapter->hw;
5477 u64 regval;
5478
5479 /*
5480 * If this bit is set, then the RX registers contain the time stamp. No
5481 * other packet will be time stamped until we read these registers, so
5482 * read the registers to make them available again. Because only one
5483 * packet can be time stamped at a time, we know that the register
5484 * values must belong to this one here and therefore we don't need to
5485 * compare any of the additional attributes stored for it.
5486 *
5487 * If nothing went wrong, then it should have a skb_shared_tx that we
5488 * can turn into a skb_shared_hwtstamps.
5489 */
757b77e2
NN		 5490 if (staterr & E1000_RXDADV_STAT_TSIP) {
5491 u32 *stamp = (u32 *)skb->data;
5492 regval = le32_to_cpu(*(stamp + 2));
5493 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5494 skb_pull(skb, IGB_TS_HDR_LEN);
5495 } else {
5496 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5497 return;
c5b9bd5e 5498
757b77e2
NN		 5499 regval = rd32(E1000_RXSTMPL);
5500 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5501 }
c5b9bd5e
AD		 5502
5503 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5504}
4c844851 5505static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
2d94d8ab
AD		 5506 union e1000_adv_rx_desc *rx_desc)
5507{
5508 /* HW will not DMA in data larger than the given buffer, even if it
5509 * parses the (NFS, of course) header to be larger. In that case, it
5510 * fills the header buffer and spills the rest into the page.
5511 */
5512 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5513 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
4c844851
AD		 5514 if (hlen > rx_ring->rx_buffer_len)
5515 hlen = rx_ring->rx_buffer_len;
2d94d8ab
AD		 5516 return hlen;
5517}
5518
047e0030
AD		 5519static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5520 int *work_done, int budget)
9d5c8243 5521{
047e0030 5522 struct igb_ring *rx_ring = q_vector->rx_ring;
e694e964 5523 struct net_device *netdev = rx_ring->netdev;
59d71989 5524 struct device *dev = rx_ring->dev;
9d5c8243
AK		 5525 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5526 struct igb_buffer *buffer_info , *next_buffer;
5527 struct sk_buff *skb;
9d5c8243
AK		 5528 bool cleaned = false;
5529 int cleaned_count = 0;
d1eff350 5530 int current_node = numa_node_id();
9d5c8243 5531 unsigned int total_bytes = 0, total_packets = 0;
73cd78f1 5532 unsigned int i;
2d94d8ab
AD		 5533 u32 staterr;
5534 u16 length;
047e0030 5535 u16 vlan_tag;
9d5c8243
AK		 5536
5537 i = rx_ring->next_to_clean;
69d3ca53 5538 buffer_info = &rx_ring->buffer_info[i];
9d5c8243
AK		 5539 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5540 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5541
5542 while (staterr & E1000_RXD_STAT_DD) {
5543 if (*work_done >= budget)
5544 break;
5545 (*work_done)++;
9d5c8243 5546
69d3ca53
AD		 5547 skb = buffer_info->skb;
5548 prefetch(skb->data - NET_IP_ALIGN);
5549 buffer_info->skb = NULL;
5550
5551 i++;
5552 if (i == rx_ring->count)
5553 i = 0;
42d0781a 5554
69d3ca53
AD		 5555 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5556 prefetch(next_rxd);
5557 next_buffer = &rx_ring->buffer_info[i];
9d5c8243
AK		 5558
5559 length = le16_to_cpu(rx_desc->wb.upper.length);
5560 cleaned = true;
5561 cleaned_count++;
5562
2d94d8ab 5563 if (buffer_info->dma) {
59d71989 5564 dma_unmap_single(dev, buffer_info->dma,
4c844851 5565 rx_ring->rx_buffer_len,
59d71989 5566 DMA_FROM_DEVICE);
91615f76 5567 buffer_info->dma = 0;
4c844851 5568 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
6ec43fe6
AD		 5569 skb_put(skb, length);
5570 goto send_up;
5571 }
4c844851 5572 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
bf36c1a0
AD		 5573 }
5574
5575 if (length) {
59d71989
AD		 5576 dma_unmap_page(dev, buffer_info->page_dma,
5577 PAGE_SIZE / 2, DMA_FROM_DEVICE);
9d5c8243 5578 buffer_info->page_dma = 0;
bf36c1a0 5579
aa913403 5580 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 5581 buffer_info->page,
5582 buffer_info->page_offset,
5583 length);
5584
d1eff350
AD		 5585 if ((page_count(buffer_info->page) != 1) ||
5586 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5587 buffer_info->page = NULL;
5588 else
5589 get_page(buffer_info->page);
9d5c8243
AK		 5590
5591 skb->len += length;
5592 skb->data_len += length;
bf36c1a0 5593 skb->truesize += length;
9d5c8243 5594 }
9d5c8243 5595
bf36c1a0 5596 if (!(staterr & E1000_RXD_STAT_EOP)) {
b2d56536
AD		 5597 buffer_info->skb = next_buffer->skb;
5598 buffer_info->dma = next_buffer->dma;
5599 next_buffer->skb = skb;
5600 next_buffer->dma = 0;
bf36c1a0
AD		 5601 goto next_desc;
5602 }
69d3ca53 5603send_up:
9d5c8243
AK		 5604 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5605 dev_kfree_skb_irq(skb);
5606 goto next_desc;
5607 }
9d5c8243 5608
757b77e2
NN		 5609 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5610 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5611 total_bytes += skb->len;
5612 total_packets++;
5613
85ad76b2 5614 igb_rx_checksum_adv(rx_ring, staterr, skb);
9d5c8243
AK		 5615
5616 skb->protocol = eth_type_trans(skb, netdev);
047e0030
AD		 5617 skb_record_rx_queue(skb, rx_ring->queue_index);
5618
5619 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5620 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
9d5c8243 5621
047e0030 5622 igb_receive_skb(q_vector, skb, vlan_tag);
9d5c8243 5623
9d5c8243
AK		 5624next_desc:
5625 rx_desc->wb.upper.status_error = 0;
5626
5627 /* return some buffers to hardware, one at a time is too slow */
5628 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
3b644cf6 5629 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5630 cleaned_count = 0;
5631 }
5632
5633 /* use prefetched values */
5634 rx_desc = next_rxd;
5635 buffer_info = next_buffer;
9d5c8243
AK		 5636 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5637 }
bf36c1a0 5638
9d5c8243 5639 rx_ring->next_to_clean = i;
c493ea45 5640 cleaned_count = igb_desc_unused(rx_ring);
9d5c8243
AK		 5641
5642 if (cleaned_count)
3b644cf6 5643 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
9d5c8243
AK		 5644
5645 rx_ring->total_packets += total_packets;
5646 rx_ring->total_bytes += total_bytes;
5647 rx_ring->rx_stats.packets += total_packets;
5648 rx_ring->rx_stats.bytes += total_bytes;
9d5c8243
AK		 5649 return cleaned;
5650}
5651
9d5c8243
AK		 5652/**
5653 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5654 * @adapter: address of board private structure
5655 **/
d7ee5b3a 5656void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
9d5c8243 5657{
e694e964 5658 struct net_device *netdev = rx_ring->netdev;
9d5c8243
AK		 5659 union e1000_adv_rx_desc *rx_desc;
5660 struct igb_buffer *buffer_info;
5661 struct sk_buff *skb;
5662 unsigned int i;
db761762 5663 int bufsz;
9d5c8243
AK		 5664
5665 i = rx_ring->next_to_use;
5666 buffer_info = &rx_ring->buffer_info[i];
5667
4c844851 5668 bufsz = rx_ring->rx_buffer_len;
db761762 5669
9d5c8243
AK		 5670 while (cleaned_count--) {
5671 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5672
6ec43fe6 5673 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
9d5c8243 5674 if (!buffer_info->page) {
42d0781a 5675 buffer_info->page = netdev_alloc_page(netdev);
bf36c1a0 5676 if (!buffer_info->page) {
04a5fcaa 5677 rx_ring->rx_stats.alloc_failed++;
bf36c1a0
AD		 5678 goto no_buffers;
5679 }
5680 buffer_info->page_offset = 0;
5681 } else {
5682 buffer_info->page_offset ^= PAGE_SIZE / 2;
9d5c8243
AK		 5683 }
5684 buffer_info->page_dma =
59d71989 5685 dma_map_page(rx_ring->dev, buffer_info->page,
bf36c1a0
AD		 5686 buffer_info->page_offset,
5687 PAGE_SIZE / 2,
59d71989
AD		 5688 DMA_FROM_DEVICE);
5689 if (dma_mapping_error(rx_ring->dev,
5690 buffer_info->page_dma)) {
42d0781a
AD		 5691 buffer_info->page_dma = 0;
5692 rx_ring->rx_stats.alloc_failed++;
5693 goto no_buffers;
5694 }
9d5c8243
AK		 5695 }
5696
42d0781a
AD		 5697 skb = buffer_info->skb;
5698 if (!skb) {
89d71a66 5699 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
9d5c8243 5700 if (!skb) {
04a5fcaa 5701 rx_ring->rx_stats.alloc_failed++;
9d5c8243
AK		 5702 goto no_buffers;
5703 }
5704
9d5c8243 5705 buffer_info->skb = skb;
42d0781a
AD		 5706 }
5707 if (!buffer_info->dma) {
59d71989 5708 buffer_info->dma = dma_map_single(rx_ring->dev,
80785298 5709 skb->data,
9d5c8243 5710 bufsz,
59d71989
AD		 5711 DMA_FROM_DEVICE);
5712 if (dma_mapping_error(rx_ring->dev,
5713 buffer_info->dma)) {
42d0781a
AD		 5714 buffer_info->dma = 0;
5715 rx_ring->rx_stats.alloc_failed++;
5716 goto no_buffers;
5717 }
9d5c8243
AK		 5718 }
5719 /* Refresh the desc even if buffer_addrs didn't change because
5720 * each write-back erases this info. */
6ec43fe6 5721 if (bufsz < IGB_RXBUFFER_1024) {
9d5c8243
AK		 5722 rx_desc->read.pkt_addr =
5723 cpu_to_le64(buffer_info->page_dma);
5724 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5725 } else {
42d0781a 5726 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
9d5c8243
AK		 5727 rx_desc->read.hdr_addr = 0;
5728 }
5729
5730 i++;
5731 if (i == rx_ring->count)
5732 i = 0;
5733 buffer_info = &rx_ring->buffer_info[i];
5734 }
5735
5736no_buffers:
5737 if (rx_ring->next_to_use != i) {
5738 rx_ring->next_to_use = i;
5739 if (i == 0)
5740 i = (rx_ring->count - 1);
5741 else
5742 i--;
5743
5744 /* Force memory writes to complete before letting h/w
5745 * know there are new descriptors to fetch. (Only
5746 * applicable for weak-ordered memory model archs,
5747 * such as IA-64). */
5748 wmb();
fce99e34 5749 writel(i, rx_ring->tail);
9d5c8243
AK		 5750 }
5751}
5752
5753/**
5754 * igb_mii_ioctl -
5755 * @netdev:
5756 * @ifreq:
5757 * @cmd:
5758 **/
5759static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5760{
5761 struct igb_adapter *adapter = netdev_priv(netdev);
5762 struct mii_ioctl_data *data = if_mii(ifr);
5763
5764 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5765 return -EOPNOTSUPP;
5766
5767 switch (cmd) {
5768 case SIOCGMIIPHY:
5769 data->phy_id = adapter->hw.phy.addr;
5770 break;
5771 case SIOCGMIIREG:
f5f4cf08
AD		 5772 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5773 &data->val_out))
9d5c8243
AK		 5774 return -EIO;
5775 break;
5776 case SIOCSMIIREG:
5777 default:
5778 return -EOPNOTSUPP;
5779 }
5780 return 0;
5781}
5782
c6cb090b
PO		 5783/**
5784 * igb_hwtstamp_ioctl - control hardware time stamping
5785 * @netdev:
5786 * @ifreq:
5787 * @cmd:
5788 *
33af6bcc
PO		 5789 * Outgoing time stamping can be enabled and disabled. Play nice and
5790 * disable it when requested, although it shouldn't case any overhead
5791 * when no packet needs it. At most one packet in the queue may be
5792 * marked for time stamping, otherwise it would be impossible to tell
5793 * for sure to which packet the hardware time stamp belongs.
5794 *
5795 * Incoming time stamping has to be configured via the hardware
5796 * filters. Not all combinations are supported, in particular event
5797 * type has to be specified. Matching the kind of event packet is
5798 * not supported, with the exception of "all V2 events regardless of
5799 * level 2 or 4".
5800 *
c6cb090b
PO		 5801 **/
5802static int igb_hwtstamp_ioctl(struct net_device *netdev,
5803 struct ifreq *ifr, int cmd)
5804{
33af6bcc
PO		 5805 struct igb_adapter *adapter = netdev_priv(netdev);
5806 struct e1000_hw *hw = &adapter->hw;
c6cb090b 5807 struct hwtstamp_config config;
c5b9bd5e
AD		 5808 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5809 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 5810 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 5811 bool is_l4 = false;
5812 bool is_l2 = false;
33af6bcc 5813 u32 regval;
c6cb090b
PO		 5814
5815 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5816 return -EFAULT;
5817
5818 /* reserved for future extensions */
5819 if (config.flags)
5820 return -EINVAL;
5821
33af6bcc
PO		 5822 switch (config.tx_type) {
5823 case HWTSTAMP_TX_OFF:
c5b9bd5e 5824 tsync_tx_ctl = 0;
33af6bcc 5825 case HWTSTAMP_TX_ON:
33af6bcc
PO		 5826 break;
5827 default:
5828 return -ERANGE;
5829 }
5830
5831 switch (config.rx_filter) {
5832 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 5833 tsync_rx_ctl = 0;
33af6bcc
PO		 5834 break;
5835 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5836 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5837 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5838 case HWTSTAMP_FILTER_ALL:
5839 /*
5840 * register TSYNCRXCFG must be set, therefore it is not
5841 * possible to time stamp both Sync and Delay_Req messages
5842 * => fall back to time stamping all packets
5843 */
c5b9bd5e 5844 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 5845 config.rx_filter = HWTSTAMP_FILTER_ALL;
5846 break;
5847 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 5848 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5849 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 5850 is_l4 = true;
33af6bcc
PO		 5851 break;
5852 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 5853 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 5854 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 5855 is_l4 = true;
33af6bcc
PO		 5856 break;
5857 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5858 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 5859 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5860 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 5861 is_l2 = true;
5862 is_l4 = true;
33af6bcc
PO		 5863 config.rx_filter = HWTSTAMP_FILTER_SOME;
5864 break;
5865 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5866 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 5867 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 5868 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 5869 is_l2 = true;
5870 is_l4 = true;
33af6bcc
PO		 5871 config.rx_filter = HWTSTAMP_FILTER_SOME;
5872 break;
5873 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5874 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5875 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 5876 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 5877 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 5878 is_l2 = true;
33af6bcc
PO		 5879 break;
5880 default:
5881 return -ERANGE;
5882 }
5883
c5b9bd5e
AD		 5884 if (hw->mac.type == e1000_82575) {
5885 if (tsync_rx_ctl | tsync_tx_ctl)
5886 return -EINVAL;
5887 return 0;
5888 }
5889
757b77e2
NN		 5890 /*
5891 * Per-packet timestamping only works if all packets are
5892 * timestamped, so enable timestamping in all packets as
5893 * long as one rx filter was configured.
5894 */
5895 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5896 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5897 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5898 }
5899
33af6bcc
PO		 5900 /* enable/disable TX */
5901 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 5902 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5903 regval |= tsync_tx_ctl;
33af6bcc
PO		 5904 wr32(E1000_TSYNCTXCTL, regval);
5905
c5b9bd5e 5906 /* enable/disable RX */
33af6bcc 5907 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 5908 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5909 regval |= tsync_rx_ctl;
33af6bcc 5910 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 5911
c5b9bd5e
AD		 5912 /* define which PTP packets are time stamped */
5913 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 5914
c5b9bd5e
AD		 5915 /* define ethertype filter for timestamped packets */
5916 if (is_l2)
5917 wr32(E1000_ETQF(3),
5918 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5919 E1000_ETQF_1588 | /* enable timestamping */
5920 ETH_P_1588)); /* 1588 eth protocol type */
5921 else
5922 wr32(E1000_ETQF(3), 0);
5923
5924#define PTP_PORT 319
5925 /* L4 Queue Filter[3]: filter by destination port and protocol */
5926 if (is_l4) {
5927 u32 ftqf = (IPPROTO_UDP /* UDP */
5928 | E1000_FTQF_VF_BP /* VF not compared */
5929 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5930 | E1000_FTQF_MASK); /* mask all inputs */
5931 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5932
5933 wr32(E1000_IMIR(3), htons(PTP_PORT));
5934 wr32(E1000_IMIREXT(3),
5935 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5936 if (hw->mac.type == e1000_82576) {
5937 /* enable source port check */
5938 wr32(E1000_SPQF(3), htons(PTP_PORT));
5939 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5940 }
5941 wr32(E1000_FTQF(3), ftqf);
5942 } else {
5943 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5944 }
33af6bcc
PO		 5945 wrfl();
5946
5947 adapter->hwtstamp_config = config;
5948
5949 /* clear TX/RX time stamp registers, just to be sure */
5950 regval = rd32(E1000_TXSTMPH);
5951 regval = rd32(E1000_RXSTMPH);
c6cb090b 5952
33af6bcc
PO		 5953 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5954 -EFAULT : 0;
c6cb090b
PO		 5955}
5956
9d5c8243
AK		 5957/**
5958 * igb_ioctl -
5959 * @netdev:
5960 * @ifreq:
5961 * @cmd:
5962 **/
5963static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5964{
5965 switch (cmd) {
5966 case SIOCGMIIPHY:
5967 case SIOCGMIIREG:
5968 case SIOCSMIIREG:
5969 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 5970 case SIOCSHWTSTAMP:
5971 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 5972 default:
5973 return -EOPNOTSUPP;
5974 }
5975}
5976
009bc06e
AD		 5977s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5978{
5979 struct igb_adapter *adapter = hw->back;
5980 u16 cap_offset;
5981
5982 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5983 if (!cap_offset)
5984 return -E1000_ERR_CONFIG;
5985
5986 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5987
5988 return 0;
5989}
5990
5991s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5992{
5993 struct igb_adapter *adapter = hw->back;
5994 u16 cap_offset;
5995
5996 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5997 if (!cap_offset)
5998 return -E1000_ERR_CONFIG;
5999
6000 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6001
6002 return 0;
6003}
6004
9d5c8243
AK		 6005static void igb_vlan_rx_register(struct net_device *netdev,
6006 struct vlan_group *grp)
6007{
6008 struct igb_adapter *adapter = netdev_priv(netdev);
6009 struct e1000_hw *hw = &adapter->hw;
6010 u32 ctrl, rctl;
6011
6012 igb_irq_disable(adapter);
6013 adapter->vlgrp = grp;
6014
6015 if (grp) {
6016 /* enable VLAN tag insert/strip */
6017 ctrl = rd32(E1000_CTRL);
6018 ctrl |= E1000_CTRL_VME;
6019 wr32(E1000_CTRL, ctrl);
6020
51466239 6021 /* Disable CFI check */
9d5c8243 6022 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6023 rctl &= ~E1000_RCTL_CFIEN;
6024 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6025 } else {
6026 /* disable VLAN tag insert/strip */
6027 ctrl = rd32(E1000_CTRL);
6028 ctrl &= ~E1000_CTRL_VME;
6029 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6030 }
6031
e1739522
AD		 6032 igb_rlpml_set(adapter);
6033
9d5c8243
AK		 6034 if (!test_bit(__IGB_DOWN, &adapter->state))
6035 igb_irq_enable(adapter);
6036}
6037
6038static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6039{
6040 struct igb_adapter *adapter = netdev_priv(netdev);
6041 struct e1000_hw *hw = &adapter->hw;
4ae196df 6042 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6043
51466239
AD		 6044 /* attempt to add filter to vlvf array */
6045 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6046
51466239
AD		 6047 /* add the filter since PF can receive vlans w/o entry in vlvf */
6048 igb_vfta_set(hw, vid, true);
9d5c8243
AK		 6049}
6050
6051static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6052{
6053 struct igb_adapter *adapter = netdev_priv(netdev);
6054 struct e1000_hw *hw = &adapter->hw;
4ae196df 6055 int pf_id = adapter->vfs_allocated_count;
51466239 6056 s32 err;
9d5c8243
AK		 6057
6058 igb_irq_disable(adapter);
6059 vlan_group_set_device(adapter->vlgrp, vid, NULL);
6060
6061 if (!test_bit(__IGB_DOWN, &adapter->state))
6062 igb_irq_enable(adapter);
6063
51466239
AD		 6064 /* remove vlan from VLVF table array */
6065 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6066
51466239
AD		 6067 /* if vid was not present in VLVF just remove it from table */
6068 if (err)
4ae196df 6069 igb_vfta_set(hw, vid, false);
9d5c8243
AK		 6070}
6071
6072static void igb_restore_vlan(struct igb_adapter *adapter)
6073{
6074 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6075
6076 if (adapter->vlgrp) {
6077 u16 vid;
6078 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
6079 if (!vlan_group_get_device(adapter->vlgrp, vid))
6080 continue;
6081 igb_vlan_rx_add_vid(adapter->netdev, vid);
6082 }
6083 }
6084}
6085
6086int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6087{
090b1795 6088 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6089 struct e1000_mac_info *mac = &adapter->hw.mac;
6090
6091 mac->autoneg = 0;
6092
9d5c8243
AK		 6093 switch (spddplx) {
6094 case SPEED_10 + DUPLEX_HALF:
6095 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6096 break;
6097 case SPEED_10 + DUPLEX_FULL:
6098 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6099 break;
6100 case SPEED_100 + DUPLEX_HALF:
6101 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6102 break;
6103 case SPEED_100 + DUPLEX_FULL:
6104 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6105 break;
6106 case SPEED_1000 + DUPLEX_FULL:
6107 mac->autoneg = 1;
6108 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6109 break;
6110 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6111 default:
090b1795 6112 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9d5c8243
AK		 6113 return -EINVAL;
6114 }
6115 return 0;
6116}
6117
3fe7c4c9 6118static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6119{
6120 struct net_device *netdev = pci_get_drvdata(pdev);
6121 struct igb_adapter *adapter = netdev_priv(netdev);
6122 struct e1000_hw *hw = &adapter->hw;
2d064c06 6123 u32 ctrl, rctl, status;
9d5c8243
AK		 6124 u32 wufc = adapter->wol;
6125#ifdef CONFIG_PM
6126 int retval = 0;
6127#endif
6128
6129 netif_device_detach(netdev);
6130
a88f10ec
AD		 6131 if (netif_running(netdev))
6132 igb_close(netdev);
6133
047e0030 6134 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6135
6136#ifdef CONFIG_PM
6137 retval = pci_save_state(pdev);
6138 if (retval)
6139 return retval;
6140#endif
6141
6142 status = rd32(E1000_STATUS);
6143 if (status & E1000_STATUS_LU)
6144 wufc &= ~E1000_WUFC_LNKC;
6145
6146 if (wufc) {
6147 igb_setup_rctl(adapter);
ff41f8dc 6148 igb_set_rx_mode(netdev);
9d5c8243
AK		 6149
6150 /* turn on all-multi mode if wake on multicast is enabled */
6151 if (wufc & E1000_WUFC_MC) {
6152 rctl = rd32(E1000_RCTL);
6153 rctl |= E1000_RCTL_MPE;
6154 wr32(E1000_RCTL, rctl);
6155 }
6156
6157 ctrl = rd32(E1000_CTRL);
6158 /* advertise wake from D3Cold */
6159 #define E1000_CTRL_ADVD3WUC 0x00100000
6160 /* phy power management enable */
6161 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6162 ctrl |= E1000_CTRL_ADVD3WUC;
6163 wr32(E1000_CTRL, ctrl);
6164
9d5c8243 6165 /* Allow time for pending master requests to run */
330a6d6a 6166 igb_disable_pcie_master(hw);
9d5c8243
AK		 6167
6168 wr32(E1000_WUC, E1000_WUC_PME_EN);
6169 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6170 } else {
6171 wr32(E1000_WUC, 0);
6172 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6173 }
6174
3fe7c4c9
RW		 6175 *enable_wake = wufc || adapter->en_mng_pt;
6176 if (!*enable_wake)
88a268c1
NN		 6177 igb_power_down_link(adapter);
6178 else
6179 igb_power_up_link(adapter);
9d5c8243
AK		 6180
6181 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6182 * would have already happened in close and is redundant. */
6183 igb_release_hw_control(adapter);
6184
6185 pci_disable_device(pdev);
6186
9d5c8243
AK		 6187 return 0;
6188}
6189
6190#ifdef CONFIG_PM
3fe7c4c9
RW		 6191static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6192{
6193 int retval;
6194 bool wake;
6195
6196 retval = __igb_shutdown(pdev, &wake);
6197 if (retval)
6198 return retval;
6199
6200 if (wake) {
6201 pci_prepare_to_sleep(pdev);
6202 } else {
6203 pci_wake_from_d3(pdev, false);
6204 pci_set_power_state(pdev, PCI_D3hot);
6205 }
6206
6207 return 0;
6208}
6209
9d5c8243
AK		 6210static int igb_resume(struct pci_dev *pdev)
6211{
6212 struct net_device *netdev = pci_get_drvdata(pdev);
6213 struct igb_adapter *adapter = netdev_priv(netdev);
6214 struct e1000_hw *hw = &adapter->hw;
6215 u32 err;
6216
6217 pci_set_power_state(pdev, PCI_D0);
6218 pci_restore_state(pdev);
b94f2d77 6219 pci_save_state(pdev);
42bfd33a 6220
aed5dec3 6221 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6222 if (err) {
6223 dev_err(&pdev->dev,
6224 "igb: Cannot enable PCI device from suspend\n");
6225 return err;
6226 }
6227 pci_set_master(pdev);
6228
6229 pci_enable_wake(pdev, PCI_D3hot, 0);
6230 pci_enable_wake(pdev, PCI_D3cold, 0);
6231
047e0030 6232 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6233 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6234 return -ENOMEM;
9d5c8243
AK		 6235 }
6236
9d5c8243 6237 igb_reset(adapter);
a8564f03
AD		 6238
6239 /* let the f/w know that the h/w is now under the control of the
6240 * driver. */
6241 igb_get_hw_control(adapter);
6242
9d5c8243
AK		 6243 wr32(E1000_WUS, ~0);
6244
a88f10ec
AD		 6245 if (netif_running(netdev)) {
6246 err = igb_open(netdev);
6247 if (err)
6248 return err;
6249 }
9d5c8243
AK		 6250
6251 netif_device_attach(netdev);
6252
9d5c8243
AK		 6253 return 0;
6254}
6255#endif
6256
6257static void igb_shutdown(struct pci_dev *pdev)
6258{
3fe7c4c9
RW		 6259 bool wake;
6260
6261 __igb_shutdown(pdev, &wake);
6262
6263 if (system_state == SYSTEM_POWER_OFF) {
6264 pci_wake_from_d3(pdev, wake);
6265 pci_set_power_state(pdev, PCI_D3hot);
6266 }
9d5c8243
AK		 6267}
6268
6269#ifdef CONFIG_NET_POLL_CONTROLLER
6270/*
6271 * Polling 'interrupt' - used by things like netconsole to send skbs
6272 * without having to re-enable interrupts. It's not called while
6273 * the interrupt routine is executing.
6274 */
6275static void igb_netpoll(struct net_device *netdev)
6276{
6277 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6278 struct e1000_hw *hw = &adapter->hw;
9d5c8243 6279 int i;
9d5c8243 6280
eebbbdba 6281 if (!adapter->msix_entries) {
047e0030 6282 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 6283 igb_irq_disable(adapter);
047e0030 6284 napi_schedule(&q_vector->napi);
eebbbdba
AD		 6285 return;
6286 }
9d5c8243 6287
047e0030
AD		 6288 for (i = 0; i < adapter->num_q_vectors; i++) {
6289 struct igb_q_vector *q_vector = adapter->q_vector[i];
6290 wr32(E1000_EIMC, q_vector->eims_value);
6291 napi_schedule(&q_vector->napi);
eebbbdba 6292 }
9d5c8243
AK		 6293}
6294#endif /* CONFIG_NET_POLL_CONTROLLER */
6295
6296/**
6297 * igb_io_error_detected - called when PCI error is detected
6298 * @pdev: Pointer to PCI device
6299 * @state: The current pci connection state
6300 *
6301 * This function is called after a PCI bus error affecting
6302 * this device has been detected.
6303 */
6304static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6305 pci_channel_state_t state)
6306{
6307 struct net_device *netdev = pci_get_drvdata(pdev);
6308 struct igb_adapter *adapter = netdev_priv(netdev);
6309
6310 netif_device_detach(netdev);
6311
59ed6eec
AD		 6312 if (state == pci_channel_io_perm_failure)
6313 return PCI_ERS_RESULT_DISCONNECT;
6314
9d5c8243
AK		 6315 if (netif_running(netdev))
6316 igb_down(adapter);
6317 pci_disable_device(pdev);
6318
6319 /* Request a slot slot reset. */
6320 return PCI_ERS_RESULT_NEED_RESET;
6321}
6322
6323/**
6324 * igb_io_slot_reset - called after the pci bus has been reset.
6325 * @pdev: Pointer to PCI device
6326 *
6327 * Restart the card from scratch, as if from a cold-boot. Implementation
6328 * resembles the first-half of the igb_resume routine.
6329 */
6330static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6331{
6332 struct net_device *netdev = pci_get_drvdata(pdev);
6333 struct igb_adapter *adapter = netdev_priv(netdev);
6334 struct e1000_hw *hw = &adapter->hw;
40a914fa 6335 pci_ers_result_t result;
42bfd33a 6336 int err;
9d5c8243 6337
aed5dec3 6338 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6339 dev_err(&pdev->dev,
6340 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6341 result = PCI_ERS_RESULT_DISCONNECT;
6342 } else {
6343 pci_set_master(pdev);
6344 pci_restore_state(pdev);
b94f2d77 6345 pci_save_state(pdev);
9d5c8243 6346
40a914fa
AD		 6347 pci_enable_wake(pdev, PCI_D3hot, 0);
6348 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6349
40a914fa
AD		 6350 igb_reset(adapter);
6351 wr32(E1000_WUS, ~0);
6352 result = PCI_ERS_RESULT_RECOVERED;
6353 }
9d5c8243 6354
ea943d41
JK		 6355 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6356 if (err) {
6357 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6358 "failed 0x%0x\n", err);
6359 /* non-fatal, continue */
6360 }
40a914fa
AD		 6361
6362 return result;
9d5c8243
AK		 6363}
6364
6365/**
6366 * igb_io_resume - called when traffic can start flowing again.
6367 * @pdev: Pointer to PCI device
6368 *
6369 * This callback is called when the error recovery driver tells us that
6370 * its OK to resume normal operation. Implementation resembles the
6371 * second-half of the igb_resume routine.
6372 */
6373static void igb_io_resume(struct pci_dev *pdev)
6374{
6375 struct net_device *netdev = pci_get_drvdata(pdev);
6376 struct igb_adapter *adapter = netdev_priv(netdev);
6377
9d5c8243
AK		 6378 if (netif_running(netdev)) {
6379 if (igb_up(adapter)) {
6380 dev_err(&pdev->dev, "igb_up failed after reset\n");
6381 return;
6382 }
6383 }
6384
6385 netif_device_attach(netdev);
6386
6387 /* let the f/w know that the h/w is now under the control of the
6388 * driver. */
6389 igb_get_hw_control(adapter);
9d5c8243
AK		 6390}
6391
26ad9178
AD		 6392static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6393 u8 qsel)
6394{
6395 u32 rar_low, rar_high;
6396 struct e1000_hw *hw = &adapter->hw;
6397
6398 /* HW expects these in little endian so we reverse the byte order
6399 * from network order (big endian) to little endian
6400 */
6401 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6402 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6403 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6404
6405 /* Indicate to hardware the Address is Valid. */
6406 rar_high |= E1000_RAH_AV;
6407
6408 if (hw->mac.type == e1000_82575)
6409 rar_high |= E1000_RAH_POOL_1 * qsel;
6410 else
6411 rar_high |= E1000_RAH_POOL_1 << qsel;
6412
6413 wr32(E1000_RAL(index), rar_low);
6414 wrfl();
6415 wr32(E1000_RAH(index), rar_high);
6416 wrfl();
6417}
6418
4ae196df
AD		 6419static int igb_set_vf_mac(struct igb_adapter *adapter,
6420 int vf, unsigned char *mac_addr)
6421{
6422 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6423 /* VF MAC addresses start at end of receive addresses and moves
6424 * torwards the first, as a result a collision should not be possible */
6425 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6426
37680117 6427 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6428
26ad9178 6429 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6430
6431 return 0;
6432}
6433
8151d294
WM		 6434static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6435{
6436 struct igb_adapter *adapter = netdev_priv(netdev);
6437 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6438 return -EINVAL;
6439 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6440 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6441 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6442 " change effective.");
6443 if (test_bit(__IGB_DOWN, &adapter->state)) {
6444 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6445 " but the PF device is not up.\n");
6446 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6447 " attempting to use the VF device.\n");
6448 }
6449 return igb_set_vf_mac(adapter, vf, mac);
6450}
6451
6452static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6453{
6454 return -EOPNOTSUPP;
6455}
6456
6457static int igb_ndo_get_vf_config(struct net_device *netdev,
6458 int vf, struct ifla_vf_info *ivi)
6459{
6460 struct igb_adapter *adapter = netdev_priv(netdev);
6461 if (vf >= adapter->vfs_allocated_count)
6462 return -EINVAL;
6463 ivi->vf = vf;
6464 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6465 ivi->tx_rate = 0;
6466 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6467 ivi->qos = adapter->vf_data[vf].pf_qos;
6468 return 0;
6469}
6470
4ae196df
AD		 6471static void igb_vmm_control(struct igb_adapter *adapter)
6472{
6473 struct e1000_hw *hw = &adapter->hw;
10d8e907 6474 u32 reg;
4ae196df 6475
52a1dd4d
AD		 6476 switch (hw->mac.type) {
6477 case e1000_82575:
6478 default:
6479 /* replication is not supported for 82575 */
4ae196df 6480 return;
52a1dd4d
AD		 6481 case e1000_82576:
6482 /* notify HW that the MAC is adding vlan tags */
6483 reg = rd32(E1000_DTXCTL);
6484 reg |= E1000_DTXCTL_VLAN_ADDED;
6485 wr32(E1000_DTXCTL, reg);
6486 case e1000_82580:
6487 /* enable replication vlan tag stripping */
6488 reg = rd32(E1000_RPLOLR);
6489 reg |= E1000_RPLOLR_STRVLAN;
6490 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6491 case e1000_i350:
6492 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6493 break;
6494 }
10d8e907 6495
d4960307
AD		 6496 if (adapter->vfs_allocated_count) {
6497 igb_vmdq_set_loopback_pf(hw, true);
6498 igb_vmdq_set_replication_pf(hw, true);
6499 } else {
6500 igb_vmdq_set_loopback_pf(hw, false);
6501 igb_vmdq_set_replication_pf(hw, false);
6502 }
4ae196df
AD		 6503}
6504
9d5c8243 6505/* igb_main.c */
Linux kernel - Deliverables
This page took 0.921515 seconds and 5 git commands to generate.