projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: fix vf lookup
[deliverable/linux.git] / drivers / net / ethernet / intel / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
6e861326 4 Copyright(c) 2007-2012 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
876d2d6f
JK		 28#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
29
9d5c8243
AK		 30#include <linux/module.h>
31#include <linux/types.h>
32#include <linux/init.h>
b2cb09b1 33#include <linux/bitops.h>
9d5c8243
AK		 34#include <linux/vmalloc.h>
35#include <linux/pagemap.h>
36#include <linux/netdevice.h>
9d5c8243 37#include <linux/ipv6.h>
5a0e3ad6 38#include <linux/slab.h>
9d5c8243
AK		 39#include <net/checksum.h>
40#include <net/ip6_checksum.h>
c6cb090b 41#include <linux/net_tstamp.h>
9d5c8243
AK		 42#include <linux/mii.h>
43#include <linux/ethtool.h>
01789349 44#include <linux/if.h>
9d5c8243
AK		 45#include <linux/if_vlan.h>
46#include <linux/pci.h>
c54106bb 47#include <linux/pci-aspm.h>
9d5c8243
AK		 48#include <linux/delay.h>
49#include <linux/interrupt.h>
7d13a7d0
AD		 50#include <linux/ip.h>
51#include <linux/tcp.h>
52#include <linux/sctp.h>
9d5c8243 53#include <linux/if_ether.h>
40a914fa 54#include <linux/aer.h>
70c71606 55#include <linux/prefetch.h>
749ab2cd 56#include <linux/pm_runtime.h>
421e02f0 57#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 58#include <linux/dca.h>
59#endif
9d5c8243
AK		 60#include "igb.h"
61
0d1fe82d 62#define MAJ 3
a28dc43f
CW		 63#define MIN 2
64#define BUILD 10
0d1fe82d 65#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
929dd047 66__stringify(BUILD) "-k"
9d5c8243
AK		 67char igb_driver_name[] = "igb";
68char igb_driver_version[] = DRV_VERSION;
69static const char igb_driver_string[] =
70 "Intel(R) Gigabit Ethernet Network Driver";
6e861326 71static const char igb_copyright[] = "Copyright (c) 2007-2012 Intel Corporation.";
9d5c8243 72
9d5c8243
AK		 73static const struct e1000_info *igb_info_tbl[] = {
74 [board_82575] = &e1000_82575_info,
75};
76
a3aa1884 77static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
6493d24f 84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
55cac248
AD		 85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
1b5dda33
GJ		 90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
2d064c06 92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 100 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
101 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
102 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
103 /* required last entry */
104 {0, }
105};
106
107MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
108
109void igb_reset(struct igb_adapter *);
110static int igb_setup_all_tx_resources(struct igb_adapter *);
111static int igb_setup_all_rx_resources(struct igb_adapter *);
112static void igb_free_all_tx_resources(struct igb_adapter *);
113static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 114static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 115static int igb_probe(struct pci_dev *, const struct pci_device_id *);
116static void __devexit igb_remove(struct pci_dev *pdev);
673b8b70 117static void igb_init_hw_timer(struct igb_adapter *adapter);
9d5c8243
AK		 118static int igb_sw_init(struct igb_adapter *);
119static int igb_open(struct net_device *);
120static int igb_close(struct net_device *);
121static void igb_configure_tx(struct igb_adapter *);
122static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 123static void igb_clean_all_tx_rings(struct igb_adapter *);
124static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 125static void igb_clean_tx_ring(struct igb_ring *);
126static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 127static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 128static void igb_update_phy_info(unsigned long);
129static void igb_watchdog(unsigned long);
130static void igb_watchdog_task(struct work_struct *);
cd392f5c 131static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 132static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
133 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 134static int igb_change_mtu(struct net_device *, int);
135static int igb_set_mac(struct net_device *, void *);
68d480c4 136static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 137static irqreturn_t igb_intr(int irq, void *);
138static irqreturn_t igb_intr_msi(int irq, void *);
139static irqreturn_t igb_msix_other(int irq, void *);
047e0030 140static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 141#ifdef CONFIG_IGB_DCA
047e0030 142static void igb_update_dca(struct igb_q_vector *);
fe4506b6 143static void igb_setup_dca(struct igb_adapter *);
421e02f0 144#endif /* CONFIG_IGB_DCA */
661086df 145static int igb_poll(struct napi_struct *, int);
13fde97a 146static bool igb_clean_tx_irq(struct igb_q_vector *);
cd392f5c 147static bool igb_clean_rx_irq(struct igb_q_vector *, int);
9d5c8243
AK		 148static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
149static void igb_tx_timeout(struct net_device *);
150static void igb_reset_task(struct work_struct *);
c8f44aff 151static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features);
8e586137
JP		 152static int igb_vlan_rx_add_vid(struct net_device *, u16);
153static int igb_vlan_rx_kill_vid(struct net_device *, u16);
9d5c8243 154static void igb_restore_vlan(struct igb_adapter *);
26ad9178 155static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 156static void igb_ping_all_vfs(struct igb_adapter *);
157static void igb_msg_task(struct igb_adapter *);
4ae196df 158static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 159static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 160static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 161static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
162static int igb_ndo_set_vf_vlan(struct net_device *netdev,
163 int vf, u16 vlan, u8 qos);
164static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
165static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
166 struct ifla_vf_info *ivi);
17dc566c 167static void igb_check_vf_rate_limit(struct igb_adapter *);
46a01698
RL		 168
169#ifdef CONFIG_PCI_IOV
0224d663
GR		 170static int igb_vf_configure(struct igb_adapter *adapter, int vf);
171static int igb_find_enabled_vfs(struct igb_adapter *adapter);
172static int igb_check_vf_assignment(struct igb_adapter *adapter);
46a01698 173#endif
9d5c8243 174
9d5c8243 175#ifdef CONFIG_PM
749ab2cd
YZ		 176static int igb_suspend(struct device *);
177static int igb_resume(struct device *);
178#ifdef CONFIG_PM_RUNTIME
179static int igb_runtime_suspend(struct device *dev);
180static int igb_runtime_resume(struct device *dev);
181static int igb_runtime_idle(struct device *dev);
182#endif
183static const struct dev_pm_ops igb_pm_ops = {
184 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
185 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
186 igb_runtime_idle)
187};
9d5c8243
AK		 188#endif
189static void igb_shutdown(struct pci_dev *);
421e02f0 190#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 191static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
192static struct notifier_block dca_notifier = {
193 .notifier_call = igb_notify_dca,
194 .next = NULL,
195 .priority = 0
196};
197#endif
9d5c8243
AK		 198#ifdef CONFIG_NET_POLL_CONTROLLER
199/* for netdump / net console */
200static void igb_netpoll(struct net_device *);
201#endif
37680117 202#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 203static unsigned int max_vfs = 0;
204module_param(max_vfs, uint, 0);
205MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
206 "per physical function");
207#endif /* CONFIG_PCI_IOV */
208
9d5c8243
AK		 209static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
210 pci_channel_state_t);
211static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
212static void igb_io_resume(struct pci_dev *);
213
214static struct pci_error_handlers igb_err_handler = {
215 .error_detected = igb_io_error_detected,
216 .slot_reset = igb_io_slot_reset,
217 .resume = igb_io_resume,
218};
219
b6e0c419 220static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
9d5c8243
AK		 221
222static struct pci_driver igb_driver = {
223 .name = igb_driver_name,
224 .id_table = igb_pci_tbl,
225 .probe = igb_probe,
226 .remove = __devexit_p(igb_remove),
227#ifdef CONFIG_PM
749ab2cd 228 .driver.pm = &igb_pm_ops,
9d5c8243
AK		 229#endif
230 .shutdown = igb_shutdown,
231 .err_handler = &igb_err_handler
232};
233
234MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
235MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
236MODULE_LICENSE("GPL");
237MODULE_VERSION(DRV_VERSION);
238
c97ec42a
TI		 239struct igb_reg_info {
240 u32 ofs;
241 char *name;
242};
243
244static const struct igb_reg_info igb_reg_info_tbl[] = {
245
246 /* General Registers */
247 {E1000_CTRL, "CTRL"},
248 {E1000_STATUS, "STATUS"},
249 {E1000_CTRL_EXT, "CTRL_EXT"},
250
251 /* Interrupt Registers */
252 {E1000_ICR, "ICR"},
253
254 /* RX Registers */
255 {E1000_RCTL, "RCTL"},
256 {E1000_RDLEN(0), "RDLEN"},
257 {E1000_RDH(0), "RDH"},
258 {E1000_RDT(0), "RDT"},
259 {E1000_RXDCTL(0), "RXDCTL"},
260 {E1000_RDBAL(0), "RDBAL"},
261 {E1000_RDBAH(0), "RDBAH"},
262
263 /* TX Registers */
264 {E1000_TCTL, "TCTL"},
265 {E1000_TDBAL(0), "TDBAL"},
266 {E1000_TDBAH(0), "TDBAH"},
267 {E1000_TDLEN(0), "TDLEN"},
268 {E1000_TDH(0), "TDH"},
269 {E1000_TDT(0), "TDT"},
270 {E1000_TXDCTL(0), "TXDCTL"},
271 {E1000_TDFH, "TDFH"},
272 {E1000_TDFT, "TDFT"},
273 {E1000_TDFHS, "TDFHS"},
274 {E1000_TDFPC, "TDFPC"},
275
276 /* List Terminator */
277 {}
278};
279
280/*
281 * igb_regdump - register printout routine
282 */
283static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
284{
285 int n = 0;
286 char rname[16];
287 u32 regs[8];
288
289 switch (reginfo->ofs) {
290 case E1000_RDLEN(0):
291 for (n = 0; n < 4; n++)
292 regs[n] = rd32(E1000_RDLEN(n));
293 break;
294 case E1000_RDH(0):
295 for (n = 0; n < 4; n++)
296 regs[n] = rd32(E1000_RDH(n));
297 break;
298 case E1000_RDT(0):
299 for (n = 0; n < 4; n++)
300 regs[n] = rd32(E1000_RDT(n));
301 break;
302 case E1000_RXDCTL(0):
303 for (n = 0; n < 4; n++)
304 regs[n] = rd32(E1000_RXDCTL(n));
305 break;
306 case E1000_RDBAL(0):
307 for (n = 0; n < 4; n++)
308 regs[n] = rd32(E1000_RDBAL(n));
309 break;
310 case E1000_RDBAH(0):
311 for (n = 0; n < 4; n++)
312 regs[n] = rd32(E1000_RDBAH(n));
313 break;
314 case E1000_TDBAL(0):
315 for (n = 0; n < 4; n++)
316 regs[n] = rd32(E1000_RDBAL(n));
317 break;
318 case E1000_TDBAH(0):
319 for (n = 0; n < 4; n++)
320 regs[n] = rd32(E1000_TDBAH(n));
321 break;
322 case E1000_TDLEN(0):
323 for (n = 0; n < 4; n++)
324 regs[n] = rd32(E1000_TDLEN(n));
325 break;
326 case E1000_TDH(0):
327 for (n = 0; n < 4; n++)
328 regs[n] = rd32(E1000_TDH(n));
329 break;
330 case E1000_TDT(0):
331 for (n = 0; n < 4; n++)
332 regs[n] = rd32(E1000_TDT(n));
333 break;
334 case E1000_TXDCTL(0):
335 for (n = 0; n < 4; n++)
336 regs[n] = rd32(E1000_TXDCTL(n));
337 break;
338 default:
876d2d6f 339 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
c97ec42a
TI		 340 return;
341 }
342
343 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
876d2d6f
JK		 344 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
345 regs[2], regs[3]);
c97ec42a
TI		 346}
347
348/*
349 * igb_dump - Print registers, tx-rings and rx-rings
350 */
351static void igb_dump(struct igb_adapter *adapter)
352{
353 struct net_device *netdev = adapter->netdev;
354 struct e1000_hw *hw = &adapter->hw;
355 struct igb_reg_info *reginfo;
c97ec42a
TI		 356 struct igb_ring *tx_ring;
357 union e1000_adv_tx_desc *tx_desc;
358 struct my_u0 { u64 a; u64 b; } *u0;
c97ec42a
TI		 359 struct igb_ring *rx_ring;
360 union e1000_adv_rx_desc *rx_desc;
361 u32 staterr;
6ad4edfc 362 u16 i, n;
c97ec42a
TI		 363
364 if (!netif_msg_hw(adapter))
365 return;
366
367 /* Print netdevice Info */
368 if (netdev) {
369 dev_info(&adapter->pdev->dev, "Net device Info\n");
876d2d6f
JK		 370 pr_info("Device Name state trans_start "
371 "last_rx\n");
372 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
373 netdev->state, netdev->trans_start, netdev->last_rx);
c97ec42a
TI		 374 }
375
376 /* Print Registers */
377 dev_info(&adapter->pdev->dev, "Register Dump\n");
876d2d6f 378 pr_info(" Register Name Value\n");
c97ec42a
TI		 379 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
380 reginfo->name; reginfo++) {
381 igb_regdump(hw, reginfo);
382 }
383
384 /* Print TX Ring Summary */
385 if (!netdev || !netif_running(netdev))
386 goto exit;
387
388 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
876d2d6f 389 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
c97ec42a 390 for (n = 0; n < adapter->num_tx_queues; n++) {
06034649 391 struct igb_tx_buffer *buffer_info;
c97ec42a 392 tx_ring = adapter->tx_ring[n];
06034649 393 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
876d2d6f
JK		 394 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
395 n, tx_ring->next_to_use, tx_ring->next_to_clean,
396 (u64)buffer_info->dma,
397 buffer_info->length,
398 buffer_info->next_to_watch,
399 (u64)buffer_info->time_stamp);
c97ec42a
TI		 400 }
401
402 /* Print TX Rings */
403 if (!netif_msg_tx_done(adapter))
404 goto rx_ring_summary;
405
406 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
407
408 /* Transmit Descriptor Formats
409 *
410 * Advanced Transmit Descriptor
411 * +--------------------------------------------------------------+
412 * 0 | Buffer Address [63:0] |
413 * +--------------------------------------------------------------+
414 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
415 * +--------------------------------------------------------------+
416 * 63 46 45 40 39 38 36 35 32 31 24 15 0
417 */
418
419 for (n = 0; n < adapter->num_tx_queues; n++) {
420 tx_ring = adapter->tx_ring[n];
876d2d6f
JK		 421 pr_info("------------------------------------\n");
422 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
423 pr_info("------------------------------------\n");
424 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] "
425 "[bi->dma ] leng ntw timestamp "
426 "bi->skb\n");
c97ec42a
TI		 427
428 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
876d2d6f 429 const char *next_desc;
06034649 430 struct igb_tx_buffer *buffer_info;
60136906 431 tx_desc = IGB_TX_DESC(tx_ring, i);
06034649 432 buffer_info = &tx_ring->tx_buffer_info[i];
c97ec42a 433 u0 = (struct my_u0 *)tx_desc;
876d2d6f
JK		 434 if (i == tx_ring->next_to_use &&
435 i == tx_ring->next_to_clean)
436 next_desc = " NTC/U";
437 else if (i == tx_ring->next_to_use)
438 next_desc = " NTU";
439 else if (i == tx_ring->next_to_clean)
440 next_desc = " NTC";
441 else
442 next_desc = "";
443
444 pr_info("T [0x%03X] %016llX %016llX %016llX"
445 " %04X %p %016llX %p%s\n", i,
c97ec42a
TI		 446 le64_to_cpu(u0->a),
447 le64_to_cpu(u0->b),
448 (u64)buffer_info->dma,
449 buffer_info->length,
450 buffer_info->next_to_watch,
451 (u64)buffer_info->time_stamp,
876d2d6f 452 buffer_info->skb, next_desc);
c97ec42a
TI		 453
454 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
455 print_hex_dump(KERN_INFO, "",
456 DUMP_PREFIX_ADDRESS,
457 16, 1, phys_to_virt(buffer_info->dma),
458 buffer_info->length, true);
459 }
460 }
461
462 /* Print RX Rings Summary */
463rx_ring_summary:
464 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
876d2d6f 465 pr_info("Queue [NTU] [NTC]\n");
c97ec42a
TI		 466 for (n = 0; n < adapter->num_rx_queues; n++) {
467 rx_ring = adapter->rx_ring[n];
876d2d6f
JK		 468 pr_info(" %5d %5X %5X\n",
469 n, rx_ring->next_to_use, rx_ring->next_to_clean);
c97ec42a
TI		 470 }
471
472 /* Print RX Rings */
473 if (!netif_msg_rx_status(adapter))
474 goto exit;
475
476 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
477
478 /* Advanced Receive Descriptor (Read) Format
479 * 63 1 0
480 * +-----------------------------------------------------+
481 * 0 | Packet Buffer Address [63:1] |A0/NSE|
482 * +----------------------------------------------+------+
483 * 8 | Header Buffer Address [63:1] | DD |
484 * +-----------------------------------------------------+
485 *
486 *
487 * Advanced Receive Descriptor (Write-Back) Format
488 *
489 * 63 48 47 32 31 30 21 20 17 16 4 3 0
490 * +------------------------------------------------------+
491 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
492 * | Checksum Ident | | | | Type | Type |
493 * +------------------------------------------------------+
494 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
495 * +------------------------------------------------------+
496 * 63 48 47 32 31 20 19 0
497 */
498
499 for (n = 0; n < adapter->num_rx_queues; n++) {
500 rx_ring = adapter->rx_ring[n];
876d2d6f
JK		 501 pr_info("------------------------------------\n");
502 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
503 pr_info("------------------------------------\n");
504 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] "
505 "[bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
506 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] -----"
507 "----------- [bi->skb] <-- Adv Rx Write-Back format\n");
c97ec42a
TI		 508
509 for (i = 0; i < rx_ring->count; i++) {
876d2d6f 510 const char *next_desc;
06034649
AD		 511 struct igb_rx_buffer *buffer_info;
512 buffer_info = &rx_ring->rx_buffer_info[i];
60136906 513 rx_desc = IGB_RX_DESC(rx_ring, i);
c97ec42a
TI		 514 u0 = (struct my_u0 *)rx_desc;
515 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
876d2d6f
JK		 516
517 if (i == rx_ring->next_to_use)
518 next_desc = " NTU";
519 else if (i == rx_ring->next_to_clean)
520 next_desc = " NTC";
521 else
522 next_desc = "";
523
c97ec42a
TI		 524 if (staterr & E1000_RXD_STAT_DD) {
525 /* Descriptor Done */
876d2d6f
JK		 526 pr_info("%s[0x%03X] %016llX %016llX -------"
527 "--------- %p%s\n", "RWB", i,
c97ec42a
TI		 528 le64_to_cpu(u0->a),
529 le64_to_cpu(u0->b),
876d2d6f 530 buffer_info->skb, next_desc);
c97ec42a 531 } else {
876d2d6f
JK		 532 pr_info("%s[0x%03X] %016llX %016llX %016llX"
533 " %p%s\n", "R ", i,
c97ec42a
TI		 534 le64_to_cpu(u0->a),
535 le64_to_cpu(u0->b),
536 (u64)buffer_info->dma,
876d2d6f 537 buffer_info->skb, next_desc);
c97ec42a
TI		 538
539 if (netif_msg_pktdata(adapter)) {
540 print_hex_dump(KERN_INFO, "",
541 DUMP_PREFIX_ADDRESS,
542 16, 1,
543 phys_to_virt(buffer_info->dma),
44390ca6
AD		 544 IGB_RX_HDR_LEN, true);
545 print_hex_dump(KERN_INFO, "",
546 DUMP_PREFIX_ADDRESS,
547 16, 1,
548 phys_to_virt(
549 buffer_info->page_dma +
550 buffer_info->page_offset),
551 PAGE_SIZE/2, true);
c97ec42a
TI		 552 }
553 }
c97ec42a
TI		 554 }
555 }
556
557exit:
558 return;
559}
560
561
38c845c7
PO		 562/**
563 * igb_read_clock - read raw cycle counter (to be used by time counter)
564 */
565static cycle_t igb_read_clock(const struct cyclecounter *tc)
566{
567 struct igb_adapter *adapter =
568 container_of(tc, struct igb_adapter, cycles);
569 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 570 u64 stamp = 0;
571 int shift = 0;
38c845c7 572
55cac248
AD		 573 /*
574 * The timestamp latches on lowest register read. For the 82580
575 * the lowest register is SYSTIMR instead of SYSTIML. However we never
576 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
577 */
06218a8d 578 if (hw->mac.type >= e1000_82580) {
55cac248
AD		 579 stamp = rd32(E1000_SYSTIMR) >> 8;
580 shift = IGB_82580_TSYNC_SHIFT;
581 }
582
c5b9bd5e
AD		 583 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
584 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 585 return stamp;
586}
587
9d5c8243 588/**
c041076a 589 * igb_get_hw_dev - return device
9d5c8243
AK		 590 * used by hardware layer to print debugging information
591 **/
c041076a 592struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 593{
594 struct igb_adapter *adapter = hw->back;
c041076a 595 return adapter->netdev;
9d5c8243 596}
38c845c7 597
9d5c8243
AK		 598/**
599 * igb_init_module - Driver Registration Routine
600 *
601 * igb_init_module is the first routine called when the driver is
602 * loaded. All it does is register with the PCI subsystem.
603 **/
604static int __init igb_init_module(void)
605{
606 int ret;
876d2d6f 607 pr_info("%s - version %s\n",
9d5c8243
AK		 608 igb_driver_string, igb_driver_version);
609
876d2d6f 610 pr_info("%s\n", igb_copyright);
9d5c8243 611
421e02f0 612#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 613 dca_register_notify(&dca_notifier);
614#endif
bbd98fe4 615 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 616 return ret;
617}
618
619module_init(igb_init_module);
620
621/**
622 * igb_exit_module - Driver Exit Cleanup Routine
623 *
624 * igb_exit_module is called just before the driver is removed
625 * from memory.
626 **/
627static void __exit igb_exit_module(void)
628{
421e02f0 629#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 630 dca_unregister_notify(&dca_notifier);
631#endif
9d5c8243
AK		 632 pci_unregister_driver(&igb_driver);
633}
634
635module_exit(igb_exit_module);
636
26bc19ec
AD		 637#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
638/**
639 * igb_cache_ring_register - Descriptor ring to register mapping
640 * @adapter: board private structure to initialize
641 *
642 * Once we know the feature-set enabled for the device, we'll cache
643 * the register offset the descriptor ring is assigned to.
644 **/
645static void igb_cache_ring_register(struct igb_adapter *adapter)
646{
ee1b9f06 647 int i = 0, j = 0;
047e0030 648 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 649
650 switch (adapter->hw.mac.type) {
651 case e1000_82576:
652 /* The queues are allocated for virtualization such that VF 0
653 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
654 * In order to avoid collision we start at the first free queue
655 * and continue consuming queues in the same sequence
656 */
ee1b9f06 657 if (adapter->vfs_allocated_count) {
a99955fc 658 for (; i < adapter->rss_queues; i++)
3025a446
AD		 659 adapter->rx_ring[i]->reg_idx = rbase_offset +
660 Q_IDX_82576(i);
ee1b9f06 661 }
26bc19ec 662 case e1000_82575:
55cac248 663 case e1000_82580:
d2ba2ed8 664 case e1000_i350:
26bc19ec 665 default:
ee1b9f06 666 for (; i < adapter->num_rx_queues; i++)
3025a446 667 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 668 for (; j < adapter->num_tx_queues; j++)
3025a446 669 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 670 break;
671 }
672}
673
047e0030
AD		 674static void igb_free_queues(struct igb_adapter *adapter)
675{
3025a446 676 int i;
047e0030 677
3025a446
AD		 678 for (i = 0; i < adapter->num_tx_queues; i++) {
679 kfree(adapter->tx_ring[i]);
680 adapter->tx_ring[i] = NULL;
681 }
682 for (i = 0; i < adapter->num_rx_queues; i++) {
683 kfree(adapter->rx_ring[i]);
684 adapter->rx_ring[i] = NULL;
685 }
047e0030
AD		 686 adapter->num_rx_queues = 0;
687 adapter->num_tx_queues = 0;
688}
689
9d5c8243
AK		 690/**
691 * igb_alloc_queues - Allocate memory for all rings
692 * @adapter: board private structure to initialize
693 *
694 * We allocate one ring per queue at run-time since we don't know the
695 * number of queues at compile-time.
696 **/
697static int igb_alloc_queues(struct igb_adapter *adapter)
698{
3025a446 699 struct igb_ring *ring;
9d5c8243 700 int i;
81c2fc22 701 int orig_node = adapter->node;
9d5c8243 702
661086df 703 for (i = 0; i < adapter->num_tx_queues; i++) {
81c2fc22
AD		 704 if (orig_node == -1) {
705 int cur_node = next_online_node(adapter->node);
706 if (cur_node == MAX_NUMNODES)
707 cur_node = first_online_node;
708 adapter->node = cur_node;
709 }
710 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
711 adapter->node);
712 if (!ring)
713 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 714 if (!ring)
715 goto err;
68fd9910 716 ring->count = adapter->tx_ring_count;
661086df 717 ring->queue_index = i;
59d71989 718 ring->dev = &adapter->pdev->dev;
e694e964 719 ring->netdev = adapter->netdev;
81c2fc22 720 ring->numa_node = adapter->node;
85ad76b2
AD		 721 /* For 82575, context index must be unique per ring. */
722 if (adapter->hw.mac.type == e1000_82575)
866cff06 723 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
3025a446 724 adapter->tx_ring[i] = ring;
661086df 725 }
81c2fc22
AD		 726 /* Restore the adapter's original node */
727 adapter->node = orig_node;
85ad76b2 728
9d5c8243 729 for (i = 0; i < adapter->num_rx_queues; i++) {
81c2fc22
AD		 730 if (orig_node == -1) {
731 int cur_node = next_online_node(adapter->node);
732 if (cur_node == MAX_NUMNODES)
733 cur_node = first_online_node;
734 adapter->node = cur_node;
735 }
736 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
737 adapter->node);
738 if (!ring)
739 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 740 if (!ring)
741 goto err;
68fd9910 742 ring->count = adapter->rx_ring_count;
844290e5 743 ring->queue_index = i;
59d71989 744 ring->dev = &adapter->pdev->dev;
e694e964 745 ring->netdev = adapter->netdev;
81c2fc22 746 ring->numa_node = adapter->node;
85ad76b2
AD		 747 /* set flag indicating ring supports SCTP checksum offload */
748 if (adapter->hw.mac.type >= e1000_82576)
866cff06 749 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
8be10e91
AD		 750
751 /* On i350, loopback VLAN packets have the tag byte-swapped. */
752 if (adapter->hw.mac.type == e1000_i350)
753 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
754
3025a446 755 adapter->rx_ring[i] = ring;
9d5c8243 756 }
81c2fc22
AD		 757 /* Restore the adapter's original node */
758 adapter->node = orig_node;
26bc19ec
AD		 759
760 igb_cache_ring_register(adapter);
9d5c8243 761
047e0030 762 return 0;
a88f10ec 763
047e0030 764err:
81c2fc22
AD		 765 /* Restore the adapter's original node */
766 adapter->node = orig_node;
047e0030 767 igb_free_queues(adapter);
d1a8c9e1 768
047e0030 769 return -ENOMEM;
a88f10ec
AD		 770}
771
4be000c8
AD		 772/**
773 * igb_write_ivar - configure ivar for given MSI-X vector
774 * @hw: pointer to the HW structure
775 * @msix_vector: vector number we are allocating to a given ring
776 * @index: row index of IVAR register to write within IVAR table
777 * @offset: column offset of in IVAR, should be multiple of 8
778 *
779 * This function is intended to handle the writing of the IVAR register
780 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
781 * each containing an cause allocation for an Rx and Tx ring, and a
782 * variable number of rows depending on the number of queues supported.
783 **/
784static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
785 int index, int offset)
786{
787 u32 ivar = array_rd32(E1000_IVAR0, index);
788
789 /* clear any bits that are currently set */
790 ivar &= ~((u32)0xFF << offset);
791
792 /* write vector and valid bit */
793 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
794
795 array_wr32(E1000_IVAR0, index, ivar);
796}
797
9d5c8243 798#define IGB_N0_QUEUE -1
047e0030 799static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243 800{
047e0030 801 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 802 struct e1000_hw *hw = &adapter->hw;
047e0030
AD		 803 int rx_queue = IGB_N0_QUEUE;
804 int tx_queue = IGB_N0_QUEUE;
4be000c8 805 u32 msixbm = 0;
047e0030 806
0ba82994
AD		 807 if (q_vector->rx.ring)
808 rx_queue = q_vector->rx.ring->reg_idx;
809 if (q_vector->tx.ring)
810 tx_queue = q_vector->tx.ring->reg_idx;
2d064c06
AD		 811
812 switch (hw->mac.type) {
813 case e1000_82575:
9d5c8243
AK		 814 /* The 82575 assigns vectors using a bitmask, which matches the
815 bitmask for the EICR/EIMS/EIMC registers. To assign one
816 or more queues to a vector, we write the appropriate bits
817 into the MSIXBM register for that vector. */
047e0030 818 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 819 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 820 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 821 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 822 if (!adapter->msix_entries && msix_vector == 0)
823 msixbm |= E1000_EIMS_OTHER;
9d5c8243 824 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 825 q_vector->eims_value = msixbm;
2d064c06
AD		 826 break;
827 case e1000_82576:
4be000c8
AD		 828 /*
829 * 82576 uses a table that essentially consists of 2 columns
830 * with 8 rows. The ordering is column-major so we use the
831 * lower 3 bits as the row index, and the 4th bit as the
832 * column offset.
833 */
834 if (rx_queue > IGB_N0_QUEUE)
835 igb_write_ivar(hw, msix_vector,
836 rx_queue & 0x7,
837 (rx_queue & 0x8) << 1);
838 if (tx_queue > IGB_N0_QUEUE)
839 igb_write_ivar(hw, msix_vector,
840 tx_queue & 0x7,
841 ((tx_queue & 0x8) << 1) + 8);
047e0030 842 q_vector->eims_value = 1 << msix_vector;
2d064c06 843 break;
55cac248 844 case e1000_82580:
d2ba2ed8 845 case e1000_i350:
4be000c8
AD		 846 /*
847 * On 82580 and newer adapters the scheme is similar to 82576
848 * however instead of ordering column-major we have things
849 * ordered row-major. So we traverse the table by using
850 * bit 0 as the column offset, and the remaining bits as the
851 * row index.
852 */
853 if (rx_queue > IGB_N0_QUEUE)
854 igb_write_ivar(hw, msix_vector,
855 rx_queue >> 1,
856 (rx_queue & 0x1) << 4);
857 if (tx_queue > IGB_N0_QUEUE)
858 igb_write_ivar(hw, msix_vector,
859 tx_queue >> 1,
860 ((tx_queue & 0x1) << 4) + 8);
55cac248
AD		 861 q_vector->eims_value = 1 << msix_vector;
862 break;
2d064c06
AD		 863 default:
864 BUG();
865 break;
866 }
26b39276
AD		 867
868 /* add q_vector eims value to global eims_enable_mask */
869 adapter->eims_enable_mask |= q_vector->eims_value;
870
871 /* configure q_vector to set itr on first interrupt */
872 q_vector->set_itr = 1;
9d5c8243
AK		 873}
874
875/**
876 * igb_configure_msix - Configure MSI-X hardware
877 *
878 * igb_configure_msix sets up the hardware to properly
879 * generate MSI-X interrupts.
880 **/
881static void igb_configure_msix(struct igb_adapter *adapter)
882{
883 u32 tmp;
884 int i, vector = 0;
885 struct e1000_hw *hw = &adapter->hw;
886
887 adapter->eims_enable_mask = 0;
9d5c8243
AK		 888
889 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 890 switch (hw->mac.type) {
891 case e1000_82575:
9d5c8243
AK		 892 tmp = rd32(E1000_CTRL_EXT);
893 /* enable MSI-X PBA support*/
894 tmp |= E1000_CTRL_EXT_PBA_CLR;
895
896 /* Auto-Mask interrupts upon ICR read. */
897 tmp |= E1000_CTRL_EXT_EIAME;
898 tmp |= E1000_CTRL_EXT_IRCA;
899
900 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 901
902 /* enable msix_other interrupt */
903 array_wr32(E1000_MSIXBM(0), vector++,
904 E1000_EIMS_OTHER);
844290e5 905 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 906
2d064c06
AD		 907 break;
908
909 case e1000_82576:
55cac248 910 case e1000_82580:
d2ba2ed8 911 case e1000_i350:
047e0030
AD		 912 /* Turn on MSI-X capability first, or our settings
913 * won't stick. And it will take days to debug. */
914 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
915 E1000_GPIE_PBA | E1000_GPIE_EIAME |
916 E1000_GPIE_NSICR);
917
918 /* enable msix_other interrupt */
919 adapter->eims_other = 1 << vector;
2d064c06 920 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 921
047e0030 922 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 923 break;
924 default:
925 /* do nothing, since nothing else supports MSI-X */
926 break;
927 } /* switch (hw->mac.type) */
047e0030
AD		 928
929 adapter->eims_enable_mask |= adapter->eims_other;
930
26b39276
AD		 931 for (i = 0; i < adapter->num_q_vectors; i++)
932 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 933
9d5c8243
AK		 934 wrfl();
935}
936
937/**
938 * igb_request_msix - Initialize MSI-X interrupts
939 *
940 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
941 * kernel.
942 **/
943static int igb_request_msix(struct igb_adapter *adapter)
944{
945 struct net_device *netdev = adapter->netdev;
047e0030 946 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 947 int i, err = 0, vector = 0;
948
047e0030 949 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 950 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 951 if (err)
952 goto out;
953 vector++;
954
955 for (i = 0; i < adapter->num_q_vectors; i++) {
956 struct igb_q_vector *q_vector = adapter->q_vector[i];
957
958 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
959
0ba82994 960 if (q_vector->rx.ring && q_vector->tx.ring)
047e0030 961 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
0ba82994
AD		 962 q_vector->rx.ring->queue_index);
963 else if (q_vector->tx.ring)
047e0030 964 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
0ba82994
AD		 965 q_vector->tx.ring->queue_index);
966 else if (q_vector->rx.ring)
047e0030 967 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
0ba82994 968 q_vector->rx.ring->queue_index);
9d5c8243 969 else
047e0030
AD		 970 sprintf(q_vector->name, "%s-unused", netdev->name);
971
9d5c8243 972 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 973 igb_msix_ring, 0, q_vector->name,
047e0030 974 q_vector);
9d5c8243
AK		 975 if (err)
976 goto out;
9d5c8243
AK		 977 vector++;
978 }
979
9d5c8243
AK		 980 igb_configure_msix(adapter);
981 return 0;
982out:
983 return err;
984}
985
986static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
987{
988 if (adapter->msix_entries) {
989 pci_disable_msix(adapter->pdev);
990 kfree(adapter->msix_entries);
991 adapter->msix_entries = NULL;
047e0030 992 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 993 pci_disable_msi(adapter->pdev);
047e0030 994 }
9d5c8243
AK		 995}
996
047e0030
AD		 997/**
998 * igb_free_q_vectors - Free memory allocated for interrupt vectors
999 * @adapter: board private structure to initialize
1000 *
1001 * This function frees the memory allocated to the q_vectors. In addition if
1002 * NAPI is enabled it will delete any references to the NAPI struct prior
1003 * to freeing the q_vector.
1004 **/
1005static void igb_free_q_vectors(struct igb_adapter *adapter)
1006{
1007 int v_idx;
1008
1009 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1010 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1011 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 1012 if (!q_vector)
1013 continue;
047e0030
AD		 1014 netif_napi_del(&q_vector->napi);
1015 kfree(q_vector);
1016 }
1017 adapter->num_q_vectors = 0;
1018}
1019
1020/**
1021 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1022 *
1023 * This function resets the device so that it has 0 rx queues, tx queues, and
1024 * MSI-X interrupts allocated.
1025 */
1026static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1027{
1028 igb_free_queues(adapter);
1029 igb_free_q_vectors(adapter);
1030 igb_reset_interrupt_capability(adapter);
1031}
9d5c8243
AK		 1032
1033/**
1034 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1035 *
1036 * Attempt to configure interrupts using the best available
1037 * capabilities of the hardware and kernel.
1038 **/
21adef3e 1039static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 1040{
1041 int err;
1042 int numvecs, i;
1043
83b7180d 1044 /* Number of supported queues. */
a99955fc 1045 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 1046 if (adapter->vfs_allocated_count)
1047 adapter->num_tx_queues = 1;
1048 else
1049 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1050
047e0030
AD		 1051 /* start with one vector for every rx queue */
1052 numvecs = adapter->num_rx_queues;
1053
3ad2f3fb 1054 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1055 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1056 numvecs += adapter->num_tx_queues;
047e0030
AD		 1057
1058 /* store the number of vectors reserved for queues */
1059 adapter->num_q_vectors = numvecs;
1060
1061 /* add 1 vector for link status interrupts */
1062 numvecs++;
9d5c8243
AK		 1063 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1064 GFP_KERNEL);
1065 if (!adapter->msix_entries)
1066 goto msi_only;
1067
1068 for (i = 0; i < numvecs; i++)
1069 adapter->msix_entries[i].entry = i;
1070
1071 err = pci_enable_msix(adapter->pdev,
1072 adapter->msix_entries,
1073 numvecs);
1074 if (err == 0)
34a20e89 1075 goto out;
9d5c8243
AK		 1076
1077 igb_reset_interrupt_capability(adapter);
1078
1079 /* If we can't do MSI-X, try MSI */
1080msi_only:
2a3abf6d
AD		 1081#ifdef CONFIG_PCI_IOV
1082 /* disable SR-IOV for non MSI-X configurations */
1083 if (adapter->vf_data) {
1084 struct e1000_hw *hw = &adapter->hw;
1085 /* disable iov and allow time for transactions to clear */
1086 pci_disable_sriov(adapter->pdev);
1087 msleep(500);
1088
1089 kfree(adapter->vf_data);
1090 adapter->vf_data = NULL;
1091 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 1092 wrfl();
2a3abf6d
AD		 1093 msleep(100);
1094 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1095 }
1096#endif
4fc82adf 1097 adapter->vfs_allocated_count = 0;
a99955fc 1098 adapter->rss_queues = 1;
4fc82adf 1099 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1100 adapter->num_rx_queues = 1;
661086df 1101 adapter->num_tx_queues = 1;
047e0030 1102 adapter->num_q_vectors = 1;
9d5c8243 1103 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1104 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1105out:
21adef3e
BH		 1106 /* Notify the stack of the (possibly) reduced queue counts. */
1107 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1108 return netif_set_real_num_rx_queues(adapter->netdev,
1109 adapter->num_rx_queues);
9d5c8243
AK		 1110}
1111
047e0030
AD		 1112/**
1113 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1114 * @adapter: board private structure to initialize
1115 *
1116 * We allocate one q_vector per queue interrupt. If allocation fails we
1117 * return -ENOMEM.
1118 **/
1119static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1120{
1121 struct igb_q_vector *q_vector;
1122 struct e1000_hw *hw = &adapter->hw;
1123 int v_idx;
81c2fc22 1124 int orig_node = adapter->node;
047e0030
AD		 1125
1126 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
81c2fc22
AD		 1127 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1128 adapter->num_tx_queues)) &&
1129 (adapter->num_rx_queues == v_idx))
1130 adapter->node = orig_node;
1131 if (orig_node == -1) {
1132 int cur_node = next_online_node(adapter->node);
1133 if (cur_node == MAX_NUMNODES)
1134 cur_node = first_online_node;
1135 adapter->node = cur_node;
1136 }
1137 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1138 adapter->node);
1139 if (!q_vector)
1140 q_vector = kzalloc(sizeof(struct igb_q_vector),
1141 GFP_KERNEL);
047e0030
AD		 1142 if (!q_vector)
1143 goto err_out;
1144 q_vector->adapter = adapter;
047e0030
AD		 1145 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1146 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1147 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1148 adapter->q_vector[v_idx] = q_vector;
1149 }
81c2fc22
AD		 1150 /* Restore the adapter's original node */
1151 adapter->node = orig_node;
1152
047e0030
AD		 1153 return 0;
1154
1155err_out:
81c2fc22
AD		 1156 /* Restore the adapter's original node */
1157 adapter->node = orig_node;
fe0592b4 1158 igb_free_q_vectors(adapter);
047e0030
AD		 1159 return -ENOMEM;
1160}
1161
1162static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1163 int ring_idx, int v_idx)
1164{
3025a446 1165 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1166
0ba82994
AD		 1167 q_vector->rx.ring = adapter->rx_ring[ring_idx];
1168 q_vector->rx.ring->q_vector = q_vector;
1169 q_vector->rx.count++;
4fc82adf
AD		 1170 q_vector->itr_val = adapter->rx_itr_setting;
1171 if (q_vector->itr_val && q_vector->itr_val <= 3)
1172 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1173}
1174
1175static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1176 int ring_idx, int v_idx)
1177{
3025a446 1178 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1179
0ba82994
AD		 1180 q_vector->tx.ring = adapter->tx_ring[ring_idx];
1181 q_vector->tx.ring->q_vector = q_vector;
1182 q_vector->tx.count++;
4fc82adf 1183 q_vector->itr_val = adapter->tx_itr_setting;
0ba82994 1184 q_vector->tx.work_limit = adapter->tx_work_limit;
4fc82adf
AD		 1185 if (q_vector->itr_val && q_vector->itr_val <= 3)
1186 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1187}
1188
1189/**
1190 * igb_map_ring_to_vector - maps allocated queues to vectors
1191 *
1192 * This function maps the recently allocated queues to vectors.
1193 **/
1194static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1195{
1196 int i;
1197 int v_idx = 0;
1198
1199 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1200 (adapter->num_q_vectors < adapter->num_tx_queues))
1201 return -ENOMEM;
1202
1203 if (adapter->num_q_vectors >=
1204 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1205 for (i = 0; i < adapter->num_rx_queues; i++)
1206 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1207 for (i = 0; i < adapter->num_tx_queues; i++)
1208 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1209 } else {
1210 for (i = 0; i < adapter->num_rx_queues; i++) {
1211 if (i < adapter->num_tx_queues)
1212 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1213 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1214 }
1215 for (; i < adapter->num_tx_queues; i++)
1216 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1217 }
1218 return 0;
1219}
1220
1221/**
1222 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1223 *
1224 * This function initializes the interrupts and allocates all of the queues.
1225 **/
1226static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1227{
1228 struct pci_dev *pdev = adapter->pdev;
1229 int err;
1230
21adef3e
BH		 1231 err = igb_set_interrupt_capability(adapter);
1232 if (err)
1233 return err;
047e0030
AD		 1234
1235 err = igb_alloc_q_vectors(adapter);
1236 if (err) {
1237 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1238 goto err_alloc_q_vectors;
1239 }
1240
1241 err = igb_alloc_queues(adapter);
1242 if (err) {
1243 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1244 goto err_alloc_queues;
1245 }
1246
1247 err = igb_map_ring_to_vector(adapter);
1248 if (err) {
1249 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1250 goto err_map_queues;
1251 }
1252
1253
1254 return 0;
1255err_map_queues:
1256 igb_free_queues(adapter);
1257err_alloc_queues:
1258 igb_free_q_vectors(adapter);
1259err_alloc_q_vectors:
1260 igb_reset_interrupt_capability(adapter);
1261 return err;
1262}
1263
9d5c8243
AK		 1264/**
1265 * igb_request_irq - initialize interrupts
1266 *
1267 * Attempts to configure interrupts using the best available
1268 * capabilities of the hardware and kernel.
1269 **/
1270static int igb_request_irq(struct igb_adapter *adapter)
1271{
1272 struct net_device *netdev = adapter->netdev;
047e0030 1273 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1274 int err = 0;
1275
1276 if (adapter->msix_entries) {
1277 err = igb_request_msix(adapter);
844290e5 1278 if (!err)
9d5c8243 1279 goto request_done;
9d5c8243 1280 /* fall back to MSI */
047e0030 1281 igb_clear_interrupt_scheme(adapter);
c74d588e 1282 if (!pci_enable_msi(pdev))
7dfc16fa 1283 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1284 igb_free_all_tx_resources(adapter);
1285 igb_free_all_rx_resources(adapter);
047e0030 1286 adapter->num_tx_queues = 1;
9d5c8243 1287 adapter->num_rx_queues = 1;
047e0030
AD		 1288 adapter->num_q_vectors = 1;
1289 err = igb_alloc_q_vectors(adapter);
1290 if (err) {
1291 dev_err(&pdev->dev,
1292 "Unable to allocate memory for vectors\n");
1293 goto request_done;
1294 }
1295 err = igb_alloc_queues(adapter);
1296 if (err) {
1297 dev_err(&pdev->dev,
1298 "Unable to allocate memory for queues\n");
1299 igb_free_q_vectors(adapter);
1300 goto request_done;
1301 }
1302 igb_setup_all_tx_resources(adapter);
1303 igb_setup_all_rx_resources(adapter);
9d5c8243 1304 }
844290e5 1305
c74d588e
AD		 1306 igb_assign_vector(adapter->q_vector[0], 0);
1307
7dfc16fa 1308 if (adapter->flags & IGB_FLAG_HAS_MSI) {
c74d588e 1309 err = request_irq(pdev->irq, igb_intr_msi, 0,
047e0030 1310 netdev->name, adapter);
9d5c8243
AK		 1311 if (!err)
1312 goto request_done;
047e0030 1313
9d5c8243
AK		 1314 /* fall back to legacy interrupts */
1315 igb_reset_interrupt_capability(adapter);
7dfc16fa 1316 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1317 }
1318
c74d588e 1319 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1320 netdev->name, adapter);
9d5c8243 1321
6cb5e577 1322 if (err)
c74d588e 1323 dev_err(&pdev->dev, "Error %d getting interrupt\n",
9d5c8243 1324 err);
9d5c8243
AK		 1325
1326request_done:
1327 return err;
1328}
1329
1330static void igb_free_irq(struct igb_adapter *adapter)
1331{
9d5c8243
AK		 1332 if (adapter->msix_entries) {
1333 int vector = 0, i;
1334
047e0030 1335 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1336
0d1ae7f4 1337 for (i = 0; i < adapter->num_q_vectors; i++)
047e0030 1338 free_irq(adapter->msix_entries[vector++].vector,
0d1ae7f4 1339 adapter->q_vector[i]);
047e0030
AD		 1340 } else {
1341 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1342 }
9d5c8243
AK		 1343}
1344
1345/**
1346 * igb_irq_disable - Mask off interrupt generation on the NIC
1347 * @adapter: board private structure
1348 **/
1349static void igb_irq_disable(struct igb_adapter *adapter)
1350{
1351 struct e1000_hw *hw = &adapter->hw;
1352
25568a53
AD		 1353 /*
1354 * we need to be careful when disabling interrupts. The VFs are also
1355 * mapped into these registers and so clearing the bits can cause
1356 * issues on the VF drivers so we only need to clear what we set
1357 */
9d5c8243 1358 if (adapter->msix_entries) {
2dfd1212
AD		 1359 u32 regval = rd32(E1000_EIAM);
1360 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1361 wr32(E1000_EIMC, adapter->eims_enable_mask);
1362 regval = rd32(E1000_EIAC);
1363 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1364 }
844290e5
PW		 1365
1366 wr32(E1000_IAM, 0);
9d5c8243
AK		 1367 wr32(E1000_IMC, ~0);
1368 wrfl();
81a61859
ET		 1369 if (adapter->msix_entries) {
1370 int i;
1371 for (i = 0; i < adapter->num_q_vectors; i++)
1372 synchronize_irq(adapter->msix_entries[i].vector);
1373 } else {
1374 synchronize_irq(adapter->pdev->irq);
1375 }
9d5c8243
AK		 1376}
1377
1378/**
1379 * igb_irq_enable - Enable default interrupt generation settings
1380 * @adapter: board private structure
1381 **/
1382static void igb_irq_enable(struct igb_adapter *adapter)
1383{
1384 struct e1000_hw *hw = &adapter->hw;
1385
1386 if (adapter->msix_entries) {
06218a8d 1387 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
2dfd1212
AD		 1388 u32 regval = rd32(E1000_EIAC);
1389 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1390 regval = rd32(E1000_EIAM);
1391 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1392 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1393 if (adapter->vfs_allocated_count) {
4ae196df 1394 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1395 ims |= E1000_IMS_VMMB;
1396 }
1397 wr32(E1000_IMS, ims);
844290e5 1398 } else {
55cac248
AD		 1399 wr32(E1000_IMS, IMS_ENABLE_MASK |
1400 E1000_IMS_DRSTA);
1401 wr32(E1000_IAM, IMS_ENABLE_MASK |
1402 E1000_IMS_DRSTA);
844290e5 1403 }
9d5c8243
AK		 1404}
1405
1406static void igb_update_mng_vlan(struct igb_adapter *adapter)
1407{
51466239 1408 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1409 u16 vid = adapter->hw.mng_cookie.vlan_id;
1410 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1411
1412 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1413 /* add VID to filter table */
1414 igb_vfta_set(hw, vid, true);
1415 adapter->mng_vlan_id = vid;
1416 } else {
1417 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1418 }
1419
1420 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1421 (vid != old_vid) &&
b2cb09b1 1422 !test_bit(old_vid, adapter->active_vlans)) {
51466239
AD		 1423 /* remove VID from filter table */
1424 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1425 }
1426}
1427
1428/**
1429 * igb_release_hw_control - release control of the h/w to f/w
1430 * @adapter: address of board private structure
1431 *
1432 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1433 * For ASF and Pass Through versions of f/w this means that the
1434 * driver is no longer loaded.
1435 *
1436 **/
1437static void igb_release_hw_control(struct igb_adapter *adapter)
1438{
1439 struct e1000_hw *hw = &adapter->hw;
1440 u32 ctrl_ext;
1441
1442 /* Let firmware take over control of h/w */
1443 ctrl_ext = rd32(E1000_CTRL_EXT);
1444 wr32(E1000_CTRL_EXT,
1445 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1446}
1447
9d5c8243
AK		 1448/**
1449 * igb_get_hw_control - get control of the h/w from f/w
1450 * @adapter: address of board private structure
1451 *
1452 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1453 * For ASF and Pass Through versions of f/w this means that
1454 * the driver is loaded.
1455 *
1456 **/
1457static void igb_get_hw_control(struct igb_adapter *adapter)
1458{
1459 struct e1000_hw *hw = &adapter->hw;
1460 u32 ctrl_ext;
1461
1462 /* Let firmware know the driver has taken over */
1463 ctrl_ext = rd32(E1000_CTRL_EXT);
1464 wr32(E1000_CTRL_EXT,
1465 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1466}
1467
9d5c8243
AK		 1468/**
1469 * igb_configure - configure the hardware for RX and TX
1470 * @adapter: private board structure
1471 **/
1472static void igb_configure(struct igb_adapter *adapter)
1473{
1474 struct net_device *netdev = adapter->netdev;
1475 int i;
1476
1477 igb_get_hw_control(adapter);
ff41f8dc 1478 igb_set_rx_mode(netdev);
9d5c8243
AK		 1479
1480 igb_restore_vlan(adapter);
9d5c8243 1481
85b430b4 1482 igb_setup_tctl(adapter);
06cf2666 1483 igb_setup_mrqc(adapter);
9d5c8243 1484 igb_setup_rctl(adapter);
85b430b4
AD		 1485
1486 igb_configure_tx(adapter);
9d5c8243 1487 igb_configure_rx(adapter);
662d7205
AD		 1488
1489 igb_rx_fifo_flush_82575(&adapter->hw);
1490
c493ea45 1491 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1492 * at least 1 descriptor unused to make sure
1493 * next_to_use != next_to_clean */
1494 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1495 struct igb_ring *ring = adapter->rx_ring[i];
cd392f5c 1496 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
9d5c8243 1497 }
9d5c8243
AK		 1498}
1499
88a268c1
NN		 1500/**
1501 * igb_power_up_link - Power up the phy/serdes link
1502 * @adapter: address of board private structure
1503 **/
1504void igb_power_up_link(struct igb_adapter *adapter)
1505{
1506 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1507 igb_power_up_phy_copper(&adapter->hw);
1508 else
1509 igb_power_up_serdes_link_82575(&adapter->hw);
a95a0744 1510 igb_reset_phy(&adapter->hw);
88a268c1
NN		 1511}
1512
1513/**
1514 * igb_power_down_link - Power down the phy/serdes link
1515 * @adapter: address of board private structure
1516 */
1517static void igb_power_down_link(struct igb_adapter *adapter)
1518{
1519 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1520 igb_power_down_phy_copper_82575(&adapter->hw);
1521 else
1522 igb_shutdown_serdes_link_82575(&adapter->hw);
1523}
9d5c8243
AK		 1524
1525/**
1526 * igb_up - Open the interface and prepare it to handle traffic
1527 * @adapter: board private structure
1528 **/
9d5c8243
AK		 1529int igb_up(struct igb_adapter *adapter)
1530{
1531 struct e1000_hw *hw = &adapter->hw;
1532 int i;
1533
1534 /* hardware has been reset, we need to reload some things */
1535 igb_configure(adapter);
1536
1537 clear_bit(__IGB_DOWN, &adapter->state);
1538
0d1ae7f4
AD		 1539 for (i = 0; i < adapter->num_q_vectors; i++)
1540 napi_enable(&(adapter->q_vector[i]->napi));
1541
844290e5 1542 if (adapter->msix_entries)
9d5c8243 1543 igb_configure_msix(adapter);
feeb2721
AD		 1544 else
1545 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1546
1547 /* Clear any pending interrupts. */
1548 rd32(E1000_ICR);
1549 igb_irq_enable(adapter);
1550
d4960307
AD		 1551 /* notify VFs that reset has been completed */
1552 if (adapter->vfs_allocated_count) {
1553 u32 reg_data = rd32(E1000_CTRL_EXT);
1554 reg_data |= E1000_CTRL_EXT_PFRSTD;
1555 wr32(E1000_CTRL_EXT, reg_data);
1556 }
1557
4cb9be7a
JB		 1558 netif_tx_start_all_queues(adapter->netdev);
1559
25568a53
AD		 1560 /* start the watchdog. */
1561 hw->mac.get_link_status = 1;
1562 schedule_work(&adapter->watchdog_task);
1563
9d5c8243
AK		 1564 return 0;
1565}
1566
1567void igb_down(struct igb_adapter *adapter)
1568{
9d5c8243 1569 struct net_device *netdev = adapter->netdev;
330a6d6a 1570 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1571 u32 tctl, rctl;
1572 int i;
1573
1574 /* signal that we're down so the interrupt handler does not
1575 * reschedule our watchdog timer */
1576 set_bit(__IGB_DOWN, &adapter->state);
1577
1578 /* disable receives in the hardware */
1579 rctl = rd32(E1000_RCTL);
1580 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1581 /* flush and sleep below */
1582
fd2ea0a7 1583 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1584
1585 /* disable transmits in the hardware */
1586 tctl = rd32(E1000_TCTL);
1587 tctl &= ~E1000_TCTL_EN;
1588 wr32(E1000_TCTL, tctl);
1589 /* flush both disables and wait for them to finish */
1590 wrfl();
1591 msleep(10);
1592
0d1ae7f4
AD		 1593 for (i = 0; i < adapter->num_q_vectors; i++)
1594 napi_disable(&(adapter->q_vector[i]->napi));
9d5c8243 1595
9d5c8243
AK		 1596 igb_irq_disable(adapter);
1597
1598 del_timer_sync(&adapter->watchdog_timer);
1599 del_timer_sync(&adapter->phy_info_timer);
1600
9d5c8243 1601 netif_carrier_off(netdev);
04fe6358
AD		 1602
1603 /* record the stats before reset*/
12dcd86b
ED		 1604 spin_lock(&adapter->stats64_lock);
1605 igb_update_stats(adapter, &adapter->stats64);
1606 spin_unlock(&adapter->stats64_lock);
04fe6358 1607
9d5c8243
AK		 1608 adapter->link_speed = 0;
1609 adapter->link_duplex = 0;
1610
3023682e
JK		 1611 if (!pci_channel_offline(adapter->pdev))
1612 igb_reset(adapter);
9d5c8243
AK		 1613 igb_clean_all_tx_rings(adapter);
1614 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1615#ifdef CONFIG_IGB_DCA
1616
1617 /* since we reset the hardware DCA settings were cleared */
1618 igb_setup_dca(adapter);
1619#endif
9d5c8243
AK		 1620}
1621
1622void igb_reinit_locked(struct igb_adapter *adapter)
1623{
1624 WARN_ON(in_interrupt());
1625 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1626 msleep(1);
1627 igb_down(adapter);
1628 igb_up(adapter);
1629 clear_bit(__IGB_RESETTING, &adapter->state);
1630}
1631
1632void igb_reset(struct igb_adapter *adapter)
1633{
090b1795 1634 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1635 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1636 struct e1000_mac_info *mac = &hw->mac;
1637 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1638 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1639 u16 hwm;
1640
1641 /* Repartition Pba for greater than 9k mtu
1642 * To take effect CTRL.RST is required.
1643 */
fa4dfae0 1644 switch (mac->type) {
d2ba2ed8 1645 case e1000_i350:
55cac248
AD		 1646 case e1000_82580:
1647 pba = rd32(E1000_RXPBS);
1648 pba = igb_rxpbs_adjust_82580(pba);
1649 break;
fa4dfae0 1650 case e1000_82576:
d249be54
AD		 1651 pba = rd32(E1000_RXPBS);
1652 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1653 break;
1654 case e1000_82575:
1655 default:
1656 pba = E1000_PBA_34K;
1657 break;
2d064c06 1658 }
9d5c8243 1659
2d064c06
AD		 1660 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1661 (mac->type < e1000_82576)) {
9d5c8243
AK		 1662 /* adjust PBA for jumbo frames */
1663 wr32(E1000_PBA, pba);
1664
1665 /* To maintain wire speed transmits, the Tx FIFO should be
1666 * large enough to accommodate two full transmit packets,
1667 * rounded up to the next 1KB and expressed in KB. Likewise,
1668 * the Rx FIFO should be large enough to accommodate at least
1669 * one full receive packet and is similarly rounded up and
1670 * expressed in KB. */
1671 pba = rd32(E1000_PBA);
1672 /* upper 16 bits has Tx packet buffer allocation size in KB */
1673 tx_space = pba >> 16;
1674 /* lower 16 bits has Rx packet buffer allocation size in KB */
1675 pba &= 0xffff;
1676 /* the tx fifo also stores 16 bytes of information about the tx
1677 * but don't include ethernet FCS because hardware appends it */
1678 min_tx_space = (adapter->max_frame_size +
85e8d004 1679 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1680 ETH_FCS_LEN) * 2;
1681 min_tx_space = ALIGN(min_tx_space, 1024);
1682 min_tx_space >>= 10;
1683 /* software strips receive CRC, so leave room for it */
1684 min_rx_space = adapter->max_frame_size;
1685 min_rx_space = ALIGN(min_rx_space, 1024);
1686 min_rx_space >>= 10;
1687
1688 /* If current Tx allocation is less than the min Tx FIFO size,
1689 * and the min Tx FIFO size is less than the current Rx FIFO
1690 * allocation, take space away from current Rx allocation */
1691 if (tx_space < min_tx_space &&
1692 ((min_tx_space - tx_space) < pba)) {
1693 pba = pba - (min_tx_space - tx_space);
1694
1695 /* if short on rx space, rx wins and must trump tx
1696 * adjustment */
1697 if (pba < min_rx_space)
1698 pba = min_rx_space;
1699 }
2d064c06 1700 wr32(E1000_PBA, pba);
9d5c8243 1701 }
9d5c8243
AK		 1702
1703 /* flow control settings */
1704 /* The high water mark must be low enough to fit one full frame
1705 * (or the size used for early receive) above it in the Rx FIFO.
1706 * Set it to the lower of:
1707 * - 90% of the Rx FIFO size, or
1708 * - the full Rx FIFO size minus one full frame */
1709 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1710 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1711
d405ea3e
AD		 1712 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1713 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1714 fc->pause_time = 0xFFFF;
1715 fc->send_xon = 1;
0cce119a 1716 fc->current_mode = fc->requested_mode;
9d5c8243 1717
4ae196df
AD		 1718 /* disable receive for all VFs and wait one second */
1719 if (adapter->vfs_allocated_count) {
1720 int i;
1721 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
8fa7e0f7 1722 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
4ae196df
AD		 1723
1724 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1725 igb_ping_all_vfs(adapter);
4ae196df
AD		 1726
1727 /* disable transmits and receives */
1728 wr32(E1000_VFRE, 0);
1729 wr32(E1000_VFTE, 0);
1730 }
1731
9d5c8243 1732 /* Allow time for pending master requests to run */
330a6d6a 1733 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1734 wr32(E1000_WUC, 0);
1735
330a6d6a 1736 if (hw->mac.ops.init_hw(hw))
090b1795 1737 dev_err(&pdev->dev, "Hardware Error\n");
831ec0b4 1738
b6e0c419 1739 igb_init_dmac(adapter, pba);
88a268c1
NN		 1740 if (!netif_running(adapter->netdev))
1741 igb_power_down_link(adapter);
1742
9d5c8243
AK		 1743 igb_update_mng_vlan(adapter);
1744
1745 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1746 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1747
330a6d6a 1748 igb_get_phy_info(hw);
9d5c8243
AK		 1749}
1750
c8f44aff
MM		 1751static netdev_features_t igb_fix_features(struct net_device *netdev,
1752 netdev_features_t features)
b2cb09b1
JP		 1753{
1754 /*
1755 * Since there is no support for separate rx/tx vlan accel
1756 * enable/disable make sure tx flag is always in same state as rx.
1757 */
1758 if (features & NETIF_F_HW_VLAN_RX)
1759 features |= NETIF_F_HW_VLAN_TX;
1760 else
1761 features &= ~NETIF_F_HW_VLAN_TX;
1762
1763 return features;
1764}
1765
c8f44aff
MM		 1766static int igb_set_features(struct net_device *netdev,
1767 netdev_features_t features)
ac52caa3 1768{
c8f44aff 1769 netdev_features_t changed = netdev->features ^ features;
ac52caa3 1770
b2cb09b1
JP		 1771 if (changed & NETIF_F_HW_VLAN_RX)
1772 igb_vlan_mode(netdev, features);
1773
ac52caa3
MM		 1774 return 0;
1775}
1776
2e5c6922 1777static const struct net_device_ops igb_netdev_ops = {
559e9c49 1778 .ndo_open = igb_open,
2e5c6922 1779 .ndo_stop = igb_close,
cd392f5c 1780 .ndo_start_xmit = igb_xmit_frame,
12dcd86b 1781 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc 1782 .ndo_set_rx_mode = igb_set_rx_mode,
2e5c6922
SH		 1783 .ndo_set_mac_address = igb_set_mac,
1784 .ndo_change_mtu = igb_change_mtu,
1785 .ndo_do_ioctl = igb_ioctl,
1786 .ndo_tx_timeout = igb_tx_timeout,
1787 .ndo_validate_addr = eth_validate_addr,
2e5c6922
SH		 1788 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1789 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1790 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1791 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1792 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1793 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1794#ifdef CONFIG_NET_POLL_CONTROLLER
1795 .ndo_poll_controller = igb_netpoll,
1796#endif
b2cb09b1
JP		 1797 .ndo_fix_features = igb_fix_features,
1798 .ndo_set_features = igb_set_features,
2e5c6922
SH		 1799};
1800
9d5c8243
AK		 1801/**
1802 * igb_probe - Device Initialization Routine
1803 * @pdev: PCI device information struct
1804 * @ent: entry in igb_pci_tbl
1805 *
1806 * Returns 0 on success, negative on failure
1807 *
1808 * igb_probe initializes an adapter identified by a pci_dev structure.
1809 * The OS initialization, configuring of the adapter private structure,
1810 * and a hardware reset occur.
1811 **/
1812static int __devinit igb_probe(struct pci_dev *pdev,
1813 const struct pci_device_id *ent)
1814{
1815 struct net_device *netdev;
1816 struct igb_adapter *adapter;
1817 struct e1000_hw *hw;
4337e993 1818 u16 eeprom_data = 0;
9835fd73 1819 s32 ret_val;
4337e993 1820 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1821 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1822 unsigned long mmio_start, mmio_len;
2d6a5e95 1823 int err, pci_using_dac;
9d5c8243 1824 u16 eeprom_apme_mask = IGB_EEPROM_APME;
9835fd73 1825 u8 part_str[E1000_PBANUM_LENGTH];
9d5c8243 1826
bded64a7
AG		 1827 /* Catch broken hardware that put the wrong VF device ID in
1828 * the PCIe SR-IOV capability.
1829 */
1830 if (pdev->is_virtfn) {
1831 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1832 pci_name(pdev), pdev->vendor, pdev->device);
1833 return -EINVAL;
1834 }
1835
aed5dec3 1836 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1837 if (err)
1838 return err;
1839
1840 pci_using_dac = 0;
59d71989 1841 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1842 if (!err) {
59d71989 1843 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1844 if (!err)
1845 pci_using_dac = 1;
1846 } else {
59d71989 1847 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1848 if (err) {
59d71989 1849 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1850 if (err) {
1851 dev_err(&pdev->dev, "No usable DMA "
1852 "configuration, aborting\n");
1853 goto err_dma;
1854 }
1855 }
1856 }
1857
aed5dec3
AD		 1858 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1859 IORESOURCE_MEM),
1860 igb_driver_name);
9d5c8243
AK		 1861 if (err)
1862 goto err_pci_reg;
1863
19d5afd4 1864 pci_enable_pcie_error_reporting(pdev);
40a914fa 1865
9d5c8243 1866 pci_set_master(pdev);
c682fc23 1867 pci_save_state(pdev);
9d5c8243
AK		 1868
1869 err = -ENOMEM;
1bfaf07b 1870 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1cc3bd87 1871 IGB_MAX_TX_QUEUES);
9d5c8243
AK		 1872 if (!netdev)
1873 goto err_alloc_etherdev;
1874
1875 SET_NETDEV_DEV(netdev, &pdev->dev);
1876
1877 pci_set_drvdata(pdev, netdev);
1878 adapter = netdev_priv(netdev);
1879 adapter->netdev = netdev;
1880 adapter->pdev = pdev;
1881 hw = &adapter->hw;
1882 hw->back = adapter;
1883 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1884
1885 mmio_start = pci_resource_start(pdev, 0);
1886 mmio_len = pci_resource_len(pdev, 0);
1887
1888 err = -EIO;
28b0759c
AD		 1889 hw->hw_addr = ioremap(mmio_start, mmio_len);
1890 if (!hw->hw_addr)
9d5c8243
AK		 1891 goto err_ioremap;
1892
2e5c6922 1893 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1894 igb_set_ethtool_ops(netdev);
9d5c8243 1895 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1896
1897 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1898
1899 netdev->mem_start = mmio_start;
1900 netdev->mem_end = mmio_start + mmio_len;
1901
9d5c8243
AK		 1902 /* PCI config space info */
1903 hw->vendor_id = pdev->vendor;
1904 hw->device_id = pdev->device;
1905 hw->revision_id = pdev->revision;
1906 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1907 hw->subsystem_device_id = pdev->subsystem_device;
1908
9d5c8243
AK		 1909 /* Copy the default MAC, PHY and NVM function pointers */
1910 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1911 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1912 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1913 /* Initialize skew-specific constants */
1914 err = ei->get_invariants(hw);
1915 if (err)
450c87c8 1916 goto err_sw_init;
9d5c8243 1917
450c87c8 1918 /* setup the private structure */
9d5c8243
AK		 1919 err = igb_sw_init(adapter);
1920 if (err)
1921 goto err_sw_init;
1922
1923 igb_get_bus_info_pcie(hw);
1924
1925 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1926
1927 /* Copper options */
1928 if (hw->phy.media_type == e1000_media_type_copper) {
1929 hw->phy.mdix = AUTO_ALL_MODES;
1930 hw->phy.disable_polarity_correction = false;
1931 hw->phy.ms_type = e1000_ms_hw_default;
1932 }
1933
1934 if (igb_check_reset_block(hw))
1935 dev_info(&pdev->dev,
1936 "PHY reset is blocked due to SOL/IDER session.\n");
1937
077887c3
AD		 1938 /*
1939 * features is initialized to 0 in allocation, it might have bits
1940 * set by igb_sw_init so we should use an or instead of an
1941 * assignment.
1942 */
1943 netdev->features |= NETIF_F_SG |
1944 NETIF_F_IP_CSUM |
1945 NETIF_F_IPV6_CSUM |
1946 NETIF_F_TSO |
1947 NETIF_F_TSO6 |
1948 NETIF_F_RXHASH |
1949 NETIF_F_RXCSUM |
1950 NETIF_F_HW_VLAN_RX |
1951 NETIF_F_HW_VLAN_TX;
1952
1953 /* copy netdev features into list of user selectable features */
1954 netdev->hw_features |= netdev->features;
1955
1956 /* set this bit last since it cannot be part of hw_features */
1957 netdev->features |= NETIF_F_HW_VLAN_FILTER;
1958
1959 netdev->vlan_features |= NETIF_F_TSO |
1960 NETIF_F_TSO6 |
1961 NETIF_F_IP_CSUM |
1962 NETIF_F_IPV6_CSUM |
1963 NETIF_F_SG;
48f29ffc 1964
7b872a55 1965 if (pci_using_dac) {
9d5c8243 1966 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 1967 netdev->vlan_features |= NETIF_F_HIGHDMA;
1968 }
9d5c8243 1969
ac52caa3
MM		 1970 if (hw->mac.type >= e1000_82576) {
1971 netdev->hw_features |= NETIF_F_SCTP_CSUM;
b9473560 1972 netdev->features |= NETIF_F_SCTP_CSUM;
ac52caa3 1973 }
b9473560 1974
01789349
JP		 1975 netdev->priv_flags |= IFF_UNICAST_FLT;
1976
330a6d6a 1977 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 1978
1979 /* before reading the NVM, reset the controller to put the device in a
1980 * known good starting state */
1981 hw->mac.ops.reset_hw(hw);
1982
1983 /* make sure the NVM is good */
4322e561 1984 if (hw->nvm.ops.validate(hw) < 0) {
9d5c8243
AK		 1985 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1986 err = -EIO;
1987 goto err_eeprom;
1988 }
1989
1990 /* copy the MAC address out of the NVM */
1991 if (hw->mac.ops.read_mac_addr(hw))
1992 dev_err(&pdev->dev, "NVM Read Error\n");
1993
1994 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1995 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1996
1997 if (!is_valid_ether_addr(netdev->perm_addr)) {
1998 dev_err(&pdev->dev, "Invalid MAC Address\n");
1999 err = -EIO;
2000 goto err_eeprom;
2001 }
2002
c061b18d 2003 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 2004 (unsigned long) adapter);
c061b18d 2005 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 2006 (unsigned long) adapter);
9d5c8243
AK		 2007
2008 INIT_WORK(&adapter->reset_task, igb_reset_task);
2009 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2010
450c87c8 2011 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 2012 adapter->fc_autoneg = true;
2013 hw->mac.autoneg = true;
2014 hw->phy.autoneg_advertised = 0x2f;
2015
0cce119a
AD		 2016 hw->fc.requested_mode = e1000_fc_default;
2017 hw->fc.current_mode = e1000_fc_default;
9d5c8243 2018
9d5c8243
AK		 2019 igb_validate_mdi_setting(hw);
2020
9d5c8243
AK		 2021 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2022 * enable the ACPI Magic Packet filter
2023 */
2024
a2cf8b6c 2025 if (hw->bus.func == 0)
312c75ae 2026 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
6d337dce 2027 else if (hw->mac.type >= e1000_82580)
55cac248
AD		 2028 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2029 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2030 &eeprom_data);
a2cf8b6c
AD		 2031 else if (hw->bus.func == 1)
2032 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 2033
2034 if (eeprom_data & eeprom_apme_mask)
2035 adapter->eeprom_wol |= E1000_WUFC_MAG;
2036
2037 /* now that we have the eeprom settings, apply the special cases where
2038 * the eeprom may be wrong or the board simply won't support wake on
2039 * lan on a particular port */
2040 switch (pdev->device) {
2041 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2042 adapter->eeprom_wol = 0;
2043 break;
2044 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 2045 case E1000_DEV_ID_82576_FIBER:
2046 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 2047 /* Wake events only supported on port A for dual fiber
2048 * regardless of eeprom setting */
2049 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2050 adapter->eeprom_wol = 0;
2051 break;
c8ea5ea9 2052 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 2053 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 2054 /* if quad port adapter, disable WoL on all but port A */
2055 if (global_quad_port_a != 0)
2056 adapter->eeprom_wol = 0;
2057 else
2058 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2059 /* Reset for multiple quad port adapters */
2060 if (++global_quad_port_a == 4)
2061 global_quad_port_a = 0;
2062 break;
9d5c8243
AK		 2063 }
2064
2065 /* initialize the wol settings based on the eeprom settings */
2066 adapter->wol = adapter->eeprom_wol;
e1b86d84 2067 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 2068
2069 /* reset the hardware with the new settings */
2070 igb_reset(adapter);
2071
2072 /* let the f/w know that the h/w is now under the control of the
2073 * driver. */
2074 igb_get_hw_control(adapter);
2075
9d5c8243
AK		 2076 strcpy(netdev->name, "eth%d");
2077 err = register_netdev(netdev);
2078 if (err)
2079 goto err_register;
2080
b168dfc5
JB		 2081 /* carrier off reporting is important to ethtool even BEFORE open */
2082 netif_carrier_off(netdev);
2083
421e02f0 2084#ifdef CONFIG_IGB_DCA
bbd98fe4 2085 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 2086 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 2087 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 2088 igb_setup_dca(adapter);
2089 }
fe4506b6 2090
38c845c7 2091#endif
673b8b70
AB		 2092 /* do hw tstamp init after resetting */
2093 igb_init_hw_timer(adapter);
2094
9d5c8243
AK		 2095 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2096 /* print bus type/speed/width info */
7c510e4b 2097 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 2098 netdev->name,
559e9c49 2099 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 2100 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 2101 "unknown"),
59c3de89
AD		 2102 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2103 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2104 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2105 "unknown"),
7c510e4b 2106 netdev->dev_addr);
9d5c8243 2107
9835fd73
CW		 2108 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2109 if (ret_val)
2110 strcpy(part_str, "Unknown");
2111 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
9d5c8243
AK		 2112 dev_info(&pdev->dev,
2113 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2114 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2115 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243 2116 adapter->num_rx_queues, adapter->num_tx_queues);
09b068d4
CW		 2117 switch (hw->mac.type) {
2118 case e1000_i350:
2119 igb_set_eee_i350(hw);
2120 break;
2121 default:
2122 break;
2123 }
749ab2cd
YZ		 2124
2125 pm_runtime_put_noidle(&pdev->dev);
9d5c8243
AK		 2126 return 0;
2127
2128err_register:
2129 igb_release_hw_control(adapter);
2130err_eeprom:
2131 if (!igb_check_reset_block(hw))
f5f4cf08 2132 igb_reset_phy(hw);
9d5c8243
AK		 2133
2134 if (hw->flash_address)
2135 iounmap(hw->flash_address);
9d5c8243 2136err_sw_init:
047e0030 2137 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2138 iounmap(hw->hw_addr);
2139err_ioremap:
2140 free_netdev(netdev);
2141err_alloc_etherdev:
559e9c49
AD		 2142 pci_release_selected_regions(pdev,
2143 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2144err_pci_reg:
2145err_dma:
2146 pci_disable_device(pdev);
2147 return err;
2148}
2149
2150/**
2151 * igb_remove - Device Removal Routine
2152 * @pdev: PCI device information struct
2153 *
2154 * igb_remove is called by the PCI subsystem to alert the driver
2155 * that it should release a PCI device. The could be caused by a
2156 * Hot-Plug event, or because the driver is going to be removed from
2157 * memory.
2158 **/
2159static void __devexit igb_remove(struct pci_dev *pdev)
2160{
2161 struct net_device *netdev = pci_get_drvdata(pdev);
2162 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2163 struct e1000_hw *hw = &adapter->hw;
9d5c8243 2164
749ab2cd
YZ		 2165 pm_runtime_get_noresume(&pdev->dev);
2166
760141a5
TH		 2167 /*
2168 * The watchdog timer may be rescheduled, so explicitly
2169 * disable watchdog from being rescheduled.
2170 */
9d5c8243
AK		 2171 set_bit(__IGB_DOWN, &adapter->state);
2172 del_timer_sync(&adapter->watchdog_timer);
2173 del_timer_sync(&adapter->phy_info_timer);
2174
760141a5
TH		 2175 cancel_work_sync(&adapter->reset_task);
2176 cancel_work_sync(&adapter->watchdog_task);
9d5c8243 2177
421e02f0 2178#ifdef CONFIG_IGB_DCA
7dfc16fa 2179 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2180 dev_info(&pdev->dev, "DCA disabled\n");
2181 dca_remove_requester(&pdev->dev);
7dfc16fa 2182 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2183 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2184 }
2185#endif
2186
9d5c8243
AK		 2187 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2188 * would have already happened in close and is redundant. */
2189 igb_release_hw_control(adapter);
2190
2191 unregister_netdev(netdev);
2192
047e0030 2193 igb_clear_interrupt_scheme(adapter);
9d5c8243 2194
37680117
AD		 2195#ifdef CONFIG_PCI_IOV
2196 /* reclaim resources allocated to VFs */
2197 if (adapter->vf_data) {
2198 /* disable iov and allow time for transactions to clear */
0224d663
GR		 2199 if (!igb_check_vf_assignment(adapter)) {
2200 pci_disable_sriov(pdev);
2201 msleep(500);
2202 } else {
2203 dev_info(&pdev->dev, "VF(s) assigned to guests!\n");
2204 }
37680117
AD		 2205
2206 kfree(adapter->vf_data);
2207 adapter->vf_data = NULL;
2208 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 2209 wrfl();
37680117
AD		 2210 msleep(100);
2211 dev_info(&pdev->dev, "IOV Disabled\n");
2212 }
2213#endif
559e9c49 2214
28b0759c
AD		 2215 iounmap(hw->hw_addr);
2216 if (hw->flash_address)
2217 iounmap(hw->flash_address);
559e9c49
AD		 2218 pci_release_selected_regions(pdev,
2219 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243 2220
1128c756 2221 kfree(adapter->shadow_vfta);
9d5c8243
AK		 2222 free_netdev(netdev);
2223
19d5afd4 2224 pci_disable_pcie_error_reporting(pdev);
40a914fa 2225
9d5c8243
AK		 2226 pci_disable_device(pdev);
2227}
2228
a6b623e0
AD		 2229/**
2230 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2231 * @adapter: board private structure to initialize
2232 *
2233 * This function initializes the vf specific data storage and then attempts to
2234 * allocate the VFs. The reason for ordering it this way is because it is much
2235 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2236 * the memory for the VFs.
2237 **/
2238static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2239{
2240#ifdef CONFIG_PCI_IOV
2241 struct pci_dev *pdev = adapter->pdev;
0224d663
GR		 2242 int old_vfs = igb_find_enabled_vfs(adapter);
2243 int i;
a6b623e0 2244
0224d663
GR		 2245 if (old_vfs) {
2246 dev_info(&pdev->dev, "%d pre-allocated VFs found - override "
2247 "max_vfs setting of %d\n", old_vfs, max_vfs);
2248 adapter->vfs_allocated_count = old_vfs;
a6b623e0
AD		 2249 }
2250
0224d663
GR		 2251 if (!adapter->vfs_allocated_count)
2252 return;
2253
2254 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2255 sizeof(struct vf_data_storage), GFP_KERNEL);
2256 /* if allocation failed then we do not support SR-IOV */
2257 if (!adapter->vf_data) {
a6b623e0 2258 adapter->vfs_allocated_count = 0;
0224d663
GR		 2259 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2260 "Data Storage\n");
2261 goto out;
a6b623e0 2262 }
0224d663
GR		 2263
2264 if (!old_vfs) {
2265 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count))
2266 goto err_out;
2267 }
2268 dev_info(&pdev->dev, "%d VFs allocated\n",
2269 adapter->vfs_allocated_count);
2270 for (i = 0; i < adapter->vfs_allocated_count; i++)
2271 igb_vf_configure(adapter, i);
2272
2273 /* DMA Coalescing is not supported in IOV mode. */
2274 adapter->flags &= ~IGB_FLAG_DMAC;
2275 goto out;
2276err_out:
2277 kfree(adapter->vf_data);
2278 adapter->vf_data = NULL;
2279 adapter->vfs_allocated_count = 0;
2280out:
2281 return;
a6b623e0
AD		 2282#endif /* CONFIG_PCI_IOV */
2283}
2284
115f459a
AD		 2285/**
2286 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2287 * @adapter: board private structure to initialize
2288 *
2289 * igb_init_hw_timer initializes the function pointer and values for the hw
2290 * timer found in hardware.
2291 **/
2292static void igb_init_hw_timer(struct igb_adapter *adapter)
2293{
2294 struct e1000_hw *hw = &adapter->hw;
2295
2296 switch (hw->mac.type) {
d2ba2ed8 2297 case e1000_i350:
55cac248
AD		 2298 case e1000_82580:
2299 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2300 adapter->cycles.read = igb_read_clock;
2301 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2302 adapter->cycles.mult = 1;
2303 /*
2304 * The 82580 timesync updates the system timer every 8ns by 8ns
2305 * and the value cannot be shifted. Instead we need to shift
2306 * the registers to generate a 64bit timer value. As a result
2307 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2308 * 24 in order to generate a larger value for synchronization.
2309 */
2310 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2311 /* disable system timer temporarily by setting bit 31 */
2312 wr32(E1000_TSAUXC, 0x80000000);
2313 wrfl();
2314
2315 /* Set registers so that rollover occurs soon to test this. */
2316 wr32(E1000_SYSTIMR, 0x00000000);
2317 wr32(E1000_SYSTIML, 0x80000000);
2318 wr32(E1000_SYSTIMH, 0x000000FF);
2319 wrfl();
2320
2321 /* enable system timer by clearing bit 31 */
2322 wr32(E1000_TSAUXC, 0x0);
2323 wrfl();
2324
2325 timecounter_init(&adapter->clock,
2326 &adapter->cycles,
2327 ktime_to_ns(ktime_get_real()));
2328 /*
2329 * Synchronize our NIC clock against system wall clock. NIC
2330 * time stamp reading requires ~3us per sample, each sample
2331 * was pretty stable even under load => only require 10
2332 * samples for each offset comparison.
2333 */
2334 memset(&adapter->compare, 0, sizeof(adapter->compare));
2335 adapter->compare.source = &adapter->clock;
2336 adapter->compare.target = ktime_get_real;
2337 adapter->compare.num_samples = 10;
2338 timecompare_update(&adapter->compare, 0);
2339 break;
115f459a
AD		 2340 case e1000_82576:
2341 /*
2342 * Initialize hardware timer: we keep it running just in case
2343 * that some program needs it later on.
2344 */
2345 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2346 adapter->cycles.read = igb_read_clock;
2347 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2348 adapter->cycles.mult = 1;
2349 /**
2350 * Scale the NIC clock cycle by a large factor so that
2351 * relatively small clock corrections can be added or
25985edc 2352 * subtracted at each clock tick. The drawbacks of a large
115f459a
AD		 2353 * factor are a) that the clock register overflows more quickly
2354 * (not such a big deal) and b) that the increment per tick has
2355 * to fit into 24 bits. As a result we need to use a shift of
2356 * 19 so we can fit a value of 16 into the TIMINCA register.
2357 */
2358 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2359 wr32(E1000_TIMINCA,
2360 (1 << E1000_TIMINCA_16NS_SHIFT) |
2361 (16 << IGB_82576_TSYNC_SHIFT));
2362
2363 /* Set registers so that rollover occurs soon to test this. */
2364 wr32(E1000_SYSTIML, 0x00000000);
2365 wr32(E1000_SYSTIMH, 0xFF800000);
2366 wrfl();
2367
2368 timecounter_init(&adapter->clock,
2369 &adapter->cycles,
2370 ktime_to_ns(ktime_get_real()));
2371 /*
2372 * Synchronize our NIC clock against system wall clock. NIC
2373 * time stamp reading requires ~3us per sample, each sample
2374 * was pretty stable even under load => only require 10
2375 * samples for each offset comparison.
2376 */
2377 memset(&adapter->compare, 0, sizeof(adapter->compare));
2378 adapter->compare.source = &adapter->clock;
2379 adapter->compare.target = ktime_get_real;
2380 adapter->compare.num_samples = 10;
2381 timecompare_update(&adapter->compare, 0);
2382 break;
2383 case e1000_82575:
2384 /* 82575 does not support timesync */
2385 default:
2386 break;
2387 }
2388
2389}
2390
9d5c8243
AK		 2391/**
2392 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2393 * @adapter: board private structure to initialize
2394 *
2395 * igb_sw_init initializes the Adapter private data structure.
2396 * Fields are initialized based on PCI device information and
2397 * OS network device settings (MTU size).
2398 **/
2399static int __devinit igb_sw_init(struct igb_adapter *adapter)
2400{
2401 struct e1000_hw *hw = &adapter->hw;
2402 struct net_device *netdev = adapter->netdev;
2403 struct pci_dev *pdev = adapter->pdev;
2404
2405 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2406
13fde97a 2407 /* set default ring sizes */
68fd9910
AD		 2408 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2409 adapter->rx_ring_count = IGB_DEFAULT_RXD;
13fde97a
AD		 2410
2411 /* set default ITR values */
4fc82adf
AD		 2412 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2413 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2414
13fde97a
AD		 2415 /* set default work limits */
2416 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2417
153285f9
AD		 2418 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2419 VLAN_HLEN;
9d5c8243
AK		 2420 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2421
81c2fc22
AD		 2422 adapter->node = -1;
2423
12dcd86b 2424 spin_lock_init(&adapter->stats64_lock);
a6b623e0 2425#ifdef CONFIG_PCI_IOV
6b78bb1d
CW		 2426 switch (hw->mac.type) {
2427 case e1000_82576:
2428 case e1000_i350:
9b082d73
SA		 2429 if (max_vfs > 7) {
2430 dev_warn(&pdev->dev,
2431 "Maximum of 7 VFs per PF, using max\n");
2432 adapter->vfs_allocated_count = 7;
2433 } else
2434 adapter->vfs_allocated_count = max_vfs;
6b78bb1d
CW		 2435 break;
2436 default:
2437 break;
2438 }
a6b623e0 2439#endif /* CONFIG_PCI_IOV */
a99955fc 2440 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
665c8c8e
WM		 2441 /* i350 cannot do RSS and SR-IOV at the same time */
2442 if (hw->mac.type == e1000_i350 && adapter->vfs_allocated_count)
2443 adapter->rss_queues = 1;
a99955fc
AD		 2444
2445 /*
2446 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2447 * then we should combine the queues into a queue pair in order to
2448 * conserve interrupts due to limited supply
2449 */
2450 if ((adapter->rss_queues > 4) ||
2451 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2452 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2453
1128c756
CW		 2454 /* Setup and initialize a copy of the hw vlan table array */
2455 adapter->shadow_vfta = kzalloc(sizeof(u32) *
2456 E1000_VLAN_FILTER_TBL_SIZE,
2457 GFP_ATOMIC);
2458
a6b623e0 2459 /* This call may decrease the number of queues */
047e0030 2460 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2461 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2462 return -ENOMEM;
2463 }
2464
a6b623e0
AD		 2465 igb_probe_vfs(adapter);
2466
9d5c8243
AK		 2467 /* Explicitly disable IRQ since the NIC can be in any state. */
2468 igb_irq_disable(adapter);
2469
831ec0b4
CW		 2470 if (hw->mac.type == e1000_i350)
2471 adapter->flags &= ~IGB_FLAG_DMAC;
2472
9d5c8243
AK		 2473 set_bit(__IGB_DOWN, &adapter->state);
2474 return 0;
2475}
2476
2477/**
2478 * igb_open - Called when a network interface is made active
2479 * @netdev: network interface device structure
2480 *
2481 * Returns 0 on success, negative value on failure
2482 *
2483 * The open entry point is called when a network interface is made
2484 * active by the system (IFF_UP). At this point all resources needed
2485 * for transmit and receive operations are allocated, the interrupt
2486 * handler is registered with the OS, the watchdog timer is started,
2487 * and the stack is notified that the interface is ready.
2488 **/
749ab2cd 2489static int __igb_open(struct net_device *netdev, bool resuming)
9d5c8243
AK		 2490{
2491 struct igb_adapter *adapter = netdev_priv(netdev);
2492 struct e1000_hw *hw = &adapter->hw;
749ab2cd 2493 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2494 int err;
2495 int i;
2496
2497 /* disallow open during test */
749ab2cd
YZ		 2498 if (test_bit(__IGB_TESTING, &adapter->state)) {
2499 WARN_ON(resuming);
9d5c8243 2500 return -EBUSY;
749ab2cd
YZ		 2501 }
2502
2503 if (!resuming)
2504 pm_runtime_get_sync(&pdev->dev);
9d5c8243 2505
b168dfc5
JB		 2506 netif_carrier_off(netdev);
2507
9d5c8243
AK		 2508 /* allocate transmit descriptors */
2509 err = igb_setup_all_tx_resources(adapter);
2510 if (err)
2511 goto err_setup_tx;
2512
2513 /* allocate receive descriptors */
2514 err = igb_setup_all_rx_resources(adapter);
2515 if (err)
2516 goto err_setup_rx;
2517
88a268c1 2518 igb_power_up_link(adapter);
9d5c8243 2519
9d5c8243
AK		 2520 /* before we allocate an interrupt, we must be ready to handle it.
2521 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2522 * as soon as we call pci_request_irq, so we have to setup our
2523 * clean_rx handler before we do so. */
2524 igb_configure(adapter);
2525
2526 err = igb_request_irq(adapter);
2527 if (err)
2528 goto err_req_irq;
2529
2530 /* From here on the code is the same as igb_up() */
2531 clear_bit(__IGB_DOWN, &adapter->state);
2532
0d1ae7f4
AD		 2533 for (i = 0; i < adapter->num_q_vectors; i++)
2534 napi_enable(&(adapter->q_vector[i]->napi));
9d5c8243
AK		 2535
2536 /* Clear any pending interrupts. */
2537 rd32(E1000_ICR);
844290e5
PW		 2538
2539 igb_irq_enable(adapter);
2540
d4960307
AD		 2541 /* notify VFs that reset has been completed */
2542 if (adapter->vfs_allocated_count) {
2543 u32 reg_data = rd32(E1000_CTRL_EXT);
2544 reg_data |= E1000_CTRL_EXT_PFRSTD;
2545 wr32(E1000_CTRL_EXT, reg_data);
2546 }
2547
d55b53ff
JK		 2548 netif_tx_start_all_queues(netdev);
2549
749ab2cd
YZ		 2550 if (!resuming)
2551 pm_runtime_put(&pdev->dev);
2552
25568a53
AD		 2553 /* start the watchdog. */
2554 hw->mac.get_link_status = 1;
2555 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2556
2557 return 0;
2558
2559err_req_irq:
2560 igb_release_hw_control(adapter);
88a268c1 2561 igb_power_down_link(adapter);
9d5c8243
AK		 2562 igb_free_all_rx_resources(adapter);
2563err_setup_rx:
2564 igb_free_all_tx_resources(adapter);
2565err_setup_tx:
2566 igb_reset(adapter);
749ab2cd
YZ		 2567 if (!resuming)
2568 pm_runtime_put(&pdev->dev);
9d5c8243
AK		 2569
2570 return err;
2571}
2572
749ab2cd
YZ		 2573static int igb_open(struct net_device *netdev)
2574{
2575 return __igb_open(netdev, false);
2576}
2577
9d5c8243
AK		 2578/**
2579 * igb_close - Disables a network interface
2580 * @netdev: network interface device structure
2581 *
2582 * Returns 0, this is not allowed to fail
2583 *
2584 * The close entry point is called when an interface is de-activated
2585 * by the OS. The hardware is still under the driver's control, but
2586 * needs to be disabled. A global MAC reset is issued to stop the
2587 * hardware, and all transmit and receive resources are freed.
2588 **/
749ab2cd 2589static int __igb_close(struct net_device *netdev, bool suspending)
9d5c8243
AK		 2590{
2591 struct igb_adapter *adapter = netdev_priv(netdev);
749ab2cd 2592 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2593
2594 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
9d5c8243 2595
749ab2cd
YZ		 2596 if (!suspending)
2597 pm_runtime_get_sync(&pdev->dev);
2598
2599 igb_down(adapter);
9d5c8243
AK		 2600 igb_free_irq(adapter);
2601
2602 igb_free_all_tx_resources(adapter);
2603 igb_free_all_rx_resources(adapter);
2604
749ab2cd
YZ		 2605 if (!suspending)
2606 pm_runtime_put_sync(&pdev->dev);
9d5c8243
AK		 2607 return 0;
2608}
2609
749ab2cd
YZ		 2610static int igb_close(struct net_device *netdev)
2611{
2612 return __igb_close(netdev, false);
2613}
2614
9d5c8243
AK		 2615/**
2616 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2617 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2618 *
2619 * Return 0 on success, negative on failure
2620 **/
80785298 2621int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2622{
59d71989 2623 struct device *dev = tx_ring->dev;
81c2fc22 2624 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2625 int size;
2626
06034649 2627 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
81c2fc22
AD		 2628 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2629 if (!tx_ring->tx_buffer_info)
2630 tx_ring->tx_buffer_info = vzalloc(size);
06034649 2631 if (!tx_ring->tx_buffer_info)
9d5c8243 2632 goto err;
9d5c8243
AK		 2633
2634 /* round up to nearest 4K */
85e8d004 2635 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2636 tx_ring->size = ALIGN(tx_ring->size, 4096);
2637
81c2fc22 2638 set_dev_node(dev, tx_ring->numa_node);
59d71989
AD		 2639 tx_ring->desc = dma_alloc_coherent(dev,
2640 tx_ring->size,
2641 &tx_ring->dma,
2642 GFP_KERNEL);
81c2fc22
AD		 2643 set_dev_node(dev, orig_node);
2644 if (!tx_ring->desc)
2645 tx_ring->desc = dma_alloc_coherent(dev,
2646 tx_ring->size,
2647 &tx_ring->dma,
2648 GFP_KERNEL);
9d5c8243
AK		 2649
2650 if (!tx_ring->desc)
2651 goto err;
2652
9d5c8243
AK		 2653 tx_ring->next_to_use = 0;
2654 tx_ring->next_to_clean = 0;
81c2fc22 2655
9d5c8243
AK		 2656 return 0;
2657
2658err:
06034649 2659 vfree(tx_ring->tx_buffer_info);
59d71989 2660 dev_err(dev,
9d5c8243
AK		 2661 "Unable to allocate memory for the transmit descriptor ring\n");
2662 return -ENOMEM;
2663}
2664
2665/**
2666 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2667 * (Descriptors) for all queues
2668 * @adapter: board private structure
2669 *
2670 * Return 0 on success, negative on failure
2671 **/
2672static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2673{
439705e1 2674 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2675 int i, err = 0;
2676
2677 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2678 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2679 if (err) {
439705e1 2680 dev_err(&pdev->dev,
9d5c8243
AK		 2681 "Allocation for Tx Queue %u failed\n", i);
2682 for (i--; i >= 0; i--)
3025a446 2683 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2684 break;
2685 }
2686 }
2687
2688 return err;
2689}
2690
2691/**
85b430b4
AD		 2692 * igb_setup_tctl - configure the transmit control registers
2693 * @adapter: Board private structure
9d5c8243 2694 **/
d7ee5b3a 2695void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2696{
9d5c8243
AK		 2697 struct e1000_hw *hw = &adapter->hw;
2698 u32 tctl;
9d5c8243 2699
85b430b4
AD		 2700 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2701 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2702
2703 /* Program the Transmit Control Register */
9d5c8243
AK		 2704 tctl = rd32(E1000_TCTL);
2705 tctl &= ~E1000_TCTL_CT;
2706 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2707 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2708
2709 igb_config_collision_dist(hw);
2710
9d5c8243
AK		 2711 /* Enable transmits */
2712 tctl |= E1000_TCTL_EN;
2713
2714 wr32(E1000_TCTL, tctl);
2715}
2716
85b430b4
AD		 2717/**
2718 * igb_configure_tx_ring - Configure transmit ring after Reset
2719 * @adapter: board private structure
2720 * @ring: tx ring to configure
2721 *
2722 * Configure a transmit ring after a reset.
2723 **/
d7ee5b3a
AD		 2724void igb_configure_tx_ring(struct igb_adapter *adapter,
2725 struct igb_ring *ring)
85b430b4
AD		 2726{
2727 struct e1000_hw *hw = &adapter->hw;
a74420e0 2728 u32 txdctl = 0;
85b430b4
AD		 2729 u64 tdba = ring->dma;
2730 int reg_idx = ring->reg_idx;
2731
2732 /* disable the queue */
a74420e0 2733 wr32(E1000_TXDCTL(reg_idx), 0);
85b430b4
AD		 2734 wrfl();
2735 mdelay(10);
2736
2737 wr32(E1000_TDLEN(reg_idx),
2738 ring->count * sizeof(union e1000_adv_tx_desc));
2739 wr32(E1000_TDBAL(reg_idx),
2740 tdba & 0x00000000ffffffffULL);
2741 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2742
fce99e34 2743 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
a74420e0 2744 wr32(E1000_TDH(reg_idx), 0);
fce99e34 2745 writel(0, ring->tail);
85b430b4
AD		 2746
2747 txdctl |= IGB_TX_PTHRESH;
2748 txdctl |= IGB_TX_HTHRESH << 8;
2749 txdctl |= IGB_TX_WTHRESH << 16;
2750
2751 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2752 wr32(E1000_TXDCTL(reg_idx), txdctl);
2753}
2754
2755/**
2756 * igb_configure_tx - Configure transmit Unit after Reset
2757 * @adapter: board private structure
2758 *
2759 * Configure the Tx unit of the MAC after a reset.
2760 **/
2761static void igb_configure_tx(struct igb_adapter *adapter)
2762{
2763 int i;
2764
2765 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2766 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2767}
2768
9d5c8243
AK		 2769/**
2770 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2771 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2772 *
2773 * Returns 0 on success, negative on failure
2774 **/
80785298 2775int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2776{
59d71989 2777 struct device *dev = rx_ring->dev;
81c2fc22 2778 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2779 int size, desc_len;
2780
06034649 2781 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
81c2fc22
AD		 2782 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
2783 if (!rx_ring->rx_buffer_info)
2784 rx_ring->rx_buffer_info = vzalloc(size);
06034649 2785 if (!rx_ring->rx_buffer_info)
9d5c8243 2786 goto err;
9d5c8243
AK		 2787
2788 desc_len = sizeof(union e1000_adv_rx_desc);
2789
2790 /* Round up to nearest 4K */
2791 rx_ring->size = rx_ring->count * desc_len;
2792 rx_ring->size = ALIGN(rx_ring->size, 4096);
2793
81c2fc22 2794 set_dev_node(dev, rx_ring->numa_node);
59d71989
AD		 2795 rx_ring->desc = dma_alloc_coherent(dev,
2796 rx_ring->size,
2797 &rx_ring->dma,
2798 GFP_KERNEL);
81c2fc22
AD		 2799 set_dev_node(dev, orig_node);
2800 if (!rx_ring->desc)
2801 rx_ring->desc = dma_alloc_coherent(dev,
2802 rx_ring->size,
2803 &rx_ring->dma,
2804 GFP_KERNEL);
9d5c8243
AK		 2805
2806 if (!rx_ring->desc)
2807 goto err;
2808
2809 rx_ring->next_to_clean = 0;
2810 rx_ring->next_to_use = 0;
9d5c8243 2811
9d5c8243
AK		 2812 return 0;
2813
2814err:
06034649
AD		 2815 vfree(rx_ring->rx_buffer_info);
2816 rx_ring->rx_buffer_info = NULL;
59d71989
AD		 2817 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2818 " ring\n");
9d5c8243
AK		 2819 return -ENOMEM;
2820}
2821
2822/**
2823 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2824 * (Descriptors) for all queues
2825 * @adapter: board private structure
2826 *
2827 * Return 0 on success, negative on failure
2828 **/
2829static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2830{
439705e1 2831 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2832 int i, err = 0;
2833
2834 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2835 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2836 if (err) {
439705e1 2837 dev_err(&pdev->dev,
9d5c8243
AK		 2838 "Allocation for Rx Queue %u failed\n", i);
2839 for (i--; i >= 0; i--)
3025a446 2840 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2841 break;
2842 }
2843 }
2844
2845 return err;
2846}
2847
06cf2666
AD		 2848/**
2849 * igb_setup_mrqc - configure the multiple receive queue control registers
2850 * @adapter: Board private structure
2851 **/
2852static void igb_setup_mrqc(struct igb_adapter *adapter)
2853{
2854 struct e1000_hw *hw = &adapter->hw;
2855 u32 mrqc, rxcsum;
2856 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2857 union e1000_reta {
2858 u32 dword;
2859 u8 bytes[4];
2860 } reta;
2861 static const u8 rsshash[40] = {
2862 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2863 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2864 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2865 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2866
2867 /* Fill out hash function seeds */
2868 for (j = 0; j < 10; j++) {
2869 u32 rsskey = rsshash[(j * 4)];
2870 rsskey |= rsshash[(j * 4) + 1] << 8;
2871 rsskey |= rsshash[(j * 4) + 2] << 16;
2872 rsskey |= rsshash[(j * 4) + 3] << 24;
2873 array_wr32(E1000_RSSRK(0), j, rsskey);
2874 }
2875
a99955fc 2876 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2877
2878 if (adapter->vfs_allocated_count) {
2879 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2880 switch (hw->mac.type) {
d2ba2ed8 2881 case e1000_i350:
55cac248
AD		 2882 case e1000_82580:
2883 num_rx_queues = 1;
2884 shift = 0;
2885 break;
06cf2666
AD		 2886 case e1000_82576:
2887 shift = 3;
2888 num_rx_queues = 2;
2889 break;
2890 case e1000_82575:
2891 shift = 2;
2892 shift2 = 6;
2893 default:
2894 break;
2895 }
2896 } else {
2897 if (hw->mac.type == e1000_82575)
2898 shift = 6;
2899 }
2900
2901 for (j = 0; j < (32 * 4); j++) {
2902 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2903 if (shift2)
2904 reta.bytes[j & 3] |= num_rx_queues << shift2;
2905 if ((j & 3) == 3)
2906 wr32(E1000_RETA(j >> 2), reta.dword);
2907 }
2908
2909 /*
2910 * Disable raw packet checksumming so that RSS hash is placed in
2911 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2912 * offloads as they are enabled by default
2913 */
2914 rxcsum = rd32(E1000_RXCSUM);
2915 rxcsum |= E1000_RXCSUM_PCSD;
2916
2917 if (adapter->hw.mac.type >= e1000_82576)
2918 /* Enable Receive Checksum Offload for SCTP */
2919 rxcsum |= E1000_RXCSUM_CRCOFL;
2920
2921 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2922 wr32(E1000_RXCSUM, rxcsum);
2923
2924 /* If VMDq is enabled then we set the appropriate mode for that, else
2925 * we default to RSS so that an RSS hash is calculated per packet even
2926 * if we are only using one queue */
2927 if (adapter->vfs_allocated_count) {
2928 if (hw->mac.type > e1000_82575) {
2929 /* Set the default pool for the PF's first queue */
2930 u32 vtctl = rd32(E1000_VT_CTL);
2931 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2932 E1000_VT_CTL_DISABLE_DEF_POOL);
2933 vtctl |= adapter->vfs_allocated_count <<
2934 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2935 wr32(E1000_VT_CTL, vtctl);
2936 }
a99955fc 2937 if (adapter->rss_queues > 1)
06cf2666
AD		 2938 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2939 else
2940 mrqc = E1000_MRQC_ENABLE_VMDQ;
2941 } else {
2942 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2943 }
2944 igb_vmm_control(adapter);
2945
4478a9cd
AD		 2946 /*
2947 * Generate RSS hash based on TCP port numbers and/or
2948 * IPv4/v6 src and dst addresses since UDP cannot be
2949 * hashed reliably due to IP fragmentation
2950 */
2951 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2952 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2953 E1000_MRQC_RSS_FIELD_IPV6 |
2954 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2955 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2956
2957 wr32(E1000_MRQC, mrqc);
2958}
2959
9d5c8243
AK		 2960/**
2961 * igb_setup_rctl - configure the receive control registers
2962 * @adapter: Board private structure
2963 **/
d7ee5b3a 2964void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2965{
2966 struct e1000_hw *hw = &adapter->hw;
2967 u32 rctl;
9d5c8243
AK		 2968
2969 rctl = rd32(E1000_RCTL);
2970
2971 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2972 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2973
69d728ba 2974 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2975 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2976
87cb7e8c
AK		 2977 /*
2978 * enable stripping of CRC. It's unlikely this will break BMC
2979 * redirection as it did with e1000. Newer features require
2980 * that the HW strips the CRC.
73cd78f1 2981 */
87cb7e8c 2982 rctl |= E1000_RCTL_SECRC;
9d5c8243 2983
559e9c49 2984 /* disable store bad packets and clear size bits. */
ec54d7d6 2985 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2986
6ec43fe6
AD		 2987 /* enable LPE to prevent packets larger than max_frame_size */
2988 rctl |= E1000_RCTL_LPE;
9d5c8243 2989
952f72a8
AD		 2990 /* disable queue 0 to prevent tail write w/o re-config */
2991 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2992
e1739522
AD		 2993 /* Attention!!! For SR-IOV PF driver operations you must enable
2994 * queue drop for all VF and PF queues to prevent head of line blocking
2995 * if an un-trusted VF does not provide descriptors to hardware.
2996 */
2997 if (adapter->vfs_allocated_count) {
e1739522
AD		 2998 /* set all queue drop enable bits */
2999 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 3000 }
3001
9d5c8243
AK		 3002 wr32(E1000_RCTL, rctl);
3003}
3004
7d5753f0
AD		 3005static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3006 int vfn)
3007{
3008 struct e1000_hw *hw = &adapter->hw;
3009 u32 vmolr;
3010
3011 /* if it isn't the PF check to see if VFs are enabled and
3012 * increase the size to support vlan tags */
3013 if (vfn < adapter->vfs_allocated_count &&
3014 adapter->vf_data[vfn].vlans_enabled)
3015 size += VLAN_TAG_SIZE;
3016
3017 vmolr = rd32(E1000_VMOLR(vfn));
3018 vmolr &= ~E1000_VMOLR_RLPML_MASK;
3019 vmolr |= size | E1000_VMOLR_LPE;
3020 wr32(E1000_VMOLR(vfn), vmolr);
3021
3022 return 0;
3023}
3024
e1739522
AD		 3025/**
3026 * igb_rlpml_set - set maximum receive packet size
3027 * @adapter: board private structure
3028 *
3029 * Configure maximum receivable packet size.
3030 **/
3031static void igb_rlpml_set(struct igb_adapter *adapter)
3032{
153285f9 3033 u32 max_frame_size = adapter->max_frame_size;
e1739522
AD		 3034 struct e1000_hw *hw = &adapter->hw;
3035 u16 pf_id = adapter->vfs_allocated_count;
3036
e1739522
AD		 3037 if (pf_id) {
3038 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
153285f9
AD		 3039 /*
3040 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3041 * to our max jumbo frame size, in case we need to enable
3042 * jumbo frames on one of the rings later.
3043 * This will not pass over-length frames into the default
3044 * queue because it's gated by the VMOLR.RLPML.
3045 */
7d5753f0 3046 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 3047 }
3048
3049 wr32(E1000_RLPML, max_frame_size);
3050}
3051
8151d294
WM		 3052static inline void igb_set_vmolr(struct igb_adapter *adapter,
3053 int vfn, bool aupe)
7d5753f0
AD		 3054{
3055 struct e1000_hw *hw = &adapter->hw;
3056 u32 vmolr;
3057
3058 /*
3059 * This register exists only on 82576 and newer so if we are older then
3060 * we should exit and do nothing
3061 */
3062 if (hw->mac.type < e1000_82576)
3063 return;
3064
3065 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 3066 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3067 if (aupe)
3068 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3069 else
3070 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 3071
3072 /* clear all bits that might not be set */
3073 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3074
a99955fc 3075 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 3076 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3077 /*
3078 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3079 * multicast packets
3080 */
3081 if (vfn <= adapter->vfs_allocated_count)
3082 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3083
3084 wr32(E1000_VMOLR(vfn), vmolr);
3085}
3086
85b430b4
AD		 3087/**
3088 * igb_configure_rx_ring - Configure a receive ring after Reset
3089 * @adapter: board private structure
3090 * @ring: receive ring to be configured
3091 *
3092 * Configure the Rx unit of the MAC after a reset.
3093 **/
d7ee5b3a
AD		 3094void igb_configure_rx_ring(struct igb_adapter *adapter,
3095 struct igb_ring *ring)
85b430b4
AD		 3096{
3097 struct e1000_hw *hw = &adapter->hw;
3098 u64 rdba = ring->dma;
3099 int reg_idx = ring->reg_idx;
a74420e0 3100 u32 srrctl = 0, rxdctl = 0;
85b430b4
AD		 3101
3102 /* disable the queue */
a74420e0 3103 wr32(E1000_RXDCTL(reg_idx), 0);
85b430b4
AD		 3104
3105 /* Set DMA base address registers */
3106 wr32(E1000_RDBAL(reg_idx),
3107 rdba & 0x00000000ffffffffULL);
3108 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3109 wr32(E1000_RDLEN(reg_idx),
3110 ring->count * sizeof(union e1000_adv_rx_desc));
3111
3112 /* initialize head and tail */
fce99e34 3113 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
a74420e0 3114 wr32(E1000_RDH(reg_idx), 0);
fce99e34 3115 writel(0, ring->tail);
85b430b4 3116
952f72a8 3117 /* set descriptor configuration */
44390ca6 3118 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
952f72a8 3119#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
44390ca6 3120 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3121#else
44390ca6 3122 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3123#endif
44390ca6 3124 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
06218a8d 3125 if (hw->mac.type >= e1000_82580)
757b77e2 3126 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 3127 /* Only set Drop Enable if we are supporting multiple queues */
3128 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3129 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 3130
3131 wr32(E1000_SRRCTL(reg_idx), srrctl);
3132
7d5753f0 3133 /* set filtering for VMDQ pools */
8151d294 3134 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 3135
85b430b4
AD		 3136 rxdctl |= IGB_RX_PTHRESH;
3137 rxdctl |= IGB_RX_HTHRESH << 8;
3138 rxdctl |= IGB_RX_WTHRESH << 16;
a74420e0
AD		 3139
3140 /* enable receive descriptor fetching */
3141 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
85b430b4
AD		 3142 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3143}
3144
9d5c8243
AK		 3145/**
3146 * igb_configure_rx - Configure receive Unit after Reset
3147 * @adapter: board private structure
3148 *
3149 * Configure the Rx unit of the MAC after a reset.
3150 **/
3151static void igb_configure_rx(struct igb_adapter *adapter)
3152{
9107584e 3153 int i;
9d5c8243 3154
68d480c4
AD		 3155 /* set UTA to appropriate mode */
3156 igb_set_uta(adapter);
3157
26ad9178
AD		 3158 /* set the correct pool for the PF default MAC address in entry 0 */
3159 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3160 adapter->vfs_allocated_count);
3161
06cf2666
AD		 3162 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3163 * the Base and Length of the Rx Descriptor Ring */
3164 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3165 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 3166}
3167
3168/**
3169 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 3170 * @tx_ring: Tx descriptor ring for a specific queue
3171 *
3172 * Free all transmit software resources
3173 **/
68fd9910 3174void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 3175{
3b644cf6 3176 igb_clean_tx_ring(tx_ring);
9d5c8243 3177
06034649
AD		 3178 vfree(tx_ring->tx_buffer_info);
3179 tx_ring->tx_buffer_info = NULL;
9d5c8243 3180
439705e1
AD		 3181 /* if not set, then don't free */
3182 if (!tx_ring->desc)
3183 return;
3184
59d71989
AD		 3185 dma_free_coherent(tx_ring->dev, tx_ring->size,
3186 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 3187
3188 tx_ring->desc = NULL;
3189}
3190
3191/**
3192 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3193 * @adapter: board private structure
3194 *
3195 * Free all transmit software resources
3196 **/
3197static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3198{
3199 int i;
3200
3201 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3202 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3203}
3204
ebe42d16
AD		 3205void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3206 struct igb_tx_buffer *tx_buffer)
3207{
3208 if (tx_buffer->skb) {
3209 dev_kfree_skb_any(tx_buffer->skb);
3210 if (tx_buffer->dma)
3211 dma_unmap_single(ring->dev,
3212 tx_buffer->dma,
3213 tx_buffer->length,
3214 DMA_TO_DEVICE);
3215 } else if (tx_buffer->dma) {
3216 dma_unmap_page(ring->dev,
3217 tx_buffer->dma,
3218 tx_buffer->length,
3219 DMA_TO_DEVICE);
3220 }
3221 tx_buffer->next_to_watch = NULL;
3222 tx_buffer->skb = NULL;
3223 tx_buffer->dma = 0;
3224 /* buffer_info must be completely set up in the transmit path */
9d5c8243
AK		 3225}
3226
3227/**
3228 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3229 * @tx_ring: ring to be cleaned
3230 **/
3b644cf6 3231static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243 3232{
06034649 3233 struct igb_tx_buffer *buffer_info;
9d5c8243 3234 unsigned long size;
6ad4edfc 3235 u16 i;
9d5c8243 3236
06034649 3237 if (!tx_ring->tx_buffer_info)
9d5c8243
AK		 3238 return;
3239 /* Free all the Tx ring sk_buffs */
3240
3241 for (i = 0; i < tx_ring->count; i++) {
06034649 3242 buffer_info = &tx_ring->tx_buffer_info[i];
80785298 3243 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243 3244 }
bdbc0631 3245 netdev_tx_reset_queue(txring_txq(tx_ring));
9d5c8243 3246
06034649
AD		 3247 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3248 memset(tx_ring->tx_buffer_info, 0, size);
9d5c8243
AK		 3249
3250 /* Zero out the descriptor ring */
9d5c8243
AK		 3251 memset(tx_ring->desc, 0, tx_ring->size);
3252
3253 tx_ring->next_to_use = 0;
3254 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3255}
3256
3257/**
3258 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3259 * @adapter: board private structure
3260 **/
3261static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3262{
3263 int i;
3264
3265 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3266 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3267}
3268
3269/**
3270 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3271 * @rx_ring: ring to clean the resources from
3272 *
3273 * Free all receive software resources
3274 **/
68fd9910 3275void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3276{
3b644cf6 3277 igb_clean_rx_ring(rx_ring);
9d5c8243 3278
06034649
AD		 3279 vfree(rx_ring->rx_buffer_info);
3280 rx_ring->rx_buffer_info = NULL;
9d5c8243 3281
439705e1
AD		 3282 /* if not set, then don't free */
3283 if (!rx_ring->desc)
3284 return;
3285
59d71989
AD		 3286 dma_free_coherent(rx_ring->dev, rx_ring->size,
3287 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3288
3289 rx_ring->desc = NULL;
3290}
3291
3292/**
3293 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3294 * @adapter: board private structure
3295 *
3296 * Free all receive software resources
3297 **/
3298static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3299{
3300 int i;
3301
3302 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3303 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3304}
3305
3306/**
3307 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3308 * @rx_ring: ring to free buffers from
3309 **/
3b644cf6 3310static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243 3311{
9d5c8243 3312 unsigned long size;
c023cd88 3313 u16 i;
9d5c8243 3314
06034649 3315 if (!rx_ring->rx_buffer_info)
9d5c8243 3316 return;
439705e1 3317
9d5c8243
AK		 3318 /* Free all the Rx ring sk_buffs */
3319 for (i = 0; i < rx_ring->count; i++) {
06034649 3320 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
9d5c8243 3321 if (buffer_info->dma) {
59d71989 3322 dma_unmap_single(rx_ring->dev,
80785298 3323 buffer_info->dma,
44390ca6 3324 IGB_RX_HDR_LEN,
59d71989 3325 DMA_FROM_DEVICE);
9d5c8243
AK		 3326 buffer_info->dma = 0;
3327 }
3328
3329 if (buffer_info->skb) {
3330 dev_kfree_skb(buffer_info->skb);
3331 buffer_info->skb = NULL;
3332 }
6ec43fe6 3333 if (buffer_info->page_dma) {
59d71989 3334 dma_unmap_page(rx_ring->dev,
80785298 3335 buffer_info->page_dma,
6ec43fe6 3336 PAGE_SIZE / 2,
59d71989 3337 DMA_FROM_DEVICE);
6ec43fe6
AD		 3338 buffer_info->page_dma = 0;
3339 }
9d5c8243 3340 if (buffer_info->page) {
9d5c8243
AK		 3341 put_page(buffer_info->page);
3342 buffer_info->page = NULL;
bf36c1a0 3343 buffer_info->page_offset = 0;
9d5c8243
AK		 3344 }
3345 }
3346
06034649
AD		 3347 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3348 memset(rx_ring->rx_buffer_info, 0, size);
9d5c8243
AK		 3349
3350 /* Zero out the descriptor ring */
3351 memset(rx_ring->desc, 0, rx_ring->size);
3352
3353 rx_ring->next_to_clean = 0;
3354 rx_ring->next_to_use = 0;
9d5c8243
AK		 3355}
3356
3357/**
3358 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3359 * @adapter: board private structure
3360 **/
3361static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3362{
3363 int i;
3364
3365 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3366 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3367}
3368
3369/**
3370 * igb_set_mac - Change the Ethernet Address of the NIC
3371 * @netdev: network interface device structure
3372 * @p: pointer to an address structure
3373 *
3374 * Returns 0 on success, negative on failure
3375 **/
3376static int igb_set_mac(struct net_device *netdev, void *p)
3377{
3378 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3379 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3380 struct sockaddr *addr = p;
3381
3382 if (!is_valid_ether_addr(addr->sa_data))
3383 return -EADDRNOTAVAIL;
3384
3385 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3386 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3387
26ad9178
AD		 3388 /* set the correct pool for the new PF MAC address in entry 0 */
3389 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3390 adapter->vfs_allocated_count);
e1739522 3391
9d5c8243
AK		 3392 return 0;
3393}
3394
3395/**
68d480c4 3396 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3397 * @netdev: network interface device structure
3398 *
68d480c4
AD		 3399 * Writes multicast address list to the MTA hash table.
3400 * Returns: -ENOMEM on failure
3401 * 0 on no addresses written
3402 * X on writing X addresses to MTA
9d5c8243 3403 **/
68d480c4 3404static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3405{
3406 struct igb_adapter *adapter = netdev_priv(netdev);
3407 struct e1000_hw *hw = &adapter->hw;
22bedad3 3408 struct netdev_hw_addr *ha;
68d480c4 3409 u8 *mta_list;
9d5c8243
AK		 3410 int i;
3411
4cd24eaf 3412 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3413 /* nothing to program, so clear mc list */
3414 igb_update_mc_addr_list(hw, NULL, 0);
3415 igb_restore_vf_multicasts(adapter);
3416 return 0;
3417 }
9d5c8243 3418
4cd24eaf 3419 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3420 if (!mta_list)
3421 return -ENOMEM;
ff41f8dc 3422
68d480c4 3423 /* The shared function expects a packed array of only addresses. */
48e2f183 3424 i = 0;
22bedad3
JP		 3425 netdev_for_each_mc_addr(ha, netdev)
3426 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3427
68d480c4
AD		 3428 igb_update_mc_addr_list(hw, mta_list, i);
3429 kfree(mta_list);
3430
4cd24eaf 3431 return netdev_mc_count(netdev);
68d480c4
AD		 3432}
3433
3434/**
3435 * igb_write_uc_addr_list - write unicast addresses to RAR table
3436 * @netdev: network interface device structure
3437 *
3438 * Writes unicast address list to the RAR table.
3439 * Returns: -ENOMEM on failure/insufficient address space
3440 * 0 on no addresses written
3441 * X on writing X addresses to the RAR table
3442 **/
3443static int igb_write_uc_addr_list(struct net_device *netdev)
3444{
3445 struct igb_adapter *adapter = netdev_priv(netdev);
3446 struct e1000_hw *hw = &adapter->hw;
3447 unsigned int vfn = adapter->vfs_allocated_count;
3448 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3449 int count = 0;
3450
3451 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3452 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3453 return -ENOMEM;
9d5c8243 3454
32e7bfc4 3455 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3456 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3457
3458 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3459 if (!rar_entries)
3460 break;
26ad9178
AD		 3461 igb_rar_set_qsel(adapter, ha->addr,
3462 rar_entries--,
68d480c4
AD		 3463 vfn);
3464 count++;
ff41f8dc
AD		 3465 }
3466 }
3467 /* write the addresses in reverse order to avoid write combining */
3468 for (; rar_entries > 0 ; rar_entries--) {
3469 wr32(E1000_RAH(rar_entries), 0);
3470 wr32(E1000_RAL(rar_entries), 0);
3471 }
3472 wrfl();
3473
68d480c4
AD		 3474 return count;
3475}
3476
3477/**
3478 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3479 * @netdev: network interface device structure
3480 *
3481 * The set_rx_mode entry point is called whenever the unicast or multicast
3482 * address lists or the network interface flags are updated. This routine is
3483 * responsible for configuring the hardware for proper unicast, multicast,
3484 * promiscuous mode, and all-multi behavior.
3485 **/
3486static void igb_set_rx_mode(struct net_device *netdev)
3487{
3488 struct igb_adapter *adapter = netdev_priv(netdev);
3489 struct e1000_hw *hw = &adapter->hw;
3490 unsigned int vfn = adapter->vfs_allocated_count;
3491 u32 rctl, vmolr = 0;
3492 int count;
3493
3494 /* Check for Promiscuous and All Multicast modes */
3495 rctl = rd32(E1000_RCTL);
3496
3497 /* clear the effected bits */
3498 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3499
3500 if (netdev->flags & IFF_PROMISC) {
3501 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3502 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3503 } else {
3504 if (netdev->flags & IFF_ALLMULTI) {
3505 rctl |= E1000_RCTL_MPE;
3506 vmolr |= E1000_VMOLR_MPME;
3507 } else {
3508 /*
3509 * Write addresses to the MTA, if the attempt fails
25985edc 3510 * then we should just turn on promiscuous mode so
68d480c4
AD		 3511 * that we can at least receive multicast traffic
3512 */
3513 count = igb_write_mc_addr_list(netdev);
3514 if (count < 0) {
3515 rctl |= E1000_RCTL_MPE;
3516 vmolr |= E1000_VMOLR_MPME;
3517 } else if (count) {
3518 vmolr |= E1000_VMOLR_ROMPE;
3519 }
3520 }
3521 /*
3522 * Write addresses to available RAR registers, if there is not
3523 * sufficient space to store all the addresses then enable
25985edc 3524 * unicast promiscuous mode
68d480c4
AD		 3525 */
3526 count = igb_write_uc_addr_list(netdev);
3527 if (count < 0) {
3528 rctl |= E1000_RCTL_UPE;
3529 vmolr |= E1000_VMOLR_ROPE;
3530 }
3531 rctl |= E1000_RCTL_VFE;
28fc06f5 3532 }
68d480c4 3533 wr32(E1000_RCTL, rctl);
28fc06f5 3534
68d480c4
AD		 3535 /*
3536 * In order to support SR-IOV and eventually VMDq it is necessary to set
3537 * the VMOLR to enable the appropriate modes. Without this workaround
3538 * we will have issues with VLAN tag stripping not being done for frames
3539 * that are only arriving because we are the default pool
3540 */
3541 if (hw->mac.type < e1000_82576)
28fc06f5 3542 return;
9d5c8243 3543
68d480c4
AD		 3544 vmolr |= rd32(E1000_VMOLR(vfn)) &
3545 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3546 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3547 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3548}
3549
13800469
GR		 3550static void igb_check_wvbr(struct igb_adapter *adapter)
3551{
3552 struct e1000_hw *hw = &adapter->hw;
3553 u32 wvbr = 0;
3554
3555 switch (hw->mac.type) {
3556 case e1000_82576:
3557 case e1000_i350:
3558 if (!(wvbr = rd32(E1000_WVBR)))
3559 return;
3560 break;
3561 default:
3562 break;
3563 }
3564
3565 adapter->wvbr |= wvbr;
3566}
3567
3568#define IGB_STAGGERED_QUEUE_OFFSET 8
3569
3570static void igb_spoof_check(struct igb_adapter *adapter)
3571{
3572 int j;
3573
3574 if (!adapter->wvbr)
3575 return;
3576
3577 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3578 if (adapter->wvbr & (1 << j) ||
3579 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3580 dev_warn(&adapter->pdev->dev,
3581 "Spoof event(s) detected on VF %d\n", j);
3582 adapter->wvbr &=
3583 ~((1 << j) |
3584 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3585 }
3586 }
3587}
3588
9d5c8243
AK		 3589/* Need to wait a few seconds after link up to get diagnostic information from
3590 * the phy */
3591static void igb_update_phy_info(unsigned long data)
3592{
3593 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3594 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3595}
3596
4d6b725e
AD		 3597/**
3598 * igb_has_link - check shared code for link and determine up/down
3599 * @adapter: pointer to driver private info
3600 **/
3145535a 3601bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3602{
3603 struct e1000_hw *hw = &adapter->hw;
3604 bool link_active = false;
3605 s32 ret_val = 0;
3606
3607 /* get_link_status is set on LSC (link status) interrupt or
3608 * rx sequence error interrupt. get_link_status will stay
3609 * false until the e1000_check_for_link establishes link
3610 * for copper adapters ONLY
3611 */
3612 switch (hw->phy.media_type) {
3613 case e1000_media_type_copper:
3614 if (hw->mac.get_link_status) {
3615 ret_val = hw->mac.ops.check_for_link(hw);
3616 link_active = !hw->mac.get_link_status;
3617 } else {
3618 link_active = true;
3619 }
3620 break;
4d6b725e
AD		 3621 case e1000_media_type_internal_serdes:
3622 ret_val = hw->mac.ops.check_for_link(hw);
3623 link_active = hw->mac.serdes_has_link;
3624 break;
3625 default:
3626 case e1000_media_type_unknown:
3627 break;
3628 }
3629
3630 return link_active;
3631}
3632
563988dc
SA		 3633static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3634{
3635 bool ret = false;
3636 u32 ctrl_ext, thstat;
3637
3638 /* check for thermal sensor event on i350, copper only */
3639 if (hw->mac.type == e1000_i350) {
3640 thstat = rd32(E1000_THSTAT);
3641 ctrl_ext = rd32(E1000_CTRL_EXT);
3642
3643 if ((hw->phy.media_type == e1000_media_type_copper) &&
3644 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3645 ret = !!(thstat & event);
3646 }
3647 }
3648
3649 return ret;
3650}
3651
9d5c8243
AK		 3652/**
3653 * igb_watchdog - Timer Call-back
3654 * @data: pointer to adapter cast into an unsigned long
3655 **/
3656static void igb_watchdog(unsigned long data)
3657{
3658 struct igb_adapter *adapter = (struct igb_adapter *)data;
3659 /* Do the rest outside of interrupt context */
3660 schedule_work(&adapter->watchdog_task);
3661}
3662
3663static void igb_watchdog_task(struct work_struct *work)
3664{
3665 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3666 struct igb_adapter,
3667 watchdog_task);
9d5c8243 3668 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3669 struct net_device *netdev = adapter->netdev;
563988dc 3670 u32 link;
7a6ea550 3671 int i;
9d5c8243 3672
4d6b725e 3673 link = igb_has_link(adapter);
9d5c8243 3674 if (link) {
749ab2cd
YZ		 3675 /* Cancel scheduled suspend requests. */
3676 pm_runtime_resume(netdev->dev.parent);
3677
9d5c8243
AK		 3678 if (!netif_carrier_ok(netdev)) {
3679 u32 ctrl;
330a6d6a
AD		 3680 hw->mac.ops.get_speed_and_duplex(hw,
3681 &adapter->link_speed,
3682 &adapter->link_duplex);
9d5c8243
AK		 3683
3684 ctrl = rd32(E1000_CTRL);
527d47c1 3685 /* Links status message must follow this format */
876d2d6f
JK		 3686 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s "
3687 "Duplex, Flow Control: %s\n",
559e9c49
AD		 3688 netdev->name,
3689 adapter->link_speed,
3690 adapter->link_duplex == FULL_DUPLEX ?
876d2d6f
JK		 3691 "Full" : "Half",
3692 (ctrl & E1000_CTRL_TFCE) &&
3693 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
3694 (ctrl & E1000_CTRL_RFCE) ? "RX" :
3695 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None");
9d5c8243 3696
563988dc 3697 /* check for thermal sensor event */
876d2d6f
JK		 3698 if (igb_thermal_sensor_event(hw,
3699 E1000_THSTAT_LINK_THROTTLE)) {
3700 netdev_info(netdev, "The network adapter link "
3701 "speed was downshifted because it "
3702 "overheated\n");
7ef5ed1c 3703 }
563988dc 3704
d07f3e37 3705 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3706 adapter->tx_timeout_factor = 1;
3707 switch (adapter->link_speed) {
3708 case SPEED_10:
9d5c8243
AK		 3709 adapter->tx_timeout_factor = 14;
3710 break;
3711 case SPEED_100:
9d5c8243
AK		 3712 /* maybe add some timeout factor ? */
3713 break;
3714 }
3715
3716 netif_carrier_on(netdev);
9d5c8243 3717
4ae196df 3718 igb_ping_all_vfs(adapter);
17dc566c 3719 igb_check_vf_rate_limit(adapter);
4ae196df 3720
4b1a9877 3721 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3722 if (!test_bit(__IGB_DOWN, &adapter->state))
3723 mod_timer(&adapter->phy_info_timer,
3724 round_jiffies(jiffies + 2 * HZ));
3725 }
3726 } else {
3727 if (netif_carrier_ok(netdev)) {
3728 adapter->link_speed = 0;
3729 adapter->link_duplex = 0;
563988dc
SA		 3730
3731 /* check for thermal sensor event */
876d2d6f
JK		 3732 if (igb_thermal_sensor_event(hw,
3733 E1000_THSTAT_PWR_DOWN)) {
3734 netdev_err(netdev, "The network adapter was "
3735 "stopped because it overheated\n");
7ef5ed1c 3736 }
563988dc 3737
527d47c1
AD		 3738 /* Links status message must follow this format */
3739 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3740 netdev->name);
9d5c8243 3741 netif_carrier_off(netdev);
4b1a9877 3742
4ae196df
AD		 3743 igb_ping_all_vfs(adapter);
3744
4b1a9877 3745 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3746 if (!test_bit(__IGB_DOWN, &adapter->state))
3747 mod_timer(&adapter->phy_info_timer,
3748 round_jiffies(jiffies + 2 * HZ));
749ab2cd
YZ		 3749
3750 pm_schedule_suspend(netdev->dev.parent,
3751 MSEC_PER_SEC * 5);
9d5c8243
AK		 3752 }
3753 }
3754
12dcd86b
ED		 3755 spin_lock(&adapter->stats64_lock);
3756 igb_update_stats(adapter, &adapter->stats64);
3757 spin_unlock(&adapter->stats64_lock);
9d5c8243 3758
dbabb065 3759 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3760 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3761 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3762 /* We've lost link, so the controller stops DMA,
3763 * but we've got queued Tx work that's never going
3764 * to get done, so reset controller to flush Tx.
3765 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3766 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3767 adapter->tx_timeout_count++;
3768 schedule_work(&adapter->reset_task);
3769 /* return immediately since reset is imminent */
3770 return;
3771 }
9d5c8243 3772 }
9d5c8243 3773
dbabb065 3774 /* Force detection of hung controller every watchdog period */
6d095fa8 3775 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
dbabb065 3776 }
f7ba205e 3777
9d5c8243 3778 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3779 if (adapter->msix_entries) {
047e0030 3780 u32 eics = 0;
0d1ae7f4
AD		 3781 for (i = 0; i < adapter->num_q_vectors; i++)
3782 eics |= adapter->q_vector[i]->eims_value;
7a6ea550
AD		 3783 wr32(E1000_EICS, eics);
3784 } else {
3785 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3786 }
9d5c8243 3787
13800469
GR		 3788 igb_spoof_check(adapter);
3789
9d5c8243
AK		 3790 /* Reset the timer */
3791 if (!test_bit(__IGB_DOWN, &adapter->state))
3792 mod_timer(&adapter->watchdog_timer,
3793 round_jiffies(jiffies + 2 * HZ));
3794}
3795
3796enum latency_range {
3797 lowest_latency = 0,
3798 low_latency = 1,
3799 bulk_latency = 2,
3800 latency_invalid = 255
3801};
3802
6eb5a7f1
AD		 3803/**
3804 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3805 *
3806 * Stores a new ITR value based on strictly on packet size. This
3807 * algorithm is less sophisticated than that used in igb_update_itr,
3808 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3809 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3810 * were determined based on theoretical maximum wire speed and testing
3811 * data, in order to minimize response time while increasing bulk
3812 * throughput.
3813 * This functionality is controlled by the InterruptThrottleRate module
3814 * parameter (see igb_param.c)
3815 * NOTE: This function is called only when operating in a multiqueue
3816 * receive environment.
047e0030 3817 * @q_vector: pointer to q_vector
6eb5a7f1 3818 **/
047e0030 3819static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3820{
047e0030 3821 int new_val = q_vector->itr_val;
6eb5a7f1 3822 int avg_wire_size = 0;
047e0030 3823 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b 3824 unsigned int packets;
9d5c8243 3825
6eb5a7f1
AD		 3826 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3827 * ints/sec - ITR timer value of 120 ticks.
3828 */
3829 if (adapter->link_speed != SPEED_1000) {
0ba82994 3830 new_val = IGB_4K_ITR;
6eb5a7f1 3831 goto set_itr_val;
9d5c8243 3832 }
047e0030 3833
0ba82994
AD		 3834 packets = q_vector->rx.total_packets;
3835 if (packets)
3836 avg_wire_size = q_vector->rx.total_bytes / packets;
047e0030 3837
0ba82994
AD		 3838 packets = q_vector->tx.total_packets;
3839 if (packets)
3840 avg_wire_size = max_t(u32, avg_wire_size,
3841 q_vector->tx.total_bytes / packets);
047e0030
AD		 3842
3843 /* if avg_wire_size isn't set no work was done */
3844 if (!avg_wire_size)
3845 goto clear_counts;
9d5c8243 3846
6eb5a7f1
AD		 3847 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3848 avg_wire_size += 24;
3849
3850 /* Don't starve jumbo frames */
3851 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3852
6eb5a7f1
AD		 3853 /* Give a little boost to mid-size frames */
3854 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3855 new_val = avg_wire_size / 3;
3856 else
3857 new_val = avg_wire_size / 2;
9d5c8243 3858
0ba82994
AD		 3859 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3860 if (new_val < IGB_20K_ITR &&
3861 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3862 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3863 new_val = IGB_20K_ITR;
abe1c363 3864
6eb5a7f1 3865set_itr_val:
047e0030
AD		 3866 if (new_val != q_vector->itr_val) {
3867 q_vector->itr_val = new_val;
3868 q_vector->set_itr = 1;
9d5c8243 3869 }
6eb5a7f1 3870clear_counts:
0ba82994
AD		 3871 q_vector->rx.total_bytes = 0;
3872 q_vector->rx.total_packets = 0;
3873 q_vector->tx.total_bytes = 0;
3874 q_vector->tx.total_packets = 0;
9d5c8243
AK		 3875}
3876
3877/**
3878 * igb_update_itr - update the dynamic ITR value based on statistics
3879 * Stores a new ITR value based on packets and byte
3880 * counts during the last interrupt. The advantage of per interrupt
3881 * computation is faster updates and more accurate ITR for the current
3882 * traffic pattern. Constants in this function were computed
3883 * based on theoretical maximum wire speed and thresholds were set based
3884 * on testing data as well as attempting to minimize response time
3885 * while increasing bulk throughput.
3886 * this functionality is controlled by the InterruptThrottleRate module
3887 * parameter (see igb_param.c)
3888 * NOTE: These calculations are only valid when operating in a single-
3889 * queue environment.
0ba82994
AD		 3890 * @q_vector: pointer to q_vector
3891 * @ring_container: ring info to update the itr for
9d5c8243 3892 **/
0ba82994
AD		 3893static void igb_update_itr(struct igb_q_vector *q_vector,
3894 struct igb_ring_container *ring_container)
9d5c8243 3895{
0ba82994
AD		 3896 unsigned int packets = ring_container->total_packets;
3897 unsigned int bytes = ring_container->total_bytes;
3898 u8 itrval = ring_container->itr;
9d5c8243 3899
0ba82994 3900 /* no packets, exit with status unchanged */
9d5c8243 3901 if (packets == 0)
0ba82994 3902 return;
9d5c8243 3903
0ba82994 3904 switch (itrval) {
9d5c8243
AK		 3905 case lowest_latency:
3906 /* handle TSO and jumbo frames */
3907 if (bytes/packets > 8000)
0ba82994 3908 itrval = bulk_latency;
9d5c8243 3909 else if ((packets < 5) && (bytes > 512))
0ba82994 3910 itrval = low_latency;
9d5c8243
AK		 3911 break;
3912 case low_latency: /* 50 usec aka 20000 ints/s */
3913 if (bytes > 10000) {
3914 /* this if handles the TSO accounting */
3915 if (bytes/packets > 8000) {
0ba82994 3916 itrval = bulk_latency;
9d5c8243 3917 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
0ba82994 3918 itrval = bulk_latency;
9d5c8243 3919 } else if ((packets > 35)) {
0ba82994 3920 itrval = lowest_latency;
9d5c8243
AK		 3921 }
3922 } else if (bytes/packets > 2000) {
0ba82994 3923 itrval = bulk_latency;
9d5c8243 3924 } else if (packets <= 2 && bytes < 512) {
0ba82994 3925 itrval = lowest_latency;
9d5c8243
AK		 3926 }
3927 break;
3928 case bulk_latency: /* 250 usec aka 4000 ints/s */
3929 if (bytes > 25000) {
3930 if (packets > 35)
0ba82994 3931 itrval = low_latency;
1e5c3d21 3932 } else if (bytes < 1500) {
0ba82994 3933 itrval = low_latency;
9d5c8243
AK		 3934 }
3935 break;
3936 }
3937
0ba82994
AD		 3938 /* clear work counters since we have the values we need */
3939 ring_container->total_bytes = 0;
3940 ring_container->total_packets = 0;
3941
3942 /* write updated itr to ring container */
3943 ring_container->itr = itrval;
9d5c8243
AK		 3944}
3945
0ba82994 3946static void igb_set_itr(struct igb_q_vector *q_vector)
9d5c8243 3947{
0ba82994 3948 struct igb_adapter *adapter = q_vector->adapter;
047e0030 3949 u32 new_itr = q_vector->itr_val;
0ba82994 3950 u8 current_itr = 0;
9d5c8243
AK		 3951
3952 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3953 if (adapter->link_speed != SPEED_1000) {
3954 current_itr = 0;
0ba82994 3955 new_itr = IGB_4K_ITR;
9d5c8243
AK		 3956 goto set_itr_now;
3957 }
3958
0ba82994
AD		 3959 igb_update_itr(q_vector, &q_vector->tx);
3960 igb_update_itr(q_vector, &q_vector->rx);
9d5c8243 3961
0ba82994 3962 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
9d5c8243 3963
6eb5a7f1 3964 /* conservative mode (itr 3) eliminates the lowest_latency setting */
0ba82994
AD		 3965 if (current_itr == lowest_latency &&
3966 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3967 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
6eb5a7f1
AD		 3968 current_itr = low_latency;
3969
9d5c8243
AK		 3970 switch (current_itr) {
3971 /* counts and packets in update_itr are dependent on these numbers */
3972 case lowest_latency:
0ba82994 3973 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
9d5c8243
AK		 3974 break;
3975 case low_latency:
0ba82994 3976 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
9d5c8243
AK		 3977 break;
3978 case bulk_latency:
0ba82994 3979 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
9d5c8243
AK		 3980 break;
3981 default:
3982 break;
3983 }
3984
3985set_itr_now:
047e0030 3986 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 3987 /* this attempts to bias the interrupt rate towards Bulk
3988 * by adding intermediate steps when interrupt rate is
3989 * increasing */
047e0030
AD		 3990 new_itr = new_itr > q_vector->itr_val ?
3991 max((new_itr * q_vector->itr_val) /
3992 (new_itr + (q_vector->itr_val >> 2)),
0ba82994 3993 new_itr) :
9d5c8243
AK		 3994 new_itr;
3995 /* Don't write the value here; it resets the adapter's
3996 * internal timer, and causes us to delay far longer than
3997 * we should between interrupts. Instead, we write the ITR
3998 * value at the beginning of the next interrupt so the timing
3999 * ends up being correct.
4000 */
047e0030
AD		 4001 q_vector->itr_val = new_itr;
4002 q_vector->set_itr = 1;
9d5c8243 4003 }
9d5c8243
AK		 4004}
4005
c50b52a0
SH		 4006static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4007 u32 type_tucmd, u32 mss_l4len_idx)
7d13a7d0
AD		 4008{
4009 struct e1000_adv_tx_context_desc *context_desc;
4010 u16 i = tx_ring->next_to_use;
4011
4012 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4013
4014 i++;
4015 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4016
4017 /* set bits to identify this as an advanced context descriptor */
4018 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4019
4020 /* For 82575, context index must be unique per ring. */
866cff06 4021 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
7d13a7d0
AD		 4022 mss_l4len_idx |= tx_ring->reg_idx << 4;
4023
4024 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
4025 context_desc->seqnum_seed = 0;
4026 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
4027 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4028}
4029
7af40ad9
AD		 4030static int igb_tso(struct igb_ring *tx_ring,
4031 struct igb_tx_buffer *first,
4032 u8 *hdr_len)
9d5c8243 4033{
7af40ad9 4034 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4035 u32 vlan_macip_lens, type_tucmd;
4036 u32 mss_l4len_idx, l4len;
4037
4038 if (!skb_is_gso(skb))
4039 return 0;
9d5c8243
AK		 4040
4041 if (skb_header_cloned(skb)) {
7af40ad9 4042 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
9d5c8243
AK		 4043 if (err)
4044 return err;
4045 }
4046
7d13a7d0
AD		 4047 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4048 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
9d5c8243 4049
7af40ad9 4050 if (first->protocol == __constant_htons(ETH_P_IP)) {
9d5c8243
AK		 4051 struct iphdr *iph = ip_hdr(skb);
4052 iph->tot_len = 0;
4053 iph->check = 0;
4054 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4055 iph->daddr, 0,
4056 IPPROTO_TCP,
4057 0);
7d13a7d0 4058 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
7af40ad9
AD		 4059 first->tx_flags |= IGB_TX_FLAGS_TSO |
4060 IGB_TX_FLAGS_CSUM |
4061 IGB_TX_FLAGS_IPV4;
8e1e8a47 4062 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 4063 ipv6_hdr(skb)->payload_len = 0;
4064 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4065 &ipv6_hdr(skb)->daddr,
4066 0, IPPROTO_TCP, 0);
7af40ad9
AD		 4067 first->tx_flags |= IGB_TX_FLAGS_TSO |
4068 IGB_TX_FLAGS_CSUM;
9d5c8243
AK		 4069 }
4070
7af40ad9 4071 /* compute header lengths */
7d13a7d0
AD		 4072 l4len = tcp_hdrlen(skb);
4073 *hdr_len = skb_transport_offset(skb) + l4len;
9d5c8243 4074
7af40ad9
AD		 4075 /* update gso size and bytecount with header size */
4076 first->gso_segs = skb_shinfo(skb)->gso_segs;
4077 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4078
9d5c8243 4079 /* MSS L4LEN IDX */
7d13a7d0
AD		 4080 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4081 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
9d5c8243 4082
7d13a7d0
AD		 4083 /* VLAN MACLEN IPLEN */
4084 vlan_macip_lens = skb_network_header_len(skb);
4085 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4086 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4087
7d13a7d0 4088 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243 4089
7d13a7d0 4090 return 1;
9d5c8243
AK		 4091}
4092
7af40ad9 4093static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
9d5c8243 4094{
7af40ad9 4095 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4096 u32 vlan_macip_lens = 0;
4097 u32 mss_l4len_idx = 0;
4098 u32 type_tucmd = 0;
9d5c8243 4099
7d13a7d0 4100 if (skb->ip_summed != CHECKSUM_PARTIAL) {
7af40ad9
AD		 4101 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4102 return;
7d13a7d0
AD		 4103 } else {
4104 u8 l4_hdr = 0;
7af40ad9 4105 switch (first->protocol) {
7d13a7d0
AD		 4106 case __constant_htons(ETH_P_IP):
4107 vlan_macip_lens |= skb_network_header_len(skb);
4108 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4109 l4_hdr = ip_hdr(skb)->protocol;
4110 break;
4111 case __constant_htons(ETH_P_IPV6):
4112 vlan_macip_lens |= skb_network_header_len(skb);
4113 l4_hdr = ipv6_hdr(skb)->nexthdr;
4114 break;
4115 default:
4116 if (unlikely(net_ratelimit())) {
4117 dev_warn(tx_ring->dev,
4118 "partial checksum but proto=%x!\n",
7af40ad9 4119 first->protocol);
fa4a7ef3 4120 }
7d13a7d0
AD		 4121 break;
4122 }
fa4a7ef3 4123
7d13a7d0
AD		 4124 switch (l4_hdr) {
4125 case IPPROTO_TCP:
4126 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4127 mss_l4len_idx = tcp_hdrlen(skb) <<
4128 E1000_ADVTXD_L4LEN_SHIFT;
4129 break;
4130 case IPPROTO_SCTP:
4131 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4132 mss_l4len_idx = sizeof(struct sctphdr) <<
4133 E1000_ADVTXD_L4LEN_SHIFT;
4134 break;
4135 case IPPROTO_UDP:
4136 mss_l4len_idx = sizeof(struct udphdr) <<
4137 E1000_ADVTXD_L4LEN_SHIFT;
4138 break;
4139 default:
4140 if (unlikely(net_ratelimit())) {
4141 dev_warn(tx_ring->dev,
4142 "partial checksum but l4 proto=%x!\n",
4143 l4_hdr);
44b0cda3 4144 }
7d13a7d0 4145 break;
9d5c8243 4146 }
7af40ad9
AD		 4147
4148 /* update TX checksum flag */
4149 first->tx_flags |= IGB_TX_FLAGS_CSUM;
7d13a7d0 4150 }
9d5c8243 4151
7d13a7d0 4152 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4153 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4154
7d13a7d0 4155 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243
AK		 4156}
4157
e032afc8
AD		 4158static __le32 igb_tx_cmd_type(u32 tx_flags)
4159{
4160 /* set type for advanced descriptor with frame checksum insertion */
4161 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4162 E1000_ADVTXD_DCMD_IFCS |
4163 E1000_ADVTXD_DCMD_DEXT);
4164
4165 /* set HW vlan bit if vlan is present */
4166 if (tx_flags & IGB_TX_FLAGS_VLAN)
4167 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4168
4169 /* set timestamp bit if present */
4170 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4171 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4172
4173 /* set segmentation bits for TSO */
4174 if (tx_flags & IGB_TX_FLAGS_TSO)
4175 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4176
4177 return cmd_type;
4178}
4179
7af40ad9
AD		 4180static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4181 union e1000_adv_tx_desc *tx_desc,
4182 u32 tx_flags, unsigned int paylen)
e032afc8
AD		 4183{
4184 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4185
4186 /* 82575 requires a unique index per ring if any offload is enabled */
4187 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
866cff06 4188 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
e032afc8
AD		 4189 olinfo_status |= tx_ring->reg_idx << 4;
4190
4191 /* insert L4 checksum */
4192 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4193 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4194
4195 /* insert IPv4 checksum */
4196 if (tx_flags & IGB_TX_FLAGS_IPV4)
4197 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4198 }
4199
7af40ad9 4200 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
e032afc8
AD		 4201}
4202
ebe42d16
AD		 4203/*
4204 * The largest size we can write to the descriptor is 65535. In order to
4205 * maintain a power of two alignment we have to limit ourselves to 32K.
4206 */
4207#define IGB_MAX_TXD_PWR 15
7af40ad9 4208#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
9d5c8243 4209
7af40ad9
AD		 4210static void igb_tx_map(struct igb_ring *tx_ring,
4211 struct igb_tx_buffer *first,
ebe42d16 4212 const u8 hdr_len)
9d5c8243 4213{
7af40ad9 4214 struct sk_buff *skb = first->skb;
ebe42d16
AD		 4215 struct igb_tx_buffer *tx_buffer_info;
4216 union e1000_adv_tx_desc *tx_desc;
4217 dma_addr_t dma;
4218 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4219 unsigned int data_len = skb->data_len;
4220 unsigned int size = skb_headlen(skb);
4221 unsigned int paylen = skb->len - hdr_len;
4222 __le32 cmd_type;
7af40ad9 4223 u32 tx_flags = first->tx_flags;
ebe42d16 4224 u16 i = tx_ring->next_to_use;
ebe42d16
AD		 4225
4226 tx_desc = IGB_TX_DESC(tx_ring, i);
4227
7af40ad9 4228 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
ebe42d16
AD		 4229 cmd_type = igb_tx_cmd_type(tx_flags);
4230
4231 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4232 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33 4233 goto dma_error;
9d5c8243 4234
ebe42d16
AD		 4235 /* record length, and DMA address */
4236 first->length = size;
4237 first->dma = dma;
ebe42d16
AD		 4238 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4239
4240 for (;;) {
4241 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4242 tx_desc->read.cmd_type_len =
4243 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4244
4245 i++;
4246 tx_desc++;
4247 if (i == tx_ring->count) {
4248 tx_desc = IGB_TX_DESC(tx_ring, 0);
4249 i = 0;
4250 }
4251
4252 dma += IGB_MAX_DATA_PER_TXD;
4253 size -= IGB_MAX_DATA_PER_TXD;
4254
4255 tx_desc->read.olinfo_status = 0;
4256 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4257 }
4258
4259 if (likely(!data_len))
4260 break;
2bbfebe2 4261
ebe42d16 4262 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
9d5c8243 4263
65689fef 4264 i++;
ebe42d16
AD		 4265 tx_desc++;
4266 if (i == tx_ring->count) {
4267 tx_desc = IGB_TX_DESC(tx_ring, 0);
65689fef 4268 i = 0;
ebe42d16 4269 }
65689fef 4270
9e903e08 4271 size = skb_frag_size(frag);
ebe42d16
AD		 4272 data_len -= size;
4273
4274 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4275 size, DMA_TO_DEVICE);
4276 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33
AD		 4277 goto dma_error;
4278
ebe42d16
AD		 4279 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4280 tx_buffer_info->length = size;
4281 tx_buffer_info->dma = dma;
4282
4283 tx_desc->read.olinfo_status = 0;
4284 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4285
4286 frag++;
9d5c8243
AK		 4287 }
4288
bdbc0631
ED		 4289 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4290
ebe42d16
AD		 4291 /* write last descriptor with RS and EOP bits */
4292 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4293 tx_desc->read.cmd_type_len = cmd_type;
8542db05
AD		 4294
4295 /* set the timestamp */
4296 first->time_stamp = jiffies;
4297
ebe42d16
AD		 4298 /*
4299 * Force memory writes to complete before letting h/w know there
4300 * are new descriptors to fetch. (Only applicable for weak-ordered
4301 * memory model archs, such as IA-64).
4302 *
4303 * We also need this memory barrier to make certain all of the
4304 * status bits have been updated before next_to_watch is written.
4305 */
4306 wmb();
4307
8542db05 4308 /* set next_to_watch value indicating a packet is present */
ebe42d16 4309 first->next_to_watch = tx_desc;
9d5c8243 4310
ebe42d16
AD		 4311 i++;
4312 if (i == tx_ring->count)
4313 i = 0;
6366ad33 4314
ebe42d16 4315 tx_ring->next_to_use = i;
6366ad33 4316
ebe42d16 4317 writel(i, tx_ring->tail);
6366ad33 4318
ebe42d16
AD		 4319 /* we need this if more than one processor can write to our tail
4320 * at a time, it syncronizes IO on IA64/Altix systems */
4321 mmiowb();
4322
4323 return;
4324
4325dma_error:
4326 dev_err(tx_ring->dev, "TX DMA map failed\n");
4327
4328 /* clear dma mappings for failed tx_buffer_info map */
4329 for (;;) {
4330 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4331 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4332 if (tx_buffer_info == first)
4333 break;
a77ff709
NN		 4334 if (i == 0)
4335 i = tx_ring->count;
6366ad33 4336 i--;
6366ad33
AD		 4337 }
4338
9d5c8243 4339 tx_ring->next_to_use = i;
9d5c8243
AK		 4340}
4341
6ad4edfc 4342static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4343{
e694e964
AD		 4344 struct net_device *netdev = tx_ring->netdev;
4345
661086df 4346 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4347
9d5c8243
AK		 4348 /* Herbert's original patch had:
4349 * smp_mb__after_netif_stop_queue();
4350 * but since that doesn't exist yet, just open code it. */
4351 smp_mb();
4352
4353 /* We need to check again in a case another CPU has just
4354 * made room available. */
c493ea45 4355 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4356 return -EBUSY;
4357
4358 /* A reprieve! */
661086df 4359 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4360
4361 u64_stats_update_begin(&tx_ring->tx_syncp2);
4362 tx_ring->tx_stats.restart_queue2++;
4363 u64_stats_update_end(&tx_ring->tx_syncp2);
4364
9d5c8243
AK		 4365 return 0;
4366}
4367
6ad4edfc 4368static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4369{
c493ea45 4370 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4371 return 0;
e694e964 4372 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4373}
4374
cd392f5c
AD		 4375netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4376 struct igb_ring *tx_ring)
9d5c8243 4377{
8542db05 4378 struct igb_tx_buffer *first;
ebe42d16 4379 int tso;
91d4ee33 4380 u32 tx_flags = 0;
31f6adbb 4381 __be16 protocol = vlan_get_protocol(skb);
91d4ee33 4382 u8 hdr_len = 0;
9d5c8243 4383
9d5c8243
AK		 4384 /* need: 1 descriptor per page,
4385 * + 2 desc gap to keep tail from touching head,
4386 * + 1 desc for skb->data,
4387 * + 1 desc for context descriptor,
4388 * otherwise try next time */
e694e964 4389 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4390 /* this is a hard error */
9d5c8243
AK		 4391 return NETDEV_TX_BUSY;
4392 }
33af6bcc 4393
7af40ad9
AD		 4394 /* record the location of the first descriptor for this packet */
4395 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4396 first->skb = skb;
4397 first->bytecount = skb->len;
4398 first->gso_segs = 1;
4399
2244d07b
OH		 4400 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4401 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4402 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4403 }
9d5c8243 4404
eab6d18d 4405 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4406 tx_flags |= IGB_TX_FLAGS_VLAN;
4407 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4408 }
4409
7af40ad9
AD		 4410 /* record initial flags and protocol */
4411 first->tx_flags = tx_flags;
4412 first->protocol = protocol;
cdfd01fc 4413
7af40ad9
AD		 4414 tso = igb_tso(tx_ring, first, &hdr_len);
4415 if (tso < 0)
7d13a7d0 4416 goto out_drop;
7af40ad9
AD		 4417 else if (!tso)
4418 igb_tx_csum(tx_ring, first);
9d5c8243 4419
7af40ad9 4420 igb_tx_map(tx_ring, first, hdr_len);
85ad76b2
AD		 4421
4422 /* Make sure there is space in the ring for the next send. */
e694e964 4423 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4424
9d5c8243 4425 return NETDEV_TX_OK;
7d13a7d0
AD		 4426
4427out_drop:
7af40ad9
AD		 4428 igb_unmap_and_free_tx_resource(tx_ring, first);
4429
7d13a7d0 4430 return NETDEV_TX_OK;
9d5c8243
AK		 4431}
4432
1cc3bd87
AD		 4433static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4434 struct sk_buff *skb)
4435{
4436 unsigned int r_idx = skb->queue_mapping;
4437
4438 if (r_idx >= adapter->num_tx_queues)
4439 r_idx = r_idx % adapter->num_tx_queues;
4440
4441 return adapter->tx_ring[r_idx];
4442}
4443
cd392f5c
AD		 4444static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4445 struct net_device *netdev)
9d5c8243
AK		 4446{
4447 struct igb_adapter *adapter = netdev_priv(netdev);
b1a436c3
AD		 4448
4449 if (test_bit(__IGB_DOWN, &adapter->state)) {
4450 dev_kfree_skb_any(skb);
4451 return NETDEV_TX_OK;
4452 }
4453
4454 if (skb->len <= 0) {
4455 dev_kfree_skb_any(skb);
4456 return NETDEV_TX_OK;
4457 }
4458
1cc3bd87
AD		 4459 /*
4460 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4461 * in order to meet this minimum size requirement.
4462 */
4463 if (skb->len < 17) {
4464 if (skb_padto(skb, 17))
4465 return NETDEV_TX_OK;
4466 skb->len = 17;
4467 }
9d5c8243 4468
1cc3bd87 4469 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
9d5c8243
AK		 4470}
4471
4472/**
4473 * igb_tx_timeout - Respond to a Tx Hang
4474 * @netdev: network interface device structure
4475 **/
4476static void igb_tx_timeout(struct net_device *netdev)
4477{
4478 struct igb_adapter *adapter = netdev_priv(netdev);
4479 struct e1000_hw *hw = &adapter->hw;
4480
4481 /* Do the reset outside of interrupt context */
4482 adapter->tx_timeout_count++;
f7ba205e 4483
06218a8d 4484 if (hw->mac.type >= e1000_82580)
55cac248
AD		 4485 hw->dev_spec._82575.global_device_reset = true;
4486
9d5c8243 4487 schedule_work(&adapter->reset_task);
265de409
AD		 4488 wr32(E1000_EICS,
4489 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4490}
4491
4492static void igb_reset_task(struct work_struct *work)
4493{
4494 struct igb_adapter *adapter;
4495 adapter = container_of(work, struct igb_adapter, reset_task);
4496
c97ec42a
TI		 4497 igb_dump(adapter);
4498 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4499 igb_reinit_locked(adapter);
4500}
4501
4502/**
12dcd86b 4503 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4504 * @netdev: network interface device structure
12dcd86b 4505 * @stats: rtnl_link_stats64 pointer
9d5c8243 4506 *
9d5c8243 4507 **/
12dcd86b
ED		 4508static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4509 struct rtnl_link_stats64 *stats)
9d5c8243 4510{
12dcd86b
ED		 4511 struct igb_adapter *adapter = netdev_priv(netdev);
4512
4513 spin_lock(&adapter->stats64_lock);
4514 igb_update_stats(adapter, &adapter->stats64);
4515 memcpy(stats, &adapter->stats64, sizeof(*stats));
4516 spin_unlock(&adapter->stats64_lock);
4517
4518 return stats;
9d5c8243
AK		 4519}
4520
4521/**
4522 * igb_change_mtu - Change the Maximum Transfer Unit
4523 * @netdev: network interface device structure
4524 * @new_mtu: new value for maximum frame size
4525 *
4526 * Returns 0 on success, negative on failure
4527 **/
4528static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4529{
4530 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4531 struct pci_dev *pdev = adapter->pdev;
153285f9 4532 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9d5c8243 4533
c809d227 4534 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4535 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4536 return -EINVAL;
4537 }
4538
153285f9 4539#define MAX_STD_JUMBO_FRAME_SIZE 9238
9d5c8243 4540 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4541 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4542 return -EINVAL;
4543 }
4544
4545 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4546 msleep(1);
73cd78f1 4547
9d5c8243
AK		 4548 /* igb_down has a dependency on max_frame_size */
4549 adapter->max_frame_size = max_frame;
559e9c49 4550
4c844851
AD		 4551 if (netif_running(netdev))
4552 igb_down(adapter);
9d5c8243 4553
090b1795 4554 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4555 netdev->mtu, new_mtu);
4556 netdev->mtu = new_mtu;
4557
4558 if (netif_running(netdev))
4559 igb_up(adapter);
4560 else
4561 igb_reset(adapter);
4562
4563 clear_bit(__IGB_RESETTING, &adapter->state);
4564
4565 return 0;
4566}
4567
4568/**
4569 * igb_update_stats - Update the board statistics counters
4570 * @adapter: board private structure
4571 **/
4572
12dcd86b
ED		 4573void igb_update_stats(struct igb_adapter *adapter,
4574 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4575{
4576 struct e1000_hw *hw = &adapter->hw;
4577 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4578 u32 reg, mpc;
9d5c8243 4579 u16 phy_tmp;
3f9c0164
AD		 4580 int i;
4581 u64 bytes, packets;
12dcd86b
ED		 4582 unsigned int start;
4583 u64 _bytes, _packets;
9d5c8243
AK		 4584
4585#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4586
4587 /*
4588 * Prevent stats update while adapter is being reset, or if the pci
4589 * connection is down.
4590 */
4591 if (adapter->link_speed == 0)
4592 return;
4593 if (pci_channel_offline(pdev))
4594 return;
4595
3f9c0164
AD		 4596 bytes = 0;
4597 packets = 0;
4598 for (i = 0; i < adapter->num_rx_queues; i++) {
4599 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4600 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4601
3025a446 4602 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4603 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4604
4605 do {
4606 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4607 _bytes = ring->rx_stats.bytes;
4608 _packets = ring->rx_stats.packets;
4609 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4610 bytes += _bytes;
4611 packets += _packets;
3f9c0164
AD		 4612 }
4613
128e45eb
AD		 4614 net_stats->rx_bytes = bytes;
4615 net_stats->rx_packets = packets;
3f9c0164
AD		 4616
4617 bytes = 0;
4618 packets = 0;
4619 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4620 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4621 do {
4622 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4623 _bytes = ring->tx_stats.bytes;
4624 _packets = ring->tx_stats.packets;
4625 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4626 bytes += _bytes;
4627 packets += _packets;
3f9c0164 4628 }
128e45eb
AD		 4629 net_stats->tx_bytes = bytes;
4630 net_stats->tx_packets = packets;
3f9c0164
AD		 4631
4632 /* read stats registers */
9d5c8243
AK		 4633 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4634 adapter->stats.gprc += rd32(E1000_GPRC);
4635 adapter->stats.gorc += rd32(E1000_GORCL);
4636 rd32(E1000_GORCH); /* clear GORCL */
4637 adapter->stats.bprc += rd32(E1000_BPRC);
4638 adapter->stats.mprc += rd32(E1000_MPRC);
4639 adapter->stats.roc += rd32(E1000_ROC);
4640
4641 adapter->stats.prc64 += rd32(E1000_PRC64);
4642 adapter->stats.prc127 += rd32(E1000_PRC127);
4643 adapter->stats.prc255 += rd32(E1000_PRC255);
4644 adapter->stats.prc511 += rd32(E1000_PRC511);
4645 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4646 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4647 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4648 adapter->stats.sec += rd32(E1000_SEC);
4649
fa3d9a6d
MW		 4650 mpc = rd32(E1000_MPC);
4651 adapter->stats.mpc += mpc;
4652 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4653 adapter->stats.scc += rd32(E1000_SCC);
4654 adapter->stats.ecol += rd32(E1000_ECOL);
4655 adapter->stats.mcc += rd32(E1000_MCC);
4656 adapter->stats.latecol += rd32(E1000_LATECOL);
4657 adapter->stats.dc += rd32(E1000_DC);
4658 adapter->stats.rlec += rd32(E1000_RLEC);
4659 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4660 adapter->stats.xontxc += rd32(E1000_XONTXC);
4661 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4662 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4663 adapter->stats.fcruc += rd32(E1000_FCRUC);
4664 adapter->stats.gptc += rd32(E1000_GPTC);
4665 adapter->stats.gotc += rd32(E1000_GOTCL);
4666 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4667 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4668 adapter->stats.ruc += rd32(E1000_RUC);
4669 adapter->stats.rfc += rd32(E1000_RFC);
4670 adapter->stats.rjc += rd32(E1000_RJC);
4671 adapter->stats.tor += rd32(E1000_TORH);
4672 adapter->stats.tot += rd32(E1000_TOTH);
4673 adapter->stats.tpr += rd32(E1000_TPR);
4674
4675 adapter->stats.ptc64 += rd32(E1000_PTC64);
4676 adapter->stats.ptc127 += rd32(E1000_PTC127);
4677 adapter->stats.ptc255 += rd32(E1000_PTC255);
4678 adapter->stats.ptc511 += rd32(E1000_PTC511);
4679 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4680 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4681
4682 adapter->stats.mptc += rd32(E1000_MPTC);
4683 adapter->stats.bptc += rd32(E1000_BPTC);
4684
2d0b0f69
NN		 4685 adapter->stats.tpt += rd32(E1000_TPT);
4686 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4687
4688 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4689 /* read internal phy specific stats */
4690 reg = rd32(E1000_CTRL_EXT);
4691 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4692 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4693 adapter->stats.tncrs += rd32(E1000_TNCRS);
4694 }
4695
9d5c8243
AK		 4696 adapter->stats.tsctc += rd32(E1000_TSCTC);
4697 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4698
4699 adapter->stats.iac += rd32(E1000_IAC);
4700 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4701 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4702 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4703 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4704 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4705 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4706 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4707 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4708
4709 /* Fill out the OS statistics structure */
128e45eb
AD		 4710 net_stats->multicast = adapter->stats.mprc;
4711 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4712
4713 /* Rx Errors */
4714
4715 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4716 * our own version based on RUC and ROC */
128e45eb 4717 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4718 adapter->stats.crcerrs + adapter->stats.algnerrc +
4719 adapter->stats.ruc + adapter->stats.roc +
4720 adapter->stats.cexterr;
128e45eb
AD		 4721 net_stats->rx_length_errors = adapter->stats.ruc +
4722 adapter->stats.roc;
4723 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4724 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4725 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4726
4727 /* Tx Errors */
128e45eb
AD		 4728 net_stats->tx_errors = adapter->stats.ecol +
4729 adapter->stats.latecol;
4730 net_stats->tx_aborted_errors = adapter->stats.ecol;
4731 net_stats->tx_window_errors = adapter->stats.latecol;
4732 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4733
4734 /* Tx Dropped needs to be maintained elsewhere */
4735
4736 /* Phy Stats */
4737 if (hw->phy.media_type == e1000_media_type_copper) {
4738 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4739 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4740 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4741 adapter->phy_stats.idle_errors += phy_tmp;
4742 }
4743 }
4744
4745 /* Management Stats */
4746 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4747 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4748 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
0a915b95
CW		 4749
4750 /* OS2BMC Stats */
4751 reg = rd32(E1000_MANC);
4752 if (reg & E1000_MANC_EN_BMC2OS) {
4753 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4754 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4755 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4756 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4757 }
9d5c8243
AK		 4758}
4759
9d5c8243
AK		 4760static irqreturn_t igb_msix_other(int irq, void *data)
4761{
047e0030 4762 struct igb_adapter *adapter = data;
9d5c8243 4763 struct e1000_hw *hw = &adapter->hw;
844290e5 4764 u32 icr = rd32(E1000_ICR);
844290e5 4765 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4766
7f081d40
AD		 4767 if (icr & E1000_ICR_DRSTA)
4768 schedule_work(&adapter->reset_task);
4769
047e0030 4770 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4771 /* HW is reporting DMA is out of sync */
4772 adapter->stats.doosync++;
13800469
GR		 4773 /* The DMA Out of Sync is also indication of a spoof event
4774 * in IOV mode. Check the Wrong VM Behavior register to
4775 * see if it is really a spoof event. */
4776 igb_check_wvbr(adapter);
dda0e083 4777 }
eebbbdba 4778
4ae196df
AD		 4779 /* Check for a mailbox event */
4780 if (icr & E1000_ICR_VMMB)
4781 igb_msg_task(adapter);
4782
4783 if (icr & E1000_ICR_LSC) {
4784 hw->mac.get_link_status = 1;
4785 /* guard against interrupt when we're going down */
4786 if (!test_bit(__IGB_DOWN, &adapter->state))
4787 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4788 }
4789
844290e5 4790 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4791
4792 return IRQ_HANDLED;
4793}
4794
047e0030 4795static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4796{
26b39276 4797 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4798 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4799
047e0030
AD		 4800 if (!q_vector->set_itr)
4801 return;
73cd78f1 4802
047e0030
AD		 4803 if (!itr_val)
4804 itr_val = 0x4;
661086df 4805
26b39276
AD		 4806 if (adapter->hw.mac.type == e1000_82575)
4807 itr_val |= itr_val << 16;
661086df 4808 else
0ba82994 4809 itr_val |= E1000_EITR_CNT_IGNR;
661086df 4810
047e0030
AD		 4811 writel(itr_val, q_vector->itr_register);
4812 q_vector->set_itr = 0;
6eb5a7f1
AD		 4813}
4814
047e0030 4815static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4816{
047e0030 4817 struct igb_q_vector *q_vector = data;
9d5c8243 4818
047e0030
AD		 4819 /* Write the ITR value calculated from the previous interrupt. */
4820 igb_write_itr(q_vector);
9d5c8243 4821
047e0030 4822 napi_schedule(&q_vector->napi);
844290e5 4823
047e0030 4824 return IRQ_HANDLED;
fe4506b6
JC		 4825}
4826
421e02f0 4827#ifdef CONFIG_IGB_DCA
047e0030 4828static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4829{
047e0030 4830 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4831 struct e1000_hw *hw = &adapter->hw;
4832 int cpu = get_cpu();
fe4506b6 4833
047e0030
AD		 4834 if (q_vector->cpu == cpu)
4835 goto out_no_update;
4836
0ba82994
AD		 4837 if (q_vector->tx.ring) {
4838 int q = q_vector->tx.ring->reg_idx;
047e0030
AD		 4839 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4840 if (hw->mac.type == e1000_82575) {
4841 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4842 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4843 } else {
047e0030
AD		 4844 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4845 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4846 E1000_DCA_TXCTRL_CPUID_SHIFT;
4847 }
4848 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4849 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4850 }
0ba82994
AD		 4851 if (q_vector->rx.ring) {
4852 int q = q_vector->rx.ring->reg_idx;
047e0030
AD		 4853 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4854 if (hw->mac.type == e1000_82575) {
2d064c06 4855 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4856 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4857 } else {
4858 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4859 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4860 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4861 }
fe4506b6
JC		 4862 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4863 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4864 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4865 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4866 }
047e0030
AD		 4867 q_vector->cpu = cpu;
4868out_no_update:
fe4506b6
JC		 4869 put_cpu();
4870}
4871
4872static void igb_setup_dca(struct igb_adapter *adapter)
4873{
7e0e99ef 4874 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4875 int i;
4876
7dfc16fa 4877 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4878 return;
4879
7e0e99ef
AD		 4880 /* Always use CB2 mode, difference is masked in the CB driver. */
4881 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4882
047e0030 4883 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4884 adapter->q_vector[i]->cpu = -1;
4885 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4886 }
4887}
4888
4889static int __igb_notify_dca(struct device *dev, void *data)
4890{
4891 struct net_device *netdev = dev_get_drvdata(dev);
4892 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4893 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4894 struct e1000_hw *hw = &adapter->hw;
4895 unsigned long event = *(unsigned long *)data;
4896
4897 switch (event) {
4898 case DCA_PROVIDER_ADD:
4899 /* if already enabled, don't do it again */
7dfc16fa 4900 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4901 break;
fe4506b6 4902 if (dca_add_requester(dev) == 0) {
bbd98fe4 4903 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4904 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4905 igb_setup_dca(adapter);
4906 break;
4907 }
4908 /* Fall Through since DCA is disabled. */
4909 case DCA_PROVIDER_REMOVE:
7dfc16fa 4910 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4911 /* without this a class_device is left
047e0030 4912 * hanging around in the sysfs model */
fe4506b6 4913 dca_remove_requester(dev);
090b1795 4914 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4915 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4916 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4917 }
4918 break;
4919 }
bbd98fe4 4920
fe4506b6 4921 return 0;
9d5c8243
AK		 4922}
4923
fe4506b6
JC		 4924static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4925 void *p)
4926{
4927 int ret_val;
4928
4929 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4930 __igb_notify_dca);
4931
4932 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4933}
421e02f0 4934#endif /* CONFIG_IGB_DCA */
9d5c8243 4935
0224d663
GR		 4936#ifdef CONFIG_PCI_IOV
4937static int igb_vf_configure(struct igb_adapter *adapter, int vf)
4938{
4939 unsigned char mac_addr[ETH_ALEN];
4940 struct pci_dev *pdev = adapter->pdev;
4941 struct e1000_hw *hw = &adapter->hw;
4942 struct pci_dev *pvfdev;
4943 unsigned int device_id;
4944 u16 thisvf_devfn;
4945
4946 random_ether_addr(mac_addr);
4947 igb_set_vf_mac(adapter, vf, mac_addr);
4948
4949 switch (adapter->hw.mac.type) {
4950 case e1000_82576:
4951 device_id = IGB_82576_VF_DEV_ID;
4952 /* VF Stride for 82576 is 2 */
4953 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
4954 (pdev->devfn & 1);
4955 break;
4956 case e1000_i350:
4957 device_id = IGB_I350_VF_DEV_ID;
4958 /* VF Stride for I350 is 4 */
4959 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
4960 (pdev->devfn & 3);
4961 break;
4962 default:
4963 device_id = 0;
4964 thisvf_devfn = 0;
4965 break;
4966 }
4967
4968 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
4969 while (pvfdev) {
4970 if (pvfdev->devfn == thisvf_devfn)
4971 break;
4972 pvfdev = pci_get_device(hw->vendor_id,
4973 device_id, pvfdev);
4974 }
4975
4976 if (pvfdev)
4977 adapter->vf_data[vf].vfdev = pvfdev;
4978 else
4979 dev_err(&pdev->dev,
4980 "Couldn't find pci dev ptr for VF %4.4x\n",
4981 thisvf_devfn);
4982 return pvfdev != NULL;
4983}
4984
4985static int igb_find_enabled_vfs(struct igb_adapter *adapter)
4986{
4987 struct e1000_hw *hw = &adapter->hw;
4988 struct pci_dev *pdev = adapter->pdev;
4989 struct pci_dev *pvfdev;
4990 u16 vf_devfn = 0;
4991 u16 vf_stride;
4992 unsigned int device_id;
4993 int vfs_found = 0;
4994
4995 switch (adapter->hw.mac.type) {
4996 case e1000_82576:
4997 device_id = IGB_82576_VF_DEV_ID;
4998 /* VF Stride for 82576 is 2 */
4999 vf_stride = 2;
5000 break;
5001 case e1000_i350:
5002 device_id = IGB_I350_VF_DEV_ID;
5003 /* VF Stride for I350 is 4 */
5004 vf_stride = 4;
5005 break;
5006 default:
5007 device_id = 0;
5008 vf_stride = 0;
5009 break;
5010 }
5011
5012 vf_devfn = pdev->devfn + 0x80;
5013 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5014 while (pvfdev) {
06292921
GR		 5015 if (pvfdev->devfn == vf_devfn &&
5016 (pvfdev->bus->number >= pdev->bus->number))
0224d663
GR		 5017 vfs_found++;
5018 vf_devfn += vf_stride;
5019 pvfdev = pci_get_device(hw->vendor_id,
5020 device_id, pvfdev);
5021 }
5022
5023 return vfs_found;
5024}
5025
5026static int igb_check_vf_assignment(struct igb_adapter *adapter)
5027{
5028 int i;
5029 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5030 if (adapter->vf_data[i].vfdev) {
5031 if (adapter->vf_data[i].vfdev->dev_flags &
5032 PCI_DEV_FLAGS_ASSIGNED)
5033 return true;
5034 }
5035 }
5036 return false;
5037}
5038
5039#endif
4ae196df
AD		 5040static void igb_ping_all_vfs(struct igb_adapter *adapter)
5041{
5042 struct e1000_hw *hw = &adapter->hw;
5043 u32 ping;
5044 int i;
5045
5046 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5047 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 5048 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 5049 ping |= E1000_VT_MSGTYPE_CTS;
5050 igb_write_mbx(hw, &ping, 1, i);
5051 }
5052}
5053
7d5753f0
AD		 5054static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5055{
5056 struct e1000_hw *hw = &adapter->hw;
5057 u32 vmolr = rd32(E1000_VMOLR(vf));
5058 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5059
d85b9004 5060 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 5061 IGB_VF_FLAG_MULTI_PROMISC);
5062 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5063
5064 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5065 vmolr |= E1000_VMOLR_MPME;
d85b9004 5066 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 5067 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5068 } else {
5069 /*
5070 * if we have hashes and we are clearing a multicast promisc
5071 * flag we need to write the hashes to the MTA as this step
5072 * was previously skipped
5073 */
5074 if (vf_data->num_vf_mc_hashes > 30) {
5075 vmolr |= E1000_VMOLR_MPME;
5076 } else if (vf_data->num_vf_mc_hashes) {
5077 int j;
5078 vmolr |= E1000_VMOLR_ROMPE;
5079 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5080 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5081 }
5082 }
5083
5084 wr32(E1000_VMOLR(vf), vmolr);
5085
5086 /* there are flags left unprocessed, likely not supported */
5087 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5088 return -EINVAL;
5089
5090 return 0;
5091
5092}
5093
4ae196df
AD		 5094static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5095 u32 *msgbuf, u32 vf)
5096{
5097 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5098 u16 *hash_list = (u16 *)&msgbuf[1];
5099 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5100 int i;
5101
7d5753f0 5102 /* salt away the number of multicast addresses assigned
4ae196df
AD		 5103 * to this VF for later use to restore when the PF multi cast
5104 * list changes
5105 */
5106 vf_data->num_vf_mc_hashes = n;
5107
7d5753f0
AD		 5108 /* only up to 30 hash values supported */
5109 if (n > 30)
5110 n = 30;
5111
5112 /* store the hashes for later use */
4ae196df 5113 for (i = 0; i < n; i++)
a419aef8 5114 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 5115
5116 /* Flush and reset the mta with the new values */
ff41f8dc 5117 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5118
5119 return 0;
5120}
5121
5122static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5123{
5124 struct e1000_hw *hw = &adapter->hw;
5125 struct vf_data_storage *vf_data;
5126 int i, j;
5127
5128 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 5129 u32 vmolr = rd32(E1000_VMOLR(i));
5130 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5131
4ae196df 5132 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 5133
5134 if ((vf_data->num_vf_mc_hashes > 30) ||
5135 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5136 vmolr |= E1000_VMOLR_MPME;
5137 } else if (vf_data->num_vf_mc_hashes) {
5138 vmolr |= E1000_VMOLR_ROMPE;
5139 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5140 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5141 }
5142 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 5143 }
5144}
5145
5146static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5147{
5148 struct e1000_hw *hw = &adapter->hw;
5149 u32 pool_mask, reg, vid;
5150 int i;
5151
5152 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5153
5154 /* Find the vlan filter for this id */
5155 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5156 reg = rd32(E1000_VLVF(i));
5157
5158 /* remove the vf from the pool */
5159 reg &= ~pool_mask;
5160
5161 /* if pool is empty then remove entry from vfta */
5162 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5163 (reg & E1000_VLVF_VLANID_ENABLE)) {
5164 reg = 0;
5165 vid = reg & E1000_VLVF_VLANID_MASK;
5166 igb_vfta_set(hw, vid, false);
5167 }
5168
5169 wr32(E1000_VLVF(i), reg);
5170 }
ae641bdc
AD		 5171
5172 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 5173}
5174
5175static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5176{
5177 struct e1000_hw *hw = &adapter->hw;
5178 u32 reg, i;
5179
51466239
AD		 5180 /* The vlvf table only exists on 82576 hardware and newer */
5181 if (hw->mac.type < e1000_82576)
5182 return -1;
5183
5184 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 5185 if (!adapter->vfs_allocated_count)
5186 return -1;
5187
5188 /* Find the vlan filter for this id */
5189 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5190 reg = rd32(E1000_VLVF(i));
5191 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5192 vid == (reg & E1000_VLVF_VLANID_MASK))
5193 break;
5194 }
5195
5196 if (add) {
5197 if (i == E1000_VLVF_ARRAY_SIZE) {
5198 /* Did not find a matching VLAN ID entry that was
5199 * enabled. Search for a free filter entry, i.e.
5200 * one without the enable bit set
5201 */
5202 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5203 reg = rd32(E1000_VLVF(i));
5204 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5205 break;
5206 }
5207 }
5208 if (i < E1000_VLVF_ARRAY_SIZE) {
5209 /* Found an enabled/available entry */
5210 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5211
5212 /* if !enabled we need to set this up in vfta */
5213 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 5214 /* add VID to filter table */
5215 igb_vfta_set(hw, vid, true);
4ae196df
AD		 5216 reg |= E1000_VLVF_VLANID_ENABLE;
5217 }
cad6d05f
AD		 5218 reg &= ~E1000_VLVF_VLANID_MASK;
5219 reg |= vid;
4ae196df 5220 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5221
5222 /* do not modify RLPML for PF devices */
5223 if (vf >= adapter->vfs_allocated_count)
5224 return 0;
5225
5226 if (!adapter->vf_data[vf].vlans_enabled) {
5227 u32 size;
5228 reg = rd32(E1000_VMOLR(vf));
5229 size = reg & E1000_VMOLR_RLPML_MASK;
5230 size += 4;
5231 reg &= ~E1000_VMOLR_RLPML_MASK;
5232 reg |= size;
5233 wr32(E1000_VMOLR(vf), reg);
5234 }
ae641bdc 5235
51466239 5236 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 5237 }
5238 } else {
5239 if (i < E1000_VLVF_ARRAY_SIZE) {
5240 /* remove vf from the pool */
5241 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5242 /* if pool is empty then remove entry from vfta */
5243 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5244 reg = 0;
5245 igb_vfta_set(hw, vid, false);
5246 }
5247 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5248
5249 /* do not modify RLPML for PF devices */
5250 if (vf >= adapter->vfs_allocated_count)
5251 return 0;
5252
5253 adapter->vf_data[vf].vlans_enabled--;
5254 if (!adapter->vf_data[vf].vlans_enabled) {
5255 u32 size;
5256 reg = rd32(E1000_VMOLR(vf));
5257 size = reg & E1000_VMOLR_RLPML_MASK;
5258 size -= 4;
5259 reg &= ~E1000_VMOLR_RLPML_MASK;
5260 reg |= size;
5261 wr32(E1000_VMOLR(vf), reg);
5262 }
4ae196df
AD		 5263 }
5264 }
8151d294
WM		 5265 return 0;
5266}
5267
5268static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5269{
5270 struct e1000_hw *hw = &adapter->hw;
5271
5272 if (vid)
5273 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5274 else
5275 wr32(E1000_VMVIR(vf), 0);
5276}
5277
5278static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5279 int vf, u16 vlan, u8 qos)
5280{
5281 int err = 0;
5282 struct igb_adapter *adapter = netdev_priv(netdev);
5283
5284 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5285 return -EINVAL;
5286 if (vlan || qos) {
5287 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5288 if (err)
5289 goto out;
5290 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5291 igb_set_vmolr(adapter, vf, !vlan);
5292 adapter->vf_data[vf].pf_vlan = vlan;
5293 adapter->vf_data[vf].pf_qos = qos;
5294 dev_info(&adapter->pdev->dev,
5295 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5296 if (test_bit(__IGB_DOWN, &adapter->state)) {
5297 dev_warn(&adapter->pdev->dev,
5298 "The VF VLAN has been set,"
5299 " but the PF device is not up.\n");
5300 dev_warn(&adapter->pdev->dev,
5301 "Bring the PF device up before"
5302 " attempting to use the VF device.\n");
5303 }
5304 } else {
5305 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5306 false, vf);
5307 igb_set_vmvir(adapter, vlan, vf);
5308 igb_set_vmolr(adapter, vf, true);
5309 adapter->vf_data[vf].pf_vlan = 0;
5310 adapter->vf_data[vf].pf_qos = 0;
5311 }
5312out:
5313 return err;
4ae196df
AD		 5314}
5315
5316static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5317{
5318 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5319 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5320
5321 return igb_vlvf_set(adapter, vid, add, vf);
5322}
5323
f2ca0dbe 5324static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 5325{
8fa7e0f7
GR		 5326 /* clear flags - except flag that indicates PF has set the MAC */
5327 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
f2ca0dbe 5328 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 5329
5330 /* reset offloads to defaults */
8151d294 5331 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 5332
5333 /* reset vlans for device */
5334 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 5335 if (adapter->vf_data[vf].pf_vlan)
5336 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5337 adapter->vf_data[vf].pf_vlan,
5338 adapter->vf_data[vf].pf_qos);
5339 else
5340 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 5341
5342 /* reset multicast table array for vf */
5343 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5344
5345 /* Flush and reset the mta with the new values */
ff41f8dc 5346 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5347}
5348
f2ca0dbe
AD		 5349static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5350{
5351 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5352
5353 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 5354 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5355 random_ether_addr(vf_mac);
f2ca0dbe
AD		 5356
5357 /* process remaining reset events */
5358 igb_vf_reset(adapter, vf);
5359}
5360
5361static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5362{
5363 struct e1000_hw *hw = &adapter->hw;
5364 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5365 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5366 u32 reg, msgbuf[3];
5367 u8 *addr = (u8 *)(&msgbuf[1]);
5368
5369 /* process all the same items cleared in a function level reset */
f2ca0dbe 5370 igb_vf_reset(adapter, vf);
4ae196df
AD		 5371
5372 /* set vf mac address */
26ad9178 5373 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5374
5375 /* enable transmit and receive for vf */
5376 reg = rd32(E1000_VFTE);
5377 wr32(E1000_VFTE, reg | (1 << vf));
5378 reg = rd32(E1000_VFRE);
5379 wr32(E1000_VFRE, reg | (1 << vf));
5380
8fa7e0f7 5381 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
4ae196df
AD		 5382
5383 /* reply to reset with ack and vf mac address */
5384 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5385 memcpy(addr, vf_mac, 6);
5386 igb_write_mbx(hw, msgbuf, 3, vf);
5387}
5388
5389static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5390{
de42edde
GR		 5391 /*
5392 * The VF MAC Address is stored in a packed array of bytes
5393 * starting at the second 32 bit word of the msg array
5394 */
f2ca0dbe
AD		 5395 unsigned char *addr = (char *)&msg[1];
5396 int err = -1;
4ae196df 5397
f2ca0dbe
AD		 5398 if (is_valid_ether_addr(addr))
5399 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5400
f2ca0dbe 5401 return err;
4ae196df
AD		 5402}
5403
5404static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5405{
5406 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5407 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5408 u32 msg = E1000_VT_MSGTYPE_NACK;
5409
5410 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5411 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5412 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5413 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5414 vf_data->last_nack = jiffies;
4ae196df
AD		 5415 }
5416}
5417
f2ca0dbe 5418static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5419{
f2ca0dbe
AD		 5420 struct pci_dev *pdev = adapter->pdev;
5421 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5422 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5423 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5424 s32 retval;
5425
f2ca0dbe 5426 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5427
fef45f4c
AD		 5428 if (retval) {
5429 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5430 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5431 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5432 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5433 return;
5434 goto out;
5435 }
4ae196df
AD		 5436
5437 /* this is a message we already processed, do nothing */
5438 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5439 return;
4ae196df
AD		 5440
5441 /*
5442 * until the vf completes a reset it should not be
5443 * allowed to start any configuration.
5444 */
5445
5446 if (msgbuf[0] == E1000_VF_RESET) {
5447 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5448 return;
4ae196df
AD		 5449 }
5450
f2ca0dbe 5451 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5452 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5453 return;
5454 retval = -1;
5455 goto out;
4ae196df
AD		 5456 }
5457
5458 switch ((msgbuf[0] & 0xFFFF)) {
5459 case E1000_VF_SET_MAC_ADDR:
a6b5ea35
GR		 5460 retval = -EINVAL;
5461 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5462 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5463 else
5464 dev_warn(&pdev->dev,
5465 "VF %d attempted to override administratively "
5466 "set MAC address\nReload the VF driver to "
5467 "resume operations\n", vf);
4ae196df 5468 break;
7d5753f0
AD		 5469 case E1000_VF_SET_PROMISC:
5470 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5471 break;
4ae196df
AD		 5472 case E1000_VF_SET_MULTICAST:
5473 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5474 break;
5475 case E1000_VF_SET_LPE:
5476 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5477 break;
5478 case E1000_VF_SET_VLAN:
a6b5ea35
GR		 5479 retval = -1;
5480 if (vf_data->pf_vlan)
5481 dev_warn(&pdev->dev,
5482 "VF %d attempted to override administratively "
5483 "set VLAN tag\nReload the VF driver to "
5484 "resume operations\n", vf);
8151d294
WM		 5485 else
5486 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5487 break;
5488 default:
090b1795 5489 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5490 retval = -1;
5491 break;
5492 }
5493
fef45f4c
AD		 5494 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5495out:
4ae196df
AD		 5496 /* notify the VF of the results of what it sent us */
5497 if (retval)
5498 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5499 else
5500 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5501
4ae196df 5502 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5503}
4ae196df 5504
f2ca0dbe
AD		 5505static void igb_msg_task(struct igb_adapter *adapter)
5506{
5507 struct e1000_hw *hw = &adapter->hw;
5508 u32 vf;
5509
5510 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5511 /* process any reset requests */
5512 if (!igb_check_for_rst(hw, vf))
5513 igb_vf_reset_event(adapter, vf);
5514
5515 /* process any messages pending */
5516 if (!igb_check_for_msg(hw, vf))
5517 igb_rcv_msg_from_vf(adapter, vf);
5518
5519 /* process any acks */
5520 if (!igb_check_for_ack(hw, vf))
5521 igb_rcv_ack_from_vf(adapter, vf);
5522 }
4ae196df
AD		 5523}
5524
68d480c4
AD		 5525/**
5526 * igb_set_uta - Set unicast filter table address
5527 * @adapter: board private structure
5528 *
5529 * The unicast table address is a register array of 32-bit registers.
5530 * The table is meant to be used in a way similar to how the MTA is used
5531 * however due to certain limitations in the hardware it is necessary to
25985edc
LDM		 5532 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5533 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
68d480c4
AD		 5534 **/
5535static void igb_set_uta(struct igb_adapter *adapter)
5536{
5537 struct e1000_hw *hw = &adapter->hw;
5538 int i;
5539
5540 /* The UTA table only exists on 82576 hardware and newer */
5541 if (hw->mac.type < e1000_82576)
5542 return;
5543
5544 /* we only need to do this if VMDq is enabled */
5545 if (!adapter->vfs_allocated_count)
5546 return;
5547
5548 for (i = 0; i < hw->mac.uta_reg_count; i++)
5549 array_wr32(E1000_UTA, i, ~0);
5550}
5551
9d5c8243
AK		 5552/**
5553 * igb_intr_msi - Interrupt Handler
5554 * @irq: interrupt number
5555 * @data: pointer to a network interface device structure
5556 **/
5557static irqreturn_t igb_intr_msi(int irq, void *data)
5558{
047e0030
AD		 5559 struct igb_adapter *adapter = data;
5560 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5561 struct e1000_hw *hw = &adapter->hw;
5562 /* read ICR disables interrupts using IAM */
5563 u32 icr = rd32(E1000_ICR);
5564
047e0030 5565 igb_write_itr(q_vector);
9d5c8243 5566
7f081d40
AD		 5567 if (icr & E1000_ICR_DRSTA)
5568 schedule_work(&adapter->reset_task);
5569
047e0030 5570 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5571 /* HW is reporting DMA is out of sync */
5572 adapter->stats.doosync++;
5573 }
5574
9d5c8243
AK		 5575 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5576 hw->mac.get_link_status = 1;
5577 if (!test_bit(__IGB_DOWN, &adapter->state))
5578 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5579 }
5580
047e0030 5581 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5582
5583 return IRQ_HANDLED;
5584}
5585
5586/**
4a3c6433 5587 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5588 * @irq: interrupt number
5589 * @data: pointer to a network interface device structure
5590 **/
5591static irqreturn_t igb_intr(int irq, void *data)
5592{
047e0030
AD		 5593 struct igb_adapter *adapter = data;
5594 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5595 struct e1000_hw *hw = &adapter->hw;
5596 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5597 * need for the IMC write */
5598 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5599
5600 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5601 * not set, then the adapter didn't send an interrupt */
5602 if (!(icr & E1000_ICR_INT_ASSERTED))
5603 return IRQ_NONE;
5604
0ba82994
AD		 5605 igb_write_itr(q_vector);
5606
7f081d40
AD		 5607 if (icr & E1000_ICR_DRSTA)
5608 schedule_work(&adapter->reset_task);
5609
047e0030 5610 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5611 /* HW is reporting DMA is out of sync */
5612 adapter->stats.doosync++;
5613 }
5614
9d5c8243
AK		 5615 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5616 hw->mac.get_link_status = 1;
5617 /* guard against interrupt when we're going down */
5618 if (!test_bit(__IGB_DOWN, &adapter->state))
5619 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5620 }
5621
047e0030 5622 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5623
5624 return IRQ_HANDLED;
5625}
5626
c50b52a0 5627static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5628{
047e0030 5629 struct igb_adapter *adapter = q_vector->adapter;
46544258 5630 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5631
0ba82994
AD		 5632 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
5633 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
5634 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
5635 igb_set_itr(q_vector);
46544258 5636 else
047e0030 5637 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5638 }
5639
46544258
AD		 5640 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5641 if (adapter->msix_entries)
047e0030 5642 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5643 else
5644 igb_irq_enable(adapter);
5645 }
9d5c8243
AK		 5646}
5647
46544258
AD		 5648/**
5649 * igb_poll - NAPI Rx polling callback
5650 * @napi: napi polling structure
5651 * @budget: count of how many packets we should handle
5652 **/
5653static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5654{
047e0030
AD		 5655 struct igb_q_vector *q_vector = container_of(napi,
5656 struct igb_q_vector,
5657 napi);
16eb8815 5658 bool clean_complete = true;
9d5c8243 5659
421e02f0 5660#ifdef CONFIG_IGB_DCA
047e0030
AD		 5661 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5662 igb_update_dca(q_vector);
fe4506b6 5663#endif
0ba82994 5664 if (q_vector->tx.ring)
13fde97a 5665 clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5666
0ba82994 5667 if (q_vector->rx.ring)
cd392f5c 5668 clean_complete &= igb_clean_rx_irq(q_vector, budget);
047e0030 5669
16eb8815
AD		 5670 /* If all work not completed, return budget and keep polling */
5671 if (!clean_complete)
5672 return budget;
46544258 5673
9d5c8243 5674 /* If not enough Rx work done, exit the polling mode */
16eb8815
AD		 5675 napi_complete(napi);
5676 igb_ring_irq_enable(q_vector);
9d5c8243 5677
16eb8815 5678 return 0;
9d5c8243 5679}
6d8126f9 5680
33af6bcc 5681/**
c5b9bd5e 5682 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5683 * @adapter: board private structure
c5b9bd5e
AD		 5684 * @shhwtstamps: timestamp structure to update
5685 * @regval: unsigned 64bit system time value.
5686 *
5687 * We need to convert the system time value stored in the RX/TXSTMP registers
5688 * into a hwtstamp which can be used by the upper level timestamping functions
5689 */
5690static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5691 struct skb_shared_hwtstamps *shhwtstamps,
5692 u64 regval)
5693{
5694 u64 ns;
5695
55cac248
AD		 5696 /*
5697 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5698 * 24 to match clock shift we setup earlier.
5699 */
06218a8d 5700 if (adapter->hw.mac.type >= e1000_82580)
55cac248
AD		 5701 regval <<= IGB_82580_TSYNC_SHIFT;
5702
c5b9bd5e
AD		 5703 ns = timecounter_cyc2time(&adapter->clock, regval);
5704 timecompare_update(&adapter->compare, ns);
5705 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5706 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5707 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5708}
5709
5710/**
5711 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5712 * @q_vector: pointer to q_vector containing needed info
06034649 5713 * @buffer: pointer to igb_tx_buffer structure
33af6bcc
PO		 5714 *
5715 * If we were asked to do hardware stamping and such a time stamp is
5716 * available, then it must have been for this skb here because we only
5717 * allow only one such packet into the queue.
5718 */
06034649
AD		 5719static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
5720 struct igb_tx_buffer *buffer_info)
33af6bcc 5721{
c5b9bd5e 5722 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5723 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5724 struct skb_shared_hwtstamps shhwtstamps;
5725 u64 regval;
33af6bcc 5726
c5b9bd5e 5727 /* if skb does not support hw timestamp or TX stamp not valid exit */
2bbfebe2 5728 if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
c5b9bd5e
AD		 5729 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5730 return;
5731
5732 regval = rd32(E1000_TXSTMPL);
5733 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5734
5735 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5736 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5737}
5738
9d5c8243
AK		 5739/**
5740 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5741 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5742 * returns true if ring is completely cleaned
5743 **/
047e0030 5744static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5745{
047e0030 5746 struct igb_adapter *adapter = q_vector->adapter;
0ba82994 5747 struct igb_ring *tx_ring = q_vector->tx.ring;
06034649 5748 struct igb_tx_buffer *tx_buffer;
8542db05 5749 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5750 unsigned int total_bytes = 0, total_packets = 0;
0ba82994 5751 unsigned int budget = q_vector->tx.work_limit;
8542db05 5752 unsigned int i = tx_ring->next_to_clean;
9d5c8243 5753
13fde97a
AD		 5754 if (test_bit(__IGB_DOWN, &adapter->state))
5755 return true;
0e014cb1 5756
06034649 5757 tx_buffer = &tx_ring->tx_buffer_info[i];
13fde97a 5758 tx_desc = IGB_TX_DESC(tx_ring, i);
8542db05 5759 i -= tx_ring->count;
9d5c8243 5760
13fde97a 5761 for (; budget; budget--) {
8542db05 5762 eop_desc = tx_buffer->next_to_watch;
13fde97a 5763
8542db05
AD		 5764 /* prevent any other reads prior to eop_desc */
5765 rmb();
5766
5767 /* if next_to_watch is not set then there is no work pending */
5768 if (!eop_desc)
5769 break;
13fde97a
AD		 5770
5771 /* if DD is not set pending work has not been completed */
5772 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5773 break;
5774
8542db05
AD		 5775 /* clear next_to_watch to prevent false hangs */
5776 tx_buffer->next_to_watch = NULL;
9d5c8243 5777
ebe42d16
AD		 5778 /* update the statistics for this packet */
5779 total_bytes += tx_buffer->bytecount;
5780 total_packets += tx_buffer->gso_segs;
13fde97a 5781
ebe42d16
AD		 5782 /* retrieve hardware timestamp */
5783 igb_tx_hwtstamp(q_vector, tx_buffer);
5784
5785 /* free the skb */
5786 dev_kfree_skb_any(tx_buffer->skb);
5787 tx_buffer->skb = NULL;
13fde97a 5788
ebe42d16
AD		 5789 /* unmap skb header data */
5790 dma_unmap_single(tx_ring->dev,
5791 tx_buffer->dma,
5792 tx_buffer->length,
5793 DMA_TO_DEVICE);
5794
5795 /* clear last DMA location and unmap remaining buffers */
5796 while (tx_desc != eop_desc) {
5797 tx_buffer->dma = 0;
9d5c8243 5798
13fde97a
AD		 5799 tx_buffer++;
5800 tx_desc++;
9d5c8243 5801 i++;
8542db05
AD		 5802 if (unlikely(!i)) {
5803 i -= tx_ring->count;
06034649 5804 tx_buffer = tx_ring->tx_buffer_info;
13fde97a
AD		 5805 tx_desc = IGB_TX_DESC(tx_ring, 0);
5806 }
ebe42d16
AD		 5807
5808 /* unmap any remaining paged data */
5809 if (tx_buffer->dma) {
5810 dma_unmap_page(tx_ring->dev,
5811 tx_buffer->dma,
5812 tx_buffer->length,
5813 DMA_TO_DEVICE);
5814 }
5815 }
5816
5817 /* clear last DMA location */
5818 tx_buffer->dma = 0;
5819
5820 /* move us one more past the eop_desc for start of next pkt */
5821 tx_buffer++;
5822 tx_desc++;
5823 i++;
5824 if (unlikely(!i)) {
5825 i -= tx_ring->count;
5826 tx_buffer = tx_ring->tx_buffer_info;
5827 tx_desc = IGB_TX_DESC(tx_ring, 0);
5828 }
0e014cb1
AD		 5829 }
5830
bdbc0631
ED		 5831 netdev_tx_completed_queue(txring_txq(tx_ring),
5832 total_packets, total_bytes);
8542db05 5833 i += tx_ring->count;
9d5c8243 5834 tx_ring->next_to_clean = i;
13fde97a
AD		 5835 u64_stats_update_begin(&tx_ring->tx_syncp);
5836 tx_ring->tx_stats.bytes += total_bytes;
5837 tx_ring->tx_stats.packets += total_packets;
5838 u64_stats_update_end(&tx_ring->tx_syncp);
0ba82994
AD		 5839 q_vector->tx.total_bytes += total_bytes;
5840 q_vector->tx.total_packets += total_packets;
9d5c8243 5841
6d095fa8 5842 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
13fde97a 5843 struct e1000_hw *hw = &adapter->hw;
12dcd86b 5844
8542db05 5845 eop_desc = tx_buffer->next_to_watch;
9d5c8243 5846
9d5c8243
AK		 5847 /* Detect a transmit hang in hardware, this serializes the
5848 * check with the clearing of time_stamp and movement of i */
6d095fa8 5849 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
8542db05
AD		 5850 if (eop_desc &&
5851 time_after(jiffies, tx_buffer->time_stamp +
8e95a202
JP		 5852 (adapter->tx_timeout_factor * HZ)) &&
5853 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5854
9d5c8243 5855 /* detected Tx unit hang */
59d71989 5856 dev_err(tx_ring->dev,
9d5c8243 5857 "Detected Tx Unit Hang\n"
2d064c06 5858 " Tx Queue <%d>\n"
9d5c8243
AK		 5859 " TDH <%x>\n"
5860 " TDT <%x>\n"
5861 " next_to_use <%x>\n"
5862 " next_to_clean <%x>\n"
9d5c8243
AK		 5863 "buffer_info[next_to_clean]\n"
5864 " time_stamp <%lx>\n"
8542db05 5865 " next_to_watch <%p>\n"
9d5c8243
AK		 5866 " jiffies <%lx>\n"
5867 " desc.status <%x>\n",
2d064c06 5868 tx_ring->queue_index,
238ac817 5869 rd32(E1000_TDH(tx_ring->reg_idx)),
fce99e34 5870 readl(tx_ring->tail),
9d5c8243
AK		 5871 tx_ring->next_to_use,
5872 tx_ring->next_to_clean,
8542db05
AD		 5873 tx_buffer->time_stamp,
5874 eop_desc,
9d5c8243 5875 jiffies,
0e014cb1 5876 eop_desc->wb.status);
13fde97a
AD		 5877 netif_stop_subqueue(tx_ring->netdev,
5878 tx_ring->queue_index);
5879
5880 /* we are about to reset, no point in enabling stuff */
5881 return true;
9d5c8243
AK		 5882 }
5883 }
13fde97a
AD		 5884
5885 if (unlikely(total_packets &&
5886 netif_carrier_ok(tx_ring->netdev) &&
5887 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5888 /* Make sure that anybody stopping the queue after this
5889 * sees the new next_to_clean.
5890 */
5891 smp_mb();
5892 if (__netif_subqueue_stopped(tx_ring->netdev,
5893 tx_ring->queue_index) &&
5894 !(test_bit(__IGB_DOWN, &adapter->state))) {
5895 netif_wake_subqueue(tx_ring->netdev,
5896 tx_ring->queue_index);
5897
5898 u64_stats_update_begin(&tx_ring->tx_syncp);
5899 tx_ring->tx_stats.restart_queue++;
5900 u64_stats_update_end(&tx_ring->tx_syncp);
5901 }
5902 }
5903
5904 return !!budget;
9d5c8243
AK		 5905}
5906
cd392f5c 5907static inline void igb_rx_checksum(struct igb_ring *ring,
3ceb90fd
AD		 5908 union e1000_adv_rx_desc *rx_desc,
5909 struct sk_buff *skb)
9d5c8243 5910{
bc8acf2c 5911 skb_checksum_none_assert(skb);
9d5c8243 5912
294e7d78 5913 /* Ignore Checksum bit is set */
3ceb90fd 5914 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
294e7d78
AD		 5915 return;
5916
5917 /* Rx checksum disabled via ethtool */
5918 if (!(ring->netdev->features & NETIF_F_RXCSUM))
9d5c8243 5919 return;
85ad76b2 5920
9d5c8243 5921 /* TCP/UDP checksum error bit is set */
3ceb90fd
AD		 5922 if (igb_test_staterr(rx_desc,
5923 E1000_RXDEXT_STATERR_TCPE |
5924 E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5925 /*
5926 * work around errata with sctp packets where the TCPE aka
5927 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5928 * packets, (aka let the stack check the crc32c)
5929 */
866cff06
AD		 5930 if (!((skb->len == 60) &&
5931 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
12dcd86b 5932 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5933 ring->rx_stats.csum_err++;
12dcd86b
ED		 5934 u64_stats_update_end(&ring->rx_syncp);
5935 }
9d5c8243 5936 /* let the stack verify checksum errors */
9d5c8243
AK		 5937 return;
5938 }
5939 /* It must be a TCP or UDP packet with a valid checksum */
3ceb90fd
AD		 5940 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
5941 E1000_RXD_STAT_UDPCS))
9d5c8243
AK		 5942 skb->ip_summed = CHECKSUM_UNNECESSARY;
5943
3ceb90fd
AD		 5944 dev_dbg(ring->dev, "cksum success: bits %08X\n",
5945 le32_to_cpu(rx_desc->wb.upper.status_error));
9d5c8243
AK		 5946}
5947
077887c3
AD		 5948static inline void igb_rx_hash(struct igb_ring *ring,
5949 union e1000_adv_rx_desc *rx_desc,
5950 struct sk_buff *skb)
5951{
5952 if (ring->netdev->features & NETIF_F_RXHASH)
5953 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
5954}
5955
3ceb90fd
AD		 5956static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
5957 union e1000_adv_rx_desc *rx_desc,
5958 struct sk_buff *skb)
c5b9bd5e
AD		 5959{
5960 struct igb_adapter *adapter = q_vector->adapter;
5961 struct e1000_hw *hw = &adapter->hw;
5962 u64 regval;
5963
3ceb90fd
AD		 5964 if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
5965 E1000_RXDADV_STAT_TS))
5966 return;
5967
c5b9bd5e
AD		 5968 /*
5969 * If this bit is set, then the RX registers contain the time stamp. No
5970 * other packet will be time stamped until we read these registers, so
5971 * read the registers to make them available again. Because only one
5972 * packet can be time stamped at a time, we know that the register
5973 * values must belong to this one here and therefore we don't need to
5974 * compare any of the additional attributes stored for it.
5975 *
2244d07b 5976 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 5977 * can turn into a skb_shared_hwtstamps.
5978 */
3ceb90fd 5979 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
757b77e2
NN		 5980 u32 *stamp = (u32 *)skb->data;
5981 regval = le32_to_cpu(*(stamp + 2));
5982 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5983 skb_pull(skb, IGB_TS_HDR_LEN);
5984 } else {
5985 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5986 return;
c5b9bd5e 5987
757b77e2
NN		 5988 regval = rd32(E1000_RXSTMPL);
5989 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5990 }
c5b9bd5e
AD		 5991
5992 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5993}
8be10e91
AD		 5994
5995static void igb_rx_vlan(struct igb_ring *ring,
5996 union e1000_adv_rx_desc *rx_desc,
5997 struct sk_buff *skb)
5998{
5999 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
6000 u16 vid;
6001 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
6002 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
6003 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
6004 else
6005 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
6006
6007 __vlan_hwaccel_put_tag(skb, vid);
6008 }
6009}
6010
44390ca6 6011static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
2d94d8ab
AD		 6012{
6013 /* HW will not DMA in data larger than the given buffer, even if it
6014 * parses the (NFS, of course) header to be larger. In that case, it
6015 * fills the header buffer and spills the rest into the page.
6016 */
6017 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
6018 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
44390ca6
AD		 6019 if (hlen > IGB_RX_HDR_LEN)
6020 hlen = IGB_RX_HDR_LEN;
2d94d8ab
AD		 6021 return hlen;
6022}
6023
cd392f5c 6024static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
9d5c8243 6025{
0ba82994 6026 struct igb_ring *rx_ring = q_vector->rx.ring;
16eb8815
AD		 6027 union e1000_adv_rx_desc *rx_desc;
6028 const int current_node = numa_node_id();
9d5c8243 6029 unsigned int total_bytes = 0, total_packets = 0;
16eb8815
AD		 6030 u16 cleaned_count = igb_desc_unused(rx_ring);
6031 u16 i = rx_ring->next_to_clean;
9d5c8243 6032
60136906 6033 rx_desc = IGB_RX_DESC(rx_ring, i);
9d5c8243 6034
3ceb90fd 6035 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
06034649 6036 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
16eb8815
AD		 6037 struct sk_buff *skb = buffer_info->skb;
6038 union e1000_adv_rx_desc *next_rxd;
9d5c8243 6039
69d3ca53 6040 buffer_info->skb = NULL;
16eb8815 6041 prefetch(skb->data);
69d3ca53
AD		 6042
6043 i++;
6044 if (i == rx_ring->count)
6045 i = 0;
42d0781a 6046
60136906 6047 next_rxd = IGB_RX_DESC(rx_ring, i);
69d3ca53 6048 prefetch(next_rxd);
9d5c8243 6049
16eb8815
AD		 6050 /*
6051 * This memory barrier is needed to keep us from reading
6052 * any other fields out of the rx_desc until we know the
6053 * RXD_STAT_DD bit is set
6054 */
6055 rmb();
9d5c8243 6056
16eb8815
AD		 6057 if (!skb_is_nonlinear(skb)) {
6058 __skb_put(skb, igb_get_hlen(rx_desc));
6059 dma_unmap_single(rx_ring->dev, buffer_info->dma,
44390ca6 6060 IGB_RX_HDR_LEN,
59d71989 6061 DMA_FROM_DEVICE);
91615f76 6062 buffer_info->dma = 0;
bf36c1a0
AD		 6063 }
6064
16eb8815
AD		 6065 if (rx_desc->wb.upper.length) {
6066 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
bf36c1a0 6067
aa913403 6068 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 6069 buffer_info->page,
6070 buffer_info->page_offset,
6071 length);
6072
16eb8815
AD		 6073 skb->len += length;
6074 skb->data_len += length;
95b9c1df 6075 skb->truesize += PAGE_SIZE / 2;
16eb8815 6076
d1eff350
AD		 6077 if ((page_count(buffer_info->page) != 1) ||
6078 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 6079 buffer_info->page = NULL;
6080 else
6081 get_page(buffer_info->page);
9d5c8243 6082
16eb8815
AD		 6083 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
6084 PAGE_SIZE / 2, DMA_FROM_DEVICE);
6085 buffer_info->page_dma = 0;
9d5c8243 6086 }
9d5c8243 6087
3ceb90fd 6088 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
06034649
AD		 6089 struct igb_rx_buffer *next_buffer;
6090 next_buffer = &rx_ring->rx_buffer_info[i];
b2d56536
AD		 6091 buffer_info->skb = next_buffer->skb;
6092 buffer_info->dma = next_buffer->dma;
6093 next_buffer->skb = skb;
6094 next_buffer->dma = 0;
bf36c1a0
AD		 6095 goto next_desc;
6096 }
44390ca6 6097
3ceb90fd
AD		 6098 if (igb_test_staterr(rx_desc,
6099 E1000_RXDEXT_ERR_FRAME_ERR_MASK)) {
16eb8815 6100 dev_kfree_skb_any(skb);
9d5c8243
AK		 6101 goto next_desc;
6102 }
9d5c8243 6103
3ceb90fd 6104 igb_rx_hwtstamp(q_vector, rx_desc, skb);
077887c3 6105 igb_rx_hash(rx_ring, rx_desc, skb);
3ceb90fd 6106 igb_rx_checksum(rx_ring, rx_desc, skb);
8be10e91 6107 igb_rx_vlan(rx_ring, rx_desc, skb);
3ceb90fd
AD		 6108
6109 total_bytes += skb->len;
6110 total_packets++;
6111
6112 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
6113
b2cb09b1 6114 napi_gro_receive(&q_vector->napi, skb);
9d5c8243 6115
16eb8815 6116 budget--;
9d5c8243 6117next_desc:
16eb8815
AD		 6118 if (!budget)
6119 break;
6120
6121 cleaned_count++;
9d5c8243
AK		 6122 /* return some buffers to hardware, one at a time is too slow */
6123 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
cd392f5c 6124 igb_alloc_rx_buffers(rx_ring, cleaned_count);
9d5c8243
AK		 6125 cleaned_count = 0;
6126 }
6127
6128 /* use prefetched values */
6129 rx_desc = next_rxd;
9d5c8243 6130 }
bf36c1a0 6131
9d5c8243 6132 rx_ring->next_to_clean = i;
12dcd86b 6133 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 6134 rx_ring->rx_stats.packets += total_packets;
6135 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 6136 u64_stats_update_end(&rx_ring->rx_syncp);
0ba82994
AD		 6137 q_vector->rx.total_packets += total_packets;
6138 q_vector->rx.total_bytes += total_bytes;
c023cd88
AD		 6139
6140 if (cleaned_count)
cd392f5c 6141 igb_alloc_rx_buffers(rx_ring, cleaned_count);
c023cd88 6142
16eb8815 6143 return !!budget;
9d5c8243
AK		 6144}
6145
c023cd88 6146static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
06034649 6147 struct igb_rx_buffer *bi)
c023cd88
AD		 6148{
6149 struct sk_buff *skb = bi->skb;
6150 dma_addr_t dma = bi->dma;
6151
6152 if (dma)
6153 return true;
6154
6155 if (likely(!skb)) {
6156 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6157 IGB_RX_HDR_LEN);
6158 bi->skb = skb;
6159 if (!skb) {
6160 rx_ring->rx_stats.alloc_failed++;
6161 return false;
6162 }
6163
6164 /* initialize skb for ring */
6165 skb_record_rx_queue(skb, rx_ring->queue_index);
6166 }
6167
6168 dma = dma_map_single(rx_ring->dev, skb->data,
6169 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6170
6171 if (dma_mapping_error(rx_ring->dev, dma)) {
6172 rx_ring->rx_stats.alloc_failed++;
6173 return false;
6174 }
6175
6176 bi->dma = dma;
6177 return true;
6178}
6179
6180static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
06034649 6181 struct igb_rx_buffer *bi)
c023cd88
AD		 6182{
6183 struct page *page = bi->page;
6184 dma_addr_t page_dma = bi->page_dma;
6185 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6186
6187 if (page_dma)
6188 return true;
6189
6190 if (!page) {
1f2149c1 6191 page = alloc_page(GFP_ATOMIC | __GFP_COLD);
c023cd88
AD		 6192 bi->page = page;
6193 if (unlikely(!page)) {
6194 rx_ring->rx_stats.alloc_failed++;
6195 return false;
6196 }
6197 }
6198
6199 page_dma = dma_map_page(rx_ring->dev, page,
6200 page_offset, PAGE_SIZE / 2,
6201 DMA_FROM_DEVICE);
6202
6203 if (dma_mapping_error(rx_ring->dev, page_dma)) {
6204 rx_ring->rx_stats.alloc_failed++;
6205 return false;
6206 }
6207
6208 bi->page_dma = page_dma;
6209 bi->page_offset = page_offset;
6210 return true;
6211}
6212
9d5c8243 6213/**
cd392f5c 6214 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
9d5c8243
AK		 6215 * @adapter: address of board private structure
6216 **/
cd392f5c 6217void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9d5c8243 6218{
9d5c8243 6219 union e1000_adv_rx_desc *rx_desc;
06034649 6220 struct igb_rx_buffer *bi;
c023cd88 6221 u16 i = rx_ring->next_to_use;
9d5c8243 6222
60136906 6223 rx_desc = IGB_RX_DESC(rx_ring, i);
06034649 6224 bi = &rx_ring->rx_buffer_info[i];
c023cd88 6225 i -= rx_ring->count;
9d5c8243
AK		 6226
6227 while (cleaned_count--) {
c023cd88
AD		 6228 if (!igb_alloc_mapped_skb(rx_ring, bi))
6229 break;
9d5c8243 6230
c023cd88
AD		 6231 /* Refresh the desc even if buffer_addrs didn't change
6232 * because each write-back erases this info. */
6233 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
9d5c8243 6234
c023cd88
AD		 6235 if (!igb_alloc_mapped_page(rx_ring, bi))
6236 break;
6237
6238 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
9d5c8243 6239
c023cd88
AD		 6240 rx_desc++;
6241 bi++;
9d5c8243 6242 i++;
c023cd88 6243 if (unlikely(!i)) {
60136906 6244 rx_desc = IGB_RX_DESC(rx_ring, 0);
06034649 6245 bi = rx_ring->rx_buffer_info;
c023cd88
AD		 6246 i -= rx_ring->count;
6247 }
6248
6249 /* clear the hdr_addr for the next_to_use descriptor */
6250 rx_desc->read.hdr_addr = 0;
9d5c8243
AK		 6251 }
6252
c023cd88
AD		 6253 i += rx_ring->count;
6254
9d5c8243
AK		 6255 if (rx_ring->next_to_use != i) {
6256 rx_ring->next_to_use = i;
9d5c8243
AK		 6257
6258 /* Force memory writes to complete before letting h/w
6259 * know there are new descriptors to fetch. (Only
6260 * applicable for weak-ordered memory model archs,
6261 * such as IA-64). */
6262 wmb();
fce99e34 6263 writel(i, rx_ring->tail);
9d5c8243
AK		 6264 }
6265}
6266
6267/**
6268 * igb_mii_ioctl -
6269 * @netdev:
6270 * @ifreq:
6271 * @cmd:
6272 **/
6273static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6274{
6275 struct igb_adapter *adapter = netdev_priv(netdev);
6276 struct mii_ioctl_data *data = if_mii(ifr);
6277
6278 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6279 return -EOPNOTSUPP;
6280
6281 switch (cmd) {
6282 case SIOCGMIIPHY:
6283 data->phy_id = adapter->hw.phy.addr;
6284 break;
6285 case SIOCGMIIREG:
f5f4cf08
AD		 6286 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6287 &data->val_out))
9d5c8243
AK		 6288 return -EIO;
6289 break;
6290 case SIOCSMIIREG:
6291 default:
6292 return -EOPNOTSUPP;
6293 }
6294 return 0;
6295}
6296
c6cb090b
PO		 6297/**
6298 * igb_hwtstamp_ioctl - control hardware time stamping
6299 * @netdev:
6300 * @ifreq:
6301 * @cmd:
6302 *
33af6bcc
PO		 6303 * Outgoing time stamping can be enabled and disabled. Play nice and
6304 * disable it when requested, although it shouldn't case any overhead
6305 * when no packet needs it. At most one packet in the queue may be
6306 * marked for time stamping, otherwise it would be impossible to tell
6307 * for sure to which packet the hardware time stamp belongs.
6308 *
6309 * Incoming time stamping has to be configured via the hardware
6310 * filters. Not all combinations are supported, in particular event
6311 * type has to be specified. Matching the kind of event packet is
6312 * not supported, with the exception of "all V2 events regardless of
6313 * level 2 or 4".
6314 *
c6cb090b
PO		 6315 **/
6316static int igb_hwtstamp_ioctl(struct net_device *netdev,
6317 struct ifreq *ifr, int cmd)
6318{
33af6bcc
PO		 6319 struct igb_adapter *adapter = netdev_priv(netdev);
6320 struct e1000_hw *hw = &adapter->hw;
c6cb090b 6321 struct hwtstamp_config config;
c5b9bd5e
AD		 6322 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6323 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 6324 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 6325 bool is_l4 = false;
6326 bool is_l2 = false;
33af6bcc 6327 u32 regval;
c6cb090b
PO		 6328
6329 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6330 return -EFAULT;
6331
6332 /* reserved for future extensions */
6333 if (config.flags)
6334 return -EINVAL;
6335
33af6bcc
PO		 6336 switch (config.tx_type) {
6337 case HWTSTAMP_TX_OFF:
c5b9bd5e 6338 tsync_tx_ctl = 0;
33af6bcc 6339 case HWTSTAMP_TX_ON:
33af6bcc
PO		 6340 break;
6341 default:
6342 return -ERANGE;
6343 }
6344
6345 switch (config.rx_filter) {
6346 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 6347 tsync_rx_ctl = 0;
33af6bcc
PO		 6348 break;
6349 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6350 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6351 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6352 case HWTSTAMP_FILTER_ALL:
6353 /*
6354 * register TSYNCRXCFG must be set, therefore it is not
6355 * possible to time stamp both Sync and Delay_Req messages
6356 * => fall back to time stamping all packets
6357 */
c5b9bd5e 6358 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 6359 config.rx_filter = HWTSTAMP_FILTER_ALL;
6360 break;
6361 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 6362 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6363 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 6364 is_l4 = true;
33af6bcc
PO		 6365 break;
6366 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 6367 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6368 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 6369 is_l4 = true;
33af6bcc
PO		 6370 break;
6371 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6372 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 6373 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6374 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 6375 is_l2 = true;
6376 is_l4 = true;
33af6bcc
PO		 6377 config.rx_filter = HWTSTAMP_FILTER_SOME;
6378 break;
6379 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6380 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 6381 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6382 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 6383 is_l2 = true;
6384 is_l4 = true;
33af6bcc
PO		 6385 config.rx_filter = HWTSTAMP_FILTER_SOME;
6386 break;
6387 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6388 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6389 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 6390 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 6391 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 6392 is_l2 = true;
11ba69e8 6393 is_l4 = true;
33af6bcc
PO		 6394 break;
6395 default:
6396 return -ERANGE;
6397 }
6398
c5b9bd5e
AD		 6399 if (hw->mac.type == e1000_82575) {
6400 if (tsync_rx_ctl | tsync_tx_ctl)
6401 return -EINVAL;
6402 return 0;
6403 }
6404
757b77e2
NN		 6405 /*
6406 * Per-packet timestamping only works if all packets are
6407 * timestamped, so enable timestamping in all packets as
6408 * long as one rx filter was configured.
6409 */
06218a8d 6410 if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
757b77e2
NN		 6411 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6412 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6413 }
6414
33af6bcc
PO		 6415 /* enable/disable TX */
6416 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 6417 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6418 regval |= tsync_tx_ctl;
33af6bcc
PO		 6419 wr32(E1000_TSYNCTXCTL, regval);
6420
c5b9bd5e 6421 /* enable/disable RX */
33af6bcc 6422 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 6423 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6424 regval |= tsync_rx_ctl;
33af6bcc 6425 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 6426
c5b9bd5e
AD		 6427 /* define which PTP packets are time stamped */
6428 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 6429
c5b9bd5e
AD		 6430 /* define ethertype filter for timestamped packets */
6431 if (is_l2)
6432 wr32(E1000_ETQF(3),
6433 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6434 E1000_ETQF_1588 | /* enable timestamping */
6435 ETH_P_1588)); /* 1588 eth protocol type */
6436 else
6437 wr32(E1000_ETQF(3), 0);
6438
6439#define PTP_PORT 319
6440 /* L4 Queue Filter[3]: filter by destination port and protocol */
6441 if (is_l4) {
6442 u32 ftqf = (IPPROTO_UDP /* UDP */
6443 | E1000_FTQF_VF_BP /* VF not compared */
6444 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6445 | E1000_FTQF_MASK); /* mask all inputs */
6446 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6447
6448 wr32(E1000_IMIR(3), htons(PTP_PORT));
6449 wr32(E1000_IMIREXT(3),
6450 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6451 if (hw->mac.type == e1000_82576) {
6452 /* enable source port check */
6453 wr32(E1000_SPQF(3), htons(PTP_PORT));
6454 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6455 }
6456 wr32(E1000_FTQF(3), ftqf);
6457 } else {
6458 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6459 }
33af6bcc
PO		 6460 wrfl();
6461
6462 adapter->hwtstamp_config = config;
6463
6464 /* clear TX/RX time stamp registers, just to be sure */
6465 regval = rd32(E1000_TXSTMPH);
6466 regval = rd32(E1000_RXSTMPH);
c6cb090b 6467
33af6bcc
PO		 6468 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6469 -EFAULT : 0;
c6cb090b
PO		 6470}
6471
9d5c8243
AK		 6472/**
6473 * igb_ioctl -
6474 * @netdev:
6475 * @ifreq:
6476 * @cmd:
6477 **/
6478static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6479{
6480 switch (cmd) {
6481 case SIOCGMIIPHY:
6482 case SIOCGMIIREG:
6483 case SIOCSMIIREG:
6484 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 6485 case SIOCSHWTSTAMP:
6486 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 6487 default:
6488 return -EOPNOTSUPP;
6489 }
6490}
6491
009bc06e
AD		 6492s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6493{
6494 struct igb_adapter *adapter = hw->back;
6495 u16 cap_offset;
6496
bdaae04c 6497 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6498 if (!cap_offset)
6499 return -E1000_ERR_CONFIG;
6500
6501 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6502
6503 return 0;
6504}
6505
6506s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6507{
6508 struct igb_adapter *adapter = hw->back;
6509 u16 cap_offset;
6510
bdaae04c 6511 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6512 if (!cap_offset)
6513 return -E1000_ERR_CONFIG;
6514
6515 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6516
6517 return 0;
6518}
6519
c8f44aff 6520static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
9d5c8243
AK		 6521{
6522 struct igb_adapter *adapter = netdev_priv(netdev);
6523 struct e1000_hw *hw = &adapter->hw;
6524 u32 ctrl, rctl;
5faf030c 6525 bool enable = !!(features & NETIF_F_HW_VLAN_RX);
9d5c8243 6526
5faf030c 6527 if (enable) {
9d5c8243
AK		 6528 /* enable VLAN tag insert/strip */
6529 ctrl = rd32(E1000_CTRL);
6530 ctrl |= E1000_CTRL_VME;
6531 wr32(E1000_CTRL, ctrl);
6532
51466239 6533 /* Disable CFI check */
9d5c8243 6534 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6535 rctl &= ~E1000_RCTL_CFIEN;
6536 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6537 } else {
6538 /* disable VLAN tag insert/strip */
6539 ctrl = rd32(E1000_CTRL);
6540 ctrl &= ~E1000_CTRL_VME;
6541 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6542 }
6543
e1739522 6544 igb_rlpml_set(adapter);
9d5c8243
AK		 6545}
6546
8e586137 6547static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
9d5c8243
AK		 6548{
6549 struct igb_adapter *adapter = netdev_priv(netdev);
6550 struct e1000_hw *hw = &adapter->hw;
4ae196df 6551 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6552
51466239
AD		 6553 /* attempt to add filter to vlvf array */
6554 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6555
51466239
AD		 6556 /* add the filter since PF can receive vlans w/o entry in vlvf */
6557 igb_vfta_set(hw, vid, true);
b2cb09b1
JP		 6558
6559 set_bit(vid, adapter->active_vlans);
8e586137
JP		 6560
6561 return 0;
9d5c8243
AK		 6562}
6563
8e586137 6564static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
9d5c8243
AK		 6565{
6566 struct igb_adapter *adapter = netdev_priv(netdev);
6567 struct e1000_hw *hw = &adapter->hw;
4ae196df 6568 int pf_id = adapter->vfs_allocated_count;
51466239 6569 s32 err;
9d5c8243 6570
51466239
AD		 6571 /* remove vlan from VLVF table array */
6572 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6573
51466239
AD		 6574 /* if vid was not present in VLVF just remove it from table */
6575 if (err)
4ae196df 6576 igb_vfta_set(hw, vid, false);
b2cb09b1
JP		 6577
6578 clear_bit(vid, adapter->active_vlans);
8e586137
JP		 6579
6580 return 0;
9d5c8243
AK		 6581}
6582
6583static void igb_restore_vlan(struct igb_adapter *adapter)
6584{
b2cb09b1 6585 u16 vid;
9d5c8243 6586
5faf030c
AD		 6587 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
6588
b2cb09b1
JP		 6589 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6590 igb_vlan_rx_add_vid(adapter->netdev, vid);
9d5c8243
AK		 6591}
6592
14ad2513 6593int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9d5c8243 6594{
090b1795 6595 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6596 struct e1000_mac_info *mac = &adapter->hw.mac;
6597
6598 mac->autoneg = 0;
6599
14ad2513
DD		 6600 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6601 * for the switch() below to work */
6602 if ((spd & 1) || (dplx & ~1))
6603 goto err_inval;
6604
cd2638a8
CW		 6605 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6606 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
14ad2513
DD		 6607 spd != SPEED_1000 &&
6608 dplx != DUPLEX_FULL)
6609 goto err_inval;
cd2638a8 6610
14ad2513 6611 switch (spd + dplx) {
9d5c8243
AK		 6612 case SPEED_10 + DUPLEX_HALF:
6613 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6614 break;
6615 case SPEED_10 + DUPLEX_FULL:
6616 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6617 break;
6618 case SPEED_100 + DUPLEX_HALF:
6619 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6620 break;
6621 case SPEED_100 + DUPLEX_FULL:
6622 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6623 break;
6624 case SPEED_1000 + DUPLEX_FULL:
6625 mac->autoneg = 1;
6626 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6627 break;
6628 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6629 default:
14ad2513 6630 goto err_inval;
9d5c8243
AK		 6631 }
6632 return 0;
14ad2513
DD		 6633
6634err_inval:
6635 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6636 return -EINVAL;
9d5c8243
AK		 6637}
6638
749ab2cd
YZ		 6639static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
6640 bool runtime)
9d5c8243
AK		 6641{
6642 struct net_device *netdev = pci_get_drvdata(pdev);
6643 struct igb_adapter *adapter = netdev_priv(netdev);
6644 struct e1000_hw *hw = &adapter->hw;
2d064c06 6645 u32 ctrl, rctl, status;
749ab2cd 6646 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
9d5c8243
AK		 6647#ifdef CONFIG_PM
6648 int retval = 0;
6649#endif
6650
6651 netif_device_detach(netdev);
6652
a88f10ec 6653 if (netif_running(netdev))
749ab2cd 6654 __igb_close(netdev, true);
a88f10ec 6655
047e0030 6656 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6657
6658#ifdef CONFIG_PM
6659 retval = pci_save_state(pdev);
6660 if (retval)
6661 return retval;
6662#endif
6663
6664 status = rd32(E1000_STATUS);
6665 if (status & E1000_STATUS_LU)
6666 wufc &= ~E1000_WUFC_LNKC;
6667
6668 if (wufc) {
6669 igb_setup_rctl(adapter);
ff41f8dc 6670 igb_set_rx_mode(netdev);
9d5c8243
AK		 6671
6672 /* turn on all-multi mode if wake on multicast is enabled */
6673 if (wufc & E1000_WUFC_MC) {
6674 rctl = rd32(E1000_RCTL);
6675 rctl |= E1000_RCTL_MPE;
6676 wr32(E1000_RCTL, rctl);
6677 }
6678
6679 ctrl = rd32(E1000_CTRL);
6680 /* advertise wake from D3Cold */
6681 #define E1000_CTRL_ADVD3WUC 0x00100000
6682 /* phy power management enable */
6683 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6684 ctrl |= E1000_CTRL_ADVD3WUC;
6685 wr32(E1000_CTRL, ctrl);
6686
9d5c8243 6687 /* Allow time for pending master requests to run */
330a6d6a 6688 igb_disable_pcie_master(hw);
9d5c8243
AK		 6689
6690 wr32(E1000_WUC, E1000_WUC_PME_EN);
6691 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6692 } else {
6693 wr32(E1000_WUC, 0);
6694 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6695 }
6696
3fe7c4c9
RW		 6697 *enable_wake = wufc || adapter->en_mng_pt;
6698 if (!*enable_wake)
88a268c1
NN		 6699 igb_power_down_link(adapter);
6700 else
6701 igb_power_up_link(adapter);
9d5c8243
AK		 6702
6703 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6704 * would have already happened in close and is redundant. */
6705 igb_release_hw_control(adapter);
6706
6707 pci_disable_device(pdev);
6708
9d5c8243
AK		 6709 return 0;
6710}
6711
6712#ifdef CONFIG_PM
749ab2cd 6713static int igb_suspend(struct device *dev)
3fe7c4c9
RW		 6714{
6715 int retval;
6716 bool wake;
749ab2cd 6717 struct pci_dev *pdev = to_pci_dev(dev);
3fe7c4c9 6718
749ab2cd 6719 retval = __igb_shutdown(pdev, &wake, 0);
3fe7c4c9
RW		 6720 if (retval)
6721 return retval;
6722
6723 if (wake) {
6724 pci_prepare_to_sleep(pdev);
6725 } else {
6726 pci_wake_from_d3(pdev, false);
6727 pci_set_power_state(pdev, PCI_D3hot);
6728 }
6729
6730 return 0;
6731}
6732
749ab2cd 6733static int igb_resume(struct device *dev)
9d5c8243 6734{
749ab2cd 6735 struct pci_dev *pdev = to_pci_dev(dev);
9d5c8243
AK		 6736 struct net_device *netdev = pci_get_drvdata(pdev);
6737 struct igb_adapter *adapter = netdev_priv(netdev);
6738 struct e1000_hw *hw = &adapter->hw;
6739 u32 err;
6740
6741 pci_set_power_state(pdev, PCI_D0);
6742 pci_restore_state(pdev);
b94f2d77 6743 pci_save_state(pdev);
42bfd33a 6744
aed5dec3 6745 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6746 if (err) {
6747 dev_err(&pdev->dev,
6748 "igb: Cannot enable PCI device from suspend\n");
6749 return err;
6750 }
6751 pci_set_master(pdev);
6752
6753 pci_enable_wake(pdev, PCI_D3hot, 0);
6754 pci_enable_wake(pdev, PCI_D3cold, 0);
6755
749ab2cd
YZ		 6756 if (!rtnl_is_locked()) {
6757 /*
6758 * shut up ASSERT_RTNL() warning in
6759 * netif_set_real_num_tx/rx_queues.
6760 */
6761 rtnl_lock();
6762 err = igb_init_interrupt_scheme(adapter);
6763 rtnl_unlock();
6764 } else {
6765 err = igb_init_interrupt_scheme(adapter);
6766 }
6767 if (err) {
a88f10ec
AD		 6768 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6769 return -ENOMEM;
9d5c8243
AK		 6770 }
6771
9d5c8243 6772 igb_reset(adapter);
a8564f03
AD		 6773
6774 /* let the f/w know that the h/w is now under the control of the
6775 * driver. */
6776 igb_get_hw_control(adapter);
6777
9d5c8243
AK		 6778 wr32(E1000_WUS, ~0);
6779
749ab2cd
YZ		 6780 if (netdev->flags & IFF_UP) {
6781 err = __igb_open(netdev, true);
a88f10ec
AD		 6782 if (err)
6783 return err;
6784 }
9d5c8243
AK		 6785
6786 netif_device_attach(netdev);
749ab2cd
YZ		 6787 return 0;
6788}
6789
6790#ifdef CONFIG_PM_RUNTIME
6791static int igb_runtime_idle(struct device *dev)
6792{
6793 struct pci_dev *pdev = to_pci_dev(dev);
6794 struct net_device *netdev = pci_get_drvdata(pdev);
6795 struct igb_adapter *adapter = netdev_priv(netdev);
6796
6797 if (!igb_has_link(adapter))
6798 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6799
6800 return -EBUSY;
6801}
6802
6803static int igb_runtime_suspend(struct device *dev)
6804{
6805 struct pci_dev *pdev = to_pci_dev(dev);
6806 int retval;
6807 bool wake;
6808
6809 retval = __igb_shutdown(pdev, &wake, 1);
6810 if (retval)
6811 return retval;
6812
6813 if (wake) {
6814 pci_prepare_to_sleep(pdev);
6815 } else {
6816 pci_wake_from_d3(pdev, false);
6817 pci_set_power_state(pdev, PCI_D3hot);
6818 }
9d5c8243 6819
9d5c8243
AK		 6820 return 0;
6821}
749ab2cd
YZ		 6822
6823static int igb_runtime_resume(struct device *dev)
6824{
6825 return igb_resume(dev);
6826}
6827#endif /* CONFIG_PM_RUNTIME */
9d5c8243
AK		 6828#endif
6829
6830static void igb_shutdown(struct pci_dev *pdev)
6831{
3fe7c4c9
RW		 6832 bool wake;
6833
749ab2cd 6834 __igb_shutdown(pdev, &wake, 0);
3fe7c4c9
RW		 6835
6836 if (system_state == SYSTEM_POWER_OFF) {
6837 pci_wake_from_d3(pdev, wake);
6838 pci_set_power_state(pdev, PCI_D3hot);
6839 }
9d5c8243
AK		 6840}
6841
6842#ifdef CONFIG_NET_POLL_CONTROLLER
6843/*
6844 * Polling 'interrupt' - used by things like netconsole to send skbs
6845 * without having to re-enable interrupts. It's not called while
6846 * the interrupt routine is executing.
6847 */
6848static void igb_netpoll(struct net_device *netdev)
6849{
6850 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6851 struct e1000_hw *hw = &adapter->hw;
0d1ae7f4 6852 struct igb_q_vector *q_vector;
9d5c8243 6853 int i;
9d5c8243 6854
047e0030 6855 for (i = 0; i < adapter->num_q_vectors; i++) {
0d1ae7f4
AD		 6856 q_vector = adapter->q_vector[i];
6857 if (adapter->msix_entries)
6858 wr32(E1000_EIMC, q_vector->eims_value);
6859 else
6860 igb_irq_disable(adapter);
047e0030 6861 napi_schedule(&q_vector->napi);
eebbbdba 6862 }
9d5c8243
AK		 6863}
6864#endif /* CONFIG_NET_POLL_CONTROLLER */
6865
6866/**
6867 * igb_io_error_detected - called when PCI error is detected
6868 * @pdev: Pointer to PCI device
6869 * @state: The current pci connection state
6870 *
6871 * This function is called after a PCI bus error affecting
6872 * this device has been detected.
6873 */
6874static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6875 pci_channel_state_t state)
6876{
6877 struct net_device *netdev = pci_get_drvdata(pdev);
6878 struct igb_adapter *adapter = netdev_priv(netdev);
6879
6880 netif_device_detach(netdev);
6881
59ed6eec
AD		 6882 if (state == pci_channel_io_perm_failure)
6883 return PCI_ERS_RESULT_DISCONNECT;
6884
9d5c8243
AK		 6885 if (netif_running(netdev))
6886 igb_down(adapter);
6887 pci_disable_device(pdev);
6888
6889 /* Request a slot slot reset. */
6890 return PCI_ERS_RESULT_NEED_RESET;
6891}
6892
6893/**
6894 * igb_io_slot_reset - called after the pci bus has been reset.
6895 * @pdev: Pointer to PCI device
6896 *
6897 * Restart the card from scratch, as if from a cold-boot. Implementation
6898 * resembles the first-half of the igb_resume routine.
6899 */
6900static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6901{
6902 struct net_device *netdev = pci_get_drvdata(pdev);
6903 struct igb_adapter *adapter = netdev_priv(netdev);
6904 struct e1000_hw *hw = &adapter->hw;
40a914fa 6905 pci_ers_result_t result;
42bfd33a 6906 int err;
9d5c8243 6907
aed5dec3 6908 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6909 dev_err(&pdev->dev,
6910 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6911 result = PCI_ERS_RESULT_DISCONNECT;
6912 } else {
6913 pci_set_master(pdev);
6914 pci_restore_state(pdev);
b94f2d77 6915 pci_save_state(pdev);
9d5c8243 6916
40a914fa
AD		 6917 pci_enable_wake(pdev, PCI_D3hot, 0);
6918 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6919
40a914fa
AD		 6920 igb_reset(adapter);
6921 wr32(E1000_WUS, ~0);
6922 result = PCI_ERS_RESULT_RECOVERED;
6923 }
9d5c8243 6924
ea943d41
JK		 6925 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6926 if (err) {
6927 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6928 "failed 0x%0x\n", err);
6929 /* non-fatal, continue */
6930 }
40a914fa
AD		 6931
6932 return result;
9d5c8243
AK		 6933}
6934
6935/**
6936 * igb_io_resume - called when traffic can start flowing again.
6937 * @pdev: Pointer to PCI device
6938 *
6939 * This callback is called when the error recovery driver tells us that
6940 * its OK to resume normal operation. Implementation resembles the
6941 * second-half of the igb_resume routine.
6942 */
6943static void igb_io_resume(struct pci_dev *pdev)
6944{
6945 struct net_device *netdev = pci_get_drvdata(pdev);
6946 struct igb_adapter *adapter = netdev_priv(netdev);
6947
9d5c8243
AK		 6948 if (netif_running(netdev)) {
6949 if (igb_up(adapter)) {
6950 dev_err(&pdev->dev, "igb_up failed after reset\n");
6951 return;
6952 }
6953 }
6954
6955 netif_device_attach(netdev);
6956
6957 /* let the f/w know that the h/w is now under the control of the
6958 * driver. */
6959 igb_get_hw_control(adapter);
9d5c8243
AK		 6960}
6961
26ad9178
AD		 6962static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6963 u8 qsel)
6964{
6965 u32 rar_low, rar_high;
6966 struct e1000_hw *hw = &adapter->hw;
6967
6968 /* HW expects these in little endian so we reverse the byte order
6969 * from network order (big endian) to little endian
6970 */
6971 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6972 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6973 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6974
6975 /* Indicate to hardware the Address is Valid. */
6976 rar_high |= E1000_RAH_AV;
6977
6978 if (hw->mac.type == e1000_82575)
6979 rar_high |= E1000_RAH_POOL_1 * qsel;
6980 else
6981 rar_high |= E1000_RAH_POOL_1 << qsel;
6982
6983 wr32(E1000_RAL(index), rar_low);
6984 wrfl();
6985 wr32(E1000_RAH(index), rar_high);
6986 wrfl();
6987}
6988
4ae196df
AD		 6989static int igb_set_vf_mac(struct igb_adapter *adapter,
6990 int vf, unsigned char *mac_addr)
6991{
6992 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6993 /* VF MAC addresses start at end of receive addresses and moves
6994 * torwards the first, as a result a collision should not be possible */
6995 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6996
37680117 6997 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6998
26ad9178 6999 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 7000
7001 return 0;
7002}
7003
8151d294
WM		 7004static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
7005{
7006 struct igb_adapter *adapter = netdev_priv(netdev);
7007 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
7008 return -EINVAL;
7009 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
7010 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
7011 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
7012 " change effective.");
7013 if (test_bit(__IGB_DOWN, &adapter->state)) {
7014 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
7015 " but the PF device is not up.\n");
7016 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
7017 " attempting to use the VF device.\n");
7018 }
7019 return igb_set_vf_mac(adapter, vf, mac);
7020}
7021
17dc566c
LL		 7022static int igb_link_mbps(int internal_link_speed)
7023{
7024 switch (internal_link_speed) {
7025 case SPEED_100:
7026 return 100;
7027 case SPEED_1000:
7028 return 1000;
7029 default:
7030 return 0;
7031 }
7032}
7033
7034static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
7035 int link_speed)
7036{
7037 int rf_dec, rf_int;
7038 u32 bcnrc_val;
7039
7040 if (tx_rate != 0) {
7041 /* Calculate the rate factor values to set */
7042 rf_int = link_speed / tx_rate;
7043 rf_dec = (link_speed - (rf_int * tx_rate));
7044 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
7045
7046 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
7047 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
7048 E1000_RTTBCNRC_RF_INT_MASK);
7049 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
7050 } else {
7051 bcnrc_val = 0;
7052 }
7053
7054 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
7055 wr32(E1000_RTTBCNRC, bcnrc_val);
7056}
7057
7058static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
7059{
7060 int actual_link_speed, i;
7061 bool reset_rate = false;
7062
7063 /* VF TX rate limit was not set or not supported */
7064 if ((adapter->vf_rate_link_speed == 0) ||
7065 (adapter->hw.mac.type != e1000_82576))
7066 return;
7067
7068 actual_link_speed = igb_link_mbps(adapter->link_speed);
7069 if (actual_link_speed != adapter->vf_rate_link_speed) {
7070 reset_rate = true;
7071 adapter->vf_rate_link_speed = 0;
7072 dev_info(&adapter->pdev->dev,
7073 "Link speed has been changed. VF Transmit "
7074 "rate is disabled\n");
7075 }
7076
7077 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7078 if (reset_rate)
7079 adapter->vf_data[i].tx_rate = 0;
7080
7081 igb_set_vf_rate_limit(&adapter->hw, i,
7082 adapter->vf_data[i].tx_rate,
7083 actual_link_speed);
7084 }
7085}
7086
8151d294
WM		 7087static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
7088{
17dc566c
LL		 7089 struct igb_adapter *adapter = netdev_priv(netdev);
7090 struct e1000_hw *hw = &adapter->hw;
7091 int actual_link_speed;
7092
7093 if (hw->mac.type != e1000_82576)
7094 return -EOPNOTSUPP;
7095
7096 actual_link_speed = igb_link_mbps(adapter->link_speed);
7097 if ((vf >= adapter->vfs_allocated_count) ||
7098 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
7099 (tx_rate < 0) || (tx_rate > actual_link_speed))
7100 return -EINVAL;
7101
7102 adapter->vf_rate_link_speed = actual_link_speed;
7103 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
7104 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
7105
7106 return 0;
8151d294
WM		 7107}
7108
7109static int igb_ndo_get_vf_config(struct net_device *netdev,
7110 int vf, struct ifla_vf_info *ivi)
7111{
7112 struct igb_adapter *adapter = netdev_priv(netdev);
7113 if (vf >= adapter->vfs_allocated_count)
7114 return -EINVAL;
7115 ivi->vf = vf;
7116 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
17dc566c 7117 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8151d294
WM		 7118 ivi->vlan = adapter->vf_data[vf].pf_vlan;
7119 ivi->qos = adapter->vf_data[vf].pf_qos;
7120 return 0;
7121}
7122
4ae196df
AD		 7123static void igb_vmm_control(struct igb_adapter *adapter)
7124{
7125 struct e1000_hw *hw = &adapter->hw;
10d8e907 7126 u32 reg;
4ae196df 7127
52a1dd4d
AD		 7128 switch (hw->mac.type) {
7129 case e1000_82575:
7130 default:
7131 /* replication is not supported for 82575 */
4ae196df 7132 return;
52a1dd4d
AD		 7133 case e1000_82576:
7134 /* notify HW that the MAC is adding vlan tags */
7135 reg = rd32(E1000_DTXCTL);
7136 reg |= E1000_DTXCTL_VLAN_ADDED;
7137 wr32(E1000_DTXCTL, reg);
7138 case e1000_82580:
7139 /* enable replication vlan tag stripping */
7140 reg = rd32(E1000_RPLOLR);
7141 reg |= E1000_RPLOLR_STRVLAN;
7142 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 7143 case e1000_i350:
7144 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 7145 break;
7146 }
10d8e907 7147
d4960307
AD		 7148 if (adapter->vfs_allocated_count) {
7149 igb_vmdq_set_loopback_pf(hw, true);
7150 igb_vmdq_set_replication_pf(hw, true);
13800469
GR		 7151 igb_vmdq_set_anti_spoofing_pf(hw, true,
7152 adapter->vfs_allocated_count);
d4960307
AD		 7153 } else {
7154 igb_vmdq_set_loopback_pf(hw, false);
7155 igb_vmdq_set_replication_pf(hw, false);
7156 }
4ae196df
AD		 7157}
7158
b6e0c419
CW		 7159static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
7160{
7161 struct e1000_hw *hw = &adapter->hw;
7162 u32 dmac_thr;
7163 u16 hwm;
7164
7165 if (hw->mac.type > e1000_82580) {
7166 if (adapter->flags & IGB_FLAG_DMAC) {
7167 u32 reg;
7168
7169 /* force threshold to 0. */
7170 wr32(E1000_DMCTXTH, 0);
7171
7172 /*
e8c626e9
MV		 7173 * DMA Coalescing high water mark needs to be greater
7174 * than the Rx threshold. Set hwm to PBA - max frame
7175 * size in 16B units, capping it at PBA - 6KB.
b6e0c419 7176 */
e8c626e9
MV		 7177 hwm = 64 * pba - adapter->max_frame_size / 16;
7178 if (hwm < 64 * (pba - 6))
7179 hwm = 64 * (pba - 6);
7180 reg = rd32(E1000_FCRTC);
7181 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
7182 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
7183 & E1000_FCRTC_RTH_COAL_MASK);
7184 wr32(E1000_FCRTC, reg);
7185
7186 /*
7187 * Set the DMA Coalescing Rx threshold to PBA - 2 * max
7188 * frame size, capping it at PBA - 10KB.
7189 */
7190 dmac_thr = pba - adapter->max_frame_size / 512;
7191 if (dmac_thr < pba - 10)
7192 dmac_thr = pba - 10;
b6e0c419
CW		 7193 reg = rd32(E1000_DMACR);
7194 reg &= ~E1000_DMACR_DMACTHR_MASK;
b6e0c419
CW		 7195 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
7196 & E1000_DMACR_DMACTHR_MASK);
7197
7198 /* transition to L0x or L1 if available..*/
7199 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
7200
7201 /* watchdog timer= +-1000 usec in 32usec intervals */
7202 reg |= (1000 >> 5);
7203 wr32(E1000_DMACR, reg);
7204
7205 /*
7206 * no lower threshold to disable
7207 * coalescing(smart fifb)-UTRESH=0
7208 */
7209 wr32(E1000_DMCRTRH, 0);
b6e0c419
CW		 7210
7211 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
7212
7213 wr32(E1000_DMCTLX, reg);
7214
7215 /*
7216 * free space in tx packet buffer to wake from
7217 * DMA coal
7218 */
7219 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
7220 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
7221
7222 /*
7223 * make low power state decision controlled
7224 * by DMA coal
7225 */
7226 reg = rd32(E1000_PCIEMISC);
7227 reg &= ~E1000_PCIEMISC_LX_DECISION;
7228 wr32(E1000_PCIEMISC, reg);
7229 } /* endif adapter->dmac is not disabled */
7230 } else if (hw->mac.type == e1000_82580) {
7231 u32 reg = rd32(E1000_PCIEMISC);
7232 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
7233 wr32(E1000_DMACR, 0);
7234 }
7235}
7236
9d5c8243 7237/* igb_main.c */
Linux kernel - Deliverables
This page took 1.068237 seconds and 5 git commands to generate.