projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: Tidy up wrapping for CONFIG_IGB_PTP.
[deliverable/linux.git] / drivers / net / ethernet / intel / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
6e861326 4 Copyright(c) 2007-2012 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
876d2d6f
JK		 28#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
29
9d5c8243
AK		 30#include <linux/module.h>
31#include <linux/types.h>
32#include <linux/init.h>
b2cb09b1 33#include <linux/bitops.h>
9d5c8243
AK		 34#include <linux/vmalloc.h>
35#include <linux/pagemap.h>
36#include <linux/netdevice.h>
9d5c8243 37#include <linux/ipv6.h>
5a0e3ad6 38#include <linux/slab.h>
9d5c8243
AK		 39#include <net/checksum.h>
40#include <net/ip6_checksum.h>
c6cb090b 41#include <linux/net_tstamp.h>
9d5c8243
AK		 42#include <linux/mii.h>
43#include <linux/ethtool.h>
01789349 44#include <linux/if.h>
9d5c8243
AK		 45#include <linux/if_vlan.h>
46#include <linux/pci.h>
c54106bb 47#include <linux/pci-aspm.h>
9d5c8243
AK		 48#include <linux/delay.h>
49#include <linux/interrupt.h>
7d13a7d0
AD		 50#include <linux/ip.h>
51#include <linux/tcp.h>
52#include <linux/sctp.h>
9d5c8243 53#include <linux/if_ether.h>
40a914fa 54#include <linux/aer.h>
70c71606 55#include <linux/prefetch.h>
749ab2cd 56#include <linux/pm_runtime.h>
421e02f0 57#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 58#include <linux/dca.h>
59#endif
9d5c8243
AK		 60#include "igb.h"
61
200e5fd5
CW		 62#define MAJ 4
63#define MIN 0
64#define BUILD 1
0d1fe82d 65#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
929dd047 66__stringify(BUILD) "-k"
9d5c8243
AK		 67char igb_driver_name[] = "igb";
68char igb_driver_version[] = DRV_VERSION;
69static const char igb_driver_string[] =
70 "Intel(R) Gigabit Ethernet Network Driver";
6e861326 71static const char igb_copyright[] = "Copyright (c) 2007-2012 Intel Corporation.";
9d5c8243 72
9d5c8243
AK		 73static const struct e1000_info *igb_info_tbl[] = {
74 [board_82575] = &e1000_82575_info,
75};
76
a3aa1884 77static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
f96a8a0b
CW		 78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
d2ba2ed8
AD		 83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
6493d24f 89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
55cac248
AD		 90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
1b5dda33
GJ		 95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
2d064c06 97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 100 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
101 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 102 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 103 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 104 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 105 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
106 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
107 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
108 /* required last entry */
109 {0, }
110};
111
112MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
113
114void igb_reset(struct igb_adapter *);
115static int igb_setup_all_tx_resources(struct igb_adapter *);
116static int igb_setup_all_rx_resources(struct igb_adapter *);
117static void igb_free_all_tx_resources(struct igb_adapter *);
118static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 119static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 120static int igb_probe(struct pci_dev *, const struct pci_device_id *);
121static void __devexit igb_remove(struct pci_dev *pdev);
122static int igb_sw_init(struct igb_adapter *);
123static int igb_open(struct net_device *);
124static int igb_close(struct net_device *);
125static void igb_configure_tx(struct igb_adapter *);
126static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 127static void igb_clean_all_tx_rings(struct igb_adapter *);
128static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 129static void igb_clean_tx_ring(struct igb_ring *);
130static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 131static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 132static void igb_update_phy_info(unsigned long);
133static void igb_watchdog(unsigned long);
134static void igb_watchdog_task(struct work_struct *);
cd392f5c 135static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 136static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
137 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 138static int igb_change_mtu(struct net_device *, int);
139static int igb_set_mac(struct net_device *, void *);
68d480c4 140static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 141static irqreturn_t igb_intr(int irq, void *);
142static irqreturn_t igb_intr_msi(int irq, void *);
143static irqreturn_t igb_msix_other(int irq, void *);
047e0030 144static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 145#ifdef CONFIG_IGB_DCA
047e0030 146static void igb_update_dca(struct igb_q_vector *);
fe4506b6 147static void igb_setup_dca(struct igb_adapter *);
421e02f0 148#endif /* CONFIG_IGB_DCA */
661086df 149static int igb_poll(struct napi_struct *, int);
13fde97a 150static bool igb_clean_tx_irq(struct igb_q_vector *);
cd392f5c 151static bool igb_clean_rx_irq(struct igb_q_vector *, int);
9d5c8243
AK		 152static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
153static void igb_tx_timeout(struct net_device *);
154static void igb_reset_task(struct work_struct *);
c8f44aff 155static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features);
8e586137
JP		 156static int igb_vlan_rx_add_vid(struct net_device *, u16);
157static int igb_vlan_rx_kill_vid(struct net_device *, u16);
9d5c8243 158static void igb_restore_vlan(struct igb_adapter *);
26ad9178 159static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 160static void igb_ping_all_vfs(struct igb_adapter *);
161static void igb_msg_task(struct igb_adapter *);
4ae196df 162static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 163static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 164static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 165static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
166static int igb_ndo_set_vf_vlan(struct net_device *netdev,
167 int vf, u16 vlan, u8 qos);
168static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
169static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
170 struct ifla_vf_info *ivi);
17dc566c 171static void igb_check_vf_rate_limit(struct igb_adapter *);
46a01698
RL		 172
173#ifdef CONFIG_PCI_IOV
0224d663
GR		 174static int igb_vf_configure(struct igb_adapter *adapter, int vf);
175static int igb_find_enabled_vfs(struct igb_adapter *adapter);
176static int igb_check_vf_assignment(struct igb_adapter *adapter);
46a01698 177#endif
9d5c8243 178
9d5c8243 179#ifdef CONFIG_PM
d9dd966d 180#ifdef CONFIG_PM_SLEEP
749ab2cd 181static int igb_suspend(struct device *);
d9dd966d 182#endif
749ab2cd
YZ		 183static int igb_resume(struct device *);
184#ifdef CONFIG_PM_RUNTIME
185static int igb_runtime_suspend(struct device *dev);
186static int igb_runtime_resume(struct device *dev);
187static int igb_runtime_idle(struct device *dev);
188#endif
189static const struct dev_pm_ops igb_pm_ops = {
190 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
191 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
192 igb_runtime_idle)
193};
9d5c8243
AK		 194#endif
195static void igb_shutdown(struct pci_dev *);
421e02f0 196#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 197static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
198static struct notifier_block dca_notifier = {
199 .notifier_call = igb_notify_dca,
200 .next = NULL,
201 .priority = 0
202};
203#endif
9d5c8243
AK		 204#ifdef CONFIG_NET_POLL_CONTROLLER
205/* for netdump / net console */
206static void igb_netpoll(struct net_device *);
207#endif
37680117 208#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 209static unsigned int max_vfs = 0;
210module_param(max_vfs, uint, 0);
211MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
212 "per physical function");
213#endif /* CONFIG_PCI_IOV */
214
9d5c8243
AK		 215static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
216 pci_channel_state_t);
217static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
218static void igb_io_resume(struct pci_dev *);
219
220static struct pci_error_handlers igb_err_handler = {
221 .error_detected = igb_io_error_detected,
222 .slot_reset = igb_io_slot_reset,
223 .resume = igb_io_resume,
224};
225
b6e0c419 226static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
9d5c8243
AK		 227
228static struct pci_driver igb_driver = {
229 .name = igb_driver_name,
230 .id_table = igb_pci_tbl,
231 .probe = igb_probe,
232 .remove = __devexit_p(igb_remove),
233#ifdef CONFIG_PM
749ab2cd 234 .driver.pm = &igb_pm_ops,
9d5c8243
AK		 235#endif
236 .shutdown = igb_shutdown,
237 .err_handler = &igb_err_handler
238};
239
240MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
241MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
242MODULE_LICENSE("GPL");
243MODULE_VERSION(DRV_VERSION);
244
b3f4d599 245#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
246static int debug = -1;
247module_param(debug, int, 0);
248MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
249
c97ec42a
TI		 250struct igb_reg_info {
251 u32 ofs;
252 char *name;
253};
254
255static const struct igb_reg_info igb_reg_info_tbl[] = {
256
257 /* General Registers */
258 {E1000_CTRL, "CTRL"},
259 {E1000_STATUS, "STATUS"},
260 {E1000_CTRL_EXT, "CTRL_EXT"},
261
262 /* Interrupt Registers */
263 {E1000_ICR, "ICR"},
264
265 /* RX Registers */
266 {E1000_RCTL, "RCTL"},
267 {E1000_RDLEN(0), "RDLEN"},
268 {E1000_RDH(0), "RDH"},
269 {E1000_RDT(0), "RDT"},
270 {E1000_RXDCTL(0), "RXDCTL"},
271 {E1000_RDBAL(0), "RDBAL"},
272 {E1000_RDBAH(0), "RDBAH"},
273
274 /* TX Registers */
275 {E1000_TCTL, "TCTL"},
276 {E1000_TDBAL(0), "TDBAL"},
277 {E1000_TDBAH(0), "TDBAH"},
278 {E1000_TDLEN(0), "TDLEN"},
279 {E1000_TDH(0), "TDH"},
280 {E1000_TDT(0), "TDT"},
281 {E1000_TXDCTL(0), "TXDCTL"},
282 {E1000_TDFH, "TDFH"},
283 {E1000_TDFT, "TDFT"},
284 {E1000_TDFHS, "TDFHS"},
285 {E1000_TDFPC, "TDFPC"},
286
287 /* List Terminator */
288 {}
289};
290
291/*
292 * igb_regdump - register printout routine
293 */
294static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
295{
296 int n = 0;
297 char rname[16];
298 u32 regs[8];
299
300 switch (reginfo->ofs) {
301 case E1000_RDLEN(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_RDLEN(n));
304 break;
305 case E1000_RDH(0):
306 for (n = 0; n < 4; n++)
307 regs[n] = rd32(E1000_RDH(n));
308 break;
309 case E1000_RDT(0):
310 for (n = 0; n < 4; n++)
311 regs[n] = rd32(E1000_RDT(n));
312 break;
313 case E1000_RXDCTL(0):
314 for (n = 0; n < 4; n++)
315 regs[n] = rd32(E1000_RXDCTL(n));
316 break;
317 case E1000_RDBAL(0):
318 for (n = 0; n < 4; n++)
319 regs[n] = rd32(E1000_RDBAL(n));
320 break;
321 case E1000_RDBAH(0):
322 for (n = 0; n < 4; n++)
323 regs[n] = rd32(E1000_RDBAH(n));
324 break;
325 case E1000_TDBAL(0):
326 for (n = 0; n < 4; n++)
327 regs[n] = rd32(E1000_RDBAL(n));
328 break;
329 case E1000_TDBAH(0):
330 for (n = 0; n < 4; n++)
331 regs[n] = rd32(E1000_TDBAH(n));
332 break;
333 case E1000_TDLEN(0):
334 for (n = 0; n < 4; n++)
335 regs[n] = rd32(E1000_TDLEN(n));
336 break;
337 case E1000_TDH(0):
338 for (n = 0; n < 4; n++)
339 regs[n] = rd32(E1000_TDH(n));
340 break;
341 case E1000_TDT(0):
342 for (n = 0; n < 4; n++)
343 regs[n] = rd32(E1000_TDT(n));
344 break;
345 case E1000_TXDCTL(0):
346 for (n = 0; n < 4; n++)
347 regs[n] = rd32(E1000_TXDCTL(n));
348 break;
349 default:
876d2d6f 350 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
c97ec42a
TI		 351 return;
352 }
353
354 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
876d2d6f
JK		 355 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
356 regs[2], regs[3]);
c97ec42a
TI		 357}
358
359/*
360 * igb_dump - Print registers, tx-rings and rx-rings
361 */
362static void igb_dump(struct igb_adapter *adapter)
363{
364 struct net_device *netdev = adapter->netdev;
365 struct e1000_hw *hw = &adapter->hw;
366 struct igb_reg_info *reginfo;
c97ec42a
TI		 367 struct igb_ring *tx_ring;
368 union e1000_adv_tx_desc *tx_desc;
369 struct my_u0 { u64 a; u64 b; } *u0;
c97ec42a
TI		 370 struct igb_ring *rx_ring;
371 union e1000_adv_rx_desc *rx_desc;
372 u32 staterr;
6ad4edfc 373 u16 i, n;
c97ec42a
TI		 374
375 if (!netif_msg_hw(adapter))
376 return;
377
378 /* Print netdevice Info */
379 if (netdev) {
380 dev_info(&adapter->pdev->dev, "Net device Info\n");
876d2d6f
JK		 381 pr_info("Device Name state trans_start "
382 "last_rx\n");
383 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
384 netdev->state, netdev->trans_start, netdev->last_rx);
c97ec42a
TI		 385 }
386
387 /* Print Registers */
388 dev_info(&adapter->pdev->dev, "Register Dump\n");
876d2d6f 389 pr_info(" Register Name Value\n");
c97ec42a
TI		 390 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
391 reginfo->name; reginfo++) {
392 igb_regdump(hw, reginfo);
393 }
394
395 /* Print TX Ring Summary */
396 if (!netdev || !netif_running(netdev))
397 goto exit;
398
399 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
876d2d6f 400 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
c97ec42a 401 for (n = 0; n < adapter->num_tx_queues; n++) {
06034649 402 struct igb_tx_buffer *buffer_info;
c97ec42a 403 tx_ring = adapter->tx_ring[n];
06034649 404 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
876d2d6f
JK		 405 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
406 n, tx_ring->next_to_use, tx_ring->next_to_clean,
407 (u64)buffer_info->dma,
408 buffer_info->length,
409 buffer_info->next_to_watch,
410 (u64)buffer_info->time_stamp);
c97ec42a
TI		 411 }
412
413 /* Print TX Rings */
414 if (!netif_msg_tx_done(adapter))
415 goto rx_ring_summary;
416
417 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
418
419 /* Transmit Descriptor Formats
420 *
421 * Advanced Transmit Descriptor
422 * +--------------------------------------------------------------+
423 * 0 | Buffer Address [63:0] |
424 * +--------------------------------------------------------------+
425 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
426 * +--------------------------------------------------------------+
427 * 63 46 45 40 39 38 36 35 32 31 24 15 0
428 */
429
430 for (n = 0; n < adapter->num_tx_queues; n++) {
431 tx_ring = adapter->tx_ring[n];
876d2d6f
JK		 432 pr_info("------------------------------------\n");
433 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
434 pr_info("------------------------------------\n");
435 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] "
436 "[bi->dma ] leng ntw timestamp "
437 "bi->skb\n");
c97ec42a
TI		 438
439 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
876d2d6f 440 const char *next_desc;
06034649 441 struct igb_tx_buffer *buffer_info;
60136906 442 tx_desc = IGB_TX_DESC(tx_ring, i);
06034649 443 buffer_info = &tx_ring->tx_buffer_info[i];
c97ec42a 444 u0 = (struct my_u0 *)tx_desc;
876d2d6f
JK		 445 if (i == tx_ring->next_to_use &&
446 i == tx_ring->next_to_clean)
447 next_desc = " NTC/U";
448 else if (i == tx_ring->next_to_use)
449 next_desc = " NTU";
450 else if (i == tx_ring->next_to_clean)
451 next_desc = " NTC";
452 else
453 next_desc = "";
454
455 pr_info("T [0x%03X] %016llX %016llX %016llX"
456 " %04X %p %016llX %p%s\n", i,
c97ec42a
TI		 457 le64_to_cpu(u0->a),
458 le64_to_cpu(u0->b),
459 (u64)buffer_info->dma,
460 buffer_info->length,
461 buffer_info->next_to_watch,
462 (u64)buffer_info->time_stamp,
876d2d6f 463 buffer_info->skb, next_desc);
c97ec42a 464
b669588a 465 if (netif_msg_pktdata(adapter) && buffer_info->skb)
c97ec42a
TI		 466 print_hex_dump(KERN_INFO, "",
467 DUMP_PREFIX_ADDRESS,
b669588a 468 16, 1, buffer_info->skb->data,
c97ec42a
TI		 469 buffer_info->length, true);
470 }
471 }
472
473 /* Print RX Rings Summary */
474rx_ring_summary:
475 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
876d2d6f 476 pr_info("Queue [NTU] [NTC]\n");
c97ec42a
TI		 477 for (n = 0; n < adapter->num_rx_queues; n++) {
478 rx_ring = adapter->rx_ring[n];
876d2d6f
JK		 479 pr_info(" %5d %5X %5X\n",
480 n, rx_ring->next_to_use, rx_ring->next_to_clean);
c97ec42a
TI		 481 }
482
483 /* Print RX Rings */
484 if (!netif_msg_rx_status(adapter))
485 goto exit;
486
487 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
488
489 /* Advanced Receive Descriptor (Read) Format
490 * 63 1 0
491 * +-----------------------------------------------------+
492 * 0 | Packet Buffer Address [63:1] |A0/NSE|
493 * +----------------------------------------------+------+
494 * 8 | Header Buffer Address [63:1] | DD |
495 * +-----------------------------------------------------+
496 *
497 *
498 * Advanced Receive Descriptor (Write-Back) Format
499 *
500 * 63 48 47 32 31 30 21 20 17 16 4 3 0
501 * +------------------------------------------------------+
502 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
503 * | Checksum Ident | | | | Type | Type |
504 * +------------------------------------------------------+
505 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
506 * +------------------------------------------------------+
507 * 63 48 47 32 31 20 19 0
508 */
509
510 for (n = 0; n < adapter->num_rx_queues; n++) {
511 rx_ring = adapter->rx_ring[n];
876d2d6f
JK		 512 pr_info("------------------------------------\n");
513 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
514 pr_info("------------------------------------\n");
515 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] "
516 "[bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
517 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] -----"
518 "----------- [bi->skb] <-- Adv Rx Write-Back format\n");
c97ec42a
TI		 519
520 for (i = 0; i < rx_ring->count; i++) {
876d2d6f 521 const char *next_desc;
06034649
AD		 522 struct igb_rx_buffer *buffer_info;
523 buffer_info = &rx_ring->rx_buffer_info[i];
60136906 524 rx_desc = IGB_RX_DESC(rx_ring, i);
c97ec42a
TI		 525 u0 = (struct my_u0 *)rx_desc;
526 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
876d2d6f
JK		 527
528 if (i == rx_ring->next_to_use)
529 next_desc = " NTU";
530 else if (i == rx_ring->next_to_clean)
531 next_desc = " NTC";
532 else
533 next_desc = "";
534
c97ec42a
TI		 535 if (staterr & E1000_RXD_STAT_DD) {
536 /* Descriptor Done */
876d2d6f
JK		 537 pr_info("%s[0x%03X] %016llX %016llX -------"
538 "--------- %p%s\n", "RWB", i,
c97ec42a
TI		 539 le64_to_cpu(u0->a),
540 le64_to_cpu(u0->b),
876d2d6f 541 buffer_info->skb, next_desc);
c97ec42a 542 } else {
876d2d6f
JK		 543 pr_info("%s[0x%03X] %016llX %016llX %016llX"
544 " %p%s\n", "R ", i,
c97ec42a
TI		 545 le64_to_cpu(u0->a),
546 le64_to_cpu(u0->b),
547 (u64)buffer_info->dma,
876d2d6f 548 buffer_info->skb, next_desc);
c97ec42a 549
b669588a
ET		 550 if (netif_msg_pktdata(adapter) &&
551 buffer_info->dma && buffer_info->skb) {
c97ec42a 552 print_hex_dump(KERN_INFO, "",
b669588a
ET		 553 DUMP_PREFIX_ADDRESS,
554 16, 1, buffer_info->skb->data,
555 IGB_RX_HDR_LEN, true);
44390ca6
AD		 556 print_hex_dump(KERN_INFO, "",
557 DUMP_PREFIX_ADDRESS,
558 16, 1,
b669588a
ET		 559 page_address(buffer_info->page) +
560 buffer_info->page_offset,
44390ca6 561 PAGE_SIZE/2, true);
c97ec42a
TI		 562 }
563 }
c97ec42a
TI		 564 }
565 }
566
567exit:
568 return;
569}
570
9d5c8243 571/**
c041076a 572 * igb_get_hw_dev - return device
9d5c8243
AK		 573 * used by hardware layer to print debugging information
574 **/
c041076a 575struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 576{
577 struct igb_adapter *adapter = hw->back;
c041076a 578 return adapter->netdev;
9d5c8243 579}
38c845c7 580
9d5c8243
AK		 581/**
582 * igb_init_module - Driver Registration Routine
583 *
584 * igb_init_module is the first routine called when the driver is
585 * loaded. All it does is register with the PCI subsystem.
586 **/
587static int __init igb_init_module(void)
588{
589 int ret;
876d2d6f 590 pr_info("%s - version %s\n",
9d5c8243
AK		 591 igb_driver_string, igb_driver_version);
592
876d2d6f 593 pr_info("%s\n", igb_copyright);
9d5c8243 594
421e02f0 595#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 596 dca_register_notify(&dca_notifier);
597#endif
bbd98fe4 598 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 599 return ret;
600}
601
602module_init(igb_init_module);
603
604/**
605 * igb_exit_module - Driver Exit Cleanup Routine
606 *
607 * igb_exit_module is called just before the driver is removed
608 * from memory.
609 **/
610static void __exit igb_exit_module(void)
611{
421e02f0 612#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 613 dca_unregister_notify(&dca_notifier);
614#endif
9d5c8243
AK		 615 pci_unregister_driver(&igb_driver);
616}
617
618module_exit(igb_exit_module);
619
26bc19ec
AD		 620#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
621/**
622 * igb_cache_ring_register - Descriptor ring to register mapping
623 * @adapter: board private structure to initialize
624 *
625 * Once we know the feature-set enabled for the device, we'll cache
626 * the register offset the descriptor ring is assigned to.
627 **/
628static void igb_cache_ring_register(struct igb_adapter *adapter)
629{
ee1b9f06 630 int i = 0, j = 0;
047e0030 631 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 632
633 switch (adapter->hw.mac.type) {
634 case e1000_82576:
635 /* The queues are allocated for virtualization such that VF 0
636 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
637 * In order to avoid collision we start at the first free queue
638 * and continue consuming queues in the same sequence
639 */
ee1b9f06 640 if (adapter->vfs_allocated_count) {
a99955fc 641 for (; i < adapter->rss_queues; i++)
3025a446
AD		 642 adapter->rx_ring[i]->reg_idx = rbase_offset +
643 Q_IDX_82576(i);
ee1b9f06 644 }
26bc19ec 645 case e1000_82575:
55cac248 646 case e1000_82580:
d2ba2ed8 647 case e1000_i350:
f96a8a0b
CW		 648 case e1000_i210:
649 case e1000_i211:
26bc19ec 650 default:
ee1b9f06 651 for (; i < adapter->num_rx_queues; i++)
3025a446 652 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 653 for (; j < adapter->num_tx_queues; j++)
3025a446 654 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 655 break;
656 }
657}
658
047e0030
AD		 659static void igb_free_queues(struct igb_adapter *adapter)
660{
3025a446 661 int i;
047e0030 662
3025a446
AD		 663 for (i = 0; i < adapter->num_tx_queues; i++) {
664 kfree(adapter->tx_ring[i]);
665 adapter->tx_ring[i] = NULL;
666 }
667 for (i = 0; i < adapter->num_rx_queues; i++) {
668 kfree(adapter->rx_ring[i]);
669 adapter->rx_ring[i] = NULL;
670 }
047e0030
AD		 671 adapter->num_rx_queues = 0;
672 adapter->num_tx_queues = 0;
673}
674
9d5c8243
AK		 675/**
676 * igb_alloc_queues - Allocate memory for all rings
677 * @adapter: board private structure to initialize
678 *
679 * We allocate one ring per queue at run-time since we don't know the
680 * number of queues at compile-time.
681 **/
682static int igb_alloc_queues(struct igb_adapter *adapter)
683{
3025a446 684 struct igb_ring *ring;
9d5c8243 685 int i;
81c2fc22 686 int orig_node = adapter->node;
9d5c8243 687
661086df 688 for (i = 0; i < adapter->num_tx_queues; i++) {
81c2fc22
AD		 689 if (orig_node == -1) {
690 int cur_node = next_online_node(adapter->node);
691 if (cur_node == MAX_NUMNODES)
692 cur_node = first_online_node;
693 adapter->node = cur_node;
694 }
695 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
696 adapter->node);
697 if (!ring)
698 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 699 if (!ring)
700 goto err;
68fd9910 701 ring->count = adapter->tx_ring_count;
661086df 702 ring->queue_index = i;
59d71989 703 ring->dev = &adapter->pdev->dev;
e694e964 704 ring->netdev = adapter->netdev;
81c2fc22 705 ring->numa_node = adapter->node;
85ad76b2
AD		 706 /* For 82575, context index must be unique per ring. */
707 if (adapter->hw.mac.type == e1000_82575)
866cff06 708 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
3025a446 709 adapter->tx_ring[i] = ring;
661086df 710 }
81c2fc22
AD		 711 /* Restore the adapter's original node */
712 adapter->node = orig_node;
85ad76b2 713
9d5c8243 714 for (i = 0; i < adapter->num_rx_queues; i++) {
81c2fc22
AD		 715 if (orig_node == -1) {
716 int cur_node = next_online_node(adapter->node);
717 if (cur_node == MAX_NUMNODES)
718 cur_node = first_online_node;
719 adapter->node = cur_node;
720 }
721 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
722 adapter->node);
723 if (!ring)
724 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 725 if (!ring)
726 goto err;
68fd9910 727 ring->count = adapter->rx_ring_count;
844290e5 728 ring->queue_index = i;
59d71989 729 ring->dev = &adapter->pdev->dev;
e694e964 730 ring->netdev = adapter->netdev;
81c2fc22 731 ring->numa_node = adapter->node;
85ad76b2
AD		 732 /* set flag indicating ring supports SCTP checksum offload */
733 if (adapter->hw.mac.type >= e1000_82576)
866cff06 734 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
8be10e91 735
f96a8a0b
CW		 736 /*
737 * On i350, i210, and i211, loopback VLAN packets
738 * have the tag byte-swapped.
739 * */
740 if (adapter->hw.mac.type >= e1000_i350)
8be10e91
AD		 741 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
742
3025a446 743 adapter->rx_ring[i] = ring;
9d5c8243 744 }
81c2fc22
AD		 745 /* Restore the adapter's original node */
746 adapter->node = orig_node;
26bc19ec
AD		 747
748 igb_cache_ring_register(adapter);
9d5c8243 749
047e0030 750 return 0;
a88f10ec 751
047e0030 752err:
81c2fc22
AD		 753 /* Restore the adapter's original node */
754 adapter->node = orig_node;
047e0030 755 igb_free_queues(adapter);
d1a8c9e1 756
047e0030 757 return -ENOMEM;
a88f10ec
AD		 758}
759
4be000c8
AD		 760/**
761 * igb_write_ivar - configure ivar for given MSI-X vector
762 * @hw: pointer to the HW structure
763 * @msix_vector: vector number we are allocating to a given ring
764 * @index: row index of IVAR register to write within IVAR table
765 * @offset: column offset of in IVAR, should be multiple of 8
766 *
767 * This function is intended to handle the writing of the IVAR register
768 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
769 * each containing an cause allocation for an Rx and Tx ring, and a
770 * variable number of rows depending on the number of queues supported.
771 **/
772static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
773 int index, int offset)
774{
775 u32 ivar = array_rd32(E1000_IVAR0, index);
776
777 /* clear any bits that are currently set */
778 ivar &= ~((u32)0xFF << offset);
779
780 /* write vector and valid bit */
781 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
782
783 array_wr32(E1000_IVAR0, index, ivar);
784}
785
9d5c8243 786#define IGB_N0_QUEUE -1
047e0030 787static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243 788{
047e0030 789 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 790 struct e1000_hw *hw = &adapter->hw;
047e0030
AD		 791 int rx_queue = IGB_N0_QUEUE;
792 int tx_queue = IGB_N0_QUEUE;
4be000c8 793 u32 msixbm = 0;
047e0030 794
0ba82994
AD		 795 if (q_vector->rx.ring)
796 rx_queue = q_vector->rx.ring->reg_idx;
797 if (q_vector->tx.ring)
798 tx_queue = q_vector->tx.ring->reg_idx;
2d064c06
AD		 799
800 switch (hw->mac.type) {
801 case e1000_82575:
9d5c8243
AK		 802 /* The 82575 assigns vectors using a bitmask, which matches the
803 bitmask for the EICR/EIMS/EIMC registers. To assign one
804 or more queues to a vector, we write the appropriate bits
805 into the MSIXBM register for that vector. */
047e0030 806 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 807 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 808 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 809 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 810 if (!adapter->msix_entries && msix_vector == 0)
811 msixbm |= E1000_EIMS_OTHER;
9d5c8243 812 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 813 q_vector->eims_value = msixbm;
2d064c06
AD		 814 break;
815 case e1000_82576:
4be000c8
AD		 816 /*
817 * 82576 uses a table that essentially consists of 2 columns
818 * with 8 rows. The ordering is column-major so we use the
819 * lower 3 bits as the row index, and the 4th bit as the
820 * column offset.
821 */
822 if (rx_queue > IGB_N0_QUEUE)
823 igb_write_ivar(hw, msix_vector,
824 rx_queue & 0x7,
825 (rx_queue & 0x8) << 1);
826 if (tx_queue > IGB_N0_QUEUE)
827 igb_write_ivar(hw, msix_vector,
828 tx_queue & 0x7,
829 ((tx_queue & 0x8) << 1) + 8);
047e0030 830 q_vector->eims_value = 1 << msix_vector;
2d064c06 831 break;
55cac248 832 case e1000_82580:
d2ba2ed8 833 case e1000_i350:
f96a8a0b
CW		 834 case e1000_i210:
835 case e1000_i211:
4be000c8
AD		 836 /*
837 * On 82580 and newer adapters the scheme is similar to 82576
838 * however instead of ordering column-major we have things
839 * ordered row-major. So we traverse the table by using
840 * bit 0 as the column offset, and the remaining bits as the
841 * row index.
842 */
843 if (rx_queue > IGB_N0_QUEUE)
844 igb_write_ivar(hw, msix_vector,
845 rx_queue >> 1,
846 (rx_queue & 0x1) << 4);
847 if (tx_queue > IGB_N0_QUEUE)
848 igb_write_ivar(hw, msix_vector,
849 tx_queue >> 1,
850 ((tx_queue & 0x1) << 4) + 8);
55cac248
AD		 851 q_vector->eims_value = 1 << msix_vector;
852 break;
2d064c06
AD		 853 default:
854 BUG();
855 break;
856 }
26b39276
AD		 857
858 /* add q_vector eims value to global eims_enable_mask */
859 adapter->eims_enable_mask |= q_vector->eims_value;
860
861 /* configure q_vector to set itr on first interrupt */
862 q_vector->set_itr = 1;
9d5c8243
AK		 863}
864
865/**
866 * igb_configure_msix - Configure MSI-X hardware
867 *
868 * igb_configure_msix sets up the hardware to properly
869 * generate MSI-X interrupts.
870 **/
871static void igb_configure_msix(struct igb_adapter *adapter)
872{
873 u32 tmp;
874 int i, vector = 0;
875 struct e1000_hw *hw = &adapter->hw;
876
877 adapter->eims_enable_mask = 0;
9d5c8243
AK		 878
879 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 880 switch (hw->mac.type) {
881 case e1000_82575:
9d5c8243
AK		 882 tmp = rd32(E1000_CTRL_EXT);
883 /* enable MSI-X PBA support*/
884 tmp |= E1000_CTRL_EXT_PBA_CLR;
885
886 /* Auto-Mask interrupts upon ICR read. */
887 tmp |= E1000_CTRL_EXT_EIAME;
888 tmp |= E1000_CTRL_EXT_IRCA;
889
890 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 891
892 /* enable msix_other interrupt */
893 array_wr32(E1000_MSIXBM(0), vector++,
894 E1000_EIMS_OTHER);
844290e5 895 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 896
2d064c06
AD		 897 break;
898
899 case e1000_82576:
55cac248 900 case e1000_82580:
d2ba2ed8 901 case e1000_i350:
f96a8a0b
CW		 902 case e1000_i210:
903 case e1000_i211:
047e0030
AD		 904 /* Turn on MSI-X capability first, or our settings
905 * won't stick. And it will take days to debug. */
906 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
907 E1000_GPIE_PBA | E1000_GPIE_EIAME |
908 E1000_GPIE_NSICR);
909
910 /* enable msix_other interrupt */
911 adapter->eims_other = 1 << vector;
2d064c06 912 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 913
047e0030 914 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 915 break;
916 default:
917 /* do nothing, since nothing else supports MSI-X */
918 break;
919 } /* switch (hw->mac.type) */
047e0030
AD		 920
921 adapter->eims_enable_mask |= adapter->eims_other;
922
26b39276
AD		 923 for (i = 0; i < adapter->num_q_vectors; i++)
924 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 925
9d5c8243
AK		 926 wrfl();
927}
928
929/**
930 * igb_request_msix - Initialize MSI-X interrupts
931 *
932 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
933 * kernel.
934 **/
935static int igb_request_msix(struct igb_adapter *adapter)
936{
937 struct net_device *netdev = adapter->netdev;
047e0030 938 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 939 int i, err = 0, vector = 0;
940
047e0030 941 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 942 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 943 if (err)
944 goto out;
945 vector++;
946
947 for (i = 0; i < adapter->num_q_vectors; i++) {
948 struct igb_q_vector *q_vector = adapter->q_vector[i];
949
950 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
951
0ba82994 952 if (q_vector->rx.ring && q_vector->tx.ring)
047e0030 953 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
0ba82994
AD		 954 q_vector->rx.ring->queue_index);
955 else if (q_vector->tx.ring)
047e0030 956 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
0ba82994
AD		 957 q_vector->tx.ring->queue_index);
958 else if (q_vector->rx.ring)
047e0030 959 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
0ba82994 960 q_vector->rx.ring->queue_index);
9d5c8243 961 else
047e0030
AD		 962 sprintf(q_vector->name, "%s-unused", netdev->name);
963
9d5c8243 964 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 965 igb_msix_ring, 0, q_vector->name,
047e0030 966 q_vector);
9d5c8243
AK		 967 if (err)
968 goto out;
9d5c8243
AK		 969 vector++;
970 }
971
9d5c8243
AK		 972 igb_configure_msix(adapter);
973 return 0;
974out:
975 return err;
976}
977
978static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
979{
980 if (adapter->msix_entries) {
981 pci_disable_msix(adapter->pdev);
982 kfree(adapter->msix_entries);
983 adapter->msix_entries = NULL;
047e0030 984 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 985 pci_disable_msi(adapter->pdev);
047e0030 986 }
9d5c8243
AK		 987}
988
047e0030
AD		 989/**
990 * igb_free_q_vectors - Free memory allocated for interrupt vectors
991 * @adapter: board private structure to initialize
992 *
993 * This function frees the memory allocated to the q_vectors. In addition if
994 * NAPI is enabled it will delete any references to the NAPI struct prior
995 * to freeing the q_vector.
996 **/
997static void igb_free_q_vectors(struct igb_adapter *adapter)
998{
999 int v_idx;
1000
1001 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1002 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1003 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 1004 if (!q_vector)
1005 continue;
047e0030
AD		 1006 netif_napi_del(&q_vector->napi);
1007 kfree(q_vector);
1008 }
1009 adapter->num_q_vectors = 0;
1010}
1011
1012/**
1013 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1014 *
1015 * This function resets the device so that it has 0 rx queues, tx queues, and
1016 * MSI-X interrupts allocated.
1017 */
1018static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1019{
1020 igb_free_queues(adapter);
1021 igb_free_q_vectors(adapter);
1022 igb_reset_interrupt_capability(adapter);
1023}
9d5c8243
AK		 1024
1025/**
1026 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1027 *
1028 * Attempt to configure interrupts using the best available
1029 * capabilities of the hardware and kernel.
1030 **/
21adef3e 1031static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 1032{
1033 int err;
1034 int numvecs, i;
1035
83b7180d 1036 /* Number of supported queues. */
a99955fc 1037 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 1038 if (adapter->vfs_allocated_count)
1039 adapter->num_tx_queues = 1;
1040 else
1041 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1042
047e0030
AD		 1043 /* start with one vector for every rx queue */
1044 numvecs = adapter->num_rx_queues;
1045
3ad2f3fb 1046 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1047 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1048 numvecs += adapter->num_tx_queues;
047e0030
AD		 1049
1050 /* store the number of vectors reserved for queues */
1051 adapter->num_q_vectors = numvecs;
1052
1053 /* add 1 vector for link status interrupts */
1054 numvecs++;
9d5c8243
AK		 1055 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1056 GFP_KERNEL);
f96a8a0b 1057
9d5c8243
AK		 1058 if (!adapter->msix_entries)
1059 goto msi_only;
1060
1061 for (i = 0; i < numvecs; i++)
1062 adapter->msix_entries[i].entry = i;
1063
1064 err = pci_enable_msix(adapter->pdev,
1065 adapter->msix_entries,
1066 numvecs);
1067 if (err == 0)
34a20e89 1068 goto out;
9d5c8243
AK		 1069
1070 igb_reset_interrupt_capability(adapter);
1071
1072 /* If we can't do MSI-X, try MSI */
1073msi_only:
2a3abf6d
AD		 1074#ifdef CONFIG_PCI_IOV
1075 /* disable SR-IOV for non MSI-X configurations */
1076 if (adapter->vf_data) {
1077 struct e1000_hw *hw = &adapter->hw;
1078 /* disable iov and allow time for transactions to clear */
1079 pci_disable_sriov(adapter->pdev);
1080 msleep(500);
1081
1082 kfree(adapter->vf_data);
1083 adapter->vf_data = NULL;
1084 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 1085 wrfl();
2a3abf6d
AD		 1086 msleep(100);
1087 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1088 }
1089#endif
4fc82adf 1090 adapter->vfs_allocated_count = 0;
a99955fc 1091 adapter->rss_queues = 1;
4fc82adf 1092 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1093 adapter->num_rx_queues = 1;
661086df 1094 adapter->num_tx_queues = 1;
047e0030 1095 adapter->num_q_vectors = 1;
9d5c8243 1096 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1097 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1098out:
21adef3e 1099 /* Notify the stack of the (possibly) reduced queue counts. */
cfb8c3aa 1100 rtnl_lock();
21adef3e 1101 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
cfb8c3aa
BP		 1102 err = netif_set_real_num_rx_queues(adapter->netdev,
1103 adapter->num_rx_queues);
1104 rtnl_unlock();
1105 return err;
9d5c8243
AK		 1106}
1107
047e0030
AD		 1108/**
1109 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1110 * @adapter: board private structure to initialize
1111 *
1112 * We allocate one q_vector per queue interrupt. If allocation fails we
1113 * return -ENOMEM.
1114 **/
1115static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1116{
1117 struct igb_q_vector *q_vector;
1118 struct e1000_hw *hw = &adapter->hw;
1119 int v_idx;
81c2fc22 1120 int orig_node = adapter->node;
047e0030
AD		 1121
1122 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
81c2fc22
AD		 1123 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1124 adapter->num_tx_queues)) &&
1125 (adapter->num_rx_queues == v_idx))
1126 adapter->node = orig_node;
1127 if (orig_node == -1) {
1128 int cur_node = next_online_node(adapter->node);
1129 if (cur_node == MAX_NUMNODES)
1130 cur_node = first_online_node;
1131 adapter->node = cur_node;
1132 }
1133 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1134 adapter->node);
1135 if (!q_vector)
1136 q_vector = kzalloc(sizeof(struct igb_q_vector),
1137 GFP_KERNEL);
047e0030
AD		 1138 if (!q_vector)
1139 goto err_out;
1140 q_vector->adapter = adapter;
047e0030
AD		 1141 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1142 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1143 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1144 adapter->q_vector[v_idx] = q_vector;
1145 }
81c2fc22
AD		 1146 /* Restore the adapter's original node */
1147 adapter->node = orig_node;
1148
047e0030
AD		 1149 return 0;
1150
1151err_out:
81c2fc22
AD		 1152 /* Restore the adapter's original node */
1153 adapter->node = orig_node;
fe0592b4 1154 igb_free_q_vectors(adapter);
047e0030
AD		 1155 return -ENOMEM;
1156}
1157
1158static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1159 int ring_idx, int v_idx)
1160{
3025a446 1161 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1162
0ba82994
AD		 1163 q_vector->rx.ring = adapter->rx_ring[ring_idx];
1164 q_vector->rx.ring->q_vector = q_vector;
1165 q_vector->rx.count++;
4fc82adf
AD		 1166 q_vector->itr_val = adapter->rx_itr_setting;
1167 if (q_vector->itr_val && q_vector->itr_val <= 3)
1168 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1169}
1170
1171static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1172 int ring_idx, int v_idx)
1173{
3025a446 1174 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1175
0ba82994
AD		 1176 q_vector->tx.ring = adapter->tx_ring[ring_idx];
1177 q_vector->tx.ring->q_vector = q_vector;
1178 q_vector->tx.count++;
4fc82adf 1179 q_vector->itr_val = adapter->tx_itr_setting;
0ba82994 1180 q_vector->tx.work_limit = adapter->tx_work_limit;
4fc82adf
AD		 1181 if (q_vector->itr_val && q_vector->itr_val <= 3)
1182 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1183}
1184
1185/**
1186 * igb_map_ring_to_vector - maps allocated queues to vectors
1187 *
1188 * This function maps the recently allocated queues to vectors.
1189 **/
1190static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1191{
1192 int i;
1193 int v_idx = 0;
1194
1195 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1196 (adapter->num_q_vectors < adapter->num_tx_queues))
1197 return -ENOMEM;
1198
1199 if (adapter->num_q_vectors >=
1200 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1201 for (i = 0; i < adapter->num_rx_queues; i++)
1202 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1203 for (i = 0; i < adapter->num_tx_queues; i++)
1204 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1205 } else {
1206 for (i = 0; i < adapter->num_rx_queues; i++) {
1207 if (i < adapter->num_tx_queues)
1208 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1209 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1210 }
1211 for (; i < adapter->num_tx_queues; i++)
1212 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1213 }
1214 return 0;
1215}
1216
1217/**
1218 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1219 *
1220 * This function initializes the interrupts and allocates all of the queues.
1221 **/
1222static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1223{
1224 struct pci_dev *pdev = adapter->pdev;
1225 int err;
1226
21adef3e
BH		 1227 err = igb_set_interrupt_capability(adapter);
1228 if (err)
1229 return err;
047e0030
AD		 1230
1231 err = igb_alloc_q_vectors(adapter);
1232 if (err) {
1233 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1234 goto err_alloc_q_vectors;
1235 }
1236
1237 err = igb_alloc_queues(adapter);
1238 if (err) {
1239 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1240 goto err_alloc_queues;
1241 }
1242
1243 err = igb_map_ring_to_vector(adapter);
1244 if (err) {
1245 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1246 goto err_map_queues;
1247 }
1248
1249
1250 return 0;
1251err_map_queues:
1252 igb_free_queues(adapter);
1253err_alloc_queues:
1254 igb_free_q_vectors(adapter);
1255err_alloc_q_vectors:
1256 igb_reset_interrupt_capability(adapter);
1257 return err;
1258}
1259
9d5c8243
AK		 1260/**
1261 * igb_request_irq - initialize interrupts
1262 *
1263 * Attempts to configure interrupts using the best available
1264 * capabilities of the hardware and kernel.
1265 **/
1266static int igb_request_irq(struct igb_adapter *adapter)
1267{
1268 struct net_device *netdev = adapter->netdev;
047e0030 1269 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1270 int err = 0;
1271
1272 if (adapter->msix_entries) {
1273 err = igb_request_msix(adapter);
844290e5 1274 if (!err)
9d5c8243 1275 goto request_done;
9d5c8243 1276 /* fall back to MSI */
047e0030 1277 igb_clear_interrupt_scheme(adapter);
c74d588e 1278 if (!pci_enable_msi(pdev))
7dfc16fa 1279 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1280 igb_free_all_tx_resources(adapter);
1281 igb_free_all_rx_resources(adapter);
047e0030 1282 adapter->num_tx_queues = 1;
9d5c8243 1283 adapter->num_rx_queues = 1;
047e0030
AD		 1284 adapter->num_q_vectors = 1;
1285 err = igb_alloc_q_vectors(adapter);
1286 if (err) {
1287 dev_err(&pdev->dev,
1288 "Unable to allocate memory for vectors\n");
1289 goto request_done;
1290 }
1291 err = igb_alloc_queues(adapter);
1292 if (err) {
1293 dev_err(&pdev->dev,
1294 "Unable to allocate memory for queues\n");
1295 igb_free_q_vectors(adapter);
1296 goto request_done;
1297 }
1298 igb_setup_all_tx_resources(adapter);
1299 igb_setup_all_rx_resources(adapter);
9d5c8243 1300 }
844290e5 1301
c74d588e
AD		 1302 igb_assign_vector(adapter->q_vector[0], 0);
1303
7dfc16fa 1304 if (adapter->flags & IGB_FLAG_HAS_MSI) {
c74d588e 1305 err = request_irq(pdev->irq, igb_intr_msi, 0,
047e0030 1306 netdev->name, adapter);
9d5c8243
AK		 1307 if (!err)
1308 goto request_done;
047e0030 1309
9d5c8243
AK		 1310 /* fall back to legacy interrupts */
1311 igb_reset_interrupt_capability(adapter);
7dfc16fa 1312 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1313 }
1314
c74d588e 1315 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1316 netdev->name, adapter);
9d5c8243 1317
6cb5e577 1318 if (err)
c74d588e 1319 dev_err(&pdev->dev, "Error %d getting interrupt\n",
9d5c8243 1320 err);
9d5c8243
AK		 1321
1322request_done:
1323 return err;
1324}
1325
1326static void igb_free_irq(struct igb_adapter *adapter)
1327{
9d5c8243
AK		 1328 if (adapter->msix_entries) {
1329 int vector = 0, i;
1330
047e0030 1331 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1332
0d1ae7f4 1333 for (i = 0; i < adapter->num_q_vectors; i++)
047e0030 1334 free_irq(adapter->msix_entries[vector++].vector,
0d1ae7f4 1335 adapter->q_vector[i]);
047e0030
AD		 1336 } else {
1337 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1338 }
9d5c8243
AK		 1339}
1340
1341/**
1342 * igb_irq_disable - Mask off interrupt generation on the NIC
1343 * @adapter: board private structure
1344 **/
1345static void igb_irq_disable(struct igb_adapter *adapter)
1346{
1347 struct e1000_hw *hw = &adapter->hw;
1348
25568a53
AD		 1349 /*
1350 * we need to be careful when disabling interrupts. The VFs are also
1351 * mapped into these registers and so clearing the bits can cause
1352 * issues on the VF drivers so we only need to clear what we set
1353 */
9d5c8243 1354 if (adapter->msix_entries) {
2dfd1212
AD		 1355 u32 regval = rd32(E1000_EIAM);
1356 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1357 wr32(E1000_EIMC, adapter->eims_enable_mask);
1358 regval = rd32(E1000_EIAC);
1359 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1360 }
844290e5
PW		 1361
1362 wr32(E1000_IAM, 0);
9d5c8243
AK		 1363 wr32(E1000_IMC, ~0);
1364 wrfl();
81a61859
ET		 1365 if (adapter->msix_entries) {
1366 int i;
1367 for (i = 0; i < adapter->num_q_vectors; i++)
1368 synchronize_irq(adapter->msix_entries[i].vector);
1369 } else {
1370 synchronize_irq(adapter->pdev->irq);
1371 }
9d5c8243
AK		 1372}
1373
1374/**
1375 * igb_irq_enable - Enable default interrupt generation settings
1376 * @adapter: board private structure
1377 **/
1378static void igb_irq_enable(struct igb_adapter *adapter)
1379{
1380 struct e1000_hw *hw = &adapter->hw;
1381
1382 if (adapter->msix_entries) {
06218a8d 1383 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
2dfd1212
AD		 1384 u32 regval = rd32(E1000_EIAC);
1385 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1386 regval = rd32(E1000_EIAM);
1387 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1388 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1389 if (adapter->vfs_allocated_count) {
4ae196df 1390 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1391 ims |= E1000_IMS_VMMB;
1392 }
1393 wr32(E1000_IMS, ims);
844290e5 1394 } else {
55cac248
AD		 1395 wr32(E1000_IMS, IMS_ENABLE_MASK |
1396 E1000_IMS_DRSTA);
1397 wr32(E1000_IAM, IMS_ENABLE_MASK |
1398 E1000_IMS_DRSTA);
844290e5 1399 }
9d5c8243
AK		 1400}
1401
1402static void igb_update_mng_vlan(struct igb_adapter *adapter)
1403{
51466239 1404 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1405 u16 vid = adapter->hw.mng_cookie.vlan_id;
1406 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1407
1408 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1409 /* add VID to filter table */
1410 igb_vfta_set(hw, vid, true);
1411 adapter->mng_vlan_id = vid;
1412 } else {
1413 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1414 }
1415
1416 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1417 (vid != old_vid) &&
b2cb09b1 1418 !test_bit(old_vid, adapter->active_vlans)) {
51466239
AD		 1419 /* remove VID from filter table */
1420 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1421 }
1422}
1423
1424/**
1425 * igb_release_hw_control - release control of the h/w to f/w
1426 * @adapter: address of board private structure
1427 *
1428 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1429 * For ASF and Pass Through versions of f/w this means that the
1430 * driver is no longer loaded.
1431 *
1432 **/
1433static void igb_release_hw_control(struct igb_adapter *adapter)
1434{
1435 struct e1000_hw *hw = &adapter->hw;
1436 u32 ctrl_ext;
1437
1438 /* Let firmware take over control of h/w */
1439 ctrl_ext = rd32(E1000_CTRL_EXT);
1440 wr32(E1000_CTRL_EXT,
1441 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1442}
1443
9d5c8243
AK		 1444/**
1445 * igb_get_hw_control - get control of the h/w from f/w
1446 * @adapter: address of board private structure
1447 *
1448 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1449 * For ASF and Pass Through versions of f/w this means that
1450 * the driver is loaded.
1451 *
1452 **/
1453static void igb_get_hw_control(struct igb_adapter *adapter)
1454{
1455 struct e1000_hw *hw = &adapter->hw;
1456 u32 ctrl_ext;
1457
1458 /* Let firmware know the driver has taken over */
1459 ctrl_ext = rd32(E1000_CTRL_EXT);
1460 wr32(E1000_CTRL_EXT,
1461 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1462}
1463
9d5c8243
AK		 1464/**
1465 * igb_configure - configure the hardware for RX and TX
1466 * @adapter: private board structure
1467 **/
1468static void igb_configure(struct igb_adapter *adapter)
1469{
1470 struct net_device *netdev = adapter->netdev;
1471 int i;
1472
1473 igb_get_hw_control(adapter);
ff41f8dc 1474 igb_set_rx_mode(netdev);
9d5c8243
AK		 1475
1476 igb_restore_vlan(adapter);
9d5c8243 1477
85b430b4 1478 igb_setup_tctl(adapter);
06cf2666 1479 igb_setup_mrqc(adapter);
9d5c8243 1480 igb_setup_rctl(adapter);
85b430b4
AD		 1481
1482 igb_configure_tx(adapter);
9d5c8243 1483 igb_configure_rx(adapter);
662d7205
AD		 1484
1485 igb_rx_fifo_flush_82575(&adapter->hw);
1486
c493ea45 1487 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1488 * at least 1 descriptor unused to make sure
1489 * next_to_use != next_to_clean */
1490 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1491 struct igb_ring *ring = adapter->rx_ring[i];
cd392f5c 1492 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
9d5c8243 1493 }
9d5c8243
AK		 1494}
1495
88a268c1
NN		 1496/**
1497 * igb_power_up_link - Power up the phy/serdes link
1498 * @adapter: address of board private structure
1499 **/
1500void igb_power_up_link(struct igb_adapter *adapter)
1501{
76886596
AA		 1502 igb_reset_phy(&adapter->hw);
1503
88a268c1
NN		 1504 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1505 igb_power_up_phy_copper(&adapter->hw);
1506 else
1507 igb_power_up_serdes_link_82575(&adapter->hw);
1508}
1509
1510/**
1511 * igb_power_down_link - Power down the phy/serdes link
1512 * @adapter: address of board private structure
1513 */
1514static void igb_power_down_link(struct igb_adapter *adapter)
1515{
1516 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1517 igb_power_down_phy_copper_82575(&adapter->hw);
1518 else
1519 igb_shutdown_serdes_link_82575(&adapter->hw);
1520}
9d5c8243
AK		 1521
1522/**
1523 * igb_up - Open the interface and prepare it to handle traffic
1524 * @adapter: board private structure
1525 **/
9d5c8243
AK		 1526int igb_up(struct igb_adapter *adapter)
1527{
1528 struct e1000_hw *hw = &adapter->hw;
1529 int i;
1530
1531 /* hardware has been reset, we need to reload some things */
1532 igb_configure(adapter);
1533
1534 clear_bit(__IGB_DOWN, &adapter->state);
1535
0d1ae7f4
AD		 1536 for (i = 0; i < adapter->num_q_vectors; i++)
1537 napi_enable(&(adapter->q_vector[i]->napi));
1538
844290e5 1539 if (adapter->msix_entries)
9d5c8243 1540 igb_configure_msix(adapter);
feeb2721
AD		 1541 else
1542 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1543
1544 /* Clear any pending interrupts. */
1545 rd32(E1000_ICR);
1546 igb_irq_enable(adapter);
1547
d4960307
AD		 1548 /* notify VFs that reset has been completed */
1549 if (adapter->vfs_allocated_count) {
1550 u32 reg_data = rd32(E1000_CTRL_EXT);
1551 reg_data |= E1000_CTRL_EXT_PFRSTD;
1552 wr32(E1000_CTRL_EXT, reg_data);
1553 }
1554
4cb9be7a
JB		 1555 netif_tx_start_all_queues(adapter->netdev);
1556
25568a53
AD		 1557 /* start the watchdog. */
1558 hw->mac.get_link_status = 1;
1559 schedule_work(&adapter->watchdog_task);
1560
9d5c8243
AK		 1561 return 0;
1562}
1563
1564void igb_down(struct igb_adapter *adapter)
1565{
9d5c8243 1566 struct net_device *netdev = adapter->netdev;
330a6d6a 1567 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1568 u32 tctl, rctl;
1569 int i;
1570
1571 /* signal that we're down so the interrupt handler does not
1572 * reschedule our watchdog timer */
1573 set_bit(__IGB_DOWN, &adapter->state);
1574
1575 /* disable receives in the hardware */
1576 rctl = rd32(E1000_RCTL);
1577 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1578 /* flush and sleep below */
1579
fd2ea0a7 1580 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1581
1582 /* disable transmits in the hardware */
1583 tctl = rd32(E1000_TCTL);
1584 tctl &= ~E1000_TCTL_EN;
1585 wr32(E1000_TCTL, tctl);
1586 /* flush both disables and wait for them to finish */
1587 wrfl();
1588 msleep(10);
1589
0d1ae7f4
AD		 1590 for (i = 0; i < adapter->num_q_vectors; i++)
1591 napi_disable(&(adapter->q_vector[i]->napi));
9d5c8243 1592
9d5c8243
AK		 1593 igb_irq_disable(adapter);
1594
1595 del_timer_sync(&adapter->watchdog_timer);
1596 del_timer_sync(&adapter->phy_info_timer);
1597
9d5c8243 1598 netif_carrier_off(netdev);
04fe6358
AD		 1599
1600 /* record the stats before reset*/
12dcd86b
ED		 1601 spin_lock(&adapter->stats64_lock);
1602 igb_update_stats(adapter, &adapter->stats64);
1603 spin_unlock(&adapter->stats64_lock);
04fe6358 1604
9d5c8243
AK		 1605 adapter->link_speed = 0;
1606 adapter->link_duplex = 0;
1607
3023682e
JK		 1608 if (!pci_channel_offline(adapter->pdev))
1609 igb_reset(adapter);
9d5c8243
AK		 1610 igb_clean_all_tx_rings(adapter);
1611 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1612#ifdef CONFIG_IGB_DCA
1613
1614 /* since we reset the hardware DCA settings were cleared */
1615 igb_setup_dca(adapter);
1616#endif
9d5c8243
AK		 1617}
1618
1619void igb_reinit_locked(struct igb_adapter *adapter)
1620{
1621 WARN_ON(in_interrupt());
1622 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1623 msleep(1);
1624 igb_down(adapter);
1625 igb_up(adapter);
1626 clear_bit(__IGB_RESETTING, &adapter->state);
1627}
1628
1629void igb_reset(struct igb_adapter *adapter)
1630{
090b1795 1631 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1632 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1633 struct e1000_mac_info *mac = &hw->mac;
1634 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1635 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1636 u16 hwm;
1637
1638 /* Repartition Pba for greater than 9k mtu
1639 * To take effect CTRL.RST is required.
1640 */
fa4dfae0 1641 switch (mac->type) {
d2ba2ed8 1642 case e1000_i350:
55cac248
AD		 1643 case e1000_82580:
1644 pba = rd32(E1000_RXPBS);
1645 pba = igb_rxpbs_adjust_82580(pba);
1646 break;
fa4dfae0 1647 case e1000_82576:
d249be54
AD		 1648 pba = rd32(E1000_RXPBS);
1649 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1650 break;
1651 case e1000_82575:
f96a8a0b
CW		 1652 case e1000_i210:
1653 case e1000_i211:
fa4dfae0
AD		 1654 default:
1655 pba = E1000_PBA_34K;
1656 break;
2d064c06 1657 }
9d5c8243 1658
2d064c06
AD		 1659 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1660 (mac->type < e1000_82576)) {
9d5c8243
AK		 1661 /* adjust PBA for jumbo frames */
1662 wr32(E1000_PBA, pba);
1663
1664 /* To maintain wire speed transmits, the Tx FIFO should be
1665 * large enough to accommodate two full transmit packets,
1666 * rounded up to the next 1KB and expressed in KB. Likewise,
1667 * the Rx FIFO should be large enough to accommodate at least
1668 * one full receive packet and is similarly rounded up and
1669 * expressed in KB. */
1670 pba = rd32(E1000_PBA);
1671 /* upper 16 bits has Tx packet buffer allocation size in KB */
1672 tx_space = pba >> 16;
1673 /* lower 16 bits has Rx packet buffer allocation size in KB */
1674 pba &= 0xffff;
1675 /* the tx fifo also stores 16 bytes of information about the tx
1676 * but don't include ethernet FCS because hardware appends it */
1677 min_tx_space = (adapter->max_frame_size +
85e8d004 1678 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1679 ETH_FCS_LEN) * 2;
1680 min_tx_space = ALIGN(min_tx_space, 1024);
1681 min_tx_space >>= 10;
1682 /* software strips receive CRC, so leave room for it */
1683 min_rx_space = adapter->max_frame_size;
1684 min_rx_space = ALIGN(min_rx_space, 1024);
1685 min_rx_space >>= 10;
1686
1687 /* If current Tx allocation is less than the min Tx FIFO size,
1688 * and the min Tx FIFO size is less than the current Rx FIFO
1689 * allocation, take space away from current Rx allocation */
1690 if (tx_space < min_tx_space &&
1691 ((min_tx_space - tx_space) < pba)) {
1692 pba = pba - (min_tx_space - tx_space);
1693
1694 /* if short on rx space, rx wins and must trump tx
1695 * adjustment */
1696 if (pba < min_rx_space)
1697 pba = min_rx_space;
1698 }
2d064c06 1699 wr32(E1000_PBA, pba);
9d5c8243 1700 }
9d5c8243
AK		 1701
1702 /* flow control settings */
1703 /* The high water mark must be low enough to fit one full frame
1704 * (or the size used for early receive) above it in the Rx FIFO.
1705 * Set it to the lower of:
1706 * - 90% of the Rx FIFO size, or
1707 * - the full Rx FIFO size minus one full frame */
1708 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1709 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1710
d405ea3e
AD		 1711 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1712 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1713 fc->pause_time = 0xFFFF;
1714 fc->send_xon = 1;
0cce119a 1715 fc->current_mode = fc->requested_mode;
9d5c8243 1716
4ae196df
AD		 1717 /* disable receive for all VFs and wait one second */
1718 if (adapter->vfs_allocated_count) {
1719 int i;
1720 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
8fa7e0f7 1721 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
4ae196df
AD		 1722
1723 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1724 igb_ping_all_vfs(adapter);
4ae196df
AD		 1725
1726 /* disable transmits and receives */
1727 wr32(E1000_VFRE, 0);
1728 wr32(E1000_VFTE, 0);
1729 }
1730
9d5c8243 1731 /* Allow time for pending master requests to run */
330a6d6a 1732 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1733 wr32(E1000_WUC, 0);
1734
330a6d6a 1735 if (hw->mac.ops.init_hw(hw))
090b1795 1736 dev_err(&pdev->dev, "Hardware Error\n");
831ec0b4 1737
a27416bb
MV		 1738 /*
1739 * Flow control settings reset on hardware reset, so guarantee flow
1740 * control is off when forcing speed.
1741 */
1742 if (!hw->mac.autoneg)
1743 igb_force_mac_fc(hw);
1744
b6e0c419 1745 igb_init_dmac(adapter, pba);
88a268c1
NN		 1746 if (!netif_running(adapter->netdev))
1747 igb_power_down_link(adapter);
1748
9d5c8243
AK		 1749 igb_update_mng_vlan(adapter);
1750
1751 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1752 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1753
330a6d6a 1754 igb_get_phy_info(hw);
9d5c8243
AK		 1755}
1756
c8f44aff
MM		 1757static netdev_features_t igb_fix_features(struct net_device *netdev,
1758 netdev_features_t features)
b2cb09b1
JP		 1759{
1760 /*
1761 * Since there is no support for separate rx/tx vlan accel
1762 * enable/disable make sure tx flag is always in same state as rx.
1763 */
1764 if (features & NETIF_F_HW_VLAN_RX)
1765 features |= NETIF_F_HW_VLAN_TX;
1766 else
1767 features &= ~NETIF_F_HW_VLAN_TX;
1768
1769 return features;
1770}
1771
c8f44aff
MM		 1772static int igb_set_features(struct net_device *netdev,
1773 netdev_features_t features)
ac52caa3 1774{
c8f44aff 1775 netdev_features_t changed = netdev->features ^ features;
89eaefb6 1776 struct igb_adapter *adapter = netdev_priv(netdev);
ac52caa3 1777
b2cb09b1
JP		 1778 if (changed & NETIF_F_HW_VLAN_RX)
1779 igb_vlan_mode(netdev, features);
1780
89eaefb6
BG		 1781 if (!(changed & NETIF_F_RXALL))
1782 return 0;
1783
1784 netdev->features = features;
1785
1786 if (netif_running(netdev))
1787 igb_reinit_locked(adapter);
1788 else
1789 igb_reset(adapter);
1790
ac52caa3
MM		 1791 return 0;
1792}
1793
2e5c6922 1794static const struct net_device_ops igb_netdev_ops = {
559e9c49 1795 .ndo_open = igb_open,
2e5c6922 1796 .ndo_stop = igb_close,
cd392f5c 1797 .ndo_start_xmit = igb_xmit_frame,
12dcd86b 1798 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc 1799 .ndo_set_rx_mode = igb_set_rx_mode,
2e5c6922
SH		 1800 .ndo_set_mac_address = igb_set_mac,
1801 .ndo_change_mtu = igb_change_mtu,
1802 .ndo_do_ioctl = igb_ioctl,
1803 .ndo_tx_timeout = igb_tx_timeout,
1804 .ndo_validate_addr = eth_validate_addr,
2e5c6922
SH		 1805 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1806 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1807 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1808 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1809 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1810 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1811#ifdef CONFIG_NET_POLL_CONTROLLER
1812 .ndo_poll_controller = igb_netpoll,
1813#endif
b2cb09b1
JP		 1814 .ndo_fix_features = igb_fix_features,
1815 .ndo_set_features = igb_set_features,
2e5c6922
SH		 1816};
1817
d67974f0
CW		 1818/**
1819 * igb_set_fw_version - Configure version string for ethtool
1820 * @adapter: adapter struct
1821 *
1822 **/
1823void igb_set_fw_version(struct igb_adapter *adapter)
1824{
1825 struct e1000_hw *hw = &adapter->hw;
1826 u16 eeprom_verh, eeprom_verl, comb_verh, comb_verl, comb_offset;
1827 u16 major, build, patch, fw_version;
1828 u32 etrack_id;
1829
1830 hw->nvm.ops.read(hw, 5, 1, &fw_version);
1831 if (adapter->hw.mac.type != e1000_i211) {
1832 hw->nvm.ops.read(hw, NVM_ETRACK_WORD, 1, &eeprom_verh);
1833 hw->nvm.ops.read(hw, (NVM_ETRACK_WORD + 1), 1, &eeprom_verl);
1834 etrack_id = (eeprom_verh << IGB_ETRACK_SHIFT) | eeprom_verl;
1835
1836 /* combo image version needs to be found */
1837 hw->nvm.ops.read(hw, NVM_COMB_VER_PTR, 1, &comb_offset);
1838 if ((comb_offset != 0x0) &&
1839 (comb_offset != IGB_NVM_VER_INVALID)) {
1840 hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset
1841 + 1), 1, &comb_verh);
1842 hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset),
1843 1, &comb_verl);
1844
1845 /* Only display Option Rom if it exists and is valid */
1846 if ((comb_verh && comb_verl) &&
1847 ((comb_verh != IGB_NVM_VER_INVALID) &&
1848 (comb_verl != IGB_NVM_VER_INVALID))) {
1849 major = comb_verl >> IGB_COMB_VER_SHFT;
1850 build = (comb_verl << IGB_COMB_VER_SHFT) |
1851 (comb_verh >> IGB_COMB_VER_SHFT);
1852 patch = comb_verh & IGB_COMB_VER_MASK;
1853 snprintf(adapter->fw_version,
1854 sizeof(adapter->fw_version),
1855 "%d.%d%d, 0x%08x, %d.%d.%d",
1856 (fw_version & IGB_MAJOR_MASK) >>
1857 IGB_MAJOR_SHIFT,
1858 (fw_version & IGB_MINOR_MASK) >>
1859 IGB_MINOR_SHIFT,
1860 (fw_version & IGB_BUILD_MASK),
1861 etrack_id, major, build, patch);
1862 goto out;
1863 }
1864 }
1865 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1866 "%d.%d%d, 0x%08x",
1867 (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1868 (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1869 (fw_version & IGB_BUILD_MASK), etrack_id);
1870 } else {
1871 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1872 "%d.%d%d",
1873 (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1874 (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1875 (fw_version & IGB_BUILD_MASK));
1876 }
1877out:
1878 return;
1879}
1880
9d5c8243
AK		 1881/**
1882 * igb_probe - Device Initialization Routine
1883 * @pdev: PCI device information struct
1884 * @ent: entry in igb_pci_tbl
1885 *
1886 * Returns 0 on success, negative on failure
1887 *
1888 * igb_probe initializes an adapter identified by a pci_dev structure.
1889 * The OS initialization, configuring of the adapter private structure,
1890 * and a hardware reset occur.
1891 **/
1892static int __devinit igb_probe(struct pci_dev *pdev,
1893 const struct pci_device_id *ent)
1894{
1895 struct net_device *netdev;
1896 struct igb_adapter *adapter;
1897 struct e1000_hw *hw;
4337e993 1898 u16 eeprom_data = 0;
9835fd73 1899 s32 ret_val;
4337e993 1900 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1901 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1902 unsigned long mmio_start, mmio_len;
2d6a5e95 1903 int err, pci_using_dac;
9d5c8243 1904 u16 eeprom_apme_mask = IGB_EEPROM_APME;
9835fd73 1905 u8 part_str[E1000_PBANUM_LENGTH];
9d5c8243 1906
bded64a7
AG		 1907 /* Catch broken hardware that put the wrong VF device ID in
1908 * the PCIe SR-IOV capability.
1909 */
1910 if (pdev->is_virtfn) {
1911 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
f96a8a0b 1912 pci_name(pdev), pdev->vendor, pdev->device);
bded64a7
AG		 1913 return -EINVAL;
1914 }
1915
aed5dec3 1916 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1917 if (err)
1918 return err;
1919
1920 pci_using_dac = 0;
59d71989 1921 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1922 if (!err) {
59d71989 1923 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1924 if (!err)
1925 pci_using_dac = 1;
1926 } else {
59d71989 1927 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1928 if (err) {
59d71989 1929 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1930 if (err) {
1931 dev_err(&pdev->dev, "No usable DMA "
1932 "configuration, aborting\n");
1933 goto err_dma;
1934 }
1935 }
1936 }
1937
aed5dec3
AD		 1938 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1939 IORESOURCE_MEM),
1940 igb_driver_name);
9d5c8243
AK		 1941 if (err)
1942 goto err_pci_reg;
1943
19d5afd4 1944 pci_enable_pcie_error_reporting(pdev);
40a914fa 1945
9d5c8243 1946 pci_set_master(pdev);
c682fc23 1947 pci_save_state(pdev);
9d5c8243
AK		 1948
1949 err = -ENOMEM;
1bfaf07b 1950 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1cc3bd87 1951 IGB_MAX_TX_QUEUES);
9d5c8243
AK		 1952 if (!netdev)
1953 goto err_alloc_etherdev;
1954
1955 SET_NETDEV_DEV(netdev, &pdev->dev);
1956
1957 pci_set_drvdata(pdev, netdev);
1958 adapter = netdev_priv(netdev);
1959 adapter->netdev = netdev;
1960 adapter->pdev = pdev;
1961 hw = &adapter->hw;
1962 hw->back = adapter;
b3f4d599 1963 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
9d5c8243
AK		 1964
1965 mmio_start = pci_resource_start(pdev, 0);
1966 mmio_len = pci_resource_len(pdev, 0);
1967
1968 err = -EIO;
28b0759c
AD		 1969 hw->hw_addr = ioremap(mmio_start, mmio_len);
1970 if (!hw->hw_addr)
9d5c8243
AK		 1971 goto err_ioremap;
1972
2e5c6922 1973 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1974 igb_set_ethtool_ops(netdev);
9d5c8243 1975 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1976
1977 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1978
1979 netdev->mem_start = mmio_start;
1980 netdev->mem_end = mmio_start + mmio_len;
1981
9d5c8243
AK		 1982 /* PCI config space info */
1983 hw->vendor_id = pdev->vendor;
1984 hw->device_id = pdev->device;
1985 hw->revision_id = pdev->revision;
1986 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1987 hw->subsystem_device_id = pdev->subsystem_device;
1988
9d5c8243
AK		 1989 /* Copy the default MAC, PHY and NVM function pointers */
1990 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1991 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1992 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1993 /* Initialize skew-specific constants */
1994 err = ei->get_invariants(hw);
1995 if (err)
450c87c8 1996 goto err_sw_init;
9d5c8243 1997
450c87c8 1998 /* setup the private structure */
9d5c8243
AK		 1999 err = igb_sw_init(adapter);
2000 if (err)
2001 goto err_sw_init;
2002
2003 igb_get_bus_info_pcie(hw);
2004
2005 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 2006
2007 /* Copper options */
2008 if (hw->phy.media_type == e1000_media_type_copper) {
2009 hw->phy.mdix = AUTO_ALL_MODES;
2010 hw->phy.disable_polarity_correction = false;
2011 hw->phy.ms_type = e1000_ms_hw_default;
2012 }
2013
2014 if (igb_check_reset_block(hw))
2015 dev_info(&pdev->dev,
2016 "PHY reset is blocked due to SOL/IDER session.\n");
2017
077887c3
AD		 2018 /*
2019 * features is initialized to 0 in allocation, it might have bits
2020 * set by igb_sw_init so we should use an or instead of an
2021 * assignment.
2022 */
2023 netdev->features |= NETIF_F_SG |
2024 NETIF_F_IP_CSUM |
2025 NETIF_F_IPV6_CSUM |
2026 NETIF_F_TSO |
2027 NETIF_F_TSO6 |
2028 NETIF_F_RXHASH |
2029 NETIF_F_RXCSUM |
2030 NETIF_F_HW_VLAN_RX |
2031 NETIF_F_HW_VLAN_TX;
2032
2033 /* copy netdev features into list of user selectable features */
2034 netdev->hw_features |= netdev->features;
89eaefb6 2035 netdev->hw_features |= NETIF_F_RXALL;
077887c3
AD		 2036
2037 /* set this bit last since it cannot be part of hw_features */
2038 netdev->features |= NETIF_F_HW_VLAN_FILTER;
2039
2040 netdev->vlan_features |= NETIF_F_TSO |
2041 NETIF_F_TSO6 |
2042 NETIF_F_IP_CSUM |
2043 NETIF_F_IPV6_CSUM |
2044 NETIF_F_SG;
48f29ffc 2045
6b8f0922
BG		 2046 netdev->priv_flags |= IFF_SUPP_NOFCS;
2047
7b872a55 2048 if (pci_using_dac) {
9d5c8243 2049 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 2050 netdev->vlan_features |= NETIF_F_HIGHDMA;
2051 }
9d5c8243 2052
ac52caa3
MM		 2053 if (hw->mac.type >= e1000_82576) {
2054 netdev->hw_features |= NETIF_F_SCTP_CSUM;
b9473560 2055 netdev->features |= NETIF_F_SCTP_CSUM;
ac52caa3 2056 }
b9473560 2057
01789349
JP		 2058 netdev->priv_flags |= IFF_UNICAST_FLT;
2059
330a6d6a 2060 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 2061
2062 /* before reading the NVM, reset the controller to put the device in a
2063 * known good starting state */
2064 hw->mac.ops.reset_hw(hw);
2065
f96a8a0b
CW		 2066 /*
2067 * make sure the NVM is good , i211 parts have special NVM that
2068 * doesn't contain a checksum
2069 */
2070 if (hw->mac.type != e1000_i211) {
2071 if (hw->nvm.ops.validate(hw) < 0) {
2072 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
2073 err = -EIO;
2074 goto err_eeprom;
2075 }
9d5c8243
AK		 2076 }
2077
2078 /* copy the MAC address out of the NVM */
2079 if (hw->mac.ops.read_mac_addr(hw))
2080 dev_err(&pdev->dev, "NVM Read Error\n");
2081
2082 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2083 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2084
2085 if (!is_valid_ether_addr(netdev->perm_addr)) {
2086 dev_err(&pdev->dev, "Invalid MAC Address\n");
2087 err = -EIO;
2088 goto err_eeprom;
2089 }
2090
d67974f0
CW		 2091 /* get firmware version for ethtool -i */
2092 igb_set_fw_version(adapter);
2093
c061b18d 2094 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 2095 (unsigned long) adapter);
c061b18d 2096 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 2097 (unsigned long) adapter);
9d5c8243
AK		 2098
2099 INIT_WORK(&adapter->reset_task, igb_reset_task);
2100 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2101
450c87c8 2102 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 2103 adapter->fc_autoneg = true;
2104 hw->mac.autoneg = true;
2105 hw->phy.autoneg_advertised = 0x2f;
2106
0cce119a
AD		 2107 hw->fc.requested_mode = e1000_fc_default;
2108 hw->fc.current_mode = e1000_fc_default;
9d5c8243 2109
9d5c8243
AK		 2110 igb_validate_mdi_setting(hw);
2111
9d5c8243
AK		 2112 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2113 * enable the ACPI Magic Packet filter
2114 */
2115
a2cf8b6c 2116 if (hw->bus.func == 0)
312c75ae 2117 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
6d337dce 2118 else if (hw->mac.type >= e1000_82580)
55cac248
AD		 2119 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2120 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2121 &eeprom_data);
a2cf8b6c
AD		 2122 else if (hw->bus.func == 1)
2123 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 2124
2125 if (eeprom_data & eeprom_apme_mask)
2126 adapter->eeprom_wol |= E1000_WUFC_MAG;
2127
2128 /* now that we have the eeprom settings, apply the special cases where
2129 * the eeprom may be wrong or the board simply won't support wake on
2130 * lan on a particular port */
2131 switch (pdev->device) {
2132 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2133 adapter->eeprom_wol = 0;
2134 break;
2135 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 2136 case E1000_DEV_ID_82576_FIBER:
2137 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 2138 /* Wake events only supported on port A for dual fiber
2139 * regardless of eeprom setting */
2140 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2141 adapter->eeprom_wol = 0;
2142 break;
c8ea5ea9 2143 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 2144 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 2145 /* if quad port adapter, disable WoL on all but port A */
2146 if (global_quad_port_a != 0)
2147 adapter->eeprom_wol = 0;
2148 else
2149 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2150 /* Reset for multiple quad port adapters */
2151 if (++global_quad_port_a == 4)
2152 global_quad_port_a = 0;
2153 break;
9d5c8243
AK		 2154 }
2155
2156 /* initialize the wol settings based on the eeprom settings */
2157 adapter->wol = adapter->eeprom_wol;
e1b86d84 2158 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 2159
2160 /* reset the hardware with the new settings */
2161 igb_reset(adapter);
2162
2163 /* let the f/w know that the h/w is now under the control of the
2164 * driver. */
2165 igb_get_hw_control(adapter);
2166
9d5c8243
AK		 2167 strcpy(netdev->name, "eth%d");
2168 err = register_netdev(netdev);
2169 if (err)
2170 goto err_register;
2171
b168dfc5
JB		 2172 /* carrier off reporting is important to ethtool even BEFORE open */
2173 netif_carrier_off(netdev);
2174
421e02f0 2175#ifdef CONFIG_IGB_DCA
bbd98fe4 2176 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 2177 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 2178 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 2179 igb_setup_dca(adapter);
2180 }
fe4506b6 2181
38c845c7 2182#endif
3c89f6d0 2183
7ebae817 2184#ifdef CONFIG_IGB_PTP
673b8b70 2185 /* do hw tstamp init after resetting */
7ebae817 2186 igb_ptp_init(adapter);
3c89f6d0 2187#endif /* CONFIG_IGB_PTP */
673b8b70 2188
9d5c8243
AK		 2189 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2190 /* print bus type/speed/width info */
7c510e4b 2191 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 2192 netdev->name,
559e9c49 2193 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 2194 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 2195 "unknown"),
59c3de89
AD		 2196 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2197 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2198 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2199 "unknown"),
7c510e4b 2200 netdev->dev_addr);
9d5c8243 2201
9835fd73
CW		 2202 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2203 if (ret_val)
2204 strcpy(part_str, "Unknown");
2205 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
9d5c8243
AK		 2206 dev_info(&pdev->dev,
2207 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2208 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2209 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243 2210 adapter->num_rx_queues, adapter->num_tx_queues);
09b068d4
CW		 2211 switch (hw->mac.type) {
2212 case e1000_i350:
f96a8a0b
CW		 2213 case e1000_i210:
2214 case e1000_i211:
09b068d4
CW		 2215 igb_set_eee_i350(hw);
2216 break;
2217 default:
2218 break;
2219 }
749ab2cd
YZ		 2220
2221 pm_runtime_put_noidle(&pdev->dev);
9d5c8243
AK		 2222 return 0;
2223
2224err_register:
2225 igb_release_hw_control(adapter);
2226err_eeprom:
2227 if (!igb_check_reset_block(hw))
f5f4cf08 2228 igb_reset_phy(hw);
9d5c8243
AK		 2229
2230 if (hw->flash_address)
2231 iounmap(hw->flash_address);
9d5c8243 2232err_sw_init:
047e0030 2233 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2234 iounmap(hw->hw_addr);
2235err_ioremap:
2236 free_netdev(netdev);
2237err_alloc_etherdev:
559e9c49
AD		 2238 pci_release_selected_regions(pdev,
2239 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2240err_pci_reg:
2241err_dma:
2242 pci_disable_device(pdev);
2243 return err;
2244}
2245
2246/**
2247 * igb_remove - Device Removal Routine
2248 * @pdev: PCI device information struct
2249 *
2250 * igb_remove is called by the PCI subsystem to alert the driver
2251 * that it should release a PCI device. The could be caused by a
2252 * Hot-Plug event, or because the driver is going to be removed from
2253 * memory.
2254 **/
2255static void __devexit igb_remove(struct pci_dev *pdev)
2256{
2257 struct net_device *netdev = pci_get_drvdata(pdev);
2258 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2259 struct e1000_hw *hw = &adapter->hw;
9d5c8243 2260
749ab2cd 2261 pm_runtime_get_noresume(&pdev->dev);
7ebae817
RC		 2262#ifdef CONFIG_IGB_PTP
2263 igb_ptp_remove(adapter);
3c89f6d0 2264#endif /* CONFIG_IGB_PTP */
749ab2cd 2265
760141a5
TH		 2266 /*
2267 * The watchdog timer may be rescheduled, so explicitly
2268 * disable watchdog from being rescheduled.
2269 */
9d5c8243
AK		 2270 set_bit(__IGB_DOWN, &adapter->state);
2271 del_timer_sync(&adapter->watchdog_timer);
2272 del_timer_sync(&adapter->phy_info_timer);
2273
760141a5
TH		 2274 cancel_work_sync(&adapter->reset_task);
2275 cancel_work_sync(&adapter->watchdog_task);
9d5c8243 2276
421e02f0 2277#ifdef CONFIG_IGB_DCA
7dfc16fa 2278 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2279 dev_info(&pdev->dev, "DCA disabled\n");
2280 dca_remove_requester(&pdev->dev);
7dfc16fa 2281 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2282 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2283 }
2284#endif
2285
9d5c8243
AK		 2286 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2287 * would have already happened in close and is redundant. */
2288 igb_release_hw_control(adapter);
2289
2290 unregister_netdev(netdev);
2291
047e0030 2292 igb_clear_interrupt_scheme(adapter);
9d5c8243 2293
37680117
AD		 2294#ifdef CONFIG_PCI_IOV
2295 /* reclaim resources allocated to VFs */
2296 if (adapter->vf_data) {
2297 /* disable iov and allow time for transactions to clear */
0224d663
GR		 2298 if (!igb_check_vf_assignment(adapter)) {
2299 pci_disable_sriov(pdev);
2300 msleep(500);
2301 } else {
2302 dev_info(&pdev->dev, "VF(s) assigned to guests!\n");
2303 }
37680117
AD		 2304
2305 kfree(adapter->vf_data);
2306 adapter->vf_data = NULL;
2307 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 2308 wrfl();
37680117
AD		 2309 msleep(100);
2310 dev_info(&pdev->dev, "IOV Disabled\n");
2311 }
2312#endif
559e9c49 2313
28b0759c
AD		 2314 iounmap(hw->hw_addr);
2315 if (hw->flash_address)
2316 iounmap(hw->flash_address);
559e9c49
AD		 2317 pci_release_selected_regions(pdev,
2318 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243 2319
1128c756 2320 kfree(adapter->shadow_vfta);
9d5c8243
AK		 2321 free_netdev(netdev);
2322
19d5afd4 2323 pci_disable_pcie_error_reporting(pdev);
40a914fa 2324
9d5c8243
AK		 2325 pci_disable_device(pdev);
2326}
2327
a6b623e0
AD		 2328/**
2329 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2330 * @adapter: board private structure to initialize
2331 *
2332 * This function initializes the vf specific data storage and then attempts to
2333 * allocate the VFs. The reason for ordering it this way is because it is much
2334 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2335 * the memory for the VFs.
2336 **/
2337static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2338{
2339#ifdef CONFIG_PCI_IOV
2340 struct pci_dev *pdev = adapter->pdev;
f96a8a0b 2341 struct e1000_hw *hw = &adapter->hw;
0224d663
GR		 2342 int old_vfs = igb_find_enabled_vfs(adapter);
2343 int i;
a6b623e0 2344
f96a8a0b
CW		 2345 /* Virtualization features not supported on i210 family. */
2346 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
2347 return;
2348
0224d663
GR		 2349 if (old_vfs) {
2350 dev_info(&pdev->dev, "%d pre-allocated VFs found - override "
2351 "max_vfs setting of %d\n", old_vfs, max_vfs);
2352 adapter->vfs_allocated_count = old_vfs;
a6b623e0
AD		 2353 }
2354
0224d663
GR		 2355 if (!adapter->vfs_allocated_count)
2356 return;
2357
2358 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2359 sizeof(struct vf_data_storage), GFP_KERNEL);
f96a8a0b 2360
0224d663
GR		 2361 /* if allocation failed then we do not support SR-IOV */
2362 if (!adapter->vf_data) {
a6b623e0 2363 adapter->vfs_allocated_count = 0;
0224d663
GR		 2364 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2365 "Data Storage\n");
2366 goto out;
a6b623e0 2367 }
0224d663
GR		 2368
2369 if (!old_vfs) {
2370 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count))
2371 goto err_out;
2372 }
2373 dev_info(&pdev->dev, "%d VFs allocated\n",
2374 adapter->vfs_allocated_count);
2375 for (i = 0; i < adapter->vfs_allocated_count; i++)
2376 igb_vf_configure(adapter, i);
2377
2378 /* DMA Coalescing is not supported in IOV mode. */
2379 adapter->flags &= ~IGB_FLAG_DMAC;
2380 goto out;
2381err_out:
2382 kfree(adapter->vf_data);
2383 adapter->vf_data = NULL;
2384 adapter->vfs_allocated_count = 0;
2385out:
2386 return;
a6b623e0
AD		 2387#endif /* CONFIG_PCI_IOV */
2388}
2389
9d5c8243
AK		 2390/**
2391 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2392 * @adapter: board private structure to initialize
2393 *
2394 * igb_sw_init initializes the Adapter private data structure.
2395 * Fields are initialized based on PCI device information and
2396 * OS network device settings (MTU size).
2397 **/
2398static int __devinit igb_sw_init(struct igb_adapter *adapter)
2399{
2400 struct e1000_hw *hw = &adapter->hw;
2401 struct net_device *netdev = adapter->netdev;
2402 struct pci_dev *pdev = adapter->pdev;
374a542d 2403 u32 max_rss_queues;
9d5c8243
AK		 2404
2405 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2406
13fde97a 2407 /* set default ring sizes */
68fd9910
AD		 2408 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2409 adapter->rx_ring_count = IGB_DEFAULT_RXD;
13fde97a
AD		 2410
2411 /* set default ITR values */
4fc82adf
AD		 2412 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2413 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2414
13fde97a
AD		 2415 /* set default work limits */
2416 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2417
153285f9
AD		 2418 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2419 VLAN_HLEN;
9d5c8243
AK		 2420 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2421
81c2fc22
AD		 2422 adapter->node = -1;
2423
12dcd86b 2424 spin_lock_init(&adapter->stats64_lock);
a6b623e0 2425#ifdef CONFIG_PCI_IOV
6b78bb1d
CW		 2426 switch (hw->mac.type) {
2427 case e1000_82576:
2428 case e1000_i350:
9b082d73
SA		 2429 if (max_vfs > 7) {
2430 dev_warn(&pdev->dev,
2431 "Maximum of 7 VFs per PF, using max\n");
2432 adapter->vfs_allocated_count = 7;
2433 } else
2434 adapter->vfs_allocated_count = max_vfs;
6b78bb1d
CW		 2435 break;
2436 default:
2437 break;
2438 }
a6b623e0 2439#endif /* CONFIG_PCI_IOV */
374a542d
MV		 2440
2441 /* Determine the maximum number of RSS queues supported. */
f96a8a0b 2442 switch (hw->mac.type) {
374a542d
MV		 2443 case e1000_i211:
2444 max_rss_queues = IGB_MAX_RX_QUEUES_I211;
2445 break;
2446 case e1000_82575:
f96a8a0b 2447 case e1000_i210:
374a542d
MV		 2448 max_rss_queues = IGB_MAX_RX_QUEUES_82575;
2449 break;
2450 case e1000_i350:
2451 /* I350 cannot do RSS and SR-IOV at the same time */
2452 if (!!adapter->vfs_allocated_count) {
2453 max_rss_queues = 1;
2454 break;
2455 }
2456 /* fall through */
2457 case e1000_82576:
2458 if (!!adapter->vfs_allocated_count) {
2459 max_rss_queues = 2;
2460 break;
2461 }
2462 /* fall through */
2463 case e1000_82580:
2464 default:
2465 max_rss_queues = IGB_MAX_RX_QUEUES;
f96a8a0b 2466 break;
374a542d
MV		 2467 }
2468
2469 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
2470
2471 /* Determine if we need to pair queues. */
2472 switch (hw->mac.type) {
2473 case e1000_82575:
f96a8a0b 2474 case e1000_i211:
374a542d 2475 /* Device supports enough interrupts without queue pairing. */
f96a8a0b 2476 break;
374a542d
MV		 2477 case e1000_82576:
2478 /*
2479 * If VFs are going to be allocated with RSS queues then we
2480 * should pair the queues in order to conserve interrupts due
2481 * to limited supply.
2482 */
2483 if ((adapter->rss_queues > 1) &&
2484 (adapter->vfs_allocated_count > 6))
2485 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2486 /* fall through */
2487 case e1000_82580:
2488 case e1000_i350:
2489 case e1000_i210:
f96a8a0b 2490 default:
374a542d
MV		 2491 /*
2492 * If rss_queues > half of max_rss_queues, pair the queues in
2493 * order to conserve interrupts due to limited supply.
2494 */
2495 if (adapter->rss_queues > (max_rss_queues / 2))
2496 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
f96a8a0b
CW		 2497 break;
2498 }
a99955fc 2499
1128c756
CW		 2500 /* Setup and initialize a copy of the hw vlan table array */
2501 adapter->shadow_vfta = kzalloc(sizeof(u32) *
2502 E1000_VLAN_FILTER_TBL_SIZE,
2503 GFP_ATOMIC);
2504
a6b623e0 2505 /* This call may decrease the number of queues */
047e0030 2506 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2507 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2508 return -ENOMEM;
2509 }
2510
a6b623e0
AD		 2511 igb_probe_vfs(adapter);
2512
9d5c8243
AK		 2513 /* Explicitly disable IRQ since the NIC can be in any state. */
2514 igb_irq_disable(adapter);
2515
f96a8a0b 2516 if (hw->mac.type >= e1000_i350)
831ec0b4
CW		 2517 adapter->flags &= ~IGB_FLAG_DMAC;
2518
9d5c8243
AK		 2519 set_bit(__IGB_DOWN, &adapter->state);
2520 return 0;
2521}
2522
2523/**
2524 * igb_open - Called when a network interface is made active
2525 * @netdev: network interface device structure
2526 *
2527 * Returns 0 on success, negative value on failure
2528 *
2529 * The open entry point is called when a network interface is made
2530 * active by the system (IFF_UP). At this point all resources needed
2531 * for transmit and receive operations are allocated, the interrupt
2532 * handler is registered with the OS, the watchdog timer is started,
2533 * and the stack is notified that the interface is ready.
2534 **/
749ab2cd 2535static int __igb_open(struct net_device *netdev, bool resuming)
9d5c8243
AK		 2536{
2537 struct igb_adapter *adapter = netdev_priv(netdev);
2538 struct e1000_hw *hw = &adapter->hw;
749ab2cd 2539 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2540 int err;
2541 int i;
2542
2543 /* disallow open during test */
749ab2cd
YZ		 2544 if (test_bit(__IGB_TESTING, &adapter->state)) {
2545 WARN_ON(resuming);
9d5c8243 2546 return -EBUSY;
749ab2cd
YZ		 2547 }
2548
2549 if (!resuming)
2550 pm_runtime_get_sync(&pdev->dev);
9d5c8243 2551
b168dfc5
JB		 2552 netif_carrier_off(netdev);
2553
9d5c8243
AK		 2554 /* allocate transmit descriptors */
2555 err = igb_setup_all_tx_resources(adapter);
2556 if (err)
2557 goto err_setup_tx;
2558
2559 /* allocate receive descriptors */
2560 err = igb_setup_all_rx_resources(adapter);
2561 if (err)
2562 goto err_setup_rx;
2563
88a268c1 2564 igb_power_up_link(adapter);
9d5c8243 2565
9d5c8243
AK		 2566 /* before we allocate an interrupt, we must be ready to handle it.
2567 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2568 * as soon as we call pci_request_irq, so we have to setup our
2569 * clean_rx handler before we do so. */
2570 igb_configure(adapter);
2571
2572 err = igb_request_irq(adapter);
2573 if (err)
2574 goto err_req_irq;
2575
2576 /* From here on the code is the same as igb_up() */
2577 clear_bit(__IGB_DOWN, &adapter->state);
2578
0d1ae7f4
AD		 2579 for (i = 0; i < adapter->num_q_vectors; i++)
2580 napi_enable(&(adapter->q_vector[i]->napi));
9d5c8243
AK		 2581
2582 /* Clear any pending interrupts. */
2583 rd32(E1000_ICR);
844290e5
PW		 2584
2585 igb_irq_enable(adapter);
2586
d4960307
AD		 2587 /* notify VFs that reset has been completed */
2588 if (adapter->vfs_allocated_count) {
2589 u32 reg_data = rd32(E1000_CTRL_EXT);
2590 reg_data |= E1000_CTRL_EXT_PFRSTD;
2591 wr32(E1000_CTRL_EXT, reg_data);
2592 }
2593
d55b53ff
JK		 2594 netif_tx_start_all_queues(netdev);
2595
749ab2cd
YZ		 2596 if (!resuming)
2597 pm_runtime_put(&pdev->dev);
2598
25568a53
AD		 2599 /* start the watchdog. */
2600 hw->mac.get_link_status = 1;
2601 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2602
2603 return 0;
2604
2605err_req_irq:
2606 igb_release_hw_control(adapter);
88a268c1 2607 igb_power_down_link(adapter);
9d5c8243
AK		 2608 igb_free_all_rx_resources(adapter);
2609err_setup_rx:
2610 igb_free_all_tx_resources(adapter);
2611err_setup_tx:
2612 igb_reset(adapter);
749ab2cd
YZ		 2613 if (!resuming)
2614 pm_runtime_put(&pdev->dev);
9d5c8243
AK		 2615
2616 return err;
2617}
2618
749ab2cd
YZ		 2619static int igb_open(struct net_device *netdev)
2620{
2621 return __igb_open(netdev, false);
2622}
2623
9d5c8243
AK		 2624/**
2625 * igb_close - Disables a network interface
2626 * @netdev: network interface device structure
2627 *
2628 * Returns 0, this is not allowed to fail
2629 *
2630 * The close entry point is called when an interface is de-activated
2631 * by the OS. The hardware is still under the driver's control, but
2632 * needs to be disabled. A global MAC reset is issued to stop the
2633 * hardware, and all transmit and receive resources are freed.
2634 **/
749ab2cd 2635static int __igb_close(struct net_device *netdev, bool suspending)
9d5c8243
AK		 2636{
2637 struct igb_adapter *adapter = netdev_priv(netdev);
749ab2cd 2638 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2639
2640 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
9d5c8243 2641
749ab2cd
YZ		 2642 if (!suspending)
2643 pm_runtime_get_sync(&pdev->dev);
2644
2645 igb_down(adapter);
9d5c8243
AK		 2646 igb_free_irq(adapter);
2647
2648 igb_free_all_tx_resources(adapter);
2649 igb_free_all_rx_resources(adapter);
2650
749ab2cd
YZ		 2651 if (!suspending)
2652 pm_runtime_put_sync(&pdev->dev);
9d5c8243
AK		 2653 return 0;
2654}
2655
749ab2cd
YZ		 2656static int igb_close(struct net_device *netdev)
2657{
2658 return __igb_close(netdev, false);
2659}
2660
9d5c8243
AK		 2661/**
2662 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2663 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2664 *
2665 * Return 0 on success, negative on failure
2666 **/
80785298 2667int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2668{
59d71989 2669 struct device *dev = tx_ring->dev;
81c2fc22 2670 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2671 int size;
2672
06034649 2673 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
81c2fc22
AD		 2674 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2675 if (!tx_ring->tx_buffer_info)
2676 tx_ring->tx_buffer_info = vzalloc(size);
06034649 2677 if (!tx_ring->tx_buffer_info)
9d5c8243 2678 goto err;
9d5c8243
AK		 2679
2680 /* round up to nearest 4K */
85e8d004 2681 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2682 tx_ring->size = ALIGN(tx_ring->size, 4096);
2683
81c2fc22 2684 set_dev_node(dev, tx_ring->numa_node);
59d71989
AD		 2685 tx_ring->desc = dma_alloc_coherent(dev,
2686 tx_ring->size,
2687 &tx_ring->dma,
2688 GFP_KERNEL);
81c2fc22
AD		 2689 set_dev_node(dev, orig_node);
2690 if (!tx_ring->desc)
2691 tx_ring->desc = dma_alloc_coherent(dev,
2692 tx_ring->size,
2693 &tx_ring->dma,
2694 GFP_KERNEL);
9d5c8243
AK		 2695
2696 if (!tx_ring->desc)
2697 goto err;
2698
9d5c8243
AK		 2699 tx_ring->next_to_use = 0;
2700 tx_ring->next_to_clean = 0;
81c2fc22 2701
9d5c8243
AK		 2702 return 0;
2703
2704err:
06034649 2705 vfree(tx_ring->tx_buffer_info);
59d71989 2706 dev_err(dev,
9d5c8243
AK		 2707 "Unable to allocate memory for the transmit descriptor ring\n");
2708 return -ENOMEM;
2709}
2710
2711/**
2712 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2713 * (Descriptors) for all queues
2714 * @adapter: board private structure
2715 *
2716 * Return 0 on success, negative on failure
2717 **/
2718static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2719{
439705e1 2720 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2721 int i, err = 0;
2722
2723 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2724 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2725 if (err) {
439705e1 2726 dev_err(&pdev->dev,
9d5c8243
AK		 2727 "Allocation for Tx Queue %u failed\n", i);
2728 for (i--; i >= 0; i--)
3025a446 2729 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2730 break;
2731 }
2732 }
2733
2734 return err;
2735}
2736
2737/**
85b430b4
AD		 2738 * igb_setup_tctl - configure the transmit control registers
2739 * @adapter: Board private structure
9d5c8243 2740 **/
d7ee5b3a 2741void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2742{
9d5c8243
AK		 2743 struct e1000_hw *hw = &adapter->hw;
2744 u32 tctl;
9d5c8243 2745
85b430b4
AD		 2746 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2747 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2748
2749 /* Program the Transmit Control Register */
9d5c8243
AK		 2750 tctl = rd32(E1000_TCTL);
2751 tctl &= ~E1000_TCTL_CT;
2752 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2753 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2754
2755 igb_config_collision_dist(hw);
2756
9d5c8243
AK		 2757 /* Enable transmits */
2758 tctl |= E1000_TCTL_EN;
2759
2760 wr32(E1000_TCTL, tctl);
2761}
2762
85b430b4
AD		 2763/**
2764 * igb_configure_tx_ring - Configure transmit ring after Reset
2765 * @adapter: board private structure
2766 * @ring: tx ring to configure
2767 *
2768 * Configure a transmit ring after a reset.
2769 **/
d7ee5b3a
AD		 2770void igb_configure_tx_ring(struct igb_adapter *adapter,
2771 struct igb_ring *ring)
85b430b4
AD		 2772{
2773 struct e1000_hw *hw = &adapter->hw;
a74420e0 2774 u32 txdctl = 0;
85b430b4
AD		 2775 u64 tdba = ring->dma;
2776 int reg_idx = ring->reg_idx;
2777
2778 /* disable the queue */
a74420e0 2779 wr32(E1000_TXDCTL(reg_idx), 0);
85b430b4
AD		 2780 wrfl();
2781 mdelay(10);
2782
2783 wr32(E1000_TDLEN(reg_idx),
2784 ring->count * sizeof(union e1000_adv_tx_desc));
2785 wr32(E1000_TDBAL(reg_idx),
2786 tdba & 0x00000000ffffffffULL);
2787 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2788
fce99e34 2789 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
a74420e0 2790 wr32(E1000_TDH(reg_idx), 0);
fce99e34 2791 writel(0, ring->tail);
85b430b4
AD		 2792
2793 txdctl |= IGB_TX_PTHRESH;
2794 txdctl |= IGB_TX_HTHRESH << 8;
2795 txdctl |= IGB_TX_WTHRESH << 16;
2796
2797 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2798 wr32(E1000_TXDCTL(reg_idx), txdctl);
2799}
2800
2801/**
2802 * igb_configure_tx - Configure transmit Unit after Reset
2803 * @adapter: board private structure
2804 *
2805 * Configure the Tx unit of the MAC after a reset.
2806 **/
2807static void igb_configure_tx(struct igb_adapter *adapter)
2808{
2809 int i;
2810
2811 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2812 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2813}
2814
9d5c8243
AK		 2815/**
2816 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2817 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2818 *
2819 * Returns 0 on success, negative on failure
2820 **/
80785298 2821int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2822{
59d71989 2823 struct device *dev = rx_ring->dev;
81c2fc22 2824 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2825 int size, desc_len;
2826
06034649 2827 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
81c2fc22
AD		 2828 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
2829 if (!rx_ring->rx_buffer_info)
2830 rx_ring->rx_buffer_info = vzalloc(size);
06034649 2831 if (!rx_ring->rx_buffer_info)
9d5c8243 2832 goto err;
9d5c8243
AK		 2833
2834 desc_len = sizeof(union e1000_adv_rx_desc);
2835
2836 /* Round up to nearest 4K */
2837 rx_ring->size = rx_ring->count * desc_len;
2838 rx_ring->size = ALIGN(rx_ring->size, 4096);
2839
81c2fc22 2840 set_dev_node(dev, rx_ring->numa_node);
59d71989
AD		 2841 rx_ring->desc = dma_alloc_coherent(dev,
2842 rx_ring->size,
2843 &rx_ring->dma,
2844 GFP_KERNEL);
81c2fc22
AD		 2845 set_dev_node(dev, orig_node);
2846 if (!rx_ring->desc)
2847 rx_ring->desc = dma_alloc_coherent(dev,
2848 rx_ring->size,
2849 &rx_ring->dma,
2850 GFP_KERNEL);
9d5c8243
AK		 2851
2852 if (!rx_ring->desc)
2853 goto err;
2854
2855 rx_ring->next_to_clean = 0;
2856 rx_ring->next_to_use = 0;
9d5c8243 2857
9d5c8243
AK		 2858 return 0;
2859
2860err:
06034649
AD		 2861 vfree(rx_ring->rx_buffer_info);
2862 rx_ring->rx_buffer_info = NULL;
59d71989
AD		 2863 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2864 " ring\n");
9d5c8243
AK		 2865 return -ENOMEM;
2866}
2867
2868/**
2869 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2870 * (Descriptors) for all queues
2871 * @adapter: board private structure
2872 *
2873 * Return 0 on success, negative on failure
2874 **/
2875static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2876{
439705e1 2877 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2878 int i, err = 0;
2879
2880 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2881 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2882 if (err) {
439705e1 2883 dev_err(&pdev->dev,
9d5c8243
AK		 2884 "Allocation for Rx Queue %u failed\n", i);
2885 for (i--; i >= 0; i--)
3025a446 2886 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2887 break;
2888 }
2889 }
2890
2891 return err;
2892}
2893
06cf2666
AD		 2894/**
2895 * igb_setup_mrqc - configure the multiple receive queue control registers
2896 * @adapter: Board private structure
2897 **/
2898static void igb_setup_mrqc(struct igb_adapter *adapter)
2899{
2900 struct e1000_hw *hw = &adapter->hw;
2901 u32 mrqc, rxcsum;
2902 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2903 union e1000_reta {
2904 u32 dword;
2905 u8 bytes[4];
2906 } reta;
2907 static const u8 rsshash[40] = {
2908 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2909 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2910 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2911 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2912
2913 /* Fill out hash function seeds */
2914 for (j = 0; j < 10; j++) {
2915 u32 rsskey = rsshash[(j * 4)];
2916 rsskey |= rsshash[(j * 4) + 1] << 8;
2917 rsskey |= rsshash[(j * 4) + 2] << 16;
2918 rsskey |= rsshash[(j * 4) + 3] << 24;
2919 array_wr32(E1000_RSSRK(0), j, rsskey);
2920 }
2921
a99955fc 2922 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2923
2924 if (adapter->vfs_allocated_count) {
2925 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2926 switch (hw->mac.type) {
d2ba2ed8 2927 case e1000_i350:
55cac248
AD		 2928 case e1000_82580:
2929 num_rx_queues = 1;
2930 shift = 0;
2931 break;
06cf2666
AD		 2932 case e1000_82576:
2933 shift = 3;
2934 num_rx_queues = 2;
2935 break;
2936 case e1000_82575:
2937 shift = 2;
2938 shift2 = 6;
2939 default:
2940 break;
2941 }
2942 } else {
2943 if (hw->mac.type == e1000_82575)
2944 shift = 6;
2945 }
2946
2947 for (j = 0; j < (32 * 4); j++) {
2948 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2949 if (shift2)
2950 reta.bytes[j & 3] |= num_rx_queues << shift2;
2951 if ((j & 3) == 3)
2952 wr32(E1000_RETA(j >> 2), reta.dword);
2953 }
2954
2955 /*
2956 * Disable raw packet checksumming so that RSS hash is placed in
2957 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2958 * offloads as they are enabled by default
2959 */
2960 rxcsum = rd32(E1000_RXCSUM);
2961 rxcsum |= E1000_RXCSUM_PCSD;
2962
2963 if (adapter->hw.mac.type >= e1000_82576)
2964 /* Enable Receive Checksum Offload for SCTP */
2965 rxcsum |= E1000_RXCSUM_CRCOFL;
2966
2967 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2968 wr32(E1000_RXCSUM, rxcsum);
f96a8a0b
CW		 2969 /*
2970 * Generate RSS hash based on TCP port numbers and/or
2971 * IPv4/v6 src and dst addresses since UDP cannot be
2972 * hashed reliably due to IP fragmentation
2973 */
2974
2975 mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
2976 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2977 E1000_MRQC_RSS_FIELD_IPV6 |
2978 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2979 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2980
2981 /* If VMDq is enabled then we set the appropriate mode for that, else
2982 * we default to RSS so that an RSS hash is calculated per packet even
2983 * if we are only using one queue */
2984 if (adapter->vfs_allocated_count) {
2985 if (hw->mac.type > e1000_82575) {
2986 /* Set the default pool for the PF's first queue */
2987 u32 vtctl = rd32(E1000_VT_CTL);
2988 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2989 E1000_VT_CTL_DISABLE_DEF_POOL);
2990 vtctl |= adapter->vfs_allocated_count <<
2991 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2992 wr32(E1000_VT_CTL, vtctl);
2993 }
a99955fc 2994 if (adapter->rss_queues > 1)
f96a8a0b 2995 mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
06cf2666 2996 else
f96a8a0b 2997 mrqc |= E1000_MRQC_ENABLE_VMDQ;
06cf2666 2998 } else {
f96a8a0b
CW		 2999 if (hw->mac.type != e1000_i211)
3000 mrqc |= E1000_MRQC_ENABLE_RSS_4Q;
06cf2666
AD		 3001 }
3002 igb_vmm_control(adapter);
3003
06cf2666
AD		 3004 wr32(E1000_MRQC, mrqc);
3005}
3006
9d5c8243
AK		 3007/**
3008 * igb_setup_rctl - configure the receive control registers
3009 * @adapter: Board private structure
3010 **/
d7ee5b3a 3011void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 3012{
3013 struct e1000_hw *hw = &adapter->hw;
3014 u32 rctl;
9d5c8243
AK		 3015
3016 rctl = rd32(E1000_RCTL);
3017
3018 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 3019 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 3020
69d728ba 3021 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 3022 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 3023
87cb7e8c
AK		 3024 /*
3025 * enable stripping of CRC. It's unlikely this will break BMC
3026 * redirection as it did with e1000. Newer features require
3027 * that the HW strips the CRC.
73cd78f1 3028 */
87cb7e8c 3029 rctl |= E1000_RCTL_SECRC;
9d5c8243 3030
559e9c49 3031 /* disable store bad packets and clear size bits. */
ec54d7d6 3032 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 3033
6ec43fe6
AD		 3034 /* enable LPE to prevent packets larger than max_frame_size */
3035 rctl |= E1000_RCTL_LPE;
9d5c8243 3036
952f72a8
AD		 3037 /* disable queue 0 to prevent tail write w/o re-config */
3038 wr32(E1000_RXDCTL(0), 0);
9d5c8243 3039
e1739522
AD		 3040 /* Attention!!! For SR-IOV PF driver operations you must enable
3041 * queue drop for all VF and PF queues to prevent head of line blocking
3042 * if an un-trusted VF does not provide descriptors to hardware.
3043 */
3044 if (adapter->vfs_allocated_count) {
e1739522
AD		 3045 /* set all queue drop enable bits */
3046 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 3047 }
3048
89eaefb6
BG		 3049 /* This is useful for sniffing bad packets. */
3050 if (adapter->netdev->features & NETIF_F_RXALL) {
3051 /* UPE and MPE will be handled by normal PROMISC logic
3052 * in e1000e_set_rx_mode */
3053 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
3054 E1000_RCTL_BAM | /* RX All Bcast Pkts */
3055 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
3056
3057 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
3058 E1000_RCTL_DPF | /* Allow filtered pause */
3059 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
3060 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3061 * and that breaks VLANs.
3062 */
3063 }
3064
9d5c8243
AK		 3065 wr32(E1000_RCTL, rctl);
3066}
3067
7d5753f0
AD		 3068static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3069 int vfn)
3070{
3071 struct e1000_hw *hw = &adapter->hw;
3072 u32 vmolr;
3073
3074 /* if it isn't the PF check to see if VFs are enabled and
3075 * increase the size to support vlan tags */
3076 if (vfn < adapter->vfs_allocated_count &&
3077 adapter->vf_data[vfn].vlans_enabled)
3078 size += VLAN_TAG_SIZE;
3079
3080 vmolr = rd32(E1000_VMOLR(vfn));
3081 vmolr &= ~E1000_VMOLR_RLPML_MASK;
3082 vmolr |= size | E1000_VMOLR_LPE;
3083 wr32(E1000_VMOLR(vfn), vmolr);
3084
3085 return 0;
3086}
3087
e1739522
AD		 3088/**
3089 * igb_rlpml_set - set maximum receive packet size
3090 * @adapter: board private structure
3091 *
3092 * Configure maximum receivable packet size.
3093 **/
3094static void igb_rlpml_set(struct igb_adapter *adapter)
3095{
153285f9 3096 u32 max_frame_size = adapter->max_frame_size;
e1739522
AD		 3097 struct e1000_hw *hw = &adapter->hw;
3098 u16 pf_id = adapter->vfs_allocated_count;
3099
e1739522
AD		 3100 if (pf_id) {
3101 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
153285f9
AD		 3102 /*
3103 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3104 * to our max jumbo frame size, in case we need to enable
3105 * jumbo frames on one of the rings later.
3106 * This will not pass over-length frames into the default
3107 * queue because it's gated by the VMOLR.RLPML.
3108 */
7d5753f0 3109 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 3110 }
3111
3112 wr32(E1000_RLPML, max_frame_size);
3113}
3114
8151d294
WM		 3115static inline void igb_set_vmolr(struct igb_adapter *adapter,
3116 int vfn, bool aupe)
7d5753f0
AD		 3117{
3118 struct e1000_hw *hw = &adapter->hw;
3119 u32 vmolr;
3120
3121 /*
3122 * This register exists only on 82576 and newer so if we are older then
3123 * we should exit and do nothing
3124 */
3125 if (hw->mac.type < e1000_82576)
3126 return;
3127
3128 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 3129 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3130 if (aupe)
3131 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3132 else
3133 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 3134
3135 /* clear all bits that might not be set */
3136 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3137
a99955fc 3138 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 3139 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3140 /*
3141 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3142 * multicast packets
3143 */
3144 if (vfn <= adapter->vfs_allocated_count)
3145 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3146
3147 wr32(E1000_VMOLR(vfn), vmolr);
3148}
3149
85b430b4
AD		 3150/**
3151 * igb_configure_rx_ring - Configure a receive ring after Reset
3152 * @adapter: board private structure
3153 * @ring: receive ring to be configured
3154 *
3155 * Configure the Rx unit of the MAC after a reset.
3156 **/
d7ee5b3a
AD		 3157void igb_configure_rx_ring(struct igb_adapter *adapter,
3158 struct igb_ring *ring)
85b430b4
AD		 3159{
3160 struct e1000_hw *hw = &adapter->hw;
3161 u64 rdba = ring->dma;
3162 int reg_idx = ring->reg_idx;
a74420e0 3163 u32 srrctl = 0, rxdctl = 0;
85b430b4
AD		 3164
3165 /* disable the queue */
a74420e0 3166 wr32(E1000_RXDCTL(reg_idx), 0);
85b430b4
AD		 3167
3168 /* Set DMA base address registers */
3169 wr32(E1000_RDBAL(reg_idx),
3170 rdba & 0x00000000ffffffffULL);
3171 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3172 wr32(E1000_RDLEN(reg_idx),
3173 ring->count * sizeof(union e1000_adv_rx_desc));
3174
3175 /* initialize head and tail */
fce99e34 3176 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
a74420e0 3177 wr32(E1000_RDH(reg_idx), 0);
fce99e34 3178 writel(0, ring->tail);
85b430b4 3179
952f72a8 3180 /* set descriptor configuration */
44390ca6 3181 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
952f72a8 3182#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
44390ca6 3183 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3184#else
44390ca6 3185 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3186#endif
44390ca6 3187 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3c89f6d0 3188#ifdef CONFIG_IGB_PTP
06218a8d 3189 if (hw->mac.type >= e1000_82580)
757b77e2 3190 srrctl |= E1000_SRRCTL_TIMESTAMP;
3c89f6d0 3191#endif /* CONFIG_IGB_PTP */
e6bdb6fe
NN		 3192 /* Only set Drop Enable if we are supporting multiple queues */
3193 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3194 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 3195
3196 wr32(E1000_SRRCTL(reg_idx), srrctl);
3197
7d5753f0 3198 /* set filtering for VMDQ pools */
8151d294 3199 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 3200
85b430b4
AD		 3201 rxdctl |= IGB_RX_PTHRESH;
3202 rxdctl |= IGB_RX_HTHRESH << 8;
3203 rxdctl |= IGB_RX_WTHRESH << 16;
a74420e0
AD		 3204
3205 /* enable receive descriptor fetching */
3206 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
85b430b4
AD		 3207 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3208}
3209
9d5c8243
AK		 3210/**
3211 * igb_configure_rx - Configure receive Unit after Reset
3212 * @adapter: board private structure
3213 *
3214 * Configure the Rx unit of the MAC after a reset.
3215 **/
3216static void igb_configure_rx(struct igb_adapter *adapter)
3217{
9107584e 3218 int i;
9d5c8243 3219
68d480c4
AD		 3220 /* set UTA to appropriate mode */
3221 igb_set_uta(adapter);
3222
26ad9178
AD		 3223 /* set the correct pool for the PF default MAC address in entry 0 */
3224 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3225 adapter->vfs_allocated_count);
3226
06cf2666
AD		 3227 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3228 * the Base and Length of the Rx Descriptor Ring */
3229 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3230 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 3231}
3232
3233/**
3234 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 3235 * @tx_ring: Tx descriptor ring for a specific queue
3236 *
3237 * Free all transmit software resources
3238 **/
68fd9910 3239void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 3240{
3b644cf6 3241 igb_clean_tx_ring(tx_ring);
9d5c8243 3242
06034649
AD		 3243 vfree(tx_ring->tx_buffer_info);
3244 tx_ring->tx_buffer_info = NULL;
9d5c8243 3245
439705e1
AD		 3246 /* if not set, then don't free */
3247 if (!tx_ring->desc)
3248 return;
3249
59d71989
AD		 3250 dma_free_coherent(tx_ring->dev, tx_ring->size,
3251 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 3252
3253 tx_ring->desc = NULL;
3254}
3255
3256/**
3257 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3258 * @adapter: board private structure
3259 *
3260 * Free all transmit software resources
3261 **/
3262static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3263{
3264 int i;
3265
3266 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3267 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3268}
3269
ebe42d16
AD		 3270void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3271 struct igb_tx_buffer *tx_buffer)
3272{
3273 if (tx_buffer->skb) {
3274 dev_kfree_skb_any(tx_buffer->skb);
3275 if (tx_buffer->dma)
3276 dma_unmap_single(ring->dev,
3277 tx_buffer->dma,
3278 tx_buffer->length,
3279 DMA_TO_DEVICE);
3280 } else if (tx_buffer->dma) {
3281 dma_unmap_page(ring->dev,
3282 tx_buffer->dma,
3283 tx_buffer->length,
3284 DMA_TO_DEVICE);
3285 }
3286 tx_buffer->next_to_watch = NULL;
3287 tx_buffer->skb = NULL;
3288 tx_buffer->dma = 0;
3289 /* buffer_info must be completely set up in the transmit path */
9d5c8243
AK		 3290}
3291
3292/**
3293 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3294 * @tx_ring: ring to be cleaned
3295 **/
3b644cf6 3296static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243 3297{
06034649 3298 struct igb_tx_buffer *buffer_info;
9d5c8243 3299 unsigned long size;
6ad4edfc 3300 u16 i;
9d5c8243 3301
06034649 3302 if (!tx_ring->tx_buffer_info)
9d5c8243
AK		 3303 return;
3304 /* Free all the Tx ring sk_buffs */
3305
3306 for (i = 0; i < tx_ring->count; i++) {
06034649 3307 buffer_info = &tx_ring->tx_buffer_info[i];
80785298 3308 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3309 }
3310
dad8a3b3
JF		 3311 netdev_tx_reset_queue(txring_txq(tx_ring));
3312
06034649
AD		 3313 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3314 memset(tx_ring->tx_buffer_info, 0, size);
9d5c8243
AK		 3315
3316 /* Zero out the descriptor ring */
9d5c8243
AK		 3317 memset(tx_ring->desc, 0, tx_ring->size);
3318
3319 tx_ring->next_to_use = 0;
3320 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3321}
3322
3323/**
3324 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3325 * @adapter: board private structure
3326 **/
3327static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3328{
3329 int i;
3330
3331 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3332 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3333}
3334
3335/**
3336 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3337 * @rx_ring: ring to clean the resources from
3338 *
3339 * Free all receive software resources
3340 **/
68fd9910 3341void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3342{
3b644cf6 3343 igb_clean_rx_ring(rx_ring);
9d5c8243 3344
06034649
AD		 3345 vfree(rx_ring->rx_buffer_info);
3346 rx_ring->rx_buffer_info = NULL;
9d5c8243 3347
439705e1
AD		 3348 /* if not set, then don't free */
3349 if (!rx_ring->desc)
3350 return;
3351
59d71989
AD		 3352 dma_free_coherent(rx_ring->dev, rx_ring->size,
3353 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3354
3355 rx_ring->desc = NULL;
3356}
3357
3358/**
3359 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3360 * @adapter: board private structure
3361 *
3362 * Free all receive software resources
3363 **/
3364static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3365{
3366 int i;
3367
3368 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3369 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3370}
3371
3372/**
3373 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3374 * @rx_ring: ring to free buffers from
3375 **/
3b644cf6 3376static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243 3377{
9d5c8243 3378 unsigned long size;
c023cd88 3379 u16 i;
9d5c8243 3380
06034649 3381 if (!rx_ring->rx_buffer_info)
9d5c8243 3382 return;
439705e1 3383
9d5c8243
AK		 3384 /* Free all the Rx ring sk_buffs */
3385 for (i = 0; i < rx_ring->count; i++) {
06034649 3386 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
9d5c8243 3387 if (buffer_info->dma) {
59d71989 3388 dma_unmap_single(rx_ring->dev,
80785298 3389 buffer_info->dma,
44390ca6 3390 IGB_RX_HDR_LEN,
59d71989 3391 DMA_FROM_DEVICE);
9d5c8243
AK		 3392 buffer_info->dma = 0;
3393 }
3394
3395 if (buffer_info->skb) {
3396 dev_kfree_skb(buffer_info->skb);
3397 buffer_info->skb = NULL;
3398 }
6ec43fe6 3399 if (buffer_info->page_dma) {
59d71989 3400 dma_unmap_page(rx_ring->dev,
80785298 3401 buffer_info->page_dma,
6ec43fe6 3402 PAGE_SIZE / 2,
59d71989 3403 DMA_FROM_DEVICE);
6ec43fe6
AD		 3404 buffer_info->page_dma = 0;
3405 }
9d5c8243 3406 if (buffer_info->page) {
9d5c8243
AK		 3407 put_page(buffer_info->page);
3408 buffer_info->page = NULL;
bf36c1a0 3409 buffer_info->page_offset = 0;
9d5c8243
AK		 3410 }
3411 }
3412
06034649
AD		 3413 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3414 memset(rx_ring->rx_buffer_info, 0, size);
9d5c8243
AK		 3415
3416 /* Zero out the descriptor ring */
3417 memset(rx_ring->desc, 0, rx_ring->size);
3418
3419 rx_ring->next_to_clean = 0;
3420 rx_ring->next_to_use = 0;
9d5c8243
AK		 3421}
3422
3423/**
3424 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3425 * @adapter: board private structure
3426 **/
3427static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3428{
3429 int i;
3430
3431 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3432 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3433}
3434
3435/**
3436 * igb_set_mac - Change the Ethernet Address of the NIC
3437 * @netdev: network interface device structure
3438 * @p: pointer to an address structure
3439 *
3440 * Returns 0 on success, negative on failure
3441 **/
3442static int igb_set_mac(struct net_device *netdev, void *p)
3443{
3444 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3445 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3446 struct sockaddr *addr = p;
3447
3448 if (!is_valid_ether_addr(addr->sa_data))
3449 return -EADDRNOTAVAIL;
3450
3451 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3452 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3453
26ad9178
AD		 3454 /* set the correct pool for the new PF MAC address in entry 0 */
3455 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3456 adapter->vfs_allocated_count);
e1739522 3457
9d5c8243
AK		 3458 return 0;
3459}
3460
3461/**
68d480c4 3462 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3463 * @netdev: network interface device structure
3464 *
68d480c4
AD		 3465 * Writes multicast address list to the MTA hash table.
3466 * Returns: -ENOMEM on failure
3467 * 0 on no addresses written
3468 * X on writing X addresses to MTA
9d5c8243 3469 **/
68d480c4 3470static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3471{
3472 struct igb_adapter *adapter = netdev_priv(netdev);
3473 struct e1000_hw *hw = &adapter->hw;
22bedad3 3474 struct netdev_hw_addr *ha;
68d480c4 3475 u8 *mta_list;
9d5c8243
AK		 3476 int i;
3477
4cd24eaf 3478 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3479 /* nothing to program, so clear mc list */
3480 igb_update_mc_addr_list(hw, NULL, 0);
3481 igb_restore_vf_multicasts(adapter);
3482 return 0;
3483 }
9d5c8243 3484
4cd24eaf 3485 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3486 if (!mta_list)
3487 return -ENOMEM;
ff41f8dc 3488
68d480c4 3489 /* The shared function expects a packed array of only addresses. */
48e2f183 3490 i = 0;
22bedad3
JP		 3491 netdev_for_each_mc_addr(ha, netdev)
3492 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3493
68d480c4
AD		 3494 igb_update_mc_addr_list(hw, mta_list, i);
3495 kfree(mta_list);
3496
4cd24eaf 3497 return netdev_mc_count(netdev);
68d480c4
AD		 3498}
3499
3500/**
3501 * igb_write_uc_addr_list - write unicast addresses to RAR table
3502 * @netdev: network interface device structure
3503 *
3504 * Writes unicast address list to the RAR table.
3505 * Returns: -ENOMEM on failure/insufficient address space
3506 * 0 on no addresses written
3507 * X on writing X addresses to the RAR table
3508 **/
3509static int igb_write_uc_addr_list(struct net_device *netdev)
3510{
3511 struct igb_adapter *adapter = netdev_priv(netdev);
3512 struct e1000_hw *hw = &adapter->hw;
3513 unsigned int vfn = adapter->vfs_allocated_count;
3514 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3515 int count = 0;
3516
3517 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3518 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3519 return -ENOMEM;
9d5c8243 3520
32e7bfc4 3521 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3522 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3523
3524 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3525 if (!rar_entries)
3526 break;
26ad9178
AD		 3527 igb_rar_set_qsel(adapter, ha->addr,
3528 rar_entries--,
68d480c4
AD		 3529 vfn);
3530 count++;
ff41f8dc
AD		 3531 }
3532 }
3533 /* write the addresses in reverse order to avoid write combining */
3534 for (; rar_entries > 0 ; rar_entries--) {
3535 wr32(E1000_RAH(rar_entries), 0);
3536 wr32(E1000_RAL(rar_entries), 0);
3537 }
3538 wrfl();
3539
68d480c4
AD		 3540 return count;
3541}
3542
3543/**
3544 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3545 * @netdev: network interface device structure
3546 *
3547 * The set_rx_mode entry point is called whenever the unicast or multicast
3548 * address lists or the network interface flags are updated. This routine is
3549 * responsible for configuring the hardware for proper unicast, multicast,
3550 * promiscuous mode, and all-multi behavior.
3551 **/
3552static void igb_set_rx_mode(struct net_device *netdev)
3553{
3554 struct igb_adapter *adapter = netdev_priv(netdev);
3555 struct e1000_hw *hw = &adapter->hw;
3556 unsigned int vfn = adapter->vfs_allocated_count;
3557 u32 rctl, vmolr = 0;
3558 int count;
3559
3560 /* Check for Promiscuous and All Multicast modes */
3561 rctl = rd32(E1000_RCTL);
3562
3563 /* clear the effected bits */
3564 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3565
3566 if (netdev->flags & IFF_PROMISC) {
3567 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3568 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3569 } else {
3570 if (netdev->flags & IFF_ALLMULTI) {
3571 rctl |= E1000_RCTL_MPE;
3572 vmolr |= E1000_VMOLR_MPME;
3573 } else {
3574 /*
3575 * Write addresses to the MTA, if the attempt fails
25985edc 3576 * then we should just turn on promiscuous mode so
68d480c4
AD		 3577 * that we can at least receive multicast traffic
3578 */
3579 count = igb_write_mc_addr_list(netdev);
3580 if (count < 0) {
3581 rctl |= E1000_RCTL_MPE;
3582 vmolr |= E1000_VMOLR_MPME;
3583 } else if (count) {
3584 vmolr |= E1000_VMOLR_ROMPE;
3585 }
3586 }
3587 /*
3588 * Write addresses to available RAR registers, if there is not
3589 * sufficient space to store all the addresses then enable
25985edc 3590 * unicast promiscuous mode
68d480c4
AD		 3591 */
3592 count = igb_write_uc_addr_list(netdev);
3593 if (count < 0) {
3594 rctl |= E1000_RCTL_UPE;
3595 vmolr |= E1000_VMOLR_ROPE;
3596 }
3597 rctl |= E1000_RCTL_VFE;
28fc06f5 3598 }
68d480c4 3599 wr32(E1000_RCTL, rctl);
28fc06f5 3600
68d480c4
AD		 3601 /*
3602 * In order to support SR-IOV and eventually VMDq it is necessary to set
3603 * the VMOLR to enable the appropriate modes. Without this workaround
3604 * we will have issues with VLAN tag stripping not being done for frames
3605 * that are only arriving because we are the default pool
3606 */
f96a8a0b 3607 if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
28fc06f5 3608 return;
9d5c8243 3609
68d480c4
AD		 3610 vmolr |= rd32(E1000_VMOLR(vfn)) &
3611 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3612 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3613 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3614}
3615
13800469
GR		 3616static void igb_check_wvbr(struct igb_adapter *adapter)
3617{
3618 struct e1000_hw *hw = &adapter->hw;
3619 u32 wvbr = 0;
3620
3621 switch (hw->mac.type) {
3622 case e1000_82576:
3623 case e1000_i350:
3624 if (!(wvbr = rd32(E1000_WVBR)))
3625 return;
3626 break;
3627 default:
3628 break;
3629 }
3630
3631 adapter->wvbr |= wvbr;
3632}
3633
3634#define IGB_STAGGERED_QUEUE_OFFSET 8
3635
3636static void igb_spoof_check(struct igb_adapter *adapter)
3637{
3638 int j;
3639
3640 if (!adapter->wvbr)
3641 return;
3642
3643 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3644 if (adapter->wvbr & (1 << j) ||
3645 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3646 dev_warn(&adapter->pdev->dev,
3647 "Spoof event(s) detected on VF %d\n", j);
3648 adapter->wvbr &=
3649 ~((1 << j) |
3650 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3651 }
3652 }
3653}
3654
9d5c8243
AK		 3655/* Need to wait a few seconds after link up to get diagnostic information from
3656 * the phy */
3657static void igb_update_phy_info(unsigned long data)
3658{
3659 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3660 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3661}
3662
4d6b725e
AD		 3663/**
3664 * igb_has_link - check shared code for link and determine up/down
3665 * @adapter: pointer to driver private info
3666 **/
3145535a 3667bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3668{
3669 struct e1000_hw *hw = &adapter->hw;
3670 bool link_active = false;
3671 s32 ret_val = 0;
3672
3673 /* get_link_status is set on LSC (link status) interrupt or
3674 * rx sequence error interrupt. get_link_status will stay
3675 * false until the e1000_check_for_link establishes link
3676 * for copper adapters ONLY
3677 */
3678 switch (hw->phy.media_type) {
3679 case e1000_media_type_copper:
3680 if (hw->mac.get_link_status) {
3681 ret_val = hw->mac.ops.check_for_link(hw);
3682 link_active = !hw->mac.get_link_status;
3683 } else {
3684 link_active = true;
3685 }
3686 break;
4d6b725e
AD		 3687 case e1000_media_type_internal_serdes:
3688 ret_val = hw->mac.ops.check_for_link(hw);
3689 link_active = hw->mac.serdes_has_link;
3690 break;
3691 default:
3692 case e1000_media_type_unknown:
3693 break;
3694 }
3695
3696 return link_active;
3697}
3698
563988dc
SA		 3699static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3700{
3701 bool ret = false;
3702 u32 ctrl_ext, thstat;
3703
f96a8a0b 3704 /* check for thermal sensor event on i350 copper only */
563988dc
SA		 3705 if (hw->mac.type == e1000_i350) {
3706 thstat = rd32(E1000_THSTAT);
3707 ctrl_ext = rd32(E1000_CTRL_EXT);
3708
3709 if ((hw->phy.media_type == e1000_media_type_copper) &&
3710 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3711 ret = !!(thstat & event);
3712 }
3713 }
3714
3715 return ret;
3716}
3717
9d5c8243
AK		 3718/**
3719 * igb_watchdog - Timer Call-back
3720 * @data: pointer to adapter cast into an unsigned long
3721 **/
3722static void igb_watchdog(unsigned long data)
3723{
3724 struct igb_adapter *adapter = (struct igb_adapter *)data;
3725 /* Do the rest outside of interrupt context */
3726 schedule_work(&adapter->watchdog_task);
3727}
3728
3729static void igb_watchdog_task(struct work_struct *work)
3730{
3731 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3732 struct igb_adapter,
3733 watchdog_task);
9d5c8243 3734 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3735 struct net_device *netdev = adapter->netdev;
563988dc 3736 u32 link;
7a6ea550 3737 int i;
9d5c8243 3738
4d6b725e 3739 link = igb_has_link(adapter);
9d5c8243 3740 if (link) {
749ab2cd
YZ		 3741 /* Cancel scheduled suspend requests. */
3742 pm_runtime_resume(netdev->dev.parent);
3743
9d5c8243
AK		 3744 if (!netif_carrier_ok(netdev)) {
3745 u32 ctrl;
330a6d6a
AD		 3746 hw->mac.ops.get_speed_and_duplex(hw,
3747 &adapter->link_speed,
3748 &adapter->link_duplex);
9d5c8243
AK		 3749
3750 ctrl = rd32(E1000_CTRL);
527d47c1 3751 /* Links status message must follow this format */
876d2d6f
JK		 3752 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s "
3753 "Duplex, Flow Control: %s\n",
559e9c49
AD		 3754 netdev->name,
3755 adapter->link_speed,
3756 adapter->link_duplex == FULL_DUPLEX ?
876d2d6f
JK		 3757 "Full" : "Half",
3758 (ctrl & E1000_CTRL_TFCE) &&
3759 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
3760 (ctrl & E1000_CTRL_RFCE) ? "RX" :
3761 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None");
9d5c8243 3762
563988dc 3763 /* check for thermal sensor event */
876d2d6f
JK		 3764 if (igb_thermal_sensor_event(hw,
3765 E1000_THSTAT_LINK_THROTTLE)) {
3766 netdev_info(netdev, "The network adapter link "
3767 "speed was downshifted because it "
3768 "overheated\n");
7ef5ed1c 3769 }
563988dc 3770
d07f3e37 3771 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3772 adapter->tx_timeout_factor = 1;
3773 switch (adapter->link_speed) {
3774 case SPEED_10:
9d5c8243
AK		 3775 adapter->tx_timeout_factor = 14;
3776 break;
3777 case SPEED_100:
9d5c8243
AK		 3778 /* maybe add some timeout factor ? */
3779 break;
3780 }
3781
3782 netif_carrier_on(netdev);
9d5c8243 3783
4ae196df 3784 igb_ping_all_vfs(adapter);
17dc566c 3785 igb_check_vf_rate_limit(adapter);
4ae196df 3786
4b1a9877 3787 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3788 if (!test_bit(__IGB_DOWN, &adapter->state))
3789 mod_timer(&adapter->phy_info_timer,
3790 round_jiffies(jiffies + 2 * HZ));
3791 }
3792 } else {
3793 if (netif_carrier_ok(netdev)) {
3794 adapter->link_speed = 0;
3795 adapter->link_duplex = 0;
563988dc
SA		 3796
3797 /* check for thermal sensor event */
876d2d6f
JK		 3798 if (igb_thermal_sensor_event(hw,
3799 E1000_THSTAT_PWR_DOWN)) {
3800 netdev_err(netdev, "The network adapter was "
3801 "stopped because it overheated\n");
7ef5ed1c 3802 }
563988dc 3803
527d47c1
AD		 3804 /* Links status message must follow this format */
3805 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3806 netdev->name);
9d5c8243 3807 netif_carrier_off(netdev);
4b1a9877 3808
4ae196df
AD		 3809 igb_ping_all_vfs(adapter);
3810
4b1a9877 3811 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3812 if (!test_bit(__IGB_DOWN, &adapter->state))
3813 mod_timer(&adapter->phy_info_timer,
3814 round_jiffies(jiffies + 2 * HZ));
749ab2cd
YZ		 3815
3816 pm_schedule_suspend(netdev->dev.parent,
3817 MSEC_PER_SEC * 5);
9d5c8243
AK		 3818 }
3819 }
3820
12dcd86b
ED		 3821 spin_lock(&adapter->stats64_lock);
3822 igb_update_stats(adapter, &adapter->stats64);
3823 spin_unlock(&adapter->stats64_lock);
9d5c8243 3824
dbabb065 3825 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3826 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3827 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3828 /* We've lost link, so the controller stops DMA,
3829 * but we've got queued Tx work that's never going
3830 * to get done, so reset controller to flush Tx.
3831 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3832 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3833 adapter->tx_timeout_count++;
3834 schedule_work(&adapter->reset_task);
3835 /* return immediately since reset is imminent */
3836 return;
3837 }
9d5c8243 3838 }
9d5c8243 3839
dbabb065 3840 /* Force detection of hung controller every watchdog period */
6d095fa8 3841 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
dbabb065 3842 }
f7ba205e 3843
9d5c8243 3844 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3845 if (adapter->msix_entries) {
047e0030 3846 u32 eics = 0;
0d1ae7f4
AD		 3847 for (i = 0; i < adapter->num_q_vectors; i++)
3848 eics |= adapter->q_vector[i]->eims_value;
7a6ea550
AD		 3849 wr32(E1000_EICS, eics);
3850 } else {
3851 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3852 }
9d5c8243 3853
13800469
GR		 3854 igb_spoof_check(adapter);
3855
9d5c8243
AK		 3856 /* Reset the timer */
3857 if (!test_bit(__IGB_DOWN, &adapter->state))
3858 mod_timer(&adapter->watchdog_timer,
3859 round_jiffies(jiffies + 2 * HZ));
3860}
3861
3862enum latency_range {
3863 lowest_latency = 0,
3864 low_latency = 1,
3865 bulk_latency = 2,
3866 latency_invalid = 255
3867};
3868
6eb5a7f1
AD		 3869/**
3870 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3871 *
3872 * Stores a new ITR value based on strictly on packet size. This
3873 * algorithm is less sophisticated than that used in igb_update_itr,
3874 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3875 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3876 * were determined based on theoretical maximum wire speed and testing
3877 * data, in order to minimize response time while increasing bulk
3878 * throughput.
3879 * This functionality is controlled by the InterruptThrottleRate module
3880 * parameter (see igb_param.c)
3881 * NOTE: This function is called only when operating in a multiqueue
3882 * receive environment.
047e0030 3883 * @q_vector: pointer to q_vector
6eb5a7f1 3884 **/
047e0030 3885static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3886{
047e0030 3887 int new_val = q_vector->itr_val;
6eb5a7f1 3888 int avg_wire_size = 0;
047e0030 3889 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b 3890 unsigned int packets;
9d5c8243 3891
6eb5a7f1
AD		 3892 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3893 * ints/sec - ITR timer value of 120 ticks.
3894 */
3895 if (adapter->link_speed != SPEED_1000) {
0ba82994 3896 new_val = IGB_4K_ITR;
6eb5a7f1 3897 goto set_itr_val;
9d5c8243 3898 }
047e0030 3899
0ba82994
AD		 3900 packets = q_vector->rx.total_packets;
3901 if (packets)
3902 avg_wire_size = q_vector->rx.total_bytes / packets;
047e0030 3903
0ba82994
AD		 3904 packets = q_vector->tx.total_packets;
3905 if (packets)
3906 avg_wire_size = max_t(u32, avg_wire_size,
3907 q_vector->tx.total_bytes / packets);
047e0030
AD		 3908
3909 /* if avg_wire_size isn't set no work was done */
3910 if (!avg_wire_size)
3911 goto clear_counts;
9d5c8243 3912
6eb5a7f1
AD		 3913 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3914 avg_wire_size += 24;
3915
3916 /* Don't starve jumbo frames */
3917 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3918
6eb5a7f1
AD		 3919 /* Give a little boost to mid-size frames */
3920 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3921 new_val = avg_wire_size / 3;
3922 else
3923 new_val = avg_wire_size / 2;
9d5c8243 3924
0ba82994
AD		 3925 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3926 if (new_val < IGB_20K_ITR &&
3927 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3928 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3929 new_val = IGB_20K_ITR;
abe1c363 3930
6eb5a7f1 3931set_itr_val:
047e0030
AD		 3932 if (new_val != q_vector->itr_val) {
3933 q_vector->itr_val = new_val;
3934 q_vector->set_itr = 1;
9d5c8243 3935 }
6eb5a7f1 3936clear_counts:
0ba82994
AD		 3937 q_vector->rx.total_bytes = 0;
3938 q_vector->rx.total_packets = 0;
3939 q_vector->tx.total_bytes = 0;
3940 q_vector->tx.total_packets = 0;
9d5c8243
AK		 3941}
3942
3943/**
3944 * igb_update_itr - update the dynamic ITR value based on statistics
3945 * Stores a new ITR value based on packets and byte
3946 * counts during the last interrupt. The advantage of per interrupt
3947 * computation is faster updates and more accurate ITR for the current
3948 * traffic pattern. Constants in this function were computed
3949 * based on theoretical maximum wire speed and thresholds were set based
3950 * on testing data as well as attempting to minimize response time
3951 * while increasing bulk throughput.
3952 * this functionality is controlled by the InterruptThrottleRate module
3953 * parameter (see igb_param.c)
3954 * NOTE: These calculations are only valid when operating in a single-
3955 * queue environment.
0ba82994
AD		 3956 * @q_vector: pointer to q_vector
3957 * @ring_container: ring info to update the itr for
9d5c8243 3958 **/
0ba82994
AD		 3959static void igb_update_itr(struct igb_q_vector *q_vector,
3960 struct igb_ring_container *ring_container)
9d5c8243 3961{
0ba82994
AD		 3962 unsigned int packets = ring_container->total_packets;
3963 unsigned int bytes = ring_container->total_bytes;
3964 u8 itrval = ring_container->itr;
9d5c8243 3965
0ba82994 3966 /* no packets, exit with status unchanged */
9d5c8243 3967 if (packets == 0)
0ba82994 3968 return;
9d5c8243 3969
0ba82994 3970 switch (itrval) {
9d5c8243
AK		 3971 case lowest_latency:
3972 /* handle TSO and jumbo frames */
3973 if (bytes/packets > 8000)
0ba82994 3974 itrval = bulk_latency;
9d5c8243 3975 else if ((packets < 5) && (bytes > 512))
0ba82994 3976 itrval = low_latency;
9d5c8243
AK		 3977 break;
3978 case low_latency: /* 50 usec aka 20000 ints/s */
3979 if (bytes > 10000) {
3980 /* this if handles the TSO accounting */
3981 if (bytes/packets > 8000) {
0ba82994 3982 itrval = bulk_latency;
9d5c8243 3983 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
0ba82994 3984 itrval = bulk_latency;
9d5c8243 3985 } else if ((packets > 35)) {
0ba82994 3986 itrval = lowest_latency;
9d5c8243
AK		 3987 }
3988 } else if (bytes/packets > 2000) {
0ba82994 3989 itrval = bulk_latency;
9d5c8243 3990 } else if (packets <= 2 && bytes < 512) {
0ba82994 3991 itrval = lowest_latency;
9d5c8243
AK		 3992 }
3993 break;
3994 case bulk_latency: /* 250 usec aka 4000 ints/s */
3995 if (bytes > 25000) {
3996 if (packets > 35)
0ba82994 3997 itrval = low_latency;
1e5c3d21 3998 } else if (bytes < 1500) {
0ba82994 3999 itrval = low_latency;
9d5c8243
AK		 4000 }
4001 break;
4002 }
4003
0ba82994
AD		 4004 /* clear work counters since we have the values we need */
4005 ring_container->total_bytes = 0;
4006 ring_container->total_packets = 0;
4007
4008 /* write updated itr to ring container */
4009 ring_container->itr = itrval;
9d5c8243
AK		 4010}
4011
0ba82994 4012static void igb_set_itr(struct igb_q_vector *q_vector)
9d5c8243 4013{
0ba82994 4014 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4015 u32 new_itr = q_vector->itr_val;
0ba82994 4016 u8 current_itr = 0;
9d5c8243
AK		 4017
4018 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4019 if (adapter->link_speed != SPEED_1000) {
4020 current_itr = 0;
0ba82994 4021 new_itr = IGB_4K_ITR;
9d5c8243
AK		 4022 goto set_itr_now;
4023 }
4024
0ba82994
AD		 4025 igb_update_itr(q_vector, &q_vector->tx);
4026 igb_update_itr(q_vector, &q_vector->rx);
9d5c8243 4027
0ba82994 4028 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
9d5c8243 4029
6eb5a7f1 4030 /* conservative mode (itr 3) eliminates the lowest_latency setting */
0ba82994
AD		 4031 if (current_itr == lowest_latency &&
4032 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4033 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
6eb5a7f1
AD		 4034 current_itr = low_latency;
4035
9d5c8243
AK		 4036 switch (current_itr) {
4037 /* counts and packets in update_itr are dependent on these numbers */
4038 case lowest_latency:
0ba82994 4039 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
9d5c8243
AK		 4040 break;
4041 case low_latency:
0ba82994 4042 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
9d5c8243
AK		 4043 break;
4044 case bulk_latency:
0ba82994 4045 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
9d5c8243
AK		 4046 break;
4047 default:
4048 break;
4049 }
4050
4051set_itr_now:
047e0030 4052 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 4053 /* this attempts to bias the interrupt rate towards Bulk
4054 * by adding intermediate steps when interrupt rate is
4055 * increasing */
047e0030
AD		 4056 new_itr = new_itr > q_vector->itr_val ?
4057 max((new_itr * q_vector->itr_val) /
4058 (new_itr + (q_vector->itr_val >> 2)),
0ba82994 4059 new_itr) :
9d5c8243
AK		 4060 new_itr;
4061 /* Don't write the value here; it resets the adapter's
4062 * internal timer, and causes us to delay far longer than
4063 * we should between interrupts. Instead, we write the ITR
4064 * value at the beginning of the next interrupt so the timing
4065 * ends up being correct.
4066 */
047e0030
AD		 4067 q_vector->itr_val = new_itr;
4068 q_vector->set_itr = 1;
9d5c8243 4069 }
9d5c8243
AK		 4070}
4071
c50b52a0
SH		 4072static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4073 u32 type_tucmd, u32 mss_l4len_idx)
7d13a7d0
AD		 4074{
4075 struct e1000_adv_tx_context_desc *context_desc;
4076 u16 i = tx_ring->next_to_use;
4077
4078 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4079
4080 i++;
4081 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4082
4083 /* set bits to identify this as an advanced context descriptor */
4084 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4085
4086 /* For 82575, context index must be unique per ring. */
866cff06 4087 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
7d13a7d0
AD		 4088 mss_l4len_idx |= tx_ring->reg_idx << 4;
4089
4090 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
4091 context_desc->seqnum_seed = 0;
4092 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
4093 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4094}
4095
7af40ad9
AD		 4096static int igb_tso(struct igb_ring *tx_ring,
4097 struct igb_tx_buffer *first,
4098 u8 *hdr_len)
9d5c8243 4099{
7af40ad9 4100 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4101 u32 vlan_macip_lens, type_tucmd;
4102 u32 mss_l4len_idx, l4len;
4103
4104 if (!skb_is_gso(skb))
4105 return 0;
9d5c8243
AK		 4106
4107 if (skb_header_cloned(skb)) {
7af40ad9 4108 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
9d5c8243
AK		 4109 if (err)
4110 return err;
4111 }
4112
7d13a7d0
AD		 4113 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4114 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
9d5c8243 4115
7af40ad9 4116 if (first->protocol == __constant_htons(ETH_P_IP)) {
9d5c8243
AK		 4117 struct iphdr *iph = ip_hdr(skb);
4118 iph->tot_len = 0;
4119 iph->check = 0;
4120 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4121 iph->daddr, 0,
4122 IPPROTO_TCP,
4123 0);
7d13a7d0 4124 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
7af40ad9
AD		 4125 first->tx_flags |= IGB_TX_FLAGS_TSO |
4126 IGB_TX_FLAGS_CSUM |
4127 IGB_TX_FLAGS_IPV4;
8e1e8a47 4128 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 4129 ipv6_hdr(skb)->payload_len = 0;
4130 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4131 &ipv6_hdr(skb)->daddr,
4132 0, IPPROTO_TCP, 0);
7af40ad9
AD		 4133 first->tx_flags |= IGB_TX_FLAGS_TSO |
4134 IGB_TX_FLAGS_CSUM;
9d5c8243
AK		 4135 }
4136
7af40ad9 4137 /* compute header lengths */
7d13a7d0
AD		 4138 l4len = tcp_hdrlen(skb);
4139 *hdr_len = skb_transport_offset(skb) + l4len;
9d5c8243 4140
7af40ad9
AD		 4141 /* update gso size and bytecount with header size */
4142 first->gso_segs = skb_shinfo(skb)->gso_segs;
4143 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4144
9d5c8243 4145 /* MSS L4LEN IDX */
7d13a7d0
AD		 4146 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4147 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
9d5c8243 4148
7d13a7d0
AD		 4149 /* VLAN MACLEN IPLEN */
4150 vlan_macip_lens = skb_network_header_len(skb);
4151 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4152 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4153
7d13a7d0 4154 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243 4155
7d13a7d0 4156 return 1;
9d5c8243
AK		 4157}
4158
7af40ad9 4159static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
9d5c8243 4160{
7af40ad9 4161 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4162 u32 vlan_macip_lens = 0;
4163 u32 mss_l4len_idx = 0;
4164 u32 type_tucmd = 0;
9d5c8243 4165
7d13a7d0 4166 if (skb->ip_summed != CHECKSUM_PARTIAL) {
7af40ad9
AD		 4167 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4168 return;
7d13a7d0
AD		 4169 } else {
4170 u8 l4_hdr = 0;
7af40ad9 4171 switch (first->protocol) {
7d13a7d0
AD		 4172 case __constant_htons(ETH_P_IP):
4173 vlan_macip_lens |= skb_network_header_len(skb);
4174 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4175 l4_hdr = ip_hdr(skb)->protocol;
4176 break;
4177 case __constant_htons(ETH_P_IPV6):
4178 vlan_macip_lens |= skb_network_header_len(skb);
4179 l4_hdr = ipv6_hdr(skb)->nexthdr;
4180 break;
4181 default:
4182 if (unlikely(net_ratelimit())) {
4183 dev_warn(tx_ring->dev,
4184 "partial checksum but proto=%x!\n",
7af40ad9 4185 first->protocol);
fa4a7ef3 4186 }
7d13a7d0
AD		 4187 break;
4188 }
fa4a7ef3 4189
7d13a7d0
AD		 4190 switch (l4_hdr) {
4191 case IPPROTO_TCP:
4192 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4193 mss_l4len_idx = tcp_hdrlen(skb) <<
4194 E1000_ADVTXD_L4LEN_SHIFT;
4195 break;
4196 case IPPROTO_SCTP:
4197 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4198 mss_l4len_idx = sizeof(struct sctphdr) <<
4199 E1000_ADVTXD_L4LEN_SHIFT;
4200 break;
4201 case IPPROTO_UDP:
4202 mss_l4len_idx = sizeof(struct udphdr) <<
4203 E1000_ADVTXD_L4LEN_SHIFT;
4204 break;
4205 default:
4206 if (unlikely(net_ratelimit())) {
4207 dev_warn(tx_ring->dev,
4208 "partial checksum but l4 proto=%x!\n",
4209 l4_hdr);
44b0cda3 4210 }
7d13a7d0 4211 break;
9d5c8243 4212 }
7af40ad9
AD		 4213
4214 /* update TX checksum flag */
4215 first->tx_flags |= IGB_TX_FLAGS_CSUM;
7d13a7d0 4216 }
9d5c8243 4217
7d13a7d0 4218 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4219 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4220
7d13a7d0 4221 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243
AK		 4222}
4223
e032afc8
AD		 4224static __le32 igb_tx_cmd_type(u32 tx_flags)
4225{
4226 /* set type for advanced descriptor with frame checksum insertion */
4227 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4228 E1000_ADVTXD_DCMD_IFCS |
4229 E1000_ADVTXD_DCMD_DEXT);
4230
4231 /* set HW vlan bit if vlan is present */
4232 if (tx_flags & IGB_TX_FLAGS_VLAN)
4233 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4234
3c89f6d0 4235#ifdef CONFIG_IGB_PTP
e032afc8
AD		 4236 /* set timestamp bit if present */
4237 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4238 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
3c89f6d0 4239#endif /* CONFIG_IGB_PTP */
e032afc8
AD		 4240
4241 /* set segmentation bits for TSO */
4242 if (tx_flags & IGB_TX_FLAGS_TSO)
4243 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4244
4245 return cmd_type;
4246}
4247
7af40ad9
AD		 4248static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4249 union e1000_adv_tx_desc *tx_desc,
4250 u32 tx_flags, unsigned int paylen)
e032afc8
AD		 4251{
4252 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4253
4254 /* 82575 requires a unique index per ring if any offload is enabled */
4255 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
866cff06 4256 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
e032afc8
AD		 4257 olinfo_status |= tx_ring->reg_idx << 4;
4258
4259 /* insert L4 checksum */
4260 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4261 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4262
4263 /* insert IPv4 checksum */
4264 if (tx_flags & IGB_TX_FLAGS_IPV4)
4265 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4266 }
4267
7af40ad9 4268 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
e032afc8
AD		 4269}
4270
ebe42d16
AD		 4271/*
4272 * The largest size we can write to the descriptor is 65535. In order to
4273 * maintain a power of two alignment we have to limit ourselves to 32K.
4274 */
4275#define IGB_MAX_TXD_PWR 15
7af40ad9 4276#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
9d5c8243 4277
7af40ad9
AD		 4278static void igb_tx_map(struct igb_ring *tx_ring,
4279 struct igb_tx_buffer *first,
ebe42d16 4280 const u8 hdr_len)
9d5c8243 4281{
7af40ad9 4282 struct sk_buff *skb = first->skb;
ebe42d16
AD		 4283 struct igb_tx_buffer *tx_buffer_info;
4284 union e1000_adv_tx_desc *tx_desc;
4285 dma_addr_t dma;
4286 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4287 unsigned int data_len = skb->data_len;
4288 unsigned int size = skb_headlen(skb);
4289 unsigned int paylen = skb->len - hdr_len;
4290 __le32 cmd_type;
7af40ad9 4291 u32 tx_flags = first->tx_flags;
ebe42d16 4292 u16 i = tx_ring->next_to_use;
ebe42d16
AD		 4293
4294 tx_desc = IGB_TX_DESC(tx_ring, i);
4295
7af40ad9 4296 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
ebe42d16
AD		 4297 cmd_type = igb_tx_cmd_type(tx_flags);
4298
4299 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4300 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33 4301 goto dma_error;
9d5c8243 4302
ebe42d16
AD		 4303 /* record length, and DMA address */
4304 first->length = size;
4305 first->dma = dma;
ebe42d16
AD		 4306 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4307
4308 for (;;) {
4309 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4310 tx_desc->read.cmd_type_len =
4311 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4312
4313 i++;
4314 tx_desc++;
4315 if (i == tx_ring->count) {
4316 tx_desc = IGB_TX_DESC(tx_ring, 0);
4317 i = 0;
4318 }
4319
4320 dma += IGB_MAX_DATA_PER_TXD;
4321 size -= IGB_MAX_DATA_PER_TXD;
4322
4323 tx_desc->read.olinfo_status = 0;
4324 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4325 }
4326
4327 if (likely(!data_len))
4328 break;
2bbfebe2 4329
ebe42d16 4330 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
9d5c8243 4331
65689fef 4332 i++;
ebe42d16
AD		 4333 tx_desc++;
4334 if (i == tx_ring->count) {
4335 tx_desc = IGB_TX_DESC(tx_ring, 0);
65689fef 4336 i = 0;
ebe42d16 4337 }
65689fef 4338
9e903e08 4339 size = skb_frag_size(frag);
ebe42d16
AD		 4340 data_len -= size;
4341
4342 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4343 size, DMA_TO_DEVICE);
4344 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33
AD		 4345 goto dma_error;
4346
ebe42d16
AD		 4347 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4348 tx_buffer_info->length = size;
4349 tx_buffer_info->dma = dma;
4350
4351 tx_desc->read.olinfo_status = 0;
4352 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4353
4354 frag++;
9d5c8243
AK		 4355 }
4356
bdbc0631
ED		 4357 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4358
ebe42d16
AD		 4359 /* write last descriptor with RS and EOP bits */
4360 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
6b8f0922
BG		 4361 if (unlikely(skb->no_fcs))
4362 cmd_type &= ~(cpu_to_le32(E1000_ADVTXD_DCMD_IFCS));
ebe42d16 4363 tx_desc->read.cmd_type_len = cmd_type;
8542db05
AD		 4364
4365 /* set the timestamp */
4366 first->time_stamp = jiffies;
4367
ebe42d16
AD		 4368 /*
4369 * Force memory writes to complete before letting h/w know there
4370 * are new descriptors to fetch. (Only applicable for weak-ordered
4371 * memory model archs, such as IA-64).
4372 *
4373 * We also need this memory barrier to make certain all of the
4374 * status bits have been updated before next_to_watch is written.
4375 */
4376 wmb();
4377
8542db05 4378 /* set next_to_watch value indicating a packet is present */
ebe42d16 4379 first->next_to_watch = tx_desc;
9d5c8243 4380
ebe42d16
AD		 4381 i++;
4382 if (i == tx_ring->count)
4383 i = 0;
6366ad33 4384
ebe42d16 4385 tx_ring->next_to_use = i;
6366ad33 4386
ebe42d16 4387 writel(i, tx_ring->tail);
6366ad33 4388
ebe42d16
AD		 4389 /* we need this if more than one processor can write to our tail
4390 * at a time, it syncronizes IO on IA64/Altix systems */
4391 mmiowb();
4392
4393 return;
4394
4395dma_error:
4396 dev_err(tx_ring->dev, "TX DMA map failed\n");
4397
4398 /* clear dma mappings for failed tx_buffer_info map */
4399 for (;;) {
4400 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4401 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4402 if (tx_buffer_info == first)
4403 break;
a77ff709
NN		 4404 if (i == 0)
4405 i = tx_ring->count;
6366ad33 4406 i--;
6366ad33
AD		 4407 }
4408
9d5c8243 4409 tx_ring->next_to_use = i;
9d5c8243
AK		 4410}
4411
6ad4edfc 4412static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4413{
e694e964
AD		 4414 struct net_device *netdev = tx_ring->netdev;
4415
661086df 4416 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4417
9d5c8243
AK		 4418 /* Herbert's original patch had:
4419 * smp_mb__after_netif_stop_queue();
4420 * but since that doesn't exist yet, just open code it. */
4421 smp_mb();
4422
4423 /* We need to check again in a case another CPU has just
4424 * made room available. */
c493ea45 4425 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4426 return -EBUSY;
4427
4428 /* A reprieve! */
661086df 4429 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4430
4431 u64_stats_update_begin(&tx_ring->tx_syncp2);
4432 tx_ring->tx_stats.restart_queue2++;
4433 u64_stats_update_end(&tx_ring->tx_syncp2);
4434
9d5c8243
AK		 4435 return 0;
4436}
4437
6ad4edfc 4438static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
9d5c8243 4439{
c493ea45 4440 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4441 return 0;
e694e964 4442 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4443}
4444
cd392f5c
AD		 4445netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4446 struct igb_ring *tx_ring)
9d5c8243 4447{
8542db05 4448 struct igb_tx_buffer *first;
ebe42d16 4449 int tso;
91d4ee33 4450 u32 tx_flags = 0;
31f6adbb 4451 __be16 protocol = vlan_get_protocol(skb);
91d4ee33 4452 u8 hdr_len = 0;
9d5c8243 4453
9d5c8243
AK		 4454 /* need: 1 descriptor per page,
4455 * + 2 desc gap to keep tail from touching head,
4456 * + 1 desc for skb->data,
4457 * + 1 desc for context descriptor,
4458 * otherwise try next time */
e694e964 4459 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4460 /* this is a hard error */
9d5c8243
AK		 4461 return NETDEV_TX_BUSY;
4462 }
33af6bcc 4463
7af40ad9
AD		 4464 /* record the location of the first descriptor for this packet */
4465 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4466 first->skb = skb;
4467 first->bytecount = skb->len;
4468 first->gso_segs = 1;
4469
3c89f6d0 4470#ifdef CONFIG_IGB_PTP
2244d07b
OH		 4471 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4472 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4473 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4474 }
3c89f6d0 4475#endif /* CONFIG_IGB_PTP */
9d5c8243 4476
eab6d18d 4477 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4478 tx_flags |= IGB_TX_FLAGS_VLAN;
4479 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4480 }
4481
7af40ad9
AD		 4482 /* record initial flags and protocol */
4483 first->tx_flags = tx_flags;
4484 first->protocol = protocol;
cdfd01fc 4485
7af40ad9
AD		 4486 tso = igb_tso(tx_ring, first, &hdr_len);
4487 if (tso < 0)
7d13a7d0 4488 goto out_drop;
7af40ad9
AD		 4489 else if (!tso)
4490 igb_tx_csum(tx_ring, first);
9d5c8243 4491
7af40ad9 4492 igb_tx_map(tx_ring, first, hdr_len);
85ad76b2
AD		 4493
4494 /* Make sure there is space in the ring for the next send. */
e694e964 4495 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4496
9d5c8243 4497 return NETDEV_TX_OK;
7d13a7d0
AD		 4498
4499out_drop:
7af40ad9
AD		 4500 igb_unmap_and_free_tx_resource(tx_ring, first);
4501
7d13a7d0 4502 return NETDEV_TX_OK;
9d5c8243
AK		 4503}
4504
1cc3bd87
AD		 4505static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4506 struct sk_buff *skb)
4507{
4508 unsigned int r_idx = skb->queue_mapping;
4509
4510 if (r_idx >= adapter->num_tx_queues)
4511 r_idx = r_idx % adapter->num_tx_queues;
4512
4513 return adapter->tx_ring[r_idx];
4514}
4515
cd392f5c
AD		 4516static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4517 struct net_device *netdev)
9d5c8243
AK		 4518{
4519 struct igb_adapter *adapter = netdev_priv(netdev);
b1a436c3
AD		 4520
4521 if (test_bit(__IGB_DOWN, &adapter->state)) {
4522 dev_kfree_skb_any(skb);
4523 return NETDEV_TX_OK;
4524 }
4525
4526 if (skb->len <= 0) {
4527 dev_kfree_skb_any(skb);
4528 return NETDEV_TX_OK;
4529 }
4530
1cc3bd87
AD		 4531 /*
4532 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4533 * in order to meet this minimum size requirement.
4534 */
4535 if (skb->len < 17) {
4536 if (skb_padto(skb, 17))
4537 return NETDEV_TX_OK;
4538 skb->len = 17;
4539 }
9d5c8243 4540
1cc3bd87 4541 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
9d5c8243
AK		 4542}
4543
4544/**
4545 * igb_tx_timeout - Respond to a Tx Hang
4546 * @netdev: network interface device structure
4547 **/
4548static void igb_tx_timeout(struct net_device *netdev)
4549{
4550 struct igb_adapter *adapter = netdev_priv(netdev);
4551 struct e1000_hw *hw = &adapter->hw;
4552
4553 /* Do the reset outside of interrupt context */
4554 adapter->tx_timeout_count++;
f7ba205e 4555
06218a8d 4556 if (hw->mac.type >= e1000_82580)
55cac248
AD		 4557 hw->dev_spec._82575.global_device_reset = true;
4558
9d5c8243 4559 schedule_work(&adapter->reset_task);
265de409
AD		 4560 wr32(E1000_EICS,
4561 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4562}
4563
4564static void igb_reset_task(struct work_struct *work)
4565{
4566 struct igb_adapter *adapter;
4567 adapter = container_of(work, struct igb_adapter, reset_task);
4568
c97ec42a
TI		 4569 igb_dump(adapter);
4570 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4571 igb_reinit_locked(adapter);
4572}
4573
4574/**
12dcd86b 4575 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4576 * @netdev: network interface device structure
12dcd86b 4577 * @stats: rtnl_link_stats64 pointer
9d5c8243 4578 *
9d5c8243 4579 **/
12dcd86b
ED		 4580static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4581 struct rtnl_link_stats64 *stats)
9d5c8243 4582{
12dcd86b
ED		 4583 struct igb_adapter *adapter = netdev_priv(netdev);
4584
4585 spin_lock(&adapter->stats64_lock);
4586 igb_update_stats(adapter, &adapter->stats64);
4587 memcpy(stats, &adapter->stats64, sizeof(*stats));
4588 spin_unlock(&adapter->stats64_lock);
4589
4590 return stats;
9d5c8243
AK		 4591}
4592
4593/**
4594 * igb_change_mtu - Change the Maximum Transfer Unit
4595 * @netdev: network interface device structure
4596 * @new_mtu: new value for maximum frame size
4597 *
4598 * Returns 0 on success, negative on failure
4599 **/
4600static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4601{
4602 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4603 struct pci_dev *pdev = adapter->pdev;
153285f9 4604 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9d5c8243 4605
c809d227 4606 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4607 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4608 return -EINVAL;
4609 }
4610
153285f9 4611#define MAX_STD_JUMBO_FRAME_SIZE 9238
9d5c8243 4612 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4613 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4614 return -EINVAL;
4615 }
4616
4617 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4618 msleep(1);
73cd78f1 4619
9d5c8243
AK		 4620 /* igb_down has a dependency on max_frame_size */
4621 adapter->max_frame_size = max_frame;
559e9c49 4622
4c844851
AD		 4623 if (netif_running(netdev))
4624 igb_down(adapter);
9d5c8243 4625
090b1795 4626 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4627 netdev->mtu, new_mtu);
4628 netdev->mtu = new_mtu;
4629
4630 if (netif_running(netdev))
4631 igb_up(adapter);
4632 else
4633 igb_reset(adapter);
4634
4635 clear_bit(__IGB_RESETTING, &adapter->state);
4636
4637 return 0;
4638}
4639
4640/**
4641 * igb_update_stats - Update the board statistics counters
4642 * @adapter: board private structure
4643 **/
4644
12dcd86b
ED		 4645void igb_update_stats(struct igb_adapter *adapter,
4646 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4647{
4648 struct e1000_hw *hw = &adapter->hw;
4649 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4650 u32 reg, mpc;
9d5c8243 4651 u16 phy_tmp;
3f9c0164
AD		 4652 int i;
4653 u64 bytes, packets;
12dcd86b
ED		 4654 unsigned int start;
4655 u64 _bytes, _packets;
9d5c8243
AK		 4656
4657#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4658
4659 /*
4660 * Prevent stats update while adapter is being reset, or if the pci
4661 * connection is down.
4662 */
4663 if (adapter->link_speed == 0)
4664 return;
4665 if (pci_channel_offline(pdev))
4666 return;
4667
3f9c0164
AD		 4668 bytes = 0;
4669 packets = 0;
4670 for (i = 0; i < adapter->num_rx_queues; i++) {
4671 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4672 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4673
3025a446 4674 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4675 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4676
4677 do {
4678 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4679 _bytes = ring->rx_stats.bytes;
4680 _packets = ring->rx_stats.packets;
4681 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4682 bytes += _bytes;
4683 packets += _packets;
3f9c0164
AD		 4684 }
4685
128e45eb
AD		 4686 net_stats->rx_bytes = bytes;
4687 net_stats->rx_packets = packets;
3f9c0164
AD		 4688
4689 bytes = 0;
4690 packets = 0;
4691 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4692 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4693 do {
4694 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4695 _bytes = ring->tx_stats.bytes;
4696 _packets = ring->tx_stats.packets;
4697 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4698 bytes += _bytes;
4699 packets += _packets;
3f9c0164 4700 }
128e45eb
AD		 4701 net_stats->tx_bytes = bytes;
4702 net_stats->tx_packets = packets;
3f9c0164
AD		 4703
4704 /* read stats registers */
9d5c8243
AK		 4705 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4706 adapter->stats.gprc += rd32(E1000_GPRC);
4707 adapter->stats.gorc += rd32(E1000_GORCL);
4708 rd32(E1000_GORCH); /* clear GORCL */
4709 adapter->stats.bprc += rd32(E1000_BPRC);
4710 adapter->stats.mprc += rd32(E1000_MPRC);
4711 adapter->stats.roc += rd32(E1000_ROC);
4712
4713 adapter->stats.prc64 += rd32(E1000_PRC64);
4714 adapter->stats.prc127 += rd32(E1000_PRC127);
4715 adapter->stats.prc255 += rd32(E1000_PRC255);
4716 adapter->stats.prc511 += rd32(E1000_PRC511);
4717 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4718 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4719 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4720 adapter->stats.sec += rd32(E1000_SEC);
4721
fa3d9a6d
MW		 4722 mpc = rd32(E1000_MPC);
4723 adapter->stats.mpc += mpc;
4724 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4725 adapter->stats.scc += rd32(E1000_SCC);
4726 adapter->stats.ecol += rd32(E1000_ECOL);
4727 adapter->stats.mcc += rd32(E1000_MCC);
4728 adapter->stats.latecol += rd32(E1000_LATECOL);
4729 adapter->stats.dc += rd32(E1000_DC);
4730 adapter->stats.rlec += rd32(E1000_RLEC);
4731 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4732 adapter->stats.xontxc += rd32(E1000_XONTXC);
4733 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4734 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4735 adapter->stats.fcruc += rd32(E1000_FCRUC);
4736 adapter->stats.gptc += rd32(E1000_GPTC);
4737 adapter->stats.gotc += rd32(E1000_GOTCL);
4738 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4739 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4740 adapter->stats.ruc += rd32(E1000_RUC);
4741 adapter->stats.rfc += rd32(E1000_RFC);
4742 adapter->stats.rjc += rd32(E1000_RJC);
4743 adapter->stats.tor += rd32(E1000_TORH);
4744 adapter->stats.tot += rd32(E1000_TOTH);
4745 adapter->stats.tpr += rd32(E1000_TPR);
4746
4747 adapter->stats.ptc64 += rd32(E1000_PTC64);
4748 adapter->stats.ptc127 += rd32(E1000_PTC127);
4749 adapter->stats.ptc255 += rd32(E1000_PTC255);
4750 adapter->stats.ptc511 += rd32(E1000_PTC511);
4751 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4752 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4753
4754 adapter->stats.mptc += rd32(E1000_MPTC);
4755 adapter->stats.bptc += rd32(E1000_BPTC);
4756
2d0b0f69
NN		 4757 adapter->stats.tpt += rd32(E1000_TPT);
4758 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4759
4760 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4761 /* read internal phy specific stats */
4762 reg = rd32(E1000_CTRL_EXT);
4763 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4764 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4765 adapter->stats.tncrs += rd32(E1000_TNCRS);
4766 }
4767
9d5c8243
AK		 4768 adapter->stats.tsctc += rd32(E1000_TSCTC);
4769 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4770
4771 adapter->stats.iac += rd32(E1000_IAC);
4772 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4773 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4774 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4775 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4776 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4777 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4778 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4779 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4780
4781 /* Fill out the OS statistics structure */
128e45eb
AD		 4782 net_stats->multicast = adapter->stats.mprc;
4783 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4784
4785 /* Rx Errors */
4786
4787 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4788 * our own version based on RUC and ROC */
128e45eb 4789 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4790 adapter->stats.crcerrs + adapter->stats.algnerrc +
4791 adapter->stats.ruc + adapter->stats.roc +
4792 adapter->stats.cexterr;
128e45eb
AD		 4793 net_stats->rx_length_errors = adapter->stats.ruc +
4794 adapter->stats.roc;
4795 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4796 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4797 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4798
4799 /* Tx Errors */
128e45eb
AD		 4800 net_stats->tx_errors = adapter->stats.ecol +
4801 adapter->stats.latecol;
4802 net_stats->tx_aborted_errors = adapter->stats.ecol;
4803 net_stats->tx_window_errors = adapter->stats.latecol;
4804 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4805
4806 /* Tx Dropped needs to be maintained elsewhere */
4807
4808 /* Phy Stats */
4809 if (hw->phy.media_type == e1000_media_type_copper) {
4810 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4811 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4812 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4813 adapter->phy_stats.idle_errors += phy_tmp;
4814 }
4815 }
4816
4817 /* Management Stats */
4818 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4819 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4820 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
0a915b95
CW		 4821
4822 /* OS2BMC Stats */
4823 reg = rd32(E1000_MANC);
4824 if (reg & E1000_MANC_EN_BMC2OS) {
4825 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4826 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4827 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4828 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4829 }
9d5c8243
AK		 4830}
4831
9d5c8243
AK		 4832static irqreturn_t igb_msix_other(int irq, void *data)
4833{
047e0030 4834 struct igb_adapter *adapter = data;
9d5c8243 4835 struct e1000_hw *hw = &adapter->hw;
844290e5 4836 u32 icr = rd32(E1000_ICR);
844290e5 4837 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4838
7f081d40
AD		 4839 if (icr & E1000_ICR_DRSTA)
4840 schedule_work(&adapter->reset_task);
4841
047e0030 4842 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4843 /* HW is reporting DMA is out of sync */
4844 adapter->stats.doosync++;
13800469
GR		 4845 /* The DMA Out of Sync is also indication of a spoof event
4846 * in IOV mode. Check the Wrong VM Behavior register to
4847 * see if it is really a spoof event. */
4848 igb_check_wvbr(adapter);
dda0e083 4849 }
eebbbdba 4850
4ae196df
AD		 4851 /* Check for a mailbox event */
4852 if (icr & E1000_ICR_VMMB)
4853 igb_msg_task(adapter);
4854
4855 if (icr & E1000_ICR_LSC) {
4856 hw->mac.get_link_status = 1;
4857 /* guard against interrupt when we're going down */
4858 if (!test_bit(__IGB_DOWN, &adapter->state))
4859 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4860 }
4861
844290e5 4862 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4863
4864 return IRQ_HANDLED;
4865}
4866
047e0030 4867static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4868{
26b39276 4869 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4870 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4871
047e0030
AD		 4872 if (!q_vector->set_itr)
4873 return;
73cd78f1 4874
047e0030
AD		 4875 if (!itr_val)
4876 itr_val = 0x4;
661086df 4877
26b39276
AD		 4878 if (adapter->hw.mac.type == e1000_82575)
4879 itr_val |= itr_val << 16;
661086df 4880 else
0ba82994 4881 itr_val |= E1000_EITR_CNT_IGNR;
661086df 4882
047e0030
AD		 4883 writel(itr_val, q_vector->itr_register);
4884 q_vector->set_itr = 0;
6eb5a7f1
AD		 4885}
4886
047e0030 4887static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4888{
047e0030 4889 struct igb_q_vector *q_vector = data;
9d5c8243 4890
047e0030
AD		 4891 /* Write the ITR value calculated from the previous interrupt. */
4892 igb_write_itr(q_vector);
9d5c8243 4893
047e0030 4894 napi_schedule(&q_vector->napi);
844290e5 4895
047e0030 4896 return IRQ_HANDLED;
fe4506b6
JC		 4897}
4898
421e02f0 4899#ifdef CONFIG_IGB_DCA
047e0030 4900static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4901{
047e0030 4902 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4903 struct e1000_hw *hw = &adapter->hw;
4904 int cpu = get_cpu();
fe4506b6 4905
047e0030
AD		 4906 if (q_vector->cpu == cpu)
4907 goto out_no_update;
4908
0ba82994
AD		 4909 if (q_vector->tx.ring) {
4910 int q = q_vector->tx.ring->reg_idx;
047e0030
AD		 4911 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4912 if (hw->mac.type == e1000_82575) {
4913 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4914 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4915 } else {
047e0030
AD		 4916 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4917 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4918 E1000_DCA_TXCTRL_CPUID_SHIFT;
4919 }
4920 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4921 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4922 }
0ba82994
AD		 4923 if (q_vector->rx.ring) {
4924 int q = q_vector->rx.ring->reg_idx;
047e0030
AD		 4925 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4926 if (hw->mac.type == e1000_82575) {
2d064c06 4927 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4928 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4929 } else {
4930 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4931 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4932 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4933 }
fe4506b6
JC		 4934 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4935 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4936 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4937 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4938 }
047e0030
AD		 4939 q_vector->cpu = cpu;
4940out_no_update:
fe4506b6
JC		 4941 put_cpu();
4942}
4943
4944static void igb_setup_dca(struct igb_adapter *adapter)
4945{
7e0e99ef 4946 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4947 int i;
4948
7dfc16fa 4949 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4950 return;
4951
7e0e99ef
AD		 4952 /* Always use CB2 mode, difference is masked in the CB driver. */
4953 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4954
047e0030 4955 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4956 adapter->q_vector[i]->cpu = -1;
4957 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4958 }
4959}
4960
4961static int __igb_notify_dca(struct device *dev, void *data)
4962{
4963 struct net_device *netdev = dev_get_drvdata(dev);
4964 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4965 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4966 struct e1000_hw *hw = &adapter->hw;
4967 unsigned long event = *(unsigned long *)data;
4968
4969 switch (event) {
4970 case DCA_PROVIDER_ADD:
4971 /* if already enabled, don't do it again */
7dfc16fa 4972 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4973 break;
fe4506b6 4974 if (dca_add_requester(dev) == 0) {
bbd98fe4 4975 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4976 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4977 igb_setup_dca(adapter);
4978 break;
4979 }
4980 /* Fall Through since DCA is disabled. */
4981 case DCA_PROVIDER_REMOVE:
7dfc16fa 4982 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4983 /* without this a class_device is left
047e0030 4984 * hanging around in the sysfs model */
fe4506b6 4985 dca_remove_requester(dev);
090b1795 4986 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4987 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4988 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4989 }
4990 break;
4991 }
bbd98fe4 4992
fe4506b6 4993 return 0;
9d5c8243
AK		 4994}
4995
fe4506b6
JC		 4996static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4997 void *p)
4998{
4999 int ret_val;
5000
5001 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
5002 __igb_notify_dca);
5003
5004 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
5005}
421e02f0 5006#endif /* CONFIG_IGB_DCA */
9d5c8243 5007
0224d663
GR		 5008#ifdef CONFIG_PCI_IOV
5009static int igb_vf_configure(struct igb_adapter *adapter, int vf)
5010{
5011 unsigned char mac_addr[ETH_ALEN];
5012 struct pci_dev *pdev = adapter->pdev;
5013 struct e1000_hw *hw = &adapter->hw;
5014 struct pci_dev *pvfdev;
5015 unsigned int device_id;
5016 u16 thisvf_devfn;
5017
7efd26d0 5018 eth_random_addr(mac_addr);
0224d663
GR		 5019 igb_set_vf_mac(adapter, vf, mac_addr);
5020
5021 switch (adapter->hw.mac.type) {
5022 case e1000_82576:
5023 device_id = IGB_82576_VF_DEV_ID;
5024 /* VF Stride for 82576 is 2 */
5025 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
5026 (pdev->devfn & 1);
5027 break;
5028 case e1000_i350:
5029 device_id = IGB_I350_VF_DEV_ID;
5030 /* VF Stride for I350 is 4 */
5031 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
5032 (pdev->devfn & 3);
5033 break;
5034 default:
5035 device_id = 0;
5036 thisvf_devfn = 0;
5037 break;
5038 }
5039
5040 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5041 while (pvfdev) {
5042 if (pvfdev->devfn == thisvf_devfn)
5043 break;
5044 pvfdev = pci_get_device(hw->vendor_id,
5045 device_id, pvfdev);
5046 }
5047
5048 if (pvfdev)
5049 adapter->vf_data[vf].vfdev = pvfdev;
5050 else
5051 dev_err(&pdev->dev,
5052 "Couldn't find pci dev ptr for VF %4.4x\n",
5053 thisvf_devfn);
5054 return pvfdev != NULL;
5055}
5056
5057static int igb_find_enabled_vfs(struct igb_adapter *adapter)
5058{
5059 struct e1000_hw *hw = &adapter->hw;
5060 struct pci_dev *pdev = adapter->pdev;
5061 struct pci_dev *pvfdev;
5062 u16 vf_devfn = 0;
5063 u16 vf_stride;
5064 unsigned int device_id;
5065 int vfs_found = 0;
5066
5067 switch (adapter->hw.mac.type) {
5068 case e1000_82576:
5069 device_id = IGB_82576_VF_DEV_ID;
5070 /* VF Stride for 82576 is 2 */
5071 vf_stride = 2;
5072 break;
5073 case e1000_i350:
5074 device_id = IGB_I350_VF_DEV_ID;
5075 /* VF Stride for I350 is 4 */
5076 vf_stride = 4;
5077 break;
5078 default:
5079 device_id = 0;
5080 vf_stride = 0;
5081 break;
5082 }
5083
5084 vf_devfn = pdev->devfn + 0x80;
5085 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5086 while (pvfdev) {
06292921
GR		 5087 if (pvfdev->devfn == vf_devfn &&
5088 (pvfdev->bus->number >= pdev->bus->number))
0224d663
GR		 5089 vfs_found++;
5090 vf_devfn += vf_stride;
5091 pvfdev = pci_get_device(hw->vendor_id,
5092 device_id, pvfdev);
5093 }
5094
5095 return vfs_found;
5096}
5097
5098static int igb_check_vf_assignment(struct igb_adapter *adapter)
5099{
5100 int i;
5101 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5102 if (adapter->vf_data[i].vfdev) {
5103 if (adapter->vf_data[i].vfdev->dev_flags &
5104 PCI_DEV_FLAGS_ASSIGNED)
5105 return true;
5106 }
5107 }
5108 return false;
5109}
5110
5111#endif
4ae196df
AD		 5112static void igb_ping_all_vfs(struct igb_adapter *adapter)
5113{
5114 struct e1000_hw *hw = &adapter->hw;
5115 u32 ping;
5116 int i;
5117
5118 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5119 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 5120 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 5121 ping |= E1000_VT_MSGTYPE_CTS;
5122 igb_write_mbx(hw, &ping, 1, i);
5123 }
5124}
5125
7d5753f0
AD		 5126static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5127{
5128 struct e1000_hw *hw = &adapter->hw;
5129 u32 vmolr = rd32(E1000_VMOLR(vf));
5130 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5131
d85b9004 5132 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 5133 IGB_VF_FLAG_MULTI_PROMISC);
5134 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5135
5136 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5137 vmolr |= E1000_VMOLR_MPME;
d85b9004 5138 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 5139 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5140 } else {
5141 /*
5142 * if we have hashes and we are clearing a multicast promisc
5143 * flag we need to write the hashes to the MTA as this step
5144 * was previously skipped
5145 */
5146 if (vf_data->num_vf_mc_hashes > 30) {
5147 vmolr |= E1000_VMOLR_MPME;
5148 } else if (vf_data->num_vf_mc_hashes) {
5149 int j;
5150 vmolr |= E1000_VMOLR_ROMPE;
5151 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5152 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5153 }
5154 }
5155
5156 wr32(E1000_VMOLR(vf), vmolr);
5157
5158 /* there are flags left unprocessed, likely not supported */
5159 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5160 return -EINVAL;
5161
5162 return 0;
5163
5164}
5165
4ae196df
AD		 5166static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5167 u32 *msgbuf, u32 vf)
5168{
5169 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5170 u16 *hash_list = (u16 *)&msgbuf[1];
5171 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5172 int i;
5173
7d5753f0 5174 /* salt away the number of multicast addresses assigned
4ae196df
AD		 5175 * to this VF for later use to restore when the PF multi cast
5176 * list changes
5177 */
5178 vf_data->num_vf_mc_hashes = n;
5179
7d5753f0
AD		 5180 /* only up to 30 hash values supported */
5181 if (n > 30)
5182 n = 30;
5183
5184 /* store the hashes for later use */
4ae196df 5185 for (i = 0; i < n; i++)
a419aef8 5186 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 5187
5188 /* Flush and reset the mta with the new values */
ff41f8dc 5189 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5190
5191 return 0;
5192}
5193
5194static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5195{
5196 struct e1000_hw *hw = &adapter->hw;
5197 struct vf_data_storage *vf_data;
5198 int i, j;
5199
5200 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 5201 u32 vmolr = rd32(E1000_VMOLR(i));
5202 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5203
4ae196df 5204 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 5205
5206 if ((vf_data->num_vf_mc_hashes > 30) ||
5207 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5208 vmolr |= E1000_VMOLR_MPME;
5209 } else if (vf_data->num_vf_mc_hashes) {
5210 vmolr |= E1000_VMOLR_ROMPE;
5211 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5212 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5213 }
5214 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 5215 }
5216}
5217
5218static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5219{
5220 struct e1000_hw *hw = &adapter->hw;
5221 u32 pool_mask, reg, vid;
5222 int i;
5223
5224 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5225
5226 /* Find the vlan filter for this id */
5227 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5228 reg = rd32(E1000_VLVF(i));
5229
5230 /* remove the vf from the pool */
5231 reg &= ~pool_mask;
5232
5233 /* if pool is empty then remove entry from vfta */
5234 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5235 (reg & E1000_VLVF_VLANID_ENABLE)) {
5236 reg = 0;
5237 vid = reg & E1000_VLVF_VLANID_MASK;
5238 igb_vfta_set(hw, vid, false);
5239 }
5240
5241 wr32(E1000_VLVF(i), reg);
5242 }
ae641bdc
AD		 5243
5244 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 5245}
5246
5247static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5248{
5249 struct e1000_hw *hw = &adapter->hw;
5250 u32 reg, i;
5251
51466239
AD		 5252 /* The vlvf table only exists on 82576 hardware and newer */
5253 if (hw->mac.type < e1000_82576)
5254 return -1;
5255
5256 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 5257 if (!adapter->vfs_allocated_count)
5258 return -1;
5259
5260 /* Find the vlan filter for this id */
5261 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5262 reg = rd32(E1000_VLVF(i));
5263 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5264 vid == (reg & E1000_VLVF_VLANID_MASK))
5265 break;
5266 }
5267
5268 if (add) {
5269 if (i == E1000_VLVF_ARRAY_SIZE) {
5270 /* Did not find a matching VLAN ID entry that was
5271 * enabled. Search for a free filter entry, i.e.
5272 * one without the enable bit set
5273 */
5274 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5275 reg = rd32(E1000_VLVF(i));
5276 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5277 break;
5278 }
5279 }
5280 if (i < E1000_VLVF_ARRAY_SIZE) {
5281 /* Found an enabled/available entry */
5282 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5283
5284 /* if !enabled we need to set this up in vfta */
5285 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 5286 /* add VID to filter table */
5287 igb_vfta_set(hw, vid, true);
4ae196df
AD		 5288 reg |= E1000_VLVF_VLANID_ENABLE;
5289 }
cad6d05f
AD		 5290 reg &= ~E1000_VLVF_VLANID_MASK;
5291 reg |= vid;
4ae196df 5292 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5293
5294 /* do not modify RLPML for PF devices */
5295 if (vf >= adapter->vfs_allocated_count)
5296 return 0;
5297
5298 if (!adapter->vf_data[vf].vlans_enabled) {
5299 u32 size;
5300 reg = rd32(E1000_VMOLR(vf));
5301 size = reg & E1000_VMOLR_RLPML_MASK;
5302 size += 4;
5303 reg &= ~E1000_VMOLR_RLPML_MASK;
5304 reg |= size;
5305 wr32(E1000_VMOLR(vf), reg);
5306 }
ae641bdc 5307
51466239 5308 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 5309 }
5310 } else {
5311 if (i < E1000_VLVF_ARRAY_SIZE) {
5312 /* remove vf from the pool */
5313 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5314 /* if pool is empty then remove entry from vfta */
5315 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5316 reg = 0;
5317 igb_vfta_set(hw, vid, false);
5318 }
5319 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5320
5321 /* do not modify RLPML for PF devices */
5322 if (vf >= adapter->vfs_allocated_count)
5323 return 0;
5324
5325 adapter->vf_data[vf].vlans_enabled--;
5326 if (!adapter->vf_data[vf].vlans_enabled) {
5327 u32 size;
5328 reg = rd32(E1000_VMOLR(vf));
5329 size = reg & E1000_VMOLR_RLPML_MASK;
5330 size -= 4;
5331 reg &= ~E1000_VMOLR_RLPML_MASK;
5332 reg |= size;
5333 wr32(E1000_VMOLR(vf), reg);
5334 }
4ae196df
AD		 5335 }
5336 }
8151d294
WM		 5337 return 0;
5338}
5339
5340static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5341{
5342 struct e1000_hw *hw = &adapter->hw;
5343
5344 if (vid)
5345 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5346 else
5347 wr32(E1000_VMVIR(vf), 0);
5348}
5349
5350static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5351 int vf, u16 vlan, u8 qos)
5352{
5353 int err = 0;
5354 struct igb_adapter *adapter = netdev_priv(netdev);
5355
5356 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5357 return -EINVAL;
5358 if (vlan || qos) {
5359 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5360 if (err)
5361 goto out;
5362 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5363 igb_set_vmolr(adapter, vf, !vlan);
5364 adapter->vf_data[vf].pf_vlan = vlan;
5365 adapter->vf_data[vf].pf_qos = qos;
5366 dev_info(&adapter->pdev->dev,
5367 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5368 if (test_bit(__IGB_DOWN, &adapter->state)) {
5369 dev_warn(&adapter->pdev->dev,
5370 "The VF VLAN has been set,"
5371 " but the PF device is not up.\n");
5372 dev_warn(&adapter->pdev->dev,
5373 "Bring the PF device up before"
5374 " attempting to use the VF device.\n");
5375 }
5376 } else {
5377 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5378 false, vf);
5379 igb_set_vmvir(adapter, vlan, vf);
5380 igb_set_vmolr(adapter, vf, true);
5381 adapter->vf_data[vf].pf_vlan = 0;
5382 adapter->vf_data[vf].pf_qos = 0;
5383 }
5384out:
5385 return err;
4ae196df
AD		 5386}
5387
5388static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5389{
5390 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5391 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5392
5393 return igb_vlvf_set(adapter, vid, add, vf);
5394}
5395
f2ca0dbe 5396static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 5397{
8fa7e0f7
GR		 5398 /* clear flags - except flag that indicates PF has set the MAC */
5399 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
f2ca0dbe 5400 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 5401
5402 /* reset offloads to defaults */
8151d294 5403 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 5404
5405 /* reset vlans for device */
5406 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 5407 if (adapter->vf_data[vf].pf_vlan)
5408 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5409 adapter->vf_data[vf].pf_vlan,
5410 adapter->vf_data[vf].pf_qos);
5411 else
5412 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 5413
5414 /* reset multicast table array for vf */
5415 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5416
5417 /* Flush and reset the mta with the new values */
ff41f8dc 5418 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5419}
5420
f2ca0dbe
AD		 5421static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5422{
5423 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5424
5425 /* generate a new mac address as we were hotplug removed/added */
8151d294 5426 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
7efd26d0 5427 eth_random_addr(vf_mac);
f2ca0dbe
AD		 5428
5429 /* process remaining reset events */
5430 igb_vf_reset(adapter, vf);
5431}
5432
5433static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5434{
5435 struct e1000_hw *hw = &adapter->hw;
5436 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5437 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5438 u32 reg, msgbuf[3];
5439 u8 *addr = (u8 *)(&msgbuf[1]);
5440
5441 /* process all the same items cleared in a function level reset */
f2ca0dbe 5442 igb_vf_reset(adapter, vf);
4ae196df
AD		 5443
5444 /* set vf mac address */
26ad9178 5445 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5446
5447 /* enable transmit and receive for vf */
5448 reg = rd32(E1000_VFTE);
5449 wr32(E1000_VFTE, reg | (1 << vf));
5450 reg = rd32(E1000_VFRE);
5451 wr32(E1000_VFRE, reg | (1 << vf));
5452
8fa7e0f7 5453 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
4ae196df
AD		 5454
5455 /* reply to reset with ack and vf mac address */
5456 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5457 memcpy(addr, vf_mac, 6);
5458 igb_write_mbx(hw, msgbuf, 3, vf);
5459}
5460
5461static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5462{
de42edde
GR		 5463 /*
5464 * The VF MAC Address is stored in a packed array of bytes
5465 * starting at the second 32 bit word of the msg array
5466 */
f2ca0dbe
AD		 5467 unsigned char *addr = (char *)&msg[1];
5468 int err = -1;
4ae196df 5469
f2ca0dbe
AD		 5470 if (is_valid_ether_addr(addr))
5471 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5472
f2ca0dbe 5473 return err;
4ae196df
AD		 5474}
5475
5476static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5477{
5478 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5479 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5480 u32 msg = E1000_VT_MSGTYPE_NACK;
5481
5482 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5483 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5484 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5485 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5486 vf_data->last_nack = jiffies;
4ae196df
AD		 5487 }
5488}
5489
f2ca0dbe 5490static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5491{
f2ca0dbe
AD		 5492 struct pci_dev *pdev = adapter->pdev;
5493 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5494 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5495 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5496 s32 retval;
5497
f2ca0dbe 5498 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5499
fef45f4c
AD		 5500 if (retval) {
5501 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5502 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5503 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5504 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5505 return;
5506 goto out;
5507 }
4ae196df
AD		 5508
5509 /* this is a message we already processed, do nothing */
5510 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5511 return;
4ae196df
AD		 5512
5513 /*
5514 * until the vf completes a reset it should not be
5515 * allowed to start any configuration.
5516 */
5517
5518 if (msgbuf[0] == E1000_VF_RESET) {
5519 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5520 return;
4ae196df
AD		 5521 }
5522
f2ca0dbe 5523 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5524 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5525 return;
5526 retval = -1;
5527 goto out;
4ae196df
AD		 5528 }
5529
5530 switch ((msgbuf[0] & 0xFFFF)) {
5531 case E1000_VF_SET_MAC_ADDR:
a6b5ea35
GR		 5532 retval = -EINVAL;
5533 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5534 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5535 else
5536 dev_warn(&pdev->dev,
5537 "VF %d attempted to override administratively "
5538 "set MAC address\nReload the VF driver to "
5539 "resume operations\n", vf);
4ae196df 5540 break;
7d5753f0
AD		 5541 case E1000_VF_SET_PROMISC:
5542 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5543 break;
4ae196df
AD		 5544 case E1000_VF_SET_MULTICAST:
5545 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5546 break;
5547 case E1000_VF_SET_LPE:
5548 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5549 break;
5550 case E1000_VF_SET_VLAN:
a6b5ea35
GR		 5551 retval = -1;
5552 if (vf_data->pf_vlan)
5553 dev_warn(&pdev->dev,
5554 "VF %d attempted to override administratively "
5555 "set VLAN tag\nReload the VF driver to "
5556 "resume operations\n", vf);
8151d294
WM		 5557 else
5558 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5559 break;
5560 default:
090b1795 5561 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5562 retval = -1;
5563 break;
5564 }
5565
fef45f4c
AD		 5566 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5567out:
4ae196df
AD		 5568 /* notify the VF of the results of what it sent us */
5569 if (retval)
5570 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5571 else
5572 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5573
4ae196df 5574 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5575}
4ae196df 5576
f2ca0dbe
AD		 5577static void igb_msg_task(struct igb_adapter *adapter)
5578{
5579 struct e1000_hw *hw = &adapter->hw;
5580 u32 vf;
5581
5582 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5583 /* process any reset requests */
5584 if (!igb_check_for_rst(hw, vf))
5585 igb_vf_reset_event(adapter, vf);
5586
5587 /* process any messages pending */
5588 if (!igb_check_for_msg(hw, vf))
5589 igb_rcv_msg_from_vf(adapter, vf);
5590
5591 /* process any acks */
5592 if (!igb_check_for_ack(hw, vf))
5593 igb_rcv_ack_from_vf(adapter, vf);
5594 }
4ae196df
AD		 5595}
5596
68d480c4
AD		 5597/**
5598 * igb_set_uta - Set unicast filter table address
5599 * @adapter: board private structure
5600 *
5601 * The unicast table address is a register array of 32-bit registers.
5602 * The table is meant to be used in a way similar to how the MTA is used
5603 * however due to certain limitations in the hardware it is necessary to
25985edc
LDM		 5604 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5605 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
68d480c4
AD		 5606 **/
5607static void igb_set_uta(struct igb_adapter *adapter)
5608{
5609 struct e1000_hw *hw = &adapter->hw;
5610 int i;
5611
5612 /* The UTA table only exists on 82576 hardware and newer */
5613 if (hw->mac.type < e1000_82576)
5614 return;
5615
5616 /* we only need to do this if VMDq is enabled */
5617 if (!adapter->vfs_allocated_count)
5618 return;
5619
5620 for (i = 0; i < hw->mac.uta_reg_count; i++)
5621 array_wr32(E1000_UTA, i, ~0);
5622}
5623
9d5c8243
AK		 5624/**
5625 * igb_intr_msi - Interrupt Handler
5626 * @irq: interrupt number
5627 * @data: pointer to a network interface device structure
5628 **/
5629static irqreturn_t igb_intr_msi(int irq, void *data)
5630{
047e0030
AD		 5631 struct igb_adapter *adapter = data;
5632 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5633 struct e1000_hw *hw = &adapter->hw;
5634 /* read ICR disables interrupts using IAM */
5635 u32 icr = rd32(E1000_ICR);
5636
047e0030 5637 igb_write_itr(q_vector);
9d5c8243 5638
7f081d40
AD		 5639 if (icr & E1000_ICR_DRSTA)
5640 schedule_work(&adapter->reset_task);
5641
047e0030 5642 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5643 /* HW is reporting DMA is out of sync */
5644 adapter->stats.doosync++;
5645 }
5646
9d5c8243
AK		 5647 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5648 hw->mac.get_link_status = 1;
5649 if (!test_bit(__IGB_DOWN, &adapter->state))
5650 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5651 }
5652
047e0030 5653 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5654
5655 return IRQ_HANDLED;
5656}
5657
5658/**
4a3c6433 5659 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5660 * @irq: interrupt number
5661 * @data: pointer to a network interface device structure
5662 **/
5663static irqreturn_t igb_intr(int irq, void *data)
5664{
047e0030
AD		 5665 struct igb_adapter *adapter = data;
5666 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5667 struct e1000_hw *hw = &adapter->hw;
5668 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5669 * need for the IMC write */
5670 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5671
5672 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5673 * not set, then the adapter didn't send an interrupt */
5674 if (!(icr & E1000_ICR_INT_ASSERTED))
5675 return IRQ_NONE;
5676
0ba82994
AD		 5677 igb_write_itr(q_vector);
5678
7f081d40
AD		 5679 if (icr & E1000_ICR_DRSTA)
5680 schedule_work(&adapter->reset_task);
5681
047e0030 5682 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5683 /* HW is reporting DMA is out of sync */
5684 adapter->stats.doosync++;
5685 }
5686
9d5c8243
AK		 5687 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5688 hw->mac.get_link_status = 1;
5689 /* guard against interrupt when we're going down */
5690 if (!test_bit(__IGB_DOWN, &adapter->state))
5691 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5692 }
5693
047e0030 5694 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5695
5696 return IRQ_HANDLED;
5697}
5698
c50b52a0 5699static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5700{
047e0030 5701 struct igb_adapter *adapter = q_vector->adapter;
46544258 5702 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5703
0ba82994
AD		 5704 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
5705 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
5706 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
5707 igb_set_itr(q_vector);
46544258 5708 else
047e0030 5709 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5710 }
5711
46544258
AD		 5712 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5713 if (adapter->msix_entries)
047e0030 5714 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5715 else
5716 igb_irq_enable(adapter);
5717 }
9d5c8243
AK		 5718}
5719
46544258
AD		 5720/**
5721 * igb_poll - NAPI Rx polling callback
5722 * @napi: napi polling structure
5723 * @budget: count of how many packets we should handle
5724 **/
5725static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5726{
047e0030
AD		 5727 struct igb_q_vector *q_vector = container_of(napi,
5728 struct igb_q_vector,
5729 napi);
16eb8815 5730 bool clean_complete = true;
9d5c8243 5731
421e02f0 5732#ifdef CONFIG_IGB_DCA
047e0030
AD		 5733 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5734 igb_update_dca(q_vector);
fe4506b6 5735#endif
0ba82994 5736 if (q_vector->tx.ring)
13fde97a 5737 clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5738
0ba82994 5739 if (q_vector->rx.ring)
cd392f5c 5740 clean_complete &= igb_clean_rx_irq(q_vector, budget);
047e0030 5741
16eb8815
AD		 5742 /* If all work not completed, return budget and keep polling */
5743 if (!clean_complete)
5744 return budget;
46544258 5745
9d5c8243 5746 /* If not enough Rx work done, exit the polling mode */
16eb8815
AD		 5747 napi_complete(napi);
5748 igb_ring_irq_enable(q_vector);
9d5c8243 5749
16eb8815 5750 return 0;
9d5c8243 5751}
6d8126f9 5752
7ebae817 5753#ifdef CONFIG_IGB_PTP
c5b9bd5e
AD		 5754/**
5755 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5756 * @q_vector: pointer to q_vector containing needed info
06034649 5757 * @buffer: pointer to igb_tx_buffer structure
33af6bcc
PO		 5758 *
5759 * If we were asked to do hardware stamping and such a time stamp is
5760 * available, then it must have been for this skb here because we only
5761 * allow only one such packet into the queue.
5762 */
06034649
AD		 5763static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
5764 struct igb_tx_buffer *buffer_info)
33af6bcc 5765{
c5b9bd5e 5766 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5767 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5768 struct skb_shared_hwtstamps shhwtstamps;
5769 u64 regval;
33af6bcc 5770
c5b9bd5e 5771 /* if skb does not support hw timestamp or TX stamp not valid exit */
2bbfebe2 5772 if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
c5b9bd5e
AD		 5773 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5774 return;
5775
5776 regval = rd32(E1000_TXSTMPL);
5777 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5778
5779 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5780 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc 5781}
3c89f6d0 5782#endif /* CONFIG_IGB_PTP */
33af6bcc 5783
9d5c8243
AK		 5784/**
5785 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5786 * @q_vector: pointer to q_vector containing needed info
49ce9c2c 5787 *
9d5c8243
AK		 5788 * returns true if ring is completely cleaned
5789 **/
047e0030 5790static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5791{
047e0030 5792 struct igb_adapter *adapter = q_vector->adapter;
0ba82994 5793 struct igb_ring *tx_ring = q_vector->tx.ring;
06034649 5794 struct igb_tx_buffer *tx_buffer;
8542db05 5795 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5796 unsigned int total_bytes = 0, total_packets = 0;
0ba82994 5797 unsigned int budget = q_vector->tx.work_limit;
8542db05 5798 unsigned int i = tx_ring->next_to_clean;
9d5c8243 5799
13fde97a
AD		 5800 if (test_bit(__IGB_DOWN, &adapter->state))
5801 return true;
0e014cb1 5802
06034649 5803 tx_buffer = &tx_ring->tx_buffer_info[i];
13fde97a 5804 tx_desc = IGB_TX_DESC(tx_ring, i);
8542db05 5805 i -= tx_ring->count;
9d5c8243 5806
13fde97a 5807 for (; budget; budget--) {
8542db05 5808 eop_desc = tx_buffer->next_to_watch;
13fde97a 5809
8542db05
AD		 5810 /* prevent any other reads prior to eop_desc */
5811 rmb();
5812
5813 /* if next_to_watch is not set then there is no work pending */
5814 if (!eop_desc)
5815 break;
13fde97a
AD		 5816
5817 /* if DD is not set pending work has not been completed */
5818 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5819 break;
5820
8542db05
AD		 5821 /* clear next_to_watch to prevent false hangs */
5822 tx_buffer->next_to_watch = NULL;
9d5c8243 5823
ebe42d16
AD		 5824 /* update the statistics for this packet */
5825 total_bytes += tx_buffer->bytecount;
5826 total_packets += tx_buffer->gso_segs;
13fde97a 5827
7ebae817 5828#ifdef CONFIG_IGB_PTP
ebe42d16
AD		 5829 /* retrieve hardware timestamp */
5830 igb_tx_hwtstamp(q_vector, tx_buffer);
3c89f6d0 5831#endif /* CONFIG_IGB_PTP */
ebe42d16
AD		 5832
5833 /* free the skb */
5834 dev_kfree_skb_any(tx_buffer->skb);
5835 tx_buffer->skb = NULL;
13fde97a 5836
ebe42d16
AD		 5837 /* unmap skb header data */
5838 dma_unmap_single(tx_ring->dev,
5839 tx_buffer->dma,
5840 tx_buffer->length,
5841 DMA_TO_DEVICE);
5842
5843 /* clear last DMA location and unmap remaining buffers */
5844 while (tx_desc != eop_desc) {
5845 tx_buffer->dma = 0;
9d5c8243 5846
13fde97a
AD		 5847 tx_buffer++;
5848 tx_desc++;
9d5c8243 5849 i++;
8542db05
AD		 5850 if (unlikely(!i)) {
5851 i -= tx_ring->count;
06034649 5852 tx_buffer = tx_ring->tx_buffer_info;
13fde97a
AD		 5853 tx_desc = IGB_TX_DESC(tx_ring, 0);
5854 }
ebe42d16
AD		 5855
5856 /* unmap any remaining paged data */
5857 if (tx_buffer->dma) {
5858 dma_unmap_page(tx_ring->dev,
5859 tx_buffer->dma,
5860 tx_buffer->length,
5861 DMA_TO_DEVICE);
5862 }
5863 }
5864
5865 /* clear last DMA location */
5866 tx_buffer->dma = 0;
5867
5868 /* move us one more past the eop_desc for start of next pkt */
5869 tx_buffer++;
5870 tx_desc++;
5871 i++;
5872 if (unlikely(!i)) {
5873 i -= tx_ring->count;
5874 tx_buffer = tx_ring->tx_buffer_info;
5875 tx_desc = IGB_TX_DESC(tx_ring, 0);
5876 }
0e014cb1
AD		 5877 }
5878
bdbc0631
ED		 5879 netdev_tx_completed_queue(txring_txq(tx_ring),
5880 total_packets, total_bytes);
8542db05 5881 i += tx_ring->count;
9d5c8243 5882 tx_ring->next_to_clean = i;
13fde97a
AD		 5883 u64_stats_update_begin(&tx_ring->tx_syncp);
5884 tx_ring->tx_stats.bytes += total_bytes;
5885 tx_ring->tx_stats.packets += total_packets;
5886 u64_stats_update_end(&tx_ring->tx_syncp);
0ba82994
AD		 5887 q_vector->tx.total_bytes += total_bytes;
5888 q_vector->tx.total_packets += total_packets;
9d5c8243 5889
6d095fa8 5890 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
13fde97a 5891 struct e1000_hw *hw = &adapter->hw;
12dcd86b 5892
8542db05 5893 eop_desc = tx_buffer->next_to_watch;
9d5c8243 5894
9d5c8243
AK		 5895 /* Detect a transmit hang in hardware, this serializes the
5896 * check with the clearing of time_stamp and movement of i */
6d095fa8 5897 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
8542db05
AD		 5898 if (eop_desc &&
5899 time_after(jiffies, tx_buffer->time_stamp +
8e95a202
JP		 5900 (adapter->tx_timeout_factor * HZ)) &&
5901 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5902
9d5c8243 5903 /* detected Tx unit hang */
59d71989 5904 dev_err(tx_ring->dev,
9d5c8243 5905 "Detected Tx Unit Hang\n"
2d064c06 5906 " Tx Queue <%d>\n"
9d5c8243
AK		 5907 " TDH <%x>\n"
5908 " TDT <%x>\n"
5909 " next_to_use <%x>\n"
5910 " next_to_clean <%x>\n"
9d5c8243
AK		 5911 "buffer_info[next_to_clean]\n"
5912 " time_stamp <%lx>\n"
8542db05 5913 " next_to_watch <%p>\n"
9d5c8243
AK		 5914 " jiffies <%lx>\n"
5915 " desc.status <%x>\n",
2d064c06 5916 tx_ring->queue_index,
238ac817 5917 rd32(E1000_TDH(tx_ring->reg_idx)),
fce99e34 5918 readl(tx_ring->tail),
9d5c8243
AK		 5919 tx_ring->next_to_use,
5920 tx_ring->next_to_clean,
8542db05
AD		 5921 tx_buffer->time_stamp,
5922 eop_desc,
9d5c8243 5923 jiffies,
0e014cb1 5924 eop_desc->wb.status);
13fde97a
AD		 5925 netif_stop_subqueue(tx_ring->netdev,
5926 tx_ring->queue_index);
5927
5928 /* we are about to reset, no point in enabling stuff */
5929 return true;
9d5c8243
AK		 5930 }
5931 }
13fde97a
AD		 5932
5933 if (unlikely(total_packets &&
5934 netif_carrier_ok(tx_ring->netdev) &&
5935 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5936 /* Make sure that anybody stopping the queue after this
5937 * sees the new next_to_clean.
5938 */
5939 smp_mb();
5940 if (__netif_subqueue_stopped(tx_ring->netdev,
5941 tx_ring->queue_index) &&
5942 !(test_bit(__IGB_DOWN, &adapter->state))) {
5943 netif_wake_subqueue(tx_ring->netdev,
5944 tx_ring->queue_index);
5945
5946 u64_stats_update_begin(&tx_ring->tx_syncp);
5947 tx_ring->tx_stats.restart_queue++;
5948 u64_stats_update_end(&tx_ring->tx_syncp);
5949 }
5950 }
5951
5952 return !!budget;
9d5c8243
AK		 5953}
5954
cd392f5c 5955static inline void igb_rx_checksum(struct igb_ring *ring,
3ceb90fd
AD		 5956 union e1000_adv_rx_desc *rx_desc,
5957 struct sk_buff *skb)
9d5c8243 5958{
bc8acf2c 5959 skb_checksum_none_assert(skb);
9d5c8243 5960
294e7d78 5961 /* Ignore Checksum bit is set */
3ceb90fd 5962 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
294e7d78
AD		 5963 return;
5964
5965 /* Rx checksum disabled via ethtool */
5966 if (!(ring->netdev->features & NETIF_F_RXCSUM))
9d5c8243 5967 return;
85ad76b2 5968
9d5c8243 5969 /* TCP/UDP checksum error bit is set */
3ceb90fd
AD		 5970 if (igb_test_staterr(rx_desc,
5971 E1000_RXDEXT_STATERR_TCPE |
5972 E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5973 /*
5974 * work around errata with sctp packets where the TCPE aka
5975 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5976 * packets, (aka let the stack check the crc32c)
5977 */
866cff06
AD		 5978 if (!((skb->len == 60) &&
5979 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
12dcd86b 5980 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5981 ring->rx_stats.csum_err++;
12dcd86b
ED		 5982 u64_stats_update_end(&ring->rx_syncp);
5983 }
9d5c8243 5984 /* let the stack verify checksum errors */
9d5c8243
AK		 5985 return;
5986 }
5987 /* It must be a TCP or UDP packet with a valid checksum */
3ceb90fd
AD		 5988 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
5989 E1000_RXD_STAT_UDPCS))
9d5c8243
AK		 5990 skb->ip_summed = CHECKSUM_UNNECESSARY;
5991
3ceb90fd
AD		 5992 dev_dbg(ring->dev, "cksum success: bits %08X\n",
5993 le32_to_cpu(rx_desc->wb.upper.status_error));
9d5c8243
AK		 5994}
5995
077887c3
AD		 5996static inline void igb_rx_hash(struct igb_ring *ring,
5997 union e1000_adv_rx_desc *rx_desc,
5998 struct sk_buff *skb)
5999{
6000 if (ring->netdev->features & NETIF_F_RXHASH)
6001 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
6002}
6003
7ebae817 6004#ifdef CONFIG_IGB_PTP
3ceb90fd
AD		 6005static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
6006 union e1000_adv_rx_desc *rx_desc,
6007 struct sk_buff *skb)
c5b9bd5e
AD		 6008{
6009 struct igb_adapter *adapter = q_vector->adapter;
6010 struct e1000_hw *hw = &adapter->hw;
6011 u64 regval;
6012
3ceb90fd
AD		 6013 if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
6014 E1000_RXDADV_STAT_TS))
6015 return;
6016
c5b9bd5e
AD		 6017 /*
6018 * If this bit is set, then the RX registers contain the time stamp. No
6019 * other packet will be time stamped until we read these registers, so
6020 * read the registers to make them available again. Because only one
6021 * packet can be time stamped at a time, we know that the register
6022 * values must belong to this one here and therefore we don't need to
6023 * compare any of the additional attributes stored for it.
6024 *
2244d07b 6025 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 6026 * can turn into a skb_shared_hwtstamps.
6027 */
3ceb90fd 6028 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
757b77e2
NN		 6029 u32 *stamp = (u32 *)skb->data;
6030 regval = le32_to_cpu(*(stamp + 2));
6031 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
6032 skb_pull(skb, IGB_TS_HDR_LEN);
6033 } else {
6034 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
6035 return;
c5b9bd5e 6036
757b77e2
NN		 6037 regval = rd32(E1000_RXSTMPL);
6038 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
6039 }
c5b9bd5e
AD		 6040
6041 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
6042}
3c89f6d0 6043#endif /* CONFIG_IGB_PTP */
8be10e91
AD		 6044
6045static void igb_rx_vlan(struct igb_ring *ring,
6046 union e1000_adv_rx_desc *rx_desc,
6047 struct sk_buff *skb)
6048{
6049 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
6050 u16 vid;
6051 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
6052 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
6053 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
6054 else
6055 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
6056
6057 __vlan_hwaccel_put_tag(skb, vid);
6058 }
6059}
6060
44390ca6 6061static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
2d94d8ab
AD		 6062{
6063 /* HW will not DMA in data larger than the given buffer, even if it
6064 * parses the (NFS, of course) header to be larger. In that case, it
6065 * fills the header buffer and spills the rest into the page.
6066 */
6067 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
6068 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
44390ca6
AD		 6069 if (hlen > IGB_RX_HDR_LEN)
6070 hlen = IGB_RX_HDR_LEN;
2d94d8ab
AD		 6071 return hlen;
6072}
6073
cd392f5c 6074static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
9d5c8243 6075{
0ba82994 6076 struct igb_ring *rx_ring = q_vector->rx.ring;
16eb8815
AD		 6077 union e1000_adv_rx_desc *rx_desc;
6078 const int current_node = numa_node_id();
9d5c8243 6079 unsigned int total_bytes = 0, total_packets = 0;
16eb8815
AD		 6080 u16 cleaned_count = igb_desc_unused(rx_ring);
6081 u16 i = rx_ring->next_to_clean;
9d5c8243 6082
60136906 6083 rx_desc = IGB_RX_DESC(rx_ring, i);
9d5c8243 6084
3ceb90fd 6085 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
06034649 6086 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
16eb8815
AD		 6087 struct sk_buff *skb = buffer_info->skb;
6088 union e1000_adv_rx_desc *next_rxd;
9d5c8243 6089
69d3ca53 6090 buffer_info->skb = NULL;
16eb8815 6091 prefetch(skb->data);
69d3ca53
AD		 6092
6093 i++;
6094 if (i == rx_ring->count)
6095 i = 0;
42d0781a 6096
60136906 6097 next_rxd = IGB_RX_DESC(rx_ring, i);
69d3ca53 6098 prefetch(next_rxd);
9d5c8243 6099
16eb8815
AD		 6100 /*
6101 * This memory barrier is needed to keep us from reading
6102 * any other fields out of the rx_desc until we know the
6103 * RXD_STAT_DD bit is set
6104 */
6105 rmb();
9d5c8243 6106
16eb8815
AD		 6107 if (!skb_is_nonlinear(skb)) {
6108 __skb_put(skb, igb_get_hlen(rx_desc));
6109 dma_unmap_single(rx_ring->dev, buffer_info->dma,
44390ca6 6110 IGB_RX_HDR_LEN,
59d71989 6111 DMA_FROM_DEVICE);
91615f76 6112 buffer_info->dma = 0;
bf36c1a0
AD		 6113 }
6114
16eb8815
AD		 6115 if (rx_desc->wb.upper.length) {
6116 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
bf36c1a0 6117
aa913403 6118 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 6119 buffer_info->page,
6120 buffer_info->page_offset,
6121 length);
6122
16eb8815
AD		 6123 skb->len += length;
6124 skb->data_len += length;
95b9c1df 6125 skb->truesize += PAGE_SIZE / 2;
16eb8815 6126
d1eff350
AD		 6127 if ((page_count(buffer_info->page) != 1) ||
6128 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 6129 buffer_info->page = NULL;
6130 else
6131 get_page(buffer_info->page);
9d5c8243 6132
16eb8815
AD		 6133 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
6134 PAGE_SIZE / 2, DMA_FROM_DEVICE);
6135 buffer_info->page_dma = 0;
9d5c8243 6136 }
9d5c8243 6137
3ceb90fd 6138 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
06034649
AD		 6139 struct igb_rx_buffer *next_buffer;
6140 next_buffer = &rx_ring->rx_buffer_info[i];
b2d56536
AD		 6141 buffer_info->skb = next_buffer->skb;
6142 buffer_info->dma = next_buffer->dma;
6143 next_buffer->skb = skb;
6144 next_buffer->dma = 0;
bf36c1a0
AD		 6145 goto next_desc;
6146 }
44390ca6 6147
89eaefb6
BG		 6148 if (unlikely((igb_test_staterr(rx_desc,
6149 E1000_RXDEXT_ERR_FRAME_ERR_MASK))
6150 && !(rx_ring->netdev->features & NETIF_F_RXALL))) {
16eb8815 6151 dev_kfree_skb_any(skb);
9d5c8243
AK		 6152 goto next_desc;
6153 }
9d5c8243 6154
7ebae817 6155#ifdef CONFIG_IGB_PTP
3ceb90fd 6156 igb_rx_hwtstamp(q_vector, rx_desc, skb);
3c89f6d0 6157#endif /* CONFIG_IGB_PTP */
077887c3 6158 igb_rx_hash(rx_ring, rx_desc, skb);
3ceb90fd 6159 igb_rx_checksum(rx_ring, rx_desc, skb);
8be10e91 6160 igb_rx_vlan(rx_ring, rx_desc, skb);
3ceb90fd
AD		 6161
6162 total_bytes += skb->len;
6163 total_packets++;
6164
6165 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
6166
b2cb09b1 6167 napi_gro_receive(&q_vector->napi, skb);
9d5c8243 6168
16eb8815 6169 budget--;
9d5c8243 6170next_desc:
16eb8815
AD		 6171 if (!budget)
6172 break;
6173
6174 cleaned_count++;
9d5c8243
AK		 6175 /* return some buffers to hardware, one at a time is too slow */
6176 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
cd392f5c 6177 igb_alloc_rx_buffers(rx_ring, cleaned_count);
9d5c8243
AK		 6178 cleaned_count = 0;
6179 }
6180
6181 /* use prefetched values */
6182 rx_desc = next_rxd;
9d5c8243 6183 }
bf36c1a0 6184
9d5c8243 6185 rx_ring->next_to_clean = i;
12dcd86b 6186 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 6187 rx_ring->rx_stats.packets += total_packets;
6188 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 6189 u64_stats_update_end(&rx_ring->rx_syncp);
0ba82994
AD		 6190 q_vector->rx.total_packets += total_packets;
6191 q_vector->rx.total_bytes += total_bytes;
c023cd88
AD		 6192
6193 if (cleaned_count)
cd392f5c 6194 igb_alloc_rx_buffers(rx_ring, cleaned_count);
c023cd88 6195
16eb8815 6196 return !!budget;
9d5c8243
AK		 6197}
6198
c023cd88 6199static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
06034649 6200 struct igb_rx_buffer *bi)
c023cd88
AD		 6201{
6202 struct sk_buff *skb = bi->skb;
6203 dma_addr_t dma = bi->dma;
6204
6205 if (dma)
6206 return true;
6207
6208 if (likely(!skb)) {
6209 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6210 IGB_RX_HDR_LEN);
6211 bi->skb = skb;
6212 if (!skb) {
6213 rx_ring->rx_stats.alloc_failed++;
6214 return false;
6215 }
6216
6217 /* initialize skb for ring */
6218 skb_record_rx_queue(skb, rx_ring->queue_index);
6219 }
6220
6221 dma = dma_map_single(rx_ring->dev, skb->data,
6222 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6223
6224 if (dma_mapping_error(rx_ring->dev, dma)) {
6225 rx_ring->rx_stats.alloc_failed++;
6226 return false;
6227 }
6228
6229 bi->dma = dma;
6230 return true;
6231}
6232
6233static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
06034649 6234 struct igb_rx_buffer *bi)
c023cd88
AD		 6235{
6236 struct page *page = bi->page;
6237 dma_addr_t page_dma = bi->page_dma;
6238 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6239
6240 if (page_dma)
6241 return true;
6242
6243 if (!page) {
0614002b 6244 page = __skb_alloc_page(GFP_ATOMIC, bi->skb);
c023cd88
AD		 6245 bi->page = page;
6246 if (unlikely(!page)) {
6247 rx_ring->rx_stats.alloc_failed++;
6248 return false;
6249 }
6250 }
6251
6252 page_dma = dma_map_page(rx_ring->dev, page,
6253 page_offset, PAGE_SIZE / 2,
6254 DMA_FROM_DEVICE);
6255
6256 if (dma_mapping_error(rx_ring->dev, page_dma)) {
6257 rx_ring->rx_stats.alloc_failed++;
6258 return false;
6259 }
6260
6261 bi->page_dma = page_dma;
6262 bi->page_offset = page_offset;
6263 return true;
6264}
6265
9d5c8243 6266/**
cd392f5c 6267 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
9d5c8243
AK		 6268 * @adapter: address of board private structure
6269 **/
cd392f5c 6270void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9d5c8243 6271{
9d5c8243 6272 union e1000_adv_rx_desc *rx_desc;
06034649 6273 struct igb_rx_buffer *bi;
c023cd88 6274 u16 i = rx_ring->next_to_use;
9d5c8243 6275
60136906 6276 rx_desc = IGB_RX_DESC(rx_ring, i);
06034649 6277 bi = &rx_ring->rx_buffer_info[i];
c023cd88 6278 i -= rx_ring->count;
9d5c8243
AK		 6279
6280 while (cleaned_count--) {
c023cd88
AD		 6281 if (!igb_alloc_mapped_skb(rx_ring, bi))
6282 break;
9d5c8243 6283
c023cd88
AD		 6284 /* Refresh the desc even if buffer_addrs didn't change
6285 * because each write-back erases this info. */
6286 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
9d5c8243 6287
c023cd88
AD		 6288 if (!igb_alloc_mapped_page(rx_ring, bi))
6289 break;
6290
6291 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
9d5c8243 6292
c023cd88
AD		 6293 rx_desc++;
6294 bi++;
9d5c8243 6295 i++;
c023cd88 6296 if (unlikely(!i)) {
60136906 6297 rx_desc = IGB_RX_DESC(rx_ring, 0);
06034649 6298 bi = rx_ring->rx_buffer_info;
c023cd88
AD		 6299 i -= rx_ring->count;
6300 }
6301
6302 /* clear the hdr_addr for the next_to_use descriptor */
6303 rx_desc->read.hdr_addr = 0;
9d5c8243
AK		 6304 }
6305
c023cd88
AD		 6306 i += rx_ring->count;
6307
9d5c8243
AK		 6308 if (rx_ring->next_to_use != i) {
6309 rx_ring->next_to_use = i;
9d5c8243
AK		 6310
6311 /* Force memory writes to complete before letting h/w
6312 * know there are new descriptors to fetch. (Only
6313 * applicable for weak-ordered memory model archs,
6314 * such as IA-64). */
6315 wmb();
fce99e34 6316 writel(i, rx_ring->tail);
9d5c8243
AK		 6317 }
6318}
6319
6320/**
6321 * igb_mii_ioctl -
6322 * @netdev:
6323 * @ifreq:
6324 * @cmd:
6325 **/
6326static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6327{
6328 struct igb_adapter *adapter = netdev_priv(netdev);
6329 struct mii_ioctl_data *data = if_mii(ifr);
6330
6331 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6332 return -EOPNOTSUPP;
6333
6334 switch (cmd) {
6335 case SIOCGMIIPHY:
6336 data->phy_id = adapter->hw.phy.addr;
6337 break;
6338 case SIOCGMIIREG:
f5f4cf08
AD		 6339 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6340 &data->val_out))
9d5c8243
AK		 6341 return -EIO;
6342 break;
6343 case SIOCSMIIREG:
6344 default:
6345 return -EOPNOTSUPP;
6346 }
6347 return 0;
6348}
6349
3c89f6d0 6350#ifdef CONFIG_IGB_PTP
c6cb090b
PO		 6351/**
6352 * igb_hwtstamp_ioctl - control hardware time stamping
6353 * @netdev:
6354 * @ifreq:
6355 * @cmd:
6356 *
33af6bcc
PO		 6357 * Outgoing time stamping can be enabled and disabled. Play nice and
6358 * disable it when requested, although it shouldn't case any overhead
6359 * when no packet needs it. At most one packet in the queue may be
6360 * marked for time stamping, otherwise it would be impossible to tell
6361 * for sure to which packet the hardware time stamp belongs.
6362 *
6363 * Incoming time stamping has to be configured via the hardware
6364 * filters. Not all combinations are supported, in particular event
6365 * type has to be specified. Matching the kind of event packet is
6366 * not supported, with the exception of "all V2 events regardless of
6367 * level 2 or 4".
6368 *
c6cb090b
PO		 6369 **/
6370static int igb_hwtstamp_ioctl(struct net_device *netdev,
6371 struct ifreq *ifr, int cmd)
6372{
33af6bcc
PO		 6373 struct igb_adapter *adapter = netdev_priv(netdev);
6374 struct e1000_hw *hw = &adapter->hw;
c6cb090b 6375 struct hwtstamp_config config;
c5b9bd5e
AD		 6376 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6377 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 6378 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 6379 bool is_l4 = false;
6380 bool is_l2 = false;
33af6bcc 6381 u32 regval;
c6cb090b
PO		 6382
6383 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6384 return -EFAULT;
6385
6386 /* reserved for future extensions */
6387 if (config.flags)
6388 return -EINVAL;
6389
33af6bcc
PO		 6390 switch (config.tx_type) {
6391 case HWTSTAMP_TX_OFF:
c5b9bd5e 6392 tsync_tx_ctl = 0;
33af6bcc 6393 case HWTSTAMP_TX_ON:
33af6bcc
PO		 6394 break;
6395 default:
6396 return -ERANGE;
6397 }
6398
6399 switch (config.rx_filter) {
6400 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 6401 tsync_rx_ctl = 0;
33af6bcc
PO		 6402 break;
6403 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6404 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6405 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6406 case HWTSTAMP_FILTER_ALL:
6407 /*
6408 * register TSYNCRXCFG must be set, therefore it is not
6409 * possible to time stamp both Sync and Delay_Req messages
6410 * => fall back to time stamping all packets
6411 */
c5b9bd5e 6412 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 6413 config.rx_filter = HWTSTAMP_FILTER_ALL;
6414 break;
6415 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 6416 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6417 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 6418 is_l4 = true;
33af6bcc
PO		 6419 break;
6420 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 6421 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6422 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 6423 is_l4 = true;
33af6bcc
PO		 6424 break;
6425 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6426 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 6427 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6428 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 6429 is_l2 = true;
6430 is_l4 = true;
33af6bcc
PO		 6431 config.rx_filter = HWTSTAMP_FILTER_SOME;
6432 break;
6433 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6434 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 6435 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6436 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 6437 is_l2 = true;
6438 is_l4 = true;
33af6bcc
PO		 6439 config.rx_filter = HWTSTAMP_FILTER_SOME;
6440 break;
6441 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6442 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6443 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 6444 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 6445 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 6446 is_l2 = true;
11ba69e8 6447 is_l4 = true;
33af6bcc
PO		 6448 break;
6449 default:
6450 return -ERANGE;
6451 }
6452
c5b9bd5e
AD		 6453 if (hw->mac.type == e1000_82575) {
6454 if (tsync_rx_ctl | tsync_tx_ctl)
6455 return -EINVAL;
6456 return 0;
6457 }
6458
757b77e2
NN		 6459 /*
6460 * Per-packet timestamping only works if all packets are
6461 * timestamped, so enable timestamping in all packets as
6462 * long as one rx filter was configured.
6463 */
06218a8d 6464 if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
757b77e2
NN		 6465 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6466 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6467 }
6468
33af6bcc
PO		 6469 /* enable/disable TX */
6470 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 6471 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6472 regval |= tsync_tx_ctl;
33af6bcc
PO		 6473 wr32(E1000_TSYNCTXCTL, regval);
6474
c5b9bd5e 6475 /* enable/disable RX */
33af6bcc 6476 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 6477 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6478 regval |= tsync_rx_ctl;
33af6bcc 6479 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 6480
c5b9bd5e
AD		 6481 /* define which PTP packets are time stamped */
6482 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 6483
c5b9bd5e
AD		 6484 /* define ethertype filter for timestamped packets */
6485 if (is_l2)
6486 wr32(E1000_ETQF(3),
6487 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6488 E1000_ETQF_1588 | /* enable timestamping */
6489 ETH_P_1588)); /* 1588 eth protocol type */
6490 else
6491 wr32(E1000_ETQF(3), 0);
6492
6493#define PTP_PORT 319
6494 /* L4 Queue Filter[3]: filter by destination port and protocol */
6495 if (is_l4) {
6496 u32 ftqf = (IPPROTO_UDP /* UDP */
6497 | E1000_FTQF_VF_BP /* VF not compared */
6498 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6499 | E1000_FTQF_MASK); /* mask all inputs */
6500 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6501
6502 wr32(E1000_IMIR(3), htons(PTP_PORT));
6503 wr32(E1000_IMIREXT(3),
6504 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6505 if (hw->mac.type == e1000_82576) {
6506 /* enable source port check */
6507 wr32(E1000_SPQF(3), htons(PTP_PORT));
6508 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6509 }
6510 wr32(E1000_FTQF(3), ftqf);
6511 } else {
6512 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6513 }
33af6bcc
PO		 6514 wrfl();
6515
6516 adapter->hwtstamp_config = config;
6517
6518 /* clear TX/RX time stamp registers, just to be sure */
6519 regval = rd32(E1000_TXSTMPH);
6520 regval = rd32(E1000_RXSTMPH);
c6cb090b 6521
33af6bcc
PO		 6522 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6523 -EFAULT : 0;
c6cb090b 6524}
3c89f6d0 6525#endif /* CONFIG_IGB_PTP */
c6cb090b 6526
9d5c8243
AK		 6527/**
6528 * igb_ioctl -
6529 * @netdev:
6530 * @ifreq:
6531 * @cmd:
6532 **/
6533static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6534{
6535 switch (cmd) {
6536 case SIOCGMIIPHY:
6537 case SIOCGMIIREG:
6538 case SIOCSMIIREG:
6539 return igb_mii_ioctl(netdev, ifr, cmd);
3c89f6d0 6540#ifdef CONFIG_IGB_PTP
c6cb090b
PO		 6541 case SIOCSHWTSTAMP:
6542 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
3c89f6d0 6543#endif /* CONFIG_IGB_PTP */
9d5c8243
AK		 6544 default:
6545 return -EOPNOTSUPP;
6546 }
6547}
6548
009bc06e
AD		 6549s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6550{
6551 struct igb_adapter *adapter = hw->back;
6552 u16 cap_offset;
6553
bdaae04c 6554 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6555 if (!cap_offset)
6556 return -E1000_ERR_CONFIG;
6557
6558 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6559
6560 return 0;
6561}
6562
6563s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6564{
6565 struct igb_adapter *adapter = hw->back;
6566 u16 cap_offset;
6567
bdaae04c 6568 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6569 if (!cap_offset)
6570 return -E1000_ERR_CONFIG;
6571
6572 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6573
6574 return 0;
6575}
6576
c8f44aff 6577static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
9d5c8243
AK		 6578{
6579 struct igb_adapter *adapter = netdev_priv(netdev);
6580 struct e1000_hw *hw = &adapter->hw;
6581 u32 ctrl, rctl;
5faf030c 6582 bool enable = !!(features & NETIF_F_HW_VLAN_RX);
9d5c8243 6583
5faf030c 6584 if (enable) {
9d5c8243
AK		 6585 /* enable VLAN tag insert/strip */
6586 ctrl = rd32(E1000_CTRL);
6587 ctrl |= E1000_CTRL_VME;
6588 wr32(E1000_CTRL, ctrl);
6589
51466239 6590 /* Disable CFI check */
9d5c8243 6591 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6592 rctl &= ~E1000_RCTL_CFIEN;
6593 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6594 } else {
6595 /* disable VLAN tag insert/strip */
6596 ctrl = rd32(E1000_CTRL);
6597 ctrl &= ~E1000_CTRL_VME;
6598 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6599 }
6600
e1739522 6601 igb_rlpml_set(adapter);
9d5c8243
AK		 6602}
6603
8e586137 6604static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
9d5c8243
AK		 6605{
6606 struct igb_adapter *adapter = netdev_priv(netdev);
6607 struct e1000_hw *hw = &adapter->hw;
4ae196df 6608 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6609
51466239
AD		 6610 /* attempt to add filter to vlvf array */
6611 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6612
51466239
AD		 6613 /* add the filter since PF can receive vlans w/o entry in vlvf */
6614 igb_vfta_set(hw, vid, true);
b2cb09b1
JP		 6615
6616 set_bit(vid, adapter->active_vlans);
8e586137
JP		 6617
6618 return 0;
9d5c8243
AK		 6619}
6620
8e586137 6621static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
9d5c8243
AK		 6622{
6623 struct igb_adapter *adapter = netdev_priv(netdev);
6624 struct e1000_hw *hw = &adapter->hw;
4ae196df 6625 int pf_id = adapter->vfs_allocated_count;
51466239 6626 s32 err;
9d5c8243 6627
51466239
AD		 6628 /* remove vlan from VLVF table array */
6629 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6630
51466239
AD		 6631 /* if vid was not present in VLVF just remove it from table */
6632 if (err)
4ae196df 6633 igb_vfta_set(hw, vid, false);
b2cb09b1
JP		 6634
6635 clear_bit(vid, adapter->active_vlans);
8e586137
JP		 6636
6637 return 0;
9d5c8243
AK		 6638}
6639
6640static void igb_restore_vlan(struct igb_adapter *adapter)
6641{
b2cb09b1 6642 u16 vid;
9d5c8243 6643
5faf030c
AD		 6644 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
6645
b2cb09b1
JP		 6646 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6647 igb_vlan_rx_add_vid(adapter->netdev, vid);
9d5c8243
AK		 6648}
6649
14ad2513 6650int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9d5c8243 6651{
090b1795 6652 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6653 struct e1000_mac_info *mac = &adapter->hw.mac;
6654
6655 mac->autoneg = 0;
6656
14ad2513
DD		 6657 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6658 * for the switch() below to work */
6659 if ((spd & 1) || (dplx & ~1))
6660 goto err_inval;
6661
cd2638a8
CW		 6662 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6663 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
14ad2513
DD		 6664 spd != SPEED_1000 &&
6665 dplx != DUPLEX_FULL)
6666 goto err_inval;
cd2638a8 6667
14ad2513 6668 switch (spd + dplx) {
9d5c8243
AK		 6669 case SPEED_10 + DUPLEX_HALF:
6670 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6671 break;
6672 case SPEED_10 + DUPLEX_FULL:
6673 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6674 break;
6675 case SPEED_100 + DUPLEX_HALF:
6676 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6677 break;
6678 case SPEED_100 + DUPLEX_FULL:
6679 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6680 break;
6681 case SPEED_1000 + DUPLEX_FULL:
6682 mac->autoneg = 1;
6683 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6684 break;
6685 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6686 default:
14ad2513 6687 goto err_inval;
9d5c8243 6688 }
8376dad0
JB		 6689
6690 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
6691 adapter->hw.phy.mdix = AUTO_ALL_MODES;
6692
9d5c8243 6693 return 0;
14ad2513
DD		 6694
6695err_inval:
6696 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6697 return -EINVAL;
9d5c8243
AK		 6698}
6699
749ab2cd
YZ		 6700static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
6701 bool runtime)
9d5c8243
AK		 6702{
6703 struct net_device *netdev = pci_get_drvdata(pdev);
6704 struct igb_adapter *adapter = netdev_priv(netdev);
6705 struct e1000_hw *hw = &adapter->hw;
2d064c06 6706 u32 ctrl, rctl, status;
749ab2cd 6707 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
9d5c8243
AK		 6708#ifdef CONFIG_PM
6709 int retval = 0;
6710#endif
6711
6712 netif_device_detach(netdev);
6713
a88f10ec 6714 if (netif_running(netdev))
749ab2cd 6715 __igb_close(netdev, true);
a88f10ec 6716
047e0030 6717 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6718
6719#ifdef CONFIG_PM
6720 retval = pci_save_state(pdev);
6721 if (retval)
6722 return retval;
6723#endif
6724
6725 status = rd32(E1000_STATUS);
6726 if (status & E1000_STATUS_LU)
6727 wufc &= ~E1000_WUFC_LNKC;
6728
6729 if (wufc) {
6730 igb_setup_rctl(adapter);
ff41f8dc 6731 igb_set_rx_mode(netdev);
9d5c8243
AK		 6732
6733 /* turn on all-multi mode if wake on multicast is enabled */
6734 if (wufc & E1000_WUFC_MC) {
6735 rctl = rd32(E1000_RCTL);
6736 rctl |= E1000_RCTL_MPE;
6737 wr32(E1000_RCTL, rctl);
6738 }
6739
6740 ctrl = rd32(E1000_CTRL);
6741 /* advertise wake from D3Cold */
6742 #define E1000_CTRL_ADVD3WUC 0x00100000
6743 /* phy power management enable */
6744 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6745 ctrl |= E1000_CTRL_ADVD3WUC;
6746 wr32(E1000_CTRL, ctrl);
6747
9d5c8243 6748 /* Allow time for pending master requests to run */
330a6d6a 6749 igb_disable_pcie_master(hw);
9d5c8243
AK		 6750
6751 wr32(E1000_WUC, E1000_WUC_PME_EN);
6752 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6753 } else {
6754 wr32(E1000_WUC, 0);
6755 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6756 }
6757
3fe7c4c9
RW		 6758 *enable_wake = wufc || adapter->en_mng_pt;
6759 if (!*enable_wake)
88a268c1
NN		 6760 igb_power_down_link(adapter);
6761 else
6762 igb_power_up_link(adapter);
9d5c8243
AK		 6763
6764 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6765 * would have already happened in close and is redundant. */
6766 igb_release_hw_control(adapter);
6767
6768 pci_disable_device(pdev);
6769
9d5c8243
AK		 6770 return 0;
6771}
6772
6773#ifdef CONFIG_PM
d9dd966d 6774#ifdef CONFIG_PM_SLEEP
749ab2cd 6775static int igb_suspend(struct device *dev)
3fe7c4c9
RW		 6776{
6777 int retval;
6778 bool wake;
749ab2cd 6779 struct pci_dev *pdev = to_pci_dev(dev);
3fe7c4c9 6780
749ab2cd 6781 retval = __igb_shutdown(pdev, &wake, 0);
3fe7c4c9
RW		 6782 if (retval)
6783 return retval;
6784
6785 if (wake) {
6786 pci_prepare_to_sleep(pdev);
6787 } else {
6788 pci_wake_from_d3(pdev, false);
6789 pci_set_power_state(pdev, PCI_D3hot);
6790 }
6791
6792 return 0;
6793}
d9dd966d 6794#endif /* CONFIG_PM_SLEEP */
3fe7c4c9 6795
749ab2cd 6796static int igb_resume(struct device *dev)
9d5c8243 6797{
749ab2cd 6798 struct pci_dev *pdev = to_pci_dev(dev);
9d5c8243
AK		 6799 struct net_device *netdev = pci_get_drvdata(pdev);
6800 struct igb_adapter *adapter = netdev_priv(netdev);
6801 struct e1000_hw *hw = &adapter->hw;
6802 u32 err;
6803
6804 pci_set_power_state(pdev, PCI_D0);
6805 pci_restore_state(pdev);
b94f2d77 6806 pci_save_state(pdev);
42bfd33a 6807
aed5dec3 6808 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6809 if (err) {
6810 dev_err(&pdev->dev,
6811 "igb: Cannot enable PCI device from suspend\n");
6812 return err;
6813 }
6814 pci_set_master(pdev);
6815
6816 pci_enable_wake(pdev, PCI_D3hot, 0);
6817 pci_enable_wake(pdev, PCI_D3cold, 0);
6818
cfb8c3aa 6819 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6820 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6821 return -ENOMEM;
9d5c8243
AK		 6822 }
6823
9d5c8243 6824 igb_reset(adapter);
a8564f03
AD		 6825
6826 /* let the f/w know that the h/w is now under the control of the
6827 * driver. */
6828 igb_get_hw_control(adapter);
6829
9d5c8243
AK		 6830 wr32(E1000_WUS, ~0);
6831
749ab2cd
YZ		 6832 if (netdev->flags & IFF_UP) {
6833 err = __igb_open(netdev, true);
a88f10ec
AD		 6834 if (err)
6835 return err;
6836 }
9d5c8243
AK		 6837
6838 netif_device_attach(netdev);
749ab2cd
YZ		 6839 return 0;
6840}
6841
6842#ifdef CONFIG_PM_RUNTIME
6843static int igb_runtime_idle(struct device *dev)
6844{
6845 struct pci_dev *pdev = to_pci_dev(dev);
6846 struct net_device *netdev = pci_get_drvdata(pdev);
6847 struct igb_adapter *adapter = netdev_priv(netdev);
6848
6849 if (!igb_has_link(adapter))
6850 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6851
6852 return -EBUSY;
6853}
6854
6855static int igb_runtime_suspend(struct device *dev)
6856{
6857 struct pci_dev *pdev = to_pci_dev(dev);
6858 int retval;
6859 bool wake;
6860
6861 retval = __igb_shutdown(pdev, &wake, 1);
6862 if (retval)
6863 return retval;
6864
6865 if (wake) {
6866 pci_prepare_to_sleep(pdev);
6867 } else {
6868 pci_wake_from_d3(pdev, false);
6869 pci_set_power_state(pdev, PCI_D3hot);
6870 }
9d5c8243 6871
9d5c8243
AK		 6872 return 0;
6873}
749ab2cd
YZ		 6874
6875static int igb_runtime_resume(struct device *dev)
6876{
6877 return igb_resume(dev);
6878}
6879#endif /* CONFIG_PM_RUNTIME */
9d5c8243
AK		 6880#endif
6881
6882static void igb_shutdown(struct pci_dev *pdev)
6883{
3fe7c4c9
RW		 6884 bool wake;
6885
749ab2cd 6886 __igb_shutdown(pdev, &wake, 0);
3fe7c4c9
RW		 6887
6888 if (system_state == SYSTEM_POWER_OFF) {
6889 pci_wake_from_d3(pdev, wake);
6890 pci_set_power_state(pdev, PCI_D3hot);
6891 }
9d5c8243
AK		 6892}
6893
6894#ifdef CONFIG_NET_POLL_CONTROLLER
6895/*
6896 * Polling 'interrupt' - used by things like netconsole to send skbs
6897 * without having to re-enable interrupts. It's not called while
6898 * the interrupt routine is executing.
6899 */
6900static void igb_netpoll(struct net_device *netdev)
6901{
6902 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6903 struct e1000_hw *hw = &adapter->hw;
0d1ae7f4 6904 struct igb_q_vector *q_vector;
9d5c8243 6905 int i;
9d5c8243 6906
047e0030 6907 for (i = 0; i < adapter->num_q_vectors; i++) {
0d1ae7f4
AD		 6908 q_vector = adapter->q_vector[i];
6909 if (adapter->msix_entries)
6910 wr32(E1000_EIMC, q_vector->eims_value);
6911 else
6912 igb_irq_disable(adapter);
047e0030 6913 napi_schedule(&q_vector->napi);
eebbbdba 6914 }
9d5c8243
AK		 6915}
6916#endif /* CONFIG_NET_POLL_CONTROLLER */
6917
6918/**
6919 * igb_io_error_detected - called when PCI error is detected
6920 * @pdev: Pointer to PCI device
6921 * @state: The current pci connection state
6922 *
6923 * This function is called after a PCI bus error affecting
6924 * this device has been detected.
6925 */
6926static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6927 pci_channel_state_t state)
6928{
6929 struct net_device *netdev = pci_get_drvdata(pdev);
6930 struct igb_adapter *adapter = netdev_priv(netdev);
6931
6932 netif_device_detach(netdev);
6933
59ed6eec
AD		 6934 if (state == pci_channel_io_perm_failure)
6935 return PCI_ERS_RESULT_DISCONNECT;
6936
9d5c8243
AK		 6937 if (netif_running(netdev))
6938 igb_down(adapter);
6939 pci_disable_device(pdev);
6940
6941 /* Request a slot slot reset. */
6942 return PCI_ERS_RESULT_NEED_RESET;
6943}
6944
6945/**
6946 * igb_io_slot_reset - called after the pci bus has been reset.
6947 * @pdev: Pointer to PCI device
6948 *
6949 * Restart the card from scratch, as if from a cold-boot. Implementation
6950 * resembles the first-half of the igb_resume routine.
6951 */
6952static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6953{
6954 struct net_device *netdev = pci_get_drvdata(pdev);
6955 struct igb_adapter *adapter = netdev_priv(netdev);
6956 struct e1000_hw *hw = &adapter->hw;
40a914fa 6957 pci_ers_result_t result;
42bfd33a 6958 int err;
9d5c8243 6959
aed5dec3 6960 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6961 dev_err(&pdev->dev,
6962 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6963 result = PCI_ERS_RESULT_DISCONNECT;
6964 } else {
6965 pci_set_master(pdev);
6966 pci_restore_state(pdev);
b94f2d77 6967 pci_save_state(pdev);
9d5c8243 6968
40a914fa
AD		 6969 pci_enable_wake(pdev, PCI_D3hot, 0);
6970 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6971
40a914fa
AD		 6972 igb_reset(adapter);
6973 wr32(E1000_WUS, ~0);
6974 result = PCI_ERS_RESULT_RECOVERED;
6975 }
9d5c8243 6976
ea943d41
JK		 6977 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6978 if (err) {
6979 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6980 "failed 0x%0x\n", err);
6981 /* non-fatal, continue */
6982 }
40a914fa
AD		 6983
6984 return result;
9d5c8243
AK		 6985}
6986
6987/**
6988 * igb_io_resume - called when traffic can start flowing again.
6989 * @pdev: Pointer to PCI device
6990 *
6991 * This callback is called when the error recovery driver tells us that
6992 * its OK to resume normal operation. Implementation resembles the
6993 * second-half of the igb_resume routine.
6994 */
6995static void igb_io_resume(struct pci_dev *pdev)
6996{
6997 struct net_device *netdev = pci_get_drvdata(pdev);
6998 struct igb_adapter *adapter = netdev_priv(netdev);
6999
9d5c8243
AK		 7000 if (netif_running(netdev)) {
7001 if (igb_up(adapter)) {
7002 dev_err(&pdev->dev, "igb_up failed after reset\n");
7003 return;
7004 }
7005 }
7006
7007 netif_device_attach(netdev);
7008
7009 /* let the f/w know that the h/w is now under the control of the
7010 * driver. */
7011 igb_get_hw_control(adapter);
9d5c8243
AK		 7012}
7013
26ad9178
AD		 7014static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
7015 u8 qsel)
7016{
7017 u32 rar_low, rar_high;
7018 struct e1000_hw *hw = &adapter->hw;
7019
7020 /* HW expects these in little endian so we reverse the byte order
7021 * from network order (big endian) to little endian
7022 */
7023 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
7024 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
7025 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
7026
7027 /* Indicate to hardware the Address is Valid. */
7028 rar_high |= E1000_RAH_AV;
7029
7030 if (hw->mac.type == e1000_82575)
7031 rar_high |= E1000_RAH_POOL_1 * qsel;
7032 else
7033 rar_high |= E1000_RAH_POOL_1 << qsel;
7034
7035 wr32(E1000_RAL(index), rar_low);
7036 wrfl();
7037 wr32(E1000_RAH(index), rar_high);
7038 wrfl();
7039}
7040
4ae196df
AD		 7041static int igb_set_vf_mac(struct igb_adapter *adapter,
7042 int vf, unsigned char *mac_addr)
7043{
7044 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 7045 /* VF MAC addresses start at end of receive addresses and moves
7046 * torwards the first, as a result a collision should not be possible */
7047 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 7048
37680117 7049 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 7050
26ad9178 7051 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 7052
7053 return 0;
7054}
7055
8151d294
WM		 7056static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
7057{
7058 struct igb_adapter *adapter = netdev_priv(netdev);
7059 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
7060 return -EINVAL;
7061 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
7062 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
7063 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
7064 " change effective.");
7065 if (test_bit(__IGB_DOWN, &adapter->state)) {
7066 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
7067 " but the PF device is not up.\n");
7068 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
7069 " attempting to use the VF device.\n");
7070 }
7071 return igb_set_vf_mac(adapter, vf, mac);
7072}
7073
17dc566c
LL		 7074static int igb_link_mbps(int internal_link_speed)
7075{
7076 switch (internal_link_speed) {
7077 case SPEED_100:
7078 return 100;
7079 case SPEED_1000:
7080 return 1000;
7081 default:
7082 return 0;
7083 }
7084}
7085
7086static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
7087 int link_speed)
7088{
7089 int rf_dec, rf_int;
7090 u32 bcnrc_val;
7091
7092 if (tx_rate != 0) {
7093 /* Calculate the rate factor values to set */
7094 rf_int = link_speed / tx_rate;
7095 rf_dec = (link_speed - (rf_int * tx_rate));
7096 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
7097
7098 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
7099 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
7100 E1000_RTTBCNRC_RF_INT_MASK);
7101 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
7102 } else {
7103 bcnrc_val = 0;
7104 }
7105
7106 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
f00b0da7
LL		 7107 /*
7108 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
7109 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
7110 */
7111 wr32(E1000_RTTBCNRM, 0x14);
17dc566c
LL		 7112 wr32(E1000_RTTBCNRC, bcnrc_val);
7113}
7114
7115static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
7116{
7117 int actual_link_speed, i;
7118 bool reset_rate = false;
7119
7120 /* VF TX rate limit was not set or not supported */
7121 if ((adapter->vf_rate_link_speed == 0) ||
7122 (adapter->hw.mac.type != e1000_82576))
7123 return;
7124
7125 actual_link_speed = igb_link_mbps(adapter->link_speed);
7126 if (actual_link_speed != adapter->vf_rate_link_speed) {
7127 reset_rate = true;
7128 adapter->vf_rate_link_speed = 0;
7129 dev_info(&adapter->pdev->dev,
7130 "Link speed has been changed. VF Transmit "
7131 "rate is disabled\n");
7132 }
7133
7134 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7135 if (reset_rate)
7136 adapter->vf_data[i].tx_rate = 0;
7137
7138 igb_set_vf_rate_limit(&adapter->hw, i,
7139 adapter->vf_data[i].tx_rate,
7140 actual_link_speed);
7141 }
7142}
7143
8151d294
WM		 7144static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
7145{
17dc566c
LL		 7146 struct igb_adapter *adapter = netdev_priv(netdev);
7147 struct e1000_hw *hw = &adapter->hw;
7148 int actual_link_speed;
7149
7150 if (hw->mac.type != e1000_82576)
7151 return -EOPNOTSUPP;
7152
7153 actual_link_speed = igb_link_mbps(adapter->link_speed);
7154 if ((vf >= adapter->vfs_allocated_count) ||
7155 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
7156 (tx_rate < 0) || (tx_rate > actual_link_speed))
7157 return -EINVAL;
7158
7159 adapter->vf_rate_link_speed = actual_link_speed;
7160 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
7161 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
7162
7163 return 0;
8151d294
WM		 7164}
7165
7166static int igb_ndo_get_vf_config(struct net_device *netdev,
7167 int vf, struct ifla_vf_info *ivi)
7168{
7169 struct igb_adapter *adapter = netdev_priv(netdev);
7170 if (vf >= adapter->vfs_allocated_count)
7171 return -EINVAL;
7172 ivi->vf = vf;
7173 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
17dc566c 7174 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8151d294
WM		 7175 ivi->vlan = adapter->vf_data[vf].pf_vlan;
7176 ivi->qos = adapter->vf_data[vf].pf_qos;
7177 return 0;
7178}
7179
4ae196df
AD		 7180static void igb_vmm_control(struct igb_adapter *adapter)
7181{
7182 struct e1000_hw *hw = &adapter->hw;
10d8e907 7183 u32 reg;
4ae196df 7184
52a1dd4d
AD		 7185 switch (hw->mac.type) {
7186 case e1000_82575:
f96a8a0b
CW		 7187 case e1000_i210:
7188 case e1000_i211:
52a1dd4d
AD		 7189 default:
7190 /* replication is not supported for 82575 */
4ae196df 7191 return;
52a1dd4d
AD		 7192 case e1000_82576:
7193 /* notify HW that the MAC is adding vlan tags */
7194 reg = rd32(E1000_DTXCTL);
7195 reg |= E1000_DTXCTL_VLAN_ADDED;
7196 wr32(E1000_DTXCTL, reg);
7197 case e1000_82580:
7198 /* enable replication vlan tag stripping */
7199 reg = rd32(E1000_RPLOLR);
7200 reg |= E1000_RPLOLR_STRVLAN;
7201 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 7202 case e1000_i350:
7203 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 7204 break;
7205 }
10d8e907 7206
d4960307
AD		 7207 if (adapter->vfs_allocated_count) {
7208 igb_vmdq_set_loopback_pf(hw, true);
7209 igb_vmdq_set_replication_pf(hw, true);
13800469
GR		 7210 igb_vmdq_set_anti_spoofing_pf(hw, true,
7211 adapter->vfs_allocated_count);
d4960307
AD		 7212 } else {
7213 igb_vmdq_set_loopback_pf(hw, false);
7214 igb_vmdq_set_replication_pf(hw, false);
7215 }
4ae196df
AD		 7216}
7217
b6e0c419
CW		 7218static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
7219{
7220 struct e1000_hw *hw = &adapter->hw;
7221 u32 dmac_thr;
7222 u16 hwm;
7223
7224 if (hw->mac.type > e1000_82580) {
7225 if (adapter->flags & IGB_FLAG_DMAC) {
7226 u32 reg;
7227
7228 /* force threshold to 0. */
7229 wr32(E1000_DMCTXTH, 0);
7230
7231 /*
e8c626e9
MV		 7232 * DMA Coalescing high water mark needs to be greater
7233 * than the Rx threshold. Set hwm to PBA - max frame
7234 * size in 16B units, capping it at PBA - 6KB.
b6e0c419 7235 */
e8c626e9
MV		 7236 hwm = 64 * pba - adapter->max_frame_size / 16;
7237 if (hwm < 64 * (pba - 6))
7238 hwm = 64 * (pba - 6);
7239 reg = rd32(E1000_FCRTC);
7240 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
7241 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
7242 & E1000_FCRTC_RTH_COAL_MASK);
7243 wr32(E1000_FCRTC, reg);
7244
7245 /*
7246 * Set the DMA Coalescing Rx threshold to PBA - 2 * max
7247 * frame size, capping it at PBA - 10KB.
7248 */
7249 dmac_thr = pba - adapter->max_frame_size / 512;
7250 if (dmac_thr < pba - 10)
7251 dmac_thr = pba - 10;
b6e0c419
CW		 7252 reg = rd32(E1000_DMACR);
7253 reg &= ~E1000_DMACR_DMACTHR_MASK;
b6e0c419
CW		 7254 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
7255 & E1000_DMACR_DMACTHR_MASK);
7256
7257 /* transition to L0x or L1 if available..*/
7258 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
7259
7260 /* watchdog timer= +-1000 usec in 32usec intervals */
7261 reg |= (1000 >> 5);
0c02dd98
MV		 7262
7263 /* Disable BMC-to-OS Watchdog Enable */
7264 reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
b6e0c419
CW		 7265 wr32(E1000_DMACR, reg);
7266
7267 /*
7268 * no lower threshold to disable
7269 * coalescing(smart fifb)-UTRESH=0
7270 */
7271 wr32(E1000_DMCRTRH, 0);
b6e0c419
CW		 7272
7273 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
7274
7275 wr32(E1000_DMCTLX, reg);
7276
7277 /*
7278 * free space in tx packet buffer to wake from
7279 * DMA coal
7280 */
7281 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
7282 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
7283
7284 /*
7285 * make low power state decision controlled
7286 * by DMA coal
7287 */
7288 reg = rd32(E1000_PCIEMISC);
7289 reg &= ~E1000_PCIEMISC_LX_DECISION;
7290 wr32(E1000_PCIEMISC, reg);
7291 } /* endif adapter->dmac is not disabled */
7292 } else if (hw->mac.type == e1000_82580) {
7293 u32 reg = rd32(E1000_PCIEMISC);
7294 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
7295 wr32(E1000_DMACR, 0);
7296 }
7297}
7298
9d5c8243 7299/* igb_main.c */
Linux kernel - Deliverables
This page took 1.166748 seconds and 5 git commands to generate.