projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
igb: Use node specific allocations for the q_vectors and rings
[deliverable/linux.git] / drivers / net / ethernet / intel / igb / igb_main.c
CommitLineData
9d5c8243
AK		 1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4297f99b 4 Copyright(c) 2007-2011 Intel Corporation.
9d5c8243
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
b2cb09b1 31#include <linux/bitops.h>
9d5c8243
AK		 32#include <linux/vmalloc.h>
33#include <linux/pagemap.h>
34#include <linux/netdevice.h>
9d5c8243 35#include <linux/ipv6.h>
5a0e3ad6 36#include <linux/slab.h>
9d5c8243
AK		 37#include <net/checksum.h>
38#include <net/ip6_checksum.h>
c6cb090b 39#include <linux/net_tstamp.h>
9d5c8243
AK		 40#include <linux/mii.h>
41#include <linux/ethtool.h>
01789349 42#include <linux/if.h>
9d5c8243
AK		 43#include <linux/if_vlan.h>
44#include <linux/pci.h>
c54106bb 45#include <linux/pci-aspm.h>
9d5c8243
AK		 46#include <linux/delay.h>
47#include <linux/interrupt.h>
7d13a7d0
AD		 48#include <linux/ip.h>
49#include <linux/tcp.h>
50#include <linux/sctp.h>
9d5c8243 51#include <linux/if_ether.h>
40a914fa 52#include <linux/aer.h>
70c71606 53#include <linux/prefetch.h>
421e02f0 54#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 55#include <linux/dca.h>
56#endif
9d5c8243
AK		 57#include "igb.h"
58
0d1fe82d
CW		 59#define MAJ 3
60#define MIN 0
61#define BUILD 6
0d1fe82d 62#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
929dd047 63__stringify(BUILD) "-k"
9d5c8243
AK		 64char igb_driver_name[] = "igb";
65char igb_driver_version[] = DRV_VERSION;
66static const char igb_driver_string[] =
67 "Intel(R) Gigabit Ethernet Network Driver";
4c4b42cb 68static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation.";
9d5c8243 69
9d5c8243
AK		 70static const struct e1000_info *igb_info_tbl[] = {
71 [board_82575] = &e1000_82575_info,
72};
73
a3aa1884 74static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
d2ba2ed8
AD		 75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
55cac248
AD		 79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
6493d24f 81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
55cac248
AD		 82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
308fb39a
JG		 85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
1b5dda33
GJ		 87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
2d064c06 89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
9eb2341d 90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
747d49ba 91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
2d064c06
AD		 92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
4703bf73 94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
b894fa26 95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
c8ea5ea9 96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
9d5c8243
AK		 97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
100 /* required last entry */
101 {0, }
102};
103
104MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
105
106void igb_reset(struct igb_adapter *);
107static int igb_setup_all_tx_resources(struct igb_adapter *);
108static int igb_setup_all_rx_resources(struct igb_adapter *);
109static void igb_free_all_tx_resources(struct igb_adapter *);
110static void igb_free_all_rx_resources(struct igb_adapter *);
06cf2666 111static void igb_setup_mrqc(struct igb_adapter *);
9d5c8243
AK		 112static int igb_probe(struct pci_dev *, const struct pci_device_id *);
113static void __devexit igb_remove(struct pci_dev *pdev);
673b8b70 114static void igb_init_hw_timer(struct igb_adapter *adapter);
9d5c8243
AK		 115static int igb_sw_init(struct igb_adapter *);
116static int igb_open(struct net_device *);
117static int igb_close(struct net_device *);
118static void igb_configure_tx(struct igb_adapter *);
119static void igb_configure_rx(struct igb_adapter *);
9d5c8243
AK		 120static void igb_clean_all_tx_rings(struct igb_adapter *);
121static void igb_clean_all_rx_rings(struct igb_adapter *);
3b644cf6
MW		 122static void igb_clean_tx_ring(struct igb_ring *);
123static void igb_clean_rx_ring(struct igb_ring *);
ff41f8dc 124static void igb_set_rx_mode(struct net_device *);
9d5c8243
AK		 125static void igb_update_phy_info(unsigned long);
126static void igb_watchdog(unsigned long);
127static void igb_watchdog_task(struct work_struct *);
cd392f5c 128static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
12dcd86b
ED		 129static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
130 struct rtnl_link_stats64 *stats);
9d5c8243
AK		 131static int igb_change_mtu(struct net_device *, int);
132static int igb_set_mac(struct net_device *, void *);
68d480c4 133static void igb_set_uta(struct igb_adapter *adapter);
9d5c8243
AK		 134static irqreturn_t igb_intr(int irq, void *);
135static irqreturn_t igb_intr_msi(int irq, void *);
136static irqreturn_t igb_msix_other(int irq, void *);
047e0030 137static irqreturn_t igb_msix_ring(int irq, void *);
421e02f0 138#ifdef CONFIG_IGB_DCA
047e0030 139static void igb_update_dca(struct igb_q_vector *);
fe4506b6 140static void igb_setup_dca(struct igb_adapter *);
421e02f0 141#endif /* CONFIG_IGB_DCA */
661086df 142static int igb_poll(struct napi_struct *, int);
13fde97a 143static bool igb_clean_tx_irq(struct igb_q_vector *);
cd392f5c 144static bool igb_clean_rx_irq(struct igb_q_vector *, int);
9d5c8243
AK		 145static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
146static void igb_tx_timeout(struct net_device *);
147static void igb_reset_task(struct work_struct *);
b2cb09b1 148static void igb_vlan_mode(struct net_device *netdev, u32 features);
9d5c8243
AK		 149static void igb_vlan_rx_add_vid(struct net_device *, u16);
150static void igb_vlan_rx_kill_vid(struct net_device *, u16);
151static void igb_restore_vlan(struct igb_adapter *);
26ad9178 152static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
4ae196df
AD		 153static void igb_ping_all_vfs(struct igb_adapter *);
154static void igb_msg_task(struct igb_adapter *);
4ae196df 155static void igb_vmm_control(struct igb_adapter *);
f2ca0dbe 156static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
4ae196df 157static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
8151d294
WM		 158static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
159static int igb_ndo_set_vf_vlan(struct net_device *netdev,
160 int vf, u16 vlan, u8 qos);
161static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
162static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
163 struct ifla_vf_info *ivi);
17dc566c 164static void igb_check_vf_rate_limit(struct igb_adapter *);
9d5c8243 165
9d5c8243 166#ifdef CONFIG_PM
3fe7c4c9 167static int igb_suspend(struct pci_dev *, pm_message_t);
9d5c8243
AK		 168static int igb_resume(struct pci_dev *);
169#endif
170static void igb_shutdown(struct pci_dev *);
421e02f0 171#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 172static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
173static struct notifier_block dca_notifier = {
174 .notifier_call = igb_notify_dca,
175 .next = NULL,
176 .priority = 0
177};
178#endif
9d5c8243
AK		 179#ifdef CONFIG_NET_POLL_CONTROLLER
180/* for netdump / net console */
181static void igb_netpoll(struct net_device *);
182#endif
37680117 183#ifdef CONFIG_PCI_IOV
2a3abf6d
AD		 184static unsigned int max_vfs = 0;
185module_param(max_vfs, uint, 0);
186MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
187 "per physical function");
188#endif /* CONFIG_PCI_IOV */
189
9d5c8243
AK		 190static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
191 pci_channel_state_t);
192static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
193static void igb_io_resume(struct pci_dev *);
194
195static struct pci_error_handlers igb_err_handler = {
196 .error_detected = igb_io_error_detected,
197 .slot_reset = igb_io_slot_reset,
198 .resume = igb_io_resume,
199};
200
201
202static struct pci_driver igb_driver = {
203 .name = igb_driver_name,
204 .id_table = igb_pci_tbl,
205 .probe = igb_probe,
206 .remove = __devexit_p(igb_remove),
207#ifdef CONFIG_PM
25985edc 208 /* Power Management Hooks */
9d5c8243
AK		 209 .suspend = igb_suspend,
210 .resume = igb_resume,
211#endif
212 .shutdown = igb_shutdown,
213 .err_handler = &igb_err_handler
214};
215
216MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
217MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
218MODULE_LICENSE("GPL");
219MODULE_VERSION(DRV_VERSION);
220
c97ec42a
TI		 221struct igb_reg_info {
222 u32 ofs;
223 char *name;
224};
225
226static const struct igb_reg_info igb_reg_info_tbl[] = {
227
228 /* General Registers */
229 {E1000_CTRL, "CTRL"},
230 {E1000_STATUS, "STATUS"},
231 {E1000_CTRL_EXT, "CTRL_EXT"},
232
233 /* Interrupt Registers */
234 {E1000_ICR, "ICR"},
235
236 /* RX Registers */
237 {E1000_RCTL, "RCTL"},
238 {E1000_RDLEN(0), "RDLEN"},
239 {E1000_RDH(0), "RDH"},
240 {E1000_RDT(0), "RDT"},
241 {E1000_RXDCTL(0), "RXDCTL"},
242 {E1000_RDBAL(0), "RDBAL"},
243 {E1000_RDBAH(0), "RDBAH"},
244
245 /* TX Registers */
246 {E1000_TCTL, "TCTL"},
247 {E1000_TDBAL(0), "TDBAL"},
248 {E1000_TDBAH(0), "TDBAH"},
249 {E1000_TDLEN(0), "TDLEN"},
250 {E1000_TDH(0), "TDH"},
251 {E1000_TDT(0), "TDT"},
252 {E1000_TXDCTL(0), "TXDCTL"},
253 {E1000_TDFH, "TDFH"},
254 {E1000_TDFT, "TDFT"},
255 {E1000_TDFHS, "TDFHS"},
256 {E1000_TDFPC, "TDFPC"},
257
258 /* List Terminator */
259 {}
260};
261
262/*
263 * igb_regdump - register printout routine
264 */
265static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
266{
267 int n = 0;
268 char rname[16];
269 u32 regs[8];
270
271 switch (reginfo->ofs) {
272 case E1000_RDLEN(0):
273 for (n = 0; n < 4; n++)
274 regs[n] = rd32(E1000_RDLEN(n));
275 break;
276 case E1000_RDH(0):
277 for (n = 0; n < 4; n++)
278 regs[n] = rd32(E1000_RDH(n));
279 break;
280 case E1000_RDT(0):
281 for (n = 0; n < 4; n++)
282 regs[n] = rd32(E1000_RDT(n));
283 break;
284 case E1000_RXDCTL(0):
285 for (n = 0; n < 4; n++)
286 regs[n] = rd32(E1000_RXDCTL(n));
287 break;
288 case E1000_RDBAL(0):
289 for (n = 0; n < 4; n++)
290 regs[n] = rd32(E1000_RDBAL(n));
291 break;
292 case E1000_RDBAH(0):
293 for (n = 0; n < 4; n++)
294 regs[n] = rd32(E1000_RDBAH(n));
295 break;
296 case E1000_TDBAL(0):
297 for (n = 0; n < 4; n++)
298 regs[n] = rd32(E1000_RDBAL(n));
299 break;
300 case E1000_TDBAH(0):
301 for (n = 0; n < 4; n++)
302 regs[n] = rd32(E1000_TDBAH(n));
303 break;
304 case E1000_TDLEN(0):
305 for (n = 0; n < 4; n++)
306 regs[n] = rd32(E1000_TDLEN(n));
307 break;
308 case E1000_TDH(0):
309 for (n = 0; n < 4; n++)
310 regs[n] = rd32(E1000_TDH(n));
311 break;
312 case E1000_TDT(0):
313 for (n = 0; n < 4; n++)
314 regs[n] = rd32(E1000_TDT(n));
315 break;
316 case E1000_TXDCTL(0):
317 for (n = 0; n < 4; n++)
318 regs[n] = rd32(E1000_TXDCTL(n));
319 break;
320 default:
321 printk(KERN_INFO "%-15s %08x\n",
322 reginfo->name, rd32(reginfo->ofs));
323 return;
324 }
325
326 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
327 printk(KERN_INFO "%-15s ", rname);
328 for (n = 0; n < 4; n++)
329 printk(KERN_CONT "%08x ", regs[n]);
330 printk(KERN_CONT "\n");
331}
332
333/*
334 * igb_dump - Print registers, tx-rings and rx-rings
335 */
336static void igb_dump(struct igb_adapter *adapter)
337{
338 struct net_device *netdev = adapter->netdev;
339 struct e1000_hw *hw = &adapter->hw;
340 struct igb_reg_info *reginfo;
341 int n = 0;
342 struct igb_ring *tx_ring;
343 union e1000_adv_tx_desc *tx_desc;
344 struct my_u0 { u64 a; u64 b; } *u0;
c97ec42a
TI		 345 struct igb_ring *rx_ring;
346 union e1000_adv_rx_desc *rx_desc;
347 u32 staterr;
348 int i = 0;
349
350 if (!netif_msg_hw(adapter))
351 return;
352
353 /* Print netdevice Info */
354 if (netdev) {
355 dev_info(&adapter->pdev->dev, "Net device Info\n");
356 printk(KERN_INFO "Device Name state "
357 "trans_start last_rx\n");
358 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
359 netdev->name,
360 netdev->state,
361 netdev->trans_start,
362 netdev->last_rx);
363 }
364
365 /* Print Registers */
366 dev_info(&adapter->pdev->dev, "Register Dump\n");
367 printk(KERN_INFO " Register Name Value\n");
368 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
369 reginfo->name; reginfo++) {
370 igb_regdump(hw, reginfo);
371 }
372
373 /* Print TX Ring Summary */
374 if (!netdev || !netif_running(netdev))
375 goto exit;
376
377 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
378 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
379 " leng ntw timestamp\n");
380 for (n = 0; n < adapter->num_tx_queues; n++) {
06034649 381 struct igb_tx_buffer *buffer_info;
c97ec42a 382 tx_ring = adapter->tx_ring[n];
06034649 383 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
8542db05 384 printk(KERN_INFO " %5d %5X %5X %016llX %04X %p %016llX\n",
c97ec42a
TI		 385 n, tx_ring->next_to_use, tx_ring->next_to_clean,
386 (u64)buffer_info->dma,
387 buffer_info->length,
388 buffer_info->next_to_watch,
389 (u64)buffer_info->time_stamp);
390 }
391
392 /* Print TX Rings */
393 if (!netif_msg_tx_done(adapter))
394 goto rx_ring_summary;
395
396 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
397
398 /* Transmit Descriptor Formats
399 *
400 * Advanced Transmit Descriptor
401 * +--------------------------------------------------------------+
402 * 0 | Buffer Address [63:0] |
403 * +--------------------------------------------------------------+
404 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
405 * +--------------------------------------------------------------+
406 * 63 46 45 40 39 38 36 35 32 31 24 15 0
407 */
408
409 for (n = 0; n < adapter->num_tx_queues; n++) {
410 tx_ring = adapter->tx_ring[n];
411 printk(KERN_INFO "------------------------------------\n");
412 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
413 printk(KERN_INFO "------------------------------------\n");
414 printk(KERN_INFO "T [desc] [address 63:0 ] "
415 "[PlPOCIStDDM Ln] [bi->dma ] "
416 "leng ntw timestamp bi->skb\n");
417
418 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
06034649 419 struct igb_tx_buffer *buffer_info;
60136906 420 tx_desc = IGB_TX_DESC(tx_ring, i);
06034649 421 buffer_info = &tx_ring->tx_buffer_info[i];
c97ec42a
TI		 422 u0 = (struct my_u0 *)tx_desc;
423 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
8542db05 424 " %04X %p %016llX %p", i,
c97ec42a
TI		 425 le64_to_cpu(u0->a),
426 le64_to_cpu(u0->b),
427 (u64)buffer_info->dma,
428 buffer_info->length,
429 buffer_info->next_to_watch,
430 (u64)buffer_info->time_stamp,
431 buffer_info->skb);
432 if (i == tx_ring->next_to_use &&
433 i == tx_ring->next_to_clean)
434 printk(KERN_CONT " NTC/U\n");
435 else if (i == tx_ring->next_to_use)
436 printk(KERN_CONT " NTU\n");
437 else if (i == tx_ring->next_to_clean)
438 printk(KERN_CONT " NTC\n");
439 else
440 printk(KERN_CONT "\n");
441
442 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
443 print_hex_dump(KERN_INFO, "",
444 DUMP_PREFIX_ADDRESS,
445 16, 1, phys_to_virt(buffer_info->dma),
446 buffer_info->length, true);
447 }
448 }
449
450 /* Print RX Rings Summary */
451rx_ring_summary:
452 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
453 printk(KERN_INFO "Queue [NTU] [NTC]\n");
454 for (n = 0; n < adapter->num_rx_queues; n++) {
455 rx_ring = adapter->rx_ring[n];
456 printk(KERN_INFO " %5d %5X %5X\n", n,
457 rx_ring->next_to_use, rx_ring->next_to_clean);
458 }
459
460 /* Print RX Rings */
461 if (!netif_msg_rx_status(adapter))
462 goto exit;
463
464 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
465
466 /* Advanced Receive Descriptor (Read) Format
467 * 63 1 0
468 * +-----------------------------------------------------+
469 * 0 | Packet Buffer Address [63:1] |A0/NSE|
470 * +----------------------------------------------+------+
471 * 8 | Header Buffer Address [63:1] | DD |
472 * +-----------------------------------------------------+
473 *
474 *
475 * Advanced Receive Descriptor (Write-Back) Format
476 *
477 * 63 48 47 32 31 30 21 20 17 16 4 3 0
478 * +------------------------------------------------------+
479 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
480 * | Checksum Ident | | | | Type | Type |
481 * +------------------------------------------------------+
482 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
483 * +------------------------------------------------------+
484 * 63 48 47 32 31 20 19 0
485 */
486
487 for (n = 0; n < adapter->num_rx_queues; n++) {
488 rx_ring = adapter->rx_ring[n];
489 printk(KERN_INFO "------------------------------------\n");
490 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
491 printk(KERN_INFO "------------------------------------\n");
492 printk(KERN_INFO "R [desc] [ PktBuf A0] "
493 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
494 "<-- Adv Rx Read format\n");
495 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
496 "[vl er S cks ln] ---------------- [bi->skb] "
497 "<-- Adv Rx Write-Back format\n");
498
499 for (i = 0; i < rx_ring->count; i++) {
06034649
AD		 500 struct igb_rx_buffer *buffer_info;
501 buffer_info = &rx_ring->rx_buffer_info[i];
60136906 502 rx_desc = IGB_RX_DESC(rx_ring, i);
c97ec42a
TI		 503 u0 = (struct my_u0 *)rx_desc;
504 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
505 if (staterr & E1000_RXD_STAT_DD) {
506 /* Descriptor Done */
507 printk(KERN_INFO "RWB[0x%03X] %016llX "
508 "%016llX ---------------- %p", i,
509 le64_to_cpu(u0->a),
510 le64_to_cpu(u0->b),
511 buffer_info->skb);
512 } else {
513 printk(KERN_INFO "R [0x%03X] %016llX "
514 "%016llX %016llX %p", i,
515 le64_to_cpu(u0->a),
516 le64_to_cpu(u0->b),
517 (u64)buffer_info->dma,
518 buffer_info->skb);
519
520 if (netif_msg_pktdata(adapter)) {
521 print_hex_dump(KERN_INFO, "",
522 DUMP_PREFIX_ADDRESS,
523 16, 1,
524 phys_to_virt(buffer_info->dma),
44390ca6
AD		 525 IGB_RX_HDR_LEN, true);
526 print_hex_dump(KERN_INFO, "",
527 DUMP_PREFIX_ADDRESS,
528 16, 1,
529 phys_to_virt(
530 buffer_info->page_dma +
531 buffer_info->page_offset),
532 PAGE_SIZE/2, true);
c97ec42a
TI		 533 }
534 }
535
536 if (i == rx_ring->next_to_use)
537 printk(KERN_CONT " NTU\n");
538 else if (i == rx_ring->next_to_clean)
539 printk(KERN_CONT " NTC\n");
540 else
541 printk(KERN_CONT "\n");
542
543 }
544 }
545
546exit:
547 return;
548}
549
550
38c845c7
PO		 551/**
552 * igb_read_clock - read raw cycle counter (to be used by time counter)
553 */
554static cycle_t igb_read_clock(const struct cyclecounter *tc)
555{
556 struct igb_adapter *adapter =
557 container_of(tc, struct igb_adapter, cycles);
558 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 559 u64 stamp = 0;
560 int shift = 0;
38c845c7 561
55cac248
AD		 562 /*
563 * The timestamp latches on lowest register read. For the 82580
564 * the lowest register is SYSTIMR instead of SYSTIML. However we never
565 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
566 */
567 if (hw->mac.type == e1000_82580) {
568 stamp = rd32(E1000_SYSTIMR) >> 8;
569 shift = IGB_82580_TSYNC_SHIFT;
570 }
571
c5b9bd5e
AD		 572 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
573 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
38c845c7
PO		 574 return stamp;
575}
576
9d5c8243 577/**
c041076a 578 * igb_get_hw_dev - return device
9d5c8243
AK		 579 * used by hardware layer to print debugging information
580 **/
c041076a 581struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
9d5c8243
AK		 582{
583 struct igb_adapter *adapter = hw->back;
c041076a 584 return adapter->netdev;
9d5c8243 585}
38c845c7 586
9d5c8243
AK		 587/**
588 * igb_init_module - Driver Registration Routine
589 *
590 * igb_init_module is the first routine called when the driver is
591 * loaded. All it does is register with the PCI subsystem.
592 **/
593static int __init igb_init_module(void)
594{
595 int ret;
596 printk(KERN_INFO "%s - version %s\n",
597 igb_driver_string, igb_driver_version);
598
599 printk(KERN_INFO "%s\n", igb_copyright);
600
421e02f0 601#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 602 dca_register_notify(&dca_notifier);
603#endif
bbd98fe4 604 ret = pci_register_driver(&igb_driver);
9d5c8243
AK		 605 return ret;
606}
607
608module_init(igb_init_module);
609
610/**
611 * igb_exit_module - Driver Exit Cleanup Routine
612 *
613 * igb_exit_module is called just before the driver is removed
614 * from memory.
615 **/
616static void __exit igb_exit_module(void)
617{
421e02f0 618#ifdef CONFIG_IGB_DCA
fe4506b6
JC		 619 dca_unregister_notify(&dca_notifier);
620#endif
9d5c8243
AK		 621 pci_unregister_driver(&igb_driver);
622}
623
624module_exit(igb_exit_module);
625
26bc19ec
AD		 626#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
627/**
628 * igb_cache_ring_register - Descriptor ring to register mapping
629 * @adapter: board private structure to initialize
630 *
631 * Once we know the feature-set enabled for the device, we'll cache
632 * the register offset the descriptor ring is assigned to.
633 **/
634static void igb_cache_ring_register(struct igb_adapter *adapter)
635{
ee1b9f06 636 int i = 0, j = 0;
047e0030 637 u32 rbase_offset = adapter->vfs_allocated_count;
26bc19ec
AD		 638
639 switch (adapter->hw.mac.type) {
640 case e1000_82576:
641 /* The queues are allocated for virtualization such that VF 0
642 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
643 * In order to avoid collision we start at the first free queue
644 * and continue consuming queues in the same sequence
645 */
ee1b9f06 646 if (adapter->vfs_allocated_count) {
a99955fc 647 for (; i < adapter->rss_queues; i++)
3025a446
AD		 648 adapter->rx_ring[i]->reg_idx = rbase_offset +
649 Q_IDX_82576(i);
ee1b9f06 650 }
26bc19ec 651 case e1000_82575:
55cac248 652 case e1000_82580:
d2ba2ed8 653 case e1000_i350:
26bc19ec 654 default:
ee1b9f06 655 for (; i < adapter->num_rx_queues; i++)
3025a446 656 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
ee1b9f06 657 for (; j < adapter->num_tx_queues; j++)
3025a446 658 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
26bc19ec
AD		 659 break;
660 }
661}
662
047e0030
AD		 663static void igb_free_queues(struct igb_adapter *adapter)
664{
3025a446 665 int i;
047e0030 666
3025a446
AD		 667 for (i = 0; i < adapter->num_tx_queues; i++) {
668 kfree(adapter->tx_ring[i]);
669 adapter->tx_ring[i] = NULL;
670 }
671 for (i = 0; i < adapter->num_rx_queues; i++) {
672 kfree(adapter->rx_ring[i]);
673 adapter->rx_ring[i] = NULL;
674 }
047e0030
AD		 675 adapter->num_rx_queues = 0;
676 adapter->num_tx_queues = 0;
677}
678
9d5c8243
AK		 679/**
680 * igb_alloc_queues - Allocate memory for all rings
681 * @adapter: board private structure to initialize
682 *
683 * We allocate one ring per queue at run-time since we don't know the
684 * number of queues at compile-time.
685 **/
686static int igb_alloc_queues(struct igb_adapter *adapter)
687{
3025a446 688 struct igb_ring *ring;
9d5c8243 689 int i;
81c2fc22 690 int orig_node = adapter->node;
9d5c8243 691
661086df 692 for (i = 0; i < adapter->num_tx_queues; i++) {
81c2fc22
AD		 693 if (orig_node == -1) {
694 int cur_node = next_online_node(adapter->node);
695 if (cur_node == MAX_NUMNODES)
696 cur_node = first_online_node;
697 adapter->node = cur_node;
698 }
699 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
700 adapter->node);
701 if (!ring)
702 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 703 if (!ring)
704 goto err;
68fd9910 705 ring->count = adapter->tx_ring_count;
661086df 706 ring->queue_index = i;
59d71989 707 ring->dev = &adapter->pdev->dev;
e694e964 708 ring->netdev = adapter->netdev;
81c2fc22 709 ring->numa_node = adapter->node;
85ad76b2
AD		 710 /* For 82575, context index must be unique per ring. */
711 if (adapter->hw.mac.type == e1000_82575)
712 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
3025a446 713 adapter->tx_ring[i] = ring;
661086df 714 }
81c2fc22
AD		 715 /* Restore the adapter's original node */
716 adapter->node = orig_node;
85ad76b2 717
9d5c8243 718 for (i = 0; i < adapter->num_rx_queues; i++) {
81c2fc22
AD		 719 if (orig_node == -1) {
720 int cur_node = next_online_node(adapter->node);
721 if (cur_node == MAX_NUMNODES)
722 cur_node = first_online_node;
723 adapter->node = cur_node;
724 }
725 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
726 adapter->node);
727 if (!ring)
728 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
3025a446
AD		 729 if (!ring)
730 goto err;
68fd9910 731 ring->count = adapter->rx_ring_count;
844290e5 732 ring->queue_index = i;
59d71989 733 ring->dev = &adapter->pdev->dev;
e694e964 734 ring->netdev = adapter->netdev;
81c2fc22 735 ring->numa_node = adapter->node;
85ad76b2
AD		 736 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
737 /* set flag indicating ring supports SCTP checksum offload */
738 if (adapter->hw.mac.type >= e1000_82576)
739 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
3025a446 740 adapter->rx_ring[i] = ring;
9d5c8243 741 }
81c2fc22
AD		 742 /* Restore the adapter's original node */
743 adapter->node = orig_node;
26bc19ec
AD		 744
745 igb_cache_ring_register(adapter);
9d5c8243 746
047e0030 747 return 0;
a88f10ec 748
047e0030 749err:
81c2fc22
AD		 750 /* Restore the adapter's original node */
751 adapter->node = orig_node;
047e0030 752 igb_free_queues(adapter);
d1a8c9e1 753
047e0030 754 return -ENOMEM;
a88f10ec
AD		 755}
756
9d5c8243 757#define IGB_N0_QUEUE -1
047e0030 758static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
9d5c8243
AK		 759{
760 u32 msixbm = 0;
047e0030 761 struct igb_adapter *adapter = q_vector->adapter;
9d5c8243 762 struct e1000_hw *hw = &adapter->hw;
2d064c06 763 u32 ivar, index;
047e0030
AD		 764 int rx_queue = IGB_N0_QUEUE;
765 int tx_queue = IGB_N0_QUEUE;
766
767 if (q_vector->rx_ring)
768 rx_queue = q_vector->rx_ring->reg_idx;
769 if (q_vector->tx_ring)
770 tx_queue = q_vector->tx_ring->reg_idx;
2d064c06
AD		 771
772 switch (hw->mac.type) {
773 case e1000_82575:
9d5c8243
AK		 774 /* The 82575 assigns vectors using a bitmask, which matches the
775 bitmask for the EICR/EIMS/EIMC registers. To assign one
776 or more queues to a vector, we write the appropriate bits
777 into the MSIXBM register for that vector. */
047e0030 778 if (rx_queue > IGB_N0_QUEUE)
9d5c8243 779 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
047e0030 780 if (tx_queue > IGB_N0_QUEUE)
9d5c8243 781 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
feeb2721
AD		 782 if (!adapter->msix_entries && msix_vector == 0)
783 msixbm |= E1000_EIMS_OTHER;
9d5c8243 784 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
047e0030 785 q_vector->eims_value = msixbm;
2d064c06
AD		 786 break;
787 case e1000_82576:
26bc19ec 788 /* 82576 uses a table-based method for assigning vectors.
2d064c06
AD		 789 Each queue has a single entry in the table to which we write
790 a vector number along with a "valid" bit. Sadly, the layout
791 of the table is somewhat counterintuitive. */
792 if (rx_queue > IGB_N0_QUEUE) {
047e0030 793 index = (rx_queue & 0x7);
2d064c06 794 ivar = array_rd32(E1000_IVAR0, index);
047e0030 795 if (rx_queue < 8) {
26bc19ec
AD		 796 /* vector goes into low byte of register */
797 ivar = ivar & 0xFFFFFF00;
798 ivar |= msix_vector | E1000_IVAR_VALID;
047e0030
AD		 799 } else {
800 /* vector goes into third byte of register */
801 ivar = ivar & 0xFF00FFFF;
802 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
2d064c06 803 }
2d064c06
AD		 804 array_wr32(E1000_IVAR0, index, ivar);
805 }
806 if (tx_queue > IGB_N0_QUEUE) {
047e0030 807 index = (tx_queue & 0x7);
2d064c06 808 ivar = array_rd32(E1000_IVAR0, index);
047e0030 809 if (tx_queue < 8) {
26bc19ec
AD		 810 /* vector goes into second byte of register */
811 ivar = ivar & 0xFFFF00FF;
812 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
047e0030
AD		 813 } else {
814 /* vector goes into high byte of register */
815 ivar = ivar & 0x00FFFFFF;
816 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
2d064c06 817 }
2d064c06
AD		 818 array_wr32(E1000_IVAR0, index, ivar);
819 }
047e0030 820 q_vector->eims_value = 1 << msix_vector;
2d064c06 821 break;
55cac248 822 case e1000_82580:
d2ba2ed8 823 case e1000_i350:
55cac248
AD		 824 /* 82580 uses the same table-based approach as 82576 but has fewer
825 entries as a result we carry over for queues greater than 4. */
826 if (rx_queue > IGB_N0_QUEUE) {
827 index = (rx_queue >> 1);
828 ivar = array_rd32(E1000_IVAR0, index);
829 if (rx_queue & 0x1) {
830 /* vector goes into third byte of register */
831 ivar = ivar & 0xFF00FFFF;
832 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
833 } else {
834 /* vector goes into low byte of register */
835 ivar = ivar & 0xFFFFFF00;
836 ivar |= msix_vector | E1000_IVAR_VALID;
837 }
838 array_wr32(E1000_IVAR0, index, ivar);
839 }
840 if (tx_queue > IGB_N0_QUEUE) {
841 index = (tx_queue >> 1);
842 ivar = array_rd32(E1000_IVAR0, index);
843 if (tx_queue & 0x1) {
844 /* vector goes into high byte of register */
845 ivar = ivar & 0x00FFFFFF;
846 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
847 } else {
848 /* vector goes into second byte of register */
849 ivar = ivar & 0xFFFF00FF;
850 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
851 }
852 array_wr32(E1000_IVAR0, index, ivar);
853 }
854 q_vector->eims_value = 1 << msix_vector;
855 break;
2d064c06
AD		 856 default:
857 BUG();
858 break;
859 }
26b39276
AD		 860
861 /* add q_vector eims value to global eims_enable_mask */
862 adapter->eims_enable_mask |= q_vector->eims_value;
863
864 /* configure q_vector to set itr on first interrupt */
865 q_vector->set_itr = 1;
9d5c8243
AK		 866}
867
868/**
869 * igb_configure_msix - Configure MSI-X hardware
870 *
871 * igb_configure_msix sets up the hardware to properly
872 * generate MSI-X interrupts.
873 **/
874static void igb_configure_msix(struct igb_adapter *adapter)
875{
876 u32 tmp;
877 int i, vector = 0;
878 struct e1000_hw *hw = &adapter->hw;
879
880 adapter->eims_enable_mask = 0;
9d5c8243
AK		 881
882 /* set vector for other causes, i.e. link changes */
2d064c06
AD		 883 switch (hw->mac.type) {
884 case e1000_82575:
9d5c8243
AK		 885 tmp = rd32(E1000_CTRL_EXT);
886 /* enable MSI-X PBA support*/
887 tmp |= E1000_CTRL_EXT_PBA_CLR;
888
889 /* Auto-Mask interrupts upon ICR read. */
890 tmp |= E1000_CTRL_EXT_EIAME;
891 tmp |= E1000_CTRL_EXT_IRCA;
892
893 wr32(E1000_CTRL_EXT, tmp);
047e0030
AD		 894
895 /* enable msix_other interrupt */
896 array_wr32(E1000_MSIXBM(0), vector++,
897 E1000_EIMS_OTHER);
844290e5 898 adapter->eims_other = E1000_EIMS_OTHER;
9d5c8243 899
2d064c06
AD		 900 break;
901
902 case e1000_82576:
55cac248 903 case e1000_82580:
d2ba2ed8 904 case e1000_i350:
047e0030
AD		 905 /* Turn on MSI-X capability first, or our settings
906 * won't stick. And it will take days to debug. */
907 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
908 E1000_GPIE_PBA | E1000_GPIE_EIAME |
909 E1000_GPIE_NSICR);
910
911 /* enable msix_other interrupt */
912 adapter->eims_other = 1 << vector;
2d064c06 913 tmp = (vector++ | E1000_IVAR_VALID) << 8;
2d064c06 914
047e0030 915 wr32(E1000_IVAR_MISC, tmp);
2d064c06
AD		 916 break;
917 default:
918 /* do nothing, since nothing else supports MSI-X */
919 break;
920 } /* switch (hw->mac.type) */
047e0030
AD		 921
922 adapter->eims_enable_mask |= adapter->eims_other;
923
26b39276
AD		 924 for (i = 0; i < adapter->num_q_vectors; i++)
925 igb_assign_vector(adapter->q_vector[i], vector++);
047e0030 926
9d5c8243
AK		 927 wrfl();
928}
929
930/**
931 * igb_request_msix - Initialize MSI-X interrupts
932 *
933 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
934 * kernel.
935 **/
936static int igb_request_msix(struct igb_adapter *adapter)
937{
938 struct net_device *netdev = adapter->netdev;
047e0030 939 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 940 int i, err = 0, vector = 0;
941
047e0030 942 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 943 igb_msix_other, 0, netdev->name, adapter);
047e0030
AD		 944 if (err)
945 goto out;
946 vector++;
947
948 for (i = 0; i < adapter->num_q_vectors; i++) {
949 struct igb_q_vector *q_vector = adapter->q_vector[i];
950
951 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
952
953 if (q_vector->rx_ring && q_vector->tx_ring)
954 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
955 q_vector->rx_ring->queue_index);
956 else if (q_vector->tx_ring)
957 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
958 q_vector->tx_ring->queue_index);
959 else if (q_vector->rx_ring)
960 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
961 q_vector->rx_ring->queue_index);
9d5c8243 962 else
047e0030
AD		 963 sprintf(q_vector->name, "%s-unused", netdev->name);
964
9d5c8243 965 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 966 igb_msix_ring, 0, q_vector->name,
047e0030 967 q_vector);
9d5c8243
AK		 968 if (err)
969 goto out;
9d5c8243
AK		 970 vector++;
971 }
972
9d5c8243
AK		 973 igb_configure_msix(adapter);
974 return 0;
975out:
976 return err;
977}
978
979static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
980{
981 if (adapter->msix_entries) {
982 pci_disable_msix(adapter->pdev);
983 kfree(adapter->msix_entries);
984 adapter->msix_entries = NULL;
047e0030 985 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
9d5c8243 986 pci_disable_msi(adapter->pdev);
047e0030 987 }
9d5c8243
AK		 988}
989
047e0030
AD		 990/**
991 * igb_free_q_vectors - Free memory allocated for interrupt vectors
992 * @adapter: board private structure to initialize
993 *
994 * This function frees the memory allocated to the q_vectors. In addition if
995 * NAPI is enabled it will delete any references to the NAPI struct prior
996 * to freeing the q_vector.
997 **/
998static void igb_free_q_vectors(struct igb_adapter *adapter)
999{
1000 int v_idx;
1001
1002 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1003 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1004 adapter->q_vector[v_idx] = NULL;
fe0592b4
NN		 1005 if (!q_vector)
1006 continue;
047e0030
AD		 1007 netif_napi_del(&q_vector->napi);
1008 kfree(q_vector);
1009 }
1010 adapter->num_q_vectors = 0;
1011}
1012
1013/**
1014 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1015 *
1016 * This function resets the device so that it has 0 rx queues, tx queues, and
1017 * MSI-X interrupts allocated.
1018 */
1019static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1020{
1021 igb_free_queues(adapter);
1022 igb_free_q_vectors(adapter);
1023 igb_reset_interrupt_capability(adapter);
1024}
9d5c8243
AK		 1025
1026/**
1027 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1028 *
1029 * Attempt to configure interrupts using the best available
1030 * capabilities of the hardware and kernel.
1031 **/
21adef3e 1032static int igb_set_interrupt_capability(struct igb_adapter *adapter)
9d5c8243
AK		 1033{
1034 int err;
1035 int numvecs, i;
1036
83b7180d 1037 /* Number of supported queues. */
a99955fc 1038 adapter->num_rx_queues = adapter->rss_queues;
5fa8517f
GR		 1039 if (adapter->vfs_allocated_count)
1040 adapter->num_tx_queues = 1;
1041 else
1042 adapter->num_tx_queues = adapter->rss_queues;
83b7180d 1043
047e0030
AD		 1044 /* start with one vector for every rx queue */
1045 numvecs = adapter->num_rx_queues;
1046
3ad2f3fb 1047 /* if tx handler is separate add 1 for every tx queue */
a99955fc
AD		 1048 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1049 numvecs += adapter->num_tx_queues;
047e0030
AD		 1050
1051 /* store the number of vectors reserved for queues */
1052 adapter->num_q_vectors = numvecs;
1053
1054 /* add 1 vector for link status interrupts */
1055 numvecs++;
9d5c8243
AK		 1056 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1057 GFP_KERNEL);
1058 if (!adapter->msix_entries)
1059 goto msi_only;
1060
1061 for (i = 0; i < numvecs; i++)
1062 adapter->msix_entries[i].entry = i;
1063
1064 err = pci_enable_msix(adapter->pdev,
1065 adapter->msix_entries,
1066 numvecs);
1067 if (err == 0)
34a20e89 1068 goto out;
9d5c8243
AK		 1069
1070 igb_reset_interrupt_capability(adapter);
1071
1072 /* If we can't do MSI-X, try MSI */
1073msi_only:
2a3abf6d
AD		 1074#ifdef CONFIG_PCI_IOV
1075 /* disable SR-IOV for non MSI-X configurations */
1076 if (adapter->vf_data) {
1077 struct e1000_hw *hw = &adapter->hw;
1078 /* disable iov and allow time for transactions to clear */
1079 pci_disable_sriov(adapter->pdev);
1080 msleep(500);
1081
1082 kfree(adapter->vf_data);
1083 adapter->vf_data = NULL;
1084 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 1085 wrfl();
2a3abf6d
AD		 1086 msleep(100);
1087 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1088 }
1089#endif
4fc82adf 1090 adapter->vfs_allocated_count = 0;
a99955fc 1091 adapter->rss_queues = 1;
4fc82adf 1092 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
9d5c8243 1093 adapter->num_rx_queues = 1;
661086df 1094 adapter->num_tx_queues = 1;
047e0030 1095 adapter->num_q_vectors = 1;
9d5c8243 1096 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1097 adapter->flags |= IGB_FLAG_HAS_MSI;
34a20e89 1098out:
21adef3e
BH		 1099 /* Notify the stack of the (possibly) reduced queue counts. */
1100 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1101 return netif_set_real_num_rx_queues(adapter->netdev,
1102 adapter->num_rx_queues);
9d5c8243
AK		 1103}
1104
047e0030
AD		 1105/**
1106 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1107 * @adapter: board private structure to initialize
1108 *
1109 * We allocate one q_vector per queue interrupt. If allocation fails we
1110 * return -ENOMEM.
1111 **/
1112static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1113{
1114 struct igb_q_vector *q_vector;
1115 struct e1000_hw *hw = &adapter->hw;
1116 int v_idx;
81c2fc22 1117 int orig_node = adapter->node;
047e0030
AD		 1118
1119 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
81c2fc22
AD		 1120 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1121 adapter->num_tx_queues)) &&
1122 (adapter->num_rx_queues == v_idx))
1123 adapter->node = orig_node;
1124 if (orig_node == -1) {
1125 int cur_node = next_online_node(adapter->node);
1126 if (cur_node == MAX_NUMNODES)
1127 cur_node = first_online_node;
1128 adapter->node = cur_node;
1129 }
1130 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1131 adapter->node);
1132 if (!q_vector)
1133 q_vector = kzalloc(sizeof(struct igb_q_vector),
1134 GFP_KERNEL);
047e0030
AD		 1135 if (!q_vector)
1136 goto err_out;
1137 q_vector->adapter = adapter;
047e0030
AD		 1138 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1139 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1140 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1141 adapter->q_vector[v_idx] = q_vector;
1142 }
81c2fc22
AD		 1143 /* Restore the adapter's original node */
1144 adapter->node = orig_node;
1145
047e0030
AD		 1146 return 0;
1147
1148err_out:
81c2fc22
AD		 1149 /* Restore the adapter's original node */
1150 adapter->node = orig_node;
fe0592b4 1151 igb_free_q_vectors(adapter);
047e0030
AD		 1152 return -ENOMEM;
1153}
1154
1155static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1156 int ring_idx, int v_idx)
1157{
3025a446 1158 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1159
3025a446 1160 q_vector->rx_ring = adapter->rx_ring[ring_idx];
047e0030 1161 q_vector->rx_ring->q_vector = q_vector;
4fc82adf
AD		 1162 q_vector->itr_val = adapter->rx_itr_setting;
1163 if (q_vector->itr_val && q_vector->itr_val <= 3)
1164 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1165}
1166
1167static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1168 int ring_idx, int v_idx)
1169{
3025a446 1170 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
047e0030 1171
3025a446 1172 q_vector->tx_ring = adapter->tx_ring[ring_idx];
047e0030 1173 q_vector->tx_ring->q_vector = q_vector;
4fc82adf 1174 q_vector->itr_val = adapter->tx_itr_setting;
13fde97a 1175 q_vector->tx_work_limit = adapter->tx_work_limit;
4fc82adf
AD		 1176 if (q_vector->itr_val && q_vector->itr_val <= 3)
1177 q_vector->itr_val = IGB_START_ITR;
047e0030
AD		 1178}
1179
1180/**
1181 * igb_map_ring_to_vector - maps allocated queues to vectors
1182 *
1183 * This function maps the recently allocated queues to vectors.
1184 **/
1185static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1186{
1187 int i;
1188 int v_idx = 0;
1189
1190 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1191 (adapter->num_q_vectors < adapter->num_tx_queues))
1192 return -ENOMEM;
1193
1194 if (adapter->num_q_vectors >=
1195 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1196 for (i = 0; i < adapter->num_rx_queues; i++)
1197 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1198 for (i = 0; i < adapter->num_tx_queues; i++)
1199 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1200 } else {
1201 for (i = 0; i < adapter->num_rx_queues; i++) {
1202 if (i < adapter->num_tx_queues)
1203 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1204 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1205 }
1206 for (; i < adapter->num_tx_queues; i++)
1207 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1208 }
1209 return 0;
1210}
1211
1212/**
1213 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1214 *
1215 * This function initializes the interrupts and allocates all of the queues.
1216 **/
1217static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1218{
1219 struct pci_dev *pdev = adapter->pdev;
1220 int err;
1221
21adef3e
BH		 1222 err = igb_set_interrupt_capability(adapter);
1223 if (err)
1224 return err;
047e0030
AD		 1225
1226 err = igb_alloc_q_vectors(adapter);
1227 if (err) {
1228 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1229 goto err_alloc_q_vectors;
1230 }
1231
1232 err = igb_alloc_queues(adapter);
1233 if (err) {
1234 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1235 goto err_alloc_queues;
1236 }
1237
1238 err = igb_map_ring_to_vector(adapter);
1239 if (err) {
1240 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1241 goto err_map_queues;
1242 }
1243
1244
1245 return 0;
1246err_map_queues:
1247 igb_free_queues(adapter);
1248err_alloc_queues:
1249 igb_free_q_vectors(adapter);
1250err_alloc_q_vectors:
1251 igb_reset_interrupt_capability(adapter);
1252 return err;
1253}
1254
9d5c8243
AK		 1255/**
1256 * igb_request_irq - initialize interrupts
1257 *
1258 * Attempts to configure interrupts using the best available
1259 * capabilities of the hardware and kernel.
1260 **/
1261static int igb_request_irq(struct igb_adapter *adapter)
1262{
1263 struct net_device *netdev = adapter->netdev;
047e0030 1264 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 1265 int err = 0;
1266
1267 if (adapter->msix_entries) {
1268 err = igb_request_msix(adapter);
844290e5 1269 if (!err)
9d5c8243 1270 goto request_done;
9d5c8243 1271 /* fall back to MSI */
047e0030 1272 igb_clear_interrupt_scheme(adapter);
9d5c8243 1273 if (!pci_enable_msi(adapter->pdev))
7dfc16fa 1274 adapter->flags |= IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1275 igb_free_all_tx_resources(adapter);
1276 igb_free_all_rx_resources(adapter);
047e0030 1277 adapter->num_tx_queues = 1;
9d5c8243 1278 adapter->num_rx_queues = 1;
047e0030
AD		 1279 adapter->num_q_vectors = 1;
1280 err = igb_alloc_q_vectors(adapter);
1281 if (err) {
1282 dev_err(&pdev->dev,
1283 "Unable to allocate memory for vectors\n");
1284 goto request_done;
1285 }
1286 err = igb_alloc_queues(adapter);
1287 if (err) {
1288 dev_err(&pdev->dev,
1289 "Unable to allocate memory for queues\n");
1290 igb_free_q_vectors(adapter);
1291 goto request_done;
1292 }
1293 igb_setup_all_tx_resources(adapter);
1294 igb_setup_all_rx_resources(adapter);
844290e5 1295 } else {
feeb2721 1296 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243 1297 }
844290e5 1298
7dfc16fa 1299 if (adapter->flags & IGB_FLAG_HAS_MSI) {
a0607fd3 1300 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
047e0030 1301 netdev->name, adapter);
9d5c8243
AK		 1302 if (!err)
1303 goto request_done;
047e0030 1304
9d5c8243
AK		 1305 /* fall back to legacy interrupts */
1306 igb_reset_interrupt_capability(adapter);
7dfc16fa 1307 adapter->flags &= ~IGB_FLAG_HAS_MSI;
9d5c8243
AK		 1308 }
1309
a0607fd3 1310 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
047e0030 1311 netdev->name, adapter);
9d5c8243 1312
6cb5e577 1313 if (err)
9d5c8243
AK		 1314 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1315 err);
9d5c8243
AK		 1316
1317request_done:
1318 return err;
1319}
1320
1321static void igb_free_irq(struct igb_adapter *adapter)
1322{
9d5c8243
AK		 1323 if (adapter->msix_entries) {
1324 int vector = 0, i;
1325
047e0030 1326 free_irq(adapter->msix_entries[vector++].vector, adapter);
9d5c8243 1327
047e0030
AD		 1328 for (i = 0; i < adapter->num_q_vectors; i++) {
1329 struct igb_q_vector *q_vector = adapter->q_vector[i];
1330 free_irq(adapter->msix_entries[vector++].vector,
1331 q_vector);
1332 }
1333 } else {
1334 free_irq(adapter->pdev->irq, adapter);
9d5c8243 1335 }
9d5c8243
AK		 1336}
1337
1338/**
1339 * igb_irq_disable - Mask off interrupt generation on the NIC
1340 * @adapter: board private structure
1341 **/
1342static void igb_irq_disable(struct igb_adapter *adapter)
1343{
1344 struct e1000_hw *hw = &adapter->hw;
1345
25568a53
AD		 1346 /*
1347 * we need to be careful when disabling interrupts. The VFs are also
1348 * mapped into these registers and so clearing the bits can cause
1349 * issues on the VF drivers so we only need to clear what we set
1350 */
9d5c8243 1351 if (adapter->msix_entries) {
2dfd1212
AD		 1352 u32 regval = rd32(E1000_EIAM);
1353 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1354 wr32(E1000_EIMC, adapter->eims_enable_mask);
1355 regval = rd32(E1000_EIAC);
1356 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
9d5c8243 1357 }
844290e5
PW		 1358
1359 wr32(E1000_IAM, 0);
9d5c8243
AK		 1360 wr32(E1000_IMC, ~0);
1361 wrfl();
81a61859
ET		 1362 if (adapter->msix_entries) {
1363 int i;
1364 for (i = 0; i < adapter->num_q_vectors; i++)
1365 synchronize_irq(adapter->msix_entries[i].vector);
1366 } else {
1367 synchronize_irq(adapter->pdev->irq);
1368 }
9d5c8243
AK		 1369}
1370
1371/**
1372 * igb_irq_enable - Enable default interrupt generation settings
1373 * @adapter: board private structure
1374 **/
1375static void igb_irq_enable(struct igb_adapter *adapter)
1376{
1377 struct e1000_hw *hw = &adapter->hw;
1378
1379 if (adapter->msix_entries) {
25568a53 1380 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
2dfd1212
AD		 1381 u32 regval = rd32(E1000_EIAC);
1382 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1383 regval = rd32(E1000_EIAM);
1384 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
844290e5 1385 wr32(E1000_EIMS, adapter->eims_enable_mask);
25568a53 1386 if (adapter->vfs_allocated_count) {
4ae196df 1387 wr32(E1000_MBVFIMR, 0xFF);
25568a53
AD		 1388 ims |= E1000_IMS_VMMB;
1389 }
55cac248
AD		 1390 if (adapter->hw.mac.type == e1000_82580)
1391 ims |= E1000_IMS_DRSTA;
1392
25568a53 1393 wr32(E1000_IMS, ims);
844290e5 1394 } else {
55cac248
AD		 1395 wr32(E1000_IMS, IMS_ENABLE_MASK |
1396 E1000_IMS_DRSTA);
1397 wr32(E1000_IAM, IMS_ENABLE_MASK |
1398 E1000_IMS_DRSTA);
844290e5 1399 }
9d5c8243
AK		 1400}
1401
1402static void igb_update_mng_vlan(struct igb_adapter *adapter)
1403{
51466239 1404 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1405 u16 vid = adapter->hw.mng_cookie.vlan_id;
1406 u16 old_vid = adapter->mng_vlan_id;
51466239
AD		 1407
1408 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1409 /* add VID to filter table */
1410 igb_vfta_set(hw, vid, true);
1411 adapter->mng_vlan_id = vid;
1412 } else {
1413 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1414 }
1415
1416 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1417 (vid != old_vid) &&
b2cb09b1 1418 !test_bit(old_vid, adapter->active_vlans)) {
51466239
AD		 1419 /* remove VID from filter table */
1420 igb_vfta_set(hw, old_vid, false);
9d5c8243
AK		 1421 }
1422}
1423
1424/**
1425 * igb_release_hw_control - release control of the h/w to f/w
1426 * @adapter: address of board private structure
1427 *
1428 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1429 * For ASF and Pass Through versions of f/w this means that the
1430 * driver is no longer loaded.
1431 *
1432 **/
1433static void igb_release_hw_control(struct igb_adapter *adapter)
1434{
1435 struct e1000_hw *hw = &adapter->hw;
1436 u32 ctrl_ext;
1437
1438 /* Let firmware take over control of h/w */
1439 ctrl_ext = rd32(E1000_CTRL_EXT);
1440 wr32(E1000_CTRL_EXT,
1441 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1442}
1443
9d5c8243
AK		 1444/**
1445 * igb_get_hw_control - get control of the h/w from f/w
1446 * @adapter: address of board private structure
1447 *
1448 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1449 * For ASF and Pass Through versions of f/w this means that
1450 * the driver is loaded.
1451 *
1452 **/
1453static void igb_get_hw_control(struct igb_adapter *adapter)
1454{
1455 struct e1000_hw *hw = &adapter->hw;
1456 u32 ctrl_ext;
1457
1458 /* Let firmware know the driver has taken over */
1459 ctrl_ext = rd32(E1000_CTRL_EXT);
1460 wr32(E1000_CTRL_EXT,
1461 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1462}
1463
9d5c8243
AK		 1464/**
1465 * igb_configure - configure the hardware for RX and TX
1466 * @adapter: private board structure
1467 **/
1468static void igb_configure(struct igb_adapter *adapter)
1469{
1470 struct net_device *netdev = adapter->netdev;
1471 int i;
1472
1473 igb_get_hw_control(adapter);
ff41f8dc 1474 igb_set_rx_mode(netdev);
9d5c8243
AK		 1475
1476 igb_restore_vlan(adapter);
9d5c8243 1477
85b430b4 1478 igb_setup_tctl(adapter);
06cf2666 1479 igb_setup_mrqc(adapter);
9d5c8243 1480 igb_setup_rctl(adapter);
85b430b4
AD		 1481
1482 igb_configure_tx(adapter);
9d5c8243 1483 igb_configure_rx(adapter);
662d7205
AD		 1484
1485 igb_rx_fifo_flush_82575(&adapter->hw);
1486
c493ea45 1487 /* call igb_desc_unused which always leaves
9d5c8243
AK		 1488 * at least 1 descriptor unused to make sure
1489 * next_to_use != next_to_clean */
1490 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 1491 struct igb_ring *ring = adapter->rx_ring[i];
cd392f5c 1492 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
9d5c8243 1493 }
9d5c8243
AK		 1494}
1495
88a268c1
NN		 1496/**
1497 * igb_power_up_link - Power up the phy/serdes link
1498 * @adapter: address of board private structure
1499 **/
1500void igb_power_up_link(struct igb_adapter *adapter)
1501{
1502 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1503 igb_power_up_phy_copper(&adapter->hw);
1504 else
1505 igb_power_up_serdes_link_82575(&adapter->hw);
1506}
1507
1508/**
1509 * igb_power_down_link - Power down the phy/serdes link
1510 * @adapter: address of board private structure
1511 */
1512static void igb_power_down_link(struct igb_adapter *adapter)
1513{
1514 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1515 igb_power_down_phy_copper_82575(&adapter->hw);
1516 else
1517 igb_shutdown_serdes_link_82575(&adapter->hw);
1518}
9d5c8243
AK		 1519
1520/**
1521 * igb_up - Open the interface and prepare it to handle traffic
1522 * @adapter: board private structure
1523 **/
9d5c8243
AK		 1524int igb_up(struct igb_adapter *adapter)
1525{
1526 struct e1000_hw *hw = &adapter->hw;
1527 int i;
1528
1529 /* hardware has been reset, we need to reload some things */
1530 igb_configure(adapter);
1531
1532 clear_bit(__IGB_DOWN, &adapter->state);
1533
047e0030
AD		 1534 for (i = 0; i < adapter->num_q_vectors; i++) {
1535 struct igb_q_vector *q_vector = adapter->q_vector[i];
1536 napi_enable(&q_vector->napi);
1537 }
844290e5 1538 if (adapter->msix_entries)
9d5c8243 1539 igb_configure_msix(adapter);
feeb2721
AD		 1540 else
1541 igb_assign_vector(adapter->q_vector[0], 0);
9d5c8243
AK		 1542
1543 /* Clear any pending interrupts. */
1544 rd32(E1000_ICR);
1545 igb_irq_enable(adapter);
1546
d4960307
AD		 1547 /* notify VFs that reset has been completed */
1548 if (adapter->vfs_allocated_count) {
1549 u32 reg_data = rd32(E1000_CTRL_EXT);
1550 reg_data |= E1000_CTRL_EXT_PFRSTD;
1551 wr32(E1000_CTRL_EXT, reg_data);
1552 }
1553
4cb9be7a
JB		 1554 netif_tx_start_all_queues(adapter->netdev);
1555
25568a53
AD		 1556 /* start the watchdog. */
1557 hw->mac.get_link_status = 1;
1558 schedule_work(&adapter->watchdog_task);
1559
9d5c8243
AK		 1560 return 0;
1561}
1562
1563void igb_down(struct igb_adapter *adapter)
1564{
9d5c8243 1565 struct net_device *netdev = adapter->netdev;
330a6d6a 1566 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 1567 u32 tctl, rctl;
1568 int i;
1569
1570 /* signal that we're down so the interrupt handler does not
1571 * reschedule our watchdog timer */
1572 set_bit(__IGB_DOWN, &adapter->state);
1573
1574 /* disable receives in the hardware */
1575 rctl = rd32(E1000_RCTL);
1576 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1577 /* flush and sleep below */
1578
fd2ea0a7 1579 netif_tx_stop_all_queues(netdev);
9d5c8243
AK		 1580
1581 /* disable transmits in the hardware */
1582 tctl = rd32(E1000_TCTL);
1583 tctl &= ~E1000_TCTL_EN;
1584 wr32(E1000_TCTL, tctl);
1585 /* flush both disables and wait for them to finish */
1586 wrfl();
1587 msleep(10);
1588
047e0030
AD		 1589 for (i = 0; i < adapter->num_q_vectors; i++) {
1590 struct igb_q_vector *q_vector = adapter->q_vector[i];
1591 napi_disable(&q_vector->napi);
1592 }
9d5c8243 1593
9d5c8243
AK		 1594 igb_irq_disable(adapter);
1595
1596 del_timer_sync(&adapter->watchdog_timer);
1597 del_timer_sync(&adapter->phy_info_timer);
1598
9d5c8243 1599 netif_carrier_off(netdev);
04fe6358
AD		 1600
1601 /* record the stats before reset*/
12dcd86b
ED		 1602 spin_lock(&adapter->stats64_lock);
1603 igb_update_stats(adapter, &adapter->stats64);
1604 spin_unlock(&adapter->stats64_lock);
04fe6358 1605
9d5c8243
AK		 1606 adapter->link_speed = 0;
1607 adapter->link_duplex = 0;
1608
3023682e
JK		 1609 if (!pci_channel_offline(adapter->pdev))
1610 igb_reset(adapter);
9d5c8243
AK		 1611 igb_clean_all_tx_rings(adapter);
1612 igb_clean_all_rx_rings(adapter);
7e0e99ef
AD		 1613#ifdef CONFIG_IGB_DCA
1614
1615 /* since we reset the hardware DCA settings were cleared */
1616 igb_setup_dca(adapter);
1617#endif
9d5c8243
AK		 1618}
1619
1620void igb_reinit_locked(struct igb_adapter *adapter)
1621{
1622 WARN_ON(in_interrupt());
1623 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1624 msleep(1);
1625 igb_down(adapter);
1626 igb_up(adapter);
1627 clear_bit(__IGB_RESETTING, &adapter->state);
1628}
1629
1630void igb_reset(struct igb_adapter *adapter)
1631{
090b1795 1632 struct pci_dev *pdev = adapter->pdev;
9d5c8243 1633 struct e1000_hw *hw = &adapter->hw;
2d064c06
AD		 1634 struct e1000_mac_info *mac = &hw->mac;
1635 struct e1000_fc_info *fc = &hw->fc;
9d5c8243
AK		 1636 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1637 u16 hwm;
1638
1639 /* Repartition Pba for greater than 9k mtu
1640 * To take effect CTRL.RST is required.
1641 */
fa4dfae0 1642 switch (mac->type) {
d2ba2ed8 1643 case e1000_i350:
55cac248
AD		 1644 case e1000_82580:
1645 pba = rd32(E1000_RXPBS);
1646 pba = igb_rxpbs_adjust_82580(pba);
1647 break;
fa4dfae0 1648 case e1000_82576:
d249be54
AD		 1649 pba = rd32(E1000_RXPBS);
1650 pba &= E1000_RXPBS_SIZE_MASK_82576;
fa4dfae0
AD		 1651 break;
1652 case e1000_82575:
1653 default:
1654 pba = E1000_PBA_34K;
1655 break;
2d064c06 1656 }
9d5c8243 1657
2d064c06
AD		 1658 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1659 (mac->type < e1000_82576)) {
9d5c8243
AK		 1660 /* adjust PBA for jumbo frames */
1661 wr32(E1000_PBA, pba);
1662
1663 /* To maintain wire speed transmits, the Tx FIFO should be
1664 * large enough to accommodate two full transmit packets,
1665 * rounded up to the next 1KB and expressed in KB. Likewise,
1666 * the Rx FIFO should be large enough to accommodate at least
1667 * one full receive packet and is similarly rounded up and
1668 * expressed in KB. */
1669 pba = rd32(E1000_PBA);
1670 /* upper 16 bits has Tx packet buffer allocation size in KB */
1671 tx_space = pba >> 16;
1672 /* lower 16 bits has Rx packet buffer allocation size in KB */
1673 pba &= 0xffff;
1674 /* the tx fifo also stores 16 bytes of information about the tx
1675 * but don't include ethernet FCS because hardware appends it */
1676 min_tx_space = (adapter->max_frame_size +
85e8d004 1677 sizeof(union e1000_adv_tx_desc) -
9d5c8243
AK		 1678 ETH_FCS_LEN) * 2;
1679 min_tx_space = ALIGN(min_tx_space, 1024);
1680 min_tx_space >>= 10;
1681 /* software strips receive CRC, so leave room for it */
1682 min_rx_space = adapter->max_frame_size;
1683 min_rx_space = ALIGN(min_rx_space, 1024);
1684 min_rx_space >>= 10;
1685
1686 /* If current Tx allocation is less than the min Tx FIFO size,
1687 * and the min Tx FIFO size is less than the current Rx FIFO
1688 * allocation, take space away from current Rx allocation */
1689 if (tx_space < min_tx_space &&
1690 ((min_tx_space - tx_space) < pba)) {
1691 pba = pba - (min_tx_space - tx_space);
1692
1693 /* if short on rx space, rx wins and must trump tx
1694 * adjustment */
1695 if (pba < min_rx_space)
1696 pba = min_rx_space;
1697 }
2d064c06 1698 wr32(E1000_PBA, pba);
9d5c8243 1699 }
9d5c8243
AK		 1700
1701 /* flow control settings */
1702 /* The high water mark must be low enough to fit one full frame
1703 * (or the size used for early receive) above it in the Rx FIFO.
1704 * Set it to the lower of:
1705 * - 90% of the Rx FIFO size, or
1706 * - the full Rx FIFO size minus one full frame */
1707 hwm = min(((pba << 10) * 9 / 10),
2d064c06 1708 ((pba << 10) - 2 * adapter->max_frame_size));
9d5c8243 1709
d405ea3e
AD		 1710 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1711 fc->low_water = fc->high_water - 16;
9d5c8243
AK		 1712 fc->pause_time = 0xFFFF;
1713 fc->send_xon = 1;
0cce119a 1714 fc->current_mode = fc->requested_mode;
9d5c8243 1715
4ae196df
AD		 1716 /* disable receive for all VFs and wait one second */
1717 if (adapter->vfs_allocated_count) {
1718 int i;
1719 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
8fa7e0f7 1720 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
4ae196df
AD		 1721
1722 /* ping all the active vfs to let them know we are going down */
f2ca0dbe 1723 igb_ping_all_vfs(adapter);
4ae196df
AD		 1724
1725 /* disable transmits and receives */
1726 wr32(E1000_VFRE, 0);
1727 wr32(E1000_VFTE, 0);
1728 }
1729
9d5c8243 1730 /* Allow time for pending master requests to run */
330a6d6a 1731 hw->mac.ops.reset_hw(hw);
9d5c8243
AK		 1732 wr32(E1000_WUC, 0);
1733
330a6d6a 1734 if (hw->mac.ops.init_hw(hw))
090b1795 1735 dev_err(&pdev->dev, "Hardware Error\n");
831ec0b4
CW		 1736 if (hw->mac.type > e1000_82580) {
1737 if (adapter->flags & IGB_FLAG_DMAC) {
1738 u32 reg;
1739
1740 /*
1741 * DMA Coalescing high water mark needs to be higher
1742 * than * the * Rx threshold. The Rx threshold is
1743 * currently * pba - 6, so we * should use a high water
1744 * mark of pba * - 4. */
1745 hwm = (pba - 4) << 10;
1746
1747 reg = (((pba-6) << E1000_DMACR_DMACTHR_SHIFT)
1748 & E1000_DMACR_DMACTHR_MASK);
1749
1750 /* transition to L0x or L1 if available..*/
1751 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
1752
1753 /* watchdog timer= +-1000 usec in 32usec intervals */
1754 reg |= (1000 >> 5);
1755 wr32(E1000_DMACR, reg);
1756
1757 /* no lower threshold to disable coalescing(smart fifb)
1758 * -UTRESH=0*/
1759 wr32(E1000_DMCRTRH, 0);
1760
1761 /* set hwm to PBA - 2 * max frame size */
1762 wr32(E1000_FCRTC, hwm);
1763
1764 /*
1765 * This sets the time to wait before requesting tran-
1766 * sition to * low power state to number of usecs needed
1767 * to receive 1 512 * byte frame at gigabit line rate
1768 */
1769 reg = rd32(E1000_DMCTLX);
1770 reg |= IGB_DMCTLX_DCFLUSH_DIS;
1771
1772 /* Delay 255 usec before entering Lx state. */
1773 reg |= 0xFF;
1774 wr32(E1000_DMCTLX, reg);
1775
1776 /* free space in Tx packet buffer to wake from DMAC */
1777 wr32(E1000_DMCTXTH,
1778 (IGB_MIN_TXPBSIZE -
1779 (IGB_TX_BUF_4096 + adapter->max_frame_size))
1780 >> 6);
1781
1782 /* make low power state decision controlled by DMAC */
1783 reg = rd32(E1000_PCIEMISC);
1784 reg |= E1000_PCIEMISC_LX_DECISION;
1785 wr32(E1000_PCIEMISC, reg);
1786 } /* end if IGB_FLAG_DMAC set */
1787 }
55cac248
AD		 1788 if (hw->mac.type == e1000_82580) {
1789 u32 reg = rd32(E1000_PCIEMISC);
1790 wr32(E1000_PCIEMISC,
1791 reg & ~E1000_PCIEMISC_LX_DECISION);
1792 }
88a268c1
NN		 1793 if (!netif_running(adapter->netdev))
1794 igb_power_down_link(adapter);
1795
9d5c8243
AK		 1796 igb_update_mng_vlan(adapter);
1797
1798 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1799 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1800
330a6d6a 1801 igb_get_phy_info(hw);
9d5c8243
AK		 1802}
1803
b2cb09b1
JP		 1804static u32 igb_fix_features(struct net_device *netdev, u32 features)
1805{
1806 /*
1807 * Since there is no support for separate rx/tx vlan accel
1808 * enable/disable make sure tx flag is always in same state as rx.
1809 */
1810 if (features & NETIF_F_HW_VLAN_RX)
1811 features |= NETIF_F_HW_VLAN_TX;
1812 else
1813 features &= ~NETIF_F_HW_VLAN_TX;
1814
1815 return features;
1816}
1817
ac52caa3
MM		 1818static int igb_set_features(struct net_device *netdev, u32 features)
1819{
1820 struct igb_adapter *adapter = netdev_priv(netdev);
1821 int i;
b2cb09b1 1822 u32 changed = netdev->features ^ features;
ac52caa3
MM		 1823
1824 for (i = 0; i < adapter->num_rx_queues; i++) {
1825 if (features & NETIF_F_RXCSUM)
1826 adapter->rx_ring[i]->flags |= IGB_RING_FLAG_RX_CSUM;
1827 else
1828 adapter->rx_ring[i]->flags &= ~IGB_RING_FLAG_RX_CSUM;
1829 }
1830
b2cb09b1
JP		 1831 if (changed & NETIF_F_HW_VLAN_RX)
1832 igb_vlan_mode(netdev, features);
1833
ac52caa3
MM		 1834 return 0;
1835}
1836
2e5c6922 1837static const struct net_device_ops igb_netdev_ops = {
559e9c49 1838 .ndo_open = igb_open,
2e5c6922 1839 .ndo_stop = igb_close,
cd392f5c 1840 .ndo_start_xmit = igb_xmit_frame,
12dcd86b 1841 .ndo_get_stats64 = igb_get_stats64,
ff41f8dc 1842 .ndo_set_rx_mode = igb_set_rx_mode,
2e5c6922
SH		 1843 .ndo_set_mac_address = igb_set_mac,
1844 .ndo_change_mtu = igb_change_mtu,
1845 .ndo_do_ioctl = igb_ioctl,
1846 .ndo_tx_timeout = igb_tx_timeout,
1847 .ndo_validate_addr = eth_validate_addr,
2e5c6922
SH		 1848 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1849 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
8151d294
WM		 1850 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1851 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1852 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1853 .ndo_get_vf_config = igb_ndo_get_vf_config,
2e5c6922
SH		 1854#ifdef CONFIG_NET_POLL_CONTROLLER
1855 .ndo_poll_controller = igb_netpoll,
1856#endif
b2cb09b1
JP		 1857 .ndo_fix_features = igb_fix_features,
1858 .ndo_set_features = igb_set_features,
2e5c6922
SH		 1859};
1860
9d5c8243
AK		 1861/**
1862 * igb_probe - Device Initialization Routine
1863 * @pdev: PCI device information struct
1864 * @ent: entry in igb_pci_tbl
1865 *
1866 * Returns 0 on success, negative on failure
1867 *
1868 * igb_probe initializes an adapter identified by a pci_dev structure.
1869 * The OS initialization, configuring of the adapter private structure,
1870 * and a hardware reset occur.
1871 **/
1872static int __devinit igb_probe(struct pci_dev *pdev,
1873 const struct pci_device_id *ent)
1874{
1875 struct net_device *netdev;
1876 struct igb_adapter *adapter;
1877 struct e1000_hw *hw;
4337e993 1878 u16 eeprom_data = 0;
9835fd73 1879 s32 ret_val;
4337e993 1880 static int global_quad_port_a; /* global quad port a indication */
9d5c8243
AK		 1881 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1882 unsigned long mmio_start, mmio_len;
2d6a5e95 1883 int err, pci_using_dac;
9d5c8243 1884 u16 eeprom_apme_mask = IGB_EEPROM_APME;
9835fd73 1885 u8 part_str[E1000_PBANUM_LENGTH];
9d5c8243 1886
bded64a7
AG		 1887 /* Catch broken hardware that put the wrong VF device ID in
1888 * the PCIe SR-IOV capability.
1889 */
1890 if (pdev->is_virtfn) {
1891 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1892 pci_name(pdev), pdev->vendor, pdev->device);
1893 return -EINVAL;
1894 }
1895
aed5dec3 1896 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 1897 if (err)
1898 return err;
1899
1900 pci_using_dac = 0;
59d71989 1901 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243 1902 if (!err) {
59d71989 1903 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
9d5c8243
AK		 1904 if (!err)
1905 pci_using_dac = 1;
1906 } else {
59d71989 1907 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243 1908 if (err) {
59d71989 1909 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
9d5c8243
AK		 1910 if (err) {
1911 dev_err(&pdev->dev, "No usable DMA "
1912 "configuration, aborting\n");
1913 goto err_dma;
1914 }
1915 }
1916 }
1917
aed5dec3
AD		 1918 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1919 IORESOURCE_MEM),
1920 igb_driver_name);
9d5c8243
AK		 1921 if (err)
1922 goto err_pci_reg;
1923
19d5afd4 1924 pci_enable_pcie_error_reporting(pdev);
40a914fa 1925
9d5c8243 1926 pci_set_master(pdev);
c682fc23 1927 pci_save_state(pdev);
9d5c8243
AK		 1928
1929 err = -ENOMEM;
1bfaf07b 1930 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1cc3bd87 1931 IGB_MAX_TX_QUEUES);
9d5c8243
AK		 1932 if (!netdev)
1933 goto err_alloc_etherdev;
1934
1935 SET_NETDEV_DEV(netdev, &pdev->dev);
1936
1937 pci_set_drvdata(pdev, netdev);
1938 adapter = netdev_priv(netdev);
1939 adapter->netdev = netdev;
1940 adapter->pdev = pdev;
1941 hw = &adapter->hw;
1942 hw->back = adapter;
1943 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1944
1945 mmio_start = pci_resource_start(pdev, 0);
1946 mmio_len = pci_resource_len(pdev, 0);
1947
1948 err = -EIO;
28b0759c
AD		 1949 hw->hw_addr = ioremap(mmio_start, mmio_len);
1950 if (!hw->hw_addr)
9d5c8243
AK		 1951 goto err_ioremap;
1952
2e5c6922 1953 netdev->netdev_ops = &igb_netdev_ops;
9d5c8243 1954 igb_set_ethtool_ops(netdev);
9d5c8243 1955 netdev->watchdog_timeo = 5 * HZ;
9d5c8243
AK		 1956
1957 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1958
1959 netdev->mem_start = mmio_start;
1960 netdev->mem_end = mmio_start + mmio_len;
1961
9d5c8243
AK		 1962 /* PCI config space info */
1963 hw->vendor_id = pdev->vendor;
1964 hw->device_id = pdev->device;
1965 hw->revision_id = pdev->revision;
1966 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1967 hw->subsystem_device_id = pdev->subsystem_device;
1968
9d5c8243
AK		 1969 /* Copy the default MAC, PHY and NVM function pointers */
1970 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1971 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1972 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1973 /* Initialize skew-specific constants */
1974 err = ei->get_invariants(hw);
1975 if (err)
450c87c8 1976 goto err_sw_init;
9d5c8243 1977
450c87c8 1978 /* setup the private structure */
9d5c8243
AK		 1979 err = igb_sw_init(adapter);
1980 if (err)
1981 goto err_sw_init;
1982
1983 igb_get_bus_info_pcie(hw);
1984
1985 hw->phy.autoneg_wait_to_complete = false;
9d5c8243
AK		 1986
1987 /* Copper options */
1988 if (hw->phy.media_type == e1000_media_type_copper) {
1989 hw->phy.mdix = AUTO_ALL_MODES;
1990 hw->phy.disable_polarity_correction = false;
1991 hw->phy.ms_type = e1000_ms_hw_default;
1992 }
1993
1994 if (igb_check_reset_block(hw))
1995 dev_info(&pdev->dev,
1996 "PHY reset is blocked due to SOL/IDER session.\n");
1997
ac52caa3 1998 netdev->hw_features = NETIF_F_SG |
7d8eb29e 1999 NETIF_F_IP_CSUM |
ac52caa3
MM		 2000 NETIF_F_IPV6_CSUM |
2001 NETIF_F_TSO |
2002 NETIF_F_TSO6 |
b2cb09b1
JP		 2003 NETIF_F_RXCSUM |
2004 NETIF_F_HW_VLAN_RX;
ac52caa3
MM		 2005
2006 netdev->features = netdev->hw_features |
9d5c8243 2007 NETIF_F_HW_VLAN_TX |
9d5c8243
AK		 2008 NETIF_F_HW_VLAN_FILTER;
2009
48f29ffc
JK		 2010 netdev->vlan_features |= NETIF_F_TSO;
2011 netdev->vlan_features |= NETIF_F_TSO6;
7d8eb29e 2012 netdev->vlan_features |= NETIF_F_IP_CSUM;
cd1da503 2013 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
48f29ffc
JK		 2014 netdev->vlan_features |= NETIF_F_SG;
2015
7b872a55 2016 if (pci_using_dac) {
9d5c8243 2017 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 2018 netdev->vlan_features |= NETIF_F_HIGHDMA;
2019 }
9d5c8243 2020
ac52caa3
MM		 2021 if (hw->mac.type >= e1000_82576) {
2022 netdev->hw_features |= NETIF_F_SCTP_CSUM;
b9473560 2023 netdev->features |= NETIF_F_SCTP_CSUM;
ac52caa3 2024 }
b9473560 2025
01789349
JP		 2026 netdev->priv_flags |= IFF_UNICAST_FLT;
2027
330a6d6a 2028 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
9d5c8243
AK		 2029
2030 /* before reading the NVM, reset the controller to put the device in a
2031 * known good starting state */
2032 hw->mac.ops.reset_hw(hw);
2033
2034 /* make sure the NVM is good */
4322e561 2035 if (hw->nvm.ops.validate(hw) < 0) {
9d5c8243
AK		 2036 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
2037 err = -EIO;
2038 goto err_eeprom;
2039 }
2040
2041 /* copy the MAC address out of the NVM */
2042 if (hw->mac.ops.read_mac_addr(hw))
2043 dev_err(&pdev->dev, "NVM Read Error\n");
2044
2045 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2046 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2047
2048 if (!is_valid_ether_addr(netdev->perm_addr)) {
2049 dev_err(&pdev->dev, "Invalid MAC Address\n");
2050 err = -EIO;
2051 goto err_eeprom;
2052 }
2053
c061b18d 2054 setup_timer(&adapter->watchdog_timer, igb_watchdog,
0e340485 2055 (unsigned long) adapter);
c061b18d 2056 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
0e340485 2057 (unsigned long) adapter);
9d5c8243
AK		 2058
2059 INIT_WORK(&adapter->reset_task, igb_reset_task);
2060 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2061
450c87c8 2062 /* Initialize link properties that are user-changeable */
9d5c8243
AK		 2063 adapter->fc_autoneg = true;
2064 hw->mac.autoneg = true;
2065 hw->phy.autoneg_advertised = 0x2f;
2066
0cce119a
AD		 2067 hw->fc.requested_mode = e1000_fc_default;
2068 hw->fc.current_mode = e1000_fc_default;
9d5c8243 2069
9d5c8243
AK		 2070 igb_validate_mdi_setting(hw);
2071
9d5c8243
AK		 2072 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2073 * enable the ACPI Magic Packet filter
2074 */
2075
a2cf8b6c 2076 if (hw->bus.func == 0)
312c75ae 2077 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
6d337dce 2078 else if (hw->mac.type >= e1000_82580)
55cac248
AD		 2079 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2080 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2081 &eeprom_data);
a2cf8b6c
AD		 2082 else if (hw->bus.func == 1)
2083 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
9d5c8243
AK		 2084
2085 if (eeprom_data & eeprom_apme_mask)
2086 adapter->eeprom_wol |= E1000_WUFC_MAG;
2087
2088 /* now that we have the eeprom settings, apply the special cases where
2089 * the eeprom may be wrong or the board simply won't support wake on
2090 * lan on a particular port */
2091 switch (pdev->device) {
2092 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2093 adapter->eeprom_wol = 0;
2094 break;
2095 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2d064c06
AD		 2096 case E1000_DEV_ID_82576_FIBER:
2097 case E1000_DEV_ID_82576_SERDES:
9d5c8243
AK		 2098 /* Wake events only supported on port A for dual fiber
2099 * regardless of eeprom setting */
2100 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2101 adapter->eeprom_wol = 0;
2102 break;
c8ea5ea9 2103 case E1000_DEV_ID_82576_QUAD_COPPER:
d5aa2252 2104 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
c8ea5ea9
AD		 2105 /* if quad port adapter, disable WoL on all but port A */
2106 if (global_quad_port_a != 0)
2107 adapter->eeprom_wol = 0;
2108 else
2109 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2110 /* Reset for multiple quad port adapters */
2111 if (++global_quad_port_a == 4)
2112 global_quad_port_a = 0;
2113 break;
9d5c8243
AK		 2114 }
2115
2116 /* initialize the wol settings based on the eeprom settings */
2117 adapter->wol = adapter->eeprom_wol;
e1b86d84 2118 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9d5c8243
AK		 2119
2120 /* reset the hardware with the new settings */
2121 igb_reset(adapter);
2122
2123 /* let the f/w know that the h/w is now under the control of the
2124 * driver. */
2125 igb_get_hw_control(adapter);
2126
9d5c8243
AK		 2127 strcpy(netdev->name, "eth%d");
2128 err = register_netdev(netdev);
2129 if (err)
2130 goto err_register;
2131
b2cb09b1
JP		 2132 igb_vlan_mode(netdev, netdev->features);
2133
b168dfc5
JB		 2134 /* carrier off reporting is important to ethtool even BEFORE open */
2135 netif_carrier_off(netdev);
2136
421e02f0 2137#ifdef CONFIG_IGB_DCA
bbd98fe4 2138 if (dca_add_requester(&pdev->dev) == 0) {
7dfc16fa 2139 adapter->flags |= IGB_FLAG_DCA_ENABLED;
fe4506b6 2140 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 2141 igb_setup_dca(adapter);
2142 }
fe4506b6 2143
38c845c7 2144#endif
673b8b70
AB		 2145 /* do hw tstamp init after resetting */
2146 igb_init_hw_timer(adapter);
2147
9d5c8243
AK		 2148 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2149 /* print bus type/speed/width info */
7c510e4b 2150 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
9d5c8243 2151 netdev->name,
559e9c49 2152 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
ff846f52 2153 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
559e9c49 2154 "unknown"),
59c3de89
AD		 2155 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2156 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2157 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2158 "unknown"),
7c510e4b 2159 netdev->dev_addr);
9d5c8243 2160
9835fd73
CW		 2161 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2162 if (ret_val)
2163 strcpy(part_str, "Unknown");
2164 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
9d5c8243
AK		 2165 dev_info(&pdev->dev,
2166 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2167 adapter->msix_entries ? "MSI-X" :
7dfc16fa 2168 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9d5c8243 2169 adapter->num_rx_queues, adapter->num_tx_queues);
09b068d4
CW		 2170 switch (hw->mac.type) {
2171 case e1000_i350:
2172 igb_set_eee_i350(hw);
2173 break;
2174 default:
2175 break;
2176 }
9d5c8243
AK		 2177 return 0;
2178
2179err_register:
2180 igb_release_hw_control(adapter);
2181err_eeprom:
2182 if (!igb_check_reset_block(hw))
f5f4cf08 2183 igb_reset_phy(hw);
9d5c8243
AK		 2184
2185 if (hw->flash_address)
2186 iounmap(hw->flash_address);
9d5c8243 2187err_sw_init:
047e0030 2188 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 2189 iounmap(hw->hw_addr);
2190err_ioremap:
2191 free_netdev(netdev);
2192err_alloc_etherdev:
559e9c49
AD		 2193 pci_release_selected_regions(pdev,
2194 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2195err_pci_reg:
2196err_dma:
2197 pci_disable_device(pdev);
2198 return err;
2199}
2200
2201/**
2202 * igb_remove - Device Removal Routine
2203 * @pdev: PCI device information struct
2204 *
2205 * igb_remove is called by the PCI subsystem to alert the driver
2206 * that it should release a PCI device. The could be caused by a
2207 * Hot-Plug event, or because the driver is going to be removed from
2208 * memory.
2209 **/
2210static void __devexit igb_remove(struct pci_dev *pdev)
2211{
2212 struct net_device *netdev = pci_get_drvdata(pdev);
2213 struct igb_adapter *adapter = netdev_priv(netdev);
fe4506b6 2214 struct e1000_hw *hw = &adapter->hw;
9d5c8243 2215
760141a5
TH		 2216 /*
2217 * The watchdog timer may be rescheduled, so explicitly
2218 * disable watchdog from being rescheduled.
2219 */
9d5c8243
AK		 2220 set_bit(__IGB_DOWN, &adapter->state);
2221 del_timer_sync(&adapter->watchdog_timer);
2222 del_timer_sync(&adapter->phy_info_timer);
2223
760141a5
TH		 2224 cancel_work_sync(&adapter->reset_task);
2225 cancel_work_sync(&adapter->watchdog_task);
9d5c8243 2226
421e02f0 2227#ifdef CONFIG_IGB_DCA
7dfc16fa 2228 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6
JC		 2229 dev_info(&pdev->dev, "DCA disabled\n");
2230 dca_remove_requester(&pdev->dev);
7dfc16fa 2231 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 2232 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 2233 }
2234#endif
2235
9d5c8243
AK		 2236 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2237 * would have already happened in close and is redundant. */
2238 igb_release_hw_control(adapter);
2239
2240 unregister_netdev(netdev);
2241
047e0030 2242 igb_clear_interrupt_scheme(adapter);
9d5c8243 2243
37680117
AD		 2244#ifdef CONFIG_PCI_IOV
2245 /* reclaim resources allocated to VFs */
2246 if (adapter->vf_data) {
2247 /* disable iov and allow time for transactions to clear */
2248 pci_disable_sriov(pdev);
2249 msleep(500);
2250
2251 kfree(adapter->vf_data);
2252 adapter->vf_data = NULL;
2253 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
945a5151 2254 wrfl();
37680117
AD		 2255 msleep(100);
2256 dev_info(&pdev->dev, "IOV Disabled\n");
2257 }
2258#endif
559e9c49 2259
28b0759c
AD		 2260 iounmap(hw->hw_addr);
2261 if (hw->flash_address)
2262 iounmap(hw->flash_address);
559e9c49
AD		 2263 pci_release_selected_regions(pdev,
2264 pci_select_bars(pdev, IORESOURCE_MEM));
9d5c8243
AK		 2265
2266 free_netdev(netdev);
2267
19d5afd4 2268 pci_disable_pcie_error_reporting(pdev);
40a914fa 2269
9d5c8243
AK		 2270 pci_disable_device(pdev);
2271}
2272
a6b623e0
AD		 2273/**
2274 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2275 * @adapter: board private structure to initialize
2276 *
2277 * This function initializes the vf specific data storage and then attempts to
2278 * allocate the VFs. The reason for ordering it this way is because it is much
2279 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2280 * the memory for the VFs.
2281 **/
2282static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2283{
2284#ifdef CONFIG_PCI_IOV
2285 struct pci_dev *pdev = adapter->pdev;
2286
a6b623e0
AD		 2287 if (adapter->vfs_allocated_count) {
2288 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2289 sizeof(struct vf_data_storage),
2290 GFP_KERNEL);
2291 /* if allocation failed then we do not support SR-IOV */
2292 if (!adapter->vf_data) {
2293 adapter->vfs_allocated_count = 0;
2294 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2295 "Data Storage\n");
2296 }
2297 }
2298
2299 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2300 kfree(adapter->vf_data);
2301 adapter->vf_data = NULL;
2302#endif /* CONFIG_PCI_IOV */
2303 adapter->vfs_allocated_count = 0;
2304#ifdef CONFIG_PCI_IOV
2305 } else {
2306 unsigned char mac_addr[ETH_ALEN];
2307 int i;
2308 dev_info(&pdev->dev, "%d vfs allocated\n",
2309 adapter->vfs_allocated_count);
2310 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2311 random_ether_addr(mac_addr);
2312 igb_set_vf_mac(adapter, i, mac_addr);
2313 }
831ec0b4
CW		 2314 /* DMA Coalescing is not supported in IOV mode. */
2315 if (adapter->flags & IGB_FLAG_DMAC)
2316 adapter->flags &= ~IGB_FLAG_DMAC;
a6b623e0
AD		 2317 }
2318#endif /* CONFIG_PCI_IOV */
2319}
2320
115f459a
AD		 2321
2322/**
2323 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2324 * @adapter: board private structure to initialize
2325 *
2326 * igb_init_hw_timer initializes the function pointer and values for the hw
2327 * timer found in hardware.
2328 **/
2329static void igb_init_hw_timer(struct igb_adapter *adapter)
2330{
2331 struct e1000_hw *hw = &adapter->hw;
2332
2333 switch (hw->mac.type) {
d2ba2ed8 2334 case e1000_i350:
55cac248
AD		 2335 case e1000_82580:
2336 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2337 adapter->cycles.read = igb_read_clock;
2338 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2339 adapter->cycles.mult = 1;
2340 /*
2341 * The 82580 timesync updates the system timer every 8ns by 8ns
2342 * and the value cannot be shifted. Instead we need to shift
2343 * the registers to generate a 64bit timer value. As a result
2344 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2345 * 24 in order to generate a larger value for synchronization.
2346 */
2347 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2348 /* disable system timer temporarily by setting bit 31 */
2349 wr32(E1000_TSAUXC, 0x80000000);
2350 wrfl();
2351
2352 /* Set registers so that rollover occurs soon to test this. */
2353 wr32(E1000_SYSTIMR, 0x00000000);
2354 wr32(E1000_SYSTIML, 0x80000000);
2355 wr32(E1000_SYSTIMH, 0x000000FF);
2356 wrfl();
2357
2358 /* enable system timer by clearing bit 31 */
2359 wr32(E1000_TSAUXC, 0x0);
2360 wrfl();
2361
2362 timecounter_init(&adapter->clock,
2363 &adapter->cycles,
2364 ktime_to_ns(ktime_get_real()));
2365 /*
2366 * Synchronize our NIC clock against system wall clock. NIC
2367 * time stamp reading requires ~3us per sample, each sample
2368 * was pretty stable even under load => only require 10
2369 * samples for each offset comparison.
2370 */
2371 memset(&adapter->compare, 0, sizeof(adapter->compare));
2372 adapter->compare.source = &adapter->clock;
2373 adapter->compare.target = ktime_get_real;
2374 adapter->compare.num_samples = 10;
2375 timecompare_update(&adapter->compare, 0);
2376 break;
115f459a
AD		 2377 case e1000_82576:
2378 /*
2379 * Initialize hardware timer: we keep it running just in case
2380 * that some program needs it later on.
2381 */
2382 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2383 adapter->cycles.read = igb_read_clock;
2384 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2385 adapter->cycles.mult = 1;
2386 /**
2387 * Scale the NIC clock cycle by a large factor so that
2388 * relatively small clock corrections can be added or
25985edc 2389 * subtracted at each clock tick. The drawbacks of a large
115f459a
AD		 2390 * factor are a) that the clock register overflows more quickly
2391 * (not such a big deal) and b) that the increment per tick has
2392 * to fit into 24 bits. As a result we need to use a shift of
2393 * 19 so we can fit a value of 16 into the TIMINCA register.
2394 */
2395 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2396 wr32(E1000_TIMINCA,
2397 (1 << E1000_TIMINCA_16NS_SHIFT) |
2398 (16 << IGB_82576_TSYNC_SHIFT));
2399
2400 /* Set registers so that rollover occurs soon to test this. */
2401 wr32(E1000_SYSTIML, 0x00000000);
2402 wr32(E1000_SYSTIMH, 0xFF800000);
2403 wrfl();
2404
2405 timecounter_init(&adapter->clock,
2406 &adapter->cycles,
2407 ktime_to_ns(ktime_get_real()));
2408 /*
2409 * Synchronize our NIC clock against system wall clock. NIC
2410 * time stamp reading requires ~3us per sample, each sample
2411 * was pretty stable even under load => only require 10
2412 * samples for each offset comparison.
2413 */
2414 memset(&adapter->compare, 0, sizeof(adapter->compare));
2415 adapter->compare.source = &adapter->clock;
2416 adapter->compare.target = ktime_get_real;
2417 adapter->compare.num_samples = 10;
2418 timecompare_update(&adapter->compare, 0);
2419 break;
2420 case e1000_82575:
2421 /* 82575 does not support timesync */
2422 default:
2423 break;
2424 }
2425
2426}
2427
9d5c8243
AK		 2428/**
2429 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2430 * @adapter: board private structure to initialize
2431 *
2432 * igb_sw_init initializes the Adapter private data structure.
2433 * Fields are initialized based on PCI device information and
2434 * OS network device settings (MTU size).
2435 **/
2436static int __devinit igb_sw_init(struct igb_adapter *adapter)
2437{
2438 struct e1000_hw *hw = &adapter->hw;
2439 struct net_device *netdev = adapter->netdev;
2440 struct pci_dev *pdev = adapter->pdev;
2441
2442 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2443
13fde97a 2444 /* set default ring sizes */
68fd9910
AD		 2445 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2446 adapter->rx_ring_count = IGB_DEFAULT_RXD;
13fde97a
AD		 2447
2448 /* set default ITR values */
4fc82adf
AD		 2449 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2450 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2451
13fde97a
AD		 2452 /* set default work limits */
2453 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2454
153285f9
AD		 2455 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2456 VLAN_HLEN;
9d5c8243
AK		 2457 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2458
81c2fc22
AD		 2459 adapter->node = -1;
2460
12dcd86b 2461 spin_lock_init(&adapter->stats64_lock);
a6b623e0 2462#ifdef CONFIG_PCI_IOV
6b78bb1d
CW		 2463 switch (hw->mac.type) {
2464 case e1000_82576:
2465 case e1000_i350:
9b082d73
SA		 2466 if (max_vfs > 7) {
2467 dev_warn(&pdev->dev,
2468 "Maximum of 7 VFs per PF, using max\n");
2469 adapter->vfs_allocated_count = 7;
2470 } else
2471 adapter->vfs_allocated_count = max_vfs;
6b78bb1d
CW		 2472 break;
2473 default:
2474 break;
2475 }
a6b623e0 2476#endif /* CONFIG_PCI_IOV */
a99955fc 2477 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
665c8c8e
WM		 2478 /* i350 cannot do RSS and SR-IOV at the same time */
2479 if (hw->mac.type == e1000_i350 && adapter->vfs_allocated_count)
2480 adapter->rss_queues = 1;
a99955fc
AD		 2481
2482 /*
2483 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2484 * then we should combine the queues into a queue pair in order to
2485 * conserve interrupts due to limited supply
2486 */
2487 if ((adapter->rss_queues > 4) ||
2488 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2489 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2490
a6b623e0 2491 /* This call may decrease the number of queues */
047e0030 2492 if (igb_init_interrupt_scheme(adapter)) {
9d5c8243
AK		 2493 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2494 return -ENOMEM;
2495 }
2496
a6b623e0
AD		 2497 igb_probe_vfs(adapter);
2498
9d5c8243
AK		 2499 /* Explicitly disable IRQ since the NIC can be in any state. */
2500 igb_irq_disable(adapter);
2501
831ec0b4
CW		 2502 if (hw->mac.type == e1000_i350)
2503 adapter->flags &= ~IGB_FLAG_DMAC;
2504
9d5c8243
AK		 2505 set_bit(__IGB_DOWN, &adapter->state);
2506 return 0;
2507}
2508
2509/**
2510 * igb_open - Called when a network interface is made active
2511 * @netdev: network interface device structure
2512 *
2513 * Returns 0 on success, negative value on failure
2514 *
2515 * The open entry point is called when a network interface is made
2516 * active by the system (IFF_UP). At this point all resources needed
2517 * for transmit and receive operations are allocated, the interrupt
2518 * handler is registered with the OS, the watchdog timer is started,
2519 * and the stack is notified that the interface is ready.
2520 **/
2521static int igb_open(struct net_device *netdev)
2522{
2523 struct igb_adapter *adapter = netdev_priv(netdev);
2524 struct e1000_hw *hw = &adapter->hw;
2525 int err;
2526 int i;
2527
2528 /* disallow open during test */
2529 if (test_bit(__IGB_TESTING, &adapter->state))
2530 return -EBUSY;
2531
b168dfc5
JB		 2532 netif_carrier_off(netdev);
2533
9d5c8243
AK		 2534 /* allocate transmit descriptors */
2535 err = igb_setup_all_tx_resources(adapter);
2536 if (err)
2537 goto err_setup_tx;
2538
2539 /* allocate receive descriptors */
2540 err = igb_setup_all_rx_resources(adapter);
2541 if (err)
2542 goto err_setup_rx;
2543
88a268c1 2544 igb_power_up_link(adapter);
9d5c8243 2545
9d5c8243
AK		 2546 /* before we allocate an interrupt, we must be ready to handle it.
2547 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2548 * as soon as we call pci_request_irq, so we have to setup our
2549 * clean_rx handler before we do so. */
2550 igb_configure(adapter);
2551
2552 err = igb_request_irq(adapter);
2553 if (err)
2554 goto err_req_irq;
2555
2556 /* From here on the code is the same as igb_up() */
2557 clear_bit(__IGB_DOWN, &adapter->state);
2558
047e0030
AD		 2559 for (i = 0; i < adapter->num_q_vectors; i++) {
2560 struct igb_q_vector *q_vector = adapter->q_vector[i];
2561 napi_enable(&q_vector->napi);
2562 }
9d5c8243
AK		 2563
2564 /* Clear any pending interrupts. */
2565 rd32(E1000_ICR);
844290e5
PW		 2566
2567 igb_irq_enable(adapter);
2568
d4960307
AD		 2569 /* notify VFs that reset has been completed */
2570 if (adapter->vfs_allocated_count) {
2571 u32 reg_data = rd32(E1000_CTRL_EXT);
2572 reg_data |= E1000_CTRL_EXT_PFRSTD;
2573 wr32(E1000_CTRL_EXT, reg_data);
2574 }
2575
d55b53ff
JK		 2576 netif_tx_start_all_queues(netdev);
2577
25568a53
AD		 2578 /* start the watchdog. */
2579 hw->mac.get_link_status = 1;
2580 schedule_work(&adapter->watchdog_task);
9d5c8243
AK		 2581
2582 return 0;
2583
2584err_req_irq:
2585 igb_release_hw_control(adapter);
88a268c1 2586 igb_power_down_link(adapter);
9d5c8243
AK		 2587 igb_free_all_rx_resources(adapter);
2588err_setup_rx:
2589 igb_free_all_tx_resources(adapter);
2590err_setup_tx:
2591 igb_reset(adapter);
2592
2593 return err;
2594}
2595
2596/**
2597 * igb_close - Disables a network interface
2598 * @netdev: network interface device structure
2599 *
2600 * Returns 0, this is not allowed to fail
2601 *
2602 * The close entry point is called when an interface is de-activated
2603 * by the OS. The hardware is still under the driver's control, but
2604 * needs to be disabled. A global MAC reset is issued to stop the
2605 * hardware, and all transmit and receive resources are freed.
2606 **/
2607static int igb_close(struct net_device *netdev)
2608{
2609 struct igb_adapter *adapter = netdev_priv(netdev);
2610
2611 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2612 igb_down(adapter);
2613
2614 igb_free_irq(adapter);
2615
2616 igb_free_all_tx_resources(adapter);
2617 igb_free_all_rx_resources(adapter);
2618
9d5c8243
AK		 2619 return 0;
2620}
2621
2622/**
2623 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
9d5c8243
AK		 2624 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2625 *
2626 * Return 0 on success, negative on failure
2627 **/
80785298 2628int igb_setup_tx_resources(struct igb_ring *tx_ring)
9d5c8243 2629{
59d71989 2630 struct device *dev = tx_ring->dev;
81c2fc22 2631 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2632 int size;
2633
06034649 2634 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
81c2fc22
AD		 2635 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2636 if (!tx_ring->tx_buffer_info)
2637 tx_ring->tx_buffer_info = vzalloc(size);
06034649 2638 if (!tx_ring->tx_buffer_info)
9d5c8243 2639 goto err;
9d5c8243
AK		 2640
2641 /* round up to nearest 4K */
85e8d004 2642 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
9d5c8243
AK		 2643 tx_ring->size = ALIGN(tx_ring->size, 4096);
2644
81c2fc22 2645 set_dev_node(dev, tx_ring->numa_node);
59d71989
AD		 2646 tx_ring->desc = dma_alloc_coherent(dev,
2647 tx_ring->size,
2648 &tx_ring->dma,
2649 GFP_KERNEL);
81c2fc22
AD		 2650 set_dev_node(dev, orig_node);
2651 if (!tx_ring->desc)
2652 tx_ring->desc = dma_alloc_coherent(dev,
2653 tx_ring->size,
2654 &tx_ring->dma,
2655 GFP_KERNEL);
9d5c8243
AK		 2656
2657 if (!tx_ring->desc)
2658 goto err;
2659
9d5c8243
AK		 2660 tx_ring->next_to_use = 0;
2661 tx_ring->next_to_clean = 0;
81c2fc22 2662
9d5c8243
AK		 2663 return 0;
2664
2665err:
06034649 2666 vfree(tx_ring->tx_buffer_info);
59d71989 2667 dev_err(dev,
9d5c8243
AK		 2668 "Unable to allocate memory for the transmit descriptor ring\n");
2669 return -ENOMEM;
2670}
2671
2672/**
2673 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2674 * (Descriptors) for all queues
2675 * @adapter: board private structure
2676 *
2677 * Return 0 on success, negative on failure
2678 **/
2679static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2680{
439705e1 2681 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2682 int i, err = 0;
2683
2684 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 2685 err = igb_setup_tx_resources(adapter->tx_ring[i]);
9d5c8243 2686 if (err) {
439705e1 2687 dev_err(&pdev->dev,
9d5c8243
AK		 2688 "Allocation for Tx Queue %u failed\n", i);
2689 for (i--; i >= 0; i--)
3025a446 2690 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 2691 break;
2692 }
2693 }
2694
2695 return err;
2696}
2697
2698/**
85b430b4
AD		 2699 * igb_setup_tctl - configure the transmit control registers
2700 * @adapter: Board private structure
9d5c8243 2701 **/
d7ee5b3a 2702void igb_setup_tctl(struct igb_adapter *adapter)
9d5c8243 2703{
9d5c8243
AK		 2704 struct e1000_hw *hw = &adapter->hw;
2705 u32 tctl;
9d5c8243 2706
85b430b4
AD		 2707 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2708 wr32(E1000_TXDCTL(0), 0);
9d5c8243
AK		 2709
2710 /* Program the Transmit Control Register */
9d5c8243
AK		 2711 tctl = rd32(E1000_TCTL);
2712 tctl &= ~E1000_TCTL_CT;
2713 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2714 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2715
2716 igb_config_collision_dist(hw);
2717
9d5c8243
AK		 2718 /* Enable transmits */
2719 tctl |= E1000_TCTL_EN;
2720
2721 wr32(E1000_TCTL, tctl);
2722}
2723
85b430b4
AD		 2724/**
2725 * igb_configure_tx_ring - Configure transmit ring after Reset
2726 * @adapter: board private structure
2727 * @ring: tx ring to configure
2728 *
2729 * Configure a transmit ring after a reset.
2730 **/
d7ee5b3a
AD		 2731void igb_configure_tx_ring(struct igb_adapter *adapter,
2732 struct igb_ring *ring)
85b430b4
AD		 2733{
2734 struct e1000_hw *hw = &adapter->hw;
a74420e0 2735 u32 txdctl = 0;
85b430b4
AD		 2736 u64 tdba = ring->dma;
2737 int reg_idx = ring->reg_idx;
2738
2739 /* disable the queue */
a74420e0 2740 wr32(E1000_TXDCTL(reg_idx), 0);
85b430b4
AD		 2741 wrfl();
2742 mdelay(10);
2743
2744 wr32(E1000_TDLEN(reg_idx),
2745 ring->count * sizeof(union e1000_adv_tx_desc));
2746 wr32(E1000_TDBAL(reg_idx),
2747 tdba & 0x00000000ffffffffULL);
2748 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2749
fce99e34 2750 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
a74420e0 2751 wr32(E1000_TDH(reg_idx), 0);
fce99e34 2752 writel(0, ring->tail);
85b430b4
AD		 2753
2754 txdctl |= IGB_TX_PTHRESH;
2755 txdctl |= IGB_TX_HTHRESH << 8;
2756 txdctl |= IGB_TX_WTHRESH << 16;
2757
2758 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2759 wr32(E1000_TXDCTL(reg_idx), txdctl);
2760}
2761
2762/**
2763 * igb_configure_tx - Configure transmit Unit after Reset
2764 * @adapter: board private structure
2765 *
2766 * Configure the Tx unit of the MAC after a reset.
2767 **/
2768static void igb_configure_tx(struct igb_adapter *adapter)
2769{
2770 int i;
2771
2772 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 2773 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
85b430b4
AD		 2774}
2775
9d5c8243
AK		 2776/**
2777 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
9d5c8243
AK		 2778 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2779 *
2780 * Returns 0 on success, negative on failure
2781 **/
80785298 2782int igb_setup_rx_resources(struct igb_ring *rx_ring)
9d5c8243 2783{
59d71989 2784 struct device *dev = rx_ring->dev;
81c2fc22 2785 int orig_node = dev_to_node(dev);
9d5c8243
AK		 2786 int size, desc_len;
2787
06034649 2788 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
81c2fc22
AD		 2789 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
2790 if (!rx_ring->rx_buffer_info)
2791 rx_ring->rx_buffer_info = vzalloc(size);
06034649 2792 if (!rx_ring->rx_buffer_info)
9d5c8243 2793 goto err;
9d5c8243
AK		 2794
2795 desc_len = sizeof(union e1000_adv_rx_desc);
2796
2797 /* Round up to nearest 4K */
2798 rx_ring->size = rx_ring->count * desc_len;
2799 rx_ring->size = ALIGN(rx_ring->size, 4096);
2800
81c2fc22 2801 set_dev_node(dev, rx_ring->numa_node);
59d71989
AD		 2802 rx_ring->desc = dma_alloc_coherent(dev,
2803 rx_ring->size,
2804 &rx_ring->dma,
2805 GFP_KERNEL);
81c2fc22
AD		 2806 set_dev_node(dev, orig_node);
2807 if (!rx_ring->desc)
2808 rx_ring->desc = dma_alloc_coherent(dev,
2809 rx_ring->size,
2810 &rx_ring->dma,
2811 GFP_KERNEL);
9d5c8243
AK		 2812
2813 if (!rx_ring->desc)
2814 goto err;
2815
2816 rx_ring->next_to_clean = 0;
2817 rx_ring->next_to_use = 0;
9d5c8243 2818
9d5c8243
AK		 2819 return 0;
2820
2821err:
06034649
AD		 2822 vfree(rx_ring->rx_buffer_info);
2823 rx_ring->rx_buffer_info = NULL;
59d71989
AD		 2824 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2825 " ring\n");
9d5c8243
AK		 2826 return -ENOMEM;
2827}
2828
2829/**
2830 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2831 * (Descriptors) for all queues
2832 * @adapter: board private structure
2833 *
2834 * Return 0 on success, negative on failure
2835 **/
2836static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2837{
439705e1 2838 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 2839 int i, err = 0;
2840
2841 for (i = 0; i < adapter->num_rx_queues; i++) {
3025a446 2842 err = igb_setup_rx_resources(adapter->rx_ring[i]);
9d5c8243 2843 if (err) {
439705e1 2844 dev_err(&pdev->dev,
9d5c8243
AK		 2845 "Allocation for Rx Queue %u failed\n", i);
2846 for (i--; i >= 0; i--)
3025a446 2847 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 2848 break;
2849 }
2850 }
2851
2852 return err;
2853}
2854
06cf2666
AD		 2855/**
2856 * igb_setup_mrqc - configure the multiple receive queue control registers
2857 * @adapter: Board private structure
2858 **/
2859static void igb_setup_mrqc(struct igb_adapter *adapter)
2860{
2861 struct e1000_hw *hw = &adapter->hw;
2862 u32 mrqc, rxcsum;
2863 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2864 union e1000_reta {
2865 u32 dword;
2866 u8 bytes[4];
2867 } reta;
2868 static const u8 rsshash[40] = {
2869 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2870 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2871 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2872 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2873
2874 /* Fill out hash function seeds */
2875 for (j = 0; j < 10; j++) {
2876 u32 rsskey = rsshash[(j * 4)];
2877 rsskey |= rsshash[(j * 4) + 1] << 8;
2878 rsskey |= rsshash[(j * 4) + 2] << 16;
2879 rsskey |= rsshash[(j * 4) + 3] << 24;
2880 array_wr32(E1000_RSSRK(0), j, rsskey);
2881 }
2882
a99955fc 2883 num_rx_queues = adapter->rss_queues;
06cf2666
AD		 2884
2885 if (adapter->vfs_allocated_count) {
2886 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2887 switch (hw->mac.type) {
d2ba2ed8 2888 case e1000_i350:
55cac248
AD		 2889 case e1000_82580:
2890 num_rx_queues = 1;
2891 shift = 0;
2892 break;
06cf2666
AD		 2893 case e1000_82576:
2894 shift = 3;
2895 num_rx_queues = 2;
2896 break;
2897 case e1000_82575:
2898 shift = 2;
2899 shift2 = 6;
2900 default:
2901 break;
2902 }
2903 } else {
2904 if (hw->mac.type == e1000_82575)
2905 shift = 6;
2906 }
2907
2908 for (j = 0; j < (32 * 4); j++) {
2909 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2910 if (shift2)
2911 reta.bytes[j & 3] |= num_rx_queues << shift2;
2912 if ((j & 3) == 3)
2913 wr32(E1000_RETA(j >> 2), reta.dword);
2914 }
2915
2916 /*
2917 * Disable raw packet checksumming so that RSS hash is placed in
2918 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2919 * offloads as they are enabled by default
2920 */
2921 rxcsum = rd32(E1000_RXCSUM);
2922 rxcsum |= E1000_RXCSUM_PCSD;
2923
2924 if (adapter->hw.mac.type >= e1000_82576)
2925 /* Enable Receive Checksum Offload for SCTP */
2926 rxcsum |= E1000_RXCSUM_CRCOFL;
2927
2928 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2929 wr32(E1000_RXCSUM, rxcsum);
2930
2931 /* If VMDq is enabled then we set the appropriate mode for that, else
2932 * we default to RSS so that an RSS hash is calculated per packet even
2933 * if we are only using one queue */
2934 if (adapter->vfs_allocated_count) {
2935 if (hw->mac.type > e1000_82575) {
2936 /* Set the default pool for the PF's first queue */
2937 u32 vtctl = rd32(E1000_VT_CTL);
2938 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2939 E1000_VT_CTL_DISABLE_DEF_POOL);
2940 vtctl |= adapter->vfs_allocated_count <<
2941 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2942 wr32(E1000_VT_CTL, vtctl);
2943 }
a99955fc 2944 if (adapter->rss_queues > 1)
06cf2666
AD		 2945 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2946 else
2947 mrqc = E1000_MRQC_ENABLE_VMDQ;
2948 } else {
2949 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2950 }
2951 igb_vmm_control(adapter);
2952
4478a9cd
AD		 2953 /*
2954 * Generate RSS hash based on TCP port numbers and/or
2955 * IPv4/v6 src and dst addresses since UDP cannot be
2956 * hashed reliably due to IP fragmentation
2957 */
2958 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2959 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2960 E1000_MRQC_RSS_FIELD_IPV6 |
2961 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2962 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
06cf2666
AD		 2963
2964 wr32(E1000_MRQC, mrqc);
2965}
2966
9d5c8243
AK		 2967/**
2968 * igb_setup_rctl - configure the receive control registers
2969 * @adapter: Board private structure
2970 **/
d7ee5b3a 2971void igb_setup_rctl(struct igb_adapter *adapter)
9d5c8243
AK		 2972{
2973 struct e1000_hw *hw = &adapter->hw;
2974 u32 rctl;
9d5c8243
AK		 2975
2976 rctl = rd32(E1000_RCTL);
2977
2978 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
69d728ba 2979 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
9d5c8243 2980
69d728ba 2981 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
28b0759c 2982 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
9d5c8243 2983
87cb7e8c
AK		 2984 /*
2985 * enable stripping of CRC. It's unlikely this will break BMC
2986 * redirection as it did with e1000. Newer features require
2987 * that the HW strips the CRC.
73cd78f1 2988 */
87cb7e8c 2989 rctl |= E1000_RCTL_SECRC;
9d5c8243 2990
559e9c49 2991 /* disable store bad packets and clear size bits. */
ec54d7d6 2992 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
9d5c8243 2993
6ec43fe6
AD		 2994 /* enable LPE to prevent packets larger than max_frame_size */
2995 rctl |= E1000_RCTL_LPE;
9d5c8243 2996
952f72a8
AD		 2997 /* disable queue 0 to prevent tail write w/o re-config */
2998 wr32(E1000_RXDCTL(0), 0);
9d5c8243 2999
e1739522
AD		 3000 /* Attention!!! For SR-IOV PF driver operations you must enable
3001 * queue drop for all VF and PF queues to prevent head of line blocking
3002 * if an un-trusted VF does not provide descriptors to hardware.
3003 */
3004 if (adapter->vfs_allocated_count) {
e1739522
AD		 3005 /* set all queue drop enable bits */
3006 wr32(E1000_QDE, ALL_QUEUES);
e1739522
AD		 3007 }
3008
9d5c8243
AK		 3009 wr32(E1000_RCTL, rctl);
3010}
3011
7d5753f0
AD		 3012static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3013 int vfn)
3014{
3015 struct e1000_hw *hw = &adapter->hw;
3016 u32 vmolr;
3017
3018 /* if it isn't the PF check to see if VFs are enabled and
3019 * increase the size to support vlan tags */
3020 if (vfn < adapter->vfs_allocated_count &&
3021 adapter->vf_data[vfn].vlans_enabled)
3022 size += VLAN_TAG_SIZE;
3023
3024 vmolr = rd32(E1000_VMOLR(vfn));
3025 vmolr &= ~E1000_VMOLR_RLPML_MASK;
3026 vmolr |= size | E1000_VMOLR_LPE;
3027 wr32(E1000_VMOLR(vfn), vmolr);
3028
3029 return 0;
3030}
3031
e1739522
AD		 3032/**
3033 * igb_rlpml_set - set maximum receive packet size
3034 * @adapter: board private structure
3035 *
3036 * Configure maximum receivable packet size.
3037 **/
3038static void igb_rlpml_set(struct igb_adapter *adapter)
3039{
153285f9 3040 u32 max_frame_size = adapter->max_frame_size;
e1739522
AD		 3041 struct e1000_hw *hw = &adapter->hw;
3042 u16 pf_id = adapter->vfs_allocated_count;
3043
e1739522
AD		 3044 if (pf_id) {
3045 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
153285f9
AD		 3046 /*
3047 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3048 * to our max jumbo frame size, in case we need to enable
3049 * jumbo frames on one of the rings later.
3050 * This will not pass over-length frames into the default
3051 * queue because it's gated by the VMOLR.RLPML.
3052 */
7d5753f0 3053 max_frame_size = MAX_JUMBO_FRAME_SIZE;
e1739522
AD		 3054 }
3055
3056 wr32(E1000_RLPML, max_frame_size);
3057}
3058
8151d294
WM		 3059static inline void igb_set_vmolr(struct igb_adapter *adapter,
3060 int vfn, bool aupe)
7d5753f0
AD		 3061{
3062 struct e1000_hw *hw = &adapter->hw;
3063 u32 vmolr;
3064
3065 /*
3066 * This register exists only on 82576 and newer so if we are older then
3067 * we should exit and do nothing
3068 */
3069 if (hw->mac.type < e1000_82576)
3070 return;
3071
3072 vmolr = rd32(E1000_VMOLR(vfn));
8151d294
WM		 3073 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3074 if (aupe)
3075 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3076 else
3077 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
7d5753f0
AD		 3078
3079 /* clear all bits that might not be set */
3080 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3081
a99955fc 3082 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
7d5753f0
AD		 3083 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3084 /*
3085 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3086 * multicast packets
3087 */
3088 if (vfn <= adapter->vfs_allocated_count)
3089 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3090
3091 wr32(E1000_VMOLR(vfn), vmolr);
3092}
3093
85b430b4
AD		 3094/**
3095 * igb_configure_rx_ring - Configure a receive ring after Reset
3096 * @adapter: board private structure
3097 * @ring: receive ring to be configured
3098 *
3099 * Configure the Rx unit of the MAC after a reset.
3100 **/
d7ee5b3a
AD		 3101void igb_configure_rx_ring(struct igb_adapter *adapter,
3102 struct igb_ring *ring)
85b430b4
AD		 3103{
3104 struct e1000_hw *hw = &adapter->hw;
3105 u64 rdba = ring->dma;
3106 int reg_idx = ring->reg_idx;
a74420e0 3107 u32 srrctl = 0, rxdctl = 0;
85b430b4
AD		 3108
3109 /* disable the queue */
a74420e0 3110 wr32(E1000_RXDCTL(reg_idx), 0);
85b430b4
AD		 3111
3112 /* Set DMA base address registers */
3113 wr32(E1000_RDBAL(reg_idx),
3114 rdba & 0x00000000ffffffffULL);
3115 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3116 wr32(E1000_RDLEN(reg_idx),
3117 ring->count * sizeof(union e1000_adv_rx_desc));
3118
3119 /* initialize head and tail */
fce99e34 3120 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
a74420e0 3121 wr32(E1000_RDH(reg_idx), 0);
fce99e34 3122 writel(0, ring->tail);
85b430b4 3123
952f72a8 3124 /* set descriptor configuration */
44390ca6 3125 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
952f72a8 3126#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
44390ca6 3127 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3128#else
44390ca6 3129 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
952f72a8 3130#endif
44390ca6 3131 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
757b77e2
NN		 3132 if (hw->mac.type == e1000_82580)
3133 srrctl |= E1000_SRRCTL_TIMESTAMP;
e6bdb6fe
NN		 3134 /* Only set Drop Enable if we are supporting multiple queues */
3135 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3136 srrctl |= E1000_SRRCTL_DROP_EN;
952f72a8
AD		 3137
3138 wr32(E1000_SRRCTL(reg_idx), srrctl);
3139
7d5753f0 3140 /* set filtering for VMDQ pools */
8151d294 3141 igb_set_vmolr(adapter, reg_idx & 0x7, true);
7d5753f0 3142
85b430b4
AD		 3143 rxdctl |= IGB_RX_PTHRESH;
3144 rxdctl |= IGB_RX_HTHRESH << 8;
3145 rxdctl |= IGB_RX_WTHRESH << 16;
a74420e0
AD		 3146
3147 /* enable receive descriptor fetching */
3148 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
85b430b4
AD		 3149 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3150}
3151
9d5c8243
AK		 3152/**
3153 * igb_configure_rx - Configure receive Unit after Reset
3154 * @adapter: board private structure
3155 *
3156 * Configure the Rx unit of the MAC after a reset.
3157 **/
3158static void igb_configure_rx(struct igb_adapter *adapter)
3159{
9107584e 3160 int i;
9d5c8243 3161
68d480c4
AD		 3162 /* set UTA to appropriate mode */
3163 igb_set_uta(adapter);
3164
26ad9178
AD		 3165 /* set the correct pool for the PF default MAC address in entry 0 */
3166 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3167 adapter->vfs_allocated_count);
3168
06cf2666
AD		 3169 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3170 * the Base and Length of the Rx Descriptor Ring */
3171 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3172 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
9d5c8243
AK		 3173}
3174
3175/**
3176 * igb_free_tx_resources - Free Tx Resources per Queue
9d5c8243
AK		 3177 * @tx_ring: Tx descriptor ring for a specific queue
3178 *
3179 * Free all transmit software resources
3180 **/
68fd9910 3181void igb_free_tx_resources(struct igb_ring *tx_ring)
9d5c8243 3182{
3b644cf6 3183 igb_clean_tx_ring(tx_ring);
9d5c8243 3184
06034649
AD		 3185 vfree(tx_ring->tx_buffer_info);
3186 tx_ring->tx_buffer_info = NULL;
9d5c8243 3187
439705e1
AD		 3188 /* if not set, then don't free */
3189 if (!tx_ring->desc)
3190 return;
3191
59d71989
AD		 3192 dma_free_coherent(tx_ring->dev, tx_ring->size,
3193 tx_ring->desc, tx_ring->dma);
9d5c8243
AK		 3194
3195 tx_ring->desc = NULL;
3196}
3197
3198/**
3199 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3200 * @adapter: board private structure
3201 *
3202 * Free all transmit software resources
3203 **/
3204static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3205{
3206 int i;
3207
3208 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3209 igb_free_tx_resources(adapter->tx_ring[i]);
9d5c8243
AK		 3210}
3211
ebe42d16
AD		 3212void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3213 struct igb_tx_buffer *tx_buffer)
3214{
3215 if (tx_buffer->skb) {
3216 dev_kfree_skb_any(tx_buffer->skb);
3217 if (tx_buffer->dma)
3218 dma_unmap_single(ring->dev,
3219 tx_buffer->dma,
3220 tx_buffer->length,
3221 DMA_TO_DEVICE);
3222 } else if (tx_buffer->dma) {
3223 dma_unmap_page(ring->dev,
3224 tx_buffer->dma,
3225 tx_buffer->length,
3226 DMA_TO_DEVICE);
3227 }
3228 tx_buffer->next_to_watch = NULL;
3229 tx_buffer->skb = NULL;
3230 tx_buffer->dma = 0;
3231 /* buffer_info must be completely set up in the transmit path */
9d5c8243
AK		 3232}
3233
3234/**
3235 * igb_clean_tx_ring - Free Tx Buffers
9d5c8243
AK		 3236 * @tx_ring: ring to be cleaned
3237 **/
3b644cf6 3238static void igb_clean_tx_ring(struct igb_ring *tx_ring)
9d5c8243 3239{
06034649 3240 struct igb_tx_buffer *buffer_info;
9d5c8243
AK		 3241 unsigned long size;
3242 unsigned int i;
3243
06034649 3244 if (!tx_ring->tx_buffer_info)
9d5c8243
AK		 3245 return;
3246 /* Free all the Tx ring sk_buffs */
3247
3248 for (i = 0; i < tx_ring->count; i++) {
06034649 3249 buffer_info = &tx_ring->tx_buffer_info[i];
80785298 3250 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
9d5c8243
AK		 3251 }
3252
06034649
AD		 3253 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3254 memset(tx_ring->tx_buffer_info, 0, size);
9d5c8243
AK		 3255
3256 /* Zero out the descriptor ring */
9d5c8243
AK		 3257 memset(tx_ring->desc, 0, tx_ring->size);
3258
3259 tx_ring->next_to_use = 0;
3260 tx_ring->next_to_clean = 0;
9d5c8243
AK		 3261}
3262
3263/**
3264 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3265 * @adapter: board private structure
3266 **/
3267static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3268{
3269 int i;
3270
3271 for (i = 0; i < adapter->num_tx_queues; i++)
3025a446 3272 igb_clean_tx_ring(adapter->tx_ring[i]);
9d5c8243
AK		 3273}
3274
3275/**
3276 * igb_free_rx_resources - Free Rx Resources
9d5c8243
AK		 3277 * @rx_ring: ring to clean the resources from
3278 *
3279 * Free all receive software resources
3280 **/
68fd9910 3281void igb_free_rx_resources(struct igb_ring *rx_ring)
9d5c8243 3282{
3b644cf6 3283 igb_clean_rx_ring(rx_ring);
9d5c8243 3284
06034649
AD		 3285 vfree(rx_ring->rx_buffer_info);
3286 rx_ring->rx_buffer_info = NULL;
9d5c8243 3287
439705e1
AD		 3288 /* if not set, then don't free */
3289 if (!rx_ring->desc)
3290 return;
3291
59d71989
AD		 3292 dma_free_coherent(rx_ring->dev, rx_ring->size,
3293 rx_ring->desc, rx_ring->dma);
9d5c8243
AK		 3294
3295 rx_ring->desc = NULL;
3296}
3297
3298/**
3299 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3300 * @adapter: board private structure
3301 *
3302 * Free all receive software resources
3303 **/
3304static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3305{
3306 int i;
3307
3308 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3309 igb_free_rx_resources(adapter->rx_ring[i]);
9d5c8243
AK		 3310}
3311
3312/**
3313 * igb_clean_rx_ring - Free Rx Buffers per Queue
9d5c8243
AK		 3314 * @rx_ring: ring to free buffers from
3315 **/
3b644cf6 3316static void igb_clean_rx_ring(struct igb_ring *rx_ring)
9d5c8243 3317{
9d5c8243 3318 unsigned long size;
c023cd88 3319 u16 i;
9d5c8243 3320
06034649 3321 if (!rx_ring->rx_buffer_info)
9d5c8243 3322 return;
439705e1 3323
9d5c8243
AK		 3324 /* Free all the Rx ring sk_buffs */
3325 for (i = 0; i < rx_ring->count; i++) {
06034649 3326 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
9d5c8243 3327 if (buffer_info->dma) {
59d71989 3328 dma_unmap_single(rx_ring->dev,
80785298 3329 buffer_info->dma,
44390ca6 3330 IGB_RX_HDR_LEN,
59d71989 3331 DMA_FROM_DEVICE);
9d5c8243
AK		 3332 buffer_info->dma = 0;
3333 }
3334
3335 if (buffer_info->skb) {
3336 dev_kfree_skb(buffer_info->skb);
3337 buffer_info->skb = NULL;
3338 }
6ec43fe6 3339 if (buffer_info->page_dma) {
59d71989 3340 dma_unmap_page(rx_ring->dev,
80785298 3341 buffer_info->page_dma,
6ec43fe6 3342 PAGE_SIZE / 2,
59d71989 3343 DMA_FROM_DEVICE);
6ec43fe6
AD		 3344 buffer_info->page_dma = 0;
3345 }
9d5c8243 3346 if (buffer_info->page) {
9d5c8243
AK		 3347 put_page(buffer_info->page);
3348 buffer_info->page = NULL;
bf36c1a0 3349 buffer_info->page_offset = 0;
9d5c8243
AK		 3350 }
3351 }
3352
06034649
AD		 3353 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3354 memset(rx_ring->rx_buffer_info, 0, size);
9d5c8243
AK		 3355
3356 /* Zero out the descriptor ring */
3357 memset(rx_ring->desc, 0, rx_ring->size);
3358
3359 rx_ring->next_to_clean = 0;
3360 rx_ring->next_to_use = 0;
9d5c8243
AK		 3361}
3362
3363/**
3364 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3365 * @adapter: board private structure
3366 **/
3367static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3368{
3369 int i;
3370
3371 for (i = 0; i < adapter->num_rx_queues; i++)
3025a446 3372 igb_clean_rx_ring(adapter->rx_ring[i]);
9d5c8243
AK		 3373}
3374
3375/**
3376 * igb_set_mac - Change the Ethernet Address of the NIC
3377 * @netdev: network interface device structure
3378 * @p: pointer to an address structure
3379 *
3380 * Returns 0 on success, negative on failure
3381 **/
3382static int igb_set_mac(struct net_device *netdev, void *p)
3383{
3384 struct igb_adapter *adapter = netdev_priv(netdev);
28b0759c 3385 struct e1000_hw *hw = &adapter->hw;
9d5c8243
AK		 3386 struct sockaddr *addr = p;
3387
3388 if (!is_valid_ether_addr(addr->sa_data))
3389 return -EADDRNOTAVAIL;
3390
3391 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
28b0759c 3392 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9d5c8243 3393
26ad9178
AD		 3394 /* set the correct pool for the new PF MAC address in entry 0 */
3395 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3396 adapter->vfs_allocated_count);
e1739522 3397
9d5c8243
AK		 3398 return 0;
3399}
3400
3401/**
68d480c4 3402 * igb_write_mc_addr_list - write multicast addresses to MTA
9d5c8243
AK		 3403 * @netdev: network interface device structure
3404 *
68d480c4
AD		 3405 * Writes multicast address list to the MTA hash table.
3406 * Returns: -ENOMEM on failure
3407 * 0 on no addresses written
3408 * X on writing X addresses to MTA
9d5c8243 3409 **/
68d480c4 3410static int igb_write_mc_addr_list(struct net_device *netdev)
9d5c8243
AK		 3411{
3412 struct igb_adapter *adapter = netdev_priv(netdev);
3413 struct e1000_hw *hw = &adapter->hw;
22bedad3 3414 struct netdev_hw_addr *ha;
68d480c4 3415 u8 *mta_list;
9d5c8243
AK		 3416 int i;
3417
4cd24eaf 3418 if (netdev_mc_empty(netdev)) {
68d480c4
AD		 3419 /* nothing to program, so clear mc list */
3420 igb_update_mc_addr_list(hw, NULL, 0);
3421 igb_restore_vf_multicasts(adapter);
3422 return 0;
3423 }
9d5c8243 3424
4cd24eaf 3425 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
68d480c4
AD		 3426 if (!mta_list)
3427 return -ENOMEM;
ff41f8dc 3428
68d480c4 3429 /* The shared function expects a packed array of only addresses. */
48e2f183 3430 i = 0;
22bedad3
JP		 3431 netdev_for_each_mc_addr(ha, netdev)
3432 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
68d480c4 3433
68d480c4
AD		 3434 igb_update_mc_addr_list(hw, mta_list, i);
3435 kfree(mta_list);
3436
4cd24eaf 3437 return netdev_mc_count(netdev);
68d480c4
AD		 3438}
3439
3440/**
3441 * igb_write_uc_addr_list - write unicast addresses to RAR table
3442 * @netdev: network interface device structure
3443 *
3444 * Writes unicast address list to the RAR table.
3445 * Returns: -ENOMEM on failure/insufficient address space
3446 * 0 on no addresses written
3447 * X on writing X addresses to the RAR table
3448 **/
3449static int igb_write_uc_addr_list(struct net_device *netdev)
3450{
3451 struct igb_adapter *adapter = netdev_priv(netdev);
3452 struct e1000_hw *hw = &adapter->hw;
3453 unsigned int vfn = adapter->vfs_allocated_count;
3454 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3455 int count = 0;
3456
3457 /* return ENOMEM indicating insufficient memory for addresses */
32e7bfc4 3458 if (netdev_uc_count(netdev) > rar_entries)
68d480c4 3459 return -ENOMEM;
9d5c8243 3460
32e7bfc4 3461 if (!netdev_uc_empty(netdev) && rar_entries) {
ff41f8dc 3462 struct netdev_hw_addr *ha;
32e7bfc4
JP		 3463
3464 netdev_for_each_uc_addr(ha, netdev) {
ff41f8dc
AD		 3465 if (!rar_entries)
3466 break;
26ad9178
AD		 3467 igb_rar_set_qsel(adapter, ha->addr,
3468 rar_entries--,
68d480c4
AD		 3469 vfn);
3470 count++;
ff41f8dc
AD		 3471 }
3472 }
3473 /* write the addresses in reverse order to avoid write combining */
3474 for (; rar_entries > 0 ; rar_entries--) {
3475 wr32(E1000_RAH(rar_entries), 0);
3476 wr32(E1000_RAL(rar_entries), 0);
3477 }
3478 wrfl();
3479
68d480c4
AD		 3480 return count;
3481}
3482
3483/**
3484 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3485 * @netdev: network interface device structure
3486 *
3487 * The set_rx_mode entry point is called whenever the unicast or multicast
3488 * address lists or the network interface flags are updated. This routine is
3489 * responsible for configuring the hardware for proper unicast, multicast,
3490 * promiscuous mode, and all-multi behavior.
3491 **/
3492static void igb_set_rx_mode(struct net_device *netdev)
3493{
3494 struct igb_adapter *adapter = netdev_priv(netdev);
3495 struct e1000_hw *hw = &adapter->hw;
3496 unsigned int vfn = adapter->vfs_allocated_count;
3497 u32 rctl, vmolr = 0;
3498 int count;
3499
3500 /* Check for Promiscuous and All Multicast modes */
3501 rctl = rd32(E1000_RCTL);
3502
3503 /* clear the effected bits */
3504 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3505
3506 if (netdev->flags & IFF_PROMISC) {
3507 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3508 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3509 } else {
3510 if (netdev->flags & IFF_ALLMULTI) {
3511 rctl |= E1000_RCTL_MPE;
3512 vmolr |= E1000_VMOLR_MPME;
3513 } else {
3514 /*
3515 * Write addresses to the MTA, if the attempt fails
25985edc 3516 * then we should just turn on promiscuous mode so
68d480c4
AD		 3517 * that we can at least receive multicast traffic
3518 */
3519 count = igb_write_mc_addr_list(netdev);
3520 if (count < 0) {
3521 rctl |= E1000_RCTL_MPE;
3522 vmolr |= E1000_VMOLR_MPME;
3523 } else if (count) {
3524 vmolr |= E1000_VMOLR_ROMPE;
3525 }
3526 }
3527 /*
3528 * Write addresses to available RAR registers, if there is not
3529 * sufficient space to store all the addresses then enable
25985edc 3530 * unicast promiscuous mode
68d480c4
AD		 3531 */
3532 count = igb_write_uc_addr_list(netdev);
3533 if (count < 0) {
3534 rctl |= E1000_RCTL_UPE;
3535 vmolr |= E1000_VMOLR_ROPE;
3536 }
3537 rctl |= E1000_RCTL_VFE;
28fc06f5 3538 }
68d480c4 3539 wr32(E1000_RCTL, rctl);
28fc06f5 3540
68d480c4
AD		 3541 /*
3542 * In order to support SR-IOV and eventually VMDq it is necessary to set
3543 * the VMOLR to enable the appropriate modes. Without this workaround
3544 * we will have issues with VLAN tag stripping not being done for frames
3545 * that are only arriving because we are the default pool
3546 */
3547 if (hw->mac.type < e1000_82576)
28fc06f5 3548 return;
9d5c8243 3549
68d480c4
AD		 3550 vmolr |= rd32(E1000_VMOLR(vfn)) &
3551 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3552 wr32(E1000_VMOLR(vfn), vmolr);
28fc06f5 3553 igb_restore_vf_multicasts(adapter);
9d5c8243
AK		 3554}
3555
13800469
GR		 3556static void igb_check_wvbr(struct igb_adapter *adapter)
3557{
3558 struct e1000_hw *hw = &adapter->hw;
3559 u32 wvbr = 0;
3560
3561 switch (hw->mac.type) {
3562 case e1000_82576:
3563 case e1000_i350:
3564 if (!(wvbr = rd32(E1000_WVBR)))
3565 return;
3566 break;
3567 default:
3568 break;
3569 }
3570
3571 adapter->wvbr |= wvbr;
3572}
3573
3574#define IGB_STAGGERED_QUEUE_OFFSET 8
3575
3576static void igb_spoof_check(struct igb_adapter *adapter)
3577{
3578 int j;
3579
3580 if (!adapter->wvbr)
3581 return;
3582
3583 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3584 if (adapter->wvbr & (1 << j) ||
3585 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3586 dev_warn(&adapter->pdev->dev,
3587 "Spoof event(s) detected on VF %d\n", j);
3588 adapter->wvbr &=
3589 ~((1 << j) |
3590 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3591 }
3592 }
3593}
3594
9d5c8243
AK		 3595/* Need to wait a few seconds after link up to get diagnostic information from
3596 * the phy */
3597static void igb_update_phy_info(unsigned long data)
3598{
3599 struct igb_adapter *adapter = (struct igb_adapter *) data;
f5f4cf08 3600 igb_get_phy_info(&adapter->hw);
9d5c8243
AK		 3601}
3602
4d6b725e
AD		 3603/**
3604 * igb_has_link - check shared code for link and determine up/down
3605 * @adapter: pointer to driver private info
3606 **/
3145535a 3607bool igb_has_link(struct igb_adapter *adapter)
4d6b725e
AD		 3608{
3609 struct e1000_hw *hw = &adapter->hw;
3610 bool link_active = false;
3611 s32 ret_val = 0;
3612
3613 /* get_link_status is set on LSC (link status) interrupt or
3614 * rx sequence error interrupt. get_link_status will stay
3615 * false until the e1000_check_for_link establishes link
3616 * for copper adapters ONLY
3617 */
3618 switch (hw->phy.media_type) {
3619 case e1000_media_type_copper:
3620 if (hw->mac.get_link_status) {
3621 ret_val = hw->mac.ops.check_for_link(hw);
3622 link_active = !hw->mac.get_link_status;
3623 } else {
3624 link_active = true;
3625 }
3626 break;
4d6b725e
AD		 3627 case e1000_media_type_internal_serdes:
3628 ret_val = hw->mac.ops.check_for_link(hw);
3629 link_active = hw->mac.serdes_has_link;
3630 break;
3631 default:
3632 case e1000_media_type_unknown:
3633 break;
3634 }
3635
3636 return link_active;
3637}
3638
563988dc
SA		 3639static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3640{
3641 bool ret = false;
3642 u32 ctrl_ext, thstat;
3643
3644 /* check for thermal sensor event on i350, copper only */
3645 if (hw->mac.type == e1000_i350) {
3646 thstat = rd32(E1000_THSTAT);
3647 ctrl_ext = rd32(E1000_CTRL_EXT);
3648
3649 if ((hw->phy.media_type == e1000_media_type_copper) &&
3650 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3651 ret = !!(thstat & event);
3652 }
3653 }
3654
3655 return ret;
3656}
3657
9d5c8243
AK		 3658/**
3659 * igb_watchdog - Timer Call-back
3660 * @data: pointer to adapter cast into an unsigned long
3661 **/
3662static void igb_watchdog(unsigned long data)
3663{
3664 struct igb_adapter *adapter = (struct igb_adapter *)data;
3665 /* Do the rest outside of interrupt context */
3666 schedule_work(&adapter->watchdog_task);
3667}
3668
3669static void igb_watchdog_task(struct work_struct *work)
3670{
3671 struct igb_adapter *adapter = container_of(work,
559e9c49
AD		 3672 struct igb_adapter,
3673 watchdog_task);
9d5c8243 3674 struct e1000_hw *hw = &adapter->hw;
9d5c8243 3675 struct net_device *netdev = adapter->netdev;
563988dc 3676 u32 link;
7a6ea550 3677 int i;
9d5c8243 3678
4d6b725e 3679 link = igb_has_link(adapter);
9d5c8243
AK		 3680 if (link) {
3681 if (!netif_carrier_ok(netdev)) {
3682 u32 ctrl;
330a6d6a
AD		 3683 hw->mac.ops.get_speed_and_duplex(hw,
3684 &adapter->link_speed,
3685 &adapter->link_duplex);
9d5c8243
AK		 3686
3687 ctrl = rd32(E1000_CTRL);
527d47c1
AD		 3688 /* Links status message must follow this format */
3689 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
9d5c8243 3690 "Flow Control: %s\n",
559e9c49
AD		 3691 netdev->name,
3692 adapter->link_speed,
3693 adapter->link_duplex == FULL_DUPLEX ?
9d5c8243 3694 "Full Duplex" : "Half Duplex",
559e9c49
AD		 3695 ((ctrl & E1000_CTRL_TFCE) &&
3696 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3697 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3698 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
9d5c8243 3699
563988dc
SA		 3700 /* check for thermal sensor event */
3701 if (igb_thermal_sensor_event(hw, E1000_THSTAT_LINK_THROTTLE)) {
3702 printk(KERN_INFO "igb: %s The network adapter "
3703 "link speed was downshifted "
3704 "because it overheated.\n",
3705 netdev->name);
7ef5ed1c 3706 }
563988dc 3707
d07f3e37 3708 /* adjust timeout factor according to speed/duplex */
9d5c8243
AK		 3709 adapter->tx_timeout_factor = 1;
3710 switch (adapter->link_speed) {
3711 case SPEED_10:
9d5c8243
AK		 3712 adapter->tx_timeout_factor = 14;
3713 break;
3714 case SPEED_100:
9d5c8243
AK		 3715 /* maybe add some timeout factor ? */
3716 break;
3717 }
3718
3719 netif_carrier_on(netdev);
9d5c8243 3720
4ae196df 3721 igb_ping_all_vfs(adapter);
17dc566c 3722 igb_check_vf_rate_limit(adapter);
4ae196df 3723
4b1a9877 3724 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3725 if (!test_bit(__IGB_DOWN, &adapter->state))
3726 mod_timer(&adapter->phy_info_timer,
3727 round_jiffies(jiffies + 2 * HZ));
3728 }
3729 } else {
3730 if (netif_carrier_ok(netdev)) {
3731 adapter->link_speed = 0;
3732 adapter->link_duplex = 0;
563988dc
SA		 3733
3734 /* check for thermal sensor event */
3735 if (igb_thermal_sensor_event(hw, E1000_THSTAT_PWR_DOWN)) {
3736 printk(KERN_ERR "igb: %s The network adapter "
3737 "was stopped because it "
3738 "overheated.\n",
7ef5ed1c 3739 netdev->name);
7ef5ed1c 3740 }
563988dc 3741
527d47c1
AD		 3742 /* Links status message must follow this format */
3743 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3744 netdev->name);
9d5c8243 3745 netif_carrier_off(netdev);
4b1a9877 3746
4ae196df
AD		 3747 igb_ping_all_vfs(adapter);
3748
4b1a9877 3749 /* link state has changed, schedule phy info update */
9d5c8243
AK		 3750 if (!test_bit(__IGB_DOWN, &adapter->state))
3751 mod_timer(&adapter->phy_info_timer,
3752 round_jiffies(jiffies + 2 * HZ));
3753 }
3754 }
3755
12dcd86b
ED		 3756 spin_lock(&adapter->stats64_lock);
3757 igb_update_stats(adapter, &adapter->stats64);
3758 spin_unlock(&adapter->stats64_lock);
9d5c8243 3759
dbabb065 3760 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 3761 struct igb_ring *tx_ring = adapter->tx_ring[i];
dbabb065 3762 if (!netif_carrier_ok(netdev)) {
9d5c8243
AK		 3763 /* We've lost link, so the controller stops DMA,
3764 * but we've got queued Tx work that's never going
3765 * to get done, so reset controller to flush Tx.
3766 * (Do the reset outside of interrupt context). */
dbabb065
AD		 3767 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3768 adapter->tx_timeout_count++;
3769 schedule_work(&adapter->reset_task);
3770 /* return immediately since reset is imminent */
3771 return;
3772 }
9d5c8243 3773 }
9d5c8243 3774
dbabb065
AD		 3775 /* Force detection of hung controller every watchdog period */
3776 tx_ring->detect_tx_hung = true;
3777 }
f7ba205e 3778
9d5c8243 3779 /* Cause software interrupt to ensure rx ring is cleaned */
7a6ea550 3780 if (adapter->msix_entries) {
047e0030
AD		 3781 u32 eics = 0;
3782 for (i = 0; i < adapter->num_q_vectors; i++) {
3783 struct igb_q_vector *q_vector = adapter->q_vector[i];
3784 eics |= q_vector->eims_value;
3785 }
7a6ea550
AD		 3786 wr32(E1000_EICS, eics);
3787 } else {
3788 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3789 }
9d5c8243 3790
13800469
GR		 3791 igb_spoof_check(adapter);
3792
9d5c8243
AK		 3793 /* Reset the timer */
3794 if (!test_bit(__IGB_DOWN, &adapter->state))
3795 mod_timer(&adapter->watchdog_timer,
3796 round_jiffies(jiffies + 2 * HZ));
3797}
3798
3799enum latency_range {
3800 lowest_latency = 0,
3801 low_latency = 1,
3802 bulk_latency = 2,
3803 latency_invalid = 255
3804};
3805
6eb5a7f1
AD		 3806/**
3807 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3808 *
3809 * Stores a new ITR value based on strictly on packet size. This
3810 * algorithm is less sophisticated than that used in igb_update_itr,
3811 * due to the difficulty of synchronizing statistics across multiple
eef35c2d 3812 * receive rings. The divisors and thresholds used by this function
6eb5a7f1
AD		 3813 * were determined based on theoretical maximum wire speed and testing
3814 * data, in order to minimize response time while increasing bulk
3815 * throughput.
3816 * This functionality is controlled by the InterruptThrottleRate module
3817 * parameter (see igb_param.c)
3818 * NOTE: This function is called only when operating in a multiqueue
3819 * receive environment.
047e0030 3820 * @q_vector: pointer to q_vector
6eb5a7f1 3821 **/
047e0030 3822static void igb_update_ring_itr(struct igb_q_vector *q_vector)
9d5c8243 3823{
047e0030 3824 int new_val = q_vector->itr_val;
6eb5a7f1 3825 int avg_wire_size = 0;
047e0030 3826 struct igb_adapter *adapter = q_vector->adapter;
12dcd86b
ED		 3827 struct igb_ring *ring;
3828 unsigned int packets;
9d5c8243 3829
6eb5a7f1
AD		 3830 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3831 * ints/sec - ITR timer value of 120 ticks.
3832 */
3833 if (adapter->link_speed != SPEED_1000) {
047e0030 3834 new_val = 976;
6eb5a7f1 3835 goto set_itr_val;
9d5c8243 3836 }
047e0030 3837
12dcd86b
ED		 3838 ring = q_vector->rx_ring;
3839 if (ring) {
3840 packets = ACCESS_ONCE(ring->total_packets);
3841
3842 if (packets)
3843 avg_wire_size = ring->total_bytes / packets;
047e0030
AD		 3844 }
3845
12dcd86b
ED		 3846 ring = q_vector->tx_ring;
3847 if (ring) {
3848 packets = ACCESS_ONCE(ring->total_packets);
3849
3850 if (packets)
3851 avg_wire_size = max_t(u32, avg_wire_size,
3852 ring->total_bytes / packets);
047e0030
AD		 3853 }
3854
3855 /* if avg_wire_size isn't set no work was done */
3856 if (!avg_wire_size)
3857 goto clear_counts;
9d5c8243 3858
6eb5a7f1
AD		 3859 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3860 avg_wire_size += 24;
3861
3862 /* Don't starve jumbo frames */
3863 avg_wire_size = min(avg_wire_size, 3000);
9d5c8243 3864
6eb5a7f1
AD		 3865 /* Give a little boost to mid-size frames */
3866 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3867 new_val = avg_wire_size / 3;
3868 else
3869 new_val = avg_wire_size / 2;
9d5c8243 3870
abe1c363
NN		 3871 /* when in itr mode 3 do not exceed 20K ints/sec */
3872 if (adapter->rx_itr_setting == 3 && new_val < 196)
3873 new_val = 196;
3874
6eb5a7f1 3875set_itr_val:
047e0030
AD		 3876 if (new_val != q_vector->itr_val) {
3877 q_vector->itr_val = new_val;
3878 q_vector->set_itr = 1;
9d5c8243 3879 }
6eb5a7f1 3880clear_counts:
047e0030
AD		 3881 if (q_vector->rx_ring) {
3882 q_vector->rx_ring->total_bytes = 0;
3883 q_vector->rx_ring->total_packets = 0;
3884 }
3885 if (q_vector->tx_ring) {
3886 q_vector->tx_ring->total_bytes = 0;
3887 q_vector->tx_ring->total_packets = 0;
3888 }
9d5c8243
AK		 3889}
3890
3891/**
3892 * igb_update_itr - update the dynamic ITR value based on statistics
3893 * Stores a new ITR value based on packets and byte
3894 * counts during the last interrupt. The advantage of per interrupt
3895 * computation is faster updates and more accurate ITR for the current
3896 * traffic pattern. Constants in this function were computed
3897 * based on theoretical maximum wire speed and thresholds were set based
3898 * on testing data as well as attempting to minimize response time
3899 * while increasing bulk throughput.
3900 * this functionality is controlled by the InterruptThrottleRate module
3901 * parameter (see igb_param.c)
3902 * NOTE: These calculations are only valid when operating in a single-
3903 * queue environment.
3904 * @adapter: pointer to adapter
047e0030 3905 * @itr_setting: current q_vector->itr_val
9d5c8243
AK		 3906 * @packets: the number of packets during this measurement interval
3907 * @bytes: the number of bytes during this measurement interval
3908 **/
3909static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3910 int packets, int bytes)
3911{
3912 unsigned int retval = itr_setting;
3913
3914 if (packets == 0)
3915 goto update_itr_done;
3916
3917 switch (itr_setting) {
3918 case lowest_latency:
3919 /* handle TSO and jumbo frames */
3920 if (bytes/packets > 8000)
3921 retval = bulk_latency;
3922 else if ((packets < 5) && (bytes > 512))
3923 retval = low_latency;
3924 break;
3925 case low_latency: /* 50 usec aka 20000 ints/s */
3926 if (bytes > 10000) {
3927 /* this if handles the TSO accounting */
3928 if (bytes/packets > 8000) {
3929 retval = bulk_latency;
3930 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3931 retval = bulk_latency;
3932 } else if ((packets > 35)) {
3933 retval = lowest_latency;
3934 }
3935 } else if (bytes/packets > 2000) {
3936 retval = bulk_latency;
3937 } else if (packets <= 2 && bytes < 512) {
3938 retval = lowest_latency;
3939 }
3940 break;
3941 case bulk_latency: /* 250 usec aka 4000 ints/s */
3942 if (bytes > 25000) {
3943 if (packets > 35)
3944 retval = low_latency;
1e5c3d21 3945 } else if (bytes < 1500) {
9d5c8243
AK		 3946 retval = low_latency;
3947 }
3948 break;
3949 }
3950
3951update_itr_done:
3952 return retval;
3953}
3954
6eb5a7f1 3955static void igb_set_itr(struct igb_adapter *adapter)
9d5c8243 3956{
047e0030 3957 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243 3958 u16 current_itr;
047e0030 3959 u32 new_itr = q_vector->itr_val;
9d5c8243
AK		 3960
3961 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3962 if (adapter->link_speed != SPEED_1000) {
3963 current_itr = 0;
3964 new_itr = 4000;
3965 goto set_itr_now;
3966 }
3967
3968 adapter->rx_itr = igb_update_itr(adapter,
3969 adapter->rx_itr,
3025a446
AD		 3970 q_vector->rx_ring->total_packets,
3971 q_vector->rx_ring->total_bytes);
9d5c8243 3972
047e0030
AD		 3973 adapter->tx_itr = igb_update_itr(adapter,
3974 adapter->tx_itr,
3025a446
AD		 3975 q_vector->tx_ring->total_packets,
3976 q_vector->tx_ring->total_bytes);
047e0030 3977 current_itr = max(adapter->rx_itr, adapter->tx_itr);
9d5c8243 3978
6eb5a7f1 3979 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4fc82adf 3980 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
6eb5a7f1
AD		 3981 current_itr = low_latency;
3982
9d5c8243
AK		 3983 switch (current_itr) {
3984 /* counts and packets in update_itr are dependent on these numbers */
3985 case lowest_latency:
78b1f607 3986 new_itr = 56; /* aka 70,000 ints/sec */
9d5c8243
AK		 3987 break;
3988 case low_latency:
78b1f607 3989 new_itr = 196; /* aka 20,000 ints/sec */
9d5c8243
AK		 3990 break;
3991 case bulk_latency:
78b1f607 3992 new_itr = 980; /* aka 4,000 ints/sec */
9d5c8243
AK		 3993 break;
3994 default:
3995 break;
3996 }
3997
3998set_itr_now:
3025a446
AD		 3999 q_vector->rx_ring->total_bytes = 0;
4000 q_vector->rx_ring->total_packets = 0;
4001 q_vector->tx_ring->total_bytes = 0;
4002 q_vector->tx_ring->total_packets = 0;
6eb5a7f1 4003
047e0030 4004 if (new_itr != q_vector->itr_val) {
9d5c8243
AK		 4005 /* this attempts to bias the interrupt rate towards Bulk
4006 * by adding intermediate steps when interrupt rate is
4007 * increasing */
047e0030
AD		 4008 new_itr = new_itr > q_vector->itr_val ?
4009 max((new_itr * q_vector->itr_val) /
4010 (new_itr + (q_vector->itr_val >> 2)),
4011 new_itr) :
9d5c8243
AK		 4012 new_itr;
4013 /* Don't write the value here; it resets the adapter's
4014 * internal timer, and causes us to delay far longer than
4015 * we should between interrupts. Instead, we write the ITR
4016 * value at the beginning of the next interrupt so the timing
4017 * ends up being correct.
4018 */
047e0030
AD		 4019 q_vector->itr_val = new_itr;
4020 q_vector->set_itr = 1;
9d5c8243 4021 }
9d5c8243
AK		 4022}
4023
7d13a7d0
AD		 4024void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4025 u32 type_tucmd, u32 mss_l4len_idx)
4026{
4027 struct e1000_adv_tx_context_desc *context_desc;
4028 u16 i = tx_ring->next_to_use;
4029
4030 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4031
4032 i++;
4033 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4034
4035 /* set bits to identify this as an advanced context descriptor */
4036 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4037
4038 /* For 82575, context index must be unique per ring. */
4039 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
4040 mss_l4len_idx |= tx_ring->reg_idx << 4;
4041
4042 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
4043 context_desc->seqnum_seed = 0;
4044 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
4045 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4046}
4047
7af40ad9
AD		 4048static int igb_tso(struct igb_ring *tx_ring,
4049 struct igb_tx_buffer *first,
4050 u8 *hdr_len)
9d5c8243 4051{
7af40ad9 4052 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4053 u32 vlan_macip_lens, type_tucmd;
4054 u32 mss_l4len_idx, l4len;
4055
4056 if (!skb_is_gso(skb))
4057 return 0;
9d5c8243
AK		 4058
4059 if (skb_header_cloned(skb)) {
7af40ad9 4060 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
9d5c8243
AK		 4061 if (err)
4062 return err;
4063 }
4064
7d13a7d0
AD		 4065 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4066 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
9d5c8243 4067
7af40ad9 4068 if (first->protocol == __constant_htons(ETH_P_IP)) {
9d5c8243
AK		 4069 struct iphdr *iph = ip_hdr(skb);
4070 iph->tot_len = 0;
4071 iph->check = 0;
4072 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4073 iph->daddr, 0,
4074 IPPROTO_TCP,
4075 0);
7d13a7d0 4076 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
7af40ad9
AD		 4077 first->tx_flags |= IGB_TX_FLAGS_TSO |
4078 IGB_TX_FLAGS_CSUM |
4079 IGB_TX_FLAGS_IPV4;
8e1e8a47 4080 } else if (skb_is_gso_v6(skb)) {
9d5c8243
AK		 4081 ipv6_hdr(skb)->payload_len = 0;
4082 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4083 &ipv6_hdr(skb)->daddr,
4084 0, IPPROTO_TCP, 0);
7af40ad9
AD		 4085 first->tx_flags |= IGB_TX_FLAGS_TSO |
4086 IGB_TX_FLAGS_CSUM;
9d5c8243
AK		 4087 }
4088
7af40ad9 4089 /* compute header lengths */
7d13a7d0
AD		 4090 l4len = tcp_hdrlen(skb);
4091 *hdr_len = skb_transport_offset(skb) + l4len;
9d5c8243 4092
7af40ad9
AD		 4093 /* update gso size and bytecount with header size */
4094 first->gso_segs = skb_shinfo(skb)->gso_segs;
4095 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4096
9d5c8243 4097 /* MSS L4LEN IDX */
7d13a7d0
AD		 4098 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4099 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
9d5c8243 4100
7d13a7d0
AD		 4101 /* VLAN MACLEN IPLEN */
4102 vlan_macip_lens = skb_network_header_len(skb);
4103 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4104 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4105
7d13a7d0 4106 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243 4107
7d13a7d0 4108 return 1;
9d5c8243
AK		 4109}
4110
7af40ad9 4111static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
9d5c8243 4112{
7af40ad9 4113 struct sk_buff *skb = first->skb;
7d13a7d0
AD		 4114 u32 vlan_macip_lens = 0;
4115 u32 mss_l4len_idx = 0;
4116 u32 type_tucmd = 0;
9d5c8243 4117
7d13a7d0 4118 if (skb->ip_summed != CHECKSUM_PARTIAL) {
7af40ad9
AD		 4119 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4120 return;
7d13a7d0
AD		 4121 } else {
4122 u8 l4_hdr = 0;
7af40ad9 4123 switch (first->protocol) {
7d13a7d0
AD		 4124 case __constant_htons(ETH_P_IP):
4125 vlan_macip_lens |= skb_network_header_len(skb);
4126 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4127 l4_hdr = ip_hdr(skb)->protocol;
4128 break;
4129 case __constant_htons(ETH_P_IPV6):
4130 vlan_macip_lens |= skb_network_header_len(skb);
4131 l4_hdr = ipv6_hdr(skb)->nexthdr;
4132 break;
4133 default:
4134 if (unlikely(net_ratelimit())) {
4135 dev_warn(tx_ring->dev,
4136 "partial checksum but proto=%x!\n",
7af40ad9 4137 first->protocol);
fa4a7ef3 4138 }
7d13a7d0
AD		 4139 break;
4140 }
fa4a7ef3 4141
7d13a7d0
AD		 4142 switch (l4_hdr) {
4143 case IPPROTO_TCP:
4144 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4145 mss_l4len_idx = tcp_hdrlen(skb) <<
4146 E1000_ADVTXD_L4LEN_SHIFT;
4147 break;
4148 case IPPROTO_SCTP:
4149 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4150 mss_l4len_idx = sizeof(struct sctphdr) <<
4151 E1000_ADVTXD_L4LEN_SHIFT;
4152 break;
4153 case IPPROTO_UDP:
4154 mss_l4len_idx = sizeof(struct udphdr) <<
4155 E1000_ADVTXD_L4LEN_SHIFT;
4156 break;
4157 default:
4158 if (unlikely(net_ratelimit())) {
4159 dev_warn(tx_ring->dev,
4160 "partial checksum but l4 proto=%x!\n",
4161 l4_hdr);
44b0cda3 4162 }
7d13a7d0 4163 break;
9d5c8243 4164 }
7af40ad9
AD		 4165
4166 /* update TX checksum flag */
4167 first->tx_flags |= IGB_TX_FLAGS_CSUM;
7d13a7d0 4168 }
9d5c8243 4169
7d13a7d0 4170 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
7af40ad9 4171 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
9d5c8243 4172
7d13a7d0 4173 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
9d5c8243
AK		 4174}
4175
e032afc8
AD		 4176static __le32 igb_tx_cmd_type(u32 tx_flags)
4177{
4178 /* set type for advanced descriptor with frame checksum insertion */
4179 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4180 E1000_ADVTXD_DCMD_IFCS |
4181 E1000_ADVTXD_DCMD_DEXT);
4182
4183 /* set HW vlan bit if vlan is present */
4184 if (tx_flags & IGB_TX_FLAGS_VLAN)
4185 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4186
4187 /* set timestamp bit if present */
4188 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4189 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4190
4191 /* set segmentation bits for TSO */
4192 if (tx_flags & IGB_TX_FLAGS_TSO)
4193 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4194
4195 return cmd_type;
4196}
4197
7af40ad9
AD		 4198static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4199 union e1000_adv_tx_desc *tx_desc,
4200 u32 tx_flags, unsigned int paylen)
e032afc8
AD		 4201{
4202 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4203
4204 /* 82575 requires a unique index per ring if any offload is enabled */
4205 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
4206 (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX))
4207 olinfo_status |= tx_ring->reg_idx << 4;
4208
4209 /* insert L4 checksum */
4210 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4211 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4212
4213 /* insert IPv4 checksum */
4214 if (tx_flags & IGB_TX_FLAGS_IPV4)
4215 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4216 }
4217
7af40ad9 4218 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
e032afc8
AD		 4219}
4220
ebe42d16
AD		 4221/*
4222 * The largest size we can write to the descriptor is 65535. In order to
4223 * maintain a power of two alignment we have to limit ourselves to 32K.
4224 */
4225#define IGB_MAX_TXD_PWR 15
7af40ad9 4226#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
9d5c8243 4227
7af40ad9
AD		 4228static void igb_tx_map(struct igb_ring *tx_ring,
4229 struct igb_tx_buffer *first,
ebe42d16 4230 const u8 hdr_len)
9d5c8243 4231{
7af40ad9 4232 struct sk_buff *skb = first->skb;
ebe42d16
AD		 4233 struct igb_tx_buffer *tx_buffer_info;
4234 union e1000_adv_tx_desc *tx_desc;
4235 dma_addr_t dma;
4236 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4237 unsigned int data_len = skb->data_len;
4238 unsigned int size = skb_headlen(skb);
4239 unsigned int paylen = skb->len - hdr_len;
4240 __le32 cmd_type;
7af40ad9 4241 u32 tx_flags = first->tx_flags;
ebe42d16 4242 u16 i = tx_ring->next_to_use;
ebe42d16
AD		 4243
4244 tx_desc = IGB_TX_DESC(tx_ring, i);
4245
7af40ad9 4246 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
ebe42d16
AD		 4247 cmd_type = igb_tx_cmd_type(tx_flags);
4248
4249 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4250 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33 4251 goto dma_error;
9d5c8243 4252
ebe42d16
AD		 4253 /* record length, and DMA address */
4254 first->length = size;
4255 first->dma = dma;
ebe42d16
AD		 4256 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4257
4258 for (;;) {
4259 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4260 tx_desc->read.cmd_type_len =
4261 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4262
4263 i++;
4264 tx_desc++;
4265 if (i == tx_ring->count) {
4266 tx_desc = IGB_TX_DESC(tx_ring, 0);
4267 i = 0;
4268 }
4269
4270 dma += IGB_MAX_DATA_PER_TXD;
4271 size -= IGB_MAX_DATA_PER_TXD;
4272
4273 tx_desc->read.olinfo_status = 0;
4274 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4275 }
4276
4277 if (likely(!data_len))
4278 break;
2bbfebe2 4279
ebe42d16 4280 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
9d5c8243 4281
65689fef 4282 i++;
ebe42d16
AD		 4283 tx_desc++;
4284 if (i == tx_ring->count) {
4285 tx_desc = IGB_TX_DESC(tx_ring, 0);
65689fef 4286 i = 0;
ebe42d16 4287 }
65689fef 4288
ebe42d16
AD		 4289 size = frag->size;
4290 data_len -= size;
4291
4292 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4293 size, DMA_TO_DEVICE);
4294 if (dma_mapping_error(tx_ring->dev, dma))
6366ad33
AD		 4295 goto dma_error;
4296
ebe42d16
AD		 4297 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4298 tx_buffer_info->length = size;
4299 tx_buffer_info->dma = dma;
4300
4301 tx_desc->read.olinfo_status = 0;
4302 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4303
4304 frag++;
9d5c8243
AK		 4305 }
4306
ebe42d16
AD		 4307 /* write last descriptor with RS and EOP bits */
4308 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4309 tx_desc->read.cmd_type_len = cmd_type;
8542db05
AD		 4310
4311 /* set the timestamp */
4312 first->time_stamp = jiffies;
4313
ebe42d16
AD		 4314 /*
4315 * Force memory writes to complete before letting h/w know there
4316 * are new descriptors to fetch. (Only applicable for weak-ordered
4317 * memory model archs, such as IA-64).
4318 *
4319 * We also need this memory barrier to make certain all of the
4320 * status bits have been updated before next_to_watch is written.
4321 */
4322 wmb();
4323
8542db05 4324 /* set next_to_watch value indicating a packet is present */
ebe42d16 4325 first->next_to_watch = tx_desc;
9d5c8243 4326
ebe42d16
AD		 4327 i++;
4328 if (i == tx_ring->count)
4329 i = 0;
6366ad33 4330
ebe42d16 4331 tx_ring->next_to_use = i;
6366ad33 4332
ebe42d16 4333 writel(i, tx_ring->tail);
6366ad33 4334
ebe42d16
AD		 4335 /* we need this if more than one processor can write to our tail
4336 * at a time, it syncronizes IO on IA64/Altix systems */
4337 mmiowb();
4338
4339 return;
4340
4341dma_error:
4342 dev_err(tx_ring->dev, "TX DMA map failed\n");
4343
4344 /* clear dma mappings for failed tx_buffer_info map */
4345 for (;;) {
4346 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4347 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4348 if (tx_buffer_info == first)
4349 break;
a77ff709
NN		 4350 if (i == 0)
4351 i = tx_ring->count;
6366ad33 4352 i--;
6366ad33
AD		 4353 }
4354
9d5c8243 4355 tx_ring->next_to_use = i;
9d5c8243
AK		 4356}
4357
e694e964 4358static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4359{
e694e964
AD		 4360 struct net_device *netdev = tx_ring->netdev;
4361
661086df 4362 netif_stop_subqueue(netdev, tx_ring->queue_index);
661086df 4363
9d5c8243
AK		 4364 /* Herbert's original patch had:
4365 * smp_mb__after_netif_stop_queue();
4366 * but since that doesn't exist yet, just open code it. */
4367 smp_mb();
4368
4369 /* We need to check again in a case another CPU has just
4370 * made room available. */
c493ea45 4371 if (igb_desc_unused(tx_ring) < size)
9d5c8243
AK		 4372 return -EBUSY;
4373
4374 /* A reprieve! */
661086df 4375 netif_wake_subqueue(netdev, tx_ring->queue_index);
12dcd86b
ED		 4376
4377 u64_stats_update_begin(&tx_ring->tx_syncp2);
4378 tx_ring->tx_stats.restart_queue2++;
4379 u64_stats_update_end(&tx_ring->tx_syncp2);
4380
9d5c8243
AK		 4381 return 0;
4382}
4383
717ba089 4384static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
9d5c8243 4385{
c493ea45 4386 if (igb_desc_unused(tx_ring) >= size)
9d5c8243 4387 return 0;
e694e964 4388 return __igb_maybe_stop_tx(tx_ring, size);
9d5c8243
AK		 4389}
4390
cd392f5c
AD		 4391netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4392 struct igb_ring *tx_ring)
9d5c8243 4393{
8542db05 4394 struct igb_tx_buffer *first;
ebe42d16 4395 int tso;
91d4ee33 4396 u32 tx_flags = 0;
31f6adbb 4397 __be16 protocol = vlan_get_protocol(skb);
91d4ee33 4398 u8 hdr_len = 0;
9d5c8243 4399
9d5c8243
AK		 4400 /* need: 1 descriptor per page,
4401 * + 2 desc gap to keep tail from touching head,
4402 * + 1 desc for skb->data,
4403 * + 1 desc for context descriptor,
4404 * otherwise try next time */
e694e964 4405 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
9d5c8243 4406 /* this is a hard error */
9d5c8243
AK		 4407 return NETDEV_TX_BUSY;
4408 }
33af6bcc 4409
7af40ad9
AD		 4410 /* record the location of the first descriptor for this packet */
4411 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4412 first->skb = skb;
4413 first->bytecount = skb->len;
4414 first->gso_segs = 1;
4415
2244d07b
OH		 4416 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4417 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
33af6bcc 4418 tx_flags |= IGB_TX_FLAGS_TSTAMP;
33af6bcc 4419 }
9d5c8243 4420
eab6d18d 4421 if (vlan_tx_tag_present(skb)) {
9d5c8243
AK		 4422 tx_flags |= IGB_TX_FLAGS_VLAN;
4423 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4424 }
4425
7af40ad9
AD		 4426 /* record initial flags and protocol */
4427 first->tx_flags = tx_flags;
4428 first->protocol = protocol;
cdfd01fc 4429
7af40ad9
AD		 4430 tso = igb_tso(tx_ring, first, &hdr_len);
4431 if (tso < 0)
7d13a7d0 4432 goto out_drop;
7af40ad9
AD		 4433 else if (!tso)
4434 igb_tx_csum(tx_ring, first);
9d5c8243 4435
7af40ad9 4436 igb_tx_map(tx_ring, first, hdr_len);
85ad76b2
AD		 4437
4438 /* Make sure there is space in the ring for the next send. */
e694e964 4439 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
85ad76b2 4440
9d5c8243 4441 return NETDEV_TX_OK;
7d13a7d0
AD		 4442
4443out_drop:
7af40ad9
AD		 4444 igb_unmap_and_free_tx_resource(tx_ring, first);
4445
7d13a7d0 4446 return NETDEV_TX_OK;
9d5c8243
AK		 4447}
4448
1cc3bd87
AD		 4449static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4450 struct sk_buff *skb)
4451{
4452 unsigned int r_idx = skb->queue_mapping;
4453
4454 if (r_idx >= adapter->num_tx_queues)
4455 r_idx = r_idx % adapter->num_tx_queues;
4456
4457 return adapter->tx_ring[r_idx];
4458}
4459
cd392f5c
AD		 4460static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4461 struct net_device *netdev)
9d5c8243
AK		 4462{
4463 struct igb_adapter *adapter = netdev_priv(netdev);
b1a436c3
AD		 4464
4465 if (test_bit(__IGB_DOWN, &adapter->state)) {
4466 dev_kfree_skb_any(skb);
4467 return NETDEV_TX_OK;
4468 }
4469
4470 if (skb->len <= 0) {
4471 dev_kfree_skb_any(skb);
4472 return NETDEV_TX_OK;
4473 }
4474
1cc3bd87
AD		 4475 /*
4476 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4477 * in order to meet this minimum size requirement.
4478 */
4479 if (skb->len < 17) {
4480 if (skb_padto(skb, 17))
4481 return NETDEV_TX_OK;
4482 skb->len = 17;
4483 }
9d5c8243 4484
1cc3bd87 4485 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
9d5c8243
AK		 4486}
4487
4488/**
4489 * igb_tx_timeout - Respond to a Tx Hang
4490 * @netdev: network interface device structure
4491 **/
4492static void igb_tx_timeout(struct net_device *netdev)
4493{
4494 struct igb_adapter *adapter = netdev_priv(netdev);
4495 struct e1000_hw *hw = &adapter->hw;
4496
4497 /* Do the reset outside of interrupt context */
4498 adapter->tx_timeout_count++;
f7ba205e 4499
55cac248
AD		 4500 if (hw->mac.type == e1000_82580)
4501 hw->dev_spec._82575.global_device_reset = true;
4502
9d5c8243 4503 schedule_work(&adapter->reset_task);
265de409
AD		 4504 wr32(E1000_EICS,
4505 (adapter->eims_enable_mask & ~adapter->eims_other));
9d5c8243
AK		 4506}
4507
4508static void igb_reset_task(struct work_struct *work)
4509{
4510 struct igb_adapter *adapter;
4511 adapter = container_of(work, struct igb_adapter, reset_task);
4512
c97ec42a
TI		 4513 igb_dump(adapter);
4514 netdev_err(adapter->netdev, "Reset adapter\n");
9d5c8243
AK		 4515 igb_reinit_locked(adapter);
4516}
4517
4518/**
12dcd86b 4519 * igb_get_stats64 - Get System Network Statistics
9d5c8243 4520 * @netdev: network interface device structure
12dcd86b 4521 * @stats: rtnl_link_stats64 pointer
9d5c8243 4522 *
9d5c8243 4523 **/
12dcd86b
ED		 4524static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4525 struct rtnl_link_stats64 *stats)
9d5c8243 4526{
12dcd86b
ED		 4527 struct igb_adapter *adapter = netdev_priv(netdev);
4528
4529 spin_lock(&adapter->stats64_lock);
4530 igb_update_stats(adapter, &adapter->stats64);
4531 memcpy(stats, &adapter->stats64, sizeof(*stats));
4532 spin_unlock(&adapter->stats64_lock);
4533
4534 return stats;
9d5c8243
AK		 4535}
4536
4537/**
4538 * igb_change_mtu - Change the Maximum Transfer Unit
4539 * @netdev: network interface device structure
4540 * @new_mtu: new value for maximum frame size
4541 *
4542 * Returns 0 on success, negative on failure
4543 **/
4544static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4545{
4546 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4547 struct pci_dev *pdev = adapter->pdev;
153285f9 4548 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9d5c8243 4549
c809d227 4550 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
090b1795 4551 dev_err(&pdev->dev, "Invalid MTU setting\n");
9d5c8243
AK		 4552 return -EINVAL;
4553 }
4554
153285f9 4555#define MAX_STD_JUMBO_FRAME_SIZE 9238
9d5c8243 4556 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
090b1795 4557 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
9d5c8243
AK		 4558 return -EINVAL;
4559 }
4560
4561 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4562 msleep(1);
73cd78f1 4563
9d5c8243
AK		 4564 /* igb_down has a dependency on max_frame_size */
4565 adapter->max_frame_size = max_frame;
559e9c49 4566
4c844851
AD		 4567 if (netif_running(netdev))
4568 igb_down(adapter);
9d5c8243 4569
090b1795 4570 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
9d5c8243
AK		 4571 netdev->mtu, new_mtu);
4572 netdev->mtu = new_mtu;
4573
4574 if (netif_running(netdev))
4575 igb_up(adapter);
4576 else
4577 igb_reset(adapter);
4578
4579 clear_bit(__IGB_RESETTING, &adapter->state);
4580
4581 return 0;
4582}
4583
4584/**
4585 * igb_update_stats - Update the board statistics counters
4586 * @adapter: board private structure
4587 **/
4588
12dcd86b
ED		 4589void igb_update_stats(struct igb_adapter *adapter,
4590 struct rtnl_link_stats64 *net_stats)
9d5c8243
AK		 4591{
4592 struct e1000_hw *hw = &adapter->hw;
4593 struct pci_dev *pdev = adapter->pdev;
fa3d9a6d 4594 u32 reg, mpc;
9d5c8243 4595 u16 phy_tmp;
3f9c0164
AD		 4596 int i;
4597 u64 bytes, packets;
12dcd86b
ED		 4598 unsigned int start;
4599 u64 _bytes, _packets;
9d5c8243
AK		 4600
4601#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4602
4603 /*
4604 * Prevent stats update while adapter is being reset, or if the pci
4605 * connection is down.
4606 */
4607 if (adapter->link_speed == 0)
4608 return;
4609 if (pci_channel_offline(pdev))
4610 return;
4611
3f9c0164
AD		 4612 bytes = 0;
4613 packets = 0;
4614 for (i = 0; i < adapter->num_rx_queues; i++) {
4615 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3025a446 4616 struct igb_ring *ring = adapter->rx_ring[i];
12dcd86b 4617
3025a446 4618 ring->rx_stats.drops += rqdpc_tmp;
128e45eb 4619 net_stats->rx_fifo_errors += rqdpc_tmp;
12dcd86b
ED		 4620
4621 do {
4622 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4623 _bytes = ring->rx_stats.bytes;
4624 _packets = ring->rx_stats.packets;
4625 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4626 bytes += _bytes;
4627 packets += _packets;
3f9c0164
AD		 4628 }
4629
128e45eb
AD		 4630 net_stats->rx_bytes = bytes;
4631 net_stats->rx_packets = packets;
3f9c0164
AD		 4632
4633 bytes = 0;
4634 packets = 0;
4635 for (i = 0; i < adapter->num_tx_queues; i++) {
3025a446 4636 struct igb_ring *ring = adapter->tx_ring[i];
12dcd86b
ED		 4637 do {
4638 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4639 _bytes = ring->tx_stats.bytes;
4640 _packets = ring->tx_stats.packets;
4641 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4642 bytes += _bytes;
4643 packets += _packets;
3f9c0164 4644 }
128e45eb
AD		 4645 net_stats->tx_bytes = bytes;
4646 net_stats->tx_packets = packets;
3f9c0164
AD		 4647
4648 /* read stats registers */
9d5c8243
AK		 4649 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4650 adapter->stats.gprc += rd32(E1000_GPRC);
4651 adapter->stats.gorc += rd32(E1000_GORCL);
4652 rd32(E1000_GORCH); /* clear GORCL */
4653 adapter->stats.bprc += rd32(E1000_BPRC);
4654 adapter->stats.mprc += rd32(E1000_MPRC);
4655 adapter->stats.roc += rd32(E1000_ROC);
4656
4657 adapter->stats.prc64 += rd32(E1000_PRC64);
4658 adapter->stats.prc127 += rd32(E1000_PRC127);
4659 adapter->stats.prc255 += rd32(E1000_PRC255);
4660 adapter->stats.prc511 += rd32(E1000_PRC511);
4661 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4662 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4663 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4664 adapter->stats.sec += rd32(E1000_SEC);
4665
fa3d9a6d
MW		 4666 mpc = rd32(E1000_MPC);
4667 adapter->stats.mpc += mpc;
4668 net_stats->rx_fifo_errors += mpc;
9d5c8243
AK		 4669 adapter->stats.scc += rd32(E1000_SCC);
4670 adapter->stats.ecol += rd32(E1000_ECOL);
4671 adapter->stats.mcc += rd32(E1000_MCC);
4672 adapter->stats.latecol += rd32(E1000_LATECOL);
4673 adapter->stats.dc += rd32(E1000_DC);
4674 adapter->stats.rlec += rd32(E1000_RLEC);
4675 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4676 adapter->stats.xontxc += rd32(E1000_XONTXC);
4677 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4678 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4679 adapter->stats.fcruc += rd32(E1000_FCRUC);
4680 adapter->stats.gptc += rd32(E1000_GPTC);
4681 adapter->stats.gotc += rd32(E1000_GOTCL);
4682 rd32(E1000_GOTCH); /* clear GOTCL */
fa3d9a6d 4683 adapter->stats.rnbc += rd32(E1000_RNBC);
9d5c8243
AK		 4684 adapter->stats.ruc += rd32(E1000_RUC);
4685 adapter->stats.rfc += rd32(E1000_RFC);
4686 adapter->stats.rjc += rd32(E1000_RJC);
4687 adapter->stats.tor += rd32(E1000_TORH);
4688 adapter->stats.tot += rd32(E1000_TOTH);
4689 adapter->stats.tpr += rd32(E1000_TPR);
4690
4691 adapter->stats.ptc64 += rd32(E1000_PTC64);
4692 adapter->stats.ptc127 += rd32(E1000_PTC127);
4693 adapter->stats.ptc255 += rd32(E1000_PTC255);
4694 adapter->stats.ptc511 += rd32(E1000_PTC511);
4695 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4696 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4697
4698 adapter->stats.mptc += rd32(E1000_MPTC);
4699 adapter->stats.bptc += rd32(E1000_BPTC);
4700
2d0b0f69
NN		 4701 adapter->stats.tpt += rd32(E1000_TPT);
4702 adapter->stats.colc += rd32(E1000_COLC);
9d5c8243
AK		 4703
4704 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
43915c7c
NN		 4705 /* read internal phy specific stats */
4706 reg = rd32(E1000_CTRL_EXT);
4707 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4708 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4709 adapter->stats.tncrs += rd32(E1000_TNCRS);
4710 }
4711
9d5c8243
AK		 4712 adapter->stats.tsctc += rd32(E1000_TSCTC);
4713 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4714
4715 adapter->stats.iac += rd32(E1000_IAC);
4716 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4717 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4718 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4719 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4720 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4721 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4722 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4723 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4724
4725 /* Fill out the OS statistics structure */
128e45eb
AD		 4726 net_stats->multicast = adapter->stats.mprc;
4727 net_stats->collisions = adapter->stats.colc;
9d5c8243
AK		 4728
4729 /* Rx Errors */
4730
4731 /* RLEC on some newer hardware can be incorrect so build
8c0ab70a 4732 * our own version based on RUC and ROC */
128e45eb 4733 net_stats->rx_errors = adapter->stats.rxerrc +
9d5c8243
AK		 4734 adapter->stats.crcerrs + adapter->stats.algnerrc +
4735 adapter->stats.ruc + adapter->stats.roc +
4736 adapter->stats.cexterr;
128e45eb
AD		 4737 net_stats->rx_length_errors = adapter->stats.ruc +
4738 adapter->stats.roc;
4739 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4740 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4741 net_stats->rx_missed_errors = adapter->stats.mpc;
9d5c8243
AK		 4742
4743 /* Tx Errors */
128e45eb
AD		 4744 net_stats->tx_errors = adapter->stats.ecol +
4745 adapter->stats.latecol;
4746 net_stats->tx_aborted_errors = adapter->stats.ecol;
4747 net_stats->tx_window_errors = adapter->stats.latecol;
4748 net_stats->tx_carrier_errors = adapter->stats.tncrs;
9d5c8243
AK		 4749
4750 /* Tx Dropped needs to be maintained elsewhere */
4751
4752 /* Phy Stats */
4753 if (hw->phy.media_type == e1000_media_type_copper) {
4754 if ((adapter->link_speed == SPEED_1000) &&
73cd78f1 4755 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
9d5c8243
AK		 4756 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4757 adapter->phy_stats.idle_errors += phy_tmp;
4758 }
4759 }
4760
4761 /* Management Stats */
4762 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4763 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4764 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
0a915b95
CW		 4765
4766 /* OS2BMC Stats */
4767 reg = rd32(E1000_MANC);
4768 if (reg & E1000_MANC_EN_BMC2OS) {
4769 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4770 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4771 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4772 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4773 }
9d5c8243
AK		 4774}
4775
9d5c8243
AK		 4776static irqreturn_t igb_msix_other(int irq, void *data)
4777{
047e0030 4778 struct igb_adapter *adapter = data;
9d5c8243 4779 struct e1000_hw *hw = &adapter->hw;
844290e5 4780 u32 icr = rd32(E1000_ICR);
844290e5 4781 /* reading ICR causes bit 31 of EICR to be cleared */
dda0e083 4782
7f081d40
AD		 4783 if (icr & E1000_ICR_DRSTA)
4784 schedule_work(&adapter->reset_task);
4785
047e0030 4786 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 4787 /* HW is reporting DMA is out of sync */
4788 adapter->stats.doosync++;
13800469
GR		 4789 /* The DMA Out of Sync is also indication of a spoof event
4790 * in IOV mode. Check the Wrong VM Behavior register to
4791 * see if it is really a spoof event. */
4792 igb_check_wvbr(adapter);
dda0e083 4793 }
eebbbdba 4794
4ae196df
AD		 4795 /* Check for a mailbox event */
4796 if (icr & E1000_ICR_VMMB)
4797 igb_msg_task(adapter);
4798
4799 if (icr & E1000_ICR_LSC) {
4800 hw->mac.get_link_status = 1;
4801 /* guard against interrupt when we're going down */
4802 if (!test_bit(__IGB_DOWN, &adapter->state))
4803 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4804 }
4805
25568a53
AD		 4806 if (adapter->vfs_allocated_count)
4807 wr32(E1000_IMS, E1000_IMS_LSC |
4808 E1000_IMS_VMMB |
4809 E1000_IMS_DOUTSYNC);
4810 else
4811 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
844290e5 4812 wr32(E1000_EIMS, adapter->eims_other);
9d5c8243
AK		 4813
4814 return IRQ_HANDLED;
4815}
4816
047e0030 4817static void igb_write_itr(struct igb_q_vector *q_vector)
9d5c8243 4818{
26b39276 4819 struct igb_adapter *adapter = q_vector->adapter;
047e0030 4820 u32 itr_val = q_vector->itr_val & 0x7FFC;
9d5c8243 4821
047e0030
AD		 4822 if (!q_vector->set_itr)
4823 return;
73cd78f1 4824
047e0030
AD		 4825 if (!itr_val)
4826 itr_val = 0x4;
661086df 4827
26b39276
AD		 4828 if (adapter->hw.mac.type == e1000_82575)
4829 itr_val |= itr_val << 16;
661086df 4830 else
047e0030 4831 itr_val |= 0x8000000;
661086df 4832
047e0030
AD		 4833 writel(itr_val, q_vector->itr_register);
4834 q_vector->set_itr = 0;
6eb5a7f1
AD		 4835}
4836
047e0030 4837static irqreturn_t igb_msix_ring(int irq, void *data)
9d5c8243 4838{
047e0030 4839 struct igb_q_vector *q_vector = data;
9d5c8243 4840
047e0030
AD		 4841 /* Write the ITR value calculated from the previous interrupt. */
4842 igb_write_itr(q_vector);
9d5c8243 4843
047e0030 4844 napi_schedule(&q_vector->napi);
844290e5 4845
047e0030 4846 return IRQ_HANDLED;
fe4506b6
JC		 4847}
4848
421e02f0 4849#ifdef CONFIG_IGB_DCA
047e0030 4850static void igb_update_dca(struct igb_q_vector *q_vector)
fe4506b6 4851{
047e0030 4852 struct igb_adapter *adapter = q_vector->adapter;
fe4506b6
JC		 4853 struct e1000_hw *hw = &adapter->hw;
4854 int cpu = get_cpu();
fe4506b6 4855
047e0030
AD		 4856 if (q_vector->cpu == cpu)
4857 goto out_no_update;
4858
4859 if (q_vector->tx_ring) {
4860 int q = q_vector->tx_ring->reg_idx;
4861 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4862 if (hw->mac.type == e1000_82575) {
4863 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4864 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
2d064c06 4865 } else {
047e0030
AD		 4866 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4867 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4868 E1000_DCA_TXCTRL_CPUID_SHIFT;
4869 }
4870 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4871 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4872 }
4873 if (q_vector->rx_ring) {
4874 int q = q_vector->rx_ring->reg_idx;
4875 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4876 if (hw->mac.type == e1000_82575) {
2d064c06 4877 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
92be7917 4878 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
047e0030
AD		 4879 } else {
4880 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4881 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4882 E1000_DCA_RXCTRL_CPUID_SHIFT;
2d064c06 4883 }
fe4506b6
JC		 4884 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4885 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4886 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4887 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
fe4506b6 4888 }
047e0030
AD		 4889 q_vector->cpu = cpu;
4890out_no_update:
fe4506b6
JC		 4891 put_cpu();
4892}
4893
4894static void igb_setup_dca(struct igb_adapter *adapter)
4895{
7e0e99ef 4896 struct e1000_hw *hw = &adapter->hw;
fe4506b6
JC		 4897 int i;
4898
7dfc16fa 4899 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
fe4506b6
JC		 4900 return;
4901
7e0e99ef
AD		 4902 /* Always use CB2 mode, difference is masked in the CB driver. */
4903 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4904
047e0030 4905 for (i = 0; i < adapter->num_q_vectors; i++) {
26b39276
AD		 4906 adapter->q_vector[i]->cpu = -1;
4907 igb_update_dca(adapter->q_vector[i]);
fe4506b6
JC		 4908 }
4909}
4910
4911static int __igb_notify_dca(struct device *dev, void *data)
4912{
4913 struct net_device *netdev = dev_get_drvdata(dev);
4914 struct igb_adapter *adapter = netdev_priv(netdev);
090b1795 4915 struct pci_dev *pdev = adapter->pdev;
fe4506b6
JC		 4916 struct e1000_hw *hw = &adapter->hw;
4917 unsigned long event = *(unsigned long *)data;
4918
4919 switch (event) {
4920 case DCA_PROVIDER_ADD:
4921 /* if already enabled, don't do it again */
7dfc16fa 4922 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
fe4506b6 4923 break;
fe4506b6 4924 if (dca_add_requester(dev) == 0) {
bbd98fe4 4925 adapter->flags |= IGB_FLAG_DCA_ENABLED;
090b1795 4926 dev_info(&pdev->dev, "DCA enabled\n");
fe4506b6
JC		 4927 igb_setup_dca(adapter);
4928 break;
4929 }
4930 /* Fall Through since DCA is disabled. */
4931 case DCA_PROVIDER_REMOVE:
7dfc16fa 4932 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
fe4506b6 4933 /* without this a class_device is left
047e0030 4934 * hanging around in the sysfs model */
fe4506b6 4935 dca_remove_requester(dev);
090b1795 4936 dev_info(&pdev->dev, "DCA disabled\n");
7dfc16fa 4937 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
cbd347ad 4938 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
fe4506b6
JC		 4939 }
4940 break;
4941 }
bbd98fe4 4942
fe4506b6 4943 return 0;
9d5c8243
AK		 4944}
4945
fe4506b6
JC		 4946static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4947 void *p)
4948{
4949 int ret_val;
4950
4951 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4952 __igb_notify_dca);
4953
4954 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4955}
421e02f0 4956#endif /* CONFIG_IGB_DCA */
9d5c8243 4957
4ae196df
AD		 4958static void igb_ping_all_vfs(struct igb_adapter *adapter)
4959{
4960 struct e1000_hw *hw = &adapter->hw;
4961 u32 ping;
4962 int i;
4963
4964 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4965 ping = E1000_PF_CONTROL_MSG;
f2ca0dbe 4966 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4ae196df
AD		 4967 ping |= E1000_VT_MSGTYPE_CTS;
4968 igb_write_mbx(hw, &ping, 1, i);
4969 }
4970}
4971
7d5753f0
AD		 4972static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4973{
4974 struct e1000_hw *hw = &adapter->hw;
4975 u32 vmolr = rd32(E1000_VMOLR(vf));
4976 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4977
d85b9004 4978 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7d5753f0
AD		 4979 IGB_VF_FLAG_MULTI_PROMISC);
4980 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4981
4982 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4983 vmolr |= E1000_VMOLR_MPME;
d85b9004 4984 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7d5753f0
AD		 4985 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4986 } else {
4987 /*
4988 * if we have hashes and we are clearing a multicast promisc
4989 * flag we need to write the hashes to the MTA as this step
4990 * was previously skipped
4991 */
4992 if (vf_data->num_vf_mc_hashes > 30) {
4993 vmolr |= E1000_VMOLR_MPME;
4994 } else if (vf_data->num_vf_mc_hashes) {
4995 int j;
4996 vmolr |= E1000_VMOLR_ROMPE;
4997 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4998 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4999 }
5000 }
5001
5002 wr32(E1000_VMOLR(vf), vmolr);
5003
5004 /* there are flags left unprocessed, likely not supported */
5005 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5006 return -EINVAL;
5007
5008 return 0;
5009
5010}
5011
4ae196df
AD		 5012static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5013 u32 *msgbuf, u32 vf)
5014{
5015 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5016 u16 *hash_list = (u16 *)&msgbuf[1];
5017 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5018 int i;
5019
7d5753f0 5020 /* salt away the number of multicast addresses assigned
4ae196df
AD		 5021 * to this VF for later use to restore when the PF multi cast
5022 * list changes
5023 */
5024 vf_data->num_vf_mc_hashes = n;
5025
7d5753f0
AD		 5026 /* only up to 30 hash values supported */
5027 if (n > 30)
5028 n = 30;
5029
5030 /* store the hashes for later use */
4ae196df 5031 for (i = 0; i < n; i++)
a419aef8 5032 vf_data->vf_mc_hashes[i] = hash_list[i];
4ae196df
AD		 5033
5034 /* Flush and reset the mta with the new values */
ff41f8dc 5035 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5036
5037 return 0;
5038}
5039
5040static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5041{
5042 struct e1000_hw *hw = &adapter->hw;
5043 struct vf_data_storage *vf_data;
5044 int i, j;
5045
5046 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7d5753f0
AD		 5047 u32 vmolr = rd32(E1000_VMOLR(i));
5048 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5049
4ae196df 5050 vf_data = &adapter->vf_data[i];
7d5753f0
AD		 5051
5052 if ((vf_data->num_vf_mc_hashes > 30) ||
5053 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5054 vmolr |= E1000_VMOLR_MPME;
5055 } else if (vf_data->num_vf_mc_hashes) {
5056 vmolr |= E1000_VMOLR_ROMPE;
5057 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5058 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5059 }
5060 wr32(E1000_VMOLR(i), vmolr);
4ae196df
AD		 5061 }
5062}
5063
5064static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5065{
5066 struct e1000_hw *hw = &adapter->hw;
5067 u32 pool_mask, reg, vid;
5068 int i;
5069
5070 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5071
5072 /* Find the vlan filter for this id */
5073 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5074 reg = rd32(E1000_VLVF(i));
5075
5076 /* remove the vf from the pool */
5077 reg &= ~pool_mask;
5078
5079 /* if pool is empty then remove entry from vfta */
5080 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5081 (reg & E1000_VLVF_VLANID_ENABLE)) {
5082 reg = 0;
5083 vid = reg & E1000_VLVF_VLANID_MASK;
5084 igb_vfta_set(hw, vid, false);
5085 }
5086
5087 wr32(E1000_VLVF(i), reg);
5088 }
ae641bdc
AD		 5089
5090 adapter->vf_data[vf].vlans_enabled = 0;
4ae196df
AD		 5091}
5092
5093static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5094{
5095 struct e1000_hw *hw = &adapter->hw;
5096 u32 reg, i;
5097
51466239
AD		 5098 /* The vlvf table only exists on 82576 hardware and newer */
5099 if (hw->mac.type < e1000_82576)
5100 return -1;
5101
5102 /* we only need to do this if VMDq is enabled */
4ae196df
AD		 5103 if (!adapter->vfs_allocated_count)
5104 return -1;
5105
5106 /* Find the vlan filter for this id */
5107 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5108 reg = rd32(E1000_VLVF(i));
5109 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5110 vid == (reg & E1000_VLVF_VLANID_MASK))
5111 break;
5112 }
5113
5114 if (add) {
5115 if (i == E1000_VLVF_ARRAY_SIZE) {
5116 /* Did not find a matching VLAN ID entry that was
5117 * enabled. Search for a free filter entry, i.e.
5118 * one without the enable bit set
5119 */
5120 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5121 reg = rd32(E1000_VLVF(i));
5122 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5123 break;
5124 }
5125 }
5126 if (i < E1000_VLVF_ARRAY_SIZE) {
5127 /* Found an enabled/available entry */
5128 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5129
5130 /* if !enabled we need to set this up in vfta */
5131 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
51466239
AD		 5132 /* add VID to filter table */
5133 igb_vfta_set(hw, vid, true);
4ae196df
AD		 5134 reg |= E1000_VLVF_VLANID_ENABLE;
5135 }
cad6d05f
AD		 5136 reg &= ~E1000_VLVF_VLANID_MASK;
5137 reg |= vid;
4ae196df 5138 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5139
5140 /* do not modify RLPML for PF devices */
5141 if (vf >= adapter->vfs_allocated_count)
5142 return 0;
5143
5144 if (!adapter->vf_data[vf].vlans_enabled) {
5145 u32 size;
5146 reg = rd32(E1000_VMOLR(vf));
5147 size = reg & E1000_VMOLR_RLPML_MASK;
5148 size += 4;
5149 reg &= ~E1000_VMOLR_RLPML_MASK;
5150 reg |= size;
5151 wr32(E1000_VMOLR(vf), reg);
5152 }
ae641bdc 5153
51466239 5154 adapter->vf_data[vf].vlans_enabled++;
4ae196df
AD		 5155 return 0;
5156 }
5157 } else {
5158 if (i < E1000_VLVF_ARRAY_SIZE) {
5159 /* remove vf from the pool */
5160 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5161 /* if pool is empty then remove entry from vfta */
5162 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5163 reg = 0;
5164 igb_vfta_set(hw, vid, false);
5165 }
5166 wr32(E1000_VLVF(i), reg);
ae641bdc
AD		 5167
5168 /* do not modify RLPML for PF devices */
5169 if (vf >= adapter->vfs_allocated_count)
5170 return 0;
5171
5172 adapter->vf_data[vf].vlans_enabled--;
5173 if (!adapter->vf_data[vf].vlans_enabled) {
5174 u32 size;
5175 reg = rd32(E1000_VMOLR(vf));
5176 size = reg & E1000_VMOLR_RLPML_MASK;
5177 size -= 4;
5178 reg &= ~E1000_VMOLR_RLPML_MASK;
5179 reg |= size;
5180 wr32(E1000_VMOLR(vf), reg);
5181 }
4ae196df
AD		 5182 }
5183 }
8151d294
WM		 5184 return 0;
5185}
5186
5187static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5188{
5189 struct e1000_hw *hw = &adapter->hw;
5190
5191 if (vid)
5192 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5193 else
5194 wr32(E1000_VMVIR(vf), 0);
5195}
5196
5197static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5198 int vf, u16 vlan, u8 qos)
5199{
5200 int err = 0;
5201 struct igb_adapter *adapter = netdev_priv(netdev);
5202
5203 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5204 return -EINVAL;
5205 if (vlan || qos) {
5206 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5207 if (err)
5208 goto out;
5209 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5210 igb_set_vmolr(adapter, vf, !vlan);
5211 adapter->vf_data[vf].pf_vlan = vlan;
5212 adapter->vf_data[vf].pf_qos = qos;
5213 dev_info(&adapter->pdev->dev,
5214 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5215 if (test_bit(__IGB_DOWN, &adapter->state)) {
5216 dev_warn(&adapter->pdev->dev,
5217 "The VF VLAN has been set,"
5218 " but the PF device is not up.\n");
5219 dev_warn(&adapter->pdev->dev,
5220 "Bring the PF device up before"
5221 " attempting to use the VF device.\n");
5222 }
5223 } else {
5224 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5225 false, vf);
5226 igb_set_vmvir(adapter, vlan, vf);
5227 igb_set_vmolr(adapter, vf, true);
5228 adapter->vf_data[vf].pf_vlan = 0;
5229 adapter->vf_data[vf].pf_qos = 0;
5230 }
5231out:
5232 return err;
4ae196df
AD		 5233}
5234
5235static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5236{
5237 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5238 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5239
5240 return igb_vlvf_set(adapter, vid, add, vf);
5241}
5242
f2ca0dbe 5243static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4ae196df 5244{
8fa7e0f7
GR		 5245 /* clear flags - except flag that indicates PF has set the MAC */
5246 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
f2ca0dbe 5247 adapter->vf_data[vf].last_nack = jiffies;
4ae196df
AD		 5248
5249 /* reset offloads to defaults */
8151d294 5250 igb_set_vmolr(adapter, vf, true);
4ae196df
AD		 5251
5252 /* reset vlans for device */
5253 igb_clear_vf_vfta(adapter, vf);
8151d294
WM		 5254 if (adapter->vf_data[vf].pf_vlan)
5255 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5256 adapter->vf_data[vf].pf_vlan,
5257 adapter->vf_data[vf].pf_qos);
5258 else
5259 igb_clear_vf_vfta(adapter, vf);
4ae196df
AD		 5260
5261 /* reset multicast table array for vf */
5262 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5263
5264 /* Flush and reset the mta with the new values */
ff41f8dc 5265 igb_set_rx_mode(adapter->netdev);
4ae196df
AD		 5266}
5267
f2ca0dbe
AD		 5268static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5269{
5270 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5271
5272 /* generate a new mac address as we were hotplug removed/added */
8151d294
WM		 5273 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5274 random_ether_addr(vf_mac);
f2ca0dbe
AD		 5275
5276 /* process remaining reset events */
5277 igb_vf_reset(adapter, vf);
5278}
5279
5280static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4ae196df
AD		 5281{
5282 struct e1000_hw *hw = &adapter->hw;
5283 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
ff41f8dc 5284 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df
AD		 5285 u32 reg, msgbuf[3];
5286 u8 *addr = (u8 *)(&msgbuf[1]);
5287
5288 /* process all the same items cleared in a function level reset */
f2ca0dbe 5289 igb_vf_reset(adapter, vf);
4ae196df
AD		 5290
5291 /* set vf mac address */
26ad9178 5292 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4ae196df
AD		 5293
5294 /* enable transmit and receive for vf */
5295 reg = rd32(E1000_VFTE);
5296 wr32(E1000_VFTE, reg | (1 << vf));
5297 reg = rd32(E1000_VFRE);
5298 wr32(E1000_VFRE, reg | (1 << vf));
5299
8fa7e0f7 5300 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
4ae196df
AD		 5301
5302 /* reply to reset with ack and vf mac address */
5303 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5304 memcpy(addr, vf_mac, 6);
5305 igb_write_mbx(hw, msgbuf, 3, vf);
5306}
5307
5308static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5309{
de42edde
GR		 5310 /*
5311 * The VF MAC Address is stored in a packed array of bytes
5312 * starting at the second 32 bit word of the msg array
5313 */
f2ca0dbe
AD		 5314 unsigned char *addr = (char *)&msg[1];
5315 int err = -1;
4ae196df 5316
f2ca0dbe
AD		 5317 if (is_valid_ether_addr(addr))
5318 err = igb_set_vf_mac(adapter, vf, addr);
4ae196df 5319
f2ca0dbe 5320 return err;
4ae196df
AD		 5321}
5322
5323static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5324{
5325 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5326 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5327 u32 msg = E1000_VT_MSGTYPE_NACK;
5328
5329 /* if device isn't clear to send it shouldn't be reading either */
f2ca0dbe
AD		 5330 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5331 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4ae196df 5332 igb_write_mbx(hw, &msg, 1, vf);
f2ca0dbe 5333 vf_data->last_nack = jiffies;
4ae196df
AD		 5334 }
5335}
5336
f2ca0dbe 5337static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4ae196df 5338{
f2ca0dbe
AD		 5339 struct pci_dev *pdev = adapter->pdev;
5340 u32 msgbuf[E1000_VFMAILBOX_SIZE];
4ae196df 5341 struct e1000_hw *hw = &adapter->hw;
f2ca0dbe 5342 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4ae196df
AD		 5343 s32 retval;
5344
f2ca0dbe 5345 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4ae196df 5346
fef45f4c
AD		 5347 if (retval) {
5348 /* if receive failed revoke VF CTS stats and restart init */
f2ca0dbe 5349 dev_err(&pdev->dev, "Error receiving message from VF\n");
fef45f4c
AD		 5350 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5351 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5352 return;
5353 goto out;
5354 }
4ae196df
AD		 5355
5356 /* this is a message we already processed, do nothing */
5357 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
f2ca0dbe 5358 return;
4ae196df
AD		 5359
5360 /*
5361 * until the vf completes a reset it should not be
5362 * allowed to start any configuration.
5363 */
5364
5365 if (msgbuf[0] == E1000_VF_RESET) {
5366 igb_vf_reset_msg(adapter, vf);
f2ca0dbe 5367 return;
4ae196df
AD		 5368 }
5369
f2ca0dbe 5370 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
fef45f4c
AD		 5371 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5372 return;
5373 retval = -1;
5374 goto out;
4ae196df
AD		 5375 }
5376
5377 switch ((msgbuf[0] & 0xFFFF)) {
5378 case E1000_VF_SET_MAC_ADDR:
a6b5ea35
GR		 5379 retval = -EINVAL;
5380 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5381 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5382 else
5383 dev_warn(&pdev->dev,
5384 "VF %d attempted to override administratively "
5385 "set MAC address\nReload the VF driver to "
5386 "resume operations\n", vf);
4ae196df 5387 break;
7d5753f0
AD		 5388 case E1000_VF_SET_PROMISC:
5389 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5390 break;
4ae196df
AD		 5391 case E1000_VF_SET_MULTICAST:
5392 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5393 break;
5394 case E1000_VF_SET_LPE:
5395 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5396 break;
5397 case E1000_VF_SET_VLAN:
a6b5ea35
GR		 5398 retval = -1;
5399 if (vf_data->pf_vlan)
5400 dev_warn(&pdev->dev,
5401 "VF %d attempted to override administratively "
5402 "set VLAN tag\nReload the VF driver to "
5403 "resume operations\n", vf);
8151d294
WM		 5404 else
5405 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4ae196df
AD		 5406 break;
5407 default:
090b1795 5408 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4ae196df
AD		 5409 retval = -1;
5410 break;
5411 }
5412
fef45f4c
AD		 5413 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5414out:
4ae196df
AD		 5415 /* notify the VF of the results of what it sent us */
5416 if (retval)
5417 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5418 else
5419 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5420
4ae196df 5421 igb_write_mbx(hw, msgbuf, 1, vf);
f2ca0dbe 5422}
4ae196df 5423
f2ca0dbe
AD		 5424static void igb_msg_task(struct igb_adapter *adapter)
5425{
5426 struct e1000_hw *hw = &adapter->hw;
5427 u32 vf;
5428
5429 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5430 /* process any reset requests */
5431 if (!igb_check_for_rst(hw, vf))
5432 igb_vf_reset_event(adapter, vf);
5433
5434 /* process any messages pending */
5435 if (!igb_check_for_msg(hw, vf))
5436 igb_rcv_msg_from_vf(adapter, vf);
5437
5438 /* process any acks */
5439 if (!igb_check_for_ack(hw, vf))
5440 igb_rcv_ack_from_vf(adapter, vf);
5441 }
4ae196df
AD		 5442}
5443
68d480c4
AD		 5444/**
5445 * igb_set_uta - Set unicast filter table address
5446 * @adapter: board private structure
5447 *
5448 * The unicast table address is a register array of 32-bit registers.
5449 * The table is meant to be used in a way similar to how the MTA is used
5450 * however due to certain limitations in the hardware it is necessary to
25985edc
LDM		 5451 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5452 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
68d480c4
AD		 5453 **/
5454static void igb_set_uta(struct igb_adapter *adapter)
5455{
5456 struct e1000_hw *hw = &adapter->hw;
5457 int i;
5458
5459 /* The UTA table only exists on 82576 hardware and newer */
5460 if (hw->mac.type < e1000_82576)
5461 return;
5462
5463 /* we only need to do this if VMDq is enabled */
5464 if (!adapter->vfs_allocated_count)
5465 return;
5466
5467 for (i = 0; i < hw->mac.uta_reg_count; i++)
5468 array_wr32(E1000_UTA, i, ~0);
5469}
5470
9d5c8243
AK		 5471/**
5472 * igb_intr_msi - Interrupt Handler
5473 * @irq: interrupt number
5474 * @data: pointer to a network interface device structure
5475 **/
5476static irqreturn_t igb_intr_msi(int irq, void *data)
5477{
047e0030
AD		 5478 struct igb_adapter *adapter = data;
5479 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5480 struct e1000_hw *hw = &adapter->hw;
5481 /* read ICR disables interrupts using IAM */
5482 u32 icr = rd32(E1000_ICR);
5483
047e0030 5484 igb_write_itr(q_vector);
9d5c8243 5485
7f081d40
AD		 5486 if (icr & E1000_ICR_DRSTA)
5487 schedule_work(&adapter->reset_task);
5488
047e0030 5489 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5490 /* HW is reporting DMA is out of sync */
5491 adapter->stats.doosync++;
5492 }
5493
9d5c8243
AK		 5494 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5495 hw->mac.get_link_status = 1;
5496 if (!test_bit(__IGB_DOWN, &adapter->state))
5497 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5498 }
5499
047e0030 5500 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5501
5502 return IRQ_HANDLED;
5503}
5504
5505/**
4a3c6433 5506 * igb_intr - Legacy Interrupt Handler
9d5c8243
AK		 5507 * @irq: interrupt number
5508 * @data: pointer to a network interface device structure
5509 **/
5510static irqreturn_t igb_intr(int irq, void *data)
5511{
047e0030
AD		 5512 struct igb_adapter *adapter = data;
5513 struct igb_q_vector *q_vector = adapter->q_vector[0];
9d5c8243
AK		 5514 struct e1000_hw *hw = &adapter->hw;
5515 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5516 * need for the IMC write */
5517 u32 icr = rd32(E1000_ICR);
9d5c8243
AK		 5518 if (!icr)
5519 return IRQ_NONE; /* Not our interrupt */
5520
047e0030 5521 igb_write_itr(q_vector);
9d5c8243
AK		 5522
5523 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5524 * not set, then the adapter didn't send an interrupt */
5525 if (!(icr & E1000_ICR_INT_ASSERTED))
5526 return IRQ_NONE;
5527
7f081d40
AD		 5528 if (icr & E1000_ICR_DRSTA)
5529 schedule_work(&adapter->reset_task);
5530
047e0030 5531 if (icr & E1000_ICR_DOUTSYNC) {
dda0e083
AD		 5532 /* HW is reporting DMA is out of sync */
5533 adapter->stats.doosync++;
5534 }
5535
9d5c8243
AK		 5536 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5537 hw->mac.get_link_status = 1;
5538 /* guard against interrupt when we're going down */
5539 if (!test_bit(__IGB_DOWN, &adapter->state))
5540 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5541 }
5542
047e0030 5543 napi_schedule(&q_vector->napi);
9d5c8243
AK		 5544
5545 return IRQ_HANDLED;
5546}
5547
047e0030 5548static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
9d5c8243 5549{
047e0030 5550 struct igb_adapter *adapter = q_vector->adapter;
46544258 5551 struct e1000_hw *hw = &adapter->hw;
9d5c8243 5552
4fc82adf
AD		 5553 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5554 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
047e0030 5555 if (!adapter->msix_entries)
6eb5a7f1 5556 igb_set_itr(adapter);
46544258 5557 else
047e0030 5558 igb_update_ring_itr(q_vector);
9d5c8243
AK		 5559 }
5560
46544258
AD		 5561 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5562 if (adapter->msix_entries)
047e0030 5563 wr32(E1000_EIMS, q_vector->eims_value);
46544258
AD		 5564 else
5565 igb_irq_enable(adapter);
5566 }
9d5c8243
AK		 5567}
5568
46544258
AD		 5569/**
5570 * igb_poll - NAPI Rx polling callback
5571 * @napi: napi polling structure
5572 * @budget: count of how many packets we should handle
5573 **/
5574static int igb_poll(struct napi_struct *napi, int budget)
9d5c8243 5575{
047e0030
AD		 5576 struct igb_q_vector *q_vector = container_of(napi,
5577 struct igb_q_vector,
5578 napi);
16eb8815 5579 bool clean_complete = true;
9d5c8243 5580
421e02f0 5581#ifdef CONFIG_IGB_DCA
047e0030
AD		 5582 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5583 igb_update_dca(q_vector);
fe4506b6 5584#endif
047e0030 5585 if (q_vector->tx_ring)
13fde97a 5586 clean_complete = igb_clean_tx_irq(q_vector);
9d5c8243 5587
047e0030 5588 if (q_vector->rx_ring)
cd392f5c 5589 clean_complete &= igb_clean_rx_irq(q_vector, budget);
047e0030 5590
16eb8815
AD		 5591 /* If all work not completed, return budget and keep polling */
5592 if (!clean_complete)
5593 return budget;
46544258 5594
9d5c8243 5595 /* If not enough Rx work done, exit the polling mode */
16eb8815
AD		 5596 napi_complete(napi);
5597 igb_ring_irq_enable(q_vector);
9d5c8243 5598
16eb8815 5599 return 0;
9d5c8243 5600}
6d8126f9 5601
33af6bcc 5602/**
c5b9bd5e 5603 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
33af6bcc 5604 * @adapter: board private structure
c5b9bd5e
AD		 5605 * @shhwtstamps: timestamp structure to update
5606 * @regval: unsigned 64bit system time value.
5607 *
5608 * We need to convert the system time value stored in the RX/TXSTMP registers
5609 * into a hwtstamp which can be used by the upper level timestamping functions
5610 */
5611static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5612 struct skb_shared_hwtstamps *shhwtstamps,
5613 u64 regval)
5614{
5615 u64 ns;
5616
55cac248
AD		 5617 /*
5618 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5619 * 24 to match clock shift we setup earlier.
5620 */
5621 if (adapter->hw.mac.type == e1000_82580)
5622 regval <<= IGB_82580_TSYNC_SHIFT;
5623
c5b9bd5e
AD		 5624 ns = timecounter_cyc2time(&adapter->clock, regval);
5625 timecompare_update(&adapter->compare, ns);
5626 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5627 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5628 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5629}
5630
5631/**
5632 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5633 * @q_vector: pointer to q_vector containing needed info
06034649 5634 * @buffer: pointer to igb_tx_buffer structure
33af6bcc
PO		 5635 *
5636 * If we were asked to do hardware stamping and such a time stamp is
5637 * available, then it must have been for this skb here because we only
5638 * allow only one such packet into the queue.
5639 */
06034649
AD		 5640static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
5641 struct igb_tx_buffer *buffer_info)
33af6bcc 5642{
c5b9bd5e 5643 struct igb_adapter *adapter = q_vector->adapter;
33af6bcc 5644 struct e1000_hw *hw = &adapter->hw;
c5b9bd5e
AD		 5645 struct skb_shared_hwtstamps shhwtstamps;
5646 u64 regval;
33af6bcc 5647
c5b9bd5e 5648 /* if skb does not support hw timestamp or TX stamp not valid exit */
2bbfebe2 5649 if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
c5b9bd5e
AD		 5650 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5651 return;
5652
5653 regval = rd32(E1000_TXSTMPL);
5654 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5655
5656 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
2873957d 5657 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
33af6bcc
PO		 5658}
5659
9d5c8243
AK		 5660/**
5661 * igb_clean_tx_irq - Reclaim resources after transmit completes
047e0030 5662 * @q_vector: pointer to q_vector containing needed info
9d5c8243
AK		 5663 * returns true if ring is completely cleaned
5664 **/
047e0030 5665static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
9d5c8243 5666{
047e0030
AD		 5667 struct igb_adapter *adapter = q_vector->adapter;
5668 struct igb_ring *tx_ring = q_vector->tx_ring;
06034649 5669 struct igb_tx_buffer *tx_buffer;
8542db05 5670 union e1000_adv_tx_desc *tx_desc, *eop_desc;
9d5c8243 5671 unsigned int total_bytes = 0, total_packets = 0;
13fde97a 5672 unsigned int budget = q_vector->tx_work_limit;
8542db05 5673 unsigned int i = tx_ring->next_to_clean;
9d5c8243 5674
13fde97a
AD		 5675 if (test_bit(__IGB_DOWN, &adapter->state))
5676 return true;
0e014cb1 5677
06034649 5678 tx_buffer = &tx_ring->tx_buffer_info[i];
13fde97a 5679 tx_desc = IGB_TX_DESC(tx_ring, i);
8542db05 5680 i -= tx_ring->count;
9d5c8243 5681
13fde97a 5682 for (; budget; budget--) {
8542db05 5683 eop_desc = tx_buffer->next_to_watch;
13fde97a 5684
8542db05
AD		 5685 /* prevent any other reads prior to eop_desc */
5686 rmb();
5687
5688 /* if next_to_watch is not set then there is no work pending */
5689 if (!eop_desc)
5690 break;
13fde97a
AD		 5691
5692 /* if DD is not set pending work has not been completed */
5693 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5694 break;
5695
8542db05
AD		 5696 /* clear next_to_watch to prevent false hangs */
5697 tx_buffer->next_to_watch = NULL;
9d5c8243 5698
ebe42d16
AD		 5699 /* update the statistics for this packet */
5700 total_bytes += tx_buffer->bytecount;
5701 total_packets += tx_buffer->gso_segs;
13fde97a 5702
ebe42d16
AD		 5703 /* retrieve hardware timestamp */
5704 igb_tx_hwtstamp(q_vector, tx_buffer);
5705
5706 /* free the skb */
5707 dev_kfree_skb_any(tx_buffer->skb);
5708 tx_buffer->skb = NULL;
13fde97a 5709
ebe42d16
AD		 5710 /* unmap skb header data */
5711 dma_unmap_single(tx_ring->dev,
5712 tx_buffer->dma,
5713 tx_buffer->length,
5714 DMA_TO_DEVICE);
5715
5716 /* clear last DMA location and unmap remaining buffers */
5717 while (tx_desc != eop_desc) {
5718 tx_buffer->dma = 0;
9d5c8243 5719
13fde97a
AD		 5720 tx_buffer++;
5721 tx_desc++;
9d5c8243 5722 i++;
8542db05
AD		 5723 if (unlikely(!i)) {
5724 i -= tx_ring->count;
06034649 5725 tx_buffer = tx_ring->tx_buffer_info;
13fde97a
AD		 5726 tx_desc = IGB_TX_DESC(tx_ring, 0);
5727 }
ebe42d16
AD		 5728
5729 /* unmap any remaining paged data */
5730 if (tx_buffer->dma) {
5731 dma_unmap_page(tx_ring->dev,
5732 tx_buffer->dma,
5733 tx_buffer->length,
5734 DMA_TO_DEVICE);
5735 }
5736 }
5737
5738 /* clear last DMA location */
5739 tx_buffer->dma = 0;
5740
5741 /* move us one more past the eop_desc for start of next pkt */
5742 tx_buffer++;
5743 tx_desc++;
5744 i++;
5745 if (unlikely(!i)) {
5746 i -= tx_ring->count;
5747 tx_buffer = tx_ring->tx_buffer_info;
5748 tx_desc = IGB_TX_DESC(tx_ring, 0);
5749 }
0e014cb1
AD		 5750 }
5751
8542db05 5752 i += tx_ring->count;
9d5c8243 5753 tx_ring->next_to_clean = i;
13fde97a
AD		 5754 u64_stats_update_begin(&tx_ring->tx_syncp);
5755 tx_ring->tx_stats.bytes += total_bytes;
5756 tx_ring->tx_stats.packets += total_packets;
5757 u64_stats_update_end(&tx_ring->tx_syncp);
5758 tx_ring->total_bytes += total_bytes;
5759 tx_ring->total_packets += total_packets;
9d5c8243 5760
13fde97a
AD		 5761 if (tx_ring->detect_tx_hung) {
5762 struct e1000_hw *hw = &adapter->hw;
12dcd86b 5763
8542db05 5764 eop_desc = tx_buffer->next_to_watch;
9d5c8243 5765
9d5c8243
AK		 5766 /* Detect a transmit hang in hardware, this serializes the
5767 * check with the clearing of time_stamp and movement of i */
5768 tx_ring->detect_tx_hung = false;
8542db05
AD		 5769 if (eop_desc &&
5770 time_after(jiffies, tx_buffer->time_stamp +
8e95a202
JP		 5771 (adapter->tx_timeout_factor * HZ)) &&
5772 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
9d5c8243 5773
9d5c8243 5774 /* detected Tx unit hang */
59d71989 5775 dev_err(tx_ring->dev,
9d5c8243 5776 "Detected Tx Unit Hang\n"
2d064c06 5777 " Tx Queue <%d>\n"
9d5c8243
AK		 5778 " TDH <%x>\n"
5779 " TDT <%x>\n"
5780 " next_to_use <%x>\n"
5781 " next_to_clean <%x>\n"
9d5c8243
AK		 5782 "buffer_info[next_to_clean]\n"
5783 " time_stamp <%lx>\n"
8542db05 5784 " next_to_watch <%p>\n"
9d5c8243
AK		 5785 " jiffies <%lx>\n"
5786 " desc.status <%x>\n",
2d064c06 5787 tx_ring->queue_index,
238ac817 5788 rd32(E1000_TDH(tx_ring->reg_idx)),
fce99e34 5789 readl(tx_ring->tail),
9d5c8243
AK		 5790 tx_ring->next_to_use,
5791 tx_ring->next_to_clean,
8542db05
AD		 5792 tx_buffer->time_stamp,
5793 eop_desc,
9d5c8243 5794 jiffies,
0e014cb1 5795 eop_desc->wb.status);
13fde97a
AD		 5796 netif_stop_subqueue(tx_ring->netdev,
5797 tx_ring->queue_index);
5798
5799 /* we are about to reset, no point in enabling stuff */
5800 return true;
9d5c8243
AK		 5801 }
5802 }
13fde97a
AD		 5803
5804 if (unlikely(total_packets &&
5805 netif_carrier_ok(tx_ring->netdev) &&
5806 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5807 /* Make sure that anybody stopping the queue after this
5808 * sees the new next_to_clean.
5809 */
5810 smp_mb();
5811 if (__netif_subqueue_stopped(tx_ring->netdev,
5812 tx_ring->queue_index) &&
5813 !(test_bit(__IGB_DOWN, &adapter->state))) {
5814 netif_wake_subqueue(tx_ring->netdev,
5815 tx_ring->queue_index);
5816
5817 u64_stats_update_begin(&tx_ring->tx_syncp);
5818 tx_ring->tx_stats.restart_queue++;
5819 u64_stats_update_end(&tx_ring->tx_syncp);
5820 }
5821 }
5822
5823 return !!budget;
9d5c8243
AK		 5824}
5825
cd392f5c
AD		 5826static inline void igb_rx_checksum(struct igb_ring *ring,
5827 u32 status_err, struct sk_buff *skb)
9d5c8243 5828{
bc8acf2c 5829 skb_checksum_none_assert(skb);
9d5c8243
AK		 5830
5831 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
85ad76b2
AD		 5832 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5833 (status_err & E1000_RXD_STAT_IXSM))
9d5c8243 5834 return;
85ad76b2 5835
9d5c8243
AK		 5836 /* TCP/UDP checksum error bit is set */
5837 if (status_err &
5838 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
b9473560
JB		 5839 /*
5840 * work around errata with sctp packets where the TCPE aka
5841 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5842 * packets, (aka let the stack check the crc32c)
5843 */
85ad76b2 5844 if ((skb->len == 60) &&
12dcd86b
ED		 5845 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM)) {
5846 u64_stats_update_begin(&ring->rx_syncp);
04a5fcaa 5847 ring->rx_stats.csum_err++;
12dcd86b
ED		 5848 u64_stats_update_end(&ring->rx_syncp);
5849 }
9d5c8243 5850 /* let the stack verify checksum errors */
9d5c8243
AK		 5851 return;
5852 }
5853 /* It must be a TCP or UDP packet with a valid checksum */
5854 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5855 skb->ip_summed = CHECKSUM_UNNECESSARY;
5856
59d71989 5857 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
9d5c8243
AK		 5858}
5859
757b77e2 5860static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
c5b9bd5e
AD		 5861 struct sk_buff *skb)
5862{
5863 struct igb_adapter *adapter = q_vector->adapter;
5864 struct e1000_hw *hw = &adapter->hw;
5865 u64 regval;
5866
5867 /*
5868 * If this bit is set, then the RX registers contain the time stamp. No
5869 * other packet will be time stamped until we read these registers, so
5870 * read the registers to make them available again. Because only one
5871 * packet can be time stamped at a time, we know that the register
5872 * values must belong to this one here and therefore we don't need to
5873 * compare any of the additional attributes stored for it.
5874 *
2244d07b 5875 * If nothing went wrong, then it should have a shared tx_flags that we
c5b9bd5e
AD		 5876 * can turn into a skb_shared_hwtstamps.
5877 */
757b77e2
NN		 5878 if (staterr & E1000_RXDADV_STAT_TSIP) {
5879 u32 *stamp = (u32 *)skb->data;
5880 regval = le32_to_cpu(*(stamp + 2));
5881 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5882 skb_pull(skb, IGB_TS_HDR_LEN);
5883 } else {
5884 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5885 return;
c5b9bd5e 5886
757b77e2
NN		 5887 regval = rd32(E1000_RXSTMPL);
5888 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5889 }
c5b9bd5e
AD		 5890
5891 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5892}
44390ca6 5893static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
2d94d8ab
AD		 5894{
5895 /* HW will not DMA in data larger than the given buffer, even if it
5896 * parses the (NFS, of course) header to be larger. In that case, it
5897 * fills the header buffer and spills the rest into the page.
5898 */
5899 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5900 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
44390ca6
AD		 5901 if (hlen > IGB_RX_HDR_LEN)
5902 hlen = IGB_RX_HDR_LEN;
2d94d8ab
AD		 5903 return hlen;
5904}
5905
cd392f5c 5906static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
9d5c8243 5907{
047e0030 5908 struct igb_ring *rx_ring = q_vector->rx_ring;
16eb8815
AD		 5909 union e1000_adv_rx_desc *rx_desc;
5910 const int current_node = numa_node_id();
9d5c8243 5911 unsigned int total_bytes = 0, total_packets = 0;
2d94d8ab 5912 u32 staterr;
16eb8815
AD		 5913 u16 cleaned_count = igb_desc_unused(rx_ring);
5914 u16 i = rx_ring->next_to_clean;
9d5c8243 5915
60136906 5916 rx_desc = IGB_RX_DESC(rx_ring, i);
9d5c8243
AK		 5917 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5918
5919 while (staterr & E1000_RXD_STAT_DD) {
06034649 5920 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
16eb8815
AD		 5921 struct sk_buff *skb = buffer_info->skb;
5922 union e1000_adv_rx_desc *next_rxd;
9d5c8243 5923
69d3ca53 5924 buffer_info->skb = NULL;
16eb8815 5925 prefetch(skb->data);
69d3ca53
AD		 5926
5927 i++;
5928 if (i == rx_ring->count)
5929 i = 0;
42d0781a 5930
60136906 5931 next_rxd = IGB_RX_DESC(rx_ring, i);
69d3ca53 5932 prefetch(next_rxd);
9d5c8243 5933
16eb8815
AD		 5934 /*
5935 * This memory barrier is needed to keep us from reading
5936 * any other fields out of the rx_desc until we know the
5937 * RXD_STAT_DD bit is set
5938 */
5939 rmb();
9d5c8243 5940
16eb8815
AD		 5941 if (!skb_is_nonlinear(skb)) {
5942 __skb_put(skb, igb_get_hlen(rx_desc));
5943 dma_unmap_single(rx_ring->dev, buffer_info->dma,
44390ca6 5944 IGB_RX_HDR_LEN,
59d71989 5945 DMA_FROM_DEVICE);
91615f76 5946 buffer_info->dma = 0;
bf36c1a0
AD		 5947 }
5948
16eb8815
AD		 5949 if (rx_desc->wb.upper.length) {
5950 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
bf36c1a0 5951
aa913403 5952 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
bf36c1a0
AD		 5953 buffer_info->page,
5954 buffer_info->page_offset,
5955 length);
5956
16eb8815
AD		 5957 skb->len += length;
5958 skb->data_len += length;
5959 skb->truesize += length;
5960
d1eff350
AD		 5961 if ((page_count(buffer_info->page) != 1) ||
5962 (page_to_nid(buffer_info->page) != current_node))
bf36c1a0
AD		 5963 buffer_info->page = NULL;
5964 else
5965 get_page(buffer_info->page);
9d5c8243 5966
16eb8815
AD		 5967 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
5968 PAGE_SIZE / 2, DMA_FROM_DEVICE);
5969 buffer_info->page_dma = 0;
9d5c8243 5970 }
9d5c8243 5971
bf36c1a0 5972 if (!(staterr & E1000_RXD_STAT_EOP)) {
06034649
AD		 5973 struct igb_rx_buffer *next_buffer;
5974 next_buffer = &rx_ring->rx_buffer_info[i];
b2d56536
AD		 5975 buffer_info->skb = next_buffer->skb;
5976 buffer_info->dma = next_buffer->dma;
5977 next_buffer->skb = skb;
5978 next_buffer->dma = 0;
bf36c1a0
AD		 5979 goto next_desc;
5980 }
44390ca6 5981
9d5c8243 5982 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
16eb8815 5983 dev_kfree_skb_any(skb);
9d5c8243
AK		 5984 goto next_desc;
5985 }
9d5c8243 5986
757b77e2
NN		 5987 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5988 igb_rx_hwtstamp(q_vector, staterr, skb);
9d5c8243
AK		 5989 total_bytes += skb->len;
5990 total_packets++;
5991
cd392f5c 5992 igb_rx_checksum(rx_ring, staterr, skb);
9d5c8243 5993
16eb8815 5994 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
047e0030 5995
b2cb09b1
JP		 5996 if (staterr & E1000_RXD_STAT_VP) {
5997 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
9d5c8243 5998
b2cb09b1
JP		 5999 __vlan_hwaccel_put_tag(skb, vid);
6000 }
6001 napi_gro_receive(&q_vector->napi, skb);
9d5c8243 6002
16eb8815 6003 budget--;
9d5c8243 6004next_desc:
16eb8815
AD		 6005 if (!budget)
6006 break;
6007
6008 cleaned_count++;
9d5c8243
AK		 6009 /* return some buffers to hardware, one at a time is too slow */
6010 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
cd392f5c 6011 igb_alloc_rx_buffers(rx_ring, cleaned_count);
9d5c8243
AK		 6012 cleaned_count = 0;
6013 }
6014
6015 /* use prefetched values */
6016 rx_desc = next_rxd;
9d5c8243
AK		 6017 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
6018 }
bf36c1a0 6019
9d5c8243 6020 rx_ring->next_to_clean = i;
12dcd86b 6021 u64_stats_update_begin(&rx_ring->rx_syncp);
9d5c8243
AK		 6022 rx_ring->rx_stats.packets += total_packets;
6023 rx_ring->rx_stats.bytes += total_bytes;
12dcd86b 6024 u64_stats_update_end(&rx_ring->rx_syncp);
c023cd88
AD		 6025 rx_ring->total_packets += total_packets;
6026 rx_ring->total_bytes += total_bytes;
6027
6028 if (cleaned_count)
cd392f5c 6029 igb_alloc_rx_buffers(rx_ring, cleaned_count);
c023cd88 6030
16eb8815 6031 return !!budget;
9d5c8243
AK		 6032}
6033
c023cd88 6034static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
06034649 6035 struct igb_rx_buffer *bi)
c023cd88
AD		 6036{
6037 struct sk_buff *skb = bi->skb;
6038 dma_addr_t dma = bi->dma;
6039
6040 if (dma)
6041 return true;
6042
6043 if (likely(!skb)) {
6044 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6045 IGB_RX_HDR_LEN);
6046 bi->skb = skb;
6047 if (!skb) {
6048 rx_ring->rx_stats.alloc_failed++;
6049 return false;
6050 }
6051
6052 /* initialize skb for ring */
6053 skb_record_rx_queue(skb, rx_ring->queue_index);
6054 }
6055
6056 dma = dma_map_single(rx_ring->dev, skb->data,
6057 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6058
6059 if (dma_mapping_error(rx_ring->dev, dma)) {
6060 rx_ring->rx_stats.alloc_failed++;
6061 return false;
6062 }
6063
6064 bi->dma = dma;
6065 return true;
6066}
6067
6068static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
06034649 6069 struct igb_rx_buffer *bi)
c023cd88
AD		 6070{
6071 struct page *page = bi->page;
6072 dma_addr_t page_dma = bi->page_dma;
6073 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6074
6075 if (page_dma)
6076 return true;
6077
6078 if (!page) {
6079 page = netdev_alloc_page(rx_ring->netdev);
6080 bi->page = page;
6081 if (unlikely(!page)) {
6082 rx_ring->rx_stats.alloc_failed++;
6083 return false;
6084 }
6085 }
6086
6087 page_dma = dma_map_page(rx_ring->dev, page,
6088 page_offset, PAGE_SIZE / 2,
6089 DMA_FROM_DEVICE);
6090
6091 if (dma_mapping_error(rx_ring->dev, page_dma)) {
6092 rx_ring->rx_stats.alloc_failed++;
6093 return false;
6094 }
6095
6096 bi->page_dma = page_dma;
6097 bi->page_offset = page_offset;
6098 return true;
6099}
6100
9d5c8243 6101/**
cd392f5c 6102 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
9d5c8243
AK		 6103 * @adapter: address of board private structure
6104 **/
cd392f5c 6105void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9d5c8243 6106{
9d5c8243 6107 union e1000_adv_rx_desc *rx_desc;
06034649 6108 struct igb_rx_buffer *bi;
c023cd88 6109 u16 i = rx_ring->next_to_use;
9d5c8243 6110
60136906 6111 rx_desc = IGB_RX_DESC(rx_ring, i);
06034649 6112 bi = &rx_ring->rx_buffer_info[i];
c023cd88 6113 i -= rx_ring->count;
9d5c8243
AK		 6114
6115 while (cleaned_count--) {
c023cd88
AD		 6116 if (!igb_alloc_mapped_skb(rx_ring, bi))
6117 break;
9d5c8243 6118
c023cd88
AD		 6119 /* Refresh the desc even if buffer_addrs didn't change
6120 * because each write-back erases this info. */
6121 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
9d5c8243 6122
c023cd88
AD		 6123 if (!igb_alloc_mapped_page(rx_ring, bi))
6124 break;
6125
6126 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
9d5c8243 6127
c023cd88
AD		 6128 rx_desc++;
6129 bi++;
9d5c8243 6130 i++;
c023cd88 6131 if (unlikely(!i)) {
60136906 6132 rx_desc = IGB_RX_DESC(rx_ring, 0);
06034649 6133 bi = rx_ring->rx_buffer_info;
c023cd88
AD		 6134 i -= rx_ring->count;
6135 }
6136
6137 /* clear the hdr_addr for the next_to_use descriptor */
6138 rx_desc->read.hdr_addr = 0;
9d5c8243
AK		 6139 }
6140
c023cd88
AD		 6141 i += rx_ring->count;
6142
9d5c8243
AK		 6143 if (rx_ring->next_to_use != i) {
6144 rx_ring->next_to_use = i;
9d5c8243
AK		 6145
6146 /* Force memory writes to complete before letting h/w
6147 * know there are new descriptors to fetch. (Only
6148 * applicable for weak-ordered memory model archs,
6149 * such as IA-64). */
6150 wmb();
fce99e34 6151 writel(i, rx_ring->tail);
9d5c8243
AK		 6152 }
6153}
6154
6155/**
6156 * igb_mii_ioctl -
6157 * @netdev:
6158 * @ifreq:
6159 * @cmd:
6160 **/
6161static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6162{
6163 struct igb_adapter *adapter = netdev_priv(netdev);
6164 struct mii_ioctl_data *data = if_mii(ifr);
6165
6166 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6167 return -EOPNOTSUPP;
6168
6169 switch (cmd) {
6170 case SIOCGMIIPHY:
6171 data->phy_id = adapter->hw.phy.addr;
6172 break;
6173 case SIOCGMIIREG:
f5f4cf08
AD		 6174 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6175 &data->val_out))
9d5c8243
AK		 6176 return -EIO;
6177 break;
6178 case SIOCSMIIREG:
6179 default:
6180 return -EOPNOTSUPP;
6181 }
6182 return 0;
6183}
6184
c6cb090b
PO		 6185/**
6186 * igb_hwtstamp_ioctl - control hardware time stamping
6187 * @netdev:
6188 * @ifreq:
6189 * @cmd:
6190 *
33af6bcc
PO		 6191 * Outgoing time stamping can be enabled and disabled. Play nice and
6192 * disable it when requested, although it shouldn't case any overhead
6193 * when no packet needs it. At most one packet in the queue may be
6194 * marked for time stamping, otherwise it would be impossible to tell
6195 * for sure to which packet the hardware time stamp belongs.
6196 *
6197 * Incoming time stamping has to be configured via the hardware
6198 * filters. Not all combinations are supported, in particular event
6199 * type has to be specified. Matching the kind of event packet is
6200 * not supported, with the exception of "all V2 events regardless of
6201 * level 2 or 4".
6202 *
c6cb090b
PO		 6203 **/
6204static int igb_hwtstamp_ioctl(struct net_device *netdev,
6205 struct ifreq *ifr, int cmd)
6206{
33af6bcc
PO		 6207 struct igb_adapter *adapter = netdev_priv(netdev);
6208 struct e1000_hw *hw = &adapter->hw;
c6cb090b 6209 struct hwtstamp_config config;
c5b9bd5e
AD		 6210 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6211 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
33af6bcc 6212 u32 tsync_rx_cfg = 0;
c5b9bd5e
AD		 6213 bool is_l4 = false;
6214 bool is_l2 = false;
33af6bcc 6215 u32 regval;
c6cb090b
PO		 6216
6217 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6218 return -EFAULT;
6219
6220 /* reserved for future extensions */
6221 if (config.flags)
6222 return -EINVAL;
6223
33af6bcc
PO		 6224 switch (config.tx_type) {
6225 case HWTSTAMP_TX_OFF:
c5b9bd5e 6226 tsync_tx_ctl = 0;
33af6bcc 6227 case HWTSTAMP_TX_ON:
33af6bcc
PO		 6228 break;
6229 default:
6230 return -ERANGE;
6231 }
6232
6233 switch (config.rx_filter) {
6234 case HWTSTAMP_FILTER_NONE:
c5b9bd5e 6235 tsync_rx_ctl = 0;
33af6bcc
PO		 6236 break;
6237 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6238 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6239 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6240 case HWTSTAMP_FILTER_ALL:
6241 /*
6242 * register TSYNCRXCFG must be set, therefore it is not
6243 * possible to time stamp both Sync and Delay_Req messages
6244 * => fall back to time stamping all packets
6245 */
c5b9bd5e 6246 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
33af6bcc
PO		 6247 config.rx_filter = HWTSTAMP_FILTER_ALL;
6248 break;
6249 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
c5b9bd5e 6250 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6251 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
c5b9bd5e 6252 is_l4 = true;
33af6bcc
PO		 6253 break;
6254 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
c5b9bd5e 6255 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
33af6bcc 6256 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
c5b9bd5e 6257 is_l4 = true;
33af6bcc
PO		 6258 break;
6259 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6260 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
c5b9bd5e 6261 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6262 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
c5b9bd5e
AD		 6263 is_l2 = true;
6264 is_l4 = true;
33af6bcc
PO		 6265 config.rx_filter = HWTSTAMP_FILTER_SOME;
6266 break;
6267 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6268 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
c5b9bd5e 6269 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
33af6bcc 6270 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
c5b9bd5e
AD		 6271 is_l2 = true;
6272 is_l4 = true;
33af6bcc
PO		 6273 config.rx_filter = HWTSTAMP_FILTER_SOME;
6274 break;
6275 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6276 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6277 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
c5b9bd5e 6278 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
33af6bcc 6279 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
c5b9bd5e 6280 is_l2 = true;
33af6bcc
PO		 6281 break;
6282 default:
6283 return -ERANGE;
6284 }
6285
c5b9bd5e
AD		 6286 if (hw->mac.type == e1000_82575) {
6287 if (tsync_rx_ctl | tsync_tx_ctl)
6288 return -EINVAL;
6289 return 0;
6290 }
6291
757b77e2
NN		 6292 /*
6293 * Per-packet timestamping only works if all packets are
6294 * timestamped, so enable timestamping in all packets as
6295 * long as one rx filter was configured.
6296 */
6297 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
6298 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6299 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6300 }
6301
33af6bcc
PO		 6302 /* enable/disable TX */
6303 regval = rd32(E1000_TSYNCTXCTL);
c5b9bd5e
AD		 6304 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6305 regval |= tsync_tx_ctl;
33af6bcc
PO		 6306 wr32(E1000_TSYNCTXCTL, regval);
6307
c5b9bd5e 6308 /* enable/disable RX */
33af6bcc 6309 regval = rd32(E1000_TSYNCRXCTL);
c5b9bd5e
AD		 6310 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6311 regval |= tsync_rx_ctl;
33af6bcc 6312 wr32(E1000_TSYNCRXCTL, regval);
33af6bcc 6313
c5b9bd5e
AD		 6314 /* define which PTP packets are time stamped */
6315 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
33af6bcc 6316
c5b9bd5e
AD		 6317 /* define ethertype filter for timestamped packets */
6318 if (is_l2)
6319 wr32(E1000_ETQF(3),
6320 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6321 E1000_ETQF_1588 | /* enable timestamping */
6322 ETH_P_1588)); /* 1588 eth protocol type */
6323 else
6324 wr32(E1000_ETQF(3), 0);
6325
6326#define PTP_PORT 319
6327 /* L4 Queue Filter[3]: filter by destination port and protocol */
6328 if (is_l4) {
6329 u32 ftqf = (IPPROTO_UDP /* UDP */
6330 | E1000_FTQF_VF_BP /* VF not compared */
6331 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6332 | E1000_FTQF_MASK); /* mask all inputs */
6333 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6334
6335 wr32(E1000_IMIR(3), htons(PTP_PORT));
6336 wr32(E1000_IMIREXT(3),
6337 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6338 if (hw->mac.type == e1000_82576) {
6339 /* enable source port check */
6340 wr32(E1000_SPQF(3), htons(PTP_PORT));
6341 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6342 }
6343 wr32(E1000_FTQF(3), ftqf);
6344 } else {
6345 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6346 }
33af6bcc
PO		 6347 wrfl();
6348
6349 adapter->hwtstamp_config = config;
6350
6351 /* clear TX/RX time stamp registers, just to be sure */
6352 regval = rd32(E1000_TXSTMPH);
6353 regval = rd32(E1000_RXSTMPH);
c6cb090b 6354
33af6bcc
PO		 6355 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6356 -EFAULT : 0;
c6cb090b
PO		 6357}
6358
9d5c8243
AK		 6359/**
6360 * igb_ioctl -
6361 * @netdev:
6362 * @ifreq:
6363 * @cmd:
6364 **/
6365static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6366{
6367 switch (cmd) {
6368 case SIOCGMIIPHY:
6369 case SIOCGMIIREG:
6370 case SIOCSMIIREG:
6371 return igb_mii_ioctl(netdev, ifr, cmd);
c6cb090b
PO		 6372 case SIOCSHWTSTAMP:
6373 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
9d5c8243
AK		 6374 default:
6375 return -EOPNOTSUPP;
6376 }
6377}
6378
009bc06e
AD		 6379s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6380{
6381 struct igb_adapter *adapter = hw->back;
6382 u16 cap_offset;
6383
bdaae04c 6384 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6385 if (!cap_offset)
6386 return -E1000_ERR_CONFIG;
6387
6388 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6389
6390 return 0;
6391}
6392
6393s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6394{
6395 struct igb_adapter *adapter = hw->back;
6396 u16 cap_offset;
6397
bdaae04c 6398 cap_offset = adapter->pdev->pcie_cap;
009bc06e
AD		 6399 if (!cap_offset)
6400 return -E1000_ERR_CONFIG;
6401
6402 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6403
6404 return 0;
6405}
6406
b2cb09b1 6407static void igb_vlan_mode(struct net_device *netdev, u32 features)
9d5c8243
AK		 6408{
6409 struct igb_adapter *adapter = netdev_priv(netdev);
6410 struct e1000_hw *hw = &adapter->hw;
6411 u32 ctrl, rctl;
6412
6413 igb_irq_disable(adapter);
9d5c8243 6414
b2cb09b1 6415 if (features & NETIF_F_HW_VLAN_RX) {
9d5c8243
AK		 6416 /* enable VLAN tag insert/strip */
6417 ctrl = rd32(E1000_CTRL);
6418 ctrl |= E1000_CTRL_VME;
6419 wr32(E1000_CTRL, ctrl);
6420
51466239 6421 /* Disable CFI check */
9d5c8243 6422 rctl = rd32(E1000_RCTL);
9d5c8243
AK		 6423 rctl &= ~E1000_RCTL_CFIEN;
6424 wr32(E1000_RCTL, rctl);
9d5c8243
AK		 6425 } else {
6426 /* disable VLAN tag insert/strip */
6427 ctrl = rd32(E1000_CTRL);
6428 ctrl &= ~E1000_CTRL_VME;
6429 wr32(E1000_CTRL, ctrl);
9d5c8243
AK		 6430 }
6431
e1739522
AD		 6432 igb_rlpml_set(adapter);
6433
9d5c8243
AK		 6434 if (!test_bit(__IGB_DOWN, &adapter->state))
6435 igb_irq_enable(adapter);
6436}
6437
6438static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6439{
6440 struct igb_adapter *adapter = netdev_priv(netdev);
6441 struct e1000_hw *hw = &adapter->hw;
4ae196df 6442 int pf_id = adapter->vfs_allocated_count;
9d5c8243 6443
51466239
AD		 6444 /* attempt to add filter to vlvf array */
6445 igb_vlvf_set(adapter, vid, true, pf_id);
4ae196df 6446
51466239
AD		 6447 /* add the filter since PF can receive vlans w/o entry in vlvf */
6448 igb_vfta_set(hw, vid, true);
b2cb09b1
JP		 6449
6450 set_bit(vid, adapter->active_vlans);
9d5c8243
AK		 6451}
6452
6453static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6454{
6455 struct igb_adapter *adapter = netdev_priv(netdev);
6456 struct e1000_hw *hw = &adapter->hw;
4ae196df 6457 int pf_id = adapter->vfs_allocated_count;
51466239 6458 s32 err;
9d5c8243
AK		 6459
6460 igb_irq_disable(adapter);
9d5c8243
AK		 6461
6462 if (!test_bit(__IGB_DOWN, &adapter->state))
6463 igb_irq_enable(adapter);
6464
51466239
AD		 6465 /* remove vlan from VLVF table array */
6466 err = igb_vlvf_set(adapter, vid, false, pf_id);
9d5c8243 6467
51466239
AD		 6468 /* if vid was not present in VLVF just remove it from table */
6469 if (err)
4ae196df 6470 igb_vfta_set(hw, vid, false);
b2cb09b1
JP		 6471
6472 clear_bit(vid, adapter->active_vlans);
9d5c8243
AK		 6473}
6474
6475static void igb_restore_vlan(struct igb_adapter *adapter)
6476{
b2cb09b1 6477 u16 vid;
9d5c8243 6478
b2cb09b1
JP		 6479 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6480 igb_vlan_rx_add_vid(adapter->netdev, vid);
9d5c8243
AK		 6481}
6482
14ad2513 6483int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9d5c8243 6484{
090b1795 6485 struct pci_dev *pdev = adapter->pdev;
9d5c8243
AK		 6486 struct e1000_mac_info *mac = &adapter->hw.mac;
6487
6488 mac->autoneg = 0;
6489
14ad2513
DD		 6490 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6491 * for the switch() below to work */
6492 if ((spd & 1) || (dplx & ~1))
6493 goto err_inval;
6494
cd2638a8
CW		 6495 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6496 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
14ad2513
DD		 6497 spd != SPEED_1000 &&
6498 dplx != DUPLEX_FULL)
6499 goto err_inval;
cd2638a8 6500
14ad2513 6501 switch (spd + dplx) {
9d5c8243
AK		 6502 case SPEED_10 + DUPLEX_HALF:
6503 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6504 break;
6505 case SPEED_10 + DUPLEX_FULL:
6506 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6507 break;
6508 case SPEED_100 + DUPLEX_HALF:
6509 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6510 break;
6511 case SPEED_100 + DUPLEX_FULL:
6512 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6513 break;
6514 case SPEED_1000 + DUPLEX_FULL:
6515 mac->autoneg = 1;
6516 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6517 break;
6518 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6519 default:
14ad2513 6520 goto err_inval;
9d5c8243
AK		 6521 }
6522 return 0;
14ad2513
DD		 6523
6524err_inval:
6525 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6526 return -EINVAL;
9d5c8243
AK		 6527}
6528
3fe7c4c9 6529static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
9d5c8243
AK		 6530{
6531 struct net_device *netdev = pci_get_drvdata(pdev);
6532 struct igb_adapter *adapter = netdev_priv(netdev);
6533 struct e1000_hw *hw = &adapter->hw;
2d064c06 6534 u32 ctrl, rctl, status;
9d5c8243
AK		 6535 u32 wufc = adapter->wol;
6536#ifdef CONFIG_PM
6537 int retval = 0;
6538#endif
6539
6540 netif_device_detach(netdev);
6541
a88f10ec
AD		 6542 if (netif_running(netdev))
6543 igb_close(netdev);
6544
047e0030 6545 igb_clear_interrupt_scheme(adapter);
9d5c8243
AK		 6546
6547#ifdef CONFIG_PM
6548 retval = pci_save_state(pdev);
6549 if (retval)
6550 return retval;
6551#endif
6552
6553 status = rd32(E1000_STATUS);
6554 if (status & E1000_STATUS_LU)
6555 wufc &= ~E1000_WUFC_LNKC;
6556
6557 if (wufc) {
6558 igb_setup_rctl(adapter);
ff41f8dc 6559 igb_set_rx_mode(netdev);
9d5c8243
AK		 6560
6561 /* turn on all-multi mode if wake on multicast is enabled */
6562 if (wufc & E1000_WUFC_MC) {
6563 rctl = rd32(E1000_RCTL);
6564 rctl |= E1000_RCTL_MPE;
6565 wr32(E1000_RCTL, rctl);
6566 }
6567
6568 ctrl = rd32(E1000_CTRL);
6569 /* advertise wake from D3Cold */
6570 #define E1000_CTRL_ADVD3WUC 0x00100000
6571 /* phy power management enable */
6572 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6573 ctrl |= E1000_CTRL_ADVD3WUC;
6574 wr32(E1000_CTRL, ctrl);
6575
9d5c8243 6576 /* Allow time for pending master requests to run */
330a6d6a 6577 igb_disable_pcie_master(hw);
9d5c8243
AK		 6578
6579 wr32(E1000_WUC, E1000_WUC_PME_EN);
6580 wr32(E1000_WUFC, wufc);
9d5c8243
AK		 6581 } else {
6582 wr32(E1000_WUC, 0);
6583 wr32(E1000_WUFC, 0);
9d5c8243
AK		 6584 }
6585
3fe7c4c9
RW		 6586 *enable_wake = wufc || adapter->en_mng_pt;
6587 if (!*enable_wake)
88a268c1
NN		 6588 igb_power_down_link(adapter);
6589 else
6590 igb_power_up_link(adapter);
9d5c8243
AK		 6591
6592 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6593 * would have already happened in close and is redundant. */
6594 igb_release_hw_control(adapter);
6595
6596 pci_disable_device(pdev);
6597
9d5c8243
AK		 6598 return 0;
6599}
6600
6601#ifdef CONFIG_PM
3fe7c4c9
RW		 6602static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6603{
6604 int retval;
6605 bool wake;
6606
6607 retval = __igb_shutdown(pdev, &wake);
6608 if (retval)
6609 return retval;
6610
6611 if (wake) {
6612 pci_prepare_to_sleep(pdev);
6613 } else {
6614 pci_wake_from_d3(pdev, false);
6615 pci_set_power_state(pdev, PCI_D3hot);
6616 }
6617
6618 return 0;
6619}
6620
9d5c8243
AK		 6621static int igb_resume(struct pci_dev *pdev)
6622{
6623 struct net_device *netdev = pci_get_drvdata(pdev);
6624 struct igb_adapter *adapter = netdev_priv(netdev);
6625 struct e1000_hw *hw = &adapter->hw;
6626 u32 err;
6627
6628 pci_set_power_state(pdev, PCI_D0);
6629 pci_restore_state(pdev);
b94f2d77 6630 pci_save_state(pdev);
42bfd33a 6631
aed5dec3 6632 err = pci_enable_device_mem(pdev);
9d5c8243
AK		 6633 if (err) {
6634 dev_err(&pdev->dev,
6635 "igb: Cannot enable PCI device from suspend\n");
6636 return err;
6637 }
6638 pci_set_master(pdev);
6639
6640 pci_enable_wake(pdev, PCI_D3hot, 0);
6641 pci_enable_wake(pdev, PCI_D3cold, 0);
6642
047e0030 6643 if (igb_init_interrupt_scheme(adapter)) {
a88f10ec
AD		 6644 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6645 return -ENOMEM;
9d5c8243
AK		 6646 }
6647
9d5c8243 6648 igb_reset(adapter);
a8564f03
AD		 6649
6650 /* let the f/w know that the h/w is now under the control of the
6651 * driver. */
6652 igb_get_hw_control(adapter);
6653
9d5c8243
AK		 6654 wr32(E1000_WUS, ~0);
6655
a88f10ec
AD		 6656 if (netif_running(netdev)) {
6657 err = igb_open(netdev);
6658 if (err)
6659 return err;
6660 }
9d5c8243
AK		 6661
6662 netif_device_attach(netdev);
6663
9d5c8243
AK		 6664 return 0;
6665}
6666#endif
6667
6668static void igb_shutdown(struct pci_dev *pdev)
6669{
3fe7c4c9
RW		 6670 bool wake;
6671
6672 __igb_shutdown(pdev, &wake);
6673
6674 if (system_state == SYSTEM_POWER_OFF) {
6675 pci_wake_from_d3(pdev, wake);
6676 pci_set_power_state(pdev, PCI_D3hot);
6677 }
9d5c8243
AK		 6678}
6679
6680#ifdef CONFIG_NET_POLL_CONTROLLER
6681/*
6682 * Polling 'interrupt' - used by things like netconsole to send skbs
6683 * without having to re-enable interrupts. It's not called while
6684 * the interrupt routine is executing.
6685 */
6686static void igb_netpoll(struct net_device *netdev)
6687{
6688 struct igb_adapter *adapter = netdev_priv(netdev);
eebbbdba 6689 struct e1000_hw *hw = &adapter->hw;
9d5c8243 6690 int i;
9d5c8243 6691
eebbbdba 6692 if (!adapter->msix_entries) {
047e0030 6693 struct igb_q_vector *q_vector = adapter->q_vector[0];
eebbbdba 6694 igb_irq_disable(adapter);
047e0030 6695 napi_schedule(&q_vector->napi);
eebbbdba
AD		 6696 return;
6697 }
9d5c8243 6698
047e0030
AD		 6699 for (i = 0; i < adapter->num_q_vectors; i++) {
6700 struct igb_q_vector *q_vector = adapter->q_vector[i];
6701 wr32(E1000_EIMC, q_vector->eims_value);
6702 napi_schedule(&q_vector->napi);
eebbbdba 6703 }
9d5c8243
AK		 6704}
6705#endif /* CONFIG_NET_POLL_CONTROLLER */
6706
6707/**
6708 * igb_io_error_detected - called when PCI error is detected
6709 * @pdev: Pointer to PCI device
6710 * @state: The current pci connection state
6711 *
6712 * This function is called after a PCI bus error affecting
6713 * this device has been detected.
6714 */
6715static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6716 pci_channel_state_t state)
6717{
6718 struct net_device *netdev = pci_get_drvdata(pdev);
6719 struct igb_adapter *adapter = netdev_priv(netdev);
6720
6721 netif_device_detach(netdev);
6722
59ed6eec
AD		 6723 if (state == pci_channel_io_perm_failure)
6724 return PCI_ERS_RESULT_DISCONNECT;
6725
9d5c8243
AK		 6726 if (netif_running(netdev))
6727 igb_down(adapter);
6728 pci_disable_device(pdev);
6729
6730 /* Request a slot slot reset. */
6731 return PCI_ERS_RESULT_NEED_RESET;
6732}
6733
6734/**
6735 * igb_io_slot_reset - called after the pci bus has been reset.
6736 * @pdev: Pointer to PCI device
6737 *
6738 * Restart the card from scratch, as if from a cold-boot. Implementation
6739 * resembles the first-half of the igb_resume routine.
6740 */
6741static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6742{
6743 struct net_device *netdev = pci_get_drvdata(pdev);
6744 struct igb_adapter *adapter = netdev_priv(netdev);
6745 struct e1000_hw *hw = &adapter->hw;
40a914fa 6746 pci_ers_result_t result;
42bfd33a 6747 int err;
9d5c8243 6748
aed5dec3 6749 if (pci_enable_device_mem(pdev)) {
9d5c8243
AK		 6750 dev_err(&pdev->dev,
6751 "Cannot re-enable PCI device after reset.\n");
40a914fa
AD		 6752 result = PCI_ERS_RESULT_DISCONNECT;
6753 } else {
6754 pci_set_master(pdev);
6755 pci_restore_state(pdev);
b94f2d77 6756 pci_save_state(pdev);
9d5c8243 6757
40a914fa
AD		 6758 pci_enable_wake(pdev, PCI_D3hot, 0);
6759 pci_enable_wake(pdev, PCI_D3cold, 0);
9d5c8243 6760
40a914fa
AD		 6761 igb_reset(adapter);
6762 wr32(E1000_WUS, ~0);
6763 result = PCI_ERS_RESULT_RECOVERED;
6764 }
9d5c8243 6765
ea943d41
JK		 6766 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6767 if (err) {
6768 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6769 "failed 0x%0x\n", err);
6770 /* non-fatal, continue */
6771 }
40a914fa
AD		 6772
6773 return result;
9d5c8243
AK		 6774}
6775
6776/**
6777 * igb_io_resume - called when traffic can start flowing again.
6778 * @pdev: Pointer to PCI device
6779 *
6780 * This callback is called when the error recovery driver tells us that
6781 * its OK to resume normal operation. Implementation resembles the
6782 * second-half of the igb_resume routine.
6783 */
6784static void igb_io_resume(struct pci_dev *pdev)
6785{
6786 struct net_device *netdev = pci_get_drvdata(pdev);
6787 struct igb_adapter *adapter = netdev_priv(netdev);
6788
9d5c8243
AK		 6789 if (netif_running(netdev)) {
6790 if (igb_up(adapter)) {
6791 dev_err(&pdev->dev, "igb_up failed after reset\n");
6792 return;
6793 }
6794 }
6795
6796 netif_device_attach(netdev);
6797
6798 /* let the f/w know that the h/w is now under the control of the
6799 * driver. */
6800 igb_get_hw_control(adapter);
9d5c8243
AK		 6801}
6802
26ad9178
AD		 6803static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6804 u8 qsel)
6805{
6806 u32 rar_low, rar_high;
6807 struct e1000_hw *hw = &adapter->hw;
6808
6809 /* HW expects these in little endian so we reverse the byte order
6810 * from network order (big endian) to little endian
6811 */
6812 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6813 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6814 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6815
6816 /* Indicate to hardware the Address is Valid. */
6817 rar_high |= E1000_RAH_AV;
6818
6819 if (hw->mac.type == e1000_82575)
6820 rar_high |= E1000_RAH_POOL_1 * qsel;
6821 else
6822 rar_high |= E1000_RAH_POOL_1 << qsel;
6823
6824 wr32(E1000_RAL(index), rar_low);
6825 wrfl();
6826 wr32(E1000_RAH(index), rar_high);
6827 wrfl();
6828}
6829
4ae196df
AD		 6830static int igb_set_vf_mac(struct igb_adapter *adapter,
6831 int vf, unsigned char *mac_addr)
6832{
6833 struct e1000_hw *hw = &adapter->hw;
ff41f8dc
AD		 6834 /* VF MAC addresses start at end of receive addresses and moves
6835 * torwards the first, as a result a collision should not be possible */
6836 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4ae196df 6837
37680117 6838 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
4ae196df 6839
26ad9178 6840 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
4ae196df
AD		 6841
6842 return 0;
6843}
6844
8151d294
WM		 6845static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6846{
6847 struct igb_adapter *adapter = netdev_priv(netdev);
6848 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6849 return -EINVAL;
6850 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6851 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6852 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6853 " change effective.");
6854 if (test_bit(__IGB_DOWN, &adapter->state)) {
6855 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6856 " but the PF device is not up.\n");
6857 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6858 " attempting to use the VF device.\n");
6859 }
6860 return igb_set_vf_mac(adapter, vf, mac);
6861}
6862
17dc566c
LL		 6863static int igb_link_mbps(int internal_link_speed)
6864{
6865 switch (internal_link_speed) {
6866 case SPEED_100:
6867 return 100;
6868 case SPEED_1000:
6869 return 1000;
6870 default:
6871 return 0;
6872 }
6873}
6874
6875static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6876 int link_speed)
6877{
6878 int rf_dec, rf_int;
6879 u32 bcnrc_val;
6880
6881 if (tx_rate != 0) {
6882 /* Calculate the rate factor values to set */
6883 rf_int = link_speed / tx_rate;
6884 rf_dec = (link_speed - (rf_int * tx_rate));
6885 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6886
6887 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6888 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6889 E1000_RTTBCNRC_RF_INT_MASK);
6890 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6891 } else {
6892 bcnrc_val = 0;
6893 }
6894
6895 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
6896 wr32(E1000_RTTBCNRC, bcnrc_val);
6897}
6898
6899static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
6900{
6901 int actual_link_speed, i;
6902 bool reset_rate = false;
6903
6904 /* VF TX rate limit was not set or not supported */
6905 if ((adapter->vf_rate_link_speed == 0) ||
6906 (adapter->hw.mac.type != e1000_82576))
6907 return;
6908
6909 actual_link_speed = igb_link_mbps(adapter->link_speed);
6910 if (actual_link_speed != adapter->vf_rate_link_speed) {
6911 reset_rate = true;
6912 adapter->vf_rate_link_speed = 0;
6913 dev_info(&adapter->pdev->dev,
6914 "Link speed has been changed. VF Transmit "
6915 "rate is disabled\n");
6916 }
6917
6918 for (i = 0; i < adapter->vfs_allocated_count; i++) {
6919 if (reset_rate)
6920 adapter->vf_data[i].tx_rate = 0;
6921
6922 igb_set_vf_rate_limit(&adapter->hw, i,
6923 adapter->vf_data[i].tx_rate,
6924 actual_link_speed);
6925 }
6926}
6927
8151d294
WM		 6928static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6929{
17dc566c
LL		 6930 struct igb_adapter *adapter = netdev_priv(netdev);
6931 struct e1000_hw *hw = &adapter->hw;
6932 int actual_link_speed;
6933
6934 if (hw->mac.type != e1000_82576)
6935 return -EOPNOTSUPP;
6936
6937 actual_link_speed = igb_link_mbps(adapter->link_speed);
6938 if ((vf >= adapter->vfs_allocated_count) ||
6939 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
6940 (tx_rate < 0) || (tx_rate > actual_link_speed))
6941 return -EINVAL;
6942
6943 adapter->vf_rate_link_speed = actual_link_speed;
6944 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
6945 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
6946
6947 return 0;
8151d294
WM		 6948}
6949
6950static int igb_ndo_get_vf_config(struct net_device *netdev,
6951 int vf, struct ifla_vf_info *ivi)
6952{
6953 struct igb_adapter *adapter = netdev_priv(netdev);
6954 if (vf >= adapter->vfs_allocated_count)
6955 return -EINVAL;
6956 ivi->vf = vf;
6957 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
17dc566c 6958 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8151d294
WM		 6959 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6960 ivi->qos = adapter->vf_data[vf].pf_qos;
6961 return 0;
6962}
6963
4ae196df
AD		 6964static void igb_vmm_control(struct igb_adapter *adapter)
6965{
6966 struct e1000_hw *hw = &adapter->hw;
10d8e907 6967 u32 reg;
4ae196df 6968
52a1dd4d
AD		 6969 switch (hw->mac.type) {
6970 case e1000_82575:
6971 default:
6972 /* replication is not supported for 82575 */
4ae196df 6973 return;
52a1dd4d
AD		 6974 case e1000_82576:
6975 /* notify HW that the MAC is adding vlan tags */
6976 reg = rd32(E1000_DTXCTL);
6977 reg |= E1000_DTXCTL_VLAN_ADDED;
6978 wr32(E1000_DTXCTL, reg);
6979 case e1000_82580:
6980 /* enable replication vlan tag stripping */
6981 reg = rd32(E1000_RPLOLR);
6982 reg |= E1000_RPLOLR_STRVLAN;
6983 wr32(E1000_RPLOLR, reg);
d2ba2ed8
AD		 6984 case e1000_i350:
6985 /* none of the above registers are supported by i350 */
52a1dd4d
AD		 6986 break;
6987 }
10d8e907 6988
d4960307
AD		 6989 if (adapter->vfs_allocated_count) {
6990 igb_vmdq_set_loopback_pf(hw, true);
6991 igb_vmdq_set_replication_pf(hw, true);
13800469
GR		 6992 igb_vmdq_set_anti_spoofing_pf(hw, true,
6993 adapter->vfs_allocated_count);
d4960307
AD		 6994 } else {
6995 igb_vmdq_set_loopback_pf(hw, false);
6996 igb_vmdq_set_replication_pf(hw, false);
6997 }
4ae196df
AD		 6998}
6999
9d5c8243 7000/* igb_main.c */
Linux kernel - Deliverables
This page took 1.127557 seconds and 5 git commands to generate.