1 /*******************************************************************************
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2012 Intel Corporation.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26 *******************************************************************************/
28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30 #include <linux/module.h>
31 #include <linux/types.h>
32 #include <linux/init.h>
33 #include <linux/bitops.h>
34 #include <linux/vmalloc.h>
35 #include <linux/pagemap.h>
36 #include <linux/netdevice.h>
37 #include <linux/ipv6.h>
38 #include <linux/slab.h>
39 #include <net/checksum.h>
40 #include <net/ip6_checksum.h>
41 #include <linux/net_tstamp.h>
42 #include <linux/mii.h>
43 #include <linux/ethtool.h>
45 #include <linux/if_vlan.h>
46 #include <linux/pci.h>
47 #include <linux/pci-aspm.h>
48 #include <linux/delay.h>
49 #include <linux/interrupt.h>
51 #include <linux/tcp.h>
52 #include <linux/sctp.h>
53 #include <linux/if_ether.h>
54 #include <linux/aer.h>
55 #include <linux/prefetch.h>
56 #include <linux/pm_runtime.h>
58 #include <linux/dca.h>
65 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
66 __stringify(BUILD) "-k"
67 char igb_driver_name
[] = "igb";
68 char igb_driver_version
[] = DRV_VERSION
;
69 static const char igb_driver_string
[] =
70 "Intel(R) Gigabit Ethernet Network Driver";
71 static const char igb_copyright
[] = "Copyright (c) 2007-2012 Intel Corporation.";
73 static const struct e1000_info
*igb_info_tbl
[] = {
74 [board_82575
] = &e1000_82575_info
,
77 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl
) = {
78 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I211_COPPER
), board_82575
},
79 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I210_COPPER
), board_82575
},
80 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I210_FIBER
), board_82575
},
81 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I210_SERDES
), board_82575
},
82 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I210_SGMII
), board_82575
},
83 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I350_COPPER
), board_82575
},
84 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I350_FIBER
), board_82575
},
85 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I350_SERDES
), board_82575
},
86 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_I350_SGMII
), board_82575
},
87 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82580_COPPER
), board_82575
},
88 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82580_FIBER
), board_82575
},
89 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82580_QUAD_FIBER
), board_82575
},
90 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82580_SERDES
), board_82575
},
91 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82580_SGMII
), board_82575
},
92 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82580_COPPER_DUAL
), board_82575
},
93 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_DH89XXCC_SGMII
), board_82575
},
94 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_DH89XXCC_SERDES
), board_82575
},
95 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_DH89XXCC_BACKPLANE
), board_82575
},
96 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_DH89XXCC_SFP
), board_82575
},
97 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576
), board_82575
},
98 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_NS
), board_82575
},
99 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_NS_SERDES
), board_82575
},
100 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_FIBER
), board_82575
},
101 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_SERDES
), board_82575
},
102 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_SERDES_QUAD
), board_82575
},
103 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_QUAD_COPPER_ET2
), board_82575
},
104 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82576_QUAD_COPPER
), board_82575
},
105 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82575EB_COPPER
), board_82575
},
106 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82575EB_FIBER_SERDES
), board_82575
},
107 { PCI_VDEVICE(INTEL
, E1000_DEV_ID_82575GB_QUAD_COPPER
), board_82575
},
108 /* required last entry */
112 MODULE_DEVICE_TABLE(pci
, igb_pci_tbl
);
114 void igb_reset(struct igb_adapter
*);
115 static int igb_setup_all_tx_resources(struct igb_adapter
*);
116 static int igb_setup_all_rx_resources(struct igb_adapter
*);
117 static void igb_free_all_tx_resources(struct igb_adapter
*);
118 static void igb_free_all_rx_resources(struct igb_adapter
*);
119 static void igb_setup_mrqc(struct igb_adapter
*);
120 static int igb_probe(struct pci_dev
*, const struct pci_device_id
*);
121 static void __devexit
igb_remove(struct pci_dev
*pdev
);
122 static int igb_sw_init(struct igb_adapter
*);
123 static int igb_open(struct net_device
*);
124 static int igb_close(struct net_device
*);
125 static void igb_configure_tx(struct igb_adapter
*);
126 static void igb_configure_rx(struct igb_adapter
*);
127 static void igb_clean_all_tx_rings(struct igb_adapter
*);
128 static void igb_clean_all_rx_rings(struct igb_adapter
*);
129 static void igb_clean_tx_ring(struct igb_ring
*);
130 static void igb_clean_rx_ring(struct igb_ring
*);
131 static void igb_set_rx_mode(struct net_device
*);
132 static void igb_update_phy_info(unsigned long);
133 static void igb_watchdog(unsigned long);
134 static void igb_watchdog_task(struct work_struct
*);
135 static netdev_tx_t
igb_xmit_frame(struct sk_buff
*skb
, struct net_device
*);
136 static struct rtnl_link_stats64
*igb_get_stats64(struct net_device
*dev
,
137 struct rtnl_link_stats64
*stats
);
138 static int igb_change_mtu(struct net_device
*, int);
139 static int igb_set_mac(struct net_device
*, void *);
140 static void igb_set_uta(struct igb_adapter
*adapter
);
141 static irqreturn_t
igb_intr(int irq
, void *);
142 static irqreturn_t
igb_intr_msi(int irq
, void *);
143 static irqreturn_t
igb_msix_other(int irq
, void *);
144 static irqreturn_t
igb_msix_ring(int irq
, void *);
145 #ifdef CONFIG_IGB_DCA
146 static void igb_update_dca(struct igb_q_vector
*);
147 static void igb_setup_dca(struct igb_adapter
*);
148 #endif /* CONFIG_IGB_DCA */
149 static int igb_poll(struct napi_struct
*, int);
150 static bool igb_clean_tx_irq(struct igb_q_vector
*);
151 static bool igb_clean_rx_irq(struct igb_q_vector
*, int);
152 static int igb_ioctl(struct net_device
*, struct ifreq
*, int cmd
);
153 static void igb_tx_timeout(struct net_device
*);
154 static void igb_reset_task(struct work_struct
*);
155 static void igb_vlan_mode(struct net_device
*netdev
, netdev_features_t features
);
156 static int igb_vlan_rx_add_vid(struct net_device
*, u16
);
157 static int igb_vlan_rx_kill_vid(struct net_device
*, u16
);
158 static void igb_restore_vlan(struct igb_adapter
*);
159 static void igb_rar_set_qsel(struct igb_adapter
*, u8
*, u32
, u8
);
160 static void igb_ping_all_vfs(struct igb_adapter
*);
161 static void igb_msg_task(struct igb_adapter
*);
162 static void igb_vmm_control(struct igb_adapter
*);
163 static int igb_set_vf_mac(struct igb_adapter
*, int, unsigned char *);
164 static void igb_restore_vf_multicasts(struct igb_adapter
*adapter
);
165 static int igb_ndo_set_vf_mac(struct net_device
*netdev
, int vf
, u8
*mac
);
166 static int igb_ndo_set_vf_vlan(struct net_device
*netdev
,
167 int vf
, u16 vlan
, u8 qos
);
168 static int igb_ndo_set_vf_bw(struct net_device
*netdev
, int vf
, int tx_rate
);
169 static int igb_ndo_get_vf_config(struct net_device
*netdev
, int vf
,
170 struct ifla_vf_info
*ivi
);
171 static void igb_check_vf_rate_limit(struct igb_adapter
*);
173 #ifdef CONFIG_PCI_IOV
174 static int igb_vf_configure(struct igb_adapter
*adapter
, int vf
);
175 static int igb_find_enabled_vfs(struct igb_adapter
*adapter
);
176 static int igb_check_vf_assignment(struct igb_adapter
*adapter
);
180 #ifdef CONFIG_PM_SLEEP
181 static int igb_suspend(struct device
*);
183 static int igb_resume(struct device
*);
184 #ifdef CONFIG_PM_RUNTIME
185 static int igb_runtime_suspend(struct device
*dev
);
186 static int igb_runtime_resume(struct device
*dev
);
187 static int igb_runtime_idle(struct device
*dev
);
189 static const struct dev_pm_ops igb_pm_ops
= {
190 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend
, igb_resume
)
191 SET_RUNTIME_PM_OPS(igb_runtime_suspend
, igb_runtime_resume
,
195 static void igb_shutdown(struct pci_dev
*);
196 #ifdef CONFIG_IGB_DCA
197 static int igb_notify_dca(struct notifier_block
*, unsigned long, void *);
198 static struct notifier_block dca_notifier
= {
199 .notifier_call
= igb_notify_dca
,
204 #ifdef CONFIG_NET_POLL_CONTROLLER
205 /* for netdump / net console */
206 static void igb_netpoll(struct net_device
*);
208 #ifdef CONFIG_PCI_IOV
209 static unsigned int max_vfs
= 0;
210 module_param(max_vfs
, uint
, 0);
211 MODULE_PARM_DESC(max_vfs
, "Maximum number of virtual functions to allocate "
212 "per physical function");
213 #endif /* CONFIG_PCI_IOV */
215 static pci_ers_result_t
igb_io_error_detected(struct pci_dev
*,
216 pci_channel_state_t
);
217 static pci_ers_result_t
igb_io_slot_reset(struct pci_dev
*);
218 static void igb_io_resume(struct pci_dev
*);
220 static struct pci_error_handlers igb_err_handler
= {
221 .error_detected
= igb_io_error_detected
,
222 .slot_reset
= igb_io_slot_reset
,
223 .resume
= igb_io_resume
,
226 static void igb_init_dmac(struct igb_adapter
*adapter
, u32 pba
);
228 static struct pci_driver igb_driver
= {
229 .name
= igb_driver_name
,
230 .id_table
= igb_pci_tbl
,
232 .remove
= __devexit_p(igb_remove
),
234 .driver
.pm
= &igb_pm_ops
,
236 .shutdown
= igb_shutdown
,
237 .err_handler
= &igb_err_handler
240 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
241 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
242 MODULE_LICENSE("GPL");
243 MODULE_VERSION(DRV_VERSION
);
245 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
246 static int debug
= -1;
247 module_param(debug
, int, 0);
248 MODULE_PARM_DESC(debug
, "Debug level (0=none,...,16=all)");
250 struct igb_reg_info
{
255 static const struct igb_reg_info igb_reg_info_tbl
[] = {
257 /* General Registers */
258 {E1000_CTRL
, "CTRL"},
259 {E1000_STATUS
, "STATUS"},
260 {E1000_CTRL_EXT
, "CTRL_EXT"},
262 /* Interrupt Registers */
266 {E1000_RCTL
, "RCTL"},
267 {E1000_RDLEN(0), "RDLEN"},
268 {E1000_RDH(0), "RDH"},
269 {E1000_RDT(0), "RDT"},
270 {E1000_RXDCTL(0), "RXDCTL"},
271 {E1000_RDBAL(0), "RDBAL"},
272 {E1000_RDBAH(0), "RDBAH"},
275 {E1000_TCTL
, "TCTL"},
276 {E1000_TDBAL(0), "TDBAL"},
277 {E1000_TDBAH(0), "TDBAH"},
278 {E1000_TDLEN(0), "TDLEN"},
279 {E1000_TDH(0), "TDH"},
280 {E1000_TDT(0), "TDT"},
281 {E1000_TXDCTL(0), "TXDCTL"},
282 {E1000_TDFH
, "TDFH"},
283 {E1000_TDFT
, "TDFT"},
284 {E1000_TDFHS
, "TDFHS"},
285 {E1000_TDFPC
, "TDFPC"},
287 /* List Terminator */
292 * igb_regdump - register printout routine
294 static void igb_regdump(struct e1000_hw
*hw
, struct igb_reg_info
*reginfo
)
300 switch (reginfo
->ofs
) {
302 for (n
= 0; n
< 4; n
++)
303 regs
[n
] = rd32(E1000_RDLEN(n
));
306 for (n
= 0; n
< 4; n
++)
307 regs
[n
] = rd32(E1000_RDH(n
));
310 for (n
= 0; n
< 4; n
++)
311 regs
[n
] = rd32(E1000_RDT(n
));
313 case E1000_RXDCTL(0):
314 for (n
= 0; n
< 4; n
++)
315 regs
[n
] = rd32(E1000_RXDCTL(n
));
318 for (n
= 0; n
< 4; n
++)
319 regs
[n
] = rd32(E1000_RDBAL(n
));
322 for (n
= 0; n
< 4; n
++)
323 regs
[n
] = rd32(E1000_RDBAH(n
));
326 for (n
= 0; n
< 4; n
++)
327 regs
[n
] = rd32(E1000_RDBAL(n
));
330 for (n
= 0; n
< 4; n
++)
331 regs
[n
] = rd32(E1000_TDBAH(n
));
334 for (n
= 0; n
< 4; n
++)
335 regs
[n
] = rd32(E1000_TDLEN(n
));
338 for (n
= 0; n
< 4; n
++)
339 regs
[n
] = rd32(E1000_TDH(n
));
342 for (n
= 0; n
< 4; n
++)
343 regs
[n
] = rd32(E1000_TDT(n
));
345 case E1000_TXDCTL(0):
346 for (n
= 0; n
< 4; n
++)
347 regs
[n
] = rd32(E1000_TXDCTL(n
));
350 pr_info("%-15s %08x\n", reginfo
->name
, rd32(reginfo
->ofs
));
354 snprintf(rname
, 16, "%s%s", reginfo
->name
, "[0-3]");
355 pr_info("%-15s %08x %08x %08x %08x\n", rname
, regs
[0], regs
[1],
360 * igb_dump - Print registers, tx-rings and rx-rings
362 static void igb_dump(struct igb_adapter
*adapter
)
364 struct net_device
*netdev
= adapter
->netdev
;
365 struct e1000_hw
*hw
= &adapter
->hw
;
366 struct igb_reg_info
*reginfo
;
367 struct igb_ring
*tx_ring
;
368 union e1000_adv_tx_desc
*tx_desc
;
369 struct my_u0
{ u64 a
; u64 b
; } *u0
;
370 struct igb_ring
*rx_ring
;
371 union e1000_adv_rx_desc
*rx_desc
;
375 if (!netif_msg_hw(adapter
))
378 /* Print netdevice Info */
380 dev_info(&adapter
->pdev
->dev
, "Net device Info\n");
381 pr_info("Device Name state trans_start "
383 pr_info("%-15s %016lX %016lX %016lX\n", netdev
->name
,
384 netdev
->state
, netdev
->trans_start
, netdev
->last_rx
);
387 /* Print Registers */
388 dev_info(&adapter
->pdev
->dev
, "Register Dump\n");
389 pr_info(" Register Name Value\n");
390 for (reginfo
= (struct igb_reg_info
*)igb_reg_info_tbl
;
391 reginfo
->name
; reginfo
++) {
392 igb_regdump(hw
, reginfo
);
395 /* Print TX Ring Summary */
396 if (!netdev
|| !netif_running(netdev
))
399 dev_info(&adapter
->pdev
->dev
, "TX Rings Summary\n");
400 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
401 for (n
= 0; n
< adapter
->num_tx_queues
; n
++) {
402 struct igb_tx_buffer
*buffer_info
;
403 tx_ring
= adapter
->tx_ring
[n
];
404 buffer_info
= &tx_ring
->tx_buffer_info
[tx_ring
->next_to_clean
];
405 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
406 n
, tx_ring
->next_to_use
, tx_ring
->next_to_clean
,
407 (u64
)buffer_info
->dma
,
409 buffer_info
->next_to_watch
,
410 (u64
)buffer_info
->time_stamp
);
414 if (!netif_msg_tx_done(adapter
))
415 goto rx_ring_summary
;
417 dev_info(&adapter
->pdev
->dev
, "TX Rings Dump\n");
419 /* Transmit Descriptor Formats
421 * Advanced Transmit Descriptor
422 * +--------------------------------------------------------------+
423 * 0 | Buffer Address [63:0] |
424 * +--------------------------------------------------------------+
425 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
426 * +--------------------------------------------------------------+
427 * 63 46 45 40 39 38 36 35 32 31 24 15 0
430 for (n
= 0; n
< adapter
->num_tx_queues
; n
++) {
431 tx_ring
= adapter
->tx_ring
[n
];
432 pr_info("------------------------------------\n");
433 pr_info("TX QUEUE INDEX = %d\n", tx_ring
->queue_index
);
434 pr_info("------------------------------------\n");
435 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] "
436 "[bi->dma ] leng ntw timestamp "
439 for (i
= 0; tx_ring
->desc
&& (i
< tx_ring
->count
); i
++) {
440 const char *next_desc
;
441 struct igb_tx_buffer
*buffer_info
;
442 tx_desc
= IGB_TX_DESC(tx_ring
, i
);
443 buffer_info
= &tx_ring
->tx_buffer_info
[i
];
444 u0
= (struct my_u0
*)tx_desc
;
445 if (i
== tx_ring
->next_to_use
&&
446 i
== tx_ring
->next_to_clean
)
447 next_desc
= " NTC/U";
448 else if (i
== tx_ring
->next_to_use
)
450 else if (i
== tx_ring
->next_to_clean
)
455 pr_info("T [0x%03X] %016llX %016llX %016llX"
456 " %04X %p %016llX %p%s\n", i
,
459 (u64
)buffer_info
->dma
,
461 buffer_info
->next_to_watch
,
462 (u64
)buffer_info
->time_stamp
,
463 buffer_info
->skb
, next_desc
);
465 if (netif_msg_pktdata(adapter
) && buffer_info
->skb
)
466 print_hex_dump(KERN_INFO
, "",
468 16, 1, buffer_info
->skb
->data
,
469 buffer_info
->length
, true);
473 /* Print RX Rings Summary */
475 dev_info(&adapter
->pdev
->dev
, "RX Rings Summary\n");
476 pr_info("Queue [NTU] [NTC]\n");
477 for (n
= 0; n
< adapter
->num_rx_queues
; n
++) {
478 rx_ring
= adapter
->rx_ring
[n
];
479 pr_info(" %5d %5X %5X\n",
480 n
, rx_ring
->next_to_use
, rx_ring
->next_to_clean
);
484 if (!netif_msg_rx_status(adapter
))
487 dev_info(&adapter
->pdev
->dev
, "RX Rings Dump\n");
489 /* Advanced Receive Descriptor (Read) Format
491 * +-----------------------------------------------------+
492 * 0 | Packet Buffer Address [63:1] |A0/NSE|
493 * +----------------------------------------------+------+
494 * 8 | Header Buffer Address [63:1] | DD |
495 * +-----------------------------------------------------+
498 * Advanced Receive Descriptor (Write-Back) Format
500 * 63 48 47 32 31 30 21 20 17 16 4 3 0
501 * +------------------------------------------------------+
502 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
503 * | Checksum Ident | | | | Type | Type |
504 * +------------------------------------------------------+
505 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
506 * +------------------------------------------------------+
507 * 63 48 47 32 31 20 19 0
510 for (n
= 0; n
< adapter
->num_rx_queues
; n
++) {
511 rx_ring
= adapter
->rx_ring
[n
];
512 pr_info("------------------------------------\n");
513 pr_info("RX QUEUE INDEX = %d\n", rx_ring
->queue_index
);
514 pr_info("------------------------------------\n");
515 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] "
516 "[bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
517 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] -----"
518 "----------- [bi->skb] <-- Adv Rx Write-Back format\n");
520 for (i
= 0; i
< rx_ring
->count
; i
++) {
521 const char *next_desc
;
522 struct igb_rx_buffer
*buffer_info
;
523 buffer_info
= &rx_ring
->rx_buffer_info
[i
];
524 rx_desc
= IGB_RX_DESC(rx_ring
, i
);
525 u0
= (struct my_u0
*)rx_desc
;
526 staterr
= le32_to_cpu(rx_desc
->wb
.upper
.status_error
);
528 if (i
== rx_ring
->next_to_use
)
530 else if (i
== rx_ring
->next_to_clean
)
535 if (staterr
& E1000_RXD_STAT_DD
) {
536 /* Descriptor Done */
537 pr_info("%s[0x%03X] %016llX %016llX -------"
538 "--------- %p%s\n", "RWB", i
,
541 buffer_info
->skb
, next_desc
);
543 pr_info("%s[0x%03X] %016llX %016llX %016llX"
547 (u64
)buffer_info
->dma
,
548 buffer_info
->skb
, next_desc
);
550 if (netif_msg_pktdata(adapter
) &&
551 buffer_info
->dma
&& buffer_info
->skb
) {
552 print_hex_dump(KERN_INFO
, "",
554 16, 1, buffer_info
->skb
->data
,
555 IGB_RX_HDR_LEN
, true);
556 print_hex_dump(KERN_INFO
, "",
559 page_address(buffer_info
->page
) +
560 buffer_info
->page_offset
,
572 * igb_get_hw_dev - return device
573 * used by hardware layer to print debugging information
575 struct net_device
*igb_get_hw_dev(struct e1000_hw
*hw
)
577 struct igb_adapter
*adapter
= hw
->back
;
578 return adapter
->netdev
;
582 * igb_init_module - Driver Registration Routine
584 * igb_init_module is the first routine called when the driver is
585 * loaded. All it does is register with the PCI subsystem.
587 static int __init
igb_init_module(void)
590 pr_info("%s - version %s\n",
591 igb_driver_string
, igb_driver_version
);
593 pr_info("%s\n", igb_copyright
);
595 #ifdef CONFIG_IGB_DCA
596 dca_register_notify(&dca_notifier
);
598 ret
= pci_register_driver(&igb_driver
);
602 module_init(igb_init_module
);
605 * igb_exit_module - Driver Exit Cleanup Routine
607 * igb_exit_module is called just before the driver is removed
610 static void __exit
igb_exit_module(void)
612 #ifdef CONFIG_IGB_DCA
613 dca_unregister_notify(&dca_notifier
);
615 pci_unregister_driver(&igb_driver
);
618 module_exit(igb_exit_module
);
620 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
622 * igb_cache_ring_register - Descriptor ring to register mapping
623 * @adapter: board private structure to initialize
625 * Once we know the feature-set enabled for the device, we'll cache
626 * the register offset the descriptor ring is assigned to.
628 static void igb_cache_ring_register(struct igb_adapter
*adapter
)
631 u32 rbase_offset
= adapter
->vfs_allocated_count
;
633 switch (adapter
->hw
.mac
.type
) {
635 /* The queues are allocated for virtualization such that VF 0
636 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
637 * In order to avoid collision we start at the first free queue
638 * and continue consuming queues in the same sequence
640 if (adapter
->vfs_allocated_count
) {
641 for (; i
< adapter
->rss_queues
; i
++)
642 adapter
->rx_ring
[i
]->reg_idx
= rbase_offset
+
651 for (; i
< adapter
->num_rx_queues
; i
++)
652 adapter
->rx_ring
[i
]->reg_idx
= rbase_offset
+ i
;
653 for (; j
< adapter
->num_tx_queues
; j
++)
654 adapter
->tx_ring
[j
]->reg_idx
= rbase_offset
+ j
;
659 static void igb_free_queues(struct igb_adapter
*adapter
)
663 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
664 kfree(adapter
->tx_ring
[i
]);
665 adapter
->tx_ring
[i
] = NULL
;
667 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
668 kfree(adapter
->rx_ring
[i
]);
669 adapter
->rx_ring
[i
] = NULL
;
671 adapter
->num_rx_queues
= 0;
672 adapter
->num_tx_queues
= 0;
676 * igb_alloc_queues - Allocate memory for all rings
677 * @adapter: board private structure to initialize
679 * We allocate one ring per queue at run-time since we don't know the
680 * number of queues at compile-time.
682 static int igb_alloc_queues(struct igb_adapter
*adapter
)
684 struct igb_ring
*ring
;
686 int orig_node
= adapter
->node
;
688 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
689 if (orig_node
== -1) {
690 int cur_node
= next_online_node(adapter
->node
);
691 if (cur_node
== MAX_NUMNODES
)
692 cur_node
= first_online_node
;
693 adapter
->node
= cur_node
;
695 ring
= kzalloc_node(sizeof(struct igb_ring
), GFP_KERNEL
,
698 ring
= kzalloc(sizeof(struct igb_ring
), GFP_KERNEL
);
701 ring
->count
= adapter
->tx_ring_count
;
702 ring
->queue_index
= i
;
703 ring
->dev
= &adapter
->pdev
->dev
;
704 ring
->netdev
= adapter
->netdev
;
705 ring
->numa_node
= adapter
->node
;
706 /* For 82575, context index must be unique per ring. */
707 if (adapter
->hw
.mac
.type
== e1000_82575
)
708 set_bit(IGB_RING_FLAG_TX_CTX_IDX
, &ring
->flags
);
709 adapter
->tx_ring
[i
] = ring
;
711 /* Restore the adapter's original node */
712 adapter
->node
= orig_node
;
714 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
715 if (orig_node
== -1) {
716 int cur_node
= next_online_node(adapter
->node
);
717 if (cur_node
== MAX_NUMNODES
)
718 cur_node
= first_online_node
;
719 adapter
->node
= cur_node
;
721 ring
= kzalloc_node(sizeof(struct igb_ring
), GFP_KERNEL
,
724 ring
= kzalloc(sizeof(struct igb_ring
), GFP_KERNEL
);
727 ring
->count
= adapter
->rx_ring_count
;
728 ring
->queue_index
= i
;
729 ring
->dev
= &adapter
->pdev
->dev
;
730 ring
->netdev
= adapter
->netdev
;
731 ring
->numa_node
= adapter
->node
;
732 /* set flag indicating ring supports SCTP checksum offload */
733 if (adapter
->hw
.mac
.type
>= e1000_82576
)
734 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM
, &ring
->flags
);
737 * On i350, i210, and i211, loopback VLAN packets
738 * have the tag byte-swapped.
740 if (adapter
->hw
.mac
.type
>= e1000_i350
)
741 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP
, &ring
->flags
);
743 adapter
->rx_ring
[i
] = ring
;
745 /* Restore the adapter's original node */
746 adapter
->node
= orig_node
;
748 igb_cache_ring_register(adapter
);
753 /* Restore the adapter's original node */
754 adapter
->node
= orig_node
;
755 igb_free_queues(adapter
);
761 * igb_write_ivar - configure ivar for given MSI-X vector
762 * @hw: pointer to the HW structure
763 * @msix_vector: vector number we are allocating to a given ring
764 * @index: row index of IVAR register to write within IVAR table
765 * @offset: column offset of in IVAR, should be multiple of 8
767 * This function is intended to handle the writing of the IVAR register
768 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
769 * each containing an cause allocation for an Rx and Tx ring, and a
770 * variable number of rows depending on the number of queues supported.
772 static void igb_write_ivar(struct e1000_hw
*hw
, int msix_vector
,
773 int index
, int offset
)
775 u32 ivar
= array_rd32(E1000_IVAR0
, index
);
777 /* clear any bits that are currently set */
778 ivar
&= ~((u32
)0xFF << offset
);
780 /* write vector and valid bit */
781 ivar
|= (msix_vector
| E1000_IVAR_VALID
) << offset
;
783 array_wr32(E1000_IVAR0
, index
, ivar
);
786 #define IGB_N0_QUEUE -1
787 static void igb_assign_vector(struct igb_q_vector
*q_vector
, int msix_vector
)
789 struct igb_adapter
*adapter
= q_vector
->adapter
;
790 struct e1000_hw
*hw
= &adapter
->hw
;
791 int rx_queue
= IGB_N0_QUEUE
;
792 int tx_queue
= IGB_N0_QUEUE
;
795 if (q_vector
->rx
.ring
)
796 rx_queue
= q_vector
->rx
.ring
->reg_idx
;
797 if (q_vector
->tx
.ring
)
798 tx_queue
= q_vector
->tx
.ring
->reg_idx
;
800 switch (hw
->mac
.type
) {
802 /* The 82575 assigns vectors using a bitmask, which matches the
803 bitmask for the EICR/EIMS/EIMC registers. To assign one
804 or more queues to a vector, we write the appropriate bits
805 into the MSIXBM register for that vector. */
806 if (rx_queue
> IGB_N0_QUEUE
)
807 msixbm
= E1000_EICR_RX_QUEUE0
<< rx_queue
;
808 if (tx_queue
> IGB_N0_QUEUE
)
809 msixbm
|= E1000_EICR_TX_QUEUE0
<< tx_queue
;
810 if (!adapter
->msix_entries
&& msix_vector
== 0)
811 msixbm
|= E1000_EIMS_OTHER
;
812 array_wr32(E1000_MSIXBM(0), msix_vector
, msixbm
);
813 q_vector
->eims_value
= msixbm
;
817 * 82576 uses a table that essentially consists of 2 columns
818 * with 8 rows. The ordering is column-major so we use the
819 * lower 3 bits as the row index, and the 4th bit as the
822 if (rx_queue
> IGB_N0_QUEUE
)
823 igb_write_ivar(hw
, msix_vector
,
825 (rx_queue
& 0x8) << 1);
826 if (tx_queue
> IGB_N0_QUEUE
)
827 igb_write_ivar(hw
, msix_vector
,
829 ((tx_queue
& 0x8) << 1) + 8);
830 q_vector
->eims_value
= 1 << msix_vector
;
837 * On 82580 and newer adapters the scheme is similar to 82576
838 * however instead of ordering column-major we have things
839 * ordered row-major. So we traverse the table by using
840 * bit 0 as the column offset, and the remaining bits as the
843 if (rx_queue
> IGB_N0_QUEUE
)
844 igb_write_ivar(hw
, msix_vector
,
846 (rx_queue
& 0x1) << 4);
847 if (tx_queue
> IGB_N0_QUEUE
)
848 igb_write_ivar(hw
, msix_vector
,
850 ((tx_queue
& 0x1) << 4) + 8);
851 q_vector
->eims_value
= 1 << msix_vector
;
858 /* add q_vector eims value to global eims_enable_mask */
859 adapter
->eims_enable_mask
|= q_vector
->eims_value
;
861 /* configure q_vector to set itr on first interrupt */
862 q_vector
->set_itr
= 1;
866 * igb_configure_msix - Configure MSI-X hardware
868 * igb_configure_msix sets up the hardware to properly
869 * generate MSI-X interrupts.
871 static void igb_configure_msix(struct igb_adapter
*adapter
)
875 struct e1000_hw
*hw
= &adapter
->hw
;
877 adapter
->eims_enable_mask
= 0;
879 /* set vector for other causes, i.e. link changes */
880 switch (hw
->mac
.type
) {
882 tmp
= rd32(E1000_CTRL_EXT
);
883 /* enable MSI-X PBA support*/
884 tmp
|= E1000_CTRL_EXT_PBA_CLR
;
886 /* Auto-Mask interrupts upon ICR read. */
887 tmp
|= E1000_CTRL_EXT_EIAME
;
888 tmp
|= E1000_CTRL_EXT_IRCA
;
890 wr32(E1000_CTRL_EXT
, tmp
);
892 /* enable msix_other interrupt */
893 array_wr32(E1000_MSIXBM(0), vector
++,
895 adapter
->eims_other
= E1000_EIMS_OTHER
;
904 /* Turn on MSI-X capability first, or our settings
905 * won't stick. And it will take days to debug. */
906 wr32(E1000_GPIE
, E1000_GPIE_MSIX_MODE
|
907 E1000_GPIE_PBA
| E1000_GPIE_EIAME
|
910 /* enable msix_other interrupt */
911 adapter
->eims_other
= 1 << vector
;
912 tmp
= (vector
++ | E1000_IVAR_VALID
) << 8;
914 wr32(E1000_IVAR_MISC
, tmp
);
917 /* do nothing, since nothing else supports MSI-X */
919 } /* switch (hw->mac.type) */
921 adapter
->eims_enable_mask
|= adapter
->eims_other
;
923 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
924 igb_assign_vector(adapter
->q_vector
[i
], vector
++);
930 * igb_request_msix - Initialize MSI-X interrupts
932 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
935 static int igb_request_msix(struct igb_adapter
*adapter
)
937 struct net_device
*netdev
= adapter
->netdev
;
938 struct e1000_hw
*hw
= &adapter
->hw
;
939 int i
, err
= 0, vector
= 0;
941 err
= request_irq(adapter
->msix_entries
[vector
].vector
,
942 igb_msix_other
, 0, netdev
->name
, adapter
);
947 for (i
= 0; i
< adapter
->num_q_vectors
; i
++) {
948 struct igb_q_vector
*q_vector
= adapter
->q_vector
[i
];
950 q_vector
->itr_register
= hw
->hw_addr
+ E1000_EITR(vector
);
952 if (q_vector
->rx
.ring
&& q_vector
->tx
.ring
)
953 sprintf(q_vector
->name
, "%s-TxRx-%u", netdev
->name
,
954 q_vector
->rx
.ring
->queue_index
);
955 else if (q_vector
->tx
.ring
)
956 sprintf(q_vector
->name
, "%s-tx-%u", netdev
->name
,
957 q_vector
->tx
.ring
->queue_index
);
958 else if (q_vector
->rx
.ring
)
959 sprintf(q_vector
->name
, "%s-rx-%u", netdev
->name
,
960 q_vector
->rx
.ring
->queue_index
);
962 sprintf(q_vector
->name
, "%s-unused", netdev
->name
);
964 err
= request_irq(adapter
->msix_entries
[vector
].vector
,
965 igb_msix_ring
, 0, q_vector
->name
,
972 igb_configure_msix(adapter
);
978 static void igb_reset_interrupt_capability(struct igb_adapter
*adapter
)
980 if (adapter
->msix_entries
) {
981 pci_disable_msix(adapter
->pdev
);
982 kfree(adapter
->msix_entries
);
983 adapter
->msix_entries
= NULL
;
984 } else if (adapter
->flags
& IGB_FLAG_HAS_MSI
) {
985 pci_disable_msi(adapter
->pdev
);
990 * igb_free_q_vectors - Free memory allocated for interrupt vectors
991 * @adapter: board private structure to initialize
993 * This function frees the memory allocated to the q_vectors. In addition if
994 * NAPI is enabled it will delete any references to the NAPI struct prior
995 * to freeing the q_vector.
997 static void igb_free_q_vectors(struct igb_adapter
*adapter
)
1001 for (v_idx
= 0; v_idx
< adapter
->num_q_vectors
; v_idx
++) {
1002 struct igb_q_vector
*q_vector
= adapter
->q_vector
[v_idx
];
1003 adapter
->q_vector
[v_idx
] = NULL
;
1006 netif_napi_del(&q_vector
->napi
);
1009 adapter
->num_q_vectors
= 0;
1013 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1015 * This function resets the device so that it has 0 rx queues, tx queues, and
1016 * MSI-X interrupts allocated.
1018 static void igb_clear_interrupt_scheme(struct igb_adapter
*adapter
)
1020 igb_free_queues(adapter
);
1021 igb_free_q_vectors(adapter
);
1022 igb_reset_interrupt_capability(adapter
);
1026 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1028 * Attempt to configure interrupts using the best available
1029 * capabilities of the hardware and kernel.
1031 static int igb_set_interrupt_capability(struct igb_adapter
*adapter
)
1036 /* Number of supported queues. */
1037 adapter
->num_rx_queues
= adapter
->rss_queues
;
1038 if (adapter
->vfs_allocated_count
)
1039 adapter
->num_tx_queues
= 1;
1041 adapter
->num_tx_queues
= adapter
->rss_queues
;
1043 /* start with one vector for every rx queue */
1044 numvecs
= adapter
->num_rx_queues
;
1046 /* if tx handler is separate add 1 for every tx queue */
1047 if (!(adapter
->flags
& IGB_FLAG_QUEUE_PAIRS
))
1048 numvecs
+= adapter
->num_tx_queues
;
1050 /* store the number of vectors reserved for queues */
1051 adapter
->num_q_vectors
= numvecs
;
1053 /* add 1 vector for link status interrupts */
1055 adapter
->msix_entries
= kcalloc(numvecs
, sizeof(struct msix_entry
),
1058 if (!adapter
->msix_entries
)
1061 for (i
= 0; i
< numvecs
; i
++)
1062 adapter
->msix_entries
[i
].entry
= i
;
1064 err
= pci_enable_msix(adapter
->pdev
,
1065 adapter
->msix_entries
,
1070 igb_reset_interrupt_capability(adapter
);
1072 /* If we can't do MSI-X, try MSI */
1074 #ifdef CONFIG_PCI_IOV
1075 /* disable SR-IOV for non MSI-X configurations */
1076 if (adapter
->vf_data
) {
1077 struct e1000_hw
*hw
= &adapter
->hw
;
1078 /* disable iov and allow time for transactions to clear */
1079 pci_disable_sriov(adapter
->pdev
);
1082 kfree(adapter
->vf_data
);
1083 adapter
->vf_data
= NULL
;
1084 wr32(E1000_IOVCTL
, E1000_IOVCTL_REUSE_VFQ
);
1087 dev_info(&adapter
->pdev
->dev
, "IOV Disabled\n");
1090 adapter
->vfs_allocated_count
= 0;
1091 adapter
->rss_queues
= 1;
1092 adapter
->flags
|= IGB_FLAG_QUEUE_PAIRS
;
1093 adapter
->num_rx_queues
= 1;
1094 adapter
->num_tx_queues
= 1;
1095 adapter
->num_q_vectors
= 1;
1096 if (!pci_enable_msi(adapter
->pdev
))
1097 adapter
->flags
|= IGB_FLAG_HAS_MSI
;
1099 /* Notify the stack of the (possibly) reduced queue counts. */
1101 netif_set_real_num_tx_queues(adapter
->netdev
, adapter
->num_tx_queues
);
1102 err
= netif_set_real_num_rx_queues(adapter
->netdev
,
1103 adapter
->num_rx_queues
);
1109 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1110 * @adapter: board private structure to initialize
1112 * We allocate one q_vector per queue interrupt. If allocation fails we
1115 static int igb_alloc_q_vectors(struct igb_adapter
*adapter
)
1117 struct igb_q_vector
*q_vector
;
1118 struct e1000_hw
*hw
= &adapter
->hw
;
1120 int orig_node
= adapter
->node
;
1122 for (v_idx
= 0; v_idx
< adapter
->num_q_vectors
; v_idx
++) {
1123 if ((adapter
->num_q_vectors
== (adapter
->num_rx_queues
+
1124 adapter
->num_tx_queues
)) &&
1125 (adapter
->num_rx_queues
== v_idx
))
1126 adapter
->node
= orig_node
;
1127 if (orig_node
== -1) {
1128 int cur_node
= next_online_node(adapter
->node
);
1129 if (cur_node
== MAX_NUMNODES
)
1130 cur_node
= first_online_node
;
1131 adapter
->node
= cur_node
;
1133 q_vector
= kzalloc_node(sizeof(struct igb_q_vector
), GFP_KERNEL
,
1136 q_vector
= kzalloc(sizeof(struct igb_q_vector
),
1140 q_vector
->adapter
= adapter
;
1141 q_vector
->itr_register
= hw
->hw_addr
+ E1000_EITR(0);
1142 q_vector
->itr_val
= IGB_START_ITR
;
1143 netif_napi_add(adapter
->netdev
, &q_vector
->napi
, igb_poll
, 64);
1144 adapter
->q_vector
[v_idx
] = q_vector
;
1146 /* Restore the adapter's original node */
1147 adapter
->node
= orig_node
;
1152 /* Restore the adapter's original node */
1153 adapter
->node
= orig_node
;
1154 igb_free_q_vectors(adapter
);
1158 static void igb_map_rx_ring_to_vector(struct igb_adapter
*adapter
,
1159 int ring_idx
, int v_idx
)
1161 struct igb_q_vector
*q_vector
= adapter
->q_vector
[v_idx
];
1163 q_vector
->rx
.ring
= adapter
->rx_ring
[ring_idx
];
1164 q_vector
->rx
.ring
->q_vector
= q_vector
;
1165 q_vector
->rx
.count
++;
1166 q_vector
->itr_val
= adapter
->rx_itr_setting
;
1167 if (q_vector
->itr_val
&& q_vector
->itr_val
<= 3)
1168 q_vector
->itr_val
= IGB_START_ITR
;
1171 static void igb_map_tx_ring_to_vector(struct igb_adapter
*adapter
,
1172 int ring_idx
, int v_idx
)
1174 struct igb_q_vector
*q_vector
= adapter
->q_vector
[v_idx
];
1176 q_vector
->tx
.ring
= adapter
->tx_ring
[ring_idx
];
1177 q_vector
->tx
.ring
->q_vector
= q_vector
;
1178 q_vector
->tx
.count
++;
1179 q_vector
->itr_val
= adapter
->tx_itr_setting
;
1180 q_vector
->tx
.work_limit
= adapter
->tx_work_limit
;
1181 if (q_vector
->itr_val
&& q_vector
->itr_val
<= 3)
1182 q_vector
->itr_val
= IGB_START_ITR
;
1186 * igb_map_ring_to_vector - maps allocated queues to vectors
1188 * This function maps the recently allocated queues to vectors.
1190 static int igb_map_ring_to_vector(struct igb_adapter
*adapter
)
1195 if ((adapter
->num_q_vectors
< adapter
->num_rx_queues
) ||
1196 (adapter
->num_q_vectors
< adapter
->num_tx_queues
))
1199 if (adapter
->num_q_vectors
>=
1200 (adapter
->num_rx_queues
+ adapter
->num_tx_queues
)) {
1201 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
1202 igb_map_rx_ring_to_vector(adapter
, i
, v_idx
++);
1203 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
1204 igb_map_tx_ring_to_vector(adapter
, i
, v_idx
++);
1206 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
1207 if (i
< adapter
->num_tx_queues
)
1208 igb_map_tx_ring_to_vector(adapter
, i
, v_idx
);
1209 igb_map_rx_ring_to_vector(adapter
, i
, v_idx
++);
1211 for (; i
< adapter
->num_tx_queues
; i
++)
1212 igb_map_tx_ring_to_vector(adapter
, i
, v_idx
++);
1218 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1220 * This function initializes the interrupts and allocates all of the queues.
1222 static int igb_init_interrupt_scheme(struct igb_adapter
*adapter
)
1224 struct pci_dev
*pdev
= adapter
->pdev
;
1227 err
= igb_set_interrupt_capability(adapter
);
1231 err
= igb_alloc_q_vectors(adapter
);
1233 dev_err(&pdev
->dev
, "Unable to allocate memory for vectors\n");
1234 goto err_alloc_q_vectors
;
1237 err
= igb_alloc_queues(adapter
);
1239 dev_err(&pdev
->dev
, "Unable to allocate memory for queues\n");
1240 goto err_alloc_queues
;
1243 err
= igb_map_ring_to_vector(adapter
);
1245 dev_err(&pdev
->dev
, "Invalid q_vector to ring mapping\n");
1246 goto err_map_queues
;
1252 igb_free_queues(adapter
);
1254 igb_free_q_vectors(adapter
);
1255 err_alloc_q_vectors
:
1256 igb_reset_interrupt_capability(adapter
);
1261 * igb_request_irq - initialize interrupts
1263 * Attempts to configure interrupts using the best available
1264 * capabilities of the hardware and kernel.
1266 static int igb_request_irq(struct igb_adapter
*adapter
)
1268 struct net_device
*netdev
= adapter
->netdev
;
1269 struct pci_dev
*pdev
= adapter
->pdev
;
1272 if (adapter
->msix_entries
) {
1273 err
= igb_request_msix(adapter
);
1276 /* fall back to MSI */
1277 igb_clear_interrupt_scheme(adapter
);
1278 if (!pci_enable_msi(pdev
))
1279 adapter
->flags
|= IGB_FLAG_HAS_MSI
;
1280 igb_free_all_tx_resources(adapter
);
1281 igb_free_all_rx_resources(adapter
);
1282 adapter
->num_tx_queues
= 1;
1283 adapter
->num_rx_queues
= 1;
1284 adapter
->num_q_vectors
= 1;
1285 err
= igb_alloc_q_vectors(adapter
);
1288 "Unable to allocate memory for vectors\n");
1291 err
= igb_alloc_queues(adapter
);
1294 "Unable to allocate memory for queues\n");
1295 igb_free_q_vectors(adapter
);
1298 igb_setup_all_tx_resources(adapter
);
1299 igb_setup_all_rx_resources(adapter
);
1302 igb_assign_vector(adapter
->q_vector
[0], 0);
1304 if (adapter
->flags
& IGB_FLAG_HAS_MSI
) {
1305 err
= request_irq(pdev
->irq
, igb_intr_msi
, 0,
1306 netdev
->name
, adapter
);
1310 /* fall back to legacy interrupts */
1311 igb_reset_interrupt_capability(adapter
);
1312 adapter
->flags
&= ~IGB_FLAG_HAS_MSI
;
1315 err
= request_irq(pdev
->irq
, igb_intr
, IRQF_SHARED
,
1316 netdev
->name
, adapter
);
1319 dev_err(&pdev
->dev
, "Error %d getting interrupt\n",
1326 static void igb_free_irq(struct igb_adapter
*adapter
)
1328 if (adapter
->msix_entries
) {
1331 free_irq(adapter
->msix_entries
[vector
++].vector
, adapter
);
1333 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
1334 free_irq(adapter
->msix_entries
[vector
++].vector
,
1335 adapter
->q_vector
[i
]);
1337 free_irq(adapter
->pdev
->irq
, adapter
);
1342 * igb_irq_disable - Mask off interrupt generation on the NIC
1343 * @adapter: board private structure
1345 static void igb_irq_disable(struct igb_adapter
*adapter
)
1347 struct e1000_hw
*hw
= &adapter
->hw
;
1350 * we need to be careful when disabling interrupts. The VFs are also
1351 * mapped into these registers and so clearing the bits can cause
1352 * issues on the VF drivers so we only need to clear what we set
1354 if (adapter
->msix_entries
) {
1355 u32 regval
= rd32(E1000_EIAM
);
1356 wr32(E1000_EIAM
, regval
& ~adapter
->eims_enable_mask
);
1357 wr32(E1000_EIMC
, adapter
->eims_enable_mask
);
1358 regval
= rd32(E1000_EIAC
);
1359 wr32(E1000_EIAC
, regval
& ~adapter
->eims_enable_mask
);
1363 wr32(E1000_IMC
, ~0);
1365 if (adapter
->msix_entries
) {
1367 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
1368 synchronize_irq(adapter
->msix_entries
[i
].vector
);
1370 synchronize_irq(adapter
->pdev
->irq
);
1375 * igb_irq_enable - Enable default interrupt generation settings
1376 * @adapter: board private structure
1378 static void igb_irq_enable(struct igb_adapter
*adapter
)
1380 struct e1000_hw
*hw
= &adapter
->hw
;
1382 if (adapter
->msix_entries
) {
1383 u32 ims
= E1000_IMS_LSC
| E1000_IMS_DOUTSYNC
| E1000_IMS_DRSTA
;
1384 u32 regval
= rd32(E1000_EIAC
);
1385 wr32(E1000_EIAC
, regval
| adapter
->eims_enable_mask
);
1386 regval
= rd32(E1000_EIAM
);
1387 wr32(E1000_EIAM
, regval
| adapter
->eims_enable_mask
);
1388 wr32(E1000_EIMS
, adapter
->eims_enable_mask
);
1389 if (adapter
->vfs_allocated_count
) {
1390 wr32(E1000_MBVFIMR
, 0xFF);
1391 ims
|= E1000_IMS_VMMB
;
1393 wr32(E1000_IMS
, ims
);
1395 wr32(E1000_IMS
, IMS_ENABLE_MASK
|
1397 wr32(E1000_IAM
, IMS_ENABLE_MASK
|
1402 static void igb_update_mng_vlan(struct igb_adapter
*adapter
)
1404 struct e1000_hw
*hw
= &adapter
->hw
;
1405 u16 vid
= adapter
->hw
.mng_cookie
.vlan_id
;
1406 u16 old_vid
= adapter
->mng_vlan_id
;
1408 if (hw
->mng_cookie
.status
& E1000_MNG_DHCP_COOKIE_STATUS_VLAN
) {
1409 /* add VID to filter table */
1410 igb_vfta_set(hw
, vid
, true);
1411 adapter
->mng_vlan_id
= vid
;
1413 adapter
->mng_vlan_id
= IGB_MNG_VLAN_NONE
;
1416 if ((old_vid
!= (u16
)IGB_MNG_VLAN_NONE
) &&
1418 !test_bit(old_vid
, adapter
->active_vlans
)) {
1419 /* remove VID from filter table */
1420 igb_vfta_set(hw
, old_vid
, false);
1425 * igb_release_hw_control - release control of the h/w to f/w
1426 * @adapter: address of board private structure
1428 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1429 * For ASF and Pass Through versions of f/w this means that the
1430 * driver is no longer loaded.
1433 static void igb_release_hw_control(struct igb_adapter
*adapter
)
1435 struct e1000_hw
*hw
= &adapter
->hw
;
1438 /* Let firmware take over control of h/w */
1439 ctrl_ext
= rd32(E1000_CTRL_EXT
);
1440 wr32(E1000_CTRL_EXT
,
1441 ctrl_ext
& ~E1000_CTRL_EXT_DRV_LOAD
);
1445 * igb_get_hw_control - get control of the h/w from f/w
1446 * @adapter: address of board private structure
1448 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1449 * For ASF and Pass Through versions of f/w this means that
1450 * the driver is loaded.
1453 static void igb_get_hw_control(struct igb_adapter
*adapter
)
1455 struct e1000_hw
*hw
= &adapter
->hw
;
1458 /* Let firmware know the driver has taken over */
1459 ctrl_ext
= rd32(E1000_CTRL_EXT
);
1460 wr32(E1000_CTRL_EXT
,
1461 ctrl_ext
| E1000_CTRL_EXT_DRV_LOAD
);
1465 * igb_configure - configure the hardware for RX and TX
1466 * @adapter: private board structure
1468 static void igb_configure(struct igb_adapter
*adapter
)
1470 struct net_device
*netdev
= adapter
->netdev
;
1473 igb_get_hw_control(adapter
);
1474 igb_set_rx_mode(netdev
);
1476 igb_restore_vlan(adapter
);
1478 igb_setup_tctl(adapter
);
1479 igb_setup_mrqc(adapter
);
1480 igb_setup_rctl(adapter
);
1482 igb_configure_tx(adapter
);
1483 igb_configure_rx(adapter
);
1485 igb_rx_fifo_flush_82575(&adapter
->hw
);
1487 /* call igb_desc_unused which always leaves
1488 * at least 1 descriptor unused to make sure
1489 * next_to_use != next_to_clean */
1490 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
1491 struct igb_ring
*ring
= adapter
->rx_ring
[i
];
1492 igb_alloc_rx_buffers(ring
, igb_desc_unused(ring
));
1497 * igb_power_up_link - Power up the phy/serdes link
1498 * @adapter: address of board private structure
1500 void igb_power_up_link(struct igb_adapter
*adapter
)
1502 igb_reset_phy(&adapter
->hw
);
1504 if (adapter
->hw
.phy
.media_type
== e1000_media_type_copper
)
1505 igb_power_up_phy_copper(&adapter
->hw
);
1507 igb_power_up_serdes_link_82575(&adapter
->hw
);
1511 * igb_power_down_link - Power down the phy/serdes link
1512 * @adapter: address of board private structure
1514 static void igb_power_down_link(struct igb_adapter
*adapter
)
1516 if (adapter
->hw
.phy
.media_type
== e1000_media_type_copper
)
1517 igb_power_down_phy_copper_82575(&adapter
->hw
);
1519 igb_shutdown_serdes_link_82575(&adapter
->hw
);
1523 * igb_up - Open the interface and prepare it to handle traffic
1524 * @adapter: board private structure
1526 int igb_up(struct igb_adapter
*adapter
)
1528 struct e1000_hw
*hw
= &adapter
->hw
;
1531 /* hardware has been reset, we need to reload some things */
1532 igb_configure(adapter
);
1534 clear_bit(__IGB_DOWN
, &adapter
->state
);
1536 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
1537 napi_enable(&(adapter
->q_vector
[i
]->napi
));
1539 if (adapter
->msix_entries
)
1540 igb_configure_msix(adapter
);
1542 igb_assign_vector(adapter
->q_vector
[0], 0);
1544 /* Clear any pending interrupts. */
1546 igb_irq_enable(adapter
);
1548 /* notify VFs that reset has been completed */
1549 if (adapter
->vfs_allocated_count
) {
1550 u32 reg_data
= rd32(E1000_CTRL_EXT
);
1551 reg_data
|= E1000_CTRL_EXT_PFRSTD
;
1552 wr32(E1000_CTRL_EXT
, reg_data
);
1555 netif_tx_start_all_queues(adapter
->netdev
);
1557 /* start the watchdog. */
1558 hw
->mac
.get_link_status
= 1;
1559 schedule_work(&adapter
->watchdog_task
);
1564 void igb_down(struct igb_adapter
*adapter
)
1566 struct net_device
*netdev
= adapter
->netdev
;
1567 struct e1000_hw
*hw
= &adapter
->hw
;
1571 /* signal that we're down so the interrupt handler does not
1572 * reschedule our watchdog timer */
1573 set_bit(__IGB_DOWN
, &adapter
->state
);
1575 /* disable receives in the hardware */
1576 rctl
= rd32(E1000_RCTL
);
1577 wr32(E1000_RCTL
, rctl
& ~E1000_RCTL_EN
);
1578 /* flush and sleep below */
1580 netif_tx_stop_all_queues(netdev
);
1582 /* disable transmits in the hardware */
1583 tctl
= rd32(E1000_TCTL
);
1584 tctl
&= ~E1000_TCTL_EN
;
1585 wr32(E1000_TCTL
, tctl
);
1586 /* flush both disables and wait for them to finish */
1590 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
1591 napi_disable(&(adapter
->q_vector
[i
]->napi
));
1593 igb_irq_disable(adapter
);
1595 del_timer_sync(&adapter
->watchdog_timer
);
1596 del_timer_sync(&adapter
->phy_info_timer
);
1598 netif_carrier_off(netdev
);
1600 /* record the stats before reset*/
1601 spin_lock(&adapter
->stats64_lock
);
1602 igb_update_stats(adapter
, &adapter
->stats64
);
1603 spin_unlock(&adapter
->stats64_lock
);
1605 adapter
->link_speed
= 0;
1606 adapter
->link_duplex
= 0;
1608 if (!pci_channel_offline(adapter
->pdev
))
1610 igb_clean_all_tx_rings(adapter
);
1611 igb_clean_all_rx_rings(adapter
);
1612 #ifdef CONFIG_IGB_DCA
1614 /* since we reset the hardware DCA settings were cleared */
1615 igb_setup_dca(adapter
);
1619 void igb_reinit_locked(struct igb_adapter
*adapter
)
1621 WARN_ON(in_interrupt());
1622 while (test_and_set_bit(__IGB_RESETTING
, &adapter
->state
))
1626 clear_bit(__IGB_RESETTING
, &adapter
->state
);
1629 void igb_reset(struct igb_adapter
*adapter
)
1631 struct pci_dev
*pdev
= adapter
->pdev
;
1632 struct e1000_hw
*hw
= &adapter
->hw
;
1633 struct e1000_mac_info
*mac
= &hw
->mac
;
1634 struct e1000_fc_info
*fc
= &hw
->fc
;
1635 u32 pba
= 0, tx_space
, min_tx_space
, min_rx_space
;
1638 /* Repartition Pba for greater than 9k mtu
1639 * To take effect CTRL.RST is required.
1641 switch (mac
->type
) {
1644 pba
= rd32(E1000_RXPBS
);
1645 pba
= igb_rxpbs_adjust_82580(pba
);
1648 pba
= rd32(E1000_RXPBS
);
1649 pba
&= E1000_RXPBS_SIZE_MASK_82576
;
1655 pba
= E1000_PBA_34K
;
1659 if ((adapter
->max_frame_size
> ETH_FRAME_LEN
+ ETH_FCS_LEN
) &&
1660 (mac
->type
< e1000_82576
)) {
1661 /* adjust PBA for jumbo frames */
1662 wr32(E1000_PBA
, pba
);
1664 /* To maintain wire speed transmits, the Tx FIFO should be
1665 * large enough to accommodate two full transmit packets,
1666 * rounded up to the next 1KB and expressed in KB. Likewise,
1667 * the Rx FIFO should be large enough to accommodate at least
1668 * one full receive packet and is similarly rounded up and
1669 * expressed in KB. */
1670 pba
= rd32(E1000_PBA
);
1671 /* upper 16 bits has Tx packet buffer allocation size in KB */
1672 tx_space
= pba
>> 16;
1673 /* lower 16 bits has Rx packet buffer allocation size in KB */
1675 /* the tx fifo also stores 16 bytes of information about the tx
1676 * but don't include ethernet FCS because hardware appends it */
1677 min_tx_space
= (adapter
->max_frame_size
+
1678 sizeof(union e1000_adv_tx_desc
) -
1680 min_tx_space
= ALIGN(min_tx_space
, 1024);
1681 min_tx_space
>>= 10;
1682 /* software strips receive CRC, so leave room for it */
1683 min_rx_space
= adapter
->max_frame_size
;
1684 min_rx_space
= ALIGN(min_rx_space
, 1024);
1685 min_rx_space
>>= 10;
1687 /* If current Tx allocation is less than the min Tx FIFO size,
1688 * and the min Tx FIFO size is less than the current Rx FIFO
1689 * allocation, take space away from current Rx allocation */
1690 if (tx_space
< min_tx_space
&&
1691 ((min_tx_space
- tx_space
) < pba
)) {
1692 pba
= pba
- (min_tx_space
- tx_space
);
1694 /* if short on rx space, rx wins and must trump tx
1696 if (pba
< min_rx_space
)
1699 wr32(E1000_PBA
, pba
);
1702 /* flow control settings */
1703 /* The high water mark must be low enough to fit one full frame
1704 * (or the size used for early receive) above it in the Rx FIFO.
1705 * Set it to the lower of:
1706 * - 90% of the Rx FIFO size, or
1707 * - the full Rx FIFO size minus one full frame */
1708 hwm
= min(((pba
<< 10) * 9 / 10),
1709 ((pba
<< 10) - 2 * adapter
->max_frame_size
));
1711 fc
->high_water
= hwm
& 0xFFF0; /* 16-byte granularity */
1712 fc
->low_water
= fc
->high_water
- 16;
1713 fc
->pause_time
= 0xFFFF;
1715 fc
->current_mode
= fc
->requested_mode
;
1717 /* disable receive for all VFs and wait one second */
1718 if (adapter
->vfs_allocated_count
) {
1720 for (i
= 0 ; i
< adapter
->vfs_allocated_count
; i
++)
1721 adapter
->vf_data
[i
].flags
&= IGB_VF_FLAG_PF_SET_MAC
;
1723 /* ping all the active vfs to let them know we are going down */
1724 igb_ping_all_vfs(adapter
);
1726 /* disable transmits and receives */
1727 wr32(E1000_VFRE
, 0);
1728 wr32(E1000_VFTE
, 0);
1731 /* Allow time for pending master requests to run */
1732 hw
->mac
.ops
.reset_hw(hw
);
1735 if (hw
->mac
.ops
.init_hw(hw
))
1736 dev_err(&pdev
->dev
, "Hardware Error\n");
1739 * Flow control settings reset on hardware reset, so guarantee flow
1740 * control is off when forcing speed.
1742 if (!hw
->mac
.autoneg
)
1743 igb_force_mac_fc(hw
);
1745 igb_init_dmac(adapter
, pba
);
1746 if (!netif_running(adapter
->netdev
))
1747 igb_power_down_link(adapter
);
1749 igb_update_mng_vlan(adapter
);
1751 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1752 wr32(E1000_VET
, ETHERNET_IEEE_VLAN_TYPE
);
1754 igb_get_phy_info(hw
);
1757 static netdev_features_t
igb_fix_features(struct net_device
*netdev
,
1758 netdev_features_t features
)
1761 * Since there is no support for separate rx/tx vlan accel
1762 * enable/disable make sure tx flag is always in same state as rx.
1764 if (features
& NETIF_F_HW_VLAN_RX
)
1765 features
|= NETIF_F_HW_VLAN_TX
;
1767 features
&= ~NETIF_F_HW_VLAN_TX
;
1772 static int igb_set_features(struct net_device
*netdev
,
1773 netdev_features_t features
)
1775 netdev_features_t changed
= netdev
->features
^ features
;
1776 struct igb_adapter
*adapter
= netdev_priv(netdev
);
1778 if (changed
& NETIF_F_HW_VLAN_RX
)
1779 igb_vlan_mode(netdev
, features
);
1781 if (!(changed
& NETIF_F_RXALL
))
1784 netdev
->features
= features
;
1786 if (netif_running(netdev
))
1787 igb_reinit_locked(adapter
);
1794 static const struct net_device_ops igb_netdev_ops
= {
1795 .ndo_open
= igb_open
,
1796 .ndo_stop
= igb_close
,
1797 .ndo_start_xmit
= igb_xmit_frame
,
1798 .ndo_get_stats64
= igb_get_stats64
,
1799 .ndo_set_rx_mode
= igb_set_rx_mode
,
1800 .ndo_set_mac_address
= igb_set_mac
,
1801 .ndo_change_mtu
= igb_change_mtu
,
1802 .ndo_do_ioctl
= igb_ioctl
,
1803 .ndo_tx_timeout
= igb_tx_timeout
,
1804 .ndo_validate_addr
= eth_validate_addr
,
1805 .ndo_vlan_rx_add_vid
= igb_vlan_rx_add_vid
,
1806 .ndo_vlan_rx_kill_vid
= igb_vlan_rx_kill_vid
,
1807 .ndo_set_vf_mac
= igb_ndo_set_vf_mac
,
1808 .ndo_set_vf_vlan
= igb_ndo_set_vf_vlan
,
1809 .ndo_set_vf_tx_rate
= igb_ndo_set_vf_bw
,
1810 .ndo_get_vf_config
= igb_ndo_get_vf_config
,
1811 #ifdef CONFIG_NET_POLL_CONTROLLER
1812 .ndo_poll_controller
= igb_netpoll
,
1814 .ndo_fix_features
= igb_fix_features
,
1815 .ndo_set_features
= igb_set_features
,
1819 * igb_set_fw_version - Configure version string for ethtool
1820 * @adapter: adapter struct
1823 void igb_set_fw_version(struct igb_adapter
*adapter
)
1825 struct e1000_hw
*hw
= &adapter
->hw
;
1826 u16 eeprom_verh
, eeprom_verl
, comb_verh
, comb_verl
, comb_offset
;
1827 u16 major
, build
, patch
, fw_version
;
1830 hw
->nvm
.ops
.read(hw
, 5, 1, &fw_version
);
1831 if (adapter
->hw
.mac
.type
!= e1000_i211
) {
1832 hw
->nvm
.ops
.read(hw
, NVM_ETRACK_WORD
, 1, &eeprom_verh
);
1833 hw
->nvm
.ops
.read(hw
, (NVM_ETRACK_WORD
+ 1), 1, &eeprom_verl
);
1834 etrack_id
= (eeprom_verh
<< IGB_ETRACK_SHIFT
) | eeprom_verl
;
1836 /* combo image version needs to be found */
1837 hw
->nvm
.ops
.read(hw
, NVM_COMB_VER_PTR
, 1, &comb_offset
);
1838 if ((comb_offset
!= 0x0) &&
1839 (comb_offset
!= IGB_NVM_VER_INVALID
)) {
1840 hw
->nvm
.ops
.read(hw
, (NVM_COMB_VER_OFF
+ comb_offset
1841 + 1), 1, &comb_verh
);
1842 hw
->nvm
.ops
.read(hw
, (NVM_COMB_VER_OFF
+ comb_offset
),
1845 /* Only display Option Rom if it exists and is valid */
1846 if ((comb_verh
&& comb_verl
) &&
1847 ((comb_verh
!= IGB_NVM_VER_INVALID
) &&
1848 (comb_verl
!= IGB_NVM_VER_INVALID
))) {
1849 major
= comb_verl
>> IGB_COMB_VER_SHFT
;
1850 build
= (comb_verl
<< IGB_COMB_VER_SHFT
) |
1851 (comb_verh
>> IGB_COMB_VER_SHFT
);
1852 patch
= comb_verh
& IGB_COMB_VER_MASK
;
1853 snprintf(adapter
->fw_version
,
1854 sizeof(adapter
->fw_version
),
1855 "%d.%d%d, 0x%08x, %d.%d.%d",
1856 (fw_version
& IGB_MAJOR_MASK
) >>
1858 (fw_version
& IGB_MINOR_MASK
) >>
1860 (fw_version
& IGB_BUILD_MASK
),
1861 etrack_id
, major
, build
, patch
);
1865 snprintf(adapter
->fw_version
, sizeof(adapter
->fw_version
),
1867 (fw_version
& IGB_MAJOR_MASK
) >> IGB_MAJOR_SHIFT
,
1868 (fw_version
& IGB_MINOR_MASK
) >> IGB_MINOR_SHIFT
,
1869 (fw_version
& IGB_BUILD_MASK
), etrack_id
);
1871 snprintf(adapter
->fw_version
, sizeof(adapter
->fw_version
),
1873 (fw_version
& IGB_MAJOR_MASK
) >> IGB_MAJOR_SHIFT
,
1874 (fw_version
& IGB_MINOR_MASK
) >> IGB_MINOR_SHIFT
,
1875 (fw_version
& IGB_BUILD_MASK
));
1882 * igb_probe - Device Initialization Routine
1883 * @pdev: PCI device information struct
1884 * @ent: entry in igb_pci_tbl
1886 * Returns 0 on success, negative on failure
1888 * igb_probe initializes an adapter identified by a pci_dev structure.
1889 * The OS initialization, configuring of the adapter private structure,
1890 * and a hardware reset occur.
1892 static int __devinit
igb_probe(struct pci_dev
*pdev
,
1893 const struct pci_device_id
*ent
)
1895 struct net_device
*netdev
;
1896 struct igb_adapter
*adapter
;
1897 struct e1000_hw
*hw
;
1898 u16 eeprom_data
= 0;
1900 static int global_quad_port_a
; /* global quad port a indication */
1901 const struct e1000_info
*ei
= igb_info_tbl
[ent
->driver_data
];
1902 unsigned long mmio_start
, mmio_len
;
1903 int err
, pci_using_dac
;
1904 u16 eeprom_apme_mask
= IGB_EEPROM_APME
;
1905 u8 part_str
[E1000_PBANUM_LENGTH
];
1907 /* Catch broken hardware that put the wrong VF device ID in
1908 * the PCIe SR-IOV capability.
1910 if (pdev
->is_virtfn
) {
1911 WARN(1, KERN_ERR
"%s (%hx:%hx) should not be a VF!\n",
1912 pci_name(pdev
), pdev
->vendor
, pdev
->device
);
1916 err
= pci_enable_device_mem(pdev
);
1921 err
= dma_set_mask(&pdev
->dev
, DMA_BIT_MASK(64));
1923 err
= dma_set_coherent_mask(&pdev
->dev
, DMA_BIT_MASK(64));
1927 err
= dma_set_mask(&pdev
->dev
, DMA_BIT_MASK(32));
1929 err
= dma_set_coherent_mask(&pdev
->dev
, DMA_BIT_MASK(32));
1931 dev_err(&pdev
->dev
, "No usable DMA "
1932 "configuration, aborting\n");
1938 err
= pci_request_selected_regions(pdev
, pci_select_bars(pdev
,
1944 pci_enable_pcie_error_reporting(pdev
);
1946 pci_set_master(pdev
);
1947 pci_save_state(pdev
);
1950 netdev
= alloc_etherdev_mq(sizeof(struct igb_adapter
),
1953 goto err_alloc_etherdev
;
1955 SET_NETDEV_DEV(netdev
, &pdev
->dev
);
1957 pci_set_drvdata(pdev
, netdev
);
1958 adapter
= netdev_priv(netdev
);
1959 adapter
->netdev
= netdev
;
1960 adapter
->pdev
= pdev
;
1963 adapter
->msg_enable
= netif_msg_init(debug
, DEFAULT_MSG_ENABLE
);
1965 mmio_start
= pci_resource_start(pdev
, 0);
1966 mmio_len
= pci_resource_len(pdev
, 0);
1969 hw
->hw_addr
= ioremap(mmio_start
, mmio_len
);
1973 netdev
->netdev_ops
= &igb_netdev_ops
;
1974 igb_set_ethtool_ops(netdev
);
1975 netdev
->watchdog_timeo
= 5 * HZ
;
1977 strncpy(netdev
->name
, pci_name(pdev
), sizeof(netdev
->name
) - 1);
1979 netdev
->mem_start
= mmio_start
;
1980 netdev
->mem_end
= mmio_start
+ mmio_len
;
1982 /* PCI config space info */
1983 hw
->vendor_id
= pdev
->vendor
;
1984 hw
->device_id
= pdev
->device
;
1985 hw
->revision_id
= pdev
->revision
;
1986 hw
->subsystem_vendor_id
= pdev
->subsystem_vendor
;
1987 hw
->subsystem_device_id
= pdev
->subsystem_device
;
1989 /* Copy the default MAC, PHY and NVM function pointers */
1990 memcpy(&hw
->mac
.ops
, ei
->mac_ops
, sizeof(hw
->mac
.ops
));
1991 memcpy(&hw
->phy
.ops
, ei
->phy_ops
, sizeof(hw
->phy
.ops
));
1992 memcpy(&hw
->nvm
.ops
, ei
->nvm_ops
, sizeof(hw
->nvm
.ops
));
1993 /* Initialize skew-specific constants */
1994 err
= ei
->get_invariants(hw
);
1998 /* setup the private structure */
1999 err
= igb_sw_init(adapter
);
2003 igb_get_bus_info_pcie(hw
);
2005 hw
->phy
.autoneg_wait_to_complete
= false;
2007 /* Copper options */
2008 if (hw
->phy
.media_type
== e1000_media_type_copper
) {
2009 hw
->phy
.mdix
= AUTO_ALL_MODES
;
2010 hw
->phy
.disable_polarity_correction
= false;
2011 hw
->phy
.ms_type
= e1000_ms_hw_default
;
2014 if (igb_check_reset_block(hw
))
2015 dev_info(&pdev
->dev
,
2016 "PHY reset is blocked due to SOL/IDER session.\n");
2019 * features is initialized to 0 in allocation, it might have bits
2020 * set by igb_sw_init so we should use an or instead of an
2023 netdev
->features
|= NETIF_F_SG
|
2030 NETIF_F_HW_VLAN_RX
|
2033 /* copy netdev features into list of user selectable features */
2034 netdev
->hw_features
|= netdev
->features
;
2035 netdev
->hw_features
|= NETIF_F_RXALL
;
2037 /* set this bit last since it cannot be part of hw_features */
2038 netdev
->features
|= NETIF_F_HW_VLAN_FILTER
;
2040 netdev
->vlan_features
|= NETIF_F_TSO
|
2046 netdev
->priv_flags
|= IFF_SUPP_NOFCS
;
2048 if (pci_using_dac
) {
2049 netdev
->features
|= NETIF_F_HIGHDMA
;
2050 netdev
->vlan_features
|= NETIF_F_HIGHDMA
;
2053 if (hw
->mac
.type
>= e1000_82576
) {
2054 netdev
->hw_features
|= NETIF_F_SCTP_CSUM
;
2055 netdev
->features
|= NETIF_F_SCTP_CSUM
;
2058 netdev
->priv_flags
|= IFF_UNICAST_FLT
;
2060 adapter
->en_mng_pt
= igb_enable_mng_pass_thru(hw
);
2062 /* before reading the NVM, reset the controller to put the device in a
2063 * known good starting state */
2064 hw
->mac
.ops
.reset_hw(hw
);
2067 * make sure the NVM is good , i211 parts have special NVM that
2068 * doesn't contain a checksum
2070 if (hw
->mac
.type
!= e1000_i211
) {
2071 if (hw
->nvm
.ops
.validate(hw
) < 0) {
2072 dev_err(&pdev
->dev
, "The NVM Checksum Is Not Valid\n");
2078 /* copy the MAC address out of the NVM */
2079 if (hw
->mac
.ops
.read_mac_addr(hw
))
2080 dev_err(&pdev
->dev
, "NVM Read Error\n");
2082 memcpy(netdev
->dev_addr
, hw
->mac
.addr
, netdev
->addr_len
);
2083 memcpy(netdev
->perm_addr
, hw
->mac
.addr
, netdev
->addr_len
);
2085 if (!is_valid_ether_addr(netdev
->perm_addr
)) {
2086 dev_err(&pdev
->dev
, "Invalid MAC Address\n");
2091 /* get firmware version for ethtool -i */
2092 igb_set_fw_version(adapter
);
2094 setup_timer(&adapter
->watchdog_timer
, igb_watchdog
,
2095 (unsigned long) adapter
);
2096 setup_timer(&adapter
->phy_info_timer
, igb_update_phy_info
,
2097 (unsigned long) adapter
);
2099 INIT_WORK(&adapter
->reset_task
, igb_reset_task
);
2100 INIT_WORK(&adapter
->watchdog_task
, igb_watchdog_task
);
2102 /* Initialize link properties that are user-changeable */
2103 adapter
->fc_autoneg
= true;
2104 hw
->mac
.autoneg
= true;
2105 hw
->phy
.autoneg_advertised
= 0x2f;
2107 hw
->fc
.requested_mode
= e1000_fc_default
;
2108 hw
->fc
.current_mode
= e1000_fc_default
;
2110 igb_validate_mdi_setting(hw
);
2112 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2113 * enable the ACPI Magic Packet filter
2116 if (hw
->bus
.func
== 0)
2117 hw
->nvm
.ops
.read(hw
, NVM_INIT_CONTROL3_PORT_A
, 1, &eeprom_data
);
2118 else if (hw
->mac
.type
>= e1000_82580
)
2119 hw
->nvm
.ops
.read(hw
, NVM_INIT_CONTROL3_PORT_A
+
2120 NVM_82580_LAN_FUNC_OFFSET(hw
->bus
.func
), 1,
2122 else if (hw
->bus
.func
== 1)
2123 hw
->nvm
.ops
.read(hw
, NVM_INIT_CONTROL3_PORT_B
, 1, &eeprom_data
);
2125 if (eeprom_data
& eeprom_apme_mask
)
2126 adapter
->eeprom_wol
|= E1000_WUFC_MAG
;
2128 /* now that we have the eeprom settings, apply the special cases where
2129 * the eeprom may be wrong or the board simply won't support wake on
2130 * lan on a particular port */
2131 switch (pdev
->device
) {
2132 case E1000_DEV_ID_82575GB_QUAD_COPPER
:
2133 adapter
->eeprom_wol
= 0;
2135 case E1000_DEV_ID_82575EB_FIBER_SERDES
:
2136 case E1000_DEV_ID_82576_FIBER
:
2137 case E1000_DEV_ID_82576_SERDES
:
2138 /* Wake events only supported on port A for dual fiber
2139 * regardless of eeprom setting */
2140 if (rd32(E1000_STATUS
) & E1000_STATUS_FUNC_1
)
2141 adapter
->eeprom_wol
= 0;
2143 case E1000_DEV_ID_82576_QUAD_COPPER
:
2144 case E1000_DEV_ID_82576_QUAD_COPPER_ET2
:
2145 /* if quad port adapter, disable WoL on all but port A */
2146 if (global_quad_port_a
!= 0)
2147 adapter
->eeprom_wol
= 0;
2149 adapter
->flags
|= IGB_FLAG_QUAD_PORT_A
;
2150 /* Reset for multiple quad port adapters */
2151 if (++global_quad_port_a
== 4)
2152 global_quad_port_a
= 0;
2156 /* initialize the wol settings based on the eeprom settings */
2157 adapter
->wol
= adapter
->eeprom_wol
;
2158 device_set_wakeup_enable(&adapter
->pdev
->dev
, adapter
->wol
);
2160 /* reset the hardware with the new settings */
2163 /* let the f/w know that the h/w is now under the control of the
2165 igb_get_hw_control(adapter
);
2167 strcpy(netdev
->name
, "eth%d");
2168 err
= register_netdev(netdev
);
2172 /* carrier off reporting is important to ethtool even BEFORE open */
2173 netif_carrier_off(netdev
);
2175 #ifdef CONFIG_IGB_DCA
2176 if (dca_add_requester(&pdev
->dev
) == 0) {
2177 adapter
->flags
|= IGB_FLAG_DCA_ENABLED
;
2178 dev_info(&pdev
->dev
, "DCA enabled\n");
2179 igb_setup_dca(adapter
);
2184 #ifdef CONFIG_IGB_PTP
2185 /* do hw tstamp init after resetting */
2186 igb_ptp_init(adapter
);
2187 #endif /* CONFIG_IGB_PTP */
2189 dev_info(&pdev
->dev
, "Intel(R) Gigabit Ethernet Network Connection\n");
2190 /* print bus type/speed/width info */
2191 dev_info(&pdev
->dev
, "%s: (PCIe:%s:%s) %pM\n",
2193 ((hw
->bus
.speed
== e1000_bus_speed_2500
) ? "2.5Gb/s" :
2194 (hw
->bus
.speed
== e1000_bus_speed_5000
) ? "5.0Gb/s" :
2196 ((hw
->bus
.width
== e1000_bus_width_pcie_x4
) ? "Width x4" :
2197 (hw
->bus
.width
== e1000_bus_width_pcie_x2
) ? "Width x2" :
2198 (hw
->bus
.width
== e1000_bus_width_pcie_x1
) ? "Width x1" :
2202 ret_val
= igb_read_part_string(hw
, part_str
, E1000_PBANUM_LENGTH
);
2204 strcpy(part_str
, "Unknown");
2205 dev_info(&pdev
->dev
, "%s: PBA No: %s\n", netdev
->name
, part_str
);
2206 dev_info(&pdev
->dev
,
2207 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2208 adapter
->msix_entries
? "MSI-X" :
2209 (adapter
->flags
& IGB_FLAG_HAS_MSI
) ? "MSI" : "legacy",
2210 adapter
->num_rx_queues
, adapter
->num_tx_queues
);
2211 switch (hw
->mac
.type
) {
2215 igb_set_eee_i350(hw
);
2221 pm_runtime_put_noidle(&pdev
->dev
);
2225 igb_release_hw_control(adapter
);
2227 if (!igb_check_reset_block(hw
))
2230 if (hw
->flash_address
)
2231 iounmap(hw
->flash_address
);
2233 igb_clear_interrupt_scheme(adapter
);
2234 iounmap(hw
->hw_addr
);
2236 free_netdev(netdev
);
2238 pci_release_selected_regions(pdev
,
2239 pci_select_bars(pdev
, IORESOURCE_MEM
));
2242 pci_disable_device(pdev
);
2247 * igb_remove - Device Removal Routine
2248 * @pdev: PCI device information struct
2250 * igb_remove is called by the PCI subsystem to alert the driver
2251 * that it should release a PCI device. The could be caused by a
2252 * Hot-Plug event, or because the driver is going to be removed from
2255 static void __devexit
igb_remove(struct pci_dev
*pdev
)
2257 struct net_device
*netdev
= pci_get_drvdata(pdev
);
2258 struct igb_adapter
*adapter
= netdev_priv(netdev
);
2259 struct e1000_hw
*hw
= &adapter
->hw
;
2261 pm_runtime_get_noresume(&pdev
->dev
);
2262 #ifdef CONFIG_IGB_PTP
2263 igb_ptp_remove(adapter
);
2264 #endif /* CONFIG_IGB_PTP */
2267 * The watchdog timer may be rescheduled, so explicitly
2268 * disable watchdog from being rescheduled.
2270 set_bit(__IGB_DOWN
, &adapter
->state
);
2271 del_timer_sync(&adapter
->watchdog_timer
);
2272 del_timer_sync(&adapter
->phy_info_timer
);
2274 cancel_work_sync(&adapter
->reset_task
);
2275 cancel_work_sync(&adapter
->watchdog_task
);
2277 #ifdef CONFIG_IGB_DCA
2278 if (adapter
->flags
& IGB_FLAG_DCA_ENABLED
) {
2279 dev_info(&pdev
->dev
, "DCA disabled\n");
2280 dca_remove_requester(&pdev
->dev
);
2281 adapter
->flags
&= ~IGB_FLAG_DCA_ENABLED
;
2282 wr32(E1000_DCA_CTRL
, E1000_DCA_CTRL_DCA_MODE_DISABLE
);
2286 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2287 * would have already happened in close and is redundant. */
2288 igb_release_hw_control(adapter
);
2290 unregister_netdev(netdev
);
2292 igb_clear_interrupt_scheme(adapter
);
2294 #ifdef CONFIG_PCI_IOV
2295 /* reclaim resources allocated to VFs */
2296 if (adapter
->vf_data
) {
2297 /* disable iov and allow time for transactions to clear */
2298 if (!igb_check_vf_assignment(adapter
)) {
2299 pci_disable_sriov(pdev
);
2302 dev_info(&pdev
->dev
, "VF(s) assigned to guests!\n");
2305 kfree(adapter
->vf_data
);
2306 adapter
->vf_data
= NULL
;
2307 wr32(E1000_IOVCTL
, E1000_IOVCTL_REUSE_VFQ
);
2310 dev_info(&pdev
->dev
, "IOV Disabled\n");
2314 iounmap(hw
->hw_addr
);
2315 if (hw
->flash_address
)
2316 iounmap(hw
->flash_address
);
2317 pci_release_selected_regions(pdev
,
2318 pci_select_bars(pdev
, IORESOURCE_MEM
));
2320 kfree(adapter
->shadow_vfta
);
2321 free_netdev(netdev
);
2323 pci_disable_pcie_error_reporting(pdev
);
2325 pci_disable_device(pdev
);
2329 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2330 * @adapter: board private structure to initialize
2332 * This function initializes the vf specific data storage and then attempts to
2333 * allocate the VFs. The reason for ordering it this way is because it is much
2334 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2335 * the memory for the VFs.
2337 static void __devinit
igb_probe_vfs(struct igb_adapter
* adapter
)
2339 #ifdef CONFIG_PCI_IOV
2340 struct pci_dev
*pdev
= adapter
->pdev
;
2341 struct e1000_hw
*hw
= &adapter
->hw
;
2342 int old_vfs
= igb_find_enabled_vfs(adapter
);
2345 /* Virtualization features not supported on i210 family. */
2346 if ((hw
->mac
.type
== e1000_i210
) || (hw
->mac
.type
== e1000_i211
))
2350 dev_info(&pdev
->dev
, "%d pre-allocated VFs found - override "
2351 "max_vfs setting of %d\n", old_vfs
, max_vfs
);
2352 adapter
->vfs_allocated_count
= old_vfs
;
2355 if (!adapter
->vfs_allocated_count
)
2358 adapter
->vf_data
= kcalloc(adapter
->vfs_allocated_count
,
2359 sizeof(struct vf_data_storage
), GFP_KERNEL
);
2361 /* if allocation failed then we do not support SR-IOV */
2362 if (!adapter
->vf_data
) {
2363 adapter
->vfs_allocated_count
= 0;
2364 dev_err(&pdev
->dev
, "Unable to allocate memory for VF "
2370 if (pci_enable_sriov(pdev
, adapter
->vfs_allocated_count
))
2373 dev_info(&pdev
->dev
, "%d VFs allocated\n",
2374 adapter
->vfs_allocated_count
);
2375 for (i
= 0; i
< adapter
->vfs_allocated_count
; i
++)
2376 igb_vf_configure(adapter
, i
);
2378 /* DMA Coalescing is not supported in IOV mode. */
2379 adapter
->flags
&= ~IGB_FLAG_DMAC
;
2382 kfree(adapter
->vf_data
);
2383 adapter
->vf_data
= NULL
;
2384 adapter
->vfs_allocated_count
= 0;
2387 #endif /* CONFIG_PCI_IOV */
2391 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2392 * @adapter: board private structure to initialize
2394 * igb_sw_init initializes the Adapter private data structure.
2395 * Fields are initialized based on PCI device information and
2396 * OS network device settings (MTU size).
2398 static int __devinit
igb_sw_init(struct igb_adapter
*adapter
)
2400 struct e1000_hw
*hw
= &adapter
->hw
;
2401 struct net_device
*netdev
= adapter
->netdev
;
2402 struct pci_dev
*pdev
= adapter
->pdev
;
2405 pci_read_config_word(pdev
, PCI_COMMAND
, &hw
->bus
.pci_cmd_word
);
2407 /* set default ring sizes */
2408 adapter
->tx_ring_count
= IGB_DEFAULT_TXD
;
2409 adapter
->rx_ring_count
= IGB_DEFAULT_RXD
;
2411 /* set default ITR values */
2412 adapter
->rx_itr_setting
= IGB_DEFAULT_ITR
;
2413 adapter
->tx_itr_setting
= IGB_DEFAULT_ITR
;
2415 /* set default work limits */
2416 adapter
->tx_work_limit
= IGB_DEFAULT_TX_WORK
;
2418 adapter
->max_frame_size
= netdev
->mtu
+ ETH_HLEN
+ ETH_FCS_LEN
+
2420 adapter
->min_frame_size
= ETH_ZLEN
+ ETH_FCS_LEN
;
2424 spin_lock_init(&adapter
->stats64_lock
);
2425 #ifdef CONFIG_PCI_IOV
2426 switch (hw
->mac
.type
) {
2430 dev_warn(&pdev
->dev
,
2431 "Maximum of 7 VFs per PF, using max\n");
2432 adapter
->vfs_allocated_count
= 7;
2434 adapter
->vfs_allocated_count
= max_vfs
;
2439 #endif /* CONFIG_PCI_IOV */
2441 /* Determine the maximum number of RSS queues supported. */
2442 switch (hw
->mac
.type
) {
2444 max_rss_queues
= IGB_MAX_RX_QUEUES_I211
;
2448 max_rss_queues
= IGB_MAX_RX_QUEUES_82575
;
2451 /* I350 cannot do RSS and SR-IOV at the same time */
2452 if (!!adapter
->vfs_allocated_count
) {
2458 if (!!adapter
->vfs_allocated_count
) {
2465 max_rss_queues
= IGB_MAX_RX_QUEUES
;
2469 adapter
->rss_queues
= min_t(u32
, max_rss_queues
, num_online_cpus());
2471 /* Determine if we need to pair queues. */
2472 switch (hw
->mac
.type
) {
2475 /* Device supports enough interrupts without queue pairing. */
2479 * If VFs are going to be allocated with RSS queues then we
2480 * should pair the queues in order to conserve interrupts due
2481 * to limited supply.
2483 if ((adapter
->rss_queues
> 1) &&
2484 (adapter
->vfs_allocated_count
> 6))
2485 adapter
->flags
|= IGB_FLAG_QUEUE_PAIRS
;
2492 * If rss_queues > half of max_rss_queues, pair the queues in
2493 * order to conserve interrupts due to limited supply.
2495 if (adapter
->rss_queues
> (max_rss_queues
/ 2))
2496 adapter
->flags
|= IGB_FLAG_QUEUE_PAIRS
;
2500 /* Setup and initialize a copy of the hw vlan table array */
2501 adapter
->shadow_vfta
= kzalloc(sizeof(u32
) *
2502 E1000_VLAN_FILTER_TBL_SIZE
,
2505 /* This call may decrease the number of queues */
2506 if (igb_init_interrupt_scheme(adapter
)) {
2507 dev_err(&pdev
->dev
, "Unable to allocate memory for queues\n");
2511 igb_probe_vfs(adapter
);
2513 /* Explicitly disable IRQ since the NIC can be in any state. */
2514 igb_irq_disable(adapter
);
2516 if (hw
->mac
.type
>= e1000_i350
)
2517 adapter
->flags
&= ~IGB_FLAG_DMAC
;
2519 set_bit(__IGB_DOWN
, &adapter
->state
);
2524 * igb_open - Called when a network interface is made active
2525 * @netdev: network interface device structure
2527 * Returns 0 on success, negative value on failure
2529 * The open entry point is called when a network interface is made
2530 * active by the system (IFF_UP). At this point all resources needed
2531 * for transmit and receive operations are allocated, the interrupt
2532 * handler is registered with the OS, the watchdog timer is started,
2533 * and the stack is notified that the interface is ready.
2535 static int __igb_open(struct net_device
*netdev
, bool resuming
)
2537 struct igb_adapter
*adapter
= netdev_priv(netdev
);
2538 struct e1000_hw
*hw
= &adapter
->hw
;
2539 struct pci_dev
*pdev
= adapter
->pdev
;
2543 /* disallow open during test */
2544 if (test_bit(__IGB_TESTING
, &adapter
->state
)) {
2550 pm_runtime_get_sync(&pdev
->dev
);
2552 netif_carrier_off(netdev
);
2554 /* allocate transmit descriptors */
2555 err
= igb_setup_all_tx_resources(adapter
);
2559 /* allocate receive descriptors */
2560 err
= igb_setup_all_rx_resources(adapter
);
2564 igb_power_up_link(adapter
);
2566 /* before we allocate an interrupt, we must be ready to handle it.
2567 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2568 * as soon as we call pci_request_irq, so we have to setup our
2569 * clean_rx handler before we do so. */
2570 igb_configure(adapter
);
2572 err
= igb_request_irq(adapter
);
2576 /* From here on the code is the same as igb_up() */
2577 clear_bit(__IGB_DOWN
, &adapter
->state
);
2579 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
2580 napi_enable(&(adapter
->q_vector
[i
]->napi
));
2582 /* Clear any pending interrupts. */
2585 igb_irq_enable(adapter
);
2587 /* notify VFs that reset has been completed */
2588 if (adapter
->vfs_allocated_count
) {
2589 u32 reg_data
= rd32(E1000_CTRL_EXT
);
2590 reg_data
|= E1000_CTRL_EXT_PFRSTD
;
2591 wr32(E1000_CTRL_EXT
, reg_data
);
2594 netif_tx_start_all_queues(netdev
);
2597 pm_runtime_put(&pdev
->dev
);
2599 /* start the watchdog. */
2600 hw
->mac
.get_link_status
= 1;
2601 schedule_work(&adapter
->watchdog_task
);
2606 igb_release_hw_control(adapter
);
2607 igb_power_down_link(adapter
);
2608 igb_free_all_rx_resources(adapter
);
2610 igb_free_all_tx_resources(adapter
);
2614 pm_runtime_put(&pdev
->dev
);
2619 static int igb_open(struct net_device
*netdev
)
2621 return __igb_open(netdev
, false);
2625 * igb_close - Disables a network interface
2626 * @netdev: network interface device structure
2628 * Returns 0, this is not allowed to fail
2630 * The close entry point is called when an interface is de-activated
2631 * by the OS. The hardware is still under the driver's control, but
2632 * needs to be disabled. A global MAC reset is issued to stop the
2633 * hardware, and all transmit and receive resources are freed.
2635 static int __igb_close(struct net_device
*netdev
, bool suspending
)
2637 struct igb_adapter
*adapter
= netdev_priv(netdev
);
2638 struct pci_dev
*pdev
= adapter
->pdev
;
2640 WARN_ON(test_bit(__IGB_RESETTING
, &adapter
->state
));
2643 pm_runtime_get_sync(&pdev
->dev
);
2646 igb_free_irq(adapter
);
2648 igb_free_all_tx_resources(adapter
);
2649 igb_free_all_rx_resources(adapter
);
2652 pm_runtime_put_sync(&pdev
->dev
);
2656 static int igb_close(struct net_device
*netdev
)
2658 return __igb_close(netdev
, false);
2662 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2663 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2665 * Return 0 on success, negative on failure
2667 int igb_setup_tx_resources(struct igb_ring
*tx_ring
)
2669 struct device
*dev
= tx_ring
->dev
;
2670 int orig_node
= dev_to_node(dev
);
2673 size
= sizeof(struct igb_tx_buffer
) * tx_ring
->count
;
2674 tx_ring
->tx_buffer_info
= vzalloc_node(size
, tx_ring
->numa_node
);
2675 if (!tx_ring
->tx_buffer_info
)
2676 tx_ring
->tx_buffer_info
= vzalloc(size
);
2677 if (!tx_ring
->tx_buffer_info
)
2680 /* round up to nearest 4K */
2681 tx_ring
->size
= tx_ring
->count
* sizeof(union e1000_adv_tx_desc
);
2682 tx_ring
->size
= ALIGN(tx_ring
->size
, 4096);
2684 set_dev_node(dev
, tx_ring
->numa_node
);
2685 tx_ring
->desc
= dma_alloc_coherent(dev
,
2689 set_dev_node(dev
, orig_node
);
2691 tx_ring
->desc
= dma_alloc_coherent(dev
,
2699 tx_ring
->next_to_use
= 0;
2700 tx_ring
->next_to_clean
= 0;
2705 vfree(tx_ring
->tx_buffer_info
);
2707 "Unable to allocate memory for the transmit descriptor ring\n");
2712 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2713 * (Descriptors) for all queues
2714 * @adapter: board private structure
2716 * Return 0 on success, negative on failure
2718 static int igb_setup_all_tx_resources(struct igb_adapter
*adapter
)
2720 struct pci_dev
*pdev
= adapter
->pdev
;
2723 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
2724 err
= igb_setup_tx_resources(adapter
->tx_ring
[i
]);
2727 "Allocation for Tx Queue %u failed\n", i
);
2728 for (i
--; i
>= 0; i
--)
2729 igb_free_tx_resources(adapter
->tx_ring
[i
]);
2738 * igb_setup_tctl - configure the transmit control registers
2739 * @adapter: Board private structure
2741 void igb_setup_tctl(struct igb_adapter
*adapter
)
2743 struct e1000_hw
*hw
= &adapter
->hw
;
2746 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2747 wr32(E1000_TXDCTL(0), 0);
2749 /* Program the Transmit Control Register */
2750 tctl
= rd32(E1000_TCTL
);
2751 tctl
&= ~E1000_TCTL_CT
;
2752 tctl
|= E1000_TCTL_PSP
| E1000_TCTL_RTLC
|
2753 (E1000_COLLISION_THRESHOLD
<< E1000_CT_SHIFT
);
2755 igb_config_collision_dist(hw
);
2757 /* Enable transmits */
2758 tctl
|= E1000_TCTL_EN
;
2760 wr32(E1000_TCTL
, tctl
);
2764 * igb_configure_tx_ring - Configure transmit ring after Reset
2765 * @adapter: board private structure
2766 * @ring: tx ring to configure
2768 * Configure a transmit ring after a reset.
2770 void igb_configure_tx_ring(struct igb_adapter
*adapter
,
2771 struct igb_ring
*ring
)
2773 struct e1000_hw
*hw
= &adapter
->hw
;
2775 u64 tdba
= ring
->dma
;
2776 int reg_idx
= ring
->reg_idx
;
2778 /* disable the queue */
2779 wr32(E1000_TXDCTL(reg_idx
), 0);
2783 wr32(E1000_TDLEN(reg_idx
),
2784 ring
->count
* sizeof(union e1000_adv_tx_desc
));
2785 wr32(E1000_TDBAL(reg_idx
),
2786 tdba
& 0x00000000ffffffffULL
);
2787 wr32(E1000_TDBAH(reg_idx
), tdba
>> 32);
2789 ring
->tail
= hw
->hw_addr
+ E1000_TDT(reg_idx
);
2790 wr32(E1000_TDH(reg_idx
), 0);
2791 writel(0, ring
->tail
);
2793 txdctl
|= IGB_TX_PTHRESH
;
2794 txdctl
|= IGB_TX_HTHRESH
<< 8;
2795 txdctl
|= IGB_TX_WTHRESH
<< 16;
2797 txdctl
|= E1000_TXDCTL_QUEUE_ENABLE
;
2798 wr32(E1000_TXDCTL(reg_idx
), txdctl
);
2802 * igb_configure_tx - Configure transmit Unit after Reset
2803 * @adapter: board private structure
2805 * Configure the Tx unit of the MAC after a reset.
2807 static void igb_configure_tx(struct igb_adapter
*adapter
)
2811 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
2812 igb_configure_tx_ring(adapter
, adapter
->tx_ring
[i
]);
2816 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2817 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2819 * Returns 0 on success, negative on failure
2821 int igb_setup_rx_resources(struct igb_ring
*rx_ring
)
2823 struct device
*dev
= rx_ring
->dev
;
2824 int orig_node
= dev_to_node(dev
);
2827 size
= sizeof(struct igb_rx_buffer
) * rx_ring
->count
;
2828 rx_ring
->rx_buffer_info
= vzalloc_node(size
, rx_ring
->numa_node
);
2829 if (!rx_ring
->rx_buffer_info
)
2830 rx_ring
->rx_buffer_info
= vzalloc(size
);
2831 if (!rx_ring
->rx_buffer_info
)
2834 desc_len
= sizeof(union e1000_adv_rx_desc
);
2836 /* Round up to nearest 4K */
2837 rx_ring
->size
= rx_ring
->count
* desc_len
;
2838 rx_ring
->size
= ALIGN(rx_ring
->size
, 4096);
2840 set_dev_node(dev
, rx_ring
->numa_node
);
2841 rx_ring
->desc
= dma_alloc_coherent(dev
,
2845 set_dev_node(dev
, orig_node
);
2847 rx_ring
->desc
= dma_alloc_coherent(dev
,
2855 rx_ring
->next_to_clean
= 0;
2856 rx_ring
->next_to_use
= 0;
2861 vfree(rx_ring
->rx_buffer_info
);
2862 rx_ring
->rx_buffer_info
= NULL
;
2863 dev_err(dev
, "Unable to allocate memory for the receive descriptor"
2869 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2870 * (Descriptors) for all queues
2871 * @adapter: board private structure
2873 * Return 0 on success, negative on failure
2875 static int igb_setup_all_rx_resources(struct igb_adapter
*adapter
)
2877 struct pci_dev
*pdev
= adapter
->pdev
;
2880 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
2881 err
= igb_setup_rx_resources(adapter
->rx_ring
[i
]);
2884 "Allocation for Rx Queue %u failed\n", i
);
2885 for (i
--; i
>= 0; i
--)
2886 igb_free_rx_resources(adapter
->rx_ring
[i
]);
2895 * igb_setup_mrqc - configure the multiple receive queue control registers
2896 * @adapter: Board private structure
2898 static void igb_setup_mrqc(struct igb_adapter
*adapter
)
2900 struct e1000_hw
*hw
= &adapter
->hw
;
2902 u32 j
, num_rx_queues
, shift
= 0, shift2
= 0;
2907 static const u8 rsshash
[40] = {
2908 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2909 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2910 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2911 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2913 /* Fill out hash function seeds */
2914 for (j
= 0; j
< 10; j
++) {
2915 u32 rsskey
= rsshash
[(j
* 4)];
2916 rsskey
|= rsshash
[(j
* 4) + 1] << 8;
2917 rsskey
|= rsshash
[(j
* 4) + 2] << 16;
2918 rsskey
|= rsshash
[(j
* 4) + 3] << 24;
2919 array_wr32(E1000_RSSRK(0), j
, rsskey
);
2922 num_rx_queues
= adapter
->rss_queues
;
2924 if (adapter
->vfs_allocated_count
) {
2925 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2926 switch (hw
->mac
.type
) {
2943 if (hw
->mac
.type
== e1000_82575
)
2947 for (j
= 0; j
< (32 * 4); j
++) {
2948 reta
.bytes
[j
& 3] = (j
% num_rx_queues
) << shift
;
2950 reta
.bytes
[j
& 3] |= num_rx_queues
<< shift2
;
2952 wr32(E1000_RETA(j
>> 2), reta
.dword
);
2956 * Disable raw packet checksumming so that RSS hash is placed in
2957 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2958 * offloads as they are enabled by default
2960 rxcsum
= rd32(E1000_RXCSUM
);
2961 rxcsum
|= E1000_RXCSUM_PCSD
;
2963 if (adapter
->hw
.mac
.type
>= e1000_82576
)
2964 /* Enable Receive Checksum Offload for SCTP */
2965 rxcsum
|= E1000_RXCSUM_CRCOFL
;
2967 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2968 wr32(E1000_RXCSUM
, rxcsum
);
2970 * Generate RSS hash based on TCP port numbers and/or
2971 * IPv4/v6 src and dst addresses since UDP cannot be
2972 * hashed reliably due to IP fragmentation
2975 mrqc
= E1000_MRQC_RSS_FIELD_IPV4
|
2976 E1000_MRQC_RSS_FIELD_IPV4_TCP
|
2977 E1000_MRQC_RSS_FIELD_IPV6
|
2978 E1000_MRQC_RSS_FIELD_IPV6_TCP
|
2979 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX
;
2981 /* If VMDq is enabled then we set the appropriate mode for that, else
2982 * we default to RSS so that an RSS hash is calculated per packet even
2983 * if we are only using one queue */
2984 if (adapter
->vfs_allocated_count
) {
2985 if (hw
->mac
.type
> e1000_82575
) {
2986 /* Set the default pool for the PF's first queue */
2987 u32 vtctl
= rd32(E1000_VT_CTL
);
2988 vtctl
&= ~(E1000_VT_CTL_DEFAULT_POOL_MASK
|
2989 E1000_VT_CTL_DISABLE_DEF_POOL
);
2990 vtctl
|= adapter
->vfs_allocated_count
<<
2991 E1000_VT_CTL_DEFAULT_POOL_SHIFT
;
2992 wr32(E1000_VT_CTL
, vtctl
);
2994 if (adapter
->rss_queues
> 1)
2995 mrqc
|= E1000_MRQC_ENABLE_VMDQ_RSS_2Q
;
2997 mrqc
|= E1000_MRQC_ENABLE_VMDQ
;
2999 if (hw
->mac
.type
!= e1000_i211
)
3000 mrqc
|= E1000_MRQC_ENABLE_RSS_4Q
;
3002 igb_vmm_control(adapter
);
3004 wr32(E1000_MRQC
, mrqc
);
3008 * igb_setup_rctl - configure the receive control registers
3009 * @adapter: Board private structure
3011 void igb_setup_rctl(struct igb_adapter
*adapter
)
3013 struct e1000_hw
*hw
= &adapter
->hw
;
3016 rctl
= rd32(E1000_RCTL
);
3018 rctl
&= ~(3 << E1000_RCTL_MO_SHIFT
);
3019 rctl
&= ~(E1000_RCTL_LBM_TCVR
| E1000_RCTL_LBM_MAC
);
3021 rctl
|= E1000_RCTL_EN
| E1000_RCTL_BAM
| E1000_RCTL_RDMTS_HALF
|
3022 (hw
->mac
.mc_filter_type
<< E1000_RCTL_MO_SHIFT
);
3025 * enable stripping of CRC. It's unlikely this will break BMC
3026 * redirection as it did with e1000. Newer features require
3027 * that the HW strips the CRC.
3029 rctl
|= E1000_RCTL_SECRC
;
3031 /* disable store bad packets and clear size bits. */
3032 rctl
&= ~(E1000_RCTL_SBP
| E1000_RCTL_SZ_256
);
3034 /* enable LPE to prevent packets larger than max_frame_size */
3035 rctl
|= E1000_RCTL_LPE
;
3037 /* disable queue 0 to prevent tail write w/o re-config */
3038 wr32(E1000_RXDCTL(0), 0);
3040 /* Attention!!! For SR-IOV PF driver operations you must enable
3041 * queue drop for all VF and PF queues to prevent head of line blocking
3042 * if an un-trusted VF does not provide descriptors to hardware.
3044 if (adapter
->vfs_allocated_count
) {
3045 /* set all queue drop enable bits */
3046 wr32(E1000_QDE
, ALL_QUEUES
);
3049 /* This is useful for sniffing bad packets. */
3050 if (adapter
->netdev
->features
& NETIF_F_RXALL
) {
3051 /* UPE and MPE will be handled by normal PROMISC logic
3052 * in e1000e_set_rx_mode */
3053 rctl
|= (E1000_RCTL_SBP
| /* Receive bad packets */
3054 E1000_RCTL_BAM
| /* RX All Bcast Pkts */
3055 E1000_RCTL_PMCF
); /* RX All MAC Ctrl Pkts */
3057 rctl
&= ~(E1000_RCTL_VFE
| /* Disable VLAN filter */
3058 E1000_RCTL_DPF
| /* Allow filtered pause */
3059 E1000_RCTL_CFIEN
); /* Dis VLAN CFIEN Filter */
3060 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3061 * and that breaks VLANs.
3065 wr32(E1000_RCTL
, rctl
);
3068 static inline int igb_set_vf_rlpml(struct igb_adapter
*adapter
, int size
,
3071 struct e1000_hw
*hw
= &adapter
->hw
;
3074 /* if it isn't the PF check to see if VFs are enabled and
3075 * increase the size to support vlan tags */
3076 if (vfn
< adapter
->vfs_allocated_count
&&
3077 adapter
->vf_data
[vfn
].vlans_enabled
)
3078 size
+= VLAN_TAG_SIZE
;
3080 vmolr
= rd32(E1000_VMOLR(vfn
));
3081 vmolr
&= ~E1000_VMOLR_RLPML_MASK
;
3082 vmolr
|= size
| E1000_VMOLR_LPE
;
3083 wr32(E1000_VMOLR(vfn
), vmolr
);
3089 * igb_rlpml_set - set maximum receive packet size
3090 * @adapter: board private structure
3092 * Configure maximum receivable packet size.
3094 static void igb_rlpml_set(struct igb_adapter
*adapter
)
3096 u32 max_frame_size
= adapter
->max_frame_size
;
3097 struct e1000_hw
*hw
= &adapter
->hw
;
3098 u16 pf_id
= adapter
->vfs_allocated_count
;
3101 igb_set_vf_rlpml(adapter
, max_frame_size
, pf_id
);
3103 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3104 * to our max jumbo frame size, in case we need to enable
3105 * jumbo frames on one of the rings later.
3106 * This will not pass over-length frames into the default
3107 * queue because it's gated by the VMOLR.RLPML.
3109 max_frame_size
= MAX_JUMBO_FRAME_SIZE
;
3112 wr32(E1000_RLPML
, max_frame_size
);
3115 static inline void igb_set_vmolr(struct igb_adapter
*adapter
,
3118 struct e1000_hw
*hw
= &adapter
->hw
;
3122 * This register exists only on 82576 and newer so if we are older then
3123 * we should exit and do nothing
3125 if (hw
->mac
.type
< e1000_82576
)
3128 vmolr
= rd32(E1000_VMOLR(vfn
));
3129 vmolr
|= E1000_VMOLR_STRVLAN
; /* Strip vlan tags */
3131 vmolr
|= E1000_VMOLR_AUPE
; /* Accept untagged packets */
3133 vmolr
&= ~(E1000_VMOLR_AUPE
); /* Tagged packets ONLY */
3135 /* clear all bits that might not be set */
3136 vmolr
&= ~(E1000_VMOLR_BAM
| E1000_VMOLR_RSSE
);
3138 if (adapter
->rss_queues
> 1 && vfn
== adapter
->vfs_allocated_count
)
3139 vmolr
|= E1000_VMOLR_RSSE
; /* enable RSS */
3141 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3144 if (vfn
<= adapter
->vfs_allocated_count
)
3145 vmolr
|= E1000_VMOLR_BAM
; /* Accept broadcast */
3147 wr32(E1000_VMOLR(vfn
), vmolr
);
3151 * igb_configure_rx_ring - Configure a receive ring after Reset
3152 * @adapter: board private structure
3153 * @ring: receive ring to be configured
3155 * Configure the Rx unit of the MAC after a reset.
3157 void igb_configure_rx_ring(struct igb_adapter
*adapter
,
3158 struct igb_ring
*ring
)
3160 struct e1000_hw
*hw
= &adapter
->hw
;
3161 u64 rdba
= ring
->dma
;
3162 int reg_idx
= ring
->reg_idx
;
3163 u32 srrctl
= 0, rxdctl
= 0;
3165 /* disable the queue */
3166 wr32(E1000_RXDCTL(reg_idx
), 0);
3168 /* Set DMA base address registers */
3169 wr32(E1000_RDBAL(reg_idx
),
3170 rdba
& 0x00000000ffffffffULL
);
3171 wr32(E1000_RDBAH(reg_idx
), rdba
>> 32);
3172 wr32(E1000_RDLEN(reg_idx
),
3173 ring
->count
* sizeof(union e1000_adv_rx_desc
));
3175 /* initialize head and tail */
3176 ring
->tail
= hw
->hw_addr
+ E1000_RDT(reg_idx
);
3177 wr32(E1000_RDH(reg_idx
), 0);
3178 writel(0, ring
->tail
);
3180 /* set descriptor configuration */
3181 srrctl
= IGB_RX_HDR_LEN
<< E1000_SRRCTL_BSIZEHDRSIZE_SHIFT
;
3182 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3183 srrctl
|= IGB_RXBUFFER_16384
>> E1000_SRRCTL_BSIZEPKT_SHIFT
;
3185 srrctl
|= (PAGE_SIZE
/ 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT
;
3187 srrctl
|= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS
;
3188 #ifdef CONFIG_IGB_PTP
3189 if (hw
->mac
.type
>= e1000_82580
)
3190 srrctl
|= E1000_SRRCTL_TIMESTAMP
;
3191 #endif /* CONFIG_IGB_PTP */
3192 /* Only set Drop Enable if we are supporting multiple queues */
3193 if (adapter
->vfs_allocated_count
|| adapter
->num_rx_queues
> 1)
3194 srrctl
|= E1000_SRRCTL_DROP_EN
;
3196 wr32(E1000_SRRCTL(reg_idx
), srrctl
);
3198 /* set filtering for VMDQ pools */
3199 igb_set_vmolr(adapter
, reg_idx
& 0x7, true);
3201 rxdctl
|= IGB_RX_PTHRESH
;
3202 rxdctl
|= IGB_RX_HTHRESH
<< 8;
3203 rxdctl
|= IGB_RX_WTHRESH
<< 16;
3205 /* enable receive descriptor fetching */
3206 rxdctl
|= E1000_RXDCTL_QUEUE_ENABLE
;
3207 wr32(E1000_RXDCTL(reg_idx
), rxdctl
);
3211 * igb_configure_rx - Configure receive Unit after Reset
3212 * @adapter: board private structure
3214 * Configure the Rx unit of the MAC after a reset.
3216 static void igb_configure_rx(struct igb_adapter
*adapter
)
3220 /* set UTA to appropriate mode */
3221 igb_set_uta(adapter
);
3223 /* set the correct pool for the PF default MAC address in entry 0 */
3224 igb_rar_set_qsel(adapter
, adapter
->hw
.mac
.addr
, 0,
3225 adapter
->vfs_allocated_count
);
3227 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3228 * the Base and Length of the Rx Descriptor Ring */
3229 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
3230 igb_configure_rx_ring(adapter
, adapter
->rx_ring
[i
]);
3234 * igb_free_tx_resources - Free Tx Resources per Queue
3235 * @tx_ring: Tx descriptor ring for a specific queue
3237 * Free all transmit software resources
3239 void igb_free_tx_resources(struct igb_ring
*tx_ring
)
3241 igb_clean_tx_ring(tx_ring
);
3243 vfree(tx_ring
->tx_buffer_info
);
3244 tx_ring
->tx_buffer_info
= NULL
;
3246 /* if not set, then don't free */
3250 dma_free_coherent(tx_ring
->dev
, tx_ring
->size
,
3251 tx_ring
->desc
, tx_ring
->dma
);
3253 tx_ring
->desc
= NULL
;
3257 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3258 * @adapter: board private structure
3260 * Free all transmit software resources
3262 static void igb_free_all_tx_resources(struct igb_adapter
*adapter
)
3266 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
3267 igb_free_tx_resources(adapter
->tx_ring
[i
]);
3270 void igb_unmap_and_free_tx_resource(struct igb_ring
*ring
,
3271 struct igb_tx_buffer
*tx_buffer
)
3273 if (tx_buffer
->skb
) {
3274 dev_kfree_skb_any(tx_buffer
->skb
);
3276 dma_unmap_single(ring
->dev
,
3280 } else if (tx_buffer
->dma
) {
3281 dma_unmap_page(ring
->dev
,
3286 tx_buffer
->next_to_watch
= NULL
;
3287 tx_buffer
->skb
= NULL
;
3289 /* buffer_info must be completely set up in the transmit path */
3293 * igb_clean_tx_ring - Free Tx Buffers
3294 * @tx_ring: ring to be cleaned
3296 static void igb_clean_tx_ring(struct igb_ring
*tx_ring
)
3298 struct igb_tx_buffer
*buffer_info
;
3302 if (!tx_ring
->tx_buffer_info
)
3304 /* Free all the Tx ring sk_buffs */
3306 for (i
= 0; i
< tx_ring
->count
; i
++) {
3307 buffer_info
= &tx_ring
->tx_buffer_info
[i
];
3308 igb_unmap_and_free_tx_resource(tx_ring
, buffer_info
);
3311 netdev_tx_reset_queue(txring_txq(tx_ring
));
3313 size
= sizeof(struct igb_tx_buffer
) * tx_ring
->count
;
3314 memset(tx_ring
->tx_buffer_info
, 0, size
);
3316 /* Zero out the descriptor ring */
3317 memset(tx_ring
->desc
, 0, tx_ring
->size
);
3319 tx_ring
->next_to_use
= 0;
3320 tx_ring
->next_to_clean
= 0;
3324 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3325 * @adapter: board private structure
3327 static void igb_clean_all_tx_rings(struct igb_adapter
*adapter
)
3331 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
3332 igb_clean_tx_ring(adapter
->tx_ring
[i
]);
3336 * igb_free_rx_resources - Free Rx Resources
3337 * @rx_ring: ring to clean the resources from
3339 * Free all receive software resources
3341 void igb_free_rx_resources(struct igb_ring
*rx_ring
)
3343 igb_clean_rx_ring(rx_ring
);
3345 vfree(rx_ring
->rx_buffer_info
);
3346 rx_ring
->rx_buffer_info
= NULL
;
3348 /* if not set, then don't free */
3352 dma_free_coherent(rx_ring
->dev
, rx_ring
->size
,
3353 rx_ring
->desc
, rx_ring
->dma
);
3355 rx_ring
->desc
= NULL
;
3359 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3360 * @adapter: board private structure
3362 * Free all receive software resources
3364 static void igb_free_all_rx_resources(struct igb_adapter
*adapter
)
3368 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
3369 igb_free_rx_resources(adapter
->rx_ring
[i
]);
3373 * igb_clean_rx_ring - Free Rx Buffers per Queue
3374 * @rx_ring: ring to free buffers from
3376 static void igb_clean_rx_ring(struct igb_ring
*rx_ring
)
3381 if (!rx_ring
->rx_buffer_info
)
3384 /* Free all the Rx ring sk_buffs */
3385 for (i
= 0; i
< rx_ring
->count
; i
++) {
3386 struct igb_rx_buffer
*buffer_info
= &rx_ring
->rx_buffer_info
[i
];
3387 if (buffer_info
->dma
) {
3388 dma_unmap_single(rx_ring
->dev
,
3392 buffer_info
->dma
= 0;
3395 if (buffer_info
->skb
) {
3396 dev_kfree_skb(buffer_info
->skb
);
3397 buffer_info
->skb
= NULL
;
3399 if (buffer_info
->page_dma
) {
3400 dma_unmap_page(rx_ring
->dev
,
3401 buffer_info
->page_dma
,
3404 buffer_info
->page_dma
= 0;
3406 if (buffer_info
->page
) {
3407 put_page(buffer_info
->page
);
3408 buffer_info
->page
= NULL
;
3409 buffer_info
->page_offset
= 0;
3413 size
= sizeof(struct igb_rx_buffer
) * rx_ring
->count
;
3414 memset(rx_ring
->rx_buffer_info
, 0, size
);
3416 /* Zero out the descriptor ring */
3417 memset(rx_ring
->desc
, 0, rx_ring
->size
);
3419 rx_ring
->next_to_clean
= 0;
3420 rx_ring
->next_to_use
= 0;
3424 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3425 * @adapter: board private structure
3427 static void igb_clean_all_rx_rings(struct igb_adapter
*adapter
)
3431 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
3432 igb_clean_rx_ring(adapter
->rx_ring
[i
]);
3436 * igb_set_mac - Change the Ethernet Address of the NIC
3437 * @netdev: network interface device structure
3438 * @p: pointer to an address structure
3440 * Returns 0 on success, negative on failure
3442 static int igb_set_mac(struct net_device
*netdev
, void *p
)
3444 struct igb_adapter
*adapter
= netdev_priv(netdev
);
3445 struct e1000_hw
*hw
= &adapter
->hw
;
3446 struct sockaddr
*addr
= p
;
3448 if (!is_valid_ether_addr(addr
->sa_data
))
3449 return -EADDRNOTAVAIL
;
3451 memcpy(netdev
->dev_addr
, addr
->sa_data
, netdev
->addr_len
);
3452 memcpy(hw
->mac
.addr
, addr
->sa_data
, netdev
->addr_len
);
3454 /* set the correct pool for the new PF MAC address in entry 0 */
3455 igb_rar_set_qsel(adapter
, hw
->mac
.addr
, 0,
3456 adapter
->vfs_allocated_count
);
3462 * igb_write_mc_addr_list - write multicast addresses to MTA
3463 * @netdev: network interface device structure
3465 * Writes multicast address list to the MTA hash table.
3466 * Returns: -ENOMEM on failure
3467 * 0 on no addresses written
3468 * X on writing X addresses to MTA
3470 static int igb_write_mc_addr_list(struct net_device
*netdev
)
3472 struct igb_adapter
*adapter
= netdev_priv(netdev
);
3473 struct e1000_hw
*hw
= &adapter
->hw
;
3474 struct netdev_hw_addr
*ha
;
3478 if (netdev_mc_empty(netdev
)) {
3479 /* nothing to program, so clear mc list */
3480 igb_update_mc_addr_list(hw
, NULL
, 0);
3481 igb_restore_vf_multicasts(adapter
);
3485 mta_list
= kzalloc(netdev_mc_count(netdev
) * 6, GFP_ATOMIC
);
3489 /* The shared function expects a packed array of only addresses. */
3491 netdev_for_each_mc_addr(ha
, netdev
)
3492 memcpy(mta_list
+ (i
++ * ETH_ALEN
), ha
->addr
, ETH_ALEN
);
3494 igb_update_mc_addr_list(hw
, mta_list
, i
);
3497 return netdev_mc_count(netdev
);
3501 * igb_write_uc_addr_list - write unicast addresses to RAR table
3502 * @netdev: network interface device structure
3504 * Writes unicast address list to the RAR table.
3505 * Returns: -ENOMEM on failure/insufficient address space
3506 * 0 on no addresses written
3507 * X on writing X addresses to the RAR table
3509 static int igb_write_uc_addr_list(struct net_device
*netdev
)
3511 struct igb_adapter
*adapter
= netdev_priv(netdev
);
3512 struct e1000_hw
*hw
= &adapter
->hw
;
3513 unsigned int vfn
= adapter
->vfs_allocated_count
;
3514 unsigned int rar_entries
= hw
->mac
.rar_entry_count
- (vfn
+ 1);
3517 /* return ENOMEM indicating insufficient memory for addresses */
3518 if (netdev_uc_count(netdev
) > rar_entries
)
3521 if (!netdev_uc_empty(netdev
) && rar_entries
) {
3522 struct netdev_hw_addr
*ha
;
3524 netdev_for_each_uc_addr(ha
, netdev
) {
3527 igb_rar_set_qsel(adapter
, ha
->addr
,
3533 /* write the addresses in reverse order to avoid write combining */
3534 for (; rar_entries
> 0 ; rar_entries
--) {
3535 wr32(E1000_RAH(rar_entries
), 0);
3536 wr32(E1000_RAL(rar_entries
), 0);
3544 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3545 * @netdev: network interface device structure
3547 * The set_rx_mode entry point is called whenever the unicast or multicast
3548 * address lists or the network interface flags are updated. This routine is
3549 * responsible for configuring the hardware for proper unicast, multicast,
3550 * promiscuous mode, and all-multi behavior.
3552 static void igb_set_rx_mode(struct net_device
*netdev
)
3554 struct igb_adapter
*adapter
= netdev_priv(netdev
);
3555 struct e1000_hw
*hw
= &adapter
->hw
;
3556 unsigned int vfn
= adapter
->vfs_allocated_count
;
3557 u32 rctl
, vmolr
= 0;
3560 /* Check for Promiscuous and All Multicast modes */
3561 rctl
= rd32(E1000_RCTL
);
3563 /* clear the effected bits */
3564 rctl
&= ~(E1000_RCTL_UPE
| E1000_RCTL_MPE
| E1000_RCTL_VFE
);
3566 if (netdev
->flags
& IFF_PROMISC
) {
3567 rctl
|= (E1000_RCTL_UPE
| E1000_RCTL_MPE
);
3568 vmolr
|= (E1000_VMOLR_ROPE
| E1000_VMOLR_MPME
);
3570 if (netdev
->flags
& IFF_ALLMULTI
) {
3571 rctl
|= E1000_RCTL_MPE
;
3572 vmolr
|= E1000_VMOLR_MPME
;
3575 * Write addresses to the MTA, if the attempt fails
3576 * then we should just turn on promiscuous mode so
3577 * that we can at least receive multicast traffic
3579 count
= igb_write_mc_addr_list(netdev
);
3581 rctl
|= E1000_RCTL_MPE
;
3582 vmolr
|= E1000_VMOLR_MPME
;
3584 vmolr
|= E1000_VMOLR_ROMPE
;
3588 * Write addresses to available RAR registers, if there is not
3589 * sufficient space to store all the addresses then enable
3590 * unicast promiscuous mode
3592 count
= igb_write_uc_addr_list(netdev
);
3594 rctl
|= E1000_RCTL_UPE
;
3595 vmolr
|= E1000_VMOLR_ROPE
;
3597 rctl
|= E1000_RCTL_VFE
;
3599 wr32(E1000_RCTL
, rctl
);
3602 * In order to support SR-IOV and eventually VMDq it is necessary to set
3603 * the VMOLR to enable the appropriate modes. Without this workaround
3604 * we will have issues with VLAN tag stripping not being done for frames
3605 * that are only arriving because we are the default pool
3607 if ((hw
->mac
.type
< e1000_82576
) || (hw
->mac
.type
> e1000_i350
))
3610 vmolr
|= rd32(E1000_VMOLR(vfn
)) &
3611 ~(E1000_VMOLR_ROPE
| E1000_VMOLR_MPME
| E1000_VMOLR_ROMPE
);
3612 wr32(E1000_VMOLR(vfn
), vmolr
);
3613 igb_restore_vf_multicasts(adapter
);
3616 static void igb_check_wvbr(struct igb_adapter
*adapter
)
3618 struct e1000_hw
*hw
= &adapter
->hw
;
3621 switch (hw
->mac
.type
) {
3624 if (!(wvbr
= rd32(E1000_WVBR
)))
3631 adapter
->wvbr
|= wvbr
;
3634 #define IGB_STAGGERED_QUEUE_OFFSET 8
3636 static void igb_spoof_check(struct igb_adapter
*adapter
)
3643 for(j
= 0; j
< adapter
->vfs_allocated_count
; j
++) {
3644 if (adapter
->wvbr
& (1 << j
) ||
3645 adapter
->wvbr
& (1 << (j
+ IGB_STAGGERED_QUEUE_OFFSET
))) {
3646 dev_warn(&adapter
->pdev
->dev
,
3647 "Spoof event(s) detected on VF %d\n", j
);
3650 (1 << (j
+ IGB_STAGGERED_QUEUE_OFFSET
)));
3655 /* Need to wait a few seconds after link up to get diagnostic information from
3657 static void igb_update_phy_info(unsigned long data
)
3659 struct igb_adapter
*adapter
= (struct igb_adapter
*) data
;
3660 igb_get_phy_info(&adapter
->hw
);
3664 * igb_has_link - check shared code for link and determine up/down
3665 * @adapter: pointer to driver private info
3667 bool igb_has_link(struct igb_adapter
*adapter
)
3669 struct e1000_hw
*hw
= &adapter
->hw
;
3670 bool link_active
= false;
3673 /* get_link_status is set on LSC (link status) interrupt or
3674 * rx sequence error interrupt. get_link_status will stay
3675 * false until the e1000_check_for_link establishes link
3676 * for copper adapters ONLY
3678 switch (hw
->phy
.media_type
) {
3679 case e1000_media_type_copper
:
3680 if (hw
->mac
.get_link_status
) {
3681 ret_val
= hw
->mac
.ops
.check_for_link(hw
);
3682 link_active
= !hw
->mac
.get_link_status
;
3687 case e1000_media_type_internal_serdes
:
3688 ret_val
= hw
->mac
.ops
.check_for_link(hw
);
3689 link_active
= hw
->mac
.serdes_has_link
;
3692 case e1000_media_type_unknown
:
3699 static bool igb_thermal_sensor_event(struct e1000_hw
*hw
, u32 event
)
3702 u32 ctrl_ext
, thstat
;
3704 /* check for thermal sensor event on i350 copper only */
3705 if (hw
->mac
.type
== e1000_i350
) {
3706 thstat
= rd32(E1000_THSTAT
);
3707 ctrl_ext
= rd32(E1000_CTRL_EXT
);
3709 if ((hw
->phy
.media_type
== e1000_media_type_copper
) &&
3710 !(ctrl_ext
& E1000_CTRL_EXT_LINK_MODE_SGMII
)) {
3711 ret
= !!(thstat
& event
);
3719 * igb_watchdog - Timer Call-back
3720 * @data: pointer to adapter cast into an unsigned long
3722 static void igb_watchdog(unsigned long data
)
3724 struct igb_adapter
*adapter
= (struct igb_adapter
*)data
;
3725 /* Do the rest outside of interrupt context */
3726 schedule_work(&adapter
->watchdog_task
);
3729 static void igb_watchdog_task(struct work_struct
*work
)
3731 struct igb_adapter
*adapter
= container_of(work
,
3734 struct e1000_hw
*hw
= &adapter
->hw
;
3735 struct net_device
*netdev
= adapter
->netdev
;
3739 link
= igb_has_link(adapter
);
3741 /* Cancel scheduled suspend requests. */
3742 pm_runtime_resume(netdev
->dev
.parent
);
3744 if (!netif_carrier_ok(netdev
)) {
3746 hw
->mac
.ops
.get_speed_and_duplex(hw
,
3747 &adapter
->link_speed
,
3748 &adapter
->link_duplex
);
3750 ctrl
= rd32(E1000_CTRL
);
3751 /* Links status message must follow this format */
3752 printk(KERN_INFO
"igb: %s NIC Link is Up %d Mbps %s "
3753 "Duplex, Flow Control: %s\n",
3755 adapter
->link_speed
,
3756 adapter
->link_duplex
== FULL_DUPLEX
?
3758 (ctrl
& E1000_CTRL_TFCE
) &&
3759 (ctrl
& E1000_CTRL_RFCE
) ? "RX/TX" :
3760 (ctrl
& E1000_CTRL_RFCE
) ? "RX" :
3761 (ctrl
& E1000_CTRL_TFCE
) ? "TX" : "None");
3763 /* check for thermal sensor event */
3764 if (igb_thermal_sensor_event(hw
,
3765 E1000_THSTAT_LINK_THROTTLE
)) {
3766 netdev_info(netdev
, "The network adapter link "
3767 "speed was downshifted because it "
3771 /* adjust timeout factor according to speed/duplex */
3772 adapter
->tx_timeout_factor
= 1;
3773 switch (adapter
->link_speed
) {
3775 adapter
->tx_timeout_factor
= 14;
3778 /* maybe add some timeout factor ? */
3782 netif_carrier_on(netdev
);
3784 igb_ping_all_vfs(adapter
);
3785 igb_check_vf_rate_limit(adapter
);
3787 /* link state has changed, schedule phy info update */
3788 if (!test_bit(__IGB_DOWN
, &adapter
->state
))
3789 mod_timer(&adapter
->phy_info_timer
,
3790 round_jiffies(jiffies
+ 2 * HZ
));
3793 if (netif_carrier_ok(netdev
)) {
3794 adapter
->link_speed
= 0;
3795 adapter
->link_duplex
= 0;
3797 /* check for thermal sensor event */
3798 if (igb_thermal_sensor_event(hw
,
3799 E1000_THSTAT_PWR_DOWN
)) {
3800 netdev_err(netdev
, "The network adapter was "
3801 "stopped because it overheated\n");
3804 /* Links status message must follow this format */
3805 printk(KERN_INFO
"igb: %s NIC Link is Down\n",
3807 netif_carrier_off(netdev
);
3809 igb_ping_all_vfs(adapter
);
3811 /* link state has changed, schedule phy info update */
3812 if (!test_bit(__IGB_DOWN
, &adapter
->state
))
3813 mod_timer(&adapter
->phy_info_timer
,
3814 round_jiffies(jiffies
+ 2 * HZ
));
3816 pm_schedule_suspend(netdev
->dev
.parent
,
3821 spin_lock(&adapter
->stats64_lock
);
3822 igb_update_stats(adapter
, &adapter
->stats64
);
3823 spin_unlock(&adapter
->stats64_lock
);
3825 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
3826 struct igb_ring
*tx_ring
= adapter
->tx_ring
[i
];
3827 if (!netif_carrier_ok(netdev
)) {
3828 /* We've lost link, so the controller stops DMA,
3829 * but we've got queued Tx work that's never going
3830 * to get done, so reset controller to flush Tx.
3831 * (Do the reset outside of interrupt context). */
3832 if (igb_desc_unused(tx_ring
) + 1 < tx_ring
->count
) {
3833 adapter
->tx_timeout_count
++;
3834 schedule_work(&adapter
->reset_task
);
3835 /* return immediately since reset is imminent */
3840 /* Force detection of hung controller every watchdog period */
3841 set_bit(IGB_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
);
3844 /* Cause software interrupt to ensure rx ring is cleaned */
3845 if (adapter
->msix_entries
) {
3847 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
3848 eics
|= adapter
->q_vector
[i
]->eims_value
;
3849 wr32(E1000_EICS
, eics
);
3851 wr32(E1000_ICS
, E1000_ICS_RXDMT0
);
3854 igb_spoof_check(adapter
);
3856 /* Reset the timer */
3857 if (!test_bit(__IGB_DOWN
, &adapter
->state
))
3858 mod_timer(&adapter
->watchdog_timer
,
3859 round_jiffies(jiffies
+ 2 * HZ
));
3862 enum latency_range
{
3866 latency_invalid
= 255
3870 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3872 * Stores a new ITR value based on strictly on packet size. This
3873 * algorithm is less sophisticated than that used in igb_update_itr,
3874 * due to the difficulty of synchronizing statistics across multiple
3875 * receive rings. The divisors and thresholds used by this function
3876 * were determined based on theoretical maximum wire speed and testing
3877 * data, in order to minimize response time while increasing bulk
3879 * This functionality is controlled by the InterruptThrottleRate module
3880 * parameter (see igb_param.c)
3881 * NOTE: This function is called only when operating in a multiqueue
3882 * receive environment.
3883 * @q_vector: pointer to q_vector
3885 static void igb_update_ring_itr(struct igb_q_vector
*q_vector
)
3887 int new_val
= q_vector
->itr_val
;
3888 int avg_wire_size
= 0;
3889 struct igb_adapter
*adapter
= q_vector
->adapter
;
3890 unsigned int packets
;
3892 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3893 * ints/sec - ITR timer value of 120 ticks.
3895 if (adapter
->link_speed
!= SPEED_1000
) {
3896 new_val
= IGB_4K_ITR
;
3900 packets
= q_vector
->rx
.total_packets
;
3902 avg_wire_size
= q_vector
->rx
.total_bytes
/ packets
;
3904 packets
= q_vector
->tx
.total_packets
;
3906 avg_wire_size
= max_t(u32
, avg_wire_size
,
3907 q_vector
->tx
.total_bytes
/ packets
);
3909 /* if avg_wire_size isn't set no work was done */
3913 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3914 avg_wire_size
+= 24;
3916 /* Don't starve jumbo frames */
3917 avg_wire_size
= min(avg_wire_size
, 3000);
3919 /* Give a little boost to mid-size frames */
3920 if ((avg_wire_size
> 300) && (avg_wire_size
< 1200))
3921 new_val
= avg_wire_size
/ 3;
3923 new_val
= avg_wire_size
/ 2;
3925 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3926 if (new_val
< IGB_20K_ITR
&&
3927 ((q_vector
->rx
.ring
&& adapter
->rx_itr_setting
== 3) ||
3928 (!q_vector
->rx
.ring
&& adapter
->tx_itr_setting
== 3)))
3929 new_val
= IGB_20K_ITR
;
3932 if (new_val
!= q_vector
->itr_val
) {
3933 q_vector
->itr_val
= new_val
;
3934 q_vector
->set_itr
= 1;
3937 q_vector
->rx
.total_bytes
= 0;
3938 q_vector
->rx
.total_packets
= 0;
3939 q_vector
->tx
.total_bytes
= 0;
3940 q_vector
->tx
.total_packets
= 0;
3944 * igb_update_itr - update the dynamic ITR value based on statistics
3945 * Stores a new ITR value based on packets and byte
3946 * counts during the last interrupt. The advantage of per interrupt
3947 * computation is faster updates and more accurate ITR for the current
3948 * traffic pattern. Constants in this function were computed
3949 * based on theoretical maximum wire speed and thresholds were set based
3950 * on testing data as well as attempting to minimize response time
3951 * while increasing bulk throughput.
3952 * this functionality is controlled by the InterruptThrottleRate module
3953 * parameter (see igb_param.c)
3954 * NOTE: These calculations are only valid when operating in a single-
3955 * queue environment.
3956 * @q_vector: pointer to q_vector
3957 * @ring_container: ring info to update the itr for
3959 static void igb_update_itr(struct igb_q_vector
*q_vector
,
3960 struct igb_ring_container
*ring_container
)
3962 unsigned int packets
= ring_container
->total_packets
;
3963 unsigned int bytes
= ring_container
->total_bytes
;
3964 u8 itrval
= ring_container
->itr
;
3966 /* no packets, exit with status unchanged */
3971 case lowest_latency
:
3972 /* handle TSO and jumbo frames */
3973 if (bytes
/packets
> 8000)
3974 itrval
= bulk_latency
;
3975 else if ((packets
< 5) && (bytes
> 512))
3976 itrval
= low_latency
;
3978 case low_latency
: /* 50 usec aka 20000 ints/s */
3979 if (bytes
> 10000) {
3980 /* this if handles the TSO accounting */
3981 if (bytes
/packets
> 8000) {
3982 itrval
= bulk_latency
;
3983 } else if ((packets
< 10) || ((bytes
/packets
) > 1200)) {
3984 itrval
= bulk_latency
;
3985 } else if ((packets
> 35)) {
3986 itrval
= lowest_latency
;
3988 } else if (bytes
/packets
> 2000) {
3989 itrval
= bulk_latency
;
3990 } else if (packets
<= 2 && bytes
< 512) {
3991 itrval
= lowest_latency
;
3994 case bulk_latency
: /* 250 usec aka 4000 ints/s */
3995 if (bytes
> 25000) {
3997 itrval
= low_latency
;
3998 } else if (bytes
< 1500) {
3999 itrval
= low_latency
;
4004 /* clear work counters since we have the values we need */
4005 ring_container
->total_bytes
= 0;
4006 ring_container
->total_packets
= 0;
4008 /* write updated itr to ring container */
4009 ring_container
->itr
= itrval
;
4012 static void igb_set_itr(struct igb_q_vector
*q_vector
)
4014 struct igb_adapter
*adapter
= q_vector
->adapter
;
4015 u32 new_itr
= q_vector
->itr_val
;
4018 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4019 if (adapter
->link_speed
!= SPEED_1000
) {
4021 new_itr
= IGB_4K_ITR
;
4025 igb_update_itr(q_vector
, &q_vector
->tx
);
4026 igb_update_itr(q_vector
, &q_vector
->rx
);
4028 current_itr
= max(q_vector
->rx
.itr
, q_vector
->tx
.itr
);
4030 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4031 if (current_itr
== lowest_latency
&&
4032 ((q_vector
->rx
.ring
&& adapter
->rx_itr_setting
== 3) ||
4033 (!q_vector
->rx
.ring
&& adapter
->tx_itr_setting
== 3)))
4034 current_itr
= low_latency
;
4036 switch (current_itr
) {
4037 /* counts and packets in update_itr are dependent on these numbers */
4038 case lowest_latency
:
4039 new_itr
= IGB_70K_ITR
; /* 70,000 ints/sec */
4042 new_itr
= IGB_20K_ITR
; /* 20,000 ints/sec */
4045 new_itr
= IGB_4K_ITR
; /* 4,000 ints/sec */
4052 if (new_itr
!= q_vector
->itr_val
) {
4053 /* this attempts to bias the interrupt rate towards Bulk
4054 * by adding intermediate steps when interrupt rate is
4056 new_itr
= new_itr
> q_vector
->itr_val
?
4057 max((new_itr
* q_vector
->itr_val
) /
4058 (new_itr
+ (q_vector
->itr_val
>> 2)),
4061 /* Don't write the value here; it resets the adapter's
4062 * internal timer, and causes us to delay far longer than
4063 * we should between interrupts. Instead, we write the ITR
4064 * value at the beginning of the next interrupt so the timing
4065 * ends up being correct.
4067 q_vector
->itr_val
= new_itr
;
4068 q_vector
->set_itr
= 1;
4072 static void igb_tx_ctxtdesc(struct igb_ring
*tx_ring
, u32 vlan_macip_lens
,
4073 u32 type_tucmd
, u32 mss_l4len_idx
)
4075 struct e1000_adv_tx_context_desc
*context_desc
;
4076 u16 i
= tx_ring
->next_to_use
;
4078 context_desc
= IGB_TX_CTXTDESC(tx_ring
, i
);
4081 tx_ring
->next_to_use
= (i
< tx_ring
->count
) ? i
: 0;
4083 /* set bits to identify this as an advanced context descriptor */
4084 type_tucmd
|= E1000_TXD_CMD_DEXT
| E1000_ADVTXD_DTYP_CTXT
;
4086 /* For 82575, context index must be unique per ring. */
4087 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX
, &tx_ring
->flags
))
4088 mss_l4len_idx
|= tx_ring
->reg_idx
<< 4;
4090 context_desc
->vlan_macip_lens
= cpu_to_le32(vlan_macip_lens
);
4091 context_desc
->seqnum_seed
= 0;
4092 context_desc
->type_tucmd_mlhl
= cpu_to_le32(type_tucmd
);
4093 context_desc
->mss_l4len_idx
= cpu_to_le32(mss_l4len_idx
);
4096 static int igb_tso(struct igb_ring
*tx_ring
,
4097 struct igb_tx_buffer
*first
,
4100 struct sk_buff
*skb
= first
->skb
;
4101 u32 vlan_macip_lens
, type_tucmd
;
4102 u32 mss_l4len_idx
, l4len
;
4104 if (!skb_is_gso(skb
))
4107 if (skb_header_cloned(skb
)) {
4108 int err
= pskb_expand_head(skb
, 0, 0, GFP_ATOMIC
);
4113 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4114 type_tucmd
= E1000_ADVTXD_TUCMD_L4T_TCP
;
4116 if (first
->protocol
== __constant_htons(ETH_P_IP
)) {
4117 struct iphdr
*iph
= ip_hdr(skb
);
4120 tcp_hdr(skb
)->check
= ~csum_tcpudp_magic(iph
->saddr
,
4124 type_tucmd
|= E1000_ADVTXD_TUCMD_IPV4
;
4125 first
->tx_flags
|= IGB_TX_FLAGS_TSO
|
4128 } else if (skb_is_gso_v6(skb
)) {
4129 ipv6_hdr(skb
)->payload_len
= 0;
4130 tcp_hdr(skb
)->check
= ~csum_ipv6_magic(&ipv6_hdr(skb
)->saddr
,
4131 &ipv6_hdr(skb
)->daddr
,
4133 first
->tx_flags
|= IGB_TX_FLAGS_TSO
|
4137 /* compute header lengths */
4138 l4len
= tcp_hdrlen(skb
);
4139 *hdr_len
= skb_transport_offset(skb
) + l4len
;
4141 /* update gso size and bytecount with header size */
4142 first
->gso_segs
= skb_shinfo(skb
)->gso_segs
;
4143 first
->bytecount
+= (first
->gso_segs
- 1) * *hdr_len
;
4146 mss_l4len_idx
= l4len
<< E1000_ADVTXD_L4LEN_SHIFT
;
4147 mss_l4len_idx
|= skb_shinfo(skb
)->gso_size
<< E1000_ADVTXD_MSS_SHIFT
;
4149 /* VLAN MACLEN IPLEN */
4150 vlan_macip_lens
= skb_network_header_len(skb
);
4151 vlan_macip_lens
|= skb_network_offset(skb
) << E1000_ADVTXD_MACLEN_SHIFT
;
4152 vlan_macip_lens
|= first
->tx_flags
& IGB_TX_FLAGS_VLAN_MASK
;
4154 igb_tx_ctxtdesc(tx_ring
, vlan_macip_lens
, type_tucmd
, mss_l4len_idx
);
4159 static void igb_tx_csum(struct igb_ring
*tx_ring
, struct igb_tx_buffer
*first
)
4161 struct sk_buff
*skb
= first
->skb
;
4162 u32 vlan_macip_lens
= 0;
4163 u32 mss_l4len_idx
= 0;
4166 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
) {
4167 if (!(first
->tx_flags
& IGB_TX_FLAGS_VLAN
))
4171 switch (first
->protocol
) {
4172 case __constant_htons(ETH_P_IP
):
4173 vlan_macip_lens
|= skb_network_header_len(skb
);
4174 type_tucmd
|= E1000_ADVTXD_TUCMD_IPV4
;
4175 l4_hdr
= ip_hdr(skb
)->protocol
;
4177 case __constant_htons(ETH_P_IPV6
):
4178 vlan_macip_lens
|= skb_network_header_len(skb
);
4179 l4_hdr
= ipv6_hdr(skb
)->nexthdr
;
4182 if (unlikely(net_ratelimit())) {
4183 dev_warn(tx_ring
->dev
,
4184 "partial checksum but proto=%x!\n",
4192 type_tucmd
|= E1000_ADVTXD_TUCMD_L4T_TCP
;
4193 mss_l4len_idx
= tcp_hdrlen(skb
) <<
4194 E1000_ADVTXD_L4LEN_SHIFT
;
4197 type_tucmd
|= E1000_ADVTXD_TUCMD_L4T_SCTP
;
4198 mss_l4len_idx
= sizeof(struct sctphdr
) <<
4199 E1000_ADVTXD_L4LEN_SHIFT
;
4202 mss_l4len_idx
= sizeof(struct udphdr
) <<
4203 E1000_ADVTXD_L4LEN_SHIFT
;
4206 if (unlikely(net_ratelimit())) {
4207 dev_warn(tx_ring
->dev
,
4208 "partial checksum but l4 proto=%x!\n",
4214 /* update TX checksum flag */
4215 first
->tx_flags
|= IGB_TX_FLAGS_CSUM
;
4218 vlan_macip_lens
|= skb_network_offset(skb
) << E1000_ADVTXD_MACLEN_SHIFT
;
4219 vlan_macip_lens
|= first
->tx_flags
& IGB_TX_FLAGS_VLAN_MASK
;
4221 igb_tx_ctxtdesc(tx_ring
, vlan_macip_lens
, type_tucmd
, mss_l4len_idx
);
4224 static __le32
igb_tx_cmd_type(u32 tx_flags
)
4226 /* set type for advanced descriptor with frame checksum insertion */
4227 __le32 cmd_type
= cpu_to_le32(E1000_ADVTXD_DTYP_DATA
|
4228 E1000_ADVTXD_DCMD_IFCS
|
4229 E1000_ADVTXD_DCMD_DEXT
);
4231 /* set HW vlan bit if vlan is present */
4232 if (tx_flags
& IGB_TX_FLAGS_VLAN
)
4233 cmd_type
|= cpu_to_le32(E1000_ADVTXD_DCMD_VLE
);
4235 #ifdef CONFIG_IGB_PTP
4236 /* set timestamp bit if present */
4237 if (tx_flags
& IGB_TX_FLAGS_TSTAMP
)
4238 cmd_type
|= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP
);
4239 #endif /* CONFIG_IGB_PTP */
4241 /* set segmentation bits for TSO */
4242 if (tx_flags
& IGB_TX_FLAGS_TSO
)
4243 cmd_type
|= cpu_to_le32(E1000_ADVTXD_DCMD_TSE
);
4248 static void igb_tx_olinfo_status(struct igb_ring
*tx_ring
,
4249 union e1000_adv_tx_desc
*tx_desc
,
4250 u32 tx_flags
, unsigned int paylen
)
4252 u32 olinfo_status
= paylen
<< E1000_ADVTXD_PAYLEN_SHIFT
;
4254 /* 82575 requires a unique index per ring if any offload is enabled */
4255 if ((tx_flags
& (IGB_TX_FLAGS_CSUM
| IGB_TX_FLAGS_VLAN
)) &&
4256 test_bit(IGB_RING_FLAG_TX_CTX_IDX
, &tx_ring
->flags
))
4257 olinfo_status
|= tx_ring
->reg_idx
<< 4;
4259 /* insert L4 checksum */
4260 if (tx_flags
& IGB_TX_FLAGS_CSUM
) {
4261 olinfo_status
|= E1000_TXD_POPTS_TXSM
<< 8;
4263 /* insert IPv4 checksum */
4264 if (tx_flags
& IGB_TX_FLAGS_IPV4
)
4265 olinfo_status
|= E1000_TXD_POPTS_IXSM
<< 8;
4268 tx_desc
->read
.olinfo_status
= cpu_to_le32(olinfo_status
);
4272 * The largest size we can write to the descriptor is 65535. In order to
4273 * maintain a power of two alignment we have to limit ourselves to 32K.
4275 #define IGB_MAX_TXD_PWR 15
4276 #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
4278 static void igb_tx_map(struct igb_ring
*tx_ring
,
4279 struct igb_tx_buffer
*first
,
4282 struct sk_buff
*skb
= first
->skb
;
4283 struct igb_tx_buffer
*tx_buffer_info
;
4284 union e1000_adv_tx_desc
*tx_desc
;
4286 struct skb_frag_struct
*frag
= &skb_shinfo(skb
)->frags
[0];
4287 unsigned int data_len
= skb
->data_len
;
4288 unsigned int size
= skb_headlen(skb
);
4289 unsigned int paylen
= skb
->len
- hdr_len
;
4291 u32 tx_flags
= first
->tx_flags
;
4292 u16 i
= tx_ring
->next_to_use
;
4294 tx_desc
= IGB_TX_DESC(tx_ring
, i
);
4296 igb_tx_olinfo_status(tx_ring
, tx_desc
, tx_flags
, paylen
);
4297 cmd_type
= igb_tx_cmd_type(tx_flags
);
4299 dma
= dma_map_single(tx_ring
->dev
, skb
->data
, size
, DMA_TO_DEVICE
);
4300 if (dma_mapping_error(tx_ring
->dev
, dma
))
4303 /* record length, and DMA address */
4304 first
->length
= size
;
4306 tx_desc
->read
.buffer_addr
= cpu_to_le64(dma
);
4309 while (unlikely(size
> IGB_MAX_DATA_PER_TXD
)) {
4310 tx_desc
->read
.cmd_type_len
=
4311 cmd_type
| cpu_to_le32(IGB_MAX_DATA_PER_TXD
);
4315 if (i
== tx_ring
->count
) {
4316 tx_desc
= IGB_TX_DESC(tx_ring
, 0);
4320 dma
+= IGB_MAX_DATA_PER_TXD
;
4321 size
-= IGB_MAX_DATA_PER_TXD
;
4323 tx_desc
->read
.olinfo_status
= 0;
4324 tx_desc
->read
.buffer_addr
= cpu_to_le64(dma
);
4327 if (likely(!data_len
))
4330 tx_desc
->read
.cmd_type_len
= cmd_type
| cpu_to_le32(size
);
4334 if (i
== tx_ring
->count
) {
4335 tx_desc
= IGB_TX_DESC(tx_ring
, 0);
4339 size
= skb_frag_size(frag
);
4342 dma
= skb_frag_dma_map(tx_ring
->dev
, frag
, 0,
4343 size
, DMA_TO_DEVICE
);
4344 if (dma_mapping_error(tx_ring
->dev
, dma
))
4347 tx_buffer_info
= &tx_ring
->tx_buffer_info
[i
];
4348 tx_buffer_info
->length
= size
;
4349 tx_buffer_info
->dma
= dma
;
4351 tx_desc
->read
.olinfo_status
= 0;
4352 tx_desc
->read
.buffer_addr
= cpu_to_le64(dma
);
4357 netdev_tx_sent_queue(txring_txq(tx_ring
), first
->bytecount
);
4359 /* write last descriptor with RS and EOP bits */
4360 cmd_type
|= cpu_to_le32(size
) | cpu_to_le32(IGB_TXD_DCMD
);
4361 if (unlikely(skb
->no_fcs
))
4362 cmd_type
&= ~(cpu_to_le32(E1000_ADVTXD_DCMD_IFCS
));
4363 tx_desc
->read
.cmd_type_len
= cmd_type
;
4365 /* set the timestamp */
4366 first
->time_stamp
= jiffies
;
4369 * Force memory writes to complete before letting h/w know there
4370 * are new descriptors to fetch. (Only applicable for weak-ordered
4371 * memory model archs, such as IA-64).
4373 * We also need this memory barrier to make certain all of the
4374 * status bits have been updated before next_to_watch is written.
4378 /* set next_to_watch value indicating a packet is present */
4379 first
->next_to_watch
= tx_desc
;
4382 if (i
== tx_ring
->count
)
4385 tx_ring
->next_to_use
= i
;
4387 writel(i
, tx_ring
->tail
);
4389 /* we need this if more than one processor can write to our tail
4390 * at a time, it syncronizes IO on IA64/Altix systems */
4396 dev_err(tx_ring
->dev
, "TX DMA map failed\n");
4398 /* clear dma mappings for failed tx_buffer_info map */
4400 tx_buffer_info
= &tx_ring
->tx_buffer_info
[i
];
4401 igb_unmap_and_free_tx_resource(tx_ring
, tx_buffer_info
);
4402 if (tx_buffer_info
== first
)
4409 tx_ring
->next_to_use
= i
;
4412 static int __igb_maybe_stop_tx(struct igb_ring
*tx_ring
, const u16 size
)
4414 struct net_device
*netdev
= tx_ring
->netdev
;
4416 netif_stop_subqueue(netdev
, tx_ring
->queue_index
);
4418 /* Herbert's original patch had:
4419 * smp_mb__after_netif_stop_queue();
4420 * but since that doesn't exist yet, just open code it. */
4423 /* We need to check again in a case another CPU has just
4424 * made room available. */
4425 if (igb_desc_unused(tx_ring
) < size
)
4429 netif_wake_subqueue(netdev
, tx_ring
->queue_index
);
4431 u64_stats_update_begin(&tx_ring
->tx_syncp2
);
4432 tx_ring
->tx_stats
.restart_queue2
++;
4433 u64_stats_update_end(&tx_ring
->tx_syncp2
);
4438 static inline int igb_maybe_stop_tx(struct igb_ring
*tx_ring
, const u16 size
)
4440 if (igb_desc_unused(tx_ring
) >= size
)
4442 return __igb_maybe_stop_tx(tx_ring
, size
);
4445 netdev_tx_t
igb_xmit_frame_ring(struct sk_buff
*skb
,
4446 struct igb_ring
*tx_ring
)
4448 struct igb_tx_buffer
*first
;
4451 __be16 protocol
= vlan_get_protocol(skb
);
4454 /* need: 1 descriptor per page,
4455 * + 2 desc gap to keep tail from touching head,
4456 * + 1 desc for skb->data,
4457 * + 1 desc for context descriptor,
4458 * otherwise try next time */
4459 if (igb_maybe_stop_tx(tx_ring
, skb_shinfo(skb
)->nr_frags
+ 4)) {
4460 /* this is a hard error */
4461 return NETDEV_TX_BUSY
;
4464 /* record the location of the first descriptor for this packet */
4465 first
= &tx_ring
->tx_buffer_info
[tx_ring
->next_to_use
];
4467 first
->bytecount
= skb
->len
;
4468 first
->gso_segs
= 1;
4470 #ifdef CONFIG_IGB_PTP
4471 if (unlikely(skb_shinfo(skb
)->tx_flags
& SKBTX_HW_TSTAMP
)) {
4472 skb_shinfo(skb
)->tx_flags
|= SKBTX_IN_PROGRESS
;
4473 tx_flags
|= IGB_TX_FLAGS_TSTAMP
;
4475 #endif /* CONFIG_IGB_PTP */
4477 if (vlan_tx_tag_present(skb
)) {
4478 tx_flags
|= IGB_TX_FLAGS_VLAN
;
4479 tx_flags
|= (vlan_tx_tag_get(skb
) << IGB_TX_FLAGS_VLAN_SHIFT
);
4482 /* record initial flags and protocol */
4483 first
->tx_flags
= tx_flags
;
4484 first
->protocol
= protocol
;
4486 tso
= igb_tso(tx_ring
, first
, &hdr_len
);
4490 igb_tx_csum(tx_ring
, first
);
4492 igb_tx_map(tx_ring
, first
, hdr_len
);
4494 /* Make sure there is space in the ring for the next send. */
4495 igb_maybe_stop_tx(tx_ring
, MAX_SKB_FRAGS
+ 4);
4497 return NETDEV_TX_OK
;
4500 igb_unmap_and_free_tx_resource(tx_ring
, first
);
4502 return NETDEV_TX_OK
;
4505 static inline struct igb_ring
*igb_tx_queue_mapping(struct igb_adapter
*adapter
,
4506 struct sk_buff
*skb
)
4508 unsigned int r_idx
= skb
->queue_mapping
;
4510 if (r_idx
>= adapter
->num_tx_queues
)
4511 r_idx
= r_idx
% adapter
->num_tx_queues
;
4513 return adapter
->tx_ring
[r_idx
];
4516 static netdev_tx_t
igb_xmit_frame(struct sk_buff
*skb
,
4517 struct net_device
*netdev
)
4519 struct igb_adapter
*adapter
= netdev_priv(netdev
);
4521 if (test_bit(__IGB_DOWN
, &adapter
->state
)) {
4522 dev_kfree_skb_any(skb
);
4523 return NETDEV_TX_OK
;
4526 if (skb
->len
<= 0) {
4527 dev_kfree_skb_any(skb
);
4528 return NETDEV_TX_OK
;
4532 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4533 * in order to meet this minimum size requirement.
4535 if (skb
->len
< 17) {
4536 if (skb_padto(skb
, 17))
4537 return NETDEV_TX_OK
;
4541 return igb_xmit_frame_ring(skb
, igb_tx_queue_mapping(adapter
, skb
));
4545 * igb_tx_timeout - Respond to a Tx Hang
4546 * @netdev: network interface device structure
4548 static void igb_tx_timeout(struct net_device
*netdev
)
4550 struct igb_adapter
*adapter
= netdev_priv(netdev
);
4551 struct e1000_hw
*hw
= &adapter
->hw
;
4553 /* Do the reset outside of interrupt context */
4554 adapter
->tx_timeout_count
++;
4556 if (hw
->mac
.type
>= e1000_82580
)
4557 hw
->dev_spec
._82575
.global_device_reset
= true;
4559 schedule_work(&adapter
->reset_task
);
4561 (adapter
->eims_enable_mask
& ~adapter
->eims_other
));
4564 static void igb_reset_task(struct work_struct
*work
)
4566 struct igb_adapter
*adapter
;
4567 adapter
= container_of(work
, struct igb_adapter
, reset_task
);
4570 netdev_err(adapter
->netdev
, "Reset adapter\n");
4571 igb_reinit_locked(adapter
);
4575 * igb_get_stats64 - Get System Network Statistics
4576 * @netdev: network interface device structure
4577 * @stats: rtnl_link_stats64 pointer
4580 static struct rtnl_link_stats64
*igb_get_stats64(struct net_device
*netdev
,
4581 struct rtnl_link_stats64
*stats
)
4583 struct igb_adapter
*adapter
= netdev_priv(netdev
);
4585 spin_lock(&adapter
->stats64_lock
);
4586 igb_update_stats(adapter
, &adapter
->stats64
);
4587 memcpy(stats
, &adapter
->stats64
, sizeof(*stats
));
4588 spin_unlock(&adapter
->stats64_lock
);
4594 * igb_change_mtu - Change the Maximum Transfer Unit
4595 * @netdev: network interface device structure
4596 * @new_mtu: new value for maximum frame size
4598 * Returns 0 on success, negative on failure
4600 static int igb_change_mtu(struct net_device
*netdev
, int new_mtu
)
4602 struct igb_adapter
*adapter
= netdev_priv(netdev
);
4603 struct pci_dev
*pdev
= adapter
->pdev
;
4604 int max_frame
= new_mtu
+ ETH_HLEN
+ ETH_FCS_LEN
+ VLAN_HLEN
;
4606 if ((new_mtu
< 68) || (max_frame
> MAX_JUMBO_FRAME_SIZE
)) {
4607 dev_err(&pdev
->dev
, "Invalid MTU setting\n");
4611 #define MAX_STD_JUMBO_FRAME_SIZE 9238
4612 if (max_frame
> MAX_STD_JUMBO_FRAME_SIZE
) {
4613 dev_err(&pdev
->dev
, "MTU > 9216 not supported.\n");
4617 while (test_and_set_bit(__IGB_RESETTING
, &adapter
->state
))
4620 /* igb_down has a dependency on max_frame_size */
4621 adapter
->max_frame_size
= max_frame
;
4623 if (netif_running(netdev
))
4626 dev_info(&pdev
->dev
, "changing MTU from %d to %d\n",
4627 netdev
->mtu
, new_mtu
);
4628 netdev
->mtu
= new_mtu
;
4630 if (netif_running(netdev
))
4635 clear_bit(__IGB_RESETTING
, &adapter
->state
);
4641 * igb_update_stats - Update the board statistics counters
4642 * @adapter: board private structure
4645 void igb_update_stats(struct igb_adapter
*adapter
,
4646 struct rtnl_link_stats64
*net_stats
)
4648 struct e1000_hw
*hw
= &adapter
->hw
;
4649 struct pci_dev
*pdev
= adapter
->pdev
;
4655 u64 _bytes
, _packets
;
4657 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4660 * Prevent stats update while adapter is being reset, or if the pci
4661 * connection is down.
4663 if (adapter
->link_speed
== 0)
4665 if (pci_channel_offline(pdev
))
4670 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
4671 u32 rqdpc_tmp
= rd32(E1000_RQDPC(i
)) & 0x0FFF;
4672 struct igb_ring
*ring
= adapter
->rx_ring
[i
];
4674 ring
->rx_stats
.drops
+= rqdpc_tmp
;
4675 net_stats
->rx_fifo_errors
+= rqdpc_tmp
;
4678 start
= u64_stats_fetch_begin_bh(&ring
->rx_syncp
);
4679 _bytes
= ring
->rx_stats
.bytes
;
4680 _packets
= ring
->rx_stats
.packets
;
4681 } while (u64_stats_fetch_retry_bh(&ring
->rx_syncp
, start
));
4683 packets
+= _packets
;
4686 net_stats
->rx_bytes
= bytes
;
4687 net_stats
->rx_packets
= packets
;
4691 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
4692 struct igb_ring
*ring
= adapter
->tx_ring
[i
];
4694 start
= u64_stats_fetch_begin_bh(&ring
->tx_syncp
);
4695 _bytes
= ring
->tx_stats
.bytes
;
4696 _packets
= ring
->tx_stats
.packets
;
4697 } while (u64_stats_fetch_retry_bh(&ring
->tx_syncp
, start
));
4699 packets
+= _packets
;
4701 net_stats
->tx_bytes
= bytes
;
4702 net_stats
->tx_packets
= packets
;
4704 /* read stats registers */
4705 adapter
->stats
.crcerrs
+= rd32(E1000_CRCERRS
);
4706 adapter
->stats
.gprc
+= rd32(E1000_GPRC
);
4707 adapter
->stats
.gorc
+= rd32(E1000_GORCL
);
4708 rd32(E1000_GORCH
); /* clear GORCL */
4709 adapter
->stats
.bprc
+= rd32(E1000_BPRC
);
4710 adapter
->stats
.mprc
+= rd32(E1000_MPRC
);
4711 adapter
->stats
.roc
+= rd32(E1000_ROC
);
4713 adapter
->stats
.prc64
+= rd32(E1000_PRC64
);
4714 adapter
->stats
.prc127
+= rd32(E1000_PRC127
);
4715 adapter
->stats
.prc255
+= rd32(E1000_PRC255
);
4716 adapter
->stats
.prc511
+= rd32(E1000_PRC511
);
4717 adapter
->stats
.prc1023
+= rd32(E1000_PRC1023
);
4718 adapter
->stats
.prc1522
+= rd32(E1000_PRC1522
);
4719 adapter
->stats
.symerrs
+= rd32(E1000_SYMERRS
);
4720 adapter
->stats
.sec
+= rd32(E1000_SEC
);
4722 mpc
= rd32(E1000_MPC
);
4723 adapter
->stats
.mpc
+= mpc
;
4724 net_stats
->rx_fifo_errors
+= mpc
;
4725 adapter
->stats
.scc
+= rd32(E1000_SCC
);
4726 adapter
->stats
.ecol
+= rd32(E1000_ECOL
);
4727 adapter
->stats
.mcc
+= rd32(E1000_MCC
);
4728 adapter
->stats
.latecol
+= rd32(E1000_LATECOL
);
4729 adapter
->stats
.dc
+= rd32(E1000_DC
);
4730 adapter
->stats
.rlec
+= rd32(E1000_RLEC
);
4731 adapter
->stats
.xonrxc
+= rd32(E1000_XONRXC
);
4732 adapter
->stats
.xontxc
+= rd32(E1000_XONTXC
);
4733 adapter
->stats
.xoffrxc
+= rd32(E1000_XOFFRXC
);
4734 adapter
->stats
.xofftxc
+= rd32(E1000_XOFFTXC
);
4735 adapter
->stats
.fcruc
+= rd32(E1000_FCRUC
);
4736 adapter
->stats
.gptc
+= rd32(E1000_GPTC
);
4737 adapter
->stats
.gotc
+= rd32(E1000_GOTCL
);
4738 rd32(E1000_GOTCH
); /* clear GOTCL */
4739 adapter
->stats
.rnbc
+= rd32(E1000_RNBC
);
4740 adapter
->stats
.ruc
+= rd32(E1000_RUC
);
4741 adapter
->stats
.rfc
+= rd32(E1000_RFC
);
4742 adapter
->stats
.rjc
+= rd32(E1000_RJC
);
4743 adapter
->stats
.tor
+= rd32(E1000_TORH
);
4744 adapter
->stats
.tot
+= rd32(E1000_TOTH
);
4745 adapter
->stats
.tpr
+= rd32(E1000_TPR
);
4747 adapter
->stats
.ptc64
+= rd32(E1000_PTC64
);
4748 adapter
->stats
.ptc127
+= rd32(E1000_PTC127
);
4749 adapter
->stats
.ptc255
+= rd32(E1000_PTC255
);
4750 adapter
->stats
.ptc511
+= rd32(E1000_PTC511
);
4751 adapter
->stats
.ptc1023
+= rd32(E1000_PTC1023
);
4752 adapter
->stats
.ptc1522
+= rd32(E1000_PTC1522
);
4754 adapter
->stats
.mptc
+= rd32(E1000_MPTC
);
4755 adapter
->stats
.bptc
+= rd32(E1000_BPTC
);
4757 adapter
->stats
.tpt
+= rd32(E1000_TPT
);
4758 adapter
->stats
.colc
+= rd32(E1000_COLC
);
4760 adapter
->stats
.algnerrc
+= rd32(E1000_ALGNERRC
);
4761 /* read internal phy specific stats */
4762 reg
= rd32(E1000_CTRL_EXT
);
4763 if (!(reg
& E1000_CTRL_EXT_LINK_MODE_MASK
)) {
4764 adapter
->stats
.rxerrc
+= rd32(E1000_RXERRC
);
4765 adapter
->stats
.tncrs
+= rd32(E1000_TNCRS
);
4768 adapter
->stats
.tsctc
+= rd32(E1000_TSCTC
);
4769 adapter
->stats
.tsctfc
+= rd32(E1000_TSCTFC
);
4771 adapter
->stats
.iac
+= rd32(E1000_IAC
);
4772 adapter
->stats
.icrxoc
+= rd32(E1000_ICRXOC
);
4773 adapter
->stats
.icrxptc
+= rd32(E1000_ICRXPTC
);
4774 adapter
->stats
.icrxatc
+= rd32(E1000_ICRXATC
);
4775 adapter
->stats
.ictxptc
+= rd32(E1000_ICTXPTC
);
4776 adapter
->stats
.ictxatc
+= rd32(E1000_ICTXATC
);
4777 adapter
->stats
.ictxqec
+= rd32(E1000_ICTXQEC
);
4778 adapter
->stats
.ictxqmtc
+= rd32(E1000_ICTXQMTC
);
4779 adapter
->stats
.icrxdmtc
+= rd32(E1000_ICRXDMTC
);
4781 /* Fill out the OS statistics structure */
4782 net_stats
->multicast
= adapter
->stats
.mprc
;
4783 net_stats
->collisions
= adapter
->stats
.colc
;
4787 /* RLEC on some newer hardware can be incorrect so build
4788 * our own version based on RUC and ROC */
4789 net_stats
->rx_errors
= adapter
->stats
.rxerrc
+
4790 adapter
->stats
.crcerrs
+ adapter
->stats
.algnerrc
+
4791 adapter
->stats
.ruc
+ adapter
->stats
.roc
+
4792 adapter
->stats
.cexterr
;
4793 net_stats
->rx_length_errors
= adapter
->stats
.ruc
+
4795 net_stats
->rx_crc_errors
= adapter
->stats
.crcerrs
;
4796 net_stats
->rx_frame_errors
= adapter
->stats
.algnerrc
;
4797 net_stats
->rx_missed_errors
= adapter
->stats
.mpc
;
4800 net_stats
->tx_errors
= adapter
->stats
.ecol
+
4801 adapter
->stats
.latecol
;
4802 net_stats
->tx_aborted_errors
= adapter
->stats
.ecol
;
4803 net_stats
->tx_window_errors
= adapter
->stats
.latecol
;
4804 net_stats
->tx_carrier_errors
= adapter
->stats
.tncrs
;
4806 /* Tx Dropped needs to be maintained elsewhere */
4809 if (hw
->phy
.media_type
== e1000_media_type_copper
) {
4810 if ((adapter
->link_speed
== SPEED_1000
) &&
4811 (!igb_read_phy_reg(hw
, PHY_1000T_STATUS
, &phy_tmp
))) {
4812 phy_tmp
&= PHY_IDLE_ERROR_COUNT_MASK
;
4813 adapter
->phy_stats
.idle_errors
+= phy_tmp
;
4817 /* Management Stats */
4818 adapter
->stats
.mgptc
+= rd32(E1000_MGTPTC
);
4819 adapter
->stats
.mgprc
+= rd32(E1000_MGTPRC
);
4820 adapter
->stats
.mgpdc
+= rd32(E1000_MGTPDC
);
4823 reg
= rd32(E1000_MANC
);
4824 if (reg
& E1000_MANC_EN_BMC2OS
) {
4825 adapter
->stats
.o2bgptc
+= rd32(E1000_O2BGPTC
);
4826 adapter
->stats
.o2bspc
+= rd32(E1000_O2BSPC
);
4827 adapter
->stats
.b2ospc
+= rd32(E1000_B2OSPC
);
4828 adapter
->stats
.b2ogprc
+= rd32(E1000_B2OGPRC
);
4832 static irqreturn_t
igb_msix_other(int irq
, void *data
)
4834 struct igb_adapter
*adapter
= data
;
4835 struct e1000_hw
*hw
= &adapter
->hw
;
4836 u32 icr
= rd32(E1000_ICR
);
4837 /* reading ICR causes bit 31 of EICR to be cleared */
4839 if (icr
& E1000_ICR_DRSTA
)
4840 schedule_work(&adapter
->reset_task
);
4842 if (icr
& E1000_ICR_DOUTSYNC
) {
4843 /* HW is reporting DMA is out of sync */
4844 adapter
->stats
.doosync
++;
4845 /* The DMA Out of Sync is also indication of a spoof event
4846 * in IOV mode. Check the Wrong VM Behavior register to
4847 * see if it is really a spoof event. */
4848 igb_check_wvbr(adapter
);
4851 /* Check for a mailbox event */
4852 if (icr
& E1000_ICR_VMMB
)
4853 igb_msg_task(adapter
);
4855 if (icr
& E1000_ICR_LSC
) {
4856 hw
->mac
.get_link_status
= 1;
4857 /* guard against interrupt when we're going down */
4858 if (!test_bit(__IGB_DOWN
, &adapter
->state
))
4859 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
4862 wr32(E1000_EIMS
, adapter
->eims_other
);
4867 static void igb_write_itr(struct igb_q_vector
*q_vector
)
4869 struct igb_adapter
*adapter
= q_vector
->adapter
;
4870 u32 itr_val
= q_vector
->itr_val
& 0x7FFC;
4872 if (!q_vector
->set_itr
)
4878 if (adapter
->hw
.mac
.type
== e1000_82575
)
4879 itr_val
|= itr_val
<< 16;
4881 itr_val
|= E1000_EITR_CNT_IGNR
;
4883 writel(itr_val
, q_vector
->itr_register
);
4884 q_vector
->set_itr
= 0;
4887 static irqreturn_t
igb_msix_ring(int irq
, void *data
)
4889 struct igb_q_vector
*q_vector
= data
;
4891 /* Write the ITR value calculated from the previous interrupt. */
4892 igb_write_itr(q_vector
);
4894 napi_schedule(&q_vector
->napi
);
4899 #ifdef CONFIG_IGB_DCA
4900 static void igb_update_dca(struct igb_q_vector
*q_vector
)
4902 struct igb_adapter
*adapter
= q_vector
->adapter
;
4903 struct e1000_hw
*hw
= &adapter
->hw
;
4904 int cpu
= get_cpu();
4906 if (q_vector
->cpu
== cpu
)
4909 if (q_vector
->tx
.ring
) {
4910 int q
= q_vector
->tx
.ring
->reg_idx
;
4911 u32 dca_txctrl
= rd32(E1000_DCA_TXCTRL(q
));
4912 if (hw
->mac
.type
== e1000_82575
) {
4913 dca_txctrl
&= ~E1000_DCA_TXCTRL_CPUID_MASK
;
4914 dca_txctrl
|= dca3_get_tag(&adapter
->pdev
->dev
, cpu
);
4916 dca_txctrl
&= ~E1000_DCA_TXCTRL_CPUID_MASK_82576
;
4917 dca_txctrl
|= dca3_get_tag(&adapter
->pdev
->dev
, cpu
) <<
4918 E1000_DCA_TXCTRL_CPUID_SHIFT
;
4920 dca_txctrl
|= E1000_DCA_TXCTRL_DESC_DCA_EN
;
4921 wr32(E1000_DCA_TXCTRL(q
), dca_txctrl
);
4923 if (q_vector
->rx
.ring
) {
4924 int q
= q_vector
->rx
.ring
->reg_idx
;
4925 u32 dca_rxctrl
= rd32(E1000_DCA_RXCTRL(q
));
4926 if (hw
->mac
.type
== e1000_82575
) {
4927 dca_rxctrl
&= ~E1000_DCA_RXCTRL_CPUID_MASK
;
4928 dca_rxctrl
|= dca3_get_tag(&adapter
->pdev
->dev
, cpu
);
4930 dca_rxctrl
&= ~E1000_DCA_RXCTRL_CPUID_MASK_82576
;
4931 dca_rxctrl
|= dca3_get_tag(&adapter
->pdev
->dev
, cpu
) <<
4932 E1000_DCA_RXCTRL_CPUID_SHIFT
;
4934 dca_rxctrl
|= E1000_DCA_RXCTRL_DESC_DCA_EN
;
4935 dca_rxctrl
|= E1000_DCA_RXCTRL_HEAD_DCA_EN
;
4936 dca_rxctrl
|= E1000_DCA_RXCTRL_DATA_DCA_EN
;
4937 wr32(E1000_DCA_RXCTRL(q
), dca_rxctrl
);
4939 q_vector
->cpu
= cpu
;
4944 static void igb_setup_dca(struct igb_adapter
*adapter
)
4946 struct e1000_hw
*hw
= &adapter
->hw
;
4949 if (!(adapter
->flags
& IGB_FLAG_DCA_ENABLED
))
4952 /* Always use CB2 mode, difference is masked in the CB driver. */
4953 wr32(E1000_DCA_CTRL
, E1000_DCA_CTRL_DCA_MODE_CB2
);
4955 for (i
= 0; i
< adapter
->num_q_vectors
; i
++) {
4956 adapter
->q_vector
[i
]->cpu
= -1;
4957 igb_update_dca(adapter
->q_vector
[i
]);
4961 static int __igb_notify_dca(struct device
*dev
, void *data
)
4963 struct net_device
*netdev
= dev_get_drvdata(dev
);
4964 struct igb_adapter
*adapter
= netdev_priv(netdev
);
4965 struct pci_dev
*pdev
= adapter
->pdev
;
4966 struct e1000_hw
*hw
= &adapter
->hw
;
4967 unsigned long event
= *(unsigned long *)data
;
4970 case DCA_PROVIDER_ADD
:
4971 /* if already enabled, don't do it again */
4972 if (adapter
->flags
& IGB_FLAG_DCA_ENABLED
)
4974 if (dca_add_requester(dev
) == 0) {
4975 adapter
->flags
|= IGB_FLAG_DCA_ENABLED
;
4976 dev_info(&pdev
->dev
, "DCA enabled\n");
4977 igb_setup_dca(adapter
);
4980 /* Fall Through since DCA is disabled. */
4981 case DCA_PROVIDER_REMOVE
:
4982 if (adapter
->flags
& IGB_FLAG_DCA_ENABLED
) {
4983 /* without this a class_device is left
4984 * hanging around in the sysfs model */
4985 dca_remove_requester(dev
);
4986 dev_info(&pdev
->dev
, "DCA disabled\n");
4987 adapter
->flags
&= ~IGB_FLAG_DCA_ENABLED
;
4988 wr32(E1000_DCA_CTRL
, E1000_DCA_CTRL_DCA_MODE_DISABLE
);
4996 static int igb_notify_dca(struct notifier_block
*nb
, unsigned long event
,
5001 ret_val
= driver_for_each_device(&igb_driver
.driver
, NULL
, &event
,
5004 return ret_val
? NOTIFY_BAD
: NOTIFY_DONE
;
5006 #endif /* CONFIG_IGB_DCA */
5008 #ifdef CONFIG_PCI_IOV
5009 static int igb_vf_configure(struct igb_adapter
*adapter
, int vf
)
5011 unsigned char mac_addr
[ETH_ALEN
];
5012 struct pci_dev
*pdev
= adapter
->pdev
;
5013 struct e1000_hw
*hw
= &adapter
->hw
;
5014 struct pci_dev
*pvfdev
;
5015 unsigned int device_id
;
5018 eth_random_addr(mac_addr
);
5019 igb_set_vf_mac(adapter
, vf
, mac_addr
);
5021 switch (adapter
->hw
.mac
.type
) {
5023 device_id
= IGB_82576_VF_DEV_ID
;
5024 /* VF Stride for 82576 is 2 */
5025 thisvf_devfn
= (pdev
->devfn
+ 0x80 + (vf
<< 1)) |
5029 device_id
= IGB_I350_VF_DEV_ID
;
5030 /* VF Stride for I350 is 4 */
5031 thisvf_devfn
= (pdev
->devfn
+ 0x80 + (vf
<< 2)) |
5040 pvfdev
= pci_get_device(hw
->vendor_id
, device_id
, NULL
);
5042 if (pvfdev
->devfn
== thisvf_devfn
)
5044 pvfdev
= pci_get_device(hw
->vendor_id
,
5049 adapter
->vf_data
[vf
].vfdev
= pvfdev
;
5052 "Couldn't find pci dev ptr for VF %4.4x\n",
5054 return pvfdev
!= NULL
;
5057 static int igb_find_enabled_vfs(struct igb_adapter
*adapter
)
5059 struct e1000_hw
*hw
= &adapter
->hw
;
5060 struct pci_dev
*pdev
= adapter
->pdev
;
5061 struct pci_dev
*pvfdev
;
5064 unsigned int device_id
;
5067 switch (adapter
->hw
.mac
.type
) {
5069 device_id
= IGB_82576_VF_DEV_ID
;
5070 /* VF Stride for 82576 is 2 */
5074 device_id
= IGB_I350_VF_DEV_ID
;
5075 /* VF Stride for I350 is 4 */
5084 vf_devfn
= pdev
->devfn
+ 0x80;
5085 pvfdev
= pci_get_device(hw
->vendor_id
, device_id
, NULL
);
5087 if (pvfdev
->devfn
== vf_devfn
&&
5088 (pvfdev
->bus
->number
>= pdev
->bus
->number
))
5090 vf_devfn
+= vf_stride
;
5091 pvfdev
= pci_get_device(hw
->vendor_id
,
5098 static int igb_check_vf_assignment(struct igb_adapter
*adapter
)
5101 for (i
= 0; i
< adapter
->vfs_allocated_count
; i
++) {
5102 if (adapter
->vf_data
[i
].vfdev
) {
5103 if (adapter
->vf_data
[i
].vfdev
->dev_flags
&
5104 PCI_DEV_FLAGS_ASSIGNED
)
5112 static void igb_ping_all_vfs(struct igb_adapter
*adapter
)
5114 struct e1000_hw
*hw
= &adapter
->hw
;
5118 for (i
= 0 ; i
< adapter
->vfs_allocated_count
; i
++) {
5119 ping
= E1000_PF_CONTROL_MSG
;
5120 if (adapter
->vf_data
[i
].flags
& IGB_VF_FLAG_CTS
)
5121 ping
|= E1000_VT_MSGTYPE_CTS
;
5122 igb_write_mbx(hw
, &ping
, 1, i
);
5126 static int igb_set_vf_promisc(struct igb_adapter
*adapter
, u32
*msgbuf
, u32 vf
)
5128 struct e1000_hw
*hw
= &adapter
->hw
;
5129 u32 vmolr
= rd32(E1000_VMOLR(vf
));
5130 struct vf_data_storage
*vf_data
= &adapter
->vf_data
[vf
];
5132 vf_data
->flags
&= ~(IGB_VF_FLAG_UNI_PROMISC
|
5133 IGB_VF_FLAG_MULTI_PROMISC
);
5134 vmolr
&= ~(E1000_VMOLR_ROPE
| E1000_VMOLR_ROMPE
| E1000_VMOLR_MPME
);
5136 if (*msgbuf
& E1000_VF_SET_PROMISC_MULTICAST
) {
5137 vmolr
|= E1000_VMOLR_MPME
;
5138 vf_data
->flags
|= IGB_VF_FLAG_MULTI_PROMISC
;
5139 *msgbuf
&= ~E1000_VF_SET_PROMISC_MULTICAST
;
5142 * if we have hashes and we are clearing a multicast promisc
5143 * flag we need to write the hashes to the MTA as this step
5144 * was previously skipped
5146 if (vf_data
->num_vf_mc_hashes
> 30) {
5147 vmolr
|= E1000_VMOLR_MPME
;
5148 } else if (vf_data
->num_vf_mc_hashes
) {
5150 vmolr
|= E1000_VMOLR_ROMPE
;
5151 for (j
= 0; j
< vf_data
->num_vf_mc_hashes
; j
++)
5152 igb_mta_set(hw
, vf_data
->vf_mc_hashes
[j
]);
5156 wr32(E1000_VMOLR(vf
), vmolr
);
5158 /* there are flags left unprocessed, likely not supported */
5159 if (*msgbuf
& E1000_VT_MSGINFO_MASK
)
5166 static int igb_set_vf_multicasts(struct igb_adapter
*adapter
,
5167 u32
*msgbuf
, u32 vf
)
5169 int n
= (msgbuf
[0] & E1000_VT_MSGINFO_MASK
) >> E1000_VT_MSGINFO_SHIFT
;
5170 u16
*hash_list
= (u16
*)&msgbuf
[1];
5171 struct vf_data_storage
*vf_data
= &adapter
->vf_data
[vf
];
5174 /* salt away the number of multicast addresses assigned
5175 * to this VF for later use to restore when the PF multi cast
5178 vf_data
->num_vf_mc_hashes
= n
;
5180 /* only up to 30 hash values supported */
5184 /* store the hashes for later use */
5185 for (i
= 0; i
< n
; i
++)
5186 vf_data
->vf_mc_hashes
[i
] = hash_list
[i
];
5188 /* Flush and reset the mta with the new values */
5189 igb_set_rx_mode(adapter
->netdev
);
5194 static void igb_restore_vf_multicasts(struct igb_adapter
*adapter
)
5196 struct e1000_hw
*hw
= &adapter
->hw
;
5197 struct vf_data_storage
*vf_data
;
5200 for (i
= 0; i
< adapter
->vfs_allocated_count
; i
++) {
5201 u32 vmolr
= rd32(E1000_VMOLR(i
));
5202 vmolr
&= ~(E1000_VMOLR_ROMPE
| E1000_VMOLR_MPME
);
5204 vf_data
= &adapter
->vf_data
[i
];
5206 if ((vf_data
->num_vf_mc_hashes
> 30) ||
5207 (vf_data
->flags
& IGB_VF_FLAG_MULTI_PROMISC
)) {
5208 vmolr
|= E1000_VMOLR_MPME
;
5209 } else if (vf_data
->num_vf_mc_hashes
) {
5210 vmolr
|= E1000_VMOLR_ROMPE
;
5211 for (j
= 0; j
< vf_data
->num_vf_mc_hashes
; j
++)
5212 igb_mta_set(hw
, vf_data
->vf_mc_hashes
[j
]);
5214 wr32(E1000_VMOLR(i
), vmolr
);
5218 static void igb_clear_vf_vfta(struct igb_adapter
*adapter
, u32 vf
)
5220 struct e1000_hw
*hw
= &adapter
->hw
;
5221 u32 pool_mask
, reg
, vid
;
5224 pool_mask
= 1 << (E1000_VLVF_POOLSEL_SHIFT
+ vf
);
5226 /* Find the vlan filter for this id */
5227 for (i
= 0; i
< E1000_VLVF_ARRAY_SIZE
; i
++) {
5228 reg
= rd32(E1000_VLVF(i
));
5230 /* remove the vf from the pool */
5233 /* if pool is empty then remove entry from vfta */
5234 if (!(reg
& E1000_VLVF_POOLSEL_MASK
) &&
5235 (reg
& E1000_VLVF_VLANID_ENABLE
)) {
5237 vid
= reg
& E1000_VLVF_VLANID_MASK
;
5238 igb_vfta_set(hw
, vid
, false);
5241 wr32(E1000_VLVF(i
), reg
);
5244 adapter
->vf_data
[vf
].vlans_enabled
= 0;
5247 static s32
igb_vlvf_set(struct igb_adapter
*adapter
, u32 vid
, bool add
, u32 vf
)
5249 struct e1000_hw
*hw
= &adapter
->hw
;
5252 /* The vlvf table only exists on 82576 hardware and newer */
5253 if (hw
->mac
.type
< e1000_82576
)
5256 /* we only need to do this if VMDq is enabled */
5257 if (!adapter
->vfs_allocated_count
)
5260 /* Find the vlan filter for this id */
5261 for (i
= 0; i
< E1000_VLVF_ARRAY_SIZE
; i
++) {
5262 reg
= rd32(E1000_VLVF(i
));
5263 if ((reg
& E1000_VLVF_VLANID_ENABLE
) &&
5264 vid
== (reg
& E1000_VLVF_VLANID_MASK
))
5269 if (i
== E1000_VLVF_ARRAY_SIZE
) {
5270 /* Did not find a matching VLAN ID entry that was
5271 * enabled. Search for a free filter entry, i.e.
5272 * one without the enable bit set
5274 for (i
= 0; i
< E1000_VLVF_ARRAY_SIZE
; i
++) {
5275 reg
= rd32(E1000_VLVF(i
));
5276 if (!(reg
& E1000_VLVF_VLANID_ENABLE
))
5280 if (i
< E1000_VLVF_ARRAY_SIZE
) {
5281 /* Found an enabled/available entry */
5282 reg
|= 1 << (E1000_VLVF_POOLSEL_SHIFT
+ vf
);
5284 /* if !enabled we need to set this up in vfta */
5285 if (!(reg
& E1000_VLVF_VLANID_ENABLE
)) {
5286 /* add VID to filter table */
5287 igb_vfta_set(hw
, vid
, true);
5288 reg
|= E1000_VLVF_VLANID_ENABLE
;
5290 reg
&= ~E1000_VLVF_VLANID_MASK
;
5292 wr32(E1000_VLVF(i
), reg
);
5294 /* do not modify RLPML for PF devices */
5295 if (vf
>= adapter
->vfs_allocated_count
)
5298 if (!adapter
->vf_data
[vf
].vlans_enabled
) {
5300 reg
= rd32(E1000_VMOLR(vf
));
5301 size
= reg
& E1000_VMOLR_RLPML_MASK
;
5303 reg
&= ~E1000_VMOLR_RLPML_MASK
;
5305 wr32(E1000_VMOLR(vf
), reg
);
5308 adapter
->vf_data
[vf
].vlans_enabled
++;
5311 if (i
< E1000_VLVF_ARRAY_SIZE
) {
5312 /* remove vf from the pool */
5313 reg
&= ~(1 << (E1000_VLVF_POOLSEL_SHIFT
+ vf
));
5314 /* if pool is empty then remove entry from vfta */
5315 if (!(reg
& E1000_VLVF_POOLSEL_MASK
)) {
5317 igb_vfta_set(hw
, vid
, false);
5319 wr32(E1000_VLVF(i
), reg
);
5321 /* do not modify RLPML for PF devices */
5322 if (vf
>= adapter
->vfs_allocated_count
)
5325 adapter
->vf_data
[vf
].vlans_enabled
--;
5326 if (!adapter
->vf_data
[vf
].vlans_enabled
) {
5328 reg
= rd32(E1000_VMOLR(vf
));
5329 size
= reg
& E1000_VMOLR_RLPML_MASK
;
5331 reg
&= ~E1000_VMOLR_RLPML_MASK
;
5333 wr32(E1000_VMOLR(vf
), reg
);
5340 static void igb_set_vmvir(struct igb_adapter
*adapter
, u32 vid
, u32 vf
)
5342 struct e1000_hw
*hw
= &adapter
->hw
;
5345 wr32(E1000_VMVIR(vf
), (vid
| E1000_VMVIR_VLANA_DEFAULT
));
5347 wr32(E1000_VMVIR(vf
), 0);
5350 static int igb_ndo_set_vf_vlan(struct net_device
*netdev
,
5351 int vf
, u16 vlan
, u8 qos
)
5354 struct igb_adapter
*adapter
= netdev_priv(netdev
);
5356 if ((vf
>= adapter
->vfs_allocated_count
) || (vlan
> 4095) || (qos
> 7))
5359 err
= igb_vlvf_set(adapter
, vlan
, !!vlan
, vf
);
5362 igb_set_vmvir(adapter
, vlan
| (qos
<< VLAN_PRIO_SHIFT
), vf
);
5363 igb_set_vmolr(adapter
, vf
, !vlan
);
5364 adapter
->vf_data
[vf
].pf_vlan
= vlan
;
5365 adapter
->vf_data
[vf
].pf_qos
= qos
;
5366 dev_info(&adapter
->pdev
->dev
,
5367 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan
, qos
, vf
);
5368 if (test_bit(__IGB_DOWN
, &adapter
->state
)) {
5369 dev_warn(&adapter
->pdev
->dev
,
5370 "The VF VLAN has been set,"
5371 " but the PF device is not up.\n");
5372 dev_warn(&adapter
->pdev
->dev
,
5373 "Bring the PF device up before"
5374 " attempting to use the VF device.\n");
5377 igb_vlvf_set(adapter
, adapter
->vf_data
[vf
].pf_vlan
,
5379 igb_set_vmvir(adapter
, vlan
, vf
);
5380 igb_set_vmolr(adapter
, vf
, true);
5381 adapter
->vf_data
[vf
].pf_vlan
= 0;
5382 adapter
->vf_data
[vf
].pf_qos
= 0;
5388 static int igb_set_vf_vlan(struct igb_adapter
*adapter
, u32
*msgbuf
, u32 vf
)
5390 int add
= (msgbuf
[0] & E1000_VT_MSGINFO_MASK
) >> E1000_VT_MSGINFO_SHIFT
;
5391 int vid
= (msgbuf
[1] & E1000_VLVF_VLANID_MASK
);
5393 return igb_vlvf_set(adapter
, vid
, add
, vf
);
5396 static inline void igb_vf_reset(struct igb_adapter
*adapter
, u32 vf
)
5398 /* clear flags - except flag that indicates PF has set the MAC */
5399 adapter
->vf_data
[vf
].flags
&= IGB_VF_FLAG_PF_SET_MAC
;
5400 adapter
->vf_data
[vf
].last_nack
= jiffies
;
5402 /* reset offloads to defaults */
5403 igb_set_vmolr(adapter
, vf
, true);
5405 /* reset vlans for device */
5406 igb_clear_vf_vfta(adapter
, vf
);
5407 if (adapter
->vf_data
[vf
].pf_vlan
)
5408 igb_ndo_set_vf_vlan(adapter
->netdev
, vf
,
5409 adapter
->vf_data
[vf
].pf_vlan
,
5410 adapter
->vf_data
[vf
].pf_qos
);
5412 igb_clear_vf_vfta(adapter
, vf
);
5414 /* reset multicast table array for vf */
5415 adapter
->vf_data
[vf
].num_vf_mc_hashes
= 0;
5417 /* Flush and reset the mta with the new values */
5418 igb_set_rx_mode(adapter
->netdev
);
5421 static void igb_vf_reset_event(struct igb_adapter
*adapter
, u32 vf
)
5423 unsigned char *vf_mac
= adapter
->vf_data
[vf
].vf_mac_addresses
;
5425 /* generate a new mac address as we were hotplug removed/added */
5426 if (!(adapter
->vf_data
[vf
].flags
& IGB_VF_FLAG_PF_SET_MAC
))
5427 eth_random_addr(vf_mac
);
5429 /* process remaining reset events */
5430 igb_vf_reset(adapter
, vf
);
5433 static void igb_vf_reset_msg(struct igb_adapter
*adapter
, u32 vf
)
5435 struct e1000_hw
*hw
= &adapter
->hw
;
5436 unsigned char *vf_mac
= adapter
->vf_data
[vf
].vf_mac_addresses
;
5437 int rar_entry
= hw
->mac
.rar_entry_count
- (vf
+ 1);
5439 u8
*addr
= (u8
*)(&msgbuf
[1]);
5441 /* process all the same items cleared in a function level reset */
5442 igb_vf_reset(adapter
, vf
);
5444 /* set vf mac address */
5445 igb_rar_set_qsel(adapter
, vf_mac
, rar_entry
, vf
);
5447 /* enable transmit and receive for vf */
5448 reg
= rd32(E1000_VFTE
);
5449 wr32(E1000_VFTE
, reg
| (1 << vf
));
5450 reg
= rd32(E1000_VFRE
);
5451 wr32(E1000_VFRE
, reg
| (1 << vf
));
5453 adapter
->vf_data
[vf
].flags
|= IGB_VF_FLAG_CTS
;
5455 /* reply to reset with ack and vf mac address */
5456 msgbuf
[0] = E1000_VF_RESET
| E1000_VT_MSGTYPE_ACK
;
5457 memcpy(addr
, vf_mac
, 6);
5458 igb_write_mbx(hw
, msgbuf
, 3, vf
);
5461 static int igb_set_vf_mac_addr(struct igb_adapter
*adapter
, u32
*msg
, int vf
)
5464 * The VF MAC Address is stored in a packed array of bytes
5465 * starting at the second 32 bit word of the msg array
5467 unsigned char *addr
= (char *)&msg
[1];
5470 if (is_valid_ether_addr(addr
))
5471 err
= igb_set_vf_mac(adapter
, vf
, addr
);
5476 static void igb_rcv_ack_from_vf(struct igb_adapter
*adapter
, u32 vf
)
5478 struct e1000_hw
*hw
= &adapter
->hw
;
5479 struct vf_data_storage
*vf_data
= &adapter
->vf_data
[vf
];
5480 u32 msg
= E1000_VT_MSGTYPE_NACK
;
5482 /* if device isn't clear to send it shouldn't be reading either */
5483 if (!(vf_data
->flags
& IGB_VF_FLAG_CTS
) &&
5484 time_after(jiffies
, vf_data
->last_nack
+ (2 * HZ
))) {
5485 igb_write_mbx(hw
, &msg
, 1, vf
);
5486 vf_data
->last_nack
= jiffies
;
5490 static void igb_rcv_msg_from_vf(struct igb_adapter
*adapter
, u32 vf
)
5492 struct pci_dev
*pdev
= adapter
->pdev
;
5493 u32 msgbuf
[E1000_VFMAILBOX_SIZE
];
5494 struct e1000_hw
*hw
= &adapter
->hw
;
5495 struct vf_data_storage
*vf_data
= &adapter
->vf_data
[vf
];
5498 retval
= igb_read_mbx(hw
, msgbuf
, E1000_VFMAILBOX_SIZE
, vf
);
5501 /* if receive failed revoke VF CTS stats and restart init */
5502 dev_err(&pdev
->dev
, "Error receiving message from VF\n");
5503 vf_data
->flags
&= ~IGB_VF_FLAG_CTS
;
5504 if (!time_after(jiffies
, vf_data
->last_nack
+ (2 * HZ
)))
5509 /* this is a message we already processed, do nothing */
5510 if (msgbuf
[0] & (E1000_VT_MSGTYPE_ACK
| E1000_VT_MSGTYPE_NACK
))
5514 * until the vf completes a reset it should not be
5515 * allowed to start any configuration.
5518 if (msgbuf
[0] == E1000_VF_RESET
) {
5519 igb_vf_reset_msg(adapter
, vf
);
5523 if (!(vf_data
->flags
& IGB_VF_FLAG_CTS
)) {
5524 if (!time_after(jiffies
, vf_data
->last_nack
+ (2 * HZ
)))
5530 switch ((msgbuf
[0] & 0xFFFF)) {
5531 case E1000_VF_SET_MAC_ADDR
:
5533 if (!(vf_data
->flags
& IGB_VF_FLAG_PF_SET_MAC
))
5534 retval
= igb_set_vf_mac_addr(adapter
, msgbuf
, vf
);
5536 dev_warn(&pdev
->dev
,
5537 "VF %d attempted to override administratively "
5538 "set MAC address\nReload the VF driver to "
5539 "resume operations\n", vf
);
5541 case E1000_VF_SET_PROMISC
:
5542 retval
= igb_set_vf_promisc(adapter
, msgbuf
, vf
);
5544 case E1000_VF_SET_MULTICAST
:
5545 retval
= igb_set_vf_multicasts(adapter
, msgbuf
, vf
);
5547 case E1000_VF_SET_LPE
:
5548 retval
= igb_set_vf_rlpml(adapter
, msgbuf
[1], vf
);
5550 case E1000_VF_SET_VLAN
:
5552 if (vf_data
->pf_vlan
)
5553 dev_warn(&pdev
->dev
,
5554 "VF %d attempted to override administratively "
5555 "set VLAN tag\nReload the VF driver to "
5556 "resume operations\n", vf
);
5558 retval
= igb_set_vf_vlan(adapter
, msgbuf
, vf
);
5561 dev_err(&pdev
->dev
, "Unhandled Msg %08x\n", msgbuf
[0]);
5566 msgbuf
[0] |= E1000_VT_MSGTYPE_CTS
;
5568 /* notify the VF of the results of what it sent us */
5570 msgbuf
[0] |= E1000_VT_MSGTYPE_NACK
;
5572 msgbuf
[0] |= E1000_VT_MSGTYPE_ACK
;
5574 igb_write_mbx(hw
, msgbuf
, 1, vf
);
5577 static void igb_msg_task(struct igb_adapter
*adapter
)
5579 struct e1000_hw
*hw
= &adapter
->hw
;
5582 for (vf
= 0; vf
< adapter
->vfs_allocated_count
; vf
++) {
5583 /* process any reset requests */
5584 if (!igb_check_for_rst(hw
, vf
))
5585 igb_vf_reset_event(adapter
, vf
);
5587 /* process any messages pending */
5588 if (!igb_check_for_msg(hw
, vf
))
5589 igb_rcv_msg_from_vf(adapter
, vf
);
5591 /* process any acks */
5592 if (!igb_check_for_ack(hw
, vf
))
5593 igb_rcv_ack_from_vf(adapter
, vf
);
5598 * igb_set_uta - Set unicast filter table address
5599 * @adapter: board private structure
5601 * The unicast table address is a register array of 32-bit registers.
5602 * The table is meant to be used in a way similar to how the MTA is used
5603 * however due to certain limitations in the hardware it is necessary to
5604 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5605 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
5607 static void igb_set_uta(struct igb_adapter
*adapter
)
5609 struct e1000_hw
*hw
= &adapter
->hw
;
5612 /* The UTA table only exists on 82576 hardware and newer */
5613 if (hw
->mac
.type
< e1000_82576
)
5616 /* we only need to do this if VMDq is enabled */
5617 if (!adapter
->vfs_allocated_count
)
5620 for (i
= 0; i
< hw
->mac
.uta_reg_count
; i
++)
5621 array_wr32(E1000_UTA
, i
, ~0);
5625 * igb_intr_msi - Interrupt Handler
5626 * @irq: interrupt number
5627 * @data: pointer to a network interface device structure
5629 static irqreturn_t
igb_intr_msi(int irq
, void *data
)
5631 struct igb_adapter
*adapter
= data
;
5632 struct igb_q_vector
*q_vector
= adapter
->q_vector
[0];
5633 struct e1000_hw
*hw
= &adapter
->hw
;
5634 /* read ICR disables interrupts using IAM */
5635 u32 icr
= rd32(E1000_ICR
);
5637 igb_write_itr(q_vector
);
5639 if (icr
& E1000_ICR_DRSTA
)
5640 schedule_work(&adapter
->reset_task
);
5642 if (icr
& E1000_ICR_DOUTSYNC
) {
5643 /* HW is reporting DMA is out of sync */
5644 adapter
->stats
.doosync
++;
5647 if (icr
& (E1000_ICR_RXSEQ
| E1000_ICR_LSC
)) {
5648 hw
->mac
.get_link_status
= 1;
5649 if (!test_bit(__IGB_DOWN
, &adapter
->state
))
5650 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
5653 napi_schedule(&q_vector
->napi
);
5659 * igb_intr - Legacy Interrupt Handler
5660 * @irq: interrupt number
5661 * @data: pointer to a network interface device structure
5663 static irqreturn_t
igb_intr(int irq
, void *data
)
5665 struct igb_adapter
*adapter
= data
;
5666 struct igb_q_vector
*q_vector
= adapter
->q_vector
[0];
5667 struct e1000_hw
*hw
= &adapter
->hw
;
5668 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5669 * need for the IMC write */
5670 u32 icr
= rd32(E1000_ICR
);
5672 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5673 * not set, then the adapter didn't send an interrupt */
5674 if (!(icr
& E1000_ICR_INT_ASSERTED
))
5677 igb_write_itr(q_vector
);
5679 if (icr
& E1000_ICR_DRSTA
)
5680 schedule_work(&adapter
->reset_task
);
5682 if (icr
& E1000_ICR_DOUTSYNC
) {
5683 /* HW is reporting DMA is out of sync */
5684 adapter
->stats
.doosync
++;
5687 if (icr
& (E1000_ICR_RXSEQ
| E1000_ICR_LSC
)) {
5688 hw
->mac
.get_link_status
= 1;
5689 /* guard against interrupt when we're going down */
5690 if (!test_bit(__IGB_DOWN
, &adapter
->state
))
5691 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
5694 napi_schedule(&q_vector
->napi
);
5699 static void igb_ring_irq_enable(struct igb_q_vector
*q_vector
)
5701 struct igb_adapter
*adapter
= q_vector
->adapter
;
5702 struct e1000_hw
*hw
= &adapter
->hw
;
5704 if ((q_vector
->rx
.ring
&& (adapter
->rx_itr_setting
& 3)) ||
5705 (!q_vector
->rx
.ring
&& (adapter
->tx_itr_setting
& 3))) {
5706 if ((adapter
->num_q_vectors
== 1) && !adapter
->vf_data
)
5707 igb_set_itr(q_vector
);
5709 igb_update_ring_itr(q_vector
);
5712 if (!test_bit(__IGB_DOWN
, &adapter
->state
)) {
5713 if (adapter
->msix_entries
)
5714 wr32(E1000_EIMS
, q_vector
->eims_value
);
5716 igb_irq_enable(adapter
);
5721 * igb_poll - NAPI Rx polling callback
5722 * @napi: napi polling structure
5723 * @budget: count of how many packets we should handle
5725 static int igb_poll(struct napi_struct
*napi
, int budget
)
5727 struct igb_q_vector
*q_vector
= container_of(napi
,
5728 struct igb_q_vector
,
5730 bool clean_complete
= true;
5732 #ifdef CONFIG_IGB_DCA
5733 if (q_vector
->adapter
->flags
& IGB_FLAG_DCA_ENABLED
)
5734 igb_update_dca(q_vector
);
5736 if (q_vector
->tx
.ring
)
5737 clean_complete
= igb_clean_tx_irq(q_vector
);
5739 if (q_vector
->rx
.ring
)
5740 clean_complete
&= igb_clean_rx_irq(q_vector
, budget
);
5742 /* If all work not completed, return budget and keep polling */
5743 if (!clean_complete
)
5746 /* If not enough Rx work done, exit the polling mode */
5747 napi_complete(napi
);
5748 igb_ring_irq_enable(q_vector
);
5753 #ifdef CONFIG_IGB_PTP
5755 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5756 * @q_vector: pointer to q_vector containing needed info
5757 * @buffer: pointer to igb_tx_buffer structure
5759 * If we were asked to do hardware stamping and such a time stamp is
5760 * available, then it must have been for this skb here because we only
5761 * allow only one such packet into the queue.
5763 static void igb_tx_hwtstamp(struct igb_q_vector
*q_vector
,
5764 struct igb_tx_buffer
*buffer_info
)
5766 struct igb_adapter
*adapter
= q_vector
->adapter
;
5767 struct e1000_hw
*hw
= &adapter
->hw
;
5768 struct skb_shared_hwtstamps shhwtstamps
;
5771 /* if skb does not support hw timestamp or TX stamp not valid exit */
5772 if (likely(!(buffer_info
->tx_flags
& IGB_TX_FLAGS_TSTAMP
)) ||
5773 !(rd32(E1000_TSYNCTXCTL
) & E1000_TSYNCTXCTL_VALID
))
5776 regval
= rd32(E1000_TXSTMPL
);
5777 regval
|= (u64
)rd32(E1000_TXSTMPH
) << 32;
5779 igb_systim_to_hwtstamp(adapter
, &shhwtstamps
, regval
);
5780 skb_tstamp_tx(buffer_info
->skb
, &shhwtstamps
);
5782 #endif /* CONFIG_IGB_PTP */
5785 * igb_clean_tx_irq - Reclaim resources after transmit completes
5786 * @q_vector: pointer to q_vector containing needed info
5788 * returns true if ring is completely cleaned
5790 static bool igb_clean_tx_irq(struct igb_q_vector
*q_vector
)
5792 struct igb_adapter
*adapter
= q_vector
->adapter
;
5793 struct igb_ring
*tx_ring
= q_vector
->tx
.ring
;
5794 struct igb_tx_buffer
*tx_buffer
;
5795 union e1000_adv_tx_desc
*tx_desc
, *eop_desc
;
5796 unsigned int total_bytes
= 0, total_packets
= 0;
5797 unsigned int budget
= q_vector
->tx
.work_limit
;
5798 unsigned int i
= tx_ring
->next_to_clean
;
5800 if (test_bit(__IGB_DOWN
, &adapter
->state
))
5803 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
5804 tx_desc
= IGB_TX_DESC(tx_ring
, i
);
5805 i
-= tx_ring
->count
;
5807 for (; budget
; budget
--) {
5808 eop_desc
= tx_buffer
->next_to_watch
;
5810 /* prevent any other reads prior to eop_desc */
5813 /* if next_to_watch is not set then there is no work pending */
5817 /* if DD is not set pending work has not been completed */
5818 if (!(eop_desc
->wb
.status
& cpu_to_le32(E1000_TXD_STAT_DD
)))
5821 /* clear next_to_watch to prevent false hangs */
5822 tx_buffer
->next_to_watch
= NULL
;
5824 /* update the statistics for this packet */
5825 total_bytes
+= tx_buffer
->bytecount
;
5826 total_packets
+= tx_buffer
->gso_segs
;
5828 #ifdef CONFIG_IGB_PTP
5829 /* retrieve hardware timestamp */
5830 igb_tx_hwtstamp(q_vector
, tx_buffer
);
5831 #endif /* CONFIG_IGB_PTP */
5834 dev_kfree_skb_any(tx_buffer
->skb
);
5835 tx_buffer
->skb
= NULL
;
5837 /* unmap skb header data */
5838 dma_unmap_single(tx_ring
->dev
,
5843 /* clear last DMA location and unmap remaining buffers */
5844 while (tx_desc
!= eop_desc
) {
5851 i
-= tx_ring
->count
;
5852 tx_buffer
= tx_ring
->tx_buffer_info
;
5853 tx_desc
= IGB_TX_DESC(tx_ring
, 0);
5856 /* unmap any remaining paged data */
5857 if (tx_buffer
->dma
) {
5858 dma_unmap_page(tx_ring
->dev
,
5865 /* clear last DMA location */
5868 /* move us one more past the eop_desc for start of next pkt */
5873 i
-= tx_ring
->count
;
5874 tx_buffer
= tx_ring
->tx_buffer_info
;
5875 tx_desc
= IGB_TX_DESC(tx_ring
, 0);
5879 netdev_tx_completed_queue(txring_txq(tx_ring
),
5880 total_packets
, total_bytes
);
5881 i
+= tx_ring
->count
;
5882 tx_ring
->next_to_clean
= i
;
5883 u64_stats_update_begin(&tx_ring
->tx_syncp
);
5884 tx_ring
->tx_stats
.bytes
+= total_bytes
;
5885 tx_ring
->tx_stats
.packets
+= total_packets
;
5886 u64_stats_update_end(&tx_ring
->tx_syncp
);
5887 q_vector
->tx
.total_bytes
+= total_bytes
;
5888 q_vector
->tx
.total_packets
+= total_packets
;
5890 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
)) {
5891 struct e1000_hw
*hw
= &adapter
->hw
;
5893 eop_desc
= tx_buffer
->next_to_watch
;
5895 /* Detect a transmit hang in hardware, this serializes the
5896 * check with the clearing of time_stamp and movement of i */
5897 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
);
5899 time_after(jiffies
, tx_buffer
->time_stamp
+
5900 (adapter
->tx_timeout_factor
* HZ
)) &&
5901 !(rd32(E1000_STATUS
) & E1000_STATUS_TXOFF
)) {
5903 /* detected Tx unit hang */
5904 dev_err(tx_ring
->dev
,
5905 "Detected Tx Unit Hang\n"
5909 " next_to_use <%x>\n"
5910 " next_to_clean <%x>\n"
5911 "buffer_info[next_to_clean]\n"
5912 " time_stamp <%lx>\n"
5913 " next_to_watch <%p>\n"
5915 " desc.status <%x>\n",
5916 tx_ring
->queue_index
,
5917 rd32(E1000_TDH(tx_ring
->reg_idx
)),
5918 readl(tx_ring
->tail
),
5919 tx_ring
->next_to_use
,
5920 tx_ring
->next_to_clean
,
5921 tx_buffer
->time_stamp
,
5924 eop_desc
->wb
.status
);
5925 netif_stop_subqueue(tx_ring
->netdev
,
5926 tx_ring
->queue_index
);
5928 /* we are about to reset, no point in enabling stuff */
5933 if (unlikely(total_packets
&&
5934 netif_carrier_ok(tx_ring
->netdev
) &&
5935 igb_desc_unused(tx_ring
) >= IGB_TX_QUEUE_WAKE
)) {
5936 /* Make sure that anybody stopping the queue after this
5937 * sees the new next_to_clean.
5940 if (__netif_subqueue_stopped(tx_ring
->netdev
,
5941 tx_ring
->queue_index
) &&
5942 !(test_bit(__IGB_DOWN
, &adapter
->state
))) {
5943 netif_wake_subqueue(tx_ring
->netdev
,
5944 tx_ring
->queue_index
);
5946 u64_stats_update_begin(&tx_ring
->tx_syncp
);
5947 tx_ring
->tx_stats
.restart_queue
++;
5948 u64_stats_update_end(&tx_ring
->tx_syncp
);
5955 static inline void igb_rx_checksum(struct igb_ring
*ring
,
5956 union e1000_adv_rx_desc
*rx_desc
,
5957 struct sk_buff
*skb
)
5959 skb_checksum_none_assert(skb
);
5961 /* Ignore Checksum bit is set */
5962 if (igb_test_staterr(rx_desc
, E1000_RXD_STAT_IXSM
))
5965 /* Rx checksum disabled via ethtool */
5966 if (!(ring
->netdev
->features
& NETIF_F_RXCSUM
))
5969 /* TCP/UDP checksum error bit is set */
5970 if (igb_test_staterr(rx_desc
,
5971 E1000_RXDEXT_STATERR_TCPE
|
5972 E1000_RXDEXT_STATERR_IPE
)) {
5974 * work around errata with sctp packets where the TCPE aka
5975 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5976 * packets, (aka let the stack check the crc32c)
5978 if (!((skb
->len
== 60) &&
5979 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM
, &ring
->flags
))) {
5980 u64_stats_update_begin(&ring
->rx_syncp
);
5981 ring
->rx_stats
.csum_err
++;
5982 u64_stats_update_end(&ring
->rx_syncp
);
5984 /* let the stack verify checksum errors */
5987 /* It must be a TCP or UDP packet with a valid checksum */
5988 if (igb_test_staterr(rx_desc
, E1000_RXD_STAT_TCPCS
|
5989 E1000_RXD_STAT_UDPCS
))
5990 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
5992 dev_dbg(ring
->dev
, "cksum success: bits %08X\n",
5993 le32_to_cpu(rx_desc
->wb
.upper
.status_error
));
5996 static inline void igb_rx_hash(struct igb_ring
*ring
,
5997 union e1000_adv_rx_desc
*rx_desc
,
5998 struct sk_buff
*skb
)
6000 if (ring
->netdev
->features
& NETIF_F_RXHASH
)
6001 skb
->rxhash
= le32_to_cpu(rx_desc
->wb
.lower
.hi_dword
.rss
);
6004 #ifdef CONFIG_IGB_PTP
6005 static void igb_rx_hwtstamp(struct igb_q_vector
*q_vector
,
6006 union e1000_adv_rx_desc
*rx_desc
,
6007 struct sk_buff
*skb
)
6009 struct igb_adapter
*adapter
= q_vector
->adapter
;
6010 struct e1000_hw
*hw
= &adapter
->hw
;
6013 if (!igb_test_staterr(rx_desc
, E1000_RXDADV_STAT_TSIP
|
6014 E1000_RXDADV_STAT_TS
))
6018 * If this bit is set, then the RX registers contain the time stamp. No
6019 * other packet will be time stamped until we read these registers, so
6020 * read the registers to make them available again. Because only one
6021 * packet can be time stamped at a time, we know that the register
6022 * values must belong to this one here and therefore we don't need to
6023 * compare any of the additional attributes stored for it.
6025 * If nothing went wrong, then it should have a shared tx_flags that we
6026 * can turn into a skb_shared_hwtstamps.
6028 if (igb_test_staterr(rx_desc
, E1000_RXDADV_STAT_TSIP
)) {
6029 u32
*stamp
= (u32
*)skb
->data
;
6030 regval
= le32_to_cpu(*(stamp
+ 2));
6031 regval
|= (u64
)le32_to_cpu(*(stamp
+ 3)) << 32;
6032 skb_pull(skb
, IGB_TS_HDR_LEN
);
6034 if(!(rd32(E1000_TSYNCRXCTL
) & E1000_TSYNCRXCTL_VALID
))
6037 regval
= rd32(E1000_RXSTMPL
);
6038 regval
|= (u64
)rd32(E1000_RXSTMPH
) << 32;
6041 igb_systim_to_hwtstamp(adapter
, skb_hwtstamps(skb
), regval
);
6043 #endif /* CONFIG_IGB_PTP */
6045 static void igb_rx_vlan(struct igb_ring
*ring
,
6046 union e1000_adv_rx_desc
*rx_desc
,
6047 struct sk_buff
*skb
)
6049 if (igb_test_staterr(rx_desc
, E1000_RXD_STAT_VP
)) {
6051 if (igb_test_staterr(rx_desc
, E1000_RXDEXT_STATERR_LB
) &&
6052 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP
, &ring
->flags
))
6053 vid
= be16_to_cpu(rx_desc
->wb
.upper
.vlan
);
6055 vid
= le16_to_cpu(rx_desc
->wb
.upper
.vlan
);
6057 __vlan_hwaccel_put_tag(skb
, vid
);
6061 static inline u16
igb_get_hlen(union e1000_adv_rx_desc
*rx_desc
)
6063 /* HW will not DMA in data larger than the given buffer, even if it
6064 * parses the (NFS, of course) header to be larger. In that case, it
6065 * fills the header buffer and spills the rest into the page.
6067 u16 hlen
= (le16_to_cpu(rx_desc
->wb
.lower
.lo_dword
.hdr_info
) &
6068 E1000_RXDADV_HDRBUFLEN_MASK
) >> E1000_RXDADV_HDRBUFLEN_SHIFT
;
6069 if (hlen
> IGB_RX_HDR_LEN
)
6070 hlen
= IGB_RX_HDR_LEN
;
6074 static bool igb_clean_rx_irq(struct igb_q_vector
*q_vector
, int budget
)
6076 struct igb_ring
*rx_ring
= q_vector
->rx
.ring
;
6077 union e1000_adv_rx_desc
*rx_desc
;
6078 const int current_node
= numa_node_id();
6079 unsigned int total_bytes
= 0, total_packets
= 0;
6080 u16 cleaned_count
= igb_desc_unused(rx_ring
);
6081 u16 i
= rx_ring
->next_to_clean
;
6083 rx_desc
= IGB_RX_DESC(rx_ring
, i
);
6085 while (igb_test_staterr(rx_desc
, E1000_RXD_STAT_DD
)) {
6086 struct igb_rx_buffer
*buffer_info
= &rx_ring
->rx_buffer_info
[i
];
6087 struct sk_buff
*skb
= buffer_info
->skb
;
6088 union e1000_adv_rx_desc
*next_rxd
;
6090 buffer_info
->skb
= NULL
;
6091 prefetch(skb
->data
);
6094 if (i
== rx_ring
->count
)
6097 next_rxd
= IGB_RX_DESC(rx_ring
, i
);
6101 * This memory barrier is needed to keep us from reading
6102 * any other fields out of the rx_desc until we know the
6103 * RXD_STAT_DD bit is set
6107 if (!skb_is_nonlinear(skb
)) {
6108 __skb_put(skb
, igb_get_hlen(rx_desc
));
6109 dma_unmap_single(rx_ring
->dev
, buffer_info
->dma
,
6112 buffer_info
->dma
= 0;
6115 if (rx_desc
->wb
.upper
.length
) {
6116 u16 length
= le16_to_cpu(rx_desc
->wb
.upper
.length
);
6118 skb_fill_page_desc(skb
, skb_shinfo(skb
)->nr_frags
,
6120 buffer_info
->page_offset
,
6124 skb
->data_len
+= length
;
6125 skb
->truesize
+= PAGE_SIZE
/ 2;
6127 if ((page_count(buffer_info
->page
) != 1) ||
6128 (page_to_nid(buffer_info
->page
) != current_node
))
6129 buffer_info
->page
= NULL
;
6131 get_page(buffer_info
->page
);
6133 dma_unmap_page(rx_ring
->dev
, buffer_info
->page_dma
,
6134 PAGE_SIZE
/ 2, DMA_FROM_DEVICE
);
6135 buffer_info
->page_dma
= 0;
6138 if (!igb_test_staterr(rx_desc
, E1000_RXD_STAT_EOP
)) {
6139 struct igb_rx_buffer
*next_buffer
;
6140 next_buffer
= &rx_ring
->rx_buffer_info
[i
];
6141 buffer_info
->skb
= next_buffer
->skb
;
6142 buffer_info
->dma
= next_buffer
->dma
;
6143 next_buffer
->skb
= skb
;
6144 next_buffer
->dma
= 0;
6148 if (unlikely((igb_test_staterr(rx_desc
,
6149 E1000_RXDEXT_ERR_FRAME_ERR_MASK
))
6150 && !(rx_ring
->netdev
->features
& NETIF_F_RXALL
))) {
6151 dev_kfree_skb_any(skb
);
6155 #ifdef CONFIG_IGB_PTP
6156 igb_rx_hwtstamp(q_vector
, rx_desc
, skb
);
6157 #endif /* CONFIG_IGB_PTP */
6158 igb_rx_hash(rx_ring
, rx_desc
, skb
);
6159 igb_rx_checksum(rx_ring
, rx_desc
, skb
);
6160 igb_rx_vlan(rx_ring
, rx_desc
, skb
);
6162 total_bytes
+= skb
->len
;
6165 skb
->protocol
= eth_type_trans(skb
, rx_ring
->netdev
);
6167 napi_gro_receive(&q_vector
->napi
, skb
);
6175 /* return some buffers to hardware, one at a time is too slow */
6176 if (cleaned_count
>= IGB_RX_BUFFER_WRITE
) {
6177 igb_alloc_rx_buffers(rx_ring
, cleaned_count
);
6181 /* use prefetched values */
6185 rx_ring
->next_to_clean
= i
;
6186 u64_stats_update_begin(&rx_ring
->rx_syncp
);
6187 rx_ring
->rx_stats
.packets
+= total_packets
;
6188 rx_ring
->rx_stats
.bytes
+= total_bytes
;
6189 u64_stats_update_end(&rx_ring
->rx_syncp
);
6190 q_vector
->rx
.total_packets
+= total_packets
;
6191 q_vector
->rx
.total_bytes
+= total_bytes
;
6194 igb_alloc_rx_buffers(rx_ring
, cleaned_count
);
6199 static bool igb_alloc_mapped_skb(struct igb_ring
*rx_ring
,
6200 struct igb_rx_buffer
*bi
)
6202 struct sk_buff
*skb
= bi
->skb
;
6203 dma_addr_t dma
= bi
->dma
;
6209 skb
= netdev_alloc_skb_ip_align(rx_ring
->netdev
,
6213 rx_ring
->rx_stats
.alloc_failed
++;
6217 /* initialize skb for ring */
6218 skb_record_rx_queue(skb
, rx_ring
->queue_index
);
6221 dma
= dma_map_single(rx_ring
->dev
, skb
->data
,
6222 IGB_RX_HDR_LEN
, DMA_FROM_DEVICE
);
6224 if (dma_mapping_error(rx_ring
->dev
, dma
)) {
6225 rx_ring
->rx_stats
.alloc_failed
++;
6233 static bool igb_alloc_mapped_page(struct igb_ring
*rx_ring
,
6234 struct igb_rx_buffer
*bi
)
6236 struct page
*page
= bi
->page
;
6237 dma_addr_t page_dma
= bi
->page_dma
;
6238 unsigned int page_offset
= bi
->page_offset
^ (PAGE_SIZE
/ 2);
6244 page
= __skb_alloc_page(GFP_ATOMIC
, bi
->skb
);
6246 if (unlikely(!page
)) {
6247 rx_ring
->rx_stats
.alloc_failed
++;
6252 page_dma
= dma_map_page(rx_ring
->dev
, page
,
6253 page_offset
, PAGE_SIZE
/ 2,
6256 if (dma_mapping_error(rx_ring
->dev
, page_dma
)) {
6257 rx_ring
->rx_stats
.alloc_failed
++;
6261 bi
->page_dma
= page_dma
;
6262 bi
->page_offset
= page_offset
;
6267 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
6268 * @adapter: address of board private structure
6270 void igb_alloc_rx_buffers(struct igb_ring
*rx_ring
, u16 cleaned_count
)
6272 union e1000_adv_rx_desc
*rx_desc
;
6273 struct igb_rx_buffer
*bi
;
6274 u16 i
= rx_ring
->next_to_use
;
6276 rx_desc
= IGB_RX_DESC(rx_ring
, i
);
6277 bi
= &rx_ring
->rx_buffer_info
[i
];
6278 i
-= rx_ring
->count
;
6280 while (cleaned_count
--) {
6281 if (!igb_alloc_mapped_skb(rx_ring
, bi
))
6284 /* Refresh the desc even if buffer_addrs didn't change
6285 * because each write-back erases this info. */
6286 rx_desc
->read
.hdr_addr
= cpu_to_le64(bi
->dma
);
6288 if (!igb_alloc_mapped_page(rx_ring
, bi
))
6291 rx_desc
->read
.pkt_addr
= cpu_to_le64(bi
->page_dma
);
6297 rx_desc
= IGB_RX_DESC(rx_ring
, 0);
6298 bi
= rx_ring
->rx_buffer_info
;
6299 i
-= rx_ring
->count
;
6302 /* clear the hdr_addr for the next_to_use descriptor */
6303 rx_desc
->read
.hdr_addr
= 0;
6306 i
+= rx_ring
->count
;
6308 if (rx_ring
->next_to_use
!= i
) {
6309 rx_ring
->next_to_use
= i
;
6311 /* Force memory writes to complete before letting h/w
6312 * know there are new descriptors to fetch. (Only
6313 * applicable for weak-ordered memory model archs,
6314 * such as IA-64). */
6316 writel(i
, rx_ring
->tail
);
6326 static int igb_mii_ioctl(struct net_device
*netdev
, struct ifreq
*ifr
, int cmd
)
6328 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6329 struct mii_ioctl_data
*data
= if_mii(ifr
);
6331 if (adapter
->hw
.phy
.media_type
!= e1000_media_type_copper
)
6336 data
->phy_id
= adapter
->hw
.phy
.addr
;
6339 if (igb_read_phy_reg(&adapter
->hw
, data
->reg_num
& 0x1F,
6350 #ifdef CONFIG_IGB_PTP
6352 * igb_hwtstamp_ioctl - control hardware time stamping
6357 * Outgoing time stamping can be enabled and disabled. Play nice and
6358 * disable it when requested, although it shouldn't case any overhead
6359 * when no packet needs it. At most one packet in the queue may be
6360 * marked for time stamping, otherwise it would be impossible to tell
6361 * for sure to which packet the hardware time stamp belongs.
6363 * Incoming time stamping has to be configured via the hardware
6364 * filters. Not all combinations are supported, in particular event
6365 * type has to be specified. Matching the kind of event packet is
6366 * not supported, with the exception of "all V2 events regardless of
6370 static int igb_hwtstamp_ioctl(struct net_device
*netdev
,
6371 struct ifreq
*ifr
, int cmd
)
6373 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6374 struct e1000_hw
*hw
= &adapter
->hw
;
6375 struct hwtstamp_config config
;
6376 u32 tsync_tx_ctl
= E1000_TSYNCTXCTL_ENABLED
;
6377 u32 tsync_rx_ctl
= E1000_TSYNCRXCTL_ENABLED
;
6378 u32 tsync_rx_cfg
= 0;
6383 if (copy_from_user(&config
, ifr
->ifr_data
, sizeof(config
)))
6386 /* reserved for future extensions */
6390 switch (config
.tx_type
) {
6391 case HWTSTAMP_TX_OFF
:
6393 case HWTSTAMP_TX_ON
:
6399 switch (config
.rx_filter
) {
6400 case HWTSTAMP_FILTER_NONE
:
6403 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT
:
6404 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT
:
6405 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT
:
6406 case HWTSTAMP_FILTER_ALL
:
6408 * register TSYNCRXCFG must be set, therefore it is not
6409 * possible to time stamp both Sync and Delay_Req messages
6410 * => fall back to time stamping all packets
6412 tsync_rx_ctl
|= E1000_TSYNCRXCTL_TYPE_ALL
;
6413 config
.rx_filter
= HWTSTAMP_FILTER_ALL
;
6415 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC
:
6416 tsync_rx_ctl
|= E1000_TSYNCRXCTL_TYPE_L4_V1
;
6417 tsync_rx_cfg
= E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE
;
6420 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ
:
6421 tsync_rx_ctl
|= E1000_TSYNCRXCTL_TYPE_L4_V1
;
6422 tsync_rx_cfg
= E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE
;
6425 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC
:
6426 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC
:
6427 tsync_rx_ctl
|= E1000_TSYNCRXCTL_TYPE_L2_L4_V2
;
6428 tsync_rx_cfg
= E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE
;
6431 config
.rx_filter
= HWTSTAMP_FILTER_SOME
;
6433 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ
:
6434 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ
:
6435 tsync_rx_ctl
|= E1000_TSYNCRXCTL_TYPE_L2_L4_V2
;
6436 tsync_rx_cfg
= E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE
;
6439 config
.rx_filter
= HWTSTAMP_FILTER_SOME
;
6441 case HWTSTAMP_FILTER_PTP_V2_EVENT
:
6442 case HWTSTAMP_FILTER_PTP_V2_SYNC
:
6443 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ
:
6444 tsync_rx_ctl
|= E1000_TSYNCRXCTL_TYPE_EVENT_V2
;
6445 config
.rx_filter
= HWTSTAMP_FILTER_PTP_V2_EVENT
;
6453 if (hw
->mac
.type
== e1000_82575
) {
6454 if (tsync_rx_ctl
| tsync_tx_ctl
)
6460 * Per-packet timestamping only works if all packets are
6461 * timestamped, so enable timestamping in all packets as
6462 * long as one rx filter was configured.
6464 if ((hw
->mac
.type
>= e1000_82580
) && tsync_rx_ctl
) {
6465 tsync_rx_ctl
= E1000_TSYNCRXCTL_ENABLED
;
6466 tsync_rx_ctl
|= E1000_TSYNCRXCTL_TYPE_ALL
;
6469 /* enable/disable TX */
6470 regval
= rd32(E1000_TSYNCTXCTL
);
6471 regval
&= ~E1000_TSYNCTXCTL_ENABLED
;
6472 regval
|= tsync_tx_ctl
;
6473 wr32(E1000_TSYNCTXCTL
, regval
);
6475 /* enable/disable RX */
6476 regval
= rd32(E1000_TSYNCRXCTL
);
6477 regval
&= ~(E1000_TSYNCRXCTL_ENABLED
| E1000_TSYNCRXCTL_TYPE_MASK
);
6478 regval
|= tsync_rx_ctl
;
6479 wr32(E1000_TSYNCRXCTL
, regval
);
6481 /* define which PTP packets are time stamped */
6482 wr32(E1000_TSYNCRXCFG
, tsync_rx_cfg
);
6484 /* define ethertype filter for timestamped packets */
6487 (E1000_ETQF_FILTER_ENABLE
| /* enable filter */
6488 E1000_ETQF_1588
| /* enable timestamping */
6489 ETH_P_1588
)); /* 1588 eth protocol type */
6491 wr32(E1000_ETQF(3), 0);
6493 #define PTP_PORT 319
6494 /* L4 Queue Filter[3]: filter by destination port and protocol */
6496 u32 ftqf
= (IPPROTO_UDP
/* UDP */
6497 | E1000_FTQF_VF_BP
/* VF not compared */
6498 | E1000_FTQF_1588_TIME_STAMP
/* Enable Timestamping */
6499 | E1000_FTQF_MASK
); /* mask all inputs */
6500 ftqf
&= ~E1000_FTQF_MASK_PROTO_BP
; /* enable protocol check */
6502 wr32(E1000_IMIR(3), htons(PTP_PORT
));
6503 wr32(E1000_IMIREXT(3),
6504 (E1000_IMIREXT_SIZE_BP
| E1000_IMIREXT_CTRL_BP
));
6505 if (hw
->mac
.type
== e1000_82576
) {
6506 /* enable source port check */
6507 wr32(E1000_SPQF(3), htons(PTP_PORT
));
6508 ftqf
&= ~E1000_FTQF_MASK_SOURCE_PORT_BP
;
6510 wr32(E1000_FTQF(3), ftqf
);
6512 wr32(E1000_FTQF(3), E1000_FTQF_MASK
);
6516 adapter
->hwtstamp_config
= config
;
6518 /* clear TX/RX time stamp registers, just to be sure */
6519 regval
= rd32(E1000_TXSTMPH
);
6520 regval
= rd32(E1000_RXSTMPH
);
6522 return copy_to_user(ifr
->ifr_data
, &config
, sizeof(config
)) ?
6525 #endif /* CONFIG_IGB_PTP */
6533 static int igb_ioctl(struct net_device
*netdev
, struct ifreq
*ifr
, int cmd
)
6539 return igb_mii_ioctl(netdev
, ifr
, cmd
);
6540 #ifdef CONFIG_IGB_PTP
6542 return igb_hwtstamp_ioctl(netdev
, ifr
, cmd
);
6543 #endif /* CONFIG_IGB_PTP */
6549 s32
igb_read_pcie_cap_reg(struct e1000_hw
*hw
, u32 reg
, u16
*value
)
6551 struct igb_adapter
*adapter
= hw
->back
;
6554 cap_offset
= adapter
->pdev
->pcie_cap
;
6556 return -E1000_ERR_CONFIG
;
6558 pci_read_config_word(adapter
->pdev
, cap_offset
+ reg
, value
);
6563 s32
igb_write_pcie_cap_reg(struct e1000_hw
*hw
, u32 reg
, u16
*value
)
6565 struct igb_adapter
*adapter
= hw
->back
;
6568 cap_offset
= adapter
->pdev
->pcie_cap
;
6570 return -E1000_ERR_CONFIG
;
6572 pci_write_config_word(adapter
->pdev
, cap_offset
+ reg
, *value
);
6577 static void igb_vlan_mode(struct net_device
*netdev
, netdev_features_t features
)
6579 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6580 struct e1000_hw
*hw
= &adapter
->hw
;
6582 bool enable
= !!(features
& NETIF_F_HW_VLAN_RX
);
6585 /* enable VLAN tag insert/strip */
6586 ctrl
= rd32(E1000_CTRL
);
6587 ctrl
|= E1000_CTRL_VME
;
6588 wr32(E1000_CTRL
, ctrl
);
6590 /* Disable CFI check */
6591 rctl
= rd32(E1000_RCTL
);
6592 rctl
&= ~E1000_RCTL_CFIEN
;
6593 wr32(E1000_RCTL
, rctl
);
6595 /* disable VLAN tag insert/strip */
6596 ctrl
= rd32(E1000_CTRL
);
6597 ctrl
&= ~E1000_CTRL_VME
;
6598 wr32(E1000_CTRL
, ctrl
);
6601 igb_rlpml_set(adapter
);
6604 static int igb_vlan_rx_add_vid(struct net_device
*netdev
, u16 vid
)
6606 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6607 struct e1000_hw
*hw
= &adapter
->hw
;
6608 int pf_id
= adapter
->vfs_allocated_count
;
6610 /* attempt to add filter to vlvf array */
6611 igb_vlvf_set(adapter
, vid
, true, pf_id
);
6613 /* add the filter since PF can receive vlans w/o entry in vlvf */
6614 igb_vfta_set(hw
, vid
, true);
6616 set_bit(vid
, adapter
->active_vlans
);
6621 static int igb_vlan_rx_kill_vid(struct net_device
*netdev
, u16 vid
)
6623 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6624 struct e1000_hw
*hw
= &adapter
->hw
;
6625 int pf_id
= adapter
->vfs_allocated_count
;
6628 /* remove vlan from VLVF table array */
6629 err
= igb_vlvf_set(adapter
, vid
, false, pf_id
);
6631 /* if vid was not present in VLVF just remove it from table */
6633 igb_vfta_set(hw
, vid
, false);
6635 clear_bit(vid
, adapter
->active_vlans
);
6640 static void igb_restore_vlan(struct igb_adapter
*adapter
)
6644 igb_vlan_mode(adapter
->netdev
, adapter
->netdev
->features
);
6646 for_each_set_bit(vid
, adapter
->active_vlans
, VLAN_N_VID
)
6647 igb_vlan_rx_add_vid(adapter
->netdev
, vid
);
6650 int igb_set_spd_dplx(struct igb_adapter
*adapter
, u32 spd
, u8 dplx
)
6652 struct pci_dev
*pdev
= adapter
->pdev
;
6653 struct e1000_mac_info
*mac
= &adapter
->hw
.mac
;
6657 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6658 * for the switch() below to work */
6659 if ((spd
& 1) || (dplx
& ~1))
6662 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6663 if ((adapter
->hw
.phy
.media_type
== e1000_media_type_internal_serdes
) &&
6664 spd
!= SPEED_1000
&&
6665 dplx
!= DUPLEX_FULL
)
6668 switch (spd
+ dplx
) {
6669 case SPEED_10
+ DUPLEX_HALF
:
6670 mac
->forced_speed_duplex
= ADVERTISE_10_HALF
;
6672 case SPEED_10
+ DUPLEX_FULL
:
6673 mac
->forced_speed_duplex
= ADVERTISE_10_FULL
;
6675 case SPEED_100
+ DUPLEX_HALF
:
6676 mac
->forced_speed_duplex
= ADVERTISE_100_HALF
;
6678 case SPEED_100
+ DUPLEX_FULL
:
6679 mac
->forced_speed_duplex
= ADVERTISE_100_FULL
;
6681 case SPEED_1000
+ DUPLEX_FULL
:
6683 adapter
->hw
.phy
.autoneg_advertised
= ADVERTISE_1000_FULL
;
6685 case SPEED_1000
+ DUPLEX_HALF
: /* not supported */
6690 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
6691 adapter
->hw
.phy
.mdix
= AUTO_ALL_MODES
;
6696 dev_err(&pdev
->dev
, "Unsupported Speed/Duplex configuration\n");
6700 static int __igb_shutdown(struct pci_dev
*pdev
, bool *enable_wake
,
6703 struct net_device
*netdev
= pci_get_drvdata(pdev
);
6704 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6705 struct e1000_hw
*hw
= &adapter
->hw
;
6706 u32 ctrl
, rctl
, status
;
6707 u32 wufc
= runtime
? E1000_WUFC_LNKC
: adapter
->wol
;
6712 netif_device_detach(netdev
);
6714 if (netif_running(netdev
))
6715 __igb_close(netdev
, true);
6717 igb_clear_interrupt_scheme(adapter
);
6720 retval
= pci_save_state(pdev
);
6725 status
= rd32(E1000_STATUS
);
6726 if (status
& E1000_STATUS_LU
)
6727 wufc
&= ~E1000_WUFC_LNKC
;
6730 igb_setup_rctl(adapter
);
6731 igb_set_rx_mode(netdev
);
6733 /* turn on all-multi mode if wake on multicast is enabled */
6734 if (wufc
& E1000_WUFC_MC
) {
6735 rctl
= rd32(E1000_RCTL
);
6736 rctl
|= E1000_RCTL_MPE
;
6737 wr32(E1000_RCTL
, rctl
);
6740 ctrl
= rd32(E1000_CTRL
);
6741 /* advertise wake from D3Cold */
6742 #define E1000_CTRL_ADVD3WUC 0x00100000
6743 /* phy power management enable */
6744 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6745 ctrl
|= E1000_CTRL_ADVD3WUC
;
6746 wr32(E1000_CTRL
, ctrl
);
6748 /* Allow time for pending master requests to run */
6749 igb_disable_pcie_master(hw
);
6751 wr32(E1000_WUC
, E1000_WUC_PME_EN
);
6752 wr32(E1000_WUFC
, wufc
);
6755 wr32(E1000_WUFC
, 0);
6758 *enable_wake
= wufc
|| adapter
->en_mng_pt
;
6760 igb_power_down_link(adapter
);
6762 igb_power_up_link(adapter
);
6764 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6765 * would have already happened in close and is redundant. */
6766 igb_release_hw_control(adapter
);
6768 pci_disable_device(pdev
);
6774 #ifdef CONFIG_PM_SLEEP
6775 static int igb_suspend(struct device
*dev
)
6779 struct pci_dev
*pdev
= to_pci_dev(dev
);
6781 retval
= __igb_shutdown(pdev
, &wake
, 0);
6786 pci_prepare_to_sleep(pdev
);
6788 pci_wake_from_d3(pdev
, false);
6789 pci_set_power_state(pdev
, PCI_D3hot
);
6794 #endif /* CONFIG_PM_SLEEP */
6796 static int igb_resume(struct device
*dev
)
6798 struct pci_dev
*pdev
= to_pci_dev(dev
);
6799 struct net_device
*netdev
= pci_get_drvdata(pdev
);
6800 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6801 struct e1000_hw
*hw
= &adapter
->hw
;
6804 pci_set_power_state(pdev
, PCI_D0
);
6805 pci_restore_state(pdev
);
6806 pci_save_state(pdev
);
6808 err
= pci_enable_device_mem(pdev
);
6811 "igb: Cannot enable PCI device from suspend\n");
6814 pci_set_master(pdev
);
6816 pci_enable_wake(pdev
, PCI_D3hot
, 0);
6817 pci_enable_wake(pdev
, PCI_D3cold
, 0);
6819 if (igb_init_interrupt_scheme(adapter
)) {
6820 dev_err(&pdev
->dev
, "Unable to allocate memory for queues\n");
6826 /* let the f/w know that the h/w is now under the control of the
6828 igb_get_hw_control(adapter
);
6830 wr32(E1000_WUS
, ~0);
6832 if (netdev
->flags
& IFF_UP
) {
6833 err
= __igb_open(netdev
, true);
6838 netif_device_attach(netdev
);
6842 #ifdef CONFIG_PM_RUNTIME
6843 static int igb_runtime_idle(struct device
*dev
)
6845 struct pci_dev
*pdev
= to_pci_dev(dev
);
6846 struct net_device
*netdev
= pci_get_drvdata(pdev
);
6847 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6849 if (!igb_has_link(adapter
))
6850 pm_schedule_suspend(dev
, MSEC_PER_SEC
* 5);
6855 static int igb_runtime_suspend(struct device
*dev
)
6857 struct pci_dev
*pdev
= to_pci_dev(dev
);
6861 retval
= __igb_shutdown(pdev
, &wake
, 1);
6866 pci_prepare_to_sleep(pdev
);
6868 pci_wake_from_d3(pdev
, false);
6869 pci_set_power_state(pdev
, PCI_D3hot
);
6875 static int igb_runtime_resume(struct device
*dev
)
6877 return igb_resume(dev
);
6879 #endif /* CONFIG_PM_RUNTIME */
6882 static void igb_shutdown(struct pci_dev
*pdev
)
6886 __igb_shutdown(pdev
, &wake
, 0);
6888 if (system_state
== SYSTEM_POWER_OFF
) {
6889 pci_wake_from_d3(pdev
, wake
);
6890 pci_set_power_state(pdev
, PCI_D3hot
);
6894 #ifdef CONFIG_NET_POLL_CONTROLLER
6896 * Polling 'interrupt' - used by things like netconsole to send skbs
6897 * without having to re-enable interrupts. It's not called while
6898 * the interrupt routine is executing.
6900 static void igb_netpoll(struct net_device
*netdev
)
6902 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6903 struct e1000_hw
*hw
= &adapter
->hw
;
6904 struct igb_q_vector
*q_vector
;
6907 for (i
= 0; i
< adapter
->num_q_vectors
; i
++) {
6908 q_vector
= adapter
->q_vector
[i
];
6909 if (adapter
->msix_entries
)
6910 wr32(E1000_EIMC
, q_vector
->eims_value
);
6912 igb_irq_disable(adapter
);
6913 napi_schedule(&q_vector
->napi
);
6916 #endif /* CONFIG_NET_POLL_CONTROLLER */
6919 * igb_io_error_detected - called when PCI error is detected
6920 * @pdev: Pointer to PCI device
6921 * @state: The current pci connection state
6923 * This function is called after a PCI bus error affecting
6924 * this device has been detected.
6926 static pci_ers_result_t
igb_io_error_detected(struct pci_dev
*pdev
,
6927 pci_channel_state_t state
)
6929 struct net_device
*netdev
= pci_get_drvdata(pdev
);
6930 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6932 netif_device_detach(netdev
);
6934 if (state
== pci_channel_io_perm_failure
)
6935 return PCI_ERS_RESULT_DISCONNECT
;
6937 if (netif_running(netdev
))
6939 pci_disable_device(pdev
);
6941 /* Request a slot slot reset. */
6942 return PCI_ERS_RESULT_NEED_RESET
;
6946 * igb_io_slot_reset - called after the pci bus has been reset.
6947 * @pdev: Pointer to PCI device
6949 * Restart the card from scratch, as if from a cold-boot. Implementation
6950 * resembles the first-half of the igb_resume routine.
6952 static pci_ers_result_t
igb_io_slot_reset(struct pci_dev
*pdev
)
6954 struct net_device
*netdev
= pci_get_drvdata(pdev
);
6955 struct igb_adapter
*adapter
= netdev_priv(netdev
);
6956 struct e1000_hw
*hw
= &adapter
->hw
;
6957 pci_ers_result_t result
;
6960 if (pci_enable_device_mem(pdev
)) {
6962 "Cannot re-enable PCI device after reset.\n");
6963 result
= PCI_ERS_RESULT_DISCONNECT
;
6965 pci_set_master(pdev
);
6966 pci_restore_state(pdev
);
6967 pci_save_state(pdev
);
6969 pci_enable_wake(pdev
, PCI_D3hot
, 0);
6970 pci_enable_wake(pdev
, PCI_D3cold
, 0);
6973 wr32(E1000_WUS
, ~0);
6974 result
= PCI_ERS_RESULT_RECOVERED
;
6977 err
= pci_cleanup_aer_uncorrect_error_status(pdev
);
6979 dev_err(&pdev
->dev
, "pci_cleanup_aer_uncorrect_error_status "
6980 "failed 0x%0x\n", err
);
6981 /* non-fatal, continue */
6988 * igb_io_resume - called when traffic can start flowing again.
6989 * @pdev: Pointer to PCI device
6991 * This callback is called when the error recovery driver tells us that
6992 * its OK to resume normal operation. Implementation resembles the
6993 * second-half of the igb_resume routine.
6995 static void igb_io_resume(struct pci_dev
*pdev
)
6997 struct net_device
*netdev
= pci_get_drvdata(pdev
);
6998 struct igb_adapter
*adapter
= netdev_priv(netdev
);
7000 if (netif_running(netdev
)) {
7001 if (igb_up(adapter
)) {
7002 dev_err(&pdev
->dev
, "igb_up failed after reset\n");
7007 netif_device_attach(netdev
);
7009 /* let the f/w know that the h/w is now under the control of the
7011 igb_get_hw_control(adapter
);
7014 static void igb_rar_set_qsel(struct igb_adapter
*adapter
, u8
*addr
, u32 index
,
7017 u32 rar_low
, rar_high
;
7018 struct e1000_hw
*hw
= &adapter
->hw
;
7020 /* HW expects these in little endian so we reverse the byte order
7021 * from network order (big endian) to little endian
7023 rar_low
= ((u32
) addr
[0] | ((u32
) addr
[1] << 8) |
7024 ((u32
) addr
[2] << 16) | ((u32
) addr
[3] << 24));
7025 rar_high
= ((u32
) addr
[4] | ((u32
) addr
[5] << 8));
7027 /* Indicate to hardware the Address is Valid. */
7028 rar_high
|= E1000_RAH_AV
;
7030 if (hw
->mac
.type
== e1000_82575
)
7031 rar_high
|= E1000_RAH_POOL_1
* qsel
;
7033 rar_high
|= E1000_RAH_POOL_1
<< qsel
;
7035 wr32(E1000_RAL(index
), rar_low
);
7037 wr32(E1000_RAH(index
), rar_high
);
7041 static int igb_set_vf_mac(struct igb_adapter
*adapter
,
7042 int vf
, unsigned char *mac_addr
)
7044 struct e1000_hw
*hw
= &adapter
->hw
;
7045 /* VF MAC addresses start at end of receive addresses and moves
7046 * torwards the first, as a result a collision should not be possible */
7047 int rar_entry
= hw
->mac
.rar_entry_count
- (vf
+ 1);
7049 memcpy(adapter
->vf_data
[vf
].vf_mac_addresses
, mac_addr
, ETH_ALEN
);
7051 igb_rar_set_qsel(adapter
, mac_addr
, rar_entry
, vf
);
7056 static int igb_ndo_set_vf_mac(struct net_device
*netdev
, int vf
, u8
*mac
)
7058 struct igb_adapter
*adapter
= netdev_priv(netdev
);
7059 if (!is_valid_ether_addr(mac
) || (vf
>= adapter
->vfs_allocated_count
))
7061 adapter
->vf_data
[vf
].flags
|= IGB_VF_FLAG_PF_SET_MAC
;
7062 dev_info(&adapter
->pdev
->dev
, "setting MAC %pM on VF %d\n", mac
, vf
);
7063 dev_info(&adapter
->pdev
->dev
, "Reload the VF driver to make this"
7064 " change effective.");
7065 if (test_bit(__IGB_DOWN
, &adapter
->state
)) {
7066 dev_warn(&adapter
->pdev
->dev
, "The VF MAC address has been set,"
7067 " but the PF device is not up.\n");
7068 dev_warn(&adapter
->pdev
->dev
, "Bring the PF device up before"
7069 " attempting to use the VF device.\n");
7071 return igb_set_vf_mac(adapter
, vf
, mac
);
7074 static int igb_link_mbps(int internal_link_speed
)
7076 switch (internal_link_speed
) {
7086 static void igb_set_vf_rate_limit(struct e1000_hw
*hw
, int vf
, int tx_rate
,
7093 /* Calculate the rate factor values to set */
7094 rf_int
= link_speed
/ tx_rate
;
7095 rf_dec
= (link_speed
- (rf_int
* tx_rate
));
7096 rf_dec
= (rf_dec
* (1<<E1000_RTTBCNRC_RF_INT_SHIFT
)) / tx_rate
;
7098 bcnrc_val
= E1000_RTTBCNRC_RS_ENA
;
7099 bcnrc_val
|= ((rf_int
<<E1000_RTTBCNRC_RF_INT_SHIFT
) &
7100 E1000_RTTBCNRC_RF_INT_MASK
);
7101 bcnrc_val
|= (rf_dec
& E1000_RTTBCNRC_RF_DEC_MASK
);
7106 wr32(E1000_RTTDQSEL
, vf
); /* vf X uses queue X */
7108 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
7109 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
7111 wr32(E1000_RTTBCNRM
, 0x14);
7112 wr32(E1000_RTTBCNRC
, bcnrc_val
);
7115 static void igb_check_vf_rate_limit(struct igb_adapter
*adapter
)
7117 int actual_link_speed
, i
;
7118 bool reset_rate
= false;
7120 /* VF TX rate limit was not set or not supported */
7121 if ((adapter
->vf_rate_link_speed
== 0) ||
7122 (adapter
->hw
.mac
.type
!= e1000_82576
))
7125 actual_link_speed
= igb_link_mbps(adapter
->link_speed
);
7126 if (actual_link_speed
!= adapter
->vf_rate_link_speed
) {
7128 adapter
->vf_rate_link_speed
= 0;
7129 dev_info(&adapter
->pdev
->dev
,
7130 "Link speed has been changed. VF Transmit "
7131 "rate is disabled\n");
7134 for (i
= 0; i
< adapter
->vfs_allocated_count
; i
++) {
7136 adapter
->vf_data
[i
].tx_rate
= 0;
7138 igb_set_vf_rate_limit(&adapter
->hw
, i
,
7139 adapter
->vf_data
[i
].tx_rate
,
7144 static int igb_ndo_set_vf_bw(struct net_device
*netdev
, int vf
, int tx_rate
)
7146 struct igb_adapter
*adapter
= netdev_priv(netdev
);
7147 struct e1000_hw
*hw
= &adapter
->hw
;
7148 int actual_link_speed
;
7150 if (hw
->mac
.type
!= e1000_82576
)
7153 actual_link_speed
= igb_link_mbps(adapter
->link_speed
);
7154 if ((vf
>= adapter
->vfs_allocated_count
) ||
7155 (!(rd32(E1000_STATUS
) & E1000_STATUS_LU
)) ||
7156 (tx_rate
< 0) || (tx_rate
> actual_link_speed
))
7159 adapter
->vf_rate_link_speed
= actual_link_speed
;
7160 adapter
->vf_data
[vf
].tx_rate
= (u16
)tx_rate
;
7161 igb_set_vf_rate_limit(hw
, vf
, tx_rate
, actual_link_speed
);
7166 static int igb_ndo_get_vf_config(struct net_device
*netdev
,
7167 int vf
, struct ifla_vf_info
*ivi
)
7169 struct igb_adapter
*adapter
= netdev_priv(netdev
);
7170 if (vf
>= adapter
->vfs_allocated_count
)
7173 memcpy(&ivi
->mac
, adapter
->vf_data
[vf
].vf_mac_addresses
, ETH_ALEN
);
7174 ivi
->tx_rate
= adapter
->vf_data
[vf
].tx_rate
;
7175 ivi
->vlan
= adapter
->vf_data
[vf
].pf_vlan
;
7176 ivi
->qos
= adapter
->vf_data
[vf
].pf_qos
;
7180 static void igb_vmm_control(struct igb_adapter
*adapter
)
7182 struct e1000_hw
*hw
= &adapter
->hw
;
7185 switch (hw
->mac
.type
) {
7190 /* replication is not supported for 82575 */
7193 /* notify HW that the MAC is adding vlan tags */
7194 reg
= rd32(E1000_DTXCTL
);
7195 reg
|= E1000_DTXCTL_VLAN_ADDED
;
7196 wr32(E1000_DTXCTL
, reg
);
7198 /* enable replication vlan tag stripping */
7199 reg
= rd32(E1000_RPLOLR
);
7200 reg
|= E1000_RPLOLR_STRVLAN
;
7201 wr32(E1000_RPLOLR
, reg
);
7203 /* none of the above registers are supported by i350 */
7207 if (adapter
->vfs_allocated_count
) {
7208 igb_vmdq_set_loopback_pf(hw
, true);
7209 igb_vmdq_set_replication_pf(hw
, true);
7210 igb_vmdq_set_anti_spoofing_pf(hw
, true,
7211 adapter
->vfs_allocated_count
);
7213 igb_vmdq_set_loopback_pf(hw
, false);
7214 igb_vmdq_set_replication_pf(hw
, false);
7218 static void igb_init_dmac(struct igb_adapter
*adapter
, u32 pba
)
7220 struct e1000_hw
*hw
= &adapter
->hw
;
7224 if (hw
->mac
.type
> e1000_82580
) {
7225 if (adapter
->flags
& IGB_FLAG_DMAC
) {
7228 /* force threshold to 0. */
7229 wr32(E1000_DMCTXTH
, 0);
7232 * DMA Coalescing high water mark needs to be greater
7233 * than the Rx threshold. Set hwm to PBA - max frame
7234 * size in 16B units, capping it at PBA - 6KB.
7236 hwm
= 64 * pba
- adapter
->max_frame_size
/ 16;
7237 if (hwm
< 64 * (pba
- 6))
7238 hwm
= 64 * (pba
- 6);
7239 reg
= rd32(E1000_FCRTC
);
7240 reg
&= ~E1000_FCRTC_RTH_COAL_MASK
;
7241 reg
|= ((hwm
<< E1000_FCRTC_RTH_COAL_SHIFT
)
7242 & E1000_FCRTC_RTH_COAL_MASK
);
7243 wr32(E1000_FCRTC
, reg
);
7246 * Set the DMA Coalescing Rx threshold to PBA - 2 * max
7247 * frame size, capping it at PBA - 10KB.
7249 dmac_thr
= pba
- adapter
->max_frame_size
/ 512;
7250 if (dmac_thr
< pba
- 10)
7251 dmac_thr
= pba
- 10;
7252 reg
= rd32(E1000_DMACR
);
7253 reg
&= ~E1000_DMACR_DMACTHR_MASK
;
7254 reg
|= ((dmac_thr
<< E1000_DMACR_DMACTHR_SHIFT
)
7255 & E1000_DMACR_DMACTHR_MASK
);
7257 /* transition to L0x or L1 if available..*/
7258 reg
|= (E1000_DMACR_DMAC_EN
| E1000_DMACR_DMAC_LX_MASK
);
7260 /* watchdog timer= +-1000 usec in 32usec intervals */
7263 /* Disable BMC-to-OS Watchdog Enable */
7264 reg
&= ~E1000_DMACR_DC_BMC2OSW_EN
;
7265 wr32(E1000_DMACR
, reg
);
7268 * no lower threshold to disable
7269 * coalescing(smart fifb)-UTRESH=0
7271 wr32(E1000_DMCRTRH
, 0);
7273 reg
= (IGB_DMCTLX_DCFLUSH_DIS
| 0x4);
7275 wr32(E1000_DMCTLX
, reg
);
7278 * free space in tx packet buffer to wake from
7281 wr32(E1000_DMCTXTH
, (IGB_MIN_TXPBSIZE
-
7282 (IGB_TX_BUF_4096
+ adapter
->max_frame_size
)) >> 6);
7285 * make low power state decision controlled
7288 reg
= rd32(E1000_PCIEMISC
);
7289 reg
&= ~E1000_PCIEMISC_LX_DECISION
;
7290 wr32(E1000_PCIEMISC
, reg
);
7291 } /* endif adapter->dmac is not disabled */
7292 } else if (hw
->mac
.type
== e1000_82580
) {
7293 u32 reg
= rd32(E1000_PCIEMISC
);
7294 wr32(E1000_PCIEMISC
, reg
& ~E1000_PCIEMISC_LX_DECISION
);
7295 wr32(E1000_DMACR
, 0);