projects / deliverable / linux.git / blob
? search:


6e39f0ca9fc4e738604a74b30fdd3fd85cefc51e
[deliverable/linux.git] / drivers / net / ethernet / intel / igb / igb_main.c
1 /*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2012 Intel Corporation.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26 *******************************************************************************/
27
28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
29
30 #include <linux/module.h>
31 #include <linux/types.h>
32 #include <linux/init.h>
33 #include <linux/bitops.h>
34 #include <linux/vmalloc.h>
35 #include <linux/pagemap.h>
36 #include <linux/netdevice.h>
37 #include <linux/ipv6.h>
38 #include <linux/slab.h>
39 #include <net/checksum.h>
40 #include <net/ip6_checksum.h>
41 #include <linux/net_tstamp.h>
42 #include <linux/mii.h>
43 #include <linux/ethtool.h>
44 #include <linux/if.h>
45 #include <linux/if_vlan.h>
46 #include <linux/pci.h>
47 #include <linux/pci-aspm.h>
48 #include <linux/delay.h>
49 #include <linux/interrupt.h>
50 #include <linux/ip.h>
51 #include <linux/tcp.h>
52 #include <linux/sctp.h>
53 #include <linux/if_ether.h>
54 #include <linux/aer.h>
55 #include <linux/prefetch.h>
56 #include <linux/pm_runtime.h>
57 #ifdef CONFIG_IGB_DCA
58 #include <linux/dca.h>
59 #endif
60 #include "igb.h"
61
62 #define MAJ 4
63 #define MIN 0
64 #define BUILD 1
65 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
66 __stringify(BUILD) "-k"
67 char igb_driver_name[] = "igb";
68 char igb_driver_version[] = DRV_VERSION;
69 static const char igb_driver_string[] =
70 "Intel(R) Gigabit Ethernet Network Driver";
71 static const char igb_copyright[] = "Copyright (c) 2007-2012 Intel Corporation.";
72
73 static const struct e1000_info *igb_info_tbl[] = {
74 [board_82575] = &e1000_82575_info,
75 };
76
77 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
100 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
101 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
102 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
103 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
104 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
105 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
106 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
107 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
108 /* required last entry */
109 {0, }
110 };
111
112 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
113
114 void igb_reset(struct igb_adapter *);
115 static int igb_setup_all_tx_resources(struct igb_adapter *);
116 static int igb_setup_all_rx_resources(struct igb_adapter *);
117 static void igb_free_all_tx_resources(struct igb_adapter *);
118 static void igb_free_all_rx_resources(struct igb_adapter *);
119 static void igb_setup_mrqc(struct igb_adapter *);
120 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
121 static void __devexit igb_remove(struct pci_dev *pdev);
122 static int igb_sw_init(struct igb_adapter *);
123 static int igb_open(struct net_device *);
124 static int igb_close(struct net_device *);
125 static void igb_configure_tx(struct igb_adapter *);
126 static void igb_configure_rx(struct igb_adapter *);
127 static void igb_clean_all_tx_rings(struct igb_adapter *);
128 static void igb_clean_all_rx_rings(struct igb_adapter *);
129 static void igb_clean_tx_ring(struct igb_ring *);
130 static void igb_clean_rx_ring(struct igb_ring *);
131 static void igb_set_rx_mode(struct net_device *);
132 static void igb_update_phy_info(unsigned long);
133 static void igb_watchdog(unsigned long);
134 static void igb_watchdog_task(struct work_struct *);
135 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
136 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
137 struct rtnl_link_stats64 *stats);
138 static int igb_change_mtu(struct net_device *, int);
139 static int igb_set_mac(struct net_device *, void *);
140 static void igb_set_uta(struct igb_adapter *adapter);
141 static irqreturn_t igb_intr(int irq, void *);
142 static irqreturn_t igb_intr_msi(int irq, void *);
143 static irqreturn_t igb_msix_other(int irq, void *);
144 static irqreturn_t igb_msix_ring(int irq, void *);
145 #ifdef CONFIG_IGB_DCA
146 static void igb_update_dca(struct igb_q_vector *);
147 static void igb_setup_dca(struct igb_adapter *);
148 #endif /* CONFIG_IGB_DCA */
149 static int igb_poll(struct napi_struct *, int);
150 static bool igb_clean_tx_irq(struct igb_q_vector *);
151 static bool igb_clean_rx_irq(struct igb_q_vector *, int);
152 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
153 static void igb_tx_timeout(struct net_device *);
154 static void igb_reset_task(struct work_struct *);
155 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features);
156 static int igb_vlan_rx_add_vid(struct net_device *, u16);
157 static int igb_vlan_rx_kill_vid(struct net_device *, u16);
158 static void igb_restore_vlan(struct igb_adapter *);
159 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
160 static void igb_ping_all_vfs(struct igb_adapter *);
161 static void igb_msg_task(struct igb_adapter *);
162 static void igb_vmm_control(struct igb_adapter *);
163 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
164 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
165 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
166 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
167 int vf, u16 vlan, u8 qos);
168 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
169 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
170 struct ifla_vf_info *ivi);
171 static void igb_check_vf_rate_limit(struct igb_adapter *);
172
173 #ifdef CONFIG_PCI_IOV
174 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
175 static int igb_find_enabled_vfs(struct igb_adapter *adapter);
176 static int igb_check_vf_assignment(struct igb_adapter *adapter);
177 #endif
178
179 #ifdef CONFIG_PM
180 #ifdef CONFIG_PM_SLEEP
181 static int igb_suspend(struct device *);
182 #endif
183 static int igb_resume(struct device *);
184 #ifdef CONFIG_PM_RUNTIME
185 static int igb_runtime_suspend(struct device *dev);
186 static int igb_runtime_resume(struct device *dev);
187 static int igb_runtime_idle(struct device *dev);
188 #endif
189 static const struct dev_pm_ops igb_pm_ops = {
190 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
191 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
192 igb_runtime_idle)
193 };
194 #endif
195 static void igb_shutdown(struct pci_dev *);
196 #ifdef CONFIG_IGB_DCA
197 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
198 static struct notifier_block dca_notifier = {
199 .notifier_call = igb_notify_dca,
200 .next = NULL,
201 .priority = 0
202 };
203 #endif
204 #ifdef CONFIG_NET_POLL_CONTROLLER
205 /* for netdump / net console */
206 static void igb_netpoll(struct net_device *);
207 #endif
208 #ifdef CONFIG_PCI_IOV
209 static unsigned int max_vfs = 0;
210 module_param(max_vfs, uint, 0);
211 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
212 "per physical function");
213 #endif /* CONFIG_PCI_IOV */
214
215 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
216 pci_channel_state_t);
217 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
218 static void igb_io_resume(struct pci_dev *);
219
220 static struct pci_error_handlers igb_err_handler = {
221 .error_detected = igb_io_error_detected,
222 .slot_reset = igb_io_slot_reset,
223 .resume = igb_io_resume,
224 };
225
226 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
227
228 static struct pci_driver igb_driver = {
229 .name = igb_driver_name,
230 .id_table = igb_pci_tbl,
231 .probe = igb_probe,
232 .remove = __devexit_p(igb_remove),
233 #ifdef CONFIG_PM
234 .driver.pm = &igb_pm_ops,
235 #endif
236 .shutdown = igb_shutdown,
237 .err_handler = &igb_err_handler
238 };
239
240 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
241 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
242 MODULE_LICENSE("GPL");
243 MODULE_VERSION(DRV_VERSION);
244
245 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
246 static int debug = -1;
247 module_param(debug, int, 0);
248 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
249
250 struct igb_reg_info {
251 u32 ofs;
252 char *name;
253 };
254
255 static const struct igb_reg_info igb_reg_info_tbl[] = {
256
257 /* General Registers */
258 {E1000_CTRL, "CTRL"},
259 {E1000_STATUS, "STATUS"},
260 {E1000_CTRL_EXT, "CTRL_EXT"},
261
262 /* Interrupt Registers */
263 {E1000_ICR, "ICR"},
264
265 /* RX Registers */
266 {E1000_RCTL, "RCTL"},
267 {E1000_RDLEN(0), "RDLEN"},
268 {E1000_RDH(0), "RDH"},
269 {E1000_RDT(0), "RDT"},
270 {E1000_RXDCTL(0), "RXDCTL"},
271 {E1000_RDBAL(0), "RDBAL"},
272 {E1000_RDBAH(0), "RDBAH"},
273
274 /* TX Registers */
275 {E1000_TCTL, "TCTL"},
276 {E1000_TDBAL(0), "TDBAL"},
277 {E1000_TDBAH(0), "TDBAH"},
278 {E1000_TDLEN(0), "TDLEN"},
279 {E1000_TDH(0), "TDH"},
280 {E1000_TDT(0), "TDT"},
281 {E1000_TXDCTL(0), "TXDCTL"},
282 {E1000_TDFH, "TDFH"},
283 {E1000_TDFT, "TDFT"},
284 {E1000_TDFHS, "TDFHS"},
285 {E1000_TDFPC, "TDFPC"},
286
287 /* List Terminator */
288 {}
289 };
290
291 /*
292 * igb_regdump - register printout routine
293 */
294 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
295 {
296 int n = 0;
297 char rname[16];
298 u32 regs[8];
299
300 switch (reginfo->ofs) {
301 case E1000_RDLEN(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_RDLEN(n));
304 break;
305 case E1000_RDH(0):
306 for (n = 0; n < 4; n++)
307 regs[n] = rd32(E1000_RDH(n));
308 break;
309 case E1000_RDT(0):
310 for (n = 0; n < 4; n++)
311 regs[n] = rd32(E1000_RDT(n));
312 break;
313 case E1000_RXDCTL(0):
314 for (n = 0; n < 4; n++)
315 regs[n] = rd32(E1000_RXDCTL(n));
316 break;
317 case E1000_RDBAL(0):
318 for (n = 0; n < 4; n++)
319 regs[n] = rd32(E1000_RDBAL(n));
320 break;
321 case E1000_RDBAH(0):
322 for (n = 0; n < 4; n++)
323 regs[n] = rd32(E1000_RDBAH(n));
324 break;
325 case E1000_TDBAL(0):
326 for (n = 0; n < 4; n++)
327 regs[n] = rd32(E1000_RDBAL(n));
328 break;
329 case E1000_TDBAH(0):
330 for (n = 0; n < 4; n++)
331 regs[n] = rd32(E1000_TDBAH(n));
332 break;
333 case E1000_TDLEN(0):
334 for (n = 0; n < 4; n++)
335 regs[n] = rd32(E1000_TDLEN(n));
336 break;
337 case E1000_TDH(0):
338 for (n = 0; n < 4; n++)
339 regs[n] = rd32(E1000_TDH(n));
340 break;
341 case E1000_TDT(0):
342 for (n = 0; n < 4; n++)
343 regs[n] = rd32(E1000_TDT(n));
344 break;
345 case E1000_TXDCTL(0):
346 for (n = 0; n < 4; n++)
347 regs[n] = rd32(E1000_TXDCTL(n));
348 break;
349 default:
350 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
351 return;
352 }
353
354 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
355 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
356 regs[2], regs[3]);
357 }
358
359 /*
360 * igb_dump - Print registers, tx-rings and rx-rings
361 */
362 static void igb_dump(struct igb_adapter *adapter)
363 {
364 struct net_device *netdev = adapter->netdev;
365 struct e1000_hw *hw = &adapter->hw;
366 struct igb_reg_info *reginfo;
367 struct igb_ring *tx_ring;
368 union e1000_adv_tx_desc *tx_desc;
369 struct my_u0 { u64 a; u64 b; } *u0;
370 struct igb_ring *rx_ring;
371 union e1000_adv_rx_desc *rx_desc;
372 u32 staterr;
373 u16 i, n;
374
375 if (!netif_msg_hw(adapter))
376 return;
377
378 /* Print netdevice Info */
379 if (netdev) {
380 dev_info(&adapter->pdev->dev, "Net device Info\n");
381 pr_info("Device Name state trans_start "
382 "last_rx\n");
383 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
384 netdev->state, netdev->trans_start, netdev->last_rx);
385 }
386
387 /* Print Registers */
388 dev_info(&adapter->pdev->dev, "Register Dump\n");
389 pr_info(" Register Name Value\n");
390 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
391 reginfo->name; reginfo++) {
392 igb_regdump(hw, reginfo);
393 }
394
395 /* Print TX Ring Summary */
396 if (!netdev || !netif_running(netdev))
397 goto exit;
398
399 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
400 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
401 for (n = 0; n < adapter->num_tx_queues; n++) {
402 struct igb_tx_buffer *buffer_info;
403 tx_ring = adapter->tx_ring[n];
404 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
405 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
406 n, tx_ring->next_to_use, tx_ring->next_to_clean,
407 (u64)buffer_info->dma,
408 buffer_info->length,
409 buffer_info->next_to_watch,
410 (u64)buffer_info->time_stamp);
411 }
412
413 /* Print TX Rings */
414 if (!netif_msg_tx_done(adapter))
415 goto rx_ring_summary;
416
417 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
418
419 /* Transmit Descriptor Formats
420 *
421 * Advanced Transmit Descriptor
422 * +--------------------------------------------------------------+
423 * 0 | Buffer Address [63:0] |
424 * +--------------------------------------------------------------+
425 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
426 * +--------------------------------------------------------------+
427 * 63 46 45 40 39 38 36 35 32 31 24 15 0
428 */
429
430 for (n = 0; n < adapter->num_tx_queues; n++) {
431 tx_ring = adapter->tx_ring[n];
432 pr_info("------------------------------------\n");
433 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
434 pr_info("------------------------------------\n");
435 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] "
436 "[bi->dma ] leng ntw timestamp "
437 "bi->skb\n");
438
439 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
440 const char *next_desc;
441 struct igb_tx_buffer *buffer_info;
442 tx_desc = IGB_TX_DESC(tx_ring, i);
443 buffer_info = &tx_ring->tx_buffer_info[i];
444 u0 = (struct my_u0 *)tx_desc;
445 if (i == tx_ring->next_to_use &&
446 i == tx_ring->next_to_clean)
447 next_desc = " NTC/U";
448 else if (i == tx_ring->next_to_use)
449 next_desc = " NTU";
450 else if (i == tx_ring->next_to_clean)
451 next_desc = " NTC";
452 else
453 next_desc = "";
454
455 pr_info("T [0x%03X] %016llX %016llX %016llX"
456 " %04X %p %016llX %p%s\n", i,
457 le64_to_cpu(u0->a),
458 le64_to_cpu(u0->b),
459 (u64)buffer_info->dma,
460 buffer_info->length,
461 buffer_info->next_to_watch,
462 (u64)buffer_info->time_stamp,
463 buffer_info->skb, next_desc);
464
465 if (netif_msg_pktdata(adapter) && buffer_info->skb)
466 print_hex_dump(KERN_INFO, "",
467 DUMP_PREFIX_ADDRESS,
468 16, 1, buffer_info->skb->data,
469 buffer_info->length, true);
470 }
471 }
472
473 /* Print RX Rings Summary */
474 rx_ring_summary:
475 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
476 pr_info("Queue [NTU] [NTC]\n");
477 for (n = 0; n < adapter->num_rx_queues; n++) {
478 rx_ring = adapter->rx_ring[n];
479 pr_info(" %5d %5X %5X\n",
480 n, rx_ring->next_to_use, rx_ring->next_to_clean);
481 }
482
483 /* Print RX Rings */
484 if (!netif_msg_rx_status(adapter))
485 goto exit;
486
487 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
488
489 /* Advanced Receive Descriptor (Read) Format
490 * 63 1 0
491 * +-----------------------------------------------------+
492 * 0 | Packet Buffer Address [63:1] |A0/NSE|
493 * +----------------------------------------------+------+
494 * 8 | Header Buffer Address [63:1] | DD |
495 * +-----------------------------------------------------+
496 *
497 *
498 * Advanced Receive Descriptor (Write-Back) Format
499 *
500 * 63 48 47 32 31 30 21 20 17 16 4 3 0
501 * +------------------------------------------------------+
502 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
503 * | Checksum Ident | | | | Type | Type |
504 * +------------------------------------------------------+
505 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
506 * +------------------------------------------------------+
507 * 63 48 47 32 31 20 19 0
508 */
509
510 for (n = 0; n < adapter->num_rx_queues; n++) {
511 rx_ring = adapter->rx_ring[n];
512 pr_info("------------------------------------\n");
513 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
514 pr_info("------------------------------------\n");
515 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] "
516 "[bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
517 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] -----"
518 "----------- [bi->skb] <-- Adv Rx Write-Back format\n");
519
520 for (i = 0; i < rx_ring->count; i++) {
521 const char *next_desc;
522 struct igb_rx_buffer *buffer_info;
523 buffer_info = &rx_ring->rx_buffer_info[i];
524 rx_desc = IGB_RX_DESC(rx_ring, i);
525 u0 = (struct my_u0 *)rx_desc;
526 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
527
528 if (i == rx_ring->next_to_use)
529 next_desc = " NTU";
530 else if (i == rx_ring->next_to_clean)
531 next_desc = " NTC";
532 else
533 next_desc = "";
534
535 if (staterr & E1000_RXD_STAT_DD) {
536 /* Descriptor Done */
537 pr_info("%s[0x%03X] %016llX %016llX -------"
538 "--------- %p%s\n", "RWB", i,
539 le64_to_cpu(u0->a),
540 le64_to_cpu(u0->b),
541 buffer_info->skb, next_desc);
542 } else {
543 pr_info("%s[0x%03X] %016llX %016llX %016llX"
544 " %p%s\n", "R ", i,
545 le64_to_cpu(u0->a),
546 le64_to_cpu(u0->b),
547 (u64)buffer_info->dma,
548 buffer_info->skb, next_desc);
549
550 if (netif_msg_pktdata(adapter) &&
551 buffer_info->dma && buffer_info->skb) {
552 print_hex_dump(KERN_INFO, "",
553 DUMP_PREFIX_ADDRESS,
554 16, 1, buffer_info->skb->data,
555 IGB_RX_HDR_LEN, true);
556 print_hex_dump(KERN_INFO, "",
557 DUMP_PREFIX_ADDRESS,
558 16, 1,
559 page_address(buffer_info->page) +
560 buffer_info->page_offset,
561 PAGE_SIZE/2, true);
562 }
563 }
564 }
565 }
566
567 exit:
568 return;
569 }
570
571 /**
572 * igb_get_hw_dev - return device
573 * used by hardware layer to print debugging information
574 **/
575 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
576 {
577 struct igb_adapter *adapter = hw->back;
578 return adapter->netdev;
579 }
580
581 /**
582 * igb_init_module - Driver Registration Routine
583 *
584 * igb_init_module is the first routine called when the driver is
585 * loaded. All it does is register with the PCI subsystem.
586 **/
587 static int __init igb_init_module(void)
588 {
589 int ret;
590 pr_info("%s - version %s\n",
591 igb_driver_string, igb_driver_version);
592
593 pr_info("%s\n", igb_copyright);
594
595 #ifdef CONFIG_IGB_DCA
596 dca_register_notify(&dca_notifier);
597 #endif
598 ret = pci_register_driver(&igb_driver);
599 return ret;
600 }
601
602 module_init(igb_init_module);
603
604 /**
605 * igb_exit_module - Driver Exit Cleanup Routine
606 *
607 * igb_exit_module is called just before the driver is removed
608 * from memory.
609 **/
610 static void __exit igb_exit_module(void)
611 {
612 #ifdef CONFIG_IGB_DCA
613 dca_unregister_notify(&dca_notifier);
614 #endif
615 pci_unregister_driver(&igb_driver);
616 }
617
618 module_exit(igb_exit_module);
619
620 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
621 /**
622 * igb_cache_ring_register - Descriptor ring to register mapping
623 * @adapter: board private structure to initialize
624 *
625 * Once we know the feature-set enabled for the device, we'll cache
626 * the register offset the descriptor ring is assigned to.
627 **/
628 static void igb_cache_ring_register(struct igb_adapter *adapter)
629 {
630 int i = 0, j = 0;
631 u32 rbase_offset = adapter->vfs_allocated_count;
632
633 switch (adapter->hw.mac.type) {
634 case e1000_82576:
635 /* The queues are allocated for virtualization such that VF 0
636 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
637 * In order to avoid collision we start at the first free queue
638 * and continue consuming queues in the same sequence
639 */
640 if (adapter->vfs_allocated_count) {
641 for (; i < adapter->rss_queues; i++)
642 adapter->rx_ring[i]->reg_idx = rbase_offset +
643 Q_IDX_82576(i);
644 }
645 case e1000_82575:
646 case e1000_82580:
647 case e1000_i350:
648 case e1000_i210:
649 case e1000_i211:
650 default:
651 for (; i < adapter->num_rx_queues; i++)
652 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
653 for (; j < adapter->num_tx_queues; j++)
654 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
655 break;
656 }
657 }
658
659 static void igb_free_queues(struct igb_adapter *adapter)
660 {
661 int i;
662
663 for (i = 0; i < adapter->num_tx_queues; i++) {
664 kfree(adapter->tx_ring[i]);
665 adapter->tx_ring[i] = NULL;
666 }
667 for (i = 0; i < adapter->num_rx_queues; i++) {
668 kfree(adapter->rx_ring[i]);
669 adapter->rx_ring[i] = NULL;
670 }
671 adapter->num_rx_queues = 0;
672 adapter->num_tx_queues = 0;
673 }
674
675 /**
676 * igb_alloc_queues - Allocate memory for all rings
677 * @adapter: board private structure to initialize
678 *
679 * We allocate one ring per queue at run-time since we don't know the
680 * number of queues at compile-time.
681 **/
682 static int igb_alloc_queues(struct igb_adapter *adapter)
683 {
684 struct igb_ring *ring;
685 int i;
686 int orig_node = adapter->node;
687
688 for (i = 0; i < adapter->num_tx_queues; i++) {
689 if (orig_node == -1) {
690 int cur_node = next_online_node(adapter->node);
691 if (cur_node == MAX_NUMNODES)
692 cur_node = first_online_node;
693 adapter->node = cur_node;
694 }
695 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
696 adapter->node);
697 if (!ring)
698 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
699 if (!ring)
700 goto err;
701 ring->count = adapter->tx_ring_count;
702 ring->queue_index = i;
703 ring->dev = &adapter->pdev->dev;
704 ring->netdev = adapter->netdev;
705 ring->numa_node = adapter->node;
706 /* For 82575, context index must be unique per ring. */
707 if (adapter->hw.mac.type == e1000_82575)
708 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
709 adapter->tx_ring[i] = ring;
710 }
711 /* Restore the adapter's original node */
712 adapter->node = orig_node;
713
714 for (i = 0; i < adapter->num_rx_queues; i++) {
715 if (orig_node == -1) {
716 int cur_node = next_online_node(adapter->node);
717 if (cur_node == MAX_NUMNODES)
718 cur_node = first_online_node;
719 adapter->node = cur_node;
720 }
721 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
722 adapter->node);
723 if (!ring)
724 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
725 if (!ring)
726 goto err;
727 ring->count = adapter->rx_ring_count;
728 ring->queue_index = i;
729 ring->dev = &adapter->pdev->dev;
730 ring->netdev = adapter->netdev;
731 ring->numa_node = adapter->node;
732 /* set flag indicating ring supports SCTP checksum offload */
733 if (adapter->hw.mac.type >= e1000_82576)
734 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
735
736 /*
737 * On i350, i210, and i211, loopback VLAN packets
738 * have the tag byte-swapped.
739 * */
740 if (adapter->hw.mac.type >= e1000_i350)
741 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
742
743 adapter->rx_ring[i] = ring;
744 }
745 /* Restore the adapter's original node */
746 adapter->node = orig_node;
747
748 igb_cache_ring_register(adapter);
749
750 return 0;
751
752 err:
753 /* Restore the adapter's original node */
754 adapter->node = orig_node;
755 igb_free_queues(adapter);
756
757 return -ENOMEM;
758 }
759
760 /**
761 * igb_write_ivar - configure ivar for given MSI-X vector
762 * @hw: pointer to the HW structure
763 * @msix_vector: vector number we are allocating to a given ring
764 * @index: row index of IVAR register to write within IVAR table
765 * @offset: column offset of in IVAR, should be multiple of 8
766 *
767 * This function is intended to handle the writing of the IVAR register
768 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
769 * each containing an cause allocation for an Rx and Tx ring, and a
770 * variable number of rows depending on the number of queues supported.
771 **/
772 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
773 int index, int offset)
774 {
775 u32 ivar = array_rd32(E1000_IVAR0, index);
776
777 /* clear any bits that are currently set */
778 ivar &= ~((u32)0xFF << offset);
779
780 /* write vector and valid bit */
781 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
782
783 array_wr32(E1000_IVAR0, index, ivar);
784 }
785
786 #define IGB_N0_QUEUE -1
787 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
788 {
789 struct igb_adapter *adapter = q_vector->adapter;
790 struct e1000_hw *hw = &adapter->hw;
791 int rx_queue = IGB_N0_QUEUE;
792 int tx_queue = IGB_N0_QUEUE;
793 u32 msixbm = 0;
794
795 if (q_vector->rx.ring)
796 rx_queue = q_vector->rx.ring->reg_idx;
797 if (q_vector->tx.ring)
798 tx_queue = q_vector->tx.ring->reg_idx;
799
800 switch (hw->mac.type) {
801 case e1000_82575:
802 /* The 82575 assigns vectors using a bitmask, which matches the
803 bitmask for the EICR/EIMS/EIMC registers. To assign one
804 or more queues to a vector, we write the appropriate bits
805 into the MSIXBM register for that vector. */
806 if (rx_queue > IGB_N0_QUEUE)
807 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
808 if (tx_queue > IGB_N0_QUEUE)
809 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
810 if (!adapter->msix_entries && msix_vector == 0)
811 msixbm |= E1000_EIMS_OTHER;
812 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
813 q_vector->eims_value = msixbm;
814 break;
815 case e1000_82576:
816 /*
817 * 82576 uses a table that essentially consists of 2 columns
818 * with 8 rows. The ordering is column-major so we use the
819 * lower 3 bits as the row index, and the 4th bit as the
820 * column offset.
821 */
822 if (rx_queue > IGB_N0_QUEUE)
823 igb_write_ivar(hw, msix_vector,
824 rx_queue & 0x7,
825 (rx_queue & 0x8) << 1);
826 if (tx_queue > IGB_N0_QUEUE)
827 igb_write_ivar(hw, msix_vector,
828 tx_queue & 0x7,
829 ((tx_queue & 0x8) << 1) + 8);
830 q_vector->eims_value = 1 << msix_vector;
831 break;
832 case e1000_82580:
833 case e1000_i350:
834 case e1000_i210:
835 case e1000_i211:
836 /*
837 * On 82580 and newer adapters the scheme is similar to 82576
838 * however instead of ordering column-major we have things
839 * ordered row-major. So we traverse the table by using
840 * bit 0 as the column offset, and the remaining bits as the
841 * row index.
842 */
843 if (rx_queue > IGB_N0_QUEUE)
844 igb_write_ivar(hw, msix_vector,
845 rx_queue >> 1,
846 (rx_queue & 0x1) << 4);
847 if (tx_queue > IGB_N0_QUEUE)
848 igb_write_ivar(hw, msix_vector,
849 tx_queue >> 1,
850 ((tx_queue & 0x1) << 4) + 8);
851 q_vector->eims_value = 1 << msix_vector;
852 break;
853 default:
854 BUG();
855 break;
856 }
857
858 /* add q_vector eims value to global eims_enable_mask */
859 adapter->eims_enable_mask |= q_vector->eims_value;
860
861 /* configure q_vector to set itr on first interrupt */
862 q_vector->set_itr = 1;
863 }
864
865 /**
866 * igb_configure_msix - Configure MSI-X hardware
867 *
868 * igb_configure_msix sets up the hardware to properly
869 * generate MSI-X interrupts.
870 **/
871 static void igb_configure_msix(struct igb_adapter *adapter)
872 {
873 u32 tmp;
874 int i, vector = 0;
875 struct e1000_hw *hw = &adapter->hw;
876
877 adapter->eims_enable_mask = 0;
878
879 /* set vector for other causes, i.e. link changes */
880 switch (hw->mac.type) {
881 case e1000_82575:
882 tmp = rd32(E1000_CTRL_EXT);
883 /* enable MSI-X PBA support*/
884 tmp |= E1000_CTRL_EXT_PBA_CLR;
885
886 /* Auto-Mask interrupts upon ICR read. */
887 tmp |= E1000_CTRL_EXT_EIAME;
888 tmp |= E1000_CTRL_EXT_IRCA;
889
890 wr32(E1000_CTRL_EXT, tmp);
891
892 /* enable msix_other interrupt */
893 array_wr32(E1000_MSIXBM(0), vector++,
894 E1000_EIMS_OTHER);
895 adapter->eims_other = E1000_EIMS_OTHER;
896
897 break;
898
899 case e1000_82576:
900 case e1000_82580:
901 case e1000_i350:
902 case e1000_i210:
903 case e1000_i211:
904 /* Turn on MSI-X capability first, or our settings
905 * won't stick. And it will take days to debug. */
906 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
907 E1000_GPIE_PBA | E1000_GPIE_EIAME |
908 E1000_GPIE_NSICR);
909
910 /* enable msix_other interrupt */
911 adapter->eims_other = 1 << vector;
912 tmp = (vector++ | E1000_IVAR_VALID) << 8;
913
914 wr32(E1000_IVAR_MISC, tmp);
915 break;
916 default:
917 /* do nothing, since nothing else supports MSI-X */
918 break;
919 } /* switch (hw->mac.type) */
920
921 adapter->eims_enable_mask |= adapter->eims_other;
922
923 for (i = 0; i < adapter->num_q_vectors; i++)
924 igb_assign_vector(adapter->q_vector[i], vector++);
925
926 wrfl();
927 }
928
929 /**
930 * igb_request_msix - Initialize MSI-X interrupts
931 *
932 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
933 * kernel.
934 **/
935 static int igb_request_msix(struct igb_adapter *adapter)
936 {
937 struct net_device *netdev = adapter->netdev;
938 struct e1000_hw *hw = &adapter->hw;
939 int i, err = 0, vector = 0;
940
941 err = request_irq(adapter->msix_entries[vector].vector,
942 igb_msix_other, 0, netdev->name, adapter);
943 if (err)
944 goto out;
945 vector++;
946
947 for (i = 0; i < adapter->num_q_vectors; i++) {
948 struct igb_q_vector *q_vector = adapter->q_vector[i];
949
950 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
951
952 if (q_vector->rx.ring && q_vector->tx.ring)
953 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
954 q_vector->rx.ring->queue_index);
955 else if (q_vector->tx.ring)
956 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
957 q_vector->tx.ring->queue_index);
958 else if (q_vector->rx.ring)
959 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
960 q_vector->rx.ring->queue_index);
961 else
962 sprintf(q_vector->name, "%s-unused", netdev->name);
963
964 err = request_irq(adapter->msix_entries[vector].vector,
965 igb_msix_ring, 0, q_vector->name,
966 q_vector);
967 if (err)
968 goto out;
969 vector++;
970 }
971
972 igb_configure_msix(adapter);
973 return 0;
974 out:
975 return err;
976 }
977
978 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
979 {
980 if (adapter->msix_entries) {
981 pci_disable_msix(adapter->pdev);
982 kfree(adapter->msix_entries);
983 adapter->msix_entries = NULL;
984 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
985 pci_disable_msi(adapter->pdev);
986 }
987 }
988
989 /**
990 * igb_free_q_vectors - Free memory allocated for interrupt vectors
991 * @adapter: board private structure to initialize
992 *
993 * This function frees the memory allocated to the q_vectors. In addition if
994 * NAPI is enabled it will delete any references to the NAPI struct prior
995 * to freeing the q_vector.
996 **/
997 static void igb_free_q_vectors(struct igb_adapter *adapter)
998 {
999 int v_idx;
1000
1001 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1002 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1003 adapter->q_vector[v_idx] = NULL;
1004 if (!q_vector)
1005 continue;
1006 netif_napi_del(&q_vector->napi);
1007 kfree(q_vector);
1008 }
1009 adapter->num_q_vectors = 0;
1010 }
1011
1012 /**
1013 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1014 *
1015 * This function resets the device so that it has 0 rx queues, tx queues, and
1016 * MSI-X interrupts allocated.
1017 */
1018 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1019 {
1020 igb_free_queues(adapter);
1021 igb_free_q_vectors(adapter);
1022 igb_reset_interrupt_capability(adapter);
1023 }
1024
1025 /**
1026 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1027 *
1028 * Attempt to configure interrupts using the best available
1029 * capabilities of the hardware and kernel.
1030 **/
1031 static int igb_set_interrupt_capability(struct igb_adapter *adapter)
1032 {
1033 int err;
1034 int numvecs, i;
1035
1036 /* Number of supported queues. */
1037 adapter->num_rx_queues = adapter->rss_queues;
1038 if (adapter->vfs_allocated_count)
1039 adapter->num_tx_queues = 1;
1040 else
1041 adapter->num_tx_queues = adapter->rss_queues;
1042
1043 /* start with one vector for every rx queue */
1044 numvecs = adapter->num_rx_queues;
1045
1046 /* if tx handler is separate add 1 for every tx queue */
1047 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1048 numvecs += adapter->num_tx_queues;
1049
1050 /* store the number of vectors reserved for queues */
1051 adapter->num_q_vectors = numvecs;
1052
1053 /* add 1 vector for link status interrupts */
1054 numvecs++;
1055 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1056 GFP_KERNEL);
1057
1058 if (!adapter->msix_entries)
1059 goto msi_only;
1060
1061 for (i = 0; i < numvecs; i++)
1062 adapter->msix_entries[i].entry = i;
1063
1064 err = pci_enable_msix(adapter->pdev,
1065 adapter->msix_entries,
1066 numvecs);
1067 if (err == 0)
1068 goto out;
1069
1070 igb_reset_interrupt_capability(adapter);
1071
1072 /* If we can't do MSI-X, try MSI */
1073 msi_only:
1074 #ifdef CONFIG_PCI_IOV
1075 /* disable SR-IOV for non MSI-X configurations */
1076 if (adapter->vf_data) {
1077 struct e1000_hw *hw = &adapter->hw;
1078 /* disable iov and allow time for transactions to clear */
1079 pci_disable_sriov(adapter->pdev);
1080 msleep(500);
1081
1082 kfree(adapter->vf_data);
1083 adapter->vf_data = NULL;
1084 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1085 wrfl();
1086 msleep(100);
1087 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1088 }
1089 #endif
1090 adapter->vfs_allocated_count = 0;
1091 adapter->rss_queues = 1;
1092 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1093 adapter->num_rx_queues = 1;
1094 adapter->num_tx_queues = 1;
1095 adapter->num_q_vectors = 1;
1096 if (!pci_enable_msi(adapter->pdev))
1097 adapter->flags |= IGB_FLAG_HAS_MSI;
1098 out:
1099 /* Notify the stack of the (possibly) reduced queue counts. */
1100 rtnl_lock();
1101 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1102 err = netif_set_real_num_rx_queues(adapter->netdev,
1103 adapter->num_rx_queues);
1104 rtnl_unlock();
1105 return err;
1106 }
1107
1108 /**
1109 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1110 * @adapter: board private structure to initialize
1111 *
1112 * We allocate one q_vector per queue interrupt. If allocation fails we
1113 * return -ENOMEM.
1114 **/
1115 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1116 {
1117 struct igb_q_vector *q_vector;
1118 struct e1000_hw *hw = &adapter->hw;
1119 int v_idx;
1120 int orig_node = adapter->node;
1121
1122 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1123 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1124 adapter->num_tx_queues)) &&
1125 (adapter->num_rx_queues == v_idx))
1126 adapter->node = orig_node;
1127 if (orig_node == -1) {
1128 int cur_node = next_online_node(adapter->node);
1129 if (cur_node == MAX_NUMNODES)
1130 cur_node = first_online_node;
1131 adapter->node = cur_node;
1132 }
1133 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1134 adapter->node);
1135 if (!q_vector)
1136 q_vector = kzalloc(sizeof(struct igb_q_vector),
1137 GFP_KERNEL);
1138 if (!q_vector)
1139 goto err_out;
1140 q_vector->adapter = adapter;
1141 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1142 q_vector->itr_val = IGB_START_ITR;
1143 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1144 adapter->q_vector[v_idx] = q_vector;
1145 }
1146 /* Restore the adapter's original node */
1147 adapter->node = orig_node;
1148
1149 return 0;
1150
1151 err_out:
1152 /* Restore the adapter's original node */
1153 adapter->node = orig_node;
1154 igb_free_q_vectors(adapter);
1155 return -ENOMEM;
1156 }
1157
1158 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1159 int ring_idx, int v_idx)
1160 {
1161 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1162
1163 q_vector->rx.ring = adapter->rx_ring[ring_idx];
1164 q_vector->rx.ring->q_vector = q_vector;
1165 q_vector->rx.count++;
1166 q_vector->itr_val = adapter->rx_itr_setting;
1167 if (q_vector->itr_val && q_vector->itr_val <= 3)
1168 q_vector->itr_val = IGB_START_ITR;
1169 }
1170
1171 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1172 int ring_idx, int v_idx)
1173 {
1174 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1175
1176 q_vector->tx.ring = adapter->tx_ring[ring_idx];
1177 q_vector->tx.ring->q_vector = q_vector;
1178 q_vector->tx.count++;
1179 q_vector->itr_val = adapter->tx_itr_setting;
1180 q_vector->tx.work_limit = adapter->tx_work_limit;
1181 if (q_vector->itr_val && q_vector->itr_val <= 3)
1182 q_vector->itr_val = IGB_START_ITR;
1183 }
1184
1185 /**
1186 * igb_map_ring_to_vector - maps allocated queues to vectors
1187 *
1188 * This function maps the recently allocated queues to vectors.
1189 **/
1190 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1191 {
1192 int i;
1193 int v_idx = 0;
1194
1195 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1196 (adapter->num_q_vectors < adapter->num_tx_queues))
1197 return -ENOMEM;
1198
1199 if (adapter->num_q_vectors >=
1200 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1201 for (i = 0; i < adapter->num_rx_queues; i++)
1202 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1203 for (i = 0; i < adapter->num_tx_queues; i++)
1204 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1205 } else {
1206 for (i = 0; i < adapter->num_rx_queues; i++) {
1207 if (i < adapter->num_tx_queues)
1208 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1209 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1210 }
1211 for (; i < adapter->num_tx_queues; i++)
1212 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1213 }
1214 return 0;
1215 }
1216
1217 /**
1218 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1219 *
1220 * This function initializes the interrupts and allocates all of the queues.
1221 **/
1222 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1223 {
1224 struct pci_dev *pdev = adapter->pdev;
1225 int err;
1226
1227 err = igb_set_interrupt_capability(adapter);
1228 if (err)
1229 return err;
1230
1231 err = igb_alloc_q_vectors(adapter);
1232 if (err) {
1233 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1234 goto err_alloc_q_vectors;
1235 }
1236
1237 err = igb_alloc_queues(adapter);
1238 if (err) {
1239 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1240 goto err_alloc_queues;
1241 }
1242
1243 err = igb_map_ring_to_vector(adapter);
1244 if (err) {
1245 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1246 goto err_map_queues;
1247 }
1248
1249
1250 return 0;
1251 err_map_queues:
1252 igb_free_queues(adapter);
1253 err_alloc_queues:
1254 igb_free_q_vectors(adapter);
1255 err_alloc_q_vectors:
1256 igb_reset_interrupt_capability(adapter);
1257 return err;
1258 }
1259
1260 /**
1261 * igb_request_irq - initialize interrupts
1262 *
1263 * Attempts to configure interrupts using the best available
1264 * capabilities of the hardware and kernel.
1265 **/
1266 static int igb_request_irq(struct igb_adapter *adapter)
1267 {
1268 struct net_device *netdev = adapter->netdev;
1269 struct pci_dev *pdev = adapter->pdev;
1270 int err = 0;
1271
1272 if (adapter->msix_entries) {
1273 err = igb_request_msix(adapter);
1274 if (!err)
1275 goto request_done;
1276 /* fall back to MSI */
1277 igb_clear_interrupt_scheme(adapter);
1278 if (!pci_enable_msi(pdev))
1279 adapter->flags |= IGB_FLAG_HAS_MSI;
1280 igb_free_all_tx_resources(adapter);
1281 igb_free_all_rx_resources(adapter);
1282 adapter->num_tx_queues = 1;
1283 adapter->num_rx_queues = 1;
1284 adapter->num_q_vectors = 1;
1285 err = igb_alloc_q_vectors(adapter);
1286 if (err) {
1287 dev_err(&pdev->dev,
1288 "Unable to allocate memory for vectors\n");
1289 goto request_done;
1290 }
1291 err = igb_alloc_queues(adapter);
1292 if (err) {
1293 dev_err(&pdev->dev,
1294 "Unable to allocate memory for queues\n");
1295 igb_free_q_vectors(adapter);
1296 goto request_done;
1297 }
1298 igb_setup_all_tx_resources(adapter);
1299 igb_setup_all_rx_resources(adapter);
1300 }
1301
1302 igb_assign_vector(adapter->q_vector[0], 0);
1303
1304 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1305 err = request_irq(pdev->irq, igb_intr_msi, 0,
1306 netdev->name, adapter);
1307 if (!err)
1308 goto request_done;
1309
1310 /* fall back to legacy interrupts */
1311 igb_reset_interrupt_capability(adapter);
1312 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1313 }
1314
1315 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1316 netdev->name, adapter);
1317
1318 if (err)
1319 dev_err(&pdev->dev, "Error %d getting interrupt\n",
1320 err);
1321
1322 request_done:
1323 return err;
1324 }
1325
1326 static void igb_free_irq(struct igb_adapter *adapter)
1327 {
1328 if (adapter->msix_entries) {
1329 int vector = 0, i;
1330
1331 free_irq(adapter->msix_entries[vector++].vector, adapter);
1332
1333 for (i = 0; i < adapter->num_q_vectors; i++)
1334 free_irq(adapter->msix_entries[vector++].vector,
1335 adapter->q_vector[i]);
1336 } else {
1337 free_irq(adapter->pdev->irq, adapter);
1338 }
1339 }
1340
1341 /**
1342 * igb_irq_disable - Mask off interrupt generation on the NIC
1343 * @adapter: board private structure
1344 **/
1345 static void igb_irq_disable(struct igb_adapter *adapter)
1346 {
1347 struct e1000_hw *hw = &adapter->hw;
1348
1349 /*
1350 * we need to be careful when disabling interrupts. The VFs are also
1351 * mapped into these registers and so clearing the bits can cause
1352 * issues on the VF drivers so we only need to clear what we set
1353 */
1354 if (adapter->msix_entries) {
1355 u32 regval = rd32(E1000_EIAM);
1356 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1357 wr32(E1000_EIMC, adapter->eims_enable_mask);
1358 regval = rd32(E1000_EIAC);
1359 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1360 }
1361
1362 wr32(E1000_IAM, 0);
1363 wr32(E1000_IMC, ~0);
1364 wrfl();
1365 if (adapter->msix_entries) {
1366 int i;
1367 for (i = 0; i < adapter->num_q_vectors; i++)
1368 synchronize_irq(adapter->msix_entries[i].vector);
1369 } else {
1370 synchronize_irq(adapter->pdev->irq);
1371 }
1372 }
1373
1374 /**
1375 * igb_irq_enable - Enable default interrupt generation settings
1376 * @adapter: board private structure
1377 **/
1378 static void igb_irq_enable(struct igb_adapter *adapter)
1379 {
1380 struct e1000_hw *hw = &adapter->hw;
1381
1382 if (adapter->msix_entries) {
1383 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1384 u32 regval = rd32(E1000_EIAC);
1385 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1386 regval = rd32(E1000_EIAM);
1387 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1388 wr32(E1000_EIMS, adapter->eims_enable_mask);
1389 if (adapter->vfs_allocated_count) {
1390 wr32(E1000_MBVFIMR, 0xFF);
1391 ims |= E1000_IMS_VMMB;
1392 }
1393 wr32(E1000_IMS, ims);
1394 } else {
1395 wr32(E1000_IMS, IMS_ENABLE_MASK |
1396 E1000_IMS_DRSTA);
1397 wr32(E1000_IAM, IMS_ENABLE_MASK |
1398 E1000_IMS_DRSTA);
1399 }
1400 }
1401
1402 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1403 {
1404 struct e1000_hw *hw = &adapter->hw;
1405 u16 vid = adapter->hw.mng_cookie.vlan_id;
1406 u16 old_vid = adapter->mng_vlan_id;
1407
1408 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1409 /* add VID to filter table */
1410 igb_vfta_set(hw, vid, true);
1411 adapter->mng_vlan_id = vid;
1412 } else {
1413 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1414 }
1415
1416 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1417 (vid != old_vid) &&
1418 !test_bit(old_vid, adapter->active_vlans)) {
1419 /* remove VID from filter table */
1420 igb_vfta_set(hw, old_vid, false);
1421 }
1422 }
1423
1424 /**
1425 * igb_release_hw_control - release control of the h/w to f/w
1426 * @adapter: address of board private structure
1427 *
1428 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1429 * For ASF and Pass Through versions of f/w this means that the
1430 * driver is no longer loaded.
1431 *
1432 **/
1433 static void igb_release_hw_control(struct igb_adapter *adapter)
1434 {
1435 struct e1000_hw *hw = &adapter->hw;
1436 u32 ctrl_ext;
1437
1438 /* Let firmware take over control of h/w */
1439 ctrl_ext = rd32(E1000_CTRL_EXT);
1440 wr32(E1000_CTRL_EXT,
1441 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1442 }
1443
1444 /**
1445 * igb_get_hw_control - get control of the h/w from f/w
1446 * @adapter: address of board private structure
1447 *
1448 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1449 * For ASF and Pass Through versions of f/w this means that
1450 * the driver is loaded.
1451 *
1452 **/
1453 static void igb_get_hw_control(struct igb_adapter *adapter)
1454 {
1455 struct e1000_hw *hw = &adapter->hw;
1456 u32 ctrl_ext;
1457
1458 /* Let firmware know the driver has taken over */
1459 ctrl_ext = rd32(E1000_CTRL_EXT);
1460 wr32(E1000_CTRL_EXT,
1461 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1462 }
1463
1464 /**
1465 * igb_configure - configure the hardware for RX and TX
1466 * @adapter: private board structure
1467 **/
1468 static void igb_configure(struct igb_adapter *adapter)
1469 {
1470 struct net_device *netdev = adapter->netdev;
1471 int i;
1472
1473 igb_get_hw_control(adapter);
1474 igb_set_rx_mode(netdev);
1475
1476 igb_restore_vlan(adapter);
1477
1478 igb_setup_tctl(adapter);
1479 igb_setup_mrqc(adapter);
1480 igb_setup_rctl(adapter);
1481
1482 igb_configure_tx(adapter);
1483 igb_configure_rx(adapter);
1484
1485 igb_rx_fifo_flush_82575(&adapter->hw);
1486
1487 /* call igb_desc_unused which always leaves
1488 * at least 1 descriptor unused to make sure
1489 * next_to_use != next_to_clean */
1490 for (i = 0; i < adapter->num_rx_queues; i++) {
1491 struct igb_ring *ring = adapter->rx_ring[i];
1492 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1493 }
1494 }
1495
1496 /**
1497 * igb_power_up_link - Power up the phy/serdes link
1498 * @adapter: address of board private structure
1499 **/
1500 void igb_power_up_link(struct igb_adapter *adapter)
1501 {
1502 igb_reset_phy(&adapter->hw);
1503
1504 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1505 igb_power_up_phy_copper(&adapter->hw);
1506 else
1507 igb_power_up_serdes_link_82575(&adapter->hw);
1508 }
1509
1510 /**
1511 * igb_power_down_link - Power down the phy/serdes link
1512 * @adapter: address of board private structure
1513 */
1514 static void igb_power_down_link(struct igb_adapter *adapter)
1515 {
1516 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1517 igb_power_down_phy_copper_82575(&adapter->hw);
1518 else
1519 igb_shutdown_serdes_link_82575(&adapter->hw);
1520 }
1521
1522 /**
1523 * igb_up - Open the interface and prepare it to handle traffic
1524 * @adapter: board private structure
1525 **/
1526 int igb_up(struct igb_adapter *adapter)
1527 {
1528 struct e1000_hw *hw = &adapter->hw;
1529 int i;
1530
1531 /* hardware has been reset, we need to reload some things */
1532 igb_configure(adapter);
1533
1534 clear_bit(__IGB_DOWN, &adapter->state);
1535
1536 for (i = 0; i < adapter->num_q_vectors; i++)
1537 napi_enable(&(adapter->q_vector[i]->napi));
1538
1539 if (adapter->msix_entries)
1540 igb_configure_msix(adapter);
1541 else
1542 igb_assign_vector(adapter->q_vector[0], 0);
1543
1544 /* Clear any pending interrupts. */
1545 rd32(E1000_ICR);
1546 igb_irq_enable(adapter);
1547
1548 /* notify VFs that reset has been completed */
1549 if (adapter->vfs_allocated_count) {
1550 u32 reg_data = rd32(E1000_CTRL_EXT);
1551 reg_data |= E1000_CTRL_EXT_PFRSTD;
1552 wr32(E1000_CTRL_EXT, reg_data);
1553 }
1554
1555 netif_tx_start_all_queues(adapter->netdev);
1556
1557 /* start the watchdog. */
1558 hw->mac.get_link_status = 1;
1559 schedule_work(&adapter->watchdog_task);
1560
1561 return 0;
1562 }
1563
1564 void igb_down(struct igb_adapter *adapter)
1565 {
1566 struct net_device *netdev = adapter->netdev;
1567 struct e1000_hw *hw = &adapter->hw;
1568 u32 tctl, rctl;
1569 int i;
1570
1571 /* signal that we're down so the interrupt handler does not
1572 * reschedule our watchdog timer */
1573 set_bit(__IGB_DOWN, &adapter->state);
1574
1575 /* disable receives in the hardware */
1576 rctl = rd32(E1000_RCTL);
1577 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1578 /* flush and sleep below */
1579
1580 netif_tx_stop_all_queues(netdev);
1581
1582 /* disable transmits in the hardware */
1583 tctl = rd32(E1000_TCTL);
1584 tctl &= ~E1000_TCTL_EN;
1585 wr32(E1000_TCTL, tctl);
1586 /* flush both disables and wait for them to finish */
1587 wrfl();
1588 msleep(10);
1589
1590 for (i = 0; i < adapter->num_q_vectors; i++)
1591 napi_disable(&(adapter->q_vector[i]->napi));
1592
1593 igb_irq_disable(adapter);
1594
1595 del_timer_sync(&adapter->watchdog_timer);
1596 del_timer_sync(&adapter->phy_info_timer);
1597
1598 netif_carrier_off(netdev);
1599
1600 /* record the stats before reset*/
1601 spin_lock(&adapter->stats64_lock);
1602 igb_update_stats(adapter, &adapter->stats64);
1603 spin_unlock(&adapter->stats64_lock);
1604
1605 adapter->link_speed = 0;
1606 adapter->link_duplex = 0;
1607
1608 if (!pci_channel_offline(adapter->pdev))
1609 igb_reset(adapter);
1610 igb_clean_all_tx_rings(adapter);
1611 igb_clean_all_rx_rings(adapter);
1612 #ifdef CONFIG_IGB_DCA
1613
1614 /* since we reset the hardware DCA settings were cleared */
1615 igb_setup_dca(adapter);
1616 #endif
1617 }
1618
1619 void igb_reinit_locked(struct igb_adapter *adapter)
1620 {
1621 WARN_ON(in_interrupt());
1622 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1623 msleep(1);
1624 igb_down(adapter);
1625 igb_up(adapter);
1626 clear_bit(__IGB_RESETTING, &adapter->state);
1627 }
1628
1629 void igb_reset(struct igb_adapter *adapter)
1630 {
1631 struct pci_dev *pdev = adapter->pdev;
1632 struct e1000_hw *hw = &adapter->hw;
1633 struct e1000_mac_info *mac = &hw->mac;
1634 struct e1000_fc_info *fc = &hw->fc;
1635 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1636 u16 hwm;
1637
1638 /* Repartition Pba for greater than 9k mtu
1639 * To take effect CTRL.RST is required.
1640 */
1641 switch (mac->type) {
1642 case e1000_i350:
1643 case e1000_82580:
1644 pba = rd32(E1000_RXPBS);
1645 pba = igb_rxpbs_adjust_82580(pba);
1646 break;
1647 case e1000_82576:
1648 pba = rd32(E1000_RXPBS);
1649 pba &= E1000_RXPBS_SIZE_MASK_82576;
1650 break;
1651 case e1000_82575:
1652 case e1000_i210:
1653 case e1000_i211:
1654 default:
1655 pba = E1000_PBA_34K;
1656 break;
1657 }
1658
1659 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1660 (mac->type < e1000_82576)) {
1661 /* adjust PBA for jumbo frames */
1662 wr32(E1000_PBA, pba);
1663
1664 /* To maintain wire speed transmits, the Tx FIFO should be
1665 * large enough to accommodate two full transmit packets,
1666 * rounded up to the next 1KB and expressed in KB. Likewise,
1667 * the Rx FIFO should be large enough to accommodate at least
1668 * one full receive packet and is similarly rounded up and
1669 * expressed in KB. */
1670 pba = rd32(E1000_PBA);
1671 /* upper 16 bits has Tx packet buffer allocation size in KB */
1672 tx_space = pba >> 16;
1673 /* lower 16 bits has Rx packet buffer allocation size in KB */
1674 pba &= 0xffff;
1675 /* the tx fifo also stores 16 bytes of information about the tx
1676 * but don't include ethernet FCS because hardware appends it */
1677 min_tx_space = (adapter->max_frame_size +
1678 sizeof(union e1000_adv_tx_desc) -
1679 ETH_FCS_LEN) * 2;
1680 min_tx_space = ALIGN(min_tx_space, 1024);
1681 min_tx_space >>= 10;
1682 /* software strips receive CRC, so leave room for it */
1683 min_rx_space = adapter->max_frame_size;
1684 min_rx_space = ALIGN(min_rx_space, 1024);
1685 min_rx_space >>= 10;
1686
1687 /* If current Tx allocation is less than the min Tx FIFO size,
1688 * and the min Tx FIFO size is less than the current Rx FIFO
1689 * allocation, take space away from current Rx allocation */
1690 if (tx_space < min_tx_space &&
1691 ((min_tx_space - tx_space) < pba)) {
1692 pba = pba - (min_tx_space - tx_space);
1693
1694 /* if short on rx space, rx wins and must trump tx
1695 * adjustment */
1696 if (pba < min_rx_space)
1697 pba = min_rx_space;
1698 }
1699 wr32(E1000_PBA, pba);
1700 }
1701
1702 /* flow control settings */
1703 /* The high water mark must be low enough to fit one full frame
1704 * (or the size used for early receive) above it in the Rx FIFO.
1705 * Set it to the lower of:
1706 * - 90% of the Rx FIFO size, or
1707 * - the full Rx FIFO size minus one full frame */
1708 hwm = min(((pba << 10) * 9 / 10),
1709 ((pba << 10) - 2 * adapter->max_frame_size));
1710
1711 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1712 fc->low_water = fc->high_water - 16;
1713 fc->pause_time = 0xFFFF;
1714 fc->send_xon = 1;
1715 fc->current_mode = fc->requested_mode;
1716
1717 /* disable receive for all VFs and wait one second */
1718 if (adapter->vfs_allocated_count) {
1719 int i;
1720 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1721 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1722
1723 /* ping all the active vfs to let them know we are going down */
1724 igb_ping_all_vfs(adapter);
1725
1726 /* disable transmits and receives */
1727 wr32(E1000_VFRE, 0);
1728 wr32(E1000_VFTE, 0);
1729 }
1730
1731 /* Allow time for pending master requests to run */
1732 hw->mac.ops.reset_hw(hw);
1733 wr32(E1000_WUC, 0);
1734
1735 if (hw->mac.ops.init_hw(hw))
1736 dev_err(&pdev->dev, "Hardware Error\n");
1737
1738 /*
1739 * Flow control settings reset on hardware reset, so guarantee flow
1740 * control is off when forcing speed.
1741 */
1742 if (!hw->mac.autoneg)
1743 igb_force_mac_fc(hw);
1744
1745 igb_init_dmac(adapter, pba);
1746 if (!netif_running(adapter->netdev))
1747 igb_power_down_link(adapter);
1748
1749 igb_update_mng_vlan(adapter);
1750
1751 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1752 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1753
1754 igb_get_phy_info(hw);
1755 }
1756
1757 static netdev_features_t igb_fix_features(struct net_device *netdev,
1758 netdev_features_t features)
1759 {
1760 /*
1761 * Since there is no support for separate rx/tx vlan accel
1762 * enable/disable make sure tx flag is always in same state as rx.
1763 */
1764 if (features & NETIF_F_HW_VLAN_RX)
1765 features |= NETIF_F_HW_VLAN_TX;
1766 else
1767 features &= ~NETIF_F_HW_VLAN_TX;
1768
1769 return features;
1770 }
1771
1772 static int igb_set_features(struct net_device *netdev,
1773 netdev_features_t features)
1774 {
1775 netdev_features_t changed = netdev->features ^ features;
1776 struct igb_adapter *adapter = netdev_priv(netdev);
1777
1778 if (changed & NETIF_F_HW_VLAN_RX)
1779 igb_vlan_mode(netdev, features);
1780
1781 if (!(changed & NETIF_F_RXALL))
1782 return 0;
1783
1784 netdev->features = features;
1785
1786 if (netif_running(netdev))
1787 igb_reinit_locked(adapter);
1788 else
1789 igb_reset(adapter);
1790
1791 return 0;
1792 }
1793
1794 static const struct net_device_ops igb_netdev_ops = {
1795 .ndo_open = igb_open,
1796 .ndo_stop = igb_close,
1797 .ndo_start_xmit = igb_xmit_frame,
1798 .ndo_get_stats64 = igb_get_stats64,
1799 .ndo_set_rx_mode = igb_set_rx_mode,
1800 .ndo_set_mac_address = igb_set_mac,
1801 .ndo_change_mtu = igb_change_mtu,
1802 .ndo_do_ioctl = igb_ioctl,
1803 .ndo_tx_timeout = igb_tx_timeout,
1804 .ndo_validate_addr = eth_validate_addr,
1805 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1806 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1807 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1808 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1809 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1810 .ndo_get_vf_config = igb_ndo_get_vf_config,
1811 #ifdef CONFIG_NET_POLL_CONTROLLER
1812 .ndo_poll_controller = igb_netpoll,
1813 #endif
1814 .ndo_fix_features = igb_fix_features,
1815 .ndo_set_features = igb_set_features,
1816 };
1817
1818 /**
1819 * igb_set_fw_version - Configure version string for ethtool
1820 * @adapter: adapter struct
1821 *
1822 **/
1823 void igb_set_fw_version(struct igb_adapter *adapter)
1824 {
1825 struct e1000_hw *hw = &adapter->hw;
1826 u16 eeprom_verh, eeprom_verl, comb_verh, comb_verl, comb_offset;
1827 u16 major, build, patch, fw_version;
1828 u32 etrack_id;
1829
1830 hw->nvm.ops.read(hw, 5, 1, &fw_version);
1831 if (adapter->hw.mac.type != e1000_i211) {
1832 hw->nvm.ops.read(hw, NVM_ETRACK_WORD, 1, &eeprom_verh);
1833 hw->nvm.ops.read(hw, (NVM_ETRACK_WORD + 1), 1, &eeprom_verl);
1834 etrack_id = (eeprom_verh << IGB_ETRACK_SHIFT) | eeprom_verl;
1835
1836 /* combo image version needs to be found */
1837 hw->nvm.ops.read(hw, NVM_COMB_VER_PTR, 1, &comb_offset);
1838 if ((comb_offset != 0x0) &&
1839 (comb_offset != IGB_NVM_VER_INVALID)) {
1840 hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset
1841 + 1), 1, &comb_verh);
1842 hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset),
1843 1, &comb_verl);
1844
1845 /* Only display Option Rom if it exists and is valid */
1846 if ((comb_verh && comb_verl) &&
1847 ((comb_verh != IGB_NVM_VER_INVALID) &&
1848 (comb_verl != IGB_NVM_VER_INVALID))) {
1849 major = comb_verl >> IGB_COMB_VER_SHFT;
1850 build = (comb_verl << IGB_COMB_VER_SHFT) |
1851 (comb_verh >> IGB_COMB_VER_SHFT);
1852 patch = comb_verh & IGB_COMB_VER_MASK;
1853 snprintf(adapter->fw_version,
1854 sizeof(adapter->fw_version),
1855 "%d.%d%d, 0x%08x, %d.%d.%d",
1856 (fw_version & IGB_MAJOR_MASK) >>
1857 IGB_MAJOR_SHIFT,
1858 (fw_version & IGB_MINOR_MASK) >>
1859 IGB_MINOR_SHIFT,
1860 (fw_version & IGB_BUILD_MASK),
1861 etrack_id, major, build, patch);
1862 goto out;
1863 }
1864 }
1865 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1866 "%d.%d%d, 0x%08x",
1867 (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1868 (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1869 (fw_version & IGB_BUILD_MASK), etrack_id);
1870 } else {
1871 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1872 "%d.%d%d",
1873 (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1874 (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1875 (fw_version & IGB_BUILD_MASK));
1876 }
1877 out:
1878 return;
1879 }
1880
1881 /**
1882 * igb_probe - Device Initialization Routine
1883 * @pdev: PCI device information struct
1884 * @ent: entry in igb_pci_tbl
1885 *
1886 * Returns 0 on success, negative on failure
1887 *
1888 * igb_probe initializes an adapter identified by a pci_dev structure.
1889 * The OS initialization, configuring of the adapter private structure,
1890 * and a hardware reset occur.
1891 **/
1892 static int __devinit igb_probe(struct pci_dev *pdev,
1893 const struct pci_device_id *ent)
1894 {
1895 struct net_device *netdev;
1896 struct igb_adapter *adapter;
1897 struct e1000_hw *hw;
1898 u16 eeprom_data = 0;
1899 s32 ret_val;
1900 static int global_quad_port_a; /* global quad port a indication */
1901 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1902 unsigned long mmio_start, mmio_len;
1903 int err, pci_using_dac;
1904 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1905 u8 part_str[E1000_PBANUM_LENGTH];
1906
1907 /* Catch broken hardware that put the wrong VF device ID in
1908 * the PCIe SR-IOV capability.
1909 */
1910 if (pdev->is_virtfn) {
1911 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1912 pci_name(pdev), pdev->vendor, pdev->device);
1913 return -EINVAL;
1914 }
1915
1916 err = pci_enable_device_mem(pdev);
1917 if (err)
1918 return err;
1919
1920 pci_using_dac = 0;
1921 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1922 if (!err) {
1923 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1924 if (!err)
1925 pci_using_dac = 1;
1926 } else {
1927 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1928 if (err) {
1929 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1930 if (err) {
1931 dev_err(&pdev->dev, "No usable DMA "
1932 "configuration, aborting\n");
1933 goto err_dma;
1934 }
1935 }
1936 }
1937
1938 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1939 IORESOURCE_MEM),
1940 igb_driver_name);
1941 if (err)
1942 goto err_pci_reg;
1943
1944 pci_enable_pcie_error_reporting(pdev);
1945
1946 pci_set_master(pdev);
1947 pci_save_state(pdev);
1948
1949 err = -ENOMEM;
1950 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1951 IGB_MAX_TX_QUEUES);
1952 if (!netdev)
1953 goto err_alloc_etherdev;
1954
1955 SET_NETDEV_DEV(netdev, &pdev->dev);
1956
1957 pci_set_drvdata(pdev, netdev);
1958 adapter = netdev_priv(netdev);
1959 adapter->netdev = netdev;
1960 adapter->pdev = pdev;
1961 hw = &adapter->hw;
1962 hw->back = adapter;
1963 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
1964
1965 mmio_start = pci_resource_start(pdev, 0);
1966 mmio_len = pci_resource_len(pdev, 0);
1967
1968 err = -EIO;
1969 hw->hw_addr = ioremap(mmio_start, mmio_len);
1970 if (!hw->hw_addr)
1971 goto err_ioremap;
1972
1973 netdev->netdev_ops = &igb_netdev_ops;
1974 igb_set_ethtool_ops(netdev);
1975 netdev->watchdog_timeo = 5 * HZ;
1976
1977 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1978
1979 netdev->mem_start = mmio_start;
1980 netdev->mem_end = mmio_start + mmio_len;
1981
1982 /* PCI config space info */
1983 hw->vendor_id = pdev->vendor;
1984 hw->device_id = pdev->device;
1985 hw->revision_id = pdev->revision;
1986 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1987 hw->subsystem_device_id = pdev->subsystem_device;
1988
1989 /* Copy the default MAC, PHY and NVM function pointers */
1990 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1991 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1992 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1993 /* Initialize skew-specific constants */
1994 err = ei->get_invariants(hw);
1995 if (err)
1996 goto err_sw_init;
1997
1998 /* setup the private structure */
1999 err = igb_sw_init(adapter);
2000 if (err)
2001 goto err_sw_init;
2002
2003 igb_get_bus_info_pcie(hw);
2004
2005 hw->phy.autoneg_wait_to_complete = false;
2006
2007 /* Copper options */
2008 if (hw->phy.media_type == e1000_media_type_copper) {
2009 hw->phy.mdix = AUTO_ALL_MODES;
2010 hw->phy.disable_polarity_correction = false;
2011 hw->phy.ms_type = e1000_ms_hw_default;
2012 }
2013
2014 if (igb_check_reset_block(hw))
2015 dev_info(&pdev->dev,
2016 "PHY reset is blocked due to SOL/IDER session.\n");
2017
2018 /*
2019 * features is initialized to 0 in allocation, it might have bits
2020 * set by igb_sw_init so we should use an or instead of an
2021 * assignment.
2022 */
2023 netdev->features |= NETIF_F_SG |
2024 NETIF_F_IP_CSUM |
2025 NETIF_F_IPV6_CSUM |
2026 NETIF_F_TSO |
2027 NETIF_F_TSO6 |
2028 NETIF_F_RXHASH |
2029 NETIF_F_RXCSUM |
2030 NETIF_F_HW_VLAN_RX |
2031 NETIF_F_HW_VLAN_TX;
2032
2033 /* copy netdev features into list of user selectable features */
2034 netdev->hw_features |= netdev->features;
2035 netdev->hw_features |= NETIF_F_RXALL;
2036
2037 /* set this bit last since it cannot be part of hw_features */
2038 netdev->features |= NETIF_F_HW_VLAN_FILTER;
2039
2040 netdev->vlan_features |= NETIF_F_TSO |
2041 NETIF_F_TSO6 |
2042 NETIF_F_IP_CSUM |
2043 NETIF_F_IPV6_CSUM |
2044 NETIF_F_SG;
2045
2046 netdev->priv_flags |= IFF_SUPP_NOFCS;
2047
2048 if (pci_using_dac) {
2049 netdev->features |= NETIF_F_HIGHDMA;
2050 netdev->vlan_features |= NETIF_F_HIGHDMA;
2051 }
2052
2053 if (hw->mac.type >= e1000_82576) {
2054 netdev->hw_features |= NETIF_F_SCTP_CSUM;
2055 netdev->features |= NETIF_F_SCTP_CSUM;
2056 }
2057
2058 netdev->priv_flags |= IFF_UNICAST_FLT;
2059
2060 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
2061
2062 /* before reading the NVM, reset the controller to put the device in a
2063 * known good starting state */
2064 hw->mac.ops.reset_hw(hw);
2065
2066 /*
2067 * make sure the NVM is good , i211 parts have special NVM that
2068 * doesn't contain a checksum
2069 */
2070 if (hw->mac.type != e1000_i211) {
2071 if (hw->nvm.ops.validate(hw) < 0) {
2072 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
2073 err = -EIO;
2074 goto err_eeprom;
2075 }
2076 }
2077
2078 /* copy the MAC address out of the NVM */
2079 if (hw->mac.ops.read_mac_addr(hw))
2080 dev_err(&pdev->dev, "NVM Read Error\n");
2081
2082 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2083 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2084
2085 if (!is_valid_ether_addr(netdev->perm_addr)) {
2086 dev_err(&pdev->dev, "Invalid MAC Address\n");
2087 err = -EIO;
2088 goto err_eeprom;
2089 }
2090
2091 /* get firmware version for ethtool -i */
2092 igb_set_fw_version(adapter);
2093
2094 setup_timer(&adapter->watchdog_timer, igb_watchdog,
2095 (unsigned long) adapter);
2096 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
2097 (unsigned long) adapter);
2098
2099 INIT_WORK(&adapter->reset_task, igb_reset_task);
2100 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2101
2102 /* Initialize link properties that are user-changeable */
2103 adapter->fc_autoneg = true;
2104 hw->mac.autoneg = true;
2105 hw->phy.autoneg_advertised = 0x2f;
2106
2107 hw->fc.requested_mode = e1000_fc_default;
2108 hw->fc.current_mode = e1000_fc_default;
2109
2110 igb_validate_mdi_setting(hw);
2111
2112 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2113 * enable the ACPI Magic Packet filter
2114 */
2115
2116 if (hw->bus.func == 0)
2117 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
2118 else if (hw->mac.type >= e1000_82580)
2119 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2120 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2121 &eeprom_data);
2122 else if (hw->bus.func == 1)
2123 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2124
2125 if (eeprom_data & eeprom_apme_mask)
2126 adapter->eeprom_wol |= E1000_WUFC_MAG;
2127
2128 /* now that we have the eeprom settings, apply the special cases where
2129 * the eeprom may be wrong or the board simply won't support wake on
2130 * lan on a particular port */
2131 switch (pdev->device) {
2132 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2133 adapter->eeprom_wol = 0;
2134 break;
2135 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2136 case E1000_DEV_ID_82576_FIBER:
2137 case E1000_DEV_ID_82576_SERDES:
2138 /* Wake events only supported on port A for dual fiber
2139 * regardless of eeprom setting */
2140 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2141 adapter->eeprom_wol = 0;
2142 break;
2143 case E1000_DEV_ID_82576_QUAD_COPPER:
2144 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2145 /* if quad port adapter, disable WoL on all but port A */
2146 if (global_quad_port_a != 0)
2147 adapter->eeprom_wol = 0;
2148 else
2149 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2150 /* Reset for multiple quad port adapters */
2151 if (++global_quad_port_a == 4)
2152 global_quad_port_a = 0;
2153 break;
2154 }
2155
2156 /* initialize the wol settings based on the eeprom settings */
2157 adapter->wol = adapter->eeprom_wol;
2158 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
2159
2160 /* reset the hardware with the new settings */
2161 igb_reset(adapter);
2162
2163 /* let the f/w know that the h/w is now under the control of the
2164 * driver. */
2165 igb_get_hw_control(adapter);
2166
2167 strcpy(netdev->name, "eth%d");
2168 err = register_netdev(netdev);
2169 if (err)
2170 goto err_register;
2171
2172 /* carrier off reporting is important to ethtool even BEFORE open */
2173 netif_carrier_off(netdev);
2174
2175 #ifdef CONFIG_IGB_DCA
2176 if (dca_add_requester(&pdev->dev) == 0) {
2177 adapter->flags |= IGB_FLAG_DCA_ENABLED;
2178 dev_info(&pdev->dev, "DCA enabled\n");
2179 igb_setup_dca(adapter);
2180 }
2181
2182 #endif
2183
2184 #ifdef CONFIG_IGB_PTP
2185 /* do hw tstamp init after resetting */
2186 igb_ptp_init(adapter);
2187 #endif /* CONFIG_IGB_PTP */
2188
2189 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2190 /* print bus type/speed/width info */
2191 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
2192 netdev->name,
2193 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
2194 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
2195 "unknown"),
2196 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2197 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2198 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2199 "unknown"),
2200 netdev->dev_addr);
2201
2202 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2203 if (ret_val)
2204 strcpy(part_str, "Unknown");
2205 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
2206 dev_info(&pdev->dev,
2207 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2208 adapter->msix_entries ? "MSI-X" :
2209 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2210 adapter->num_rx_queues, adapter->num_tx_queues);
2211 switch (hw->mac.type) {
2212 case e1000_i350:
2213 case e1000_i210:
2214 case e1000_i211:
2215 igb_set_eee_i350(hw);
2216 break;
2217 default:
2218 break;
2219 }
2220
2221 pm_runtime_put_noidle(&pdev->dev);
2222 return 0;
2223
2224 err_register:
2225 igb_release_hw_control(adapter);
2226 err_eeprom:
2227 if (!igb_check_reset_block(hw))
2228 igb_reset_phy(hw);
2229
2230 if (hw->flash_address)
2231 iounmap(hw->flash_address);
2232 err_sw_init:
2233 igb_clear_interrupt_scheme(adapter);
2234 iounmap(hw->hw_addr);
2235 err_ioremap:
2236 free_netdev(netdev);
2237 err_alloc_etherdev:
2238 pci_release_selected_regions(pdev,
2239 pci_select_bars(pdev, IORESOURCE_MEM));
2240 err_pci_reg:
2241 err_dma:
2242 pci_disable_device(pdev);
2243 return err;
2244 }
2245
2246 /**
2247 * igb_remove - Device Removal Routine
2248 * @pdev: PCI device information struct
2249 *
2250 * igb_remove is called by the PCI subsystem to alert the driver
2251 * that it should release a PCI device. The could be caused by a
2252 * Hot-Plug event, or because the driver is going to be removed from
2253 * memory.
2254 **/
2255 static void __devexit igb_remove(struct pci_dev *pdev)
2256 {
2257 struct net_device *netdev = pci_get_drvdata(pdev);
2258 struct igb_adapter *adapter = netdev_priv(netdev);
2259 struct e1000_hw *hw = &adapter->hw;
2260
2261 pm_runtime_get_noresume(&pdev->dev);
2262 #ifdef CONFIG_IGB_PTP
2263 igb_ptp_stop(adapter);
2264 #endif /* CONFIG_IGB_PTP */
2265
2266 /*
2267 * The watchdog timer may be rescheduled, so explicitly
2268 * disable watchdog from being rescheduled.
2269 */
2270 set_bit(__IGB_DOWN, &adapter->state);
2271 del_timer_sync(&adapter->watchdog_timer);
2272 del_timer_sync(&adapter->phy_info_timer);
2273
2274 cancel_work_sync(&adapter->reset_task);
2275 cancel_work_sync(&adapter->watchdog_task);
2276
2277 #ifdef CONFIG_IGB_DCA
2278 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2279 dev_info(&pdev->dev, "DCA disabled\n");
2280 dca_remove_requester(&pdev->dev);
2281 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2282 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2283 }
2284 #endif
2285
2286 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2287 * would have already happened in close and is redundant. */
2288 igb_release_hw_control(adapter);
2289
2290 unregister_netdev(netdev);
2291
2292 igb_clear_interrupt_scheme(adapter);
2293
2294 #ifdef CONFIG_PCI_IOV
2295 /* reclaim resources allocated to VFs */
2296 if (adapter->vf_data) {
2297 /* disable iov and allow time for transactions to clear */
2298 if (!igb_check_vf_assignment(adapter)) {
2299 pci_disable_sriov(pdev);
2300 msleep(500);
2301 } else {
2302 dev_info(&pdev->dev, "VF(s) assigned to guests!\n");
2303 }
2304
2305 kfree(adapter->vf_data);
2306 adapter->vf_data = NULL;
2307 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2308 wrfl();
2309 msleep(100);
2310 dev_info(&pdev->dev, "IOV Disabled\n");
2311 }
2312 #endif
2313
2314 iounmap(hw->hw_addr);
2315 if (hw->flash_address)
2316 iounmap(hw->flash_address);
2317 pci_release_selected_regions(pdev,
2318 pci_select_bars(pdev, IORESOURCE_MEM));
2319
2320 kfree(adapter->shadow_vfta);
2321 free_netdev(netdev);
2322
2323 pci_disable_pcie_error_reporting(pdev);
2324
2325 pci_disable_device(pdev);
2326 }
2327
2328 /**
2329 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2330 * @adapter: board private structure to initialize
2331 *
2332 * This function initializes the vf specific data storage and then attempts to
2333 * allocate the VFs. The reason for ordering it this way is because it is much
2334 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2335 * the memory for the VFs.
2336 **/
2337 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2338 {
2339 #ifdef CONFIG_PCI_IOV
2340 struct pci_dev *pdev = adapter->pdev;
2341 struct e1000_hw *hw = &adapter->hw;
2342 int old_vfs = igb_find_enabled_vfs(adapter);
2343 int i;
2344
2345 /* Virtualization features not supported on i210 family. */
2346 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
2347 return;
2348
2349 if (old_vfs) {
2350 dev_info(&pdev->dev, "%d pre-allocated VFs found - override "
2351 "max_vfs setting of %d\n", old_vfs, max_vfs);
2352 adapter->vfs_allocated_count = old_vfs;
2353 }
2354
2355 if (!adapter->vfs_allocated_count)
2356 return;
2357
2358 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2359 sizeof(struct vf_data_storage), GFP_KERNEL);
2360
2361 /* if allocation failed then we do not support SR-IOV */
2362 if (!adapter->vf_data) {
2363 adapter->vfs_allocated_count = 0;
2364 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2365 "Data Storage\n");
2366 goto out;
2367 }
2368
2369 if (!old_vfs) {
2370 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count))
2371 goto err_out;
2372 }
2373 dev_info(&pdev->dev, "%d VFs allocated\n",
2374 adapter->vfs_allocated_count);
2375 for (i = 0; i < adapter->vfs_allocated_count; i++)
2376 igb_vf_configure(adapter, i);
2377
2378 /* DMA Coalescing is not supported in IOV mode. */
2379 adapter->flags &= ~IGB_FLAG_DMAC;
2380 goto out;
2381 err_out:
2382 kfree(adapter->vf_data);
2383 adapter->vf_data = NULL;
2384 adapter->vfs_allocated_count = 0;
2385 out:
2386 return;
2387 #endif /* CONFIG_PCI_IOV */
2388 }
2389
2390 /**
2391 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2392 * @adapter: board private structure to initialize
2393 *
2394 * igb_sw_init initializes the Adapter private data structure.
2395 * Fields are initialized based on PCI device information and
2396 * OS network device settings (MTU size).
2397 **/
2398 static int __devinit igb_sw_init(struct igb_adapter *adapter)
2399 {
2400 struct e1000_hw *hw = &adapter->hw;
2401 struct net_device *netdev = adapter->netdev;
2402 struct pci_dev *pdev = adapter->pdev;
2403 u32 max_rss_queues;
2404
2405 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2406
2407 /* set default ring sizes */
2408 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2409 adapter->rx_ring_count = IGB_DEFAULT_RXD;
2410
2411 /* set default ITR values */
2412 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2413 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2414
2415 /* set default work limits */
2416 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2417
2418 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2419 VLAN_HLEN;
2420 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2421
2422 adapter->node = -1;
2423
2424 spin_lock_init(&adapter->stats64_lock);
2425 #ifdef CONFIG_PCI_IOV
2426 switch (hw->mac.type) {
2427 case e1000_82576:
2428 case e1000_i350:
2429 if (max_vfs > 7) {
2430 dev_warn(&pdev->dev,
2431 "Maximum of 7 VFs per PF, using max\n");
2432 adapter->vfs_allocated_count = 7;
2433 } else
2434 adapter->vfs_allocated_count = max_vfs;
2435 break;
2436 default:
2437 break;
2438 }
2439 #endif /* CONFIG_PCI_IOV */
2440
2441 /* Determine the maximum number of RSS queues supported. */
2442 switch (hw->mac.type) {
2443 case e1000_i211:
2444 max_rss_queues = IGB_MAX_RX_QUEUES_I211;
2445 break;
2446 case e1000_82575:
2447 case e1000_i210:
2448 max_rss_queues = IGB_MAX_RX_QUEUES_82575;
2449 break;
2450 case e1000_i350:
2451 /* I350 cannot do RSS and SR-IOV at the same time */
2452 if (!!adapter->vfs_allocated_count) {
2453 max_rss_queues = 1;
2454 break;
2455 }
2456 /* fall through */
2457 case e1000_82576:
2458 if (!!adapter->vfs_allocated_count) {
2459 max_rss_queues = 2;
2460 break;
2461 }
2462 /* fall through */
2463 case e1000_82580:
2464 default:
2465 max_rss_queues = IGB_MAX_RX_QUEUES;
2466 break;
2467 }
2468
2469 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
2470
2471 /* Determine if we need to pair queues. */
2472 switch (hw->mac.type) {
2473 case e1000_82575:
2474 case e1000_i211:
2475 /* Device supports enough interrupts without queue pairing. */
2476 break;
2477 case e1000_82576:
2478 /*
2479 * If VFs are going to be allocated with RSS queues then we
2480 * should pair the queues in order to conserve interrupts due
2481 * to limited supply.
2482 */
2483 if ((adapter->rss_queues > 1) &&
2484 (adapter->vfs_allocated_count > 6))
2485 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2486 /* fall through */
2487 case e1000_82580:
2488 case e1000_i350:
2489 case e1000_i210:
2490 default:
2491 /*
2492 * If rss_queues > half of max_rss_queues, pair the queues in
2493 * order to conserve interrupts due to limited supply.
2494 */
2495 if (adapter->rss_queues > (max_rss_queues / 2))
2496 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2497 break;
2498 }
2499
2500 /* Setup and initialize a copy of the hw vlan table array */
2501 adapter->shadow_vfta = kzalloc(sizeof(u32) *
2502 E1000_VLAN_FILTER_TBL_SIZE,
2503 GFP_ATOMIC);
2504
2505 /* This call may decrease the number of queues */
2506 if (igb_init_interrupt_scheme(adapter)) {
2507 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2508 return -ENOMEM;
2509 }
2510
2511 igb_probe_vfs(adapter);
2512
2513 /* Explicitly disable IRQ since the NIC can be in any state. */
2514 igb_irq_disable(adapter);
2515
2516 if (hw->mac.type >= e1000_i350)
2517 adapter->flags &= ~IGB_FLAG_DMAC;
2518
2519 set_bit(__IGB_DOWN, &adapter->state);
2520 return 0;
2521 }
2522
2523 /**
2524 * igb_open - Called when a network interface is made active
2525 * @netdev: network interface device structure
2526 *
2527 * Returns 0 on success, negative value on failure
2528 *
2529 * The open entry point is called when a network interface is made
2530 * active by the system (IFF_UP). At this point all resources needed
2531 * for transmit and receive operations are allocated, the interrupt
2532 * handler is registered with the OS, the watchdog timer is started,
2533 * and the stack is notified that the interface is ready.
2534 **/
2535 static int __igb_open(struct net_device *netdev, bool resuming)
2536 {
2537 struct igb_adapter *adapter = netdev_priv(netdev);
2538 struct e1000_hw *hw = &adapter->hw;
2539 struct pci_dev *pdev = adapter->pdev;
2540 int err;
2541 int i;
2542
2543 /* disallow open during test */
2544 if (test_bit(__IGB_TESTING, &adapter->state)) {
2545 WARN_ON(resuming);
2546 return -EBUSY;
2547 }
2548
2549 if (!resuming)
2550 pm_runtime_get_sync(&pdev->dev);
2551
2552 netif_carrier_off(netdev);
2553
2554 /* allocate transmit descriptors */
2555 err = igb_setup_all_tx_resources(adapter);
2556 if (err)
2557 goto err_setup_tx;
2558
2559 /* allocate receive descriptors */
2560 err = igb_setup_all_rx_resources(adapter);
2561 if (err)
2562 goto err_setup_rx;
2563
2564 igb_power_up_link(adapter);
2565
2566 /* before we allocate an interrupt, we must be ready to handle it.
2567 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2568 * as soon as we call pci_request_irq, so we have to setup our
2569 * clean_rx handler before we do so. */
2570 igb_configure(adapter);
2571
2572 err = igb_request_irq(adapter);
2573 if (err)
2574 goto err_req_irq;
2575
2576 /* From here on the code is the same as igb_up() */
2577 clear_bit(__IGB_DOWN, &adapter->state);
2578
2579 for (i = 0; i < adapter->num_q_vectors; i++)
2580 napi_enable(&(adapter->q_vector[i]->napi));
2581
2582 /* Clear any pending interrupts. */
2583 rd32(E1000_ICR);
2584
2585 igb_irq_enable(adapter);
2586
2587 /* notify VFs that reset has been completed */
2588 if (adapter->vfs_allocated_count) {
2589 u32 reg_data = rd32(E1000_CTRL_EXT);
2590 reg_data |= E1000_CTRL_EXT_PFRSTD;
2591 wr32(E1000_CTRL_EXT, reg_data);
2592 }
2593
2594 netif_tx_start_all_queues(netdev);
2595
2596 if (!resuming)
2597 pm_runtime_put(&pdev->dev);
2598
2599 /* start the watchdog. */
2600 hw->mac.get_link_status = 1;
2601 schedule_work(&adapter->watchdog_task);
2602
2603 return 0;
2604
2605 err_req_irq:
2606 igb_release_hw_control(adapter);
2607 igb_power_down_link(adapter);
2608 igb_free_all_rx_resources(adapter);
2609 err_setup_rx:
2610 igb_free_all_tx_resources(adapter);
2611 err_setup_tx:
2612 igb_reset(adapter);
2613 if (!resuming)
2614 pm_runtime_put(&pdev->dev);
2615
2616 return err;
2617 }
2618
2619 static int igb_open(struct net_device *netdev)
2620 {
2621 return __igb_open(netdev, false);
2622 }
2623
2624 /**
2625 * igb_close - Disables a network interface
2626 * @netdev: network interface device structure
2627 *
2628 * Returns 0, this is not allowed to fail
2629 *
2630 * The close entry point is called when an interface is de-activated
2631 * by the OS. The hardware is still under the driver's control, but
2632 * needs to be disabled. A global MAC reset is issued to stop the
2633 * hardware, and all transmit and receive resources are freed.
2634 **/
2635 static int __igb_close(struct net_device *netdev, bool suspending)
2636 {
2637 struct igb_adapter *adapter = netdev_priv(netdev);
2638 struct pci_dev *pdev = adapter->pdev;
2639
2640 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2641
2642 if (!suspending)
2643 pm_runtime_get_sync(&pdev->dev);
2644
2645 igb_down(adapter);
2646 igb_free_irq(adapter);
2647
2648 igb_free_all_tx_resources(adapter);
2649 igb_free_all_rx_resources(adapter);
2650
2651 if (!suspending)
2652 pm_runtime_put_sync(&pdev->dev);
2653 return 0;
2654 }
2655
2656 static int igb_close(struct net_device *netdev)
2657 {
2658 return __igb_close(netdev, false);
2659 }
2660
2661 /**
2662 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2663 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2664 *
2665 * Return 0 on success, negative on failure
2666 **/
2667 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2668 {
2669 struct device *dev = tx_ring->dev;
2670 int orig_node = dev_to_node(dev);
2671 int size;
2672
2673 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
2674 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2675 if (!tx_ring->tx_buffer_info)
2676 tx_ring->tx_buffer_info = vzalloc(size);
2677 if (!tx_ring->tx_buffer_info)
2678 goto err;
2679
2680 /* round up to nearest 4K */
2681 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2682 tx_ring->size = ALIGN(tx_ring->size, 4096);
2683
2684 set_dev_node(dev, tx_ring->numa_node);
2685 tx_ring->desc = dma_alloc_coherent(dev,
2686 tx_ring->size,
2687 &tx_ring->dma,
2688 GFP_KERNEL);
2689 set_dev_node(dev, orig_node);
2690 if (!tx_ring->desc)
2691 tx_ring->desc = dma_alloc_coherent(dev,
2692 tx_ring->size,
2693 &tx_ring->dma,
2694 GFP_KERNEL);
2695
2696 if (!tx_ring->desc)
2697 goto err;
2698
2699 tx_ring->next_to_use = 0;
2700 tx_ring->next_to_clean = 0;
2701
2702 return 0;
2703
2704 err:
2705 vfree(tx_ring->tx_buffer_info);
2706 dev_err(dev,
2707 "Unable to allocate memory for the transmit descriptor ring\n");
2708 return -ENOMEM;
2709 }
2710
2711 /**
2712 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2713 * (Descriptors) for all queues
2714 * @adapter: board private structure
2715 *
2716 * Return 0 on success, negative on failure
2717 **/
2718 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2719 {
2720 struct pci_dev *pdev = adapter->pdev;
2721 int i, err = 0;
2722
2723 for (i = 0; i < adapter->num_tx_queues; i++) {
2724 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2725 if (err) {
2726 dev_err(&pdev->dev,
2727 "Allocation for Tx Queue %u failed\n", i);
2728 for (i--; i >= 0; i--)
2729 igb_free_tx_resources(adapter->tx_ring[i]);
2730 break;
2731 }
2732 }
2733
2734 return err;
2735 }
2736
2737 /**
2738 * igb_setup_tctl - configure the transmit control registers
2739 * @adapter: Board private structure
2740 **/
2741 void igb_setup_tctl(struct igb_adapter *adapter)
2742 {
2743 struct e1000_hw *hw = &adapter->hw;
2744 u32 tctl;
2745
2746 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2747 wr32(E1000_TXDCTL(0), 0);
2748
2749 /* Program the Transmit Control Register */
2750 tctl = rd32(E1000_TCTL);
2751 tctl &= ~E1000_TCTL_CT;
2752 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2753 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2754
2755 igb_config_collision_dist(hw);
2756
2757 /* Enable transmits */
2758 tctl |= E1000_TCTL_EN;
2759
2760 wr32(E1000_TCTL, tctl);
2761 }
2762
2763 /**
2764 * igb_configure_tx_ring - Configure transmit ring after Reset
2765 * @adapter: board private structure
2766 * @ring: tx ring to configure
2767 *
2768 * Configure a transmit ring after a reset.
2769 **/
2770 void igb_configure_tx_ring(struct igb_adapter *adapter,
2771 struct igb_ring *ring)
2772 {
2773 struct e1000_hw *hw = &adapter->hw;
2774 u32 txdctl = 0;
2775 u64 tdba = ring->dma;
2776 int reg_idx = ring->reg_idx;
2777
2778 /* disable the queue */
2779 wr32(E1000_TXDCTL(reg_idx), 0);
2780 wrfl();
2781 mdelay(10);
2782
2783 wr32(E1000_TDLEN(reg_idx),
2784 ring->count * sizeof(union e1000_adv_tx_desc));
2785 wr32(E1000_TDBAL(reg_idx),
2786 tdba & 0x00000000ffffffffULL);
2787 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2788
2789 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2790 wr32(E1000_TDH(reg_idx), 0);
2791 writel(0, ring->tail);
2792
2793 txdctl |= IGB_TX_PTHRESH;
2794 txdctl |= IGB_TX_HTHRESH << 8;
2795 txdctl |= IGB_TX_WTHRESH << 16;
2796
2797 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2798 wr32(E1000_TXDCTL(reg_idx), txdctl);
2799 }
2800
2801 /**
2802 * igb_configure_tx - Configure transmit Unit after Reset
2803 * @adapter: board private structure
2804 *
2805 * Configure the Tx unit of the MAC after a reset.
2806 **/
2807 static void igb_configure_tx(struct igb_adapter *adapter)
2808 {
2809 int i;
2810
2811 for (i = 0; i < adapter->num_tx_queues; i++)
2812 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2813 }
2814
2815 /**
2816 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2817 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2818 *
2819 * Returns 0 on success, negative on failure
2820 **/
2821 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2822 {
2823 struct device *dev = rx_ring->dev;
2824 int orig_node = dev_to_node(dev);
2825 int size, desc_len;
2826
2827 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
2828 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
2829 if (!rx_ring->rx_buffer_info)
2830 rx_ring->rx_buffer_info = vzalloc(size);
2831 if (!rx_ring->rx_buffer_info)
2832 goto err;
2833
2834 desc_len = sizeof(union e1000_adv_rx_desc);
2835
2836 /* Round up to nearest 4K */
2837 rx_ring->size = rx_ring->count * desc_len;
2838 rx_ring->size = ALIGN(rx_ring->size, 4096);
2839
2840 set_dev_node(dev, rx_ring->numa_node);
2841 rx_ring->desc = dma_alloc_coherent(dev,
2842 rx_ring->size,
2843 &rx_ring->dma,
2844 GFP_KERNEL);
2845 set_dev_node(dev, orig_node);
2846 if (!rx_ring->desc)
2847 rx_ring->desc = dma_alloc_coherent(dev,
2848 rx_ring->size,
2849 &rx_ring->dma,
2850 GFP_KERNEL);
2851
2852 if (!rx_ring->desc)
2853 goto err;
2854
2855 rx_ring->next_to_clean = 0;
2856 rx_ring->next_to_use = 0;
2857
2858 return 0;
2859
2860 err:
2861 vfree(rx_ring->rx_buffer_info);
2862 rx_ring->rx_buffer_info = NULL;
2863 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2864 " ring\n");
2865 return -ENOMEM;
2866 }
2867
2868 /**
2869 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2870 * (Descriptors) for all queues
2871 * @adapter: board private structure
2872 *
2873 * Return 0 on success, negative on failure
2874 **/
2875 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2876 {
2877 struct pci_dev *pdev = adapter->pdev;
2878 int i, err = 0;
2879
2880 for (i = 0; i < adapter->num_rx_queues; i++) {
2881 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2882 if (err) {
2883 dev_err(&pdev->dev,
2884 "Allocation for Rx Queue %u failed\n", i);
2885 for (i--; i >= 0; i--)
2886 igb_free_rx_resources(adapter->rx_ring[i]);
2887 break;
2888 }
2889 }
2890
2891 return err;
2892 }
2893
2894 /**
2895 * igb_setup_mrqc - configure the multiple receive queue control registers
2896 * @adapter: Board private structure
2897 **/
2898 static void igb_setup_mrqc(struct igb_adapter *adapter)
2899 {
2900 struct e1000_hw *hw = &adapter->hw;
2901 u32 mrqc, rxcsum;
2902 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2903 union e1000_reta {
2904 u32 dword;
2905 u8 bytes[4];
2906 } reta;
2907 static const u8 rsshash[40] = {
2908 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2909 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2910 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2911 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2912
2913 /* Fill out hash function seeds */
2914 for (j = 0; j < 10; j++) {
2915 u32 rsskey = rsshash[(j * 4)];
2916 rsskey |= rsshash[(j * 4) + 1] << 8;
2917 rsskey |= rsshash[(j * 4) + 2] << 16;
2918 rsskey |= rsshash[(j * 4) + 3] << 24;
2919 array_wr32(E1000_RSSRK(0), j, rsskey);
2920 }
2921
2922 num_rx_queues = adapter->rss_queues;
2923
2924 if (adapter->vfs_allocated_count) {
2925 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2926 switch (hw->mac.type) {
2927 case e1000_i350:
2928 case e1000_82580:
2929 num_rx_queues = 1;
2930 shift = 0;
2931 break;
2932 case e1000_82576:
2933 shift = 3;
2934 num_rx_queues = 2;
2935 break;
2936 case e1000_82575:
2937 shift = 2;
2938 shift2 = 6;
2939 default:
2940 break;
2941 }
2942 } else {
2943 if (hw->mac.type == e1000_82575)
2944 shift = 6;
2945 }
2946
2947 for (j = 0; j < (32 * 4); j++) {
2948 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2949 if (shift2)
2950 reta.bytes[j & 3] |= num_rx_queues << shift2;
2951 if ((j & 3) == 3)
2952 wr32(E1000_RETA(j >> 2), reta.dword);
2953 }
2954
2955 /*
2956 * Disable raw packet checksumming so that RSS hash is placed in
2957 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2958 * offloads as they are enabled by default
2959 */
2960 rxcsum = rd32(E1000_RXCSUM);
2961 rxcsum |= E1000_RXCSUM_PCSD;
2962
2963 if (adapter->hw.mac.type >= e1000_82576)
2964 /* Enable Receive Checksum Offload for SCTP */
2965 rxcsum |= E1000_RXCSUM_CRCOFL;
2966
2967 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2968 wr32(E1000_RXCSUM, rxcsum);
2969 /*
2970 * Generate RSS hash based on TCP port numbers and/or
2971 * IPv4/v6 src and dst addresses since UDP cannot be
2972 * hashed reliably due to IP fragmentation
2973 */
2974
2975 mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
2976 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2977 E1000_MRQC_RSS_FIELD_IPV6 |
2978 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2979 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
2980
2981 /* If VMDq is enabled then we set the appropriate mode for that, else
2982 * we default to RSS so that an RSS hash is calculated per packet even
2983 * if we are only using one queue */
2984 if (adapter->vfs_allocated_count) {
2985 if (hw->mac.type > e1000_82575) {
2986 /* Set the default pool for the PF's first queue */
2987 u32 vtctl = rd32(E1000_VT_CTL);
2988 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2989 E1000_VT_CTL_DISABLE_DEF_POOL);
2990 vtctl |= adapter->vfs_allocated_count <<
2991 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2992 wr32(E1000_VT_CTL, vtctl);
2993 }
2994 if (adapter->rss_queues > 1)
2995 mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2996 else
2997 mrqc |= E1000_MRQC_ENABLE_VMDQ;
2998 } else {
2999 if (hw->mac.type != e1000_i211)
3000 mrqc |= E1000_MRQC_ENABLE_RSS_4Q;
3001 }
3002 igb_vmm_control(adapter);
3003
3004 wr32(E1000_MRQC, mrqc);
3005 }
3006
3007 /**
3008 * igb_setup_rctl - configure the receive control registers
3009 * @adapter: Board private structure
3010 **/
3011 void igb_setup_rctl(struct igb_adapter *adapter)
3012 {
3013 struct e1000_hw *hw = &adapter->hw;
3014 u32 rctl;
3015
3016 rctl = rd32(E1000_RCTL);
3017
3018 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3019 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
3020
3021 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
3022 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3023
3024 /*
3025 * enable stripping of CRC. It's unlikely this will break BMC
3026 * redirection as it did with e1000. Newer features require
3027 * that the HW strips the CRC.
3028 */
3029 rctl |= E1000_RCTL_SECRC;
3030
3031 /* disable store bad packets and clear size bits. */
3032 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
3033
3034 /* enable LPE to prevent packets larger than max_frame_size */
3035 rctl |= E1000_RCTL_LPE;
3036
3037 /* disable queue 0 to prevent tail write w/o re-config */
3038 wr32(E1000_RXDCTL(0), 0);
3039
3040 /* Attention!!! For SR-IOV PF driver operations you must enable
3041 * queue drop for all VF and PF queues to prevent head of line blocking
3042 * if an un-trusted VF does not provide descriptors to hardware.
3043 */
3044 if (adapter->vfs_allocated_count) {
3045 /* set all queue drop enable bits */
3046 wr32(E1000_QDE, ALL_QUEUES);
3047 }
3048
3049 /* This is useful for sniffing bad packets. */
3050 if (adapter->netdev->features & NETIF_F_RXALL) {
3051 /* UPE and MPE will be handled by normal PROMISC logic
3052 * in e1000e_set_rx_mode */
3053 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
3054 E1000_RCTL_BAM | /* RX All Bcast Pkts */
3055 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
3056
3057 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
3058 E1000_RCTL_DPF | /* Allow filtered pause */
3059 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
3060 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3061 * and that breaks VLANs.
3062 */
3063 }
3064
3065 wr32(E1000_RCTL, rctl);
3066 }
3067
3068 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3069 int vfn)
3070 {
3071 struct e1000_hw *hw = &adapter->hw;
3072 u32 vmolr;
3073
3074 /* if it isn't the PF check to see if VFs are enabled and
3075 * increase the size to support vlan tags */
3076 if (vfn < adapter->vfs_allocated_count &&
3077 adapter->vf_data[vfn].vlans_enabled)
3078 size += VLAN_TAG_SIZE;
3079
3080 vmolr = rd32(E1000_VMOLR(vfn));
3081 vmolr &= ~E1000_VMOLR_RLPML_MASK;
3082 vmolr |= size | E1000_VMOLR_LPE;
3083 wr32(E1000_VMOLR(vfn), vmolr);
3084
3085 return 0;
3086 }
3087
3088 /**
3089 * igb_rlpml_set - set maximum receive packet size
3090 * @adapter: board private structure
3091 *
3092 * Configure maximum receivable packet size.
3093 **/
3094 static void igb_rlpml_set(struct igb_adapter *adapter)
3095 {
3096 u32 max_frame_size = adapter->max_frame_size;
3097 struct e1000_hw *hw = &adapter->hw;
3098 u16 pf_id = adapter->vfs_allocated_count;
3099
3100 if (pf_id) {
3101 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
3102 /*
3103 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3104 * to our max jumbo frame size, in case we need to enable
3105 * jumbo frames on one of the rings later.
3106 * This will not pass over-length frames into the default
3107 * queue because it's gated by the VMOLR.RLPML.
3108 */
3109 max_frame_size = MAX_JUMBO_FRAME_SIZE;
3110 }
3111
3112 wr32(E1000_RLPML, max_frame_size);
3113 }
3114
3115 static inline void igb_set_vmolr(struct igb_adapter *adapter,
3116 int vfn, bool aupe)
3117 {
3118 struct e1000_hw *hw = &adapter->hw;
3119 u32 vmolr;
3120
3121 /*
3122 * This register exists only on 82576 and newer so if we are older then
3123 * we should exit and do nothing
3124 */
3125 if (hw->mac.type < e1000_82576)
3126 return;
3127
3128 vmolr = rd32(E1000_VMOLR(vfn));
3129 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3130 if (aupe)
3131 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3132 else
3133 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3134
3135 /* clear all bits that might not be set */
3136 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3137
3138 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3139 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3140 /*
3141 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3142 * multicast packets
3143 */
3144 if (vfn <= adapter->vfs_allocated_count)
3145 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3146
3147 wr32(E1000_VMOLR(vfn), vmolr);
3148 }
3149
3150 /**
3151 * igb_configure_rx_ring - Configure a receive ring after Reset
3152 * @adapter: board private structure
3153 * @ring: receive ring to be configured
3154 *
3155 * Configure the Rx unit of the MAC after a reset.
3156 **/
3157 void igb_configure_rx_ring(struct igb_adapter *adapter,
3158 struct igb_ring *ring)
3159 {
3160 struct e1000_hw *hw = &adapter->hw;
3161 u64 rdba = ring->dma;
3162 int reg_idx = ring->reg_idx;
3163 u32 srrctl = 0, rxdctl = 0;
3164
3165 /* disable the queue */
3166 wr32(E1000_RXDCTL(reg_idx), 0);
3167
3168 /* Set DMA base address registers */
3169 wr32(E1000_RDBAL(reg_idx),
3170 rdba & 0x00000000ffffffffULL);
3171 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3172 wr32(E1000_RDLEN(reg_idx),
3173 ring->count * sizeof(union e1000_adv_rx_desc));
3174
3175 /* initialize head and tail */
3176 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3177 wr32(E1000_RDH(reg_idx), 0);
3178 writel(0, ring->tail);
3179
3180 /* set descriptor configuration */
3181 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3182 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3183 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3184 #else
3185 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3186 #endif
3187 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3188 #ifdef CONFIG_IGB_PTP
3189 if (hw->mac.type >= e1000_82580)
3190 srrctl |= E1000_SRRCTL_TIMESTAMP;
3191 #endif /* CONFIG_IGB_PTP */
3192 /* Only set Drop Enable if we are supporting multiple queues */
3193 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3194 srrctl |= E1000_SRRCTL_DROP_EN;
3195
3196 wr32(E1000_SRRCTL(reg_idx), srrctl);
3197
3198 /* set filtering for VMDQ pools */
3199 igb_set_vmolr(adapter, reg_idx & 0x7, true);
3200
3201 rxdctl |= IGB_RX_PTHRESH;
3202 rxdctl |= IGB_RX_HTHRESH << 8;
3203 rxdctl |= IGB_RX_WTHRESH << 16;
3204
3205 /* enable receive descriptor fetching */
3206 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3207 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3208 }
3209
3210 /**
3211 * igb_configure_rx - Configure receive Unit after Reset
3212 * @adapter: board private structure
3213 *
3214 * Configure the Rx unit of the MAC after a reset.
3215 **/
3216 static void igb_configure_rx(struct igb_adapter *adapter)
3217 {
3218 int i;
3219
3220 /* set UTA to appropriate mode */
3221 igb_set_uta(adapter);
3222
3223 /* set the correct pool for the PF default MAC address in entry 0 */
3224 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3225 adapter->vfs_allocated_count);
3226
3227 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3228 * the Base and Length of the Rx Descriptor Ring */
3229 for (i = 0; i < adapter->num_rx_queues; i++)
3230 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
3231 }
3232
3233 /**
3234 * igb_free_tx_resources - Free Tx Resources per Queue
3235 * @tx_ring: Tx descriptor ring for a specific queue
3236 *
3237 * Free all transmit software resources
3238 **/
3239 void igb_free_tx_resources(struct igb_ring *tx_ring)
3240 {
3241 igb_clean_tx_ring(tx_ring);
3242
3243 vfree(tx_ring->tx_buffer_info);
3244 tx_ring->tx_buffer_info = NULL;
3245
3246 /* if not set, then don't free */
3247 if (!tx_ring->desc)
3248 return;
3249
3250 dma_free_coherent(tx_ring->dev, tx_ring->size,
3251 tx_ring->desc, tx_ring->dma);
3252
3253 tx_ring->desc = NULL;
3254 }
3255
3256 /**
3257 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3258 * @adapter: board private structure
3259 *
3260 * Free all transmit software resources
3261 **/
3262 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3263 {
3264 int i;
3265
3266 for (i = 0; i < adapter->num_tx_queues; i++)
3267 igb_free_tx_resources(adapter->tx_ring[i]);
3268 }
3269
3270 void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3271 struct igb_tx_buffer *tx_buffer)
3272 {
3273 if (tx_buffer->skb) {
3274 dev_kfree_skb_any(tx_buffer->skb);
3275 if (tx_buffer->dma)
3276 dma_unmap_single(ring->dev,
3277 tx_buffer->dma,
3278 tx_buffer->length,
3279 DMA_TO_DEVICE);
3280 } else if (tx_buffer->dma) {
3281 dma_unmap_page(ring->dev,
3282 tx_buffer->dma,
3283 tx_buffer->length,
3284 DMA_TO_DEVICE);
3285 }
3286 tx_buffer->next_to_watch = NULL;
3287 tx_buffer->skb = NULL;
3288 tx_buffer->dma = 0;
3289 /* buffer_info must be completely set up in the transmit path */
3290 }
3291
3292 /**
3293 * igb_clean_tx_ring - Free Tx Buffers
3294 * @tx_ring: ring to be cleaned
3295 **/
3296 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3297 {
3298 struct igb_tx_buffer *buffer_info;
3299 unsigned long size;
3300 u16 i;
3301
3302 if (!tx_ring->tx_buffer_info)
3303 return;
3304 /* Free all the Tx ring sk_buffs */
3305
3306 for (i = 0; i < tx_ring->count; i++) {
3307 buffer_info = &tx_ring->tx_buffer_info[i];
3308 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3309 }
3310
3311 netdev_tx_reset_queue(txring_txq(tx_ring));
3312
3313 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3314 memset(tx_ring->tx_buffer_info, 0, size);
3315
3316 /* Zero out the descriptor ring */
3317 memset(tx_ring->desc, 0, tx_ring->size);
3318
3319 tx_ring->next_to_use = 0;
3320 tx_ring->next_to_clean = 0;
3321 }
3322
3323 /**
3324 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3325 * @adapter: board private structure
3326 **/
3327 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3328 {
3329 int i;
3330
3331 for (i = 0; i < adapter->num_tx_queues; i++)
3332 igb_clean_tx_ring(adapter->tx_ring[i]);
3333 }
3334
3335 /**
3336 * igb_free_rx_resources - Free Rx Resources
3337 * @rx_ring: ring to clean the resources from
3338 *
3339 * Free all receive software resources
3340 **/
3341 void igb_free_rx_resources(struct igb_ring *rx_ring)
3342 {
3343 igb_clean_rx_ring(rx_ring);
3344
3345 vfree(rx_ring->rx_buffer_info);
3346 rx_ring->rx_buffer_info = NULL;
3347
3348 /* if not set, then don't free */
3349 if (!rx_ring->desc)
3350 return;
3351
3352 dma_free_coherent(rx_ring->dev, rx_ring->size,
3353 rx_ring->desc, rx_ring->dma);
3354
3355 rx_ring->desc = NULL;
3356 }
3357
3358 /**
3359 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3360 * @adapter: board private structure
3361 *
3362 * Free all receive software resources
3363 **/
3364 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3365 {
3366 int i;
3367
3368 for (i = 0; i < adapter->num_rx_queues; i++)
3369 igb_free_rx_resources(adapter->rx_ring[i]);
3370 }
3371
3372 /**
3373 * igb_clean_rx_ring - Free Rx Buffers per Queue
3374 * @rx_ring: ring to free buffers from
3375 **/
3376 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3377 {
3378 unsigned long size;
3379 u16 i;
3380
3381 if (!rx_ring->rx_buffer_info)
3382 return;
3383
3384 /* Free all the Rx ring sk_buffs */
3385 for (i = 0; i < rx_ring->count; i++) {
3386 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
3387 if (buffer_info->dma) {
3388 dma_unmap_single(rx_ring->dev,
3389 buffer_info->dma,
3390 IGB_RX_HDR_LEN,
3391 DMA_FROM_DEVICE);
3392 buffer_info->dma = 0;
3393 }
3394
3395 if (buffer_info->skb) {
3396 dev_kfree_skb(buffer_info->skb);
3397 buffer_info->skb = NULL;
3398 }
3399 if (buffer_info->page_dma) {
3400 dma_unmap_page(rx_ring->dev,
3401 buffer_info->page_dma,
3402 PAGE_SIZE / 2,
3403 DMA_FROM_DEVICE);
3404 buffer_info->page_dma = 0;
3405 }
3406 if (buffer_info->page) {
3407 put_page(buffer_info->page);
3408 buffer_info->page = NULL;
3409 buffer_info->page_offset = 0;
3410 }
3411 }
3412
3413 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3414 memset(rx_ring->rx_buffer_info, 0, size);
3415
3416 /* Zero out the descriptor ring */
3417 memset(rx_ring->desc, 0, rx_ring->size);
3418
3419 rx_ring->next_to_clean = 0;
3420 rx_ring->next_to_use = 0;
3421 }
3422
3423 /**
3424 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3425 * @adapter: board private structure
3426 **/
3427 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3428 {
3429 int i;
3430
3431 for (i = 0; i < adapter->num_rx_queues; i++)
3432 igb_clean_rx_ring(adapter->rx_ring[i]);
3433 }
3434
3435 /**
3436 * igb_set_mac - Change the Ethernet Address of the NIC
3437 * @netdev: network interface device structure
3438 * @p: pointer to an address structure
3439 *
3440 * Returns 0 on success, negative on failure
3441 **/
3442 static int igb_set_mac(struct net_device *netdev, void *p)
3443 {
3444 struct igb_adapter *adapter = netdev_priv(netdev);
3445 struct e1000_hw *hw = &adapter->hw;
3446 struct sockaddr *addr = p;
3447
3448 if (!is_valid_ether_addr(addr->sa_data))
3449 return -EADDRNOTAVAIL;
3450
3451 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3452 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3453
3454 /* set the correct pool for the new PF MAC address in entry 0 */
3455 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3456 adapter->vfs_allocated_count);
3457
3458 return 0;
3459 }
3460
3461 /**
3462 * igb_write_mc_addr_list - write multicast addresses to MTA
3463 * @netdev: network interface device structure
3464 *
3465 * Writes multicast address list to the MTA hash table.
3466 * Returns: -ENOMEM on failure
3467 * 0 on no addresses written
3468 * X on writing X addresses to MTA
3469 **/
3470 static int igb_write_mc_addr_list(struct net_device *netdev)
3471 {
3472 struct igb_adapter *adapter = netdev_priv(netdev);
3473 struct e1000_hw *hw = &adapter->hw;
3474 struct netdev_hw_addr *ha;
3475 u8 *mta_list;
3476 int i;
3477
3478 if (netdev_mc_empty(netdev)) {
3479 /* nothing to program, so clear mc list */
3480 igb_update_mc_addr_list(hw, NULL, 0);
3481 igb_restore_vf_multicasts(adapter);
3482 return 0;
3483 }
3484
3485 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3486 if (!mta_list)
3487 return -ENOMEM;
3488
3489 /* The shared function expects a packed array of only addresses. */
3490 i = 0;
3491 netdev_for_each_mc_addr(ha, netdev)
3492 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3493
3494 igb_update_mc_addr_list(hw, mta_list, i);
3495 kfree(mta_list);
3496
3497 return netdev_mc_count(netdev);
3498 }
3499
3500 /**
3501 * igb_write_uc_addr_list - write unicast addresses to RAR table
3502 * @netdev: network interface device structure
3503 *
3504 * Writes unicast address list to the RAR table.
3505 * Returns: -ENOMEM on failure/insufficient address space
3506 * 0 on no addresses written
3507 * X on writing X addresses to the RAR table
3508 **/
3509 static int igb_write_uc_addr_list(struct net_device *netdev)
3510 {
3511 struct igb_adapter *adapter = netdev_priv(netdev);
3512 struct e1000_hw *hw = &adapter->hw;
3513 unsigned int vfn = adapter->vfs_allocated_count;
3514 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3515 int count = 0;
3516
3517 /* return ENOMEM indicating insufficient memory for addresses */
3518 if (netdev_uc_count(netdev) > rar_entries)
3519 return -ENOMEM;
3520
3521 if (!netdev_uc_empty(netdev) && rar_entries) {
3522 struct netdev_hw_addr *ha;
3523
3524 netdev_for_each_uc_addr(ha, netdev) {
3525 if (!rar_entries)
3526 break;
3527 igb_rar_set_qsel(adapter, ha->addr,
3528 rar_entries--,
3529 vfn);
3530 count++;
3531 }
3532 }
3533 /* write the addresses in reverse order to avoid write combining */
3534 for (; rar_entries > 0 ; rar_entries--) {
3535 wr32(E1000_RAH(rar_entries), 0);
3536 wr32(E1000_RAL(rar_entries), 0);
3537 }
3538 wrfl();
3539
3540 return count;
3541 }
3542
3543 /**
3544 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3545 * @netdev: network interface device structure
3546 *
3547 * The set_rx_mode entry point is called whenever the unicast or multicast
3548 * address lists or the network interface flags are updated. This routine is
3549 * responsible for configuring the hardware for proper unicast, multicast,
3550 * promiscuous mode, and all-multi behavior.
3551 **/
3552 static void igb_set_rx_mode(struct net_device *netdev)
3553 {
3554 struct igb_adapter *adapter = netdev_priv(netdev);
3555 struct e1000_hw *hw = &adapter->hw;
3556 unsigned int vfn = adapter->vfs_allocated_count;
3557 u32 rctl, vmolr = 0;
3558 int count;
3559
3560 /* Check for Promiscuous and All Multicast modes */
3561 rctl = rd32(E1000_RCTL);
3562
3563 /* clear the effected bits */
3564 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3565
3566 if (netdev->flags & IFF_PROMISC) {
3567 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3568 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3569 } else {
3570 if (netdev->flags & IFF_ALLMULTI) {
3571 rctl |= E1000_RCTL_MPE;
3572 vmolr |= E1000_VMOLR_MPME;
3573 } else {
3574 /*
3575 * Write addresses to the MTA, if the attempt fails
3576 * then we should just turn on promiscuous mode so
3577 * that we can at least receive multicast traffic
3578 */
3579 count = igb_write_mc_addr_list(netdev);
3580 if (count < 0) {
3581 rctl |= E1000_RCTL_MPE;
3582 vmolr |= E1000_VMOLR_MPME;
3583 } else if (count) {
3584 vmolr |= E1000_VMOLR_ROMPE;
3585 }
3586 }
3587 /*
3588 * Write addresses to available RAR registers, if there is not
3589 * sufficient space to store all the addresses then enable
3590 * unicast promiscuous mode
3591 */
3592 count = igb_write_uc_addr_list(netdev);
3593 if (count < 0) {
3594 rctl |= E1000_RCTL_UPE;
3595 vmolr |= E1000_VMOLR_ROPE;
3596 }
3597 rctl |= E1000_RCTL_VFE;
3598 }
3599 wr32(E1000_RCTL, rctl);
3600
3601 /*
3602 * In order to support SR-IOV and eventually VMDq it is necessary to set
3603 * the VMOLR to enable the appropriate modes. Without this workaround
3604 * we will have issues with VLAN tag stripping not being done for frames
3605 * that are only arriving because we are the default pool
3606 */
3607 if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
3608 return;
3609
3610 vmolr |= rd32(E1000_VMOLR(vfn)) &
3611 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3612 wr32(E1000_VMOLR(vfn), vmolr);
3613 igb_restore_vf_multicasts(adapter);
3614 }
3615
3616 static void igb_check_wvbr(struct igb_adapter *adapter)
3617 {
3618 struct e1000_hw *hw = &adapter->hw;
3619 u32 wvbr = 0;
3620
3621 switch (hw->mac.type) {
3622 case e1000_82576:
3623 case e1000_i350:
3624 if (!(wvbr = rd32(E1000_WVBR)))
3625 return;
3626 break;
3627 default:
3628 break;
3629 }
3630
3631 adapter->wvbr |= wvbr;
3632 }
3633
3634 #define IGB_STAGGERED_QUEUE_OFFSET 8
3635
3636 static void igb_spoof_check(struct igb_adapter *adapter)
3637 {
3638 int j;
3639
3640 if (!adapter->wvbr)
3641 return;
3642
3643 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3644 if (adapter->wvbr & (1 << j) ||
3645 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3646 dev_warn(&adapter->pdev->dev,
3647 "Spoof event(s) detected on VF %d\n", j);
3648 adapter->wvbr &=
3649 ~((1 << j) |
3650 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3651 }
3652 }
3653 }
3654
3655 /* Need to wait a few seconds after link up to get diagnostic information from
3656 * the phy */
3657 static void igb_update_phy_info(unsigned long data)
3658 {
3659 struct igb_adapter *adapter = (struct igb_adapter *) data;
3660 igb_get_phy_info(&adapter->hw);
3661 }
3662
3663 /**
3664 * igb_has_link - check shared code for link and determine up/down
3665 * @adapter: pointer to driver private info
3666 **/
3667 bool igb_has_link(struct igb_adapter *adapter)
3668 {
3669 struct e1000_hw *hw = &adapter->hw;
3670 bool link_active = false;
3671 s32 ret_val = 0;
3672
3673 /* get_link_status is set on LSC (link status) interrupt or
3674 * rx sequence error interrupt. get_link_status will stay
3675 * false until the e1000_check_for_link establishes link
3676 * for copper adapters ONLY
3677 */
3678 switch (hw->phy.media_type) {
3679 case e1000_media_type_copper:
3680 if (hw->mac.get_link_status) {
3681 ret_val = hw->mac.ops.check_for_link(hw);
3682 link_active = !hw->mac.get_link_status;
3683 } else {
3684 link_active = true;
3685 }
3686 break;
3687 case e1000_media_type_internal_serdes:
3688 ret_val = hw->mac.ops.check_for_link(hw);
3689 link_active = hw->mac.serdes_has_link;
3690 break;
3691 default:
3692 case e1000_media_type_unknown:
3693 break;
3694 }
3695
3696 return link_active;
3697 }
3698
3699 static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3700 {
3701 bool ret = false;
3702 u32 ctrl_ext, thstat;
3703
3704 /* check for thermal sensor event on i350 copper only */
3705 if (hw->mac.type == e1000_i350) {
3706 thstat = rd32(E1000_THSTAT);
3707 ctrl_ext = rd32(E1000_CTRL_EXT);
3708
3709 if ((hw->phy.media_type == e1000_media_type_copper) &&
3710 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3711 ret = !!(thstat & event);
3712 }
3713 }
3714
3715 return ret;
3716 }
3717
3718 /**
3719 * igb_watchdog - Timer Call-back
3720 * @data: pointer to adapter cast into an unsigned long
3721 **/
3722 static void igb_watchdog(unsigned long data)
3723 {
3724 struct igb_adapter *adapter = (struct igb_adapter *)data;
3725 /* Do the rest outside of interrupt context */
3726 schedule_work(&adapter->watchdog_task);
3727 }
3728
3729 static void igb_watchdog_task(struct work_struct *work)
3730 {
3731 struct igb_adapter *adapter = container_of(work,
3732 struct igb_adapter,
3733 watchdog_task);
3734 struct e1000_hw *hw = &adapter->hw;
3735 struct net_device *netdev = adapter->netdev;
3736 u32 link;
3737 int i;
3738
3739 link = igb_has_link(adapter);
3740 if (link) {
3741 /* Cancel scheduled suspend requests. */
3742 pm_runtime_resume(netdev->dev.parent);
3743
3744 if (!netif_carrier_ok(netdev)) {
3745 u32 ctrl;
3746 hw->mac.ops.get_speed_and_duplex(hw,
3747 &adapter->link_speed,
3748 &adapter->link_duplex);
3749
3750 ctrl = rd32(E1000_CTRL);
3751 /* Links status message must follow this format */
3752 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s "
3753 "Duplex, Flow Control: %s\n",
3754 netdev->name,
3755 adapter->link_speed,
3756 adapter->link_duplex == FULL_DUPLEX ?
3757 "Full" : "Half",
3758 (ctrl & E1000_CTRL_TFCE) &&
3759 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
3760 (ctrl & E1000_CTRL_RFCE) ? "RX" :
3761 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None");
3762
3763 /* check for thermal sensor event */
3764 if (igb_thermal_sensor_event(hw,
3765 E1000_THSTAT_LINK_THROTTLE)) {
3766 netdev_info(netdev, "The network adapter link "
3767 "speed was downshifted because it "
3768 "overheated\n");
3769 }
3770
3771 /* adjust timeout factor according to speed/duplex */
3772 adapter->tx_timeout_factor = 1;
3773 switch (adapter->link_speed) {
3774 case SPEED_10:
3775 adapter->tx_timeout_factor = 14;
3776 break;
3777 case SPEED_100:
3778 /* maybe add some timeout factor ? */
3779 break;
3780 }
3781
3782 netif_carrier_on(netdev);
3783
3784 igb_ping_all_vfs(adapter);
3785 igb_check_vf_rate_limit(adapter);
3786
3787 /* link state has changed, schedule phy info update */
3788 if (!test_bit(__IGB_DOWN, &adapter->state))
3789 mod_timer(&adapter->phy_info_timer,
3790 round_jiffies(jiffies + 2 * HZ));
3791 }
3792 } else {
3793 if (netif_carrier_ok(netdev)) {
3794 adapter->link_speed = 0;
3795 adapter->link_duplex = 0;
3796
3797 /* check for thermal sensor event */
3798 if (igb_thermal_sensor_event(hw,
3799 E1000_THSTAT_PWR_DOWN)) {
3800 netdev_err(netdev, "The network adapter was "
3801 "stopped because it overheated\n");
3802 }
3803
3804 /* Links status message must follow this format */
3805 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3806 netdev->name);
3807 netif_carrier_off(netdev);
3808
3809 igb_ping_all_vfs(adapter);
3810
3811 /* link state has changed, schedule phy info update */
3812 if (!test_bit(__IGB_DOWN, &adapter->state))
3813 mod_timer(&adapter->phy_info_timer,
3814 round_jiffies(jiffies + 2 * HZ));
3815
3816 pm_schedule_suspend(netdev->dev.parent,
3817 MSEC_PER_SEC * 5);
3818 }
3819 }
3820
3821 spin_lock(&adapter->stats64_lock);
3822 igb_update_stats(adapter, &adapter->stats64);
3823 spin_unlock(&adapter->stats64_lock);
3824
3825 for (i = 0; i < adapter->num_tx_queues; i++) {
3826 struct igb_ring *tx_ring = adapter->tx_ring[i];
3827 if (!netif_carrier_ok(netdev)) {
3828 /* We've lost link, so the controller stops DMA,
3829 * but we've got queued Tx work that's never going
3830 * to get done, so reset controller to flush Tx.
3831 * (Do the reset outside of interrupt context). */
3832 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3833 adapter->tx_timeout_count++;
3834 schedule_work(&adapter->reset_task);
3835 /* return immediately since reset is imminent */
3836 return;
3837 }
3838 }
3839
3840 /* Force detection of hung controller every watchdog period */
3841 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
3842 }
3843
3844 /* Cause software interrupt to ensure rx ring is cleaned */
3845 if (adapter->msix_entries) {
3846 u32 eics = 0;
3847 for (i = 0; i < adapter->num_q_vectors; i++)
3848 eics |= adapter->q_vector[i]->eims_value;
3849 wr32(E1000_EICS, eics);
3850 } else {
3851 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3852 }
3853
3854 igb_spoof_check(adapter);
3855
3856 /* Reset the timer */
3857 if (!test_bit(__IGB_DOWN, &adapter->state))
3858 mod_timer(&adapter->watchdog_timer,
3859 round_jiffies(jiffies + 2 * HZ));
3860 }
3861
3862 enum latency_range {
3863 lowest_latency = 0,
3864 low_latency = 1,
3865 bulk_latency = 2,
3866 latency_invalid = 255
3867 };
3868
3869 /**
3870 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3871 *
3872 * Stores a new ITR value based on strictly on packet size. This
3873 * algorithm is less sophisticated than that used in igb_update_itr,
3874 * due to the difficulty of synchronizing statistics across multiple
3875 * receive rings. The divisors and thresholds used by this function
3876 * were determined based on theoretical maximum wire speed and testing
3877 * data, in order to minimize response time while increasing bulk
3878 * throughput.
3879 * This functionality is controlled by the InterruptThrottleRate module
3880 * parameter (see igb_param.c)
3881 * NOTE: This function is called only when operating in a multiqueue
3882 * receive environment.
3883 * @q_vector: pointer to q_vector
3884 **/
3885 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3886 {
3887 int new_val = q_vector->itr_val;
3888 int avg_wire_size = 0;
3889 struct igb_adapter *adapter = q_vector->adapter;
3890 unsigned int packets;
3891
3892 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3893 * ints/sec - ITR timer value of 120 ticks.
3894 */
3895 if (adapter->link_speed != SPEED_1000) {
3896 new_val = IGB_4K_ITR;
3897 goto set_itr_val;
3898 }
3899
3900 packets = q_vector->rx.total_packets;
3901 if (packets)
3902 avg_wire_size = q_vector->rx.total_bytes / packets;
3903
3904 packets = q_vector->tx.total_packets;
3905 if (packets)
3906 avg_wire_size = max_t(u32, avg_wire_size,
3907 q_vector->tx.total_bytes / packets);
3908
3909 /* if avg_wire_size isn't set no work was done */
3910 if (!avg_wire_size)
3911 goto clear_counts;
3912
3913 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3914 avg_wire_size += 24;
3915
3916 /* Don't starve jumbo frames */
3917 avg_wire_size = min(avg_wire_size, 3000);
3918
3919 /* Give a little boost to mid-size frames */
3920 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3921 new_val = avg_wire_size / 3;
3922 else
3923 new_val = avg_wire_size / 2;
3924
3925 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3926 if (new_val < IGB_20K_ITR &&
3927 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3928 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3929 new_val = IGB_20K_ITR;
3930
3931 set_itr_val:
3932 if (new_val != q_vector->itr_val) {
3933 q_vector->itr_val = new_val;
3934 q_vector->set_itr = 1;
3935 }
3936 clear_counts:
3937 q_vector->rx.total_bytes = 0;
3938 q_vector->rx.total_packets = 0;
3939 q_vector->tx.total_bytes = 0;
3940 q_vector->tx.total_packets = 0;
3941 }
3942
3943 /**
3944 * igb_update_itr - update the dynamic ITR value based on statistics
3945 * Stores a new ITR value based on packets and byte
3946 * counts during the last interrupt. The advantage of per interrupt
3947 * computation is faster updates and more accurate ITR for the current
3948 * traffic pattern. Constants in this function were computed
3949 * based on theoretical maximum wire speed and thresholds were set based
3950 * on testing data as well as attempting to minimize response time
3951 * while increasing bulk throughput.
3952 * this functionality is controlled by the InterruptThrottleRate module
3953 * parameter (see igb_param.c)
3954 * NOTE: These calculations are only valid when operating in a single-
3955 * queue environment.
3956 * @q_vector: pointer to q_vector
3957 * @ring_container: ring info to update the itr for
3958 **/
3959 static void igb_update_itr(struct igb_q_vector *q_vector,
3960 struct igb_ring_container *ring_container)
3961 {
3962 unsigned int packets = ring_container->total_packets;
3963 unsigned int bytes = ring_container->total_bytes;
3964 u8 itrval = ring_container->itr;
3965
3966 /* no packets, exit with status unchanged */
3967 if (packets == 0)
3968 return;
3969
3970 switch (itrval) {
3971 case lowest_latency:
3972 /* handle TSO and jumbo frames */
3973 if (bytes/packets > 8000)
3974 itrval = bulk_latency;
3975 else if ((packets < 5) && (bytes > 512))
3976 itrval = low_latency;
3977 break;
3978 case low_latency: /* 50 usec aka 20000 ints/s */
3979 if (bytes > 10000) {
3980 /* this if handles the TSO accounting */
3981 if (bytes/packets > 8000) {
3982 itrval = bulk_latency;
3983 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3984 itrval = bulk_latency;
3985 } else if ((packets > 35)) {
3986 itrval = lowest_latency;
3987 }
3988 } else if (bytes/packets > 2000) {
3989 itrval = bulk_latency;
3990 } else if (packets <= 2 && bytes < 512) {
3991 itrval = lowest_latency;
3992 }
3993 break;
3994 case bulk_latency: /* 250 usec aka 4000 ints/s */
3995 if (bytes > 25000) {
3996 if (packets > 35)
3997 itrval = low_latency;
3998 } else if (bytes < 1500) {
3999 itrval = low_latency;
4000 }
4001 break;
4002 }
4003
4004 /* clear work counters since we have the values we need */
4005 ring_container->total_bytes = 0;
4006 ring_container->total_packets = 0;
4007
4008 /* write updated itr to ring container */
4009 ring_container->itr = itrval;
4010 }
4011
4012 static void igb_set_itr(struct igb_q_vector *q_vector)
4013 {
4014 struct igb_adapter *adapter = q_vector->adapter;
4015 u32 new_itr = q_vector->itr_val;
4016 u8 current_itr = 0;
4017
4018 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4019 if (adapter->link_speed != SPEED_1000) {
4020 current_itr = 0;
4021 new_itr = IGB_4K_ITR;
4022 goto set_itr_now;
4023 }
4024
4025 igb_update_itr(q_vector, &q_vector->tx);
4026 igb_update_itr(q_vector, &q_vector->rx);
4027
4028 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4029
4030 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4031 if (current_itr == lowest_latency &&
4032 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4033 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4034 current_itr = low_latency;
4035
4036 switch (current_itr) {
4037 /* counts and packets in update_itr are dependent on these numbers */
4038 case lowest_latency:
4039 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
4040 break;
4041 case low_latency:
4042 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
4043 break;
4044 case bulk_latency:
4045 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
4046 break;
4047 default:
4048 break;
4049 }
4050
4051 set_itr_now:
4052 if (new_itr != q_vector->itr_val) {
4053 /* this attempts to bias the interrupt rate towards Bulk
4054 * by adding intermediate steps when interrupt rate is
4055 * increasing */
4056 new_itr = new_itr > q_vector->itr_val ?
4057 max((new_itr * q_vector->itr_val) /
4058 (new_itr + (q_vector->itr_val >> 2)),
4059 new_itr) :
4060 new_itr;
4061 /* Don't write the value here; it resets the adapter's
4062 * internal timer, and causes us to delay far longer than
4063 * we should between interrupts. Instead, we write the ITR
4064 * value at the beginning of the next interrupt so the timing
4065 * ends up being correct.
4066 */
4067 q_vector->itr_val = new_itr;
4068 q_vector->set_itr = 1;
4069 }
4070 }
4071
4072 static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4073 u32 type_tucmd, u32 mss_l4len_idx)
4074 {
4075 struct e1000_adv_tx_context_desc *context_desc;
4076 u16 i = tx_ring->next_to_use;
4077
4078 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4079
4080 i++;
4081 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4082
4083 /* set bits to identify this as an advanced context descriptor */
4084 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4085
4086 /* For 82575, context index must be unique per ring. */
4087 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4088 mss_l4len_idx |= tx_ring->reg_idx << 4;
4089
4090 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
4091 context_desc->seqnum_seed = 0;
4092 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
4093 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4094 }
4095
4096 static int igb_tso(struct igb_ring *tx_ring,
4097 struct igb_tx_buffer *first,
4098 u8 *hdr_len)
4099 {
4100 struct sk_buff *skb = first->skb;
4101 u32 vlan_macip_lens, type_tucmd;
4102 u32 mss_l4len_idx, l4len;
4103
4104 if (!skb_is_gso(skb))
4105 return 0;
4106
4107 if (skb_header_cloned(skb)) {
4108 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
4109 if (err)
4110 return err;
4111 }
4112
4113 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4114 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
4115
4116 if (first->protocol == __constant_htons(ETH_P_IP)) {
4117 struct iphdr *iph = ip_hdr(skb);
4118 iph->tot_len = 0;
4119 iph->check = 0;
4120 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4121 iph->daddr, 0,
4122 IPPROTO_TCP,
4123 0);
4124 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4125 first->tx_flags |= IGB_TX_FLAGS_TSO |
4126 IGB_TX_FLAGS_CSUM |
4127 IGB_TX_FLAGS_IPV4;
4128 } else if (skb_is_gso_v6(skb)) {
4129 ipv6_hdr(skb)->payload_len = 0;
4130 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4131 &ipv6_hdr(skb)->daddr,
4132 0, IPPROTO_TCP, 0);
4133 first->tx_flags |= IGB_TX_FLAGS_TSO |
4134 IGB_TX_FLAGS_CSUM;
4135 }
4136
4137 /* compute header lengths */
4138 l4len = tcp_hdrlen(skb);
4139 *hdr_len = skb_transport_offset(skb) + l4len;
4140
4141 /* update gso size and bytecount with header size */
4142 first->gso_segs = skb_shinfo(skb)->gso_segs;
4143 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4144
4145 /* MSS L4LEN IDX */
4146 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4147 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
4148
4149 /* VLAN MACLEN IPLEN */
4150 vlan_macip_lens = skb_network_header_len(skb);
4151 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4152 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4153
4154 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4155
4156 return 1;
4157 }
4158
4159 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
4160 {
4161 struct sk_buff *skb = first->skb;
4162 u32 vlan_macip_lens = 0;
4163 u32 mss_l4len_idx = 0;
4164 u32 type_tucmd = 0;
4165
4166 if (skb->ip_summed != CHECKSUM_PARTIAL) {
4167 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4168 return;
4169 } else {
4170 u8 l4_hdr = 0;
4171 switch (first->protocol) {
4172 case __constant_htons(ETH_P_IP):
4173 vlan_macip_lens |= skb_network_header_len(skb);
4174 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4175 l4_hdr = ip_hdr(skb)->protocol;
4176 break;
4177 case __constant_htons(ETH_P_IPV6):
4178 vlan_macip_lens |= skb_network_header_len(skb);
4179 l4_hdr = ipv6_hdr(skb)->nexthdr;
4180 break;
4181 default:
4182 if (unlikely(net_ratelimit())) {
4183 dev_warn(tx_ring->dev,
4184 "partial checksum but proto=%x!\n",
4185 first->protocol);
4186 }
4187 break;
4188 }
4189
4190 switch (l4_hdr) {
4191 case IPPROTO_TCP:
4192 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4193 mss_l4len_idx = tcp_hdrlen(skb) <<
4194 E1000_ADVTXD_L4LEN_SHIFT;
4195 break;
4196 case IPPROTO_SCTP:
4197 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4198 mss_l4len_idx = sizeof(struct sctphdr) <<
4199 E1000_ADVTXD_L4LEN_SHIFT;
4200 break;
4201 case IPPROTO_UDP:
4202 mss_l4len_idx = sizeof(struct udphdr) <<
4203 E1000_ADVTXD_L4LEN_SHIFT;
4204 break;
4205 default:
4206 if (unlikely(net_ratelimit())) {
4207 dev_warn(tx_ring->dev,
4208 "partial checksum but l4 proto=%x!\n",
4209 l4_hdr);
4210 }
4211 break;
4212 }
4213
4214 /* update TX checksum flag */
4215 first->tx_flags |= IGB_TX_FLAGS_CSUM;
4216 }
4217
4218 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4219 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4220
4221 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4222 }
4223
4224 static __le32 igb_tx_cmd_type(u32 tx_flags)
4225 {
4226 /* set type for advanced descriptor with frame checksum insertion */
4227 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4228 E1000_ADVTXD_DCMD_IFCS |
4229 E1000_ADVTXD_DCMD_DEXT);
4230
4231 /* set HW vlan bit if vlan is present */
4232 if (tx_flags & IGB_TX_FLAGS_VLAN)
4233 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4234
4235 #ifdef CONFIG_IGB_PTP
4236 /* set timestamp bit if present */
4237 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4238 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4239 #endif /* CONFIG_IGB_PTP */
4240
4241 /* set segmentation bits for TSO */
4242 if (tx_flags & IGB_TX_FLAGS_TSO)
4243 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4244
4245 return cmd_type;
4246 }
4247
4248 static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4249 union e1000_adv_tx_desc *tx_desc,
4250 u32 tx_flags, unsigned int paylen)
4251 {
4252 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4253
4254 /* 82575 requires a unique index per ring if any offload is enabled */
4255 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
4256 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4257 olinfo_status |= tx_ring->reg_idx << 4;
4258
4259 /* insert L4 checksum */
4260 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4261 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4262
4263 /* insert IPv4 checksum */
4264 if (tx_flags & IGB_TX_FLAGS_IPV4)
4265 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4266 }
4267
4268 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4269 }
4270
4271 /*
4272 * The largest size we can write to the descriptor is 65535. In order to
4273 * maintain a power of two alignment we have to limit ourselves to 32K.
4274 */
4275 #define IGB_MAX_TXD_PWR 15
4276 #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
4277
4278 static void igb_tx_map(struct igb_ring *tx_ring,
4279 struct igb_tx_buffer *first,
4280 const u8 hdr_len)
4281 {
4282 struct sk_buff *skb = first->skb;
4283 struct igb_tx_buffer *tx_buffer_info;
4284 union e1000_adv_tx_desc *tx_desc;
4285 dma_addr_t dma;
4286 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4287 unsigned int data_len = skb->data_len;
4288 unsigned int size = skb_headlen(skb);
4289 unsigned int paylen = skb->len - hdr_len;
4290 __le32 cmd_type;
4291 u32 tx_flags = first->tx_flags;
4292 u16 i = tx_ring->next_to_use;
4293
4294 tx_desc = IGB_TX_DESC(tx_ring, i);
4295
4296 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
4297 cmd_type = igb_tx_cmd_type(tx_flags);
4298
4299 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4300 if (dma_mapping_error(tx_ring->dev, dma))
4301 goto dma_error;
4302
4303 /* record length, and DMA address */
4304 first->length = size;
4305 first->dma = dma;
4306 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4307
4308 for (;;) {
4309 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4310 tx_desc->read.cmd_type_len =
4311 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4312
4313 i++;
4314 tx_desc++;
4315 if (i == tx_ring->count) {
4316 tx_desc = IGB_TX_DESC(tx_ring, 0);
4317 i = 0;
4318 }
4319
4320 dma += IGB_MAX_DATA_PER_TXD;
4321 size -= IGB_MAX_DATA_PER_TXD;
4322
4323 tx_desc->read.olinfo_status = 0;
4324 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4325 }
4326
4327 if (likely(!data_len))
4328 break;
4329
4330 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
4331
4332 i++;
4333 tx_desc++;
4334 if (i == tx_ring->count) {
4335 tx_desc = IGB_TX_DESC(tx_ring, 0);
4336 i = 0;
4337 }
4338
4339 size = skb_frag_size(frag);
4340 data_len -= size;
4341
4342 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4343 size, DMA_TO_DEVICE);
4344 if (dma_mapping_error(tx_ring->dev, dma))
4345 goto dma_error;
4346
4347 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4348 tx_buffer_info->length = size;
4349 tx_buffer_info->dma = dma;
4350
4351 tx_desc->read.olinfo_status = 0;
4352 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4353
4354 frag++;
4355 }
4356
4357 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4358
4359 /* write last descriptor with RS and EOP bits */
4360 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4361 if (unlikely(skb->no_fcs))
4362 cmd_type &= ~(cpu_to_le32(E1000_ADVTXD_DCMD_IFCS));
4363 tx_desc->read.cmd_type_len = cmd_type;
4364
4365 /* set the timestamp */
4366 first->time_stamp = jiffies;
4367
4368 /*
4369 * Force memory writes to complete before letting h/w know there
4370 * are new descriptors to fetch. (Only applicable for weak-ordered
4371 * memory model archs, such as IA-64).
4372 *
4373 * We also need this memory barrier to make certain all of the
4374 * status bits have been updated before next_to_watch is written.
4375 */
4376 wmb();
4377
4378 /* set next_to_watch value indicating a packet is present */
4379 first->next_to_watch = tx_desc;
4380
4381 i++;
4382 if (i == tx_ring->count)
4383 i = 0;
4384
4385 tx_ring->next_to_use = i;
4386
4387 writel(i, tx_ring->tail);
4388
4389 /* we need this if more than one processor can write to our tail
4390 * at a time, it syncronizes IO on IA64/Altix systems */
4391 mmiowb();
4392
4393 return;
4394
4395 dma_error:
4396 dev_err(tx_ring->dev, "TX DMA map failed\n");
4397
4398 /* clear dma mappings for failed tx_buffer_info map */
4399 for (;;) {
4400 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4401 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4402 if (tx_buffer_info == first)
4403 break;
4404 if (i == 0)
4405 i = tx_ring->count;
4406 i--;
4407 }
4408
4409 tx_ring->next_to_use = i;
4410 }
4411
4412 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4413 {
4414 struct net_device *netdev = tx_ring->netdev;
4415
4416 netif_stop_subqueue(netdev, tx_ring->queue_index);
4417
4418 /* Herbert's original patch had:
4419 * smp_mb__after_netif_stop_queue();
4420 * but since that doesn't exist yet, just open code it. */
4421 smp_mb();
4422
4423 /* We need to check again in a case another CPU has just
4424 * made room available. */
4425 if (igb_desc_unused(tx_ring) < size)
4426 return -EBUSY;
4427
4428 /* A reprieve! */
4429 netif_wake_subqueue(netdev, tx_ring->queue_index);
4430
4431 u64_stats_update_begin(&tx_ring->tx_syncp2);
4432 tx_ring->tx_stats.restart_queue2++;
4433 u64_stats_update_end(&tx_ring->tx_syncp2);
4434
4435 return 0;
4436 }
4437
4438 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4439 {
4440 if (igb_desc_unused(tx_ring) >= size)
4441 return 0;
4442 return __igb_maybe_stop_tx(tx_ring, size);
4443 }
4444
4445 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4446 struct igb_ring *tx_ring)
4447 {
4448 struct igb_tx_buffer *first;
4449 int tso;
4450 u32 tx_flags = 0;
4451 __be16 protocol = vlan_get_protocol(skb);
4452 u8 hdr_len = 0;
4453
4454 /* need: 1 descriptor per page,
4455 * + 2 desc gap to keep tail from touching head,
4456 * + 1 desc for skb->data,
4457 * + 1 desc for context descriptor,
4458 * otherwise try next time */
4459 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4460 /* this is a hard error */
4461 return NETDEV_TX_BUSY;
4462 }
4463
4464 /* record the location of the first descriptor for this packet */
4465 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4466 first->skb = skb;
4467 first->bytecount = skb->len;
4468 first->gso_segs = 1;
4469
4470 #ifdef CONFIG_IGB_PTP
4471 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4472 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4473 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4474 }
4475 #endif /* CONFIG_IGB_PTP */
4476
4477 if (vlan_tx_tag_present(skb)) {
4478 tx_flags |= IGB_TX_FLAGS_VLAN;
4479 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4480 }
4481
4482 /* record initial flags and protocol */
4483 first->tx_flags = tx_flags;
4484 first->protocol = protocol;
4485
4486 tso = igb_tso(tx_ring, first, &hdr_len);
4487 if (tso < 0)
4488 goto out_drop;
4489 else if (!tso)
4490 igb_tx_csum(tx_ring, first);
4491
4492 igb_tx_map(tx_ring, first, hdr_len);
4493
4494 /* Make sure there is space in the ring for the next send. */
4495 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4496
4497 return NETDEV_TX_OK;
4498
4499 out_drop:
4500 igb_unmap_and_free_tx_resource(tx_ring, first);
4501
4502 return NETDEV_TX_OK;
4503 }
4504
4505 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4506 struct sk_buff *skb)
4507 {
4508 unsigned int r_idx = skb->queue_mapping;
4509
4510 if (r_idx >= adapter->num_tx_queues)
4511 r_idx = r_idx % adapter->num_tx_queues;
4512
4513 return adapter->tx_ring[r_idx];
4514 }
4515
4516 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4517 struct net_device *netdev)
4518 {
4519 struct igb_adapter *adapter = netdev_priv(netdev);
4520
4521 if (test_bit(__IGB_DOWN, &adapter->state)) {
4522 dev_kfree_skb_any(skb);
4523 return NETDEV_TX_OK;
4524 }
4525
4526 if (skb->len <= 0) {
4527 dev_kfree_skb_any(skb);
4528 return NETDEV_TX_OK;
4529 }
4530
4531 /*
4532 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4533 * in order to meet this minimum size requirement.
4534 */
4535 if (skb->len < 17) {
4536 if (skb_padto(skb, 17))
4537 return NETDEV_TX_OK;
4538 skb->len = 17;
4539 }
4540
4541 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
4542 }
4543
4544 /**
4545 * igb_tx_timeout - Respond to a Tx Hang
4546 * @netdev: network interface device structure
4547 **/
4548 static void igb_tx_timeout(struct net_device *netdev)
4549 {
4550 struct igb_adapter *adapter = netdev_priv(netdev);
4551 struct e1000_hw *hw = &adapter->hw;
4552
4553 /* Do the reset outside of interrupt context */
4554 adapter->tx_timeout_count++;
4555
4556 if (hw->mac.type >= e1000_82580)
4557 hw->dev_spec._82575.global_device_reset = true;
4558
4559 schedule_work(&adapter->reset_task);
4560 wr32(E1000_EICS,
4561 (adapter->eims_enable_mask & ~adapter->eims_other));
4562 }
4563
4564 static void igb_reset_task(struct work_struct *work)
4565 {
4566 struct igb_adapter *adapter;
4567 adapter = container_of(work, struct igb_adapter, reset_task);
4568
4569 igb_dump(adapter);
4570 netdev_err(adapter->netdev, "Reset adapter\n");
4571 igb_reinit_locked(adapter);
4572 }
4573
4574 /**
4575 * igb_get_stats64 - Get System Network Statistics
4576 * @netdev: network interface device structure
4577 * @stats: rtnl_link_stats64 pointer
4578 *
4579 **/
4580 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4581 struct rtnl_link_stats64 *stats)
4582 {
4583 struct igb_adapter *adapter = netdev_priv(netdev);
4584
4585 spin_lock(&adapter->stats64_lock);
4586 igb_update_stats(adapter, &adapter->stats64);
4587 memcpy(stats, &adapter->stats64, sizeof(*stats));
4588 spin_unlock(&adapter->stats64_lock);
4589
4590 return stats;
4591 }
4592
4593 /**
4594 * igb_change_mtu - Change the Maximum Transfer Unit
4595 * @netdev: network interface device structure
4596 * @new_mtu: new value for maximum frame size
4597 *
4598 * Returns 0 on success, negative on failure
4599 **/
4600 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4601 {
4602 struct igb_adapter *adapter = netdev_priv(netdev);
4603 struct pci_dev *pdev = adapter->pdev;
4604 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4605
4606 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4607 dev_err(&pdev->dev, "Invalid MTU setting\n");
4608 return -EINVAL;
4609 }
4610
4611 #define MAX_STD_JUMBO_FRAME_SIZE 9238
4612 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4613 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4614 return -EINVAL;
4615 }
4616
4617 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4618 msleep(1);
4619
4620 /* igb_down has a dependency on max_frame_size */
4621 adapter->max_frame_size = max_frame;
4622
4623 if (netif_running(netdev))
4624 igb_down(adapter);
4625
4626 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4627 netdev->mtu, new_mtu);
4628 netdev->mtu = new_mtu;
4629
4630 if (netif_running(netdev))
4631 igb_up(adapter);
4632 else
4633 igb_reset(adapter);
4634
4635 clear_bit(__IGB_RESETTING, &adapter->state);
4636
4637 return 0;
4638 }
4639
4640 /**
4641 * igb_update_stats - Update the board statistics counters
4642 * @adapter: board private structure
4643 **/
4644
4645 void igb_update_stats(struct igb_adapter *adapter,
4646 struct rtnl_link_stats64 *net_stats)
4647 {
4648 struct e1000_hw *hw = &adapter->hw;
4649 struct pci_dev *pdev = adapter->pdev;
4650 u32 reg, mpc;
4651 u16 phy_tmp;
4652 int i;
4653 u64 bytes, packets;
4654 unsigned int start;
4655 u64 _bytes, _packets;
4656
4657 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4658
4659 /*
4660 * Prevent stats update while adapter is being reset, or if the pci
4661 * connection is down.
4662 */
4663 if (adapter->link_speed == 0)
4664 return;
4665 if (pci_channel_offline(pdev))
4666 return;
4667
4668 bytes = 0;
4669 packets = 0;
4670 for (i = 0; i < adapter->num_rx_queues; i++) {
4671 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
4672 struct igb_ring *ring = adapter->rx_ring[i];
4673
4674 ring->rx_stats.drops += rqdpc_tmp;
4675 net_stats->rx_fifo_errors += rqdpc_tmp;
4676
4677 do {
4678 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4679 _bytes = ring->rx_stats.bytes;
4680 _packets = ring->rx_stats.packets;
4681 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4682 bytes += _bytes;
4683 packets += _packets;
4684 }
4685
4686 net_stats->rx_bytes = bytes;
4687 net_stats->rx_packets = packets;
4688
4689 bytes = 0;
4690 packets = 0;
4691 for (i = 0; i < adapter->num_tx_queues; i++) {
4692 struct igb_ring *ring = adapter->tx_ring[i];
4693 do {
4694 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4695 _bytes = ring->tx_stats.bytes;
4696 _packets = ring->tx_stats.packets;
4697 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4698 bytes += _bytes;
4699 packets += _packets;
4700 }
4701 net_stats->tx_bytes = bytes;
4702 net_stats->tx_packets = packets;
4703
4704 /* read stats registers */
4705 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4706 adapter->stats.gprc += rd32(E1000_GPRC);
4707 adapter->stats.gorc += rd32(E1000_GORCL);
4708 rd32(E1000_GORCH); /* clear GORCL */
4709 adapter->stats.bprc += rd32(E1000_BPRC);
4710 adapter->stats.mprc += rd32(E1000_MPRC);
4711 adapter->stats.roc += rd32(E1000_ROC);
4712
4713 adapter->stats.prc64 += rd32(E1000_PRC64);
4714 adapter->stats.prc127 += rd32(E1000_PRC127);
4715 adapter->stats.prc255 += rd32(E1000_PRC255);
4716 adapter->stats.prc511 += rd32(E1000_PRC511);
4717 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4718 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4719 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4720 adapter->stats.sec += rd32(E1000_SEC);
4721
4722 mpc = rd32(E1000_MPC);
4723 adapter->stats.mpc += mpc;
4724 net_stats->rx_fifo_errors += mpc;
4725 adapter->stats.scc += rd32(E1000_SCC);
4726 adapter->stats.ecol += rd32(E1000_ECOL);
4727 adapter->stats.mcc += rd32(E1000_MCC);
4728 adapter->stats.latecol += rd32(E1000_LATECOL);
4729 adapter->stats.dc += rd32(E1000_DC);
4730 adapter->stats.rlec += rd32(E1000_RLEC);
4731 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4732 adapter->stats.xontxc += rd32(E1000_XONTXC);
4733 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4734 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4735 adapter->stats.fcruc += rd32(E1000_FCRUC);
4736 adapter->stats.gptc += rd32(E1000_GPTC);
4737 adapter->stats.gotc += rd32(E1000_GOTCL);
4738 rd32(E1000_GOTCH); /* clear GOTCL */
4739 adapter->stats.rnbc += rd32(E1000_RNBC);
4740 adapter->stats.ruc += rd32(E1000_RUC);
4741 adapter->stats.rfc += rd32(E1000_RFC);
4742 adapter->stats.rjc += rd32(E1000_RJC);
4743 adapter->stats.tor += rd32(E1000_TORH);
4744 adapter->stats.tot += rd32(E1000_TOTH);
4745 adapter->stats.tpr += rd32(E1000_TPR);
4746
4747 adapter->stats.ptc64 += rd32(E1000_PTC64);
4748 adapter->stats.ptc127 += rd32(E1000_PTC127);
4749 adapter->stats.ptc255 += rd32(E1000_PTC255);
4750 adapter->stats.ptc511 += rd32(E1000_PTC511);
4751 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4752 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4753
4754 adapter->stats.mptc += rd32(E1000_MPTC);
4755 adapter->stats.bptc += rd32(E1000_BPTC);
4756
4757 adapter->stats.tpt += rd32(E1000_TPT);
4758 adapter->stats.colc += rd32(E1000_COLC);
4759
4760 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4761 /* read internal phy specific stats */
4762 reg = rd32(E1000_CTRL_EXT);
4763 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4764 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4765 adapter->stats.tncrs += rd32(E1000_TNCRS);
4766 }
4767
4768 adapter->stats.tsctc += rd32(E1000_TSCTC);
4769 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4770
4771 adapter->stats.iac += rd32(E1000_IAC);
4772 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4773 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4774 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4775 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4776 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4777 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4778 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4779 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4780
4781 /* Fill out the OS statistics structure */
4782 net_stats->multicast = adapter->stats.mprc;
4783 net_stats->collisions = adapter->stats.colc;
4784
4785 /* Rx Errors */
4786
4787 /* RLEC on some newer hardware can be incorrect so build
4788 * our own version based on RUC and ROC */
4789 net_stats->rx_errors = adapter->stats.rxerrc +
4790 adapter->stats.crcerrs + adapter->stats.algnerrc +
4791 adapter->stats.ruc + adapter->stats.roc +
4792 adapter->stats.cexterr;
4793 net_stats->rx_length_errors = adapter->stats.ruc +
4794 adapter->stats.roc;
4795 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4796 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4797 net_stats->rx_missed_errors = adapter->stats.mpc;
4798
4799 /* Tx Errors */
4800 net_stats->tx_errors = adapter->stats.ecol +
4801 adapter->stats.latecol;
4802 net_stats->tx_aborted_errors = adapter->stats.ecol;
4803 net_stats->tx_window_errors = adapter->stats.latecol;
4804 net_stats->tx_carrier_errors = adapter->stats.tncrs;
4805
4806 /* Tx Dropped needs to be maintained elsewhere */
4807
4808 /* Phy Stats */
4809 if (hw->phy.media_type == e1000_media_type_copper) {
4810 if ((adapter->link_speed == SPEED_1000) &&
4811 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4812 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4813 adapter->phy_stats.idle_errors += phy_tmp;
4814 }
4815 }
4816
4817 /* Management Stats */
4818 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4819 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4820 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4821
4822 /* OS2BMC Stats */
4823 reg = rd32(E1000_MANC);
4824 if (reg & E1000_MANC_EN_BMC2OS) {
4825 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4826 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4827 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4828 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4829 }
4830 }
4831
4832 static irqreturn_t igb_msix_other(int irq, void *data)
4833 {
4834 struct igb_adapter *adapter = data;
4835 struct e1000_hw *hw = &adapter->hw;
4836 u32 icr = rd32(E1000_ICR);
4837 /* reading ICR causes bit 31 of EICR to be cleared */
4838
4839 if (icr & E1000_ICR_DRSTA)
4840 schedule_work(&adapter->reset_task);
4841
4842 if (icr & E1000_ICR_DOUTSYNC) {
4843 /* HW is reporting DMA is out of sync */
4844 adapter->stats.doosync++;
4845 /* The DMA Out of Sync is also indication of a spoof event
4846 * in IOV mode. Check the Wrong VM Behavior register to
4847 * see if it is really a spoof event. */
4848 igb_check_wvbr(adapter);
4849 }
4850
4851 /* Check for a mailbox event */
4852 if (icr & E1000_ICR_VMMB)
4853 igb_msg_task(adapter);
4854
4855 if (icr & E1000_ICR_LSC) {
4856 hw->mac.get_link_status = 1;
4857 /* guard against interrupt when we're going down */
4858 if (!test_bit(__IGB_DOWN, &adapter->state))
4859 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4860 }
4861
4862 wr32(E1000_EIMS, adapter->eims_other);
4863
4864 return IRQ_HANDLED;
4865 }
4866
4867 static void igb_write_itr(struct igb_q_vector *q_vector)
4868 {
4869 struct igb_adapter *adapter = q_vector->adapter;
4870 u32 itr_val = q_vector->itr_val & 0x7FFC;
4871
4872 if (!q_vector->set_itr)
4873 return;
4874
4875 if (!itr_val)
4876 itr_val = 0x4;
4877
4878 if (adapter->hw.mac.type == e1000_82575)
4879 itr_val |= itr_val << 16;
4880 else
4881 itr_val |= E1000_EITR_CNT_IGNR;
4882
4883 writel(itr_val, q_vector->itr_register);
4884 q_vector->set_itr = 0;
4885 }
4886
4887 static irqreturn_t igb_msix_ring(int irq, void *data)
4888 {
4889 struct igb_q_vector *q_vector = data;
4890
4891 /* Write the ITR value calculated from the previous interrupt. */
4892 igb_write_itr(q_vector);
4893
4894 napi_schedule(&q_vector->napi);
4895
4896 return IRQ_HANDLED;
4897 }
4898
4899 #ifdef CONFIG_IGB_DCA
4900 static void igb_update_dca(struct igb_q_vector *q_vector)
4901 {
4902 struct igb_adapter *adapter = q_vector->adapter;
4903 struct e1000_hw *hw = &adapter->hw;
4904 int cpu = get_cpu();
4905
4906 if (q_vector->cpu == cpu)
4907 goto out_no_update;
4908
4909 if (q_vector->tx.ring) {
4910 int q = q_vector->tx.ring->reg_idx;
4911 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4912 if (hw->mac.type == e1000_82575) {
4913 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4914 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4915 } else {
4916 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4917 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4918 E1000_DCA_TXCTRL_CPUID_SHIFT;
4919 }
4920 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4921 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4922 }
4923 if (q_vector->rx.ring) {
4924 int q = q_vector->rx.ring->reg_idx;
4925 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4926 if (hw->mac.type == e1000_82575) {
4927 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4928 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4929 } else {
4930 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4931 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4932 E1000_DCA_RXCTRL_CPUID_SHIFT;
4933 }
4934 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4935 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4936 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4937 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4938 }
4939 q_vector->cpu = cpu;
4940 out_no_update:
4941 put_cpu();
4942 }
4943
4944 static void igb_setup_dca(struct igb_adapter *adapter)
4945 {
4946 struct e1000_hw *hw = &adapter->hw;
4947 int i;
4948
4949 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4950 return;
4951
4952 /* Always use CB2 mode, difference is masked in the CB driver. */
4953 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4954
4955 for (i = 0; i < adapter->num_q_vectors; i++) {
4956 adapter->q_vector[i]->cpu = -1;
4957 igb_update_dca(adapter->q_vector[i]);
4958 }
4959 }
4960
4961 static int __igb_notify_dca(struct device *dev, void *data)
4962 {
4963 struct net_device *netdev = dev_get_drvdata(dev);
4964 struct igb_adapter *adapter = netdev_priv(netdev);
4965 struct pci_dev *pdev = adapter->pdev;
4966 struct e1000_hw *hw = &adapter->hw;
4967 unsigned long event = *(unsigned long *)data;
4968
4969 switch (event) {
4970 case DCA_PROVIDER_ADD:
4971 /* if already enabled, don't do it again */
4972 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4973 break;
4974 if (dca_add_requester(dev) == 0) {
4975 adapter->flags |= IGB_FLAG_DCA_ENABLED;
4976 dev_info(&pdev->dev, "DCA enabled\n");
4977 igb_setup_dca(adapter);
4978 break;
4979 }
4980 /* Fall Through since DCA is disabled. */
4981 case DCA_PROVIDER_REMOVE:
4982 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4983 /* without this a class_device is left
4984 * hanging around in the sysfs model */
4985 dca_remove_requester(dev);
4986 dev_info(&pdev->dev, "DCA disabled\n");
4987 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4988 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4989 }
4990 break;
4991 }
4992
4993 return 0;
4994 }
4995
4996 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4997 void *p)
4998 {
4999 int ret_val;
5000
5001 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
5002 __igb_notify_dca);
5003
5004 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
5005 }
5006 #endif /* CONFIG_IGB_DCA */
5007
5008 #ifdef CONFIG_PCI_IOV
5009 static int igb_vf_configure(struct igb_adapter *adapter, int vf)
5010 {
5011 unsigned char mac_addr[ETH_ALEN];
5012 struct pci_dev *pdev = adapter->pdev;
5013 struct e1000_hw *hw = &adapter->hw;
5014 struct pci_dev *pvfdev;
5015 unsigned int device_id;
5016 u16 thisvf_devfn;
5017
5018 eth_random_addr(mac_addr);
5019 igb_set_vf_mac(adapter, vf, mac_addr);
5020
5021 switch (adapter->hw.mac.type) {
5022 case e1000_82576:
5023 device_id = IGB_82576_VF_DEV_ID;
5024 /* VF Stride for 82576 is 2 */
5025 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
5026 (pdev->devfn & 1);
5027 break;
5028 case e1000_i350:
5029 device_id = IGB_I350_VF_DEV_ID;
5030 /* VF Stride for I350 is 4 */
5031 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
5032 (pdev->devfn & 3);
5033 break;
5034 default:
5035 device_id = 0;
5036 thisvf_devfn = 0;
5037 break;
5038 }
5039
5040 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5041 while (pvfdev) {
5042 if (pvfdev->devfn == thisvf_devfn)
5043 break;
5044 pvfdev = pci_get_device(hw->vendor_id,
5045 device_id, pvfdev);
5046 }
5047
5048 if (pvfdev)
5049 adapter->vf_data[vf].vfdev = pvfdev;
5050 else
5051 dev_err(&pdev->dev,
5052 "Couldn't find pci dev ptr for VF %4.4x\n",
5053 thisvf_devfn);
5054 return pvfdev != NULL;
5055 }
5056
5057 static int igb_find_enabled_vfs(struct igb_adapter *adapter)
5058 {
5059 struct e1000_hw *hw = &adapter->hw;
5060 struct pci_dev *pdev = adapter->pdev;
5061 struct pci_dev *pvfdev;
5062 u16 vf_devfn = 0;
5063 u16 vf_stride;
5064 unsigned int device_id;
5065 int vfs_found = 0;
5066
5067 switch (adapter->hw.mac.type) {
5068 case e1000_82576:
5069 device_id = IGB_82576_VF_DEV_ID;
5070 /* VF Stride for 82576 is 2 */
5071 vf_stride = 2;
5072 break;
5073 case e1000_i350:
5074 device_id = IGB_I350_VF_DEV_ID;
5075 /* VF Stride for I350 is 4 */
5076 vf_stride = 4;
5077 break;
5078 default:
5079 device_id = 0;
5080 vf_stride = 0;
5081 break;
5082 }
5083
5084 vf_devfn = pdev->devfn + 0x80;
5085 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5086 while (pvfdev) {
5087 if (pvfdev->devfn == vf_devfn &&
5088 (pvfdev->bus->number >= pdev->bus->number))
5089 vfs_found++;
5090 vf_devfn += vf_stride;
5091 pvfdev = pci_get_device(hw->vendor_id,
5092 device_id, pvfdev);
5093 }
5094
5095 return vfs_found;
5096 }
5097
5098 static int igb_check_vf_assignment(struct igb_adapter *adapter)
5099 {
5100 int i;
5101 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5102 if (adapter->vf_data[i].vfdev) {
5103 if (adapter->vf_data[i].vfdev->dev_flags &
5104 PCI_DEV_FLAGS_ASSIGNED)
5105 return true;
5106 }
5107 }
5108 return false;
5109 }
5110
5111 #endif
5112 static void igb_ping_all_vfs(struct igb_adapter *adapter)
5113 {
5114 struct e1000_hw *hw = &adapter->hw;
5115 u32 ping;
5116 int i;
5117
5118 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5119 ping = E1000_PF_CONTROL_MSG;
5120 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
5121 ping |= E1000_VT_MSGTYPE_CTS;
5122 igb_write_mbx(hw, &ping, 1, i);
5123 }
5124 }
5125
5126 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5127 {
5128 struct e1000_hw *hw = &adapter->hw;
5129 u32 vmolr = rd32(E1000_VMOLR(vf));
5130 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5131
5132 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
5133 IGB_VF_FLAG_MULTI_PROMISC);
5134 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5135
5136 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5137 vmolr |= E1000_VMOLR_MPME;
5138 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
5139 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5140 } else {
5141 /*
5142 * if we have hashes and we are clearing a multicast promisc
5143 * flag we need to write the hashes to the MTA as this step
5144 * was previously skipped
5145 */
5146 if (vf_data->num_vf_mc_hashes > 30) {
5147 vmolr |= E1000_VMOLR_MPME;
5148 } else if (vf_data->num_vf_mc_hashes) {
5149 int j;
5150 vmolr |= E1000_VMOLR_ROMPE;
5151 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5152 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5153 }
5154 }
5155
5156 wr32(E1000_VMOLR(vf), vmolr);
5157
5158 /* there are flags left unprocessed, likely not supported */
5159 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5160 return -EINVAL;
5161
5162 return 0;
5163
5164 }
5165
5166 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5167 u32 *msgbuf, u32 vf)
5168 {
5169 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5170 u16 *hash_list = (u16 *)&msgbuf[1];
5171 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5172 int i;
5173
5174 /* salt away the number of multicast addresses assigned
5175 * to this VF for later use to restore when the PF multi cast
5176 * list changes
5177 */
5178 vf_data->num_vf_mc_hashes = n;
5179
5180 /* only up to 30 hash values supported */
5181 if (n > 30)
5182 n = 30;
5183
5184 /* store the hashes for later use */
5185 for (i = 0; i < n; i++)
5186 vf_data->vf_mc_hashes[i] = hash_list[i];
5187
5188 /* Flush and reset the mta with the new values */
5189 igb_set_rx_mode(adapter->netdev);
5190
5191 return 0;
5192 }
5193
5194 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5195 {
5196 struct e1000_hw *hw = &adapter->hw;
5197 struct vf_data_storage *vf_data;
5198 int i, j;
5199
5200 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5201 u32 vmolr = rd32(E1000_VMOLR(i));
5202 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5203
5204 vf_data = &adapter->vf_data[i];
5205
5206 if ((vf_data->num_vf_mc_hashes > 30) ||
5207 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5208 vmolr |= E1000_VMOLR_MPME;
5209 } else if (vf_data->num_vf_mc_hashes) {
5210 vmolr |= E1000_VMOLR_ROMPE;
5211 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5212 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5213 }
5214 wr32(E1000_VMOLR(i), vmolr);
5215 }
5216 }
5217
5218 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5219 {
5220 struct e1000_hw *hw = &adapter->hw;
5221 u32 pool_mask, reg, vid;
5222 int i;
5223
5224 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5225
5226 /* Find the vlan filter for this id */
5227 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5228 reg = rd32(E1000_VLVF(i));
5229
5230 /* remove the vf from the pool */
5231 reg &= ~pool_mask;
5232
5233 /* if pool is empty then remove entry from vfta */
5234 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5235 (reg & E1000_VLVF_VLANID_ENABLE)) {
5236 reg = 0;
5237 vid = reg & E1000_VLVF_VLANID_MASK;
5238 igb_vfta_set(hw, vid, false);
5239 }
5240
5241 wr32(E1000_VLVF(i), reg);
5242 }
5243
5244 adapter->vf_data[vf].vlans_enabled = 0;
5245 }
5246
5247 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5248 {
5249 struct e1000_hw *hw = &adapter->hw;
5250 u32 reg, i;
5251
5252 /* The vlvf table only exists on 82576 hardware and newer */
5253 if (hw->mac.type < e1000_82576)
5254 return -1;
5255
5256 /* we only need to do this if VMDq is enabled */
5257 if (!adapter->vfs_allocated_count)
5258 return -1;
5259
5260 /* Find the vlan filter for this id */
5261 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5262 reg = rd32(E1000_VLVF(i));
5263 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5264 vid == (reg & E1000_VLVF_VLANID_MASK))
5265 break;
5266 }
5267
5268 if (add) {
5269 if (i == E1000_VLVF_ARRAY_SIZE) {
5270 /* Did not find a matching VLAN ID entry that was
5271 * enabled. Search for a free filter entry, i.e.
5272 * one without the enable bit set
5273 */
5274 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5275 reg = rd32(E1000_VLVF(i));
5276 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5277 break;
5278 }
5279 }
5280 if (i < E1000_VLVF_ARRAY_SIZE) {
5281 /* Found an enabled/available entry */
5282 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5283
5284 /* if !enabled we need to set this up in vfta */
5285 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
5286 /* add VID to filter table */
5287 igb_vfta_set(hw, vid, true);
5288 reg |= E1000_VLVF_VLANID_ENABLE;
5289 }
5290 reg &= ~E1000_VLVF_VLANID_MASK;
5291 reg |= vid;
5292 wr32(E1000_VLVF(i), reg);
5293
5294 /* do not modify RLPML for PF devices */
5295 if (vf >= adapter->vfs_allocated_count)
5296 return 0;
5297
5298 if (!adapter->vf_data[vf].vlans_enabled) {
5299 u32 size;
5300 reg = rd32(E1000_VMOLR(vf));
5301 size = reg & E1000_VMOLR_RLPML_MASK;
5302 size += 4;
5303 reg &= ~E1000_VMOLR_RLPML_MASK;
5304 reg |= size;
5305 wr32(E1000_VMOLR(vf), reg);
5306 }
5307
5308 adapter->vf_data[vf].vlans_enabled++;
5309 }
5310 } else {
5311 if (i < E1000_VLVF_ARRAY_SIZE) {
5312 /* remove vf from the pool */
5313 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5314 /* if pool is empty then remove entry from vfta */
5315 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5316 reg = 0;
5317 igb_vfta_set(hw, vid, false);
5318 }
5319 wr32(E1000_VLVF(i), reg);
5320
5321 /* do not modify RLPML for PF devices */
5322 if (vf >= adapter->vfs_allocated_count)
5323 return 0;
5324
5325 adapter->vf_data[vf].vlans_enabled--;
5326 if (!adapter->vf_data[vf].vlans_enabled) {
5327 u32 size;
5328 reg = rd32(E1000_VMOLR(vf));
5329 size = reg & E1000_VMOLR_RLPML_MASK;
5330 size -= 4;
5331 reg &= ~E1000_VMOLR_RLPML_MASK;
5332 reg |= size;
5333 wr32(E1000_VMOLR(vf), reg);
5334 }
5335 }
5336 }
5337 return 0;
5338 }
5339
5340 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5341 {
5342 struct e1000_hw *hw = &adapter->hw;
5343
5344 if (vid)
5345 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5346 else
5347 wr32(E1000_VMVIR(vf), 0);
5348 }
5349
5350 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5351 int vf, u16 vlan, u8 qos)
5352 {
5353 int err = 0;
5354 struct igb_adapter *adapter = netdev_priv(netdev);
5355
5356 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5357 return -EINVAL;
5358 if (vlan || qos) {
5359 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5360 if (err)
5361 goto out;
5362 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5363 igb_set_vmolr(adapter, vf, !vlan);
5364 adapter->vf_data[vf].pf_vlan = vlan;
5365 adapter->vf_data[vf].pf_qos = qos;
5366 dev_info(&adapter->pdev->dev,
5367 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5368 if (test_bit(__IGB_DOWN, &adapter->state)) {
5369 dev_warn(&adapter->pdev->dev,
5370 "The VF VLAN has been set,"
5371 " but the PF device is not up.\n");
5372 dev_warn(&adapter->pdev->dev,
5373 "Bring the PF device up before"
5374 " attempting to use the VF device.\n");
5375 }
5376 } else {
5377 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5378 false, vf);
5379 igb_set_vmvir(adapter, vlan, vf);
5380 igb_set_vmolr(adapter, vf, true);
5381 adapter->vf_data[vf].pf_vlan = 0;
5382 adapter->vf_data[vf].pf_qos = 0;
5383 }
5384 out:
5385 return err;
5386 }
5387
5388 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5389 {
5390 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5391 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5392
5393 return igb_vlvf_set(adapter, vid, add, vf);
5394 }
5395
5396 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5397 {
5398 /* clear flags - except flag that indicates PF has set the MAC */
5399 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5400 adapter->vf_data[vf].last_nack = jiffies;
5401
5402 /* reset offloads to defaults */
5403 igb_set_vmolr(adapter, vf, true);
5404
5405 /* reset vlans for device */
5406 igb_clear_vf_vfta(adapter, vf);
5407 if (adapter->vf_data[vf].pf_vlan)
5408 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5409 adapter->vf_data[vf].pf_vlan,
5410 adapter->vf_data[vf].pf_qos);
5411 else
5412 igb_clear_vf_vfta(adapter, vf);
5413
5414 /* reset multicast table array for vf */
5415 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5416
5417 /* Flush and reset the mta with the new values */
5418 igb_set_rx_mode(adapter->netdev);
5419 }
5420
5421 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5422 {
5423 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5424
5425 /* generate a new mac address as we were hotplug removed/added */
5426 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5427 eth_random_addr(vf_mac);
5428
5429 /* process remaining reset events */
5430 igb_vf_reset(adapter, vf);
5431 }
5432
5433 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5434 {
5435 struct e1000_hw *hw = &adapter->hw;
5436 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5437 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5438 u32 reg, msgbuf[3];
5439 u8 *addr = (u8 *)(&msgbuf[1]);
5440
5441 /* process all the same items cleared in a function level reset */
5442 igb_vf_reset(adapter, vf);
5443
5444 /* set vf mac address */
5445 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
5446
5447 /* enable transmit and receive for vf */
5448 reg = rd32(E1000_VFTE);
5449 wr32(E1000_VFTE, reg | (1 << vf));
5450 reg = rd32(E1000_VFRE);
5451 wr32(E1000_VFRE, reg | (1 << vf));
5452
5453 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
5454
5455 /* reply to reset with ack and vf mac address */
5456 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5457 memcpy(addr, vf_mac, 6);
5458 igb_write_mbx(hw, msgbuf, 3, vf);
5459 }
5460
5461 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5462 {
5463 /*
5464 * The VF MAC Address is stored in a packed array of bytes
5465 * starting at the second 32 bit word of the msg array
5466 */
5467 unsigned char *addr = (char *)&msg[1];
5468 int err = -1;
5469
5470 if (is_valid_ether_addr(addr))
5471 err = igb_set_vf_mac(adapter, vf, addr);
5472
5473 return err;
5474 }
5475
5476 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5477 {
5478 struct e1000_hw *hw = &adapter->hw;
5479 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5480 u32 msg = E1000_VT_MSGTYPE_NACK;
5481
5482 /* if device isn't clear to send it shouldn't be reading either */
5483 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5484 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5485 igb_write_mbx(hw, &msg, 1, vf);
5486 vf_data->last_nack = jiffies;
5487 }
5488 }
5489
5490 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5491 {
5492 struct pci_dev *pdev = adapter->pdev;
5493 u32 msgbuf[E1000_VFMAILBOX_SIZE];
5494 struct e1000_hw *hw = &adapter->hw;
5495 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5496 s32 retval;
5497
5498 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5499
5500 if (retval) {
5501 /* if receive failed revoke VF CTS stats and restart init */
5502 dev_err(&pdev->dev, "Error receiving message from VF\n");
5503 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5504 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5505 return;
5506 goto out;
5507 }
5508
5509 /* this is a message we already processed, do nothing */
5510 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5511 return;
5512
5513 /*
5514 * until the vf completes a reset it should not be
5515 * allowed to start any configuration.
5516 */
5517
5518 if (msgbuf[0] == E1000_VF_RESET) {
5519 igb_vf_reset_msg(adapter, vf);
5520 return;
5521 }
5522
5523 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5524 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5525 return;
5526 retval = -1;
5527 goto out;
5528 }
5529
5530 switch ((msgbuf[0] & 0xFFFF)) {
5531 case E1000_VF_SET_MAC_ADDR:
5532 retval = -EINVAL;
5533 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5534 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5535 else
5536 dev_warn(&pdev->dev,
5537 "VF %d attempted to override administratively "
5538 "set MAC address\nReload the VF driver to "
5539 "resume operations\n", vf);
5540 break;
5541 case E1000_VF_SET_PROMISC:
5542 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5543 break;
5544 case E1000_VF_SET_MULTICAST:
5545 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5546 break;
5547 case E1000_VF_SET_LPE:
5548 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5549 break;
5550 case E1000_VF_SET_VLAN:
5551 retval = -1;
5552 if (vf_data->pf_vlan)
5553 dev_warn(&pdev->dev,
5554 "VF %d attempted to override administratively "
5555 "set VLAN tag\nReload the VF driver to "
5556 "resume operations\n", vf);
5557 else
5558 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5559 break;
5560 default:
5561 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5562 retval = -1;
5563 break;
5564 }
5565
5566 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5567 out:
5568 /* notify the VF of the results of what it sent us */
5569 if (retval)
5570 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5571 else
5572 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5573
5574 igb_write_mbx(hw, msgbuf, 1, vf);
5575 }
5576
5577 static void igb_msg_task(struct igb_adapter *adapter)
5578 {
5579 struct e1000_hw *hw = &adapter->hw;
5580 u32 vf;
5581
5582 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5583 /* process any reset requests */
5584 if (!igb_check_for_rst(hw, vf))
5585 igb_vf_reset_event(adapter, vf);
5586
5587 /* process any messages pending */
5588 if (!igb_check_for_msg(hw, vf))
5589 igb_rcv_msg_from_vf(adapter, vf);
5590
5591 /* process any acks */
5592 if (!igb_check_for_ack(hw, vf))
5593 igb_rcv_ack_from_vf(adapter, vf);
5594 }
5595 }
5596
5597 /**
5598 * igb_set_uta - Set unicast filter table address
5599 * @adapter: board private structure
5600 *
5601 * The unicast table address is a register array of 32-bit registers.
5602 * The table is meant to be used in a way similar to how the MTA is used
5603 * however due to certain limitations in the hardware it is necessary to
5604 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5605 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
5606 **/
5607 static void igb_set_uta(struct igb_adapter *adapter)
5608 {
5609 struct e1000_hw *hw = &adapter->hw;
5610 int i;
5611
5612 /* The UTA table only exists on 82576 hardware and newer */
5613 if (hw->mac.type < e1000_82576)
5614 return;
5615
5616 /* we only need to do this if VMDq is enabled */
5617 if (!adapter->vfs_allocated_count)
5618 return;
5619
5620 for (i = 0; i < hw->mac.uta_reg_count; i++)
5621 array_wr32(E1000_UTA, i, ~0);
5622 }
5623
5624 /**
5625 * igb_intr_msi - Interrupt Handler
5626 * @irq: interrupt number
5627 * @data: pointer to a network interface device structure
5628 **/
5629 static irqreturn_t igb_intr_msi(int irq, void *data)
5630 {
5631 struct igb_adapter *adapter = data;
5632 struct igb_q_vector *q_vector = adapter->q_vector[0];
5633 struct e1000_hw *hw = &adapter->hw;
5634 /* read ICR disables interrupts using IAM */
5635 u32 icr = rd32(E1000_ICR);
5636
5637 igb_write_itr(q_vector);
5638
5639 if (icr & E1000_ICR_DRSTA)
5640 schedule_work(&adapter->reset_task);
5641
5642 if (icr & E1000_ICR_DOUTSYNC) {
5643 /* HW is reporting DMA is out of sync */
5644 adapter->stats.doosync++;
5645 }
5646
5647 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5648 hw->mac.get_link_status = 1;
5649 if (!test_bit(__IGB_DOWN, &adapter->state))
5650 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5651 }
5652
5653 napi_schedule(&q_vector->napi);
5654
5655 return IRQ_HANDLED;
5656 }
5657
5658 /**
5659 * igb_intr - Legacy Interrupt Handler
5660 * @irq: interrupt number
5661 * @data: pointer to a network interface device structure
5662 **/
5663 static irqreturn_t igb_intr(int irq, void *data)
5664 {
5665 struct igb_adapter *adapter = data;
5666 struct igb_q_vector *q_vector = adapter->q_vector[0];
5667 struct e1000_hw *hw = &adapter->hw;
5668 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5669 * need for the IMC write */
5670 u32 icr = rd32(E1000_ICR);
5671
5672 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5673 * not set, then the adapter didn't send an interrupt */
5674 if (!(icr & E1000_ICR_INT_ASSERTED))
5675 return IRQ_NONE;
5676
5677 igb_write_itr(q_vector);
5678
5679 if (icr & E1000_ICR_DRSTA)
5680 schedule_work(&adapter->reset_task);
5681
5682 if (icr & E1000_ICR_DOUTSYNC) {
5683 /* HW is reporting DMA is out of sync */
5684 adapter->stats.doosync++;
5685 }
5686
5687 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5688 hw->mac.get_link_status = 1;
5689 /* guard against interrupt when we're going down */
5690 if (!test_bit(__IGB_DOWN, &adapter->state))
5691 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5692 }
5693
5694 napi_schedule(&q_vector->napi);
5695
5696 return IRQ_HANDLED;
5697 }
5698
5699 static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5700 {
5701 struct igb_adapter *adapter = q_vector->adapter;
5702 struct e1000_hw *hw = &adapter->hw;
5703
5704 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
5705 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
5706 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
5707 igb_set_itr(q_vector);
5708 else
5709 igb_update_ring_itr(q_vector);
5710 }
5711
5712 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5713 if (adapter->msix_entries)
5714 wr32(E1000_EIMS, q_vector->eims_value);
5715 else
5716 igb_irq_enable(adapter);
5717 }
5718 }
5719
5720 /**
5721 * igb_poll - NAPI Rx polling callback
5722 * @napi: napi polling structure
5723 * @budget: count of how many packets we should handle
5724 **/
5725 static int igb_poll(struct napi_struct *napi, int budget)
5726 {
5727 struct igb_q_vector *q_vector = container_of(napi,
5728 struct igb_q_vector,
5729 napi);
5730 bool clean_complete = true;
5731
5732 #ifdef CONFIG_IGB_DCA
5733 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5734 igb_update_dca(q_vector);
5735 #endif
5736 if (q_vector->tx.ring)
5737 clean_complete = igb_clean_tx_irq(q_vector);
5738
5739 if (q_vector->rx.ring)
5740 clean_complete &= igb_clean_rx_irq(q_vector, budget);
5741
5742 /* If all work not completed, return budget and keep polling */
5743 if (!clean_complete)
5744 return budget;
5745
5746 /* If not enough Rx work done, exit the polling mode */
5747 napi_complete(napi);
5748 igb_ring_irq_enable(q_vector);
5749
5750 return 0;
5751 }
5752
5753 /**
5754 * igb_clean_tx_irq - Reclaim resources after transmit completes
5755 * @q_vector: pointer to q_vector containing needed info
5756 *
5757 * returns true if ring is completely cleaned
5758 **/
5759 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5760 {
5761 struct igb_adapter *adapter = q_vector->adapter;
5762 struct igb_ring *tx_ring = q_vector->tx.ring;
5763 struct igb_tx_buffer *tx_buffer;
5764 union e1000_adv_tx_desc *tx_desc, *eop_desc;
5765 unsigned int total_bytes = 0, total_packets = 0;
5766 unsigned int budget = q_vector->tx.work_limit;
5767 unsigned int i = tx_ring->next_to_clean;
5768
5769 if (test_bit(__IGB_DOWN, &adapter->state))
5770 return true;
5771
5772 tx_buffer = &tx_ring->tx_buffer_info[i];
5773 tx_desc = IGB_TX_DESC(tx_ring, i);
5774 i -= tx_ring->count;
5775
5776 for (; budget; budget--) {
5777 eop_desc = tx_buffer->next_to_watch;
5778
5779 /* prevent any other reads prior to eop_desc */
5780 rmb();
5781
5782 /* if next_to_watch is not set then there is no work pending */
5783 if (!eop_desc)
5784 break;
5785
5786 /* if DD is not set pending work has not been completed */
5787 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5788 break;
5789
5790 /* clear next_to_watch to prevent false hangs */
5791 tx_buffer->next_to_watch = NULL;
5792
5793 /* update the statistics for this packet */
5794 total_bytes += tx_buffer->bytecount;
5795 total_packets += tx_buffer->gso_segs;
5796
5797 #ifdef CONFIG_IGB_PTP
5798 /* retrieve hardware timestamp */
5799 igb_ptp_tx_hwtstamp(q_vector, tx_buffer);
5800 #endif /* CONFIG_IGB_PTP */
5801
5802 /* free the skb */
5803 dev_kfree_skb_any(tx_buffer->skb);
5804 tx_buffer->skb = NULL;
5805
5806 /* unmap skb header data */
5807 dma_unmap_single(tx_ring->dev,
5808 tx_buffer->dma,
5809 tx_buffer->length,
5810 DMA_TO_DEVICE);
5811
5812 /* clear last DMA location and unmap remaining buffers */
5813 while (tx_desc != eop_desc) {
5814 tx_buffer->dma = 0;
5815
5816 tx_buffer++;
5817 tx_desc++;
5818 i++;
5819 if (unlikely(!i)) {
5820 i -= tx_ring->count;
5821 tx_buffer = tx_ring->tx_buffer_info;
5822 tx_desc = IGB_TX_DESC(tx_ring, 0);
5823 }
5824
5825 /* unmap any remaining paged data */
5826 if (tx_buffer->dma) {
5827 dma_unmap_page(tx_ring->dev,
5828 tx_buffer->dma,
5829 tx_buffer->length,
5830 DMA_TO_DEVICE);
5831 }
5832 }
5833
5834 /* clear last DMA location */
5835 tx_buffer->dma = 0;
5836
5837 /* move us one more past the eop_desc for start of next pkt */
5838 tx_buffer++;
5839 tx_desc++;
5840 i++;
5841 if (unlikely(!i)) {
5842 i -= tx_ring->count;
5843 tx_buffer = tx_ring->tx_buffer_info;
5844 tx_desc = IGB_TX_DESC(tx_ring, 0);
5845 }
5846 }
5847
5848 netdev_tx_completed_queue(txring_txq(tx_ring),
5849 total_packets, total_bytes);
5850 i += tx_ring->count;
5851 tx_ring->next_to_clean = i;
5852 u64_stats_update_begin(&tx_ring->tx_syncp);
5853 tx_ring->tx_stats.bytes += total_bytes;
5854 tx_ring->tx_stats.packets += total_packets;
5855 u64_stats_update_end(&tx_ring->tx_syncp);
5856 q_vector->tx.total_bytes += total_bytes;
5857 q_vector->tx.total_packets += total_packets;
5858
5859 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
5860 struct e1000_hw *hw = &adapter->hw;
5861
5862 eop_desc = tx_buffer->next_to_watch;
5863
5864 /* Detect a transmit hang in hardware, this serializes the
5865 * check with the clearing of time_stamp and movement of i */
5866 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5867 if (eop_desc &&
5868 time_after(jiffies, tx_buffer->time_stamp +
5869 (adapter->tx_timeout_factor * HZ)) &&
5870 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5871
5872 /* detected Tx unit hang */
5873 dev_err(tx_ring->dev,
5874 "Detected Tx Unit Hang\n"
5875 " Tx Queue <%d>\n"
5876 " TDH <%x>\n"
5877 " TDT <%x>\n"
5878 " next_to_use <%x>\n"
5879 " next_to_clean <%x>\n"
5880 "buffer_info[next_to_clean]\n"
5881 " time_stamp <%lx>\n"
5882 " next_to_watch <%p>\n"
5883 " jiffies <%lx>\n"
5884 " desc.status <%x>\n",
5885 tx_ring->queue_index,
5886 rd32(E1000_TDH(tx_ring->reg_idx)),
5887 readl(tx_ring->tail),
5888 tx_ring->next_to_use,
5889 tx_ring->next_to_clean,
5890 tx_buffer->time_stamp,
5891 eop_desc,
5892 jiffies,
5893 eop_desc->wb.status);
5894 netif_stop_subqueue(tx_ring->netdev,
5895 tx_ring->queue_index);
5896
5897 /* we are about to reset, no point in enabling stuff */
5898 return true;
5899 }
5900 }
5901
5902 if (unlikely(total_packets &&
5903 netif_carrier_ok(tx_ring->netdev) &&
5904 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5905 /* Make sure that anybody stopping the queue after this
5906 * sees the new next_to_clean.
5907 */
5908 smp_mb();
5909 if (__netif_subqueue_stopped(tx_ring->netdev,
5910 tx_ring->queue_index) &&
5911 !(test_bit(__IGB_DOWN, &adapter->state))) {
5912 netif_wake_subqueue(tx_ring->netdev,
5913 tx_ring->queue_index);
5914
5915 u64_stats_update_begin(&tx_ring->tx_syncp);
5916 tx_ring->tx_stats.restart_queue++;
5917 u64_stats_update_end(&tx_ring->tx_syncp);
5918 }
5919 }
5920
5921 return !!budget;
5922 }
5923
5924 static inline void igb_rx_checksum(struct igb_ring *ring,
5925 union e1000_adv_rx_desc *rx_desc,
5926 struct sk_buff *skb)
5927 {
5928 skb_checksum_none_assert(skb);
5929
5930 /* Ignore Checksum bit is set */
5931 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
5932 return;
5933
5934 /* Rx checksum disabled via ethtool */
5935 if (!(ring->netdev->features & NETIF_F_RXCSUM))
5936 return;
5937
5938 /* TCP/UDP checksum error bit is set */
5939 if (igb_test_staterr(rx_desc,
5940 E1000_RXDEXT_STATERR_TCPE |
5941 E1000_RXDEXT_STATERR_IPE)) {
5942 /*
5943 * work around errata with sctp packets where the TCPE aka
5944 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5945 * packets, (aka let the stack check the crc32c)
5946 */
5947 if (!((skb->len == 60) &&
5948 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
5949 u64_stats_update_begin(&ring->rx_syncp);
5950 ring->rx_stats.csum_err++;
5951 u64_stats_update_end(&ring->rx_syncp);
5952 }
5953 /* let the stack verify checksum errors */
5954 return;
5955 }
5956 /* It must be a TCP or UDP packet with a valid checksum */
5957 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
5958 E1000_RXD_STAT_UDPCS))
5959 skb->ip_summed = CHECKSUM_UNNECESSARY;
5960
5961 dev_dbg(ring->dev, "cksum success: bits %08X\n",
5962 le32_to_cpu(rx_desc->wb.upper.status_error));
5963 }
5964
5965 static inline void igb_rx_hash(struct igb_ring *ring,
5966 union e1000_adv_rx_desc *rx_desc,
5967 struct sk_buff *skb)
5968 {
5969 if (ring->netdev->features & NETIF_F_RXHASH)
5970 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
5971 }
5972
5973 static void igb_rx_vlan(struct igb_ring *ring,
5974 union e1000_adv_rx_desc *rx_desc,
5975 struct sk_buff *skb)
5976 {
5977 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
5978 u16 vid;
5979 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
5980 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
5981 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
5982 else
5983 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
5984
5985 __vlan_hwaccel_put_tag(skb, vid);
5986 }
5987 }
5988
5989 static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
5990 {
5991 /* HW will not DMA in data larger than the given buffer, even if it
5992 * parses the (NFS, of course) header to be larger. In that case, it
5993 * fills the header buffer and spills the rest into the page.
5994 */
5995 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5996 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5997 if (hlen > IGB_RX_HDR_LEN)
5998 hlen = IGB_RX_HDR_LEN;
5999 return hlen;
6000 }
6001
6002 static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
6003 {
6004 struct igb_ring *rx_ring = q_vector->rx.ring;
6005 union e1000_adv_rx_desc *rx_desc;
6006 const int current_node = numa_node_id();
6007 unsigned int total_bytes = 0, total_packets = 0;
6008 u16 cleaned_count = igb_desc_unused(rx_ring);
6009 u16 i = rx_ring->next_to_clean;
6010
6011 rx_desc = IGB_RX_DESC(rx_ring, i);
6012
6013 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
6014 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
6015 struct sk_buff *skb = buffer_info->skb;
6016 union e1000_adv_rx_desc *next_rxd;
6017
6018 buffer_info->skb = NULL;
6019 prefetch(skb->data);
6020
6021 i++;
6022 if (i == rx_ring->count)
6023 i = 0;
6024
6025 next_rxd = IGB_RX_DESC(rx_ring, i);
6026 prefetch(next_rxd);
6027
6028 /*
6029 * This memory barrier is needed to keep us from reading
6030 * any other fields out of the rx_desc until we know the
6031 * RXD_STAT_DD bit is set
6032 */
6033 rmb();
6034
6035 if (!skb_is_nonlinear(skb)) {
6036 __skb_put(skb, igb_get_hlen(rx_desc));
6037 dma_unmap_single(rx_ring->dev, buffer_info->dma,
6038 IGB_RX_HDR_LEN,
6039 DMA_FROM_DEVICE);
6040 buffer_info->dma = 0;
6041 }
6042
6043 if (rx_desc->wb.upper.length) {
6044 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
6045
6046 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
6047 buffer_info->page,
6048 buffer_info->page_offset,
6049 length);
6050
6051 skb->len += length;
6052 skb->data_len += length;
6053 skb->truesize += PAGE_SIZE / 2;
6054
6055 if ((page_count(buffer_info->page) != 1) ||
6056 (page_to_nid(buffer_info->page) != current_node))
6057 buffer_info->page = NULL;
6058 else
6059 get_page(buffer_info->page);
6060
6061 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
6062 PAGE_SIZE / 2, DMA_FROM_DEVICE);
6063 buffer_info->page_dma = 0;
6064 }
6065
6066 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
6067 struct igb_rx_buffer *next_buffer;
6068 next_buffer = &rx_ring->rx_buffer_info[i];
6069 buffer_info->skb = next_buffer->skb;
6070 buffer_info->dma = next_buffer->dma;
6071 next_buffer->skb = skb;
6072 next_buffer->dma = 0;
6073 goto next_desc;
6074 }
6075
6076 if (unlikely((igb_test_staterr(rx_desc,
6077 E1000_RXDEXT_ERR_FRAME_ERR_MASK))
6078 && !(rx_ring->netdev->features & NETIF_F_RXALL))) {
6079 dev_kfree_skb_any(skb);
6080 goto next_desc;
6081 }
6082
6083 #ifdef CONFIG_IGB_PTP
6084 igb_ptp_rx_hwtstamp(q_vector, rx_desc, skb);
6085 #endif /* CONFIG_IGB_PTP */
6086 igb_rx_hash(rx_ring, rx_desc, skb);
6087 igb_rx_checksum(rx_ring, rx_desc, skb);
6088 igb_rx_vlan(rx_ring, rx_desc, skb);
6089
6090 total_bytes += skb->len;
6091 total_packets++;
6092
6093 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
6094
6095 napi_gro_receive(&q_vector->napi, skb);
6096
6097 budget--;
6098 next_desc:
6099 if (!budget)
6100 break;
6101
6102 cleaned_count++;
6103 /* return some buffers to hardware, one at a time is too slow */
6104 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
6105 igb_alloc_rx_buffers(rx_ring, cleaned_count);
6106 cleaned_count = 0;
6107 }
6108
6109 /* use prefetched values */
6110 rx_desc = next_rxd;
6111 }
6112
6113 rx_ring->next_to_clean = i;
6114 u64_stats_update_begin(&rx_ring->rx_syncp);
6115 rx_ring->rx_stats.packets += total_packets;
6116 rx_ring->rx_stats.bytes += total_bytes;
6117 u64_stats_update_end(&rx_ring->rx_syncp);
6118 q_vector->rx.total_packets += total_packets;
6119 q_vector->rx.total_bytes += total_bytes;
6120
6121 if (cleaned_count)
6122 igb_alloc_rx_buffers(rx_ring, cleaned_count);
6123
6124 return !!budget;
6125 }
6126
6127 static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
6128 struct igb_rx_buffer *bi)
6129 {
6130 struct sk_buff *skb = bi->skb;
6131 dma_addr_t dma = bi->dma;
6132
6133 if (dma)
6134 return true;
6135
6136 if (likely(!skb)) {
6137 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6138 IGB_RX_HDR_LEN);
6139 bi->skb = skb;
6140 if (!skb) {
6141 rx_ring->rx_stats.alloc_failed++;
6142 return false;
6143 }
6144
6145 /* initialize skb for ring */
6146 skb_record_rx_queue(skb, rx_ring->queue_index);
6147 }
6148
6149 dma = dma_map_single(rx_ring->dev, skb->data,
6150 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6151
6152 if (dma_mapping_error(rx_ring->dev, dma)) {
6153 rx_ring->rx_stats.alloc_failed++;
6154 return false;
6155 }
6156
6157 bi->dma = dma;
6158 return true;
6159 }
6160
6161 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
6162 struct igb_rx_buffer *bi)
6163 {
6164 struct page *page = bi->page;
6165 dma_addr_t page_dma = bi->page_dma;
6166 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6167
6168 if (page_dma)
6169 return true;
6170
6171 if (!page) {
6172 page = __skb_alloc_page(GFP_ATOMIC, bi->skb);
6173 bi->page = page;
6174 if (unlikely(!page)) {
6175 rx_ring->rx_stats.alloc_failed++;
6176 return false;
6177 }
6178 }
6179
6180 page_dma = dma_map_page(rx_ring->dev, page,
6181 page_offset, PAGE_SIZE / 2,
6182 DMA_FROM_DEVICE);
6183
6184 if (dma_mapping_error(rx_ring->dev, page_dma)) {
6185 rx_ring->rx_stats.alloc_failed++;
6186 return false;
6187 }
6188
6189 bi->page_dma = page_dma;
6190 bi->page_offset = page_offset;
6191 return true;
6192 }
6193
6194 /**
6195 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
6196 * @adapter: address of board private structure
6197 **/
6198 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
6199 {
6200 union e1000_adv_rx_desc *rx_desc;
6201 struct igb_rx_buffer *bi;
6202 u16 i = rx_ring->next_to_use;
6203
6204 rx_desc = IGB_RX_DESC(rx_ring, i);
6205 bi = &rx_ring->rx_buffer_info[i];
6206 i -= rx_ring->count;
6207
6208 while (cleaned_count--) {
6209 if (!igb_alloc_mapped_skb(rx_ring, bi))
6210 break;
6211
6212 /* Refresh the desc even if buffer_addrs didn't change
6213 * because each write-back erases this info. */
6214 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
6215
6216 if (!igb_alloc_mapped_page(rx_ring, bi))
6217 break;
6218
6219 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
6220
6221 rx_desc++;
6222 bi++;
6223 i++;
6224 if (unlikely(!i)) {
6225 rx_desc = IGB_RX_DESC(rx_ring, 0);
6226 bi = rx_ring->rx_buffer_info;
6227 i -= rx_ring->count;
6228 }
6229
6230 /* clear the hdr_addr for the next_to_use descriptor */
6231 rx_desc->read.hdr_addr = 0;
6232 }
6233
6234 i += rx_ring->count;
6235
6236 if (rx_ring->next_to_use != i) {
6237 rx_ring->next_to_use = i;
6238
6239 /* Force memory writes to complete before letting h/w
6240 * know there are new descriptors to fetch. (Only
6241 * applicable for weak-ordered memory model archs,
6242 * such as IA-64). */
6243 wmb();
6244 writel(i, rx_ring->tail);
6245 }
6246 }
6247
6248 /**
6249 * igb_mii_ioctl -
6250 * @netdev:
6251 * @ifreq:
6252 * @cmd:
6253 **/
6254 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6255 {
6256 struct igb_adapter *adapter = netdev_priv(netdev);
6257 struct mii_ioctl_data *data = if_mii(ifr);
6258
6259 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6260 return -EOPNOTSUPP;
6261
6262 switch (cmd) {
6263 case SIOCGMIIPHY:
6264 data->phy_id = adapter->hw.phy.addr;
6265 break;
6266 case SIOCGMIIREG:
6267 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6268 &data->val_out))
6269 return -EIO;
6270 break;
6271 case SIOCSMIIREG:
6272 default:
6273 return -EOPNOTSUPP;
6274 }
6275 return 0;
6276 }
6277
6278 /**
6279 * igb_ioctl -
6280 * @netdev:
6281 * @ifreq:
6282 * @cmd:
6283 **/
6284 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6285 {
6286 switch (cmd) {
6287 case SIOCGMIIPHY:
6288 case SIOCGMIIREG:
6289 case SIOCSMIIREG:
6290 return igb_mii_ioctl(netdev, ifr, cmd);
6291 #ifdef CONFIG_IGB_PTP
6292 case SIOCSHWTSTAMP:
6293 return igb_ptp_hwtstamp_ioctl(netdev, ifr, cmd);
6294 #endif /* CONFIG_IGB_PTP */
6295 default:
6296 return -EOPNOTSUPP;
6297 }
6298 }
6299
6300 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6301 {
6302 struct igb_adapter *adapter = hw->back;
6303 u16 cap_offset;
6304
6305 cap_offset = adapter->pdev->pcie_cap;
6306 if (!cap_offset)
6307 return -E1000_ERR_CONFIG;
6308
6309 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6310
6311 return 0;
6312 }
6313
6314 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6315 {
6316 struct igb_adapter *adapter = hw->back;
6317 u16 cap_offset;
6318
6319 cap_offset = adapter->pdev->pcie_cap;
6320 if (!cap_offset)
6321 return -E1000_ERR_CONFIG;
6322
6323 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6324
6325 return 0;
6326 }
6327
6328 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
6329 {
6330 struct igb_adapter *adapter = netdev_priv(netdev);
6331 struct e1000_hw *hw = &adapter->hw;
6332 u32 ctrl, rctl;
6333 bool enable = !!(features & NETIF_F_HW_VLAN_RX);
6334
6335 if (enable) {
6336 /* enable VLAN tag insert/strip */
6337 ctrl = rd32(E1000_CTRL);
6338 ctrl |= E1000_CTRL_VME;
6339 wr32(E1000_CTRL, ctrl);
6340
6341 /* Disable CFI check */
6342 rctl = rd32(E1000_RCTL);
6343 rctl &= ~E1000_RCTL_CFIEN;
6344 wr32(E1000_RCTL, rctl);
6345 } else {
6346 /* disable VLAN tag insert/strip */
6347 ctrl = rd32(E1000_CTRL);
6348 ctrl &= ~E1000_CTRL_VME;
6349 wr32(E1000_CTRL, ctrl);
6350 }
6351
6352 igb_rlpml_set(adapter);
6353 }
6354
6355 static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6356 {
6357 struct igb_adapter *adapter = netdev_priv(netdev);
6358 struct e1000_hw *hw = &adapter->hw;
6359 int pf_id = adapter->vfs_allocated_count;
6360
6361 /* attempt to add filter to vlvf array */
6362 igb_vlvf_set(adapter, vid, true, pf_id);
6363
6364 /* add the filter since PF can receive vlans w/o entry in vlvf */
6365 igb_vfta_set(hw, vid, true);
6366
6367 set_bit(vid, adapter->active_vlans);
6368
6369 return 0;
6370 }
6371
6372 static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6373 {
6374 struct igb_adapter *adapter = netdev_priv(netdev);
6375 struct e1000_hw *hw = &adapter->hw;
6376 int pf_id = adapter->vfs_allocated_count;
6377 s32 err;
6378
6379 /* remove vlan from VLVF table array */
6380 err = igb_vlvf_set(adapter, vid, false, pf_id);
6381
6382 /* if vid was not present in VLVF just remove it from table */
6383 if (err)
6384 igb_vfta_set(hw, vid, false);
6385
6386 clear_bit(vid, adapter->active_vlans);
6387
6388 return 0;
6389 }
6390
6391 static void igb_restore_vlan(struct igb_adapter *adapter)
6392 {
6393 u16 vid;
6394
6395 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
6396
6397 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6398 igb_vlan_rx_add_vid(adapter->netdev, vid);
6399 }
6400
6401 int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
6402 {
6403 struct pci_dev *pdev = adapter->pdev;
6404 struct e1000_mac_info *mac = &adapter->hw.mac;
6405
6406 mac->autoneg = 0;
6407
6408 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6409 * for the switch() below to work */
6410 if ((spd & 1) || (dplx & ~1))
6411 goto err_inval;
6412
6413 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6414 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6415 spd != SPEED_1000 &&
6416 dplx != DUPLEX_FULL)
6417 goto err_inval;
6418
6419 switch (spd + dplx) {
6420 case SPEED_10 + DUPLEX_HALF:
6421 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6422 break;
6423 case SPEED_10 + DUPLEX_FULL:
6424 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6425 break;
6426 case SPEED_100 + DUPLEX_HALF:
6427 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6428 break;
6429 case SPEED_100 + DUPLEX_FULL:
6430 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6431 break;
6432 case SPEED_1000 + DUPLEX_FULL:
6433 mac->autoneg = 1;
6434 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6435 break;
6436 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6437 default:
6438 goto err_inval;
6439 }
6440
6441 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
6442 adapter->hw.phy.mdix = AUTO_ALL_MODES;
6443
6444 return 0;
6445
6446 err_inval:
6447 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6448 return -EINVAL;
6449 }
6450
6451 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
6452 bool runtime)
6453 {
6454 struct net_device *netdev = pci_get_drvdata(pdev);
6455 struct igb_adapter *adapter = netdev_priv(netdev);
6456 struct e1000_hw *hw = &adapter->hw;
6457 u32 ctrl, rctl, status;
6458 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
6459 #ifdef CONFIG_PM
6460 int retval = 0;
6461 #endif
6462
6463 netif_device_detach(netdev);
6464
6465 if (netif_running(netdev))
6466 __igb_close(netdev, true);
6467
6468 igb_clear_interrupt_scheme(adapter);
6469
6470 #ifdef CONFIG_PM
6471 retval = pci_save_state(pdev);
6472 if (retval)
6473 return retval;
6474 #endif
6475
6476 status = rd32(E1000_STATUS);
6477 if (status & E1000_STATUS_LU)
6478 wufc &= ~E1000_WUFC_LNKC;
6479
6480 if (wufc) {
6481 igb_setup_rctl(adapter);
6482 igb_set_rx_mode(netdev);
6483
6484 /* turn on all-multi mode if wake on multicast is enabled */
6485 if (wufc & E1000_WUFC_MC) {
6486 rctl = rd32(E1000_RCTL);
6487 rctl |= E1000_RCTL_MPE;
6488 wr32(E1000_RCTL, rctl);
6489 }
6490
6491 ctrl = rd32(E1000_CTRL);
6492 /* advertise wake from D3Cold */
6493 #define E1000_CTRL_ADVD3WUC 0x00100000
6494 /* phy power management enable */
6495 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6496 ctrl |= E1000_CTRL_ADVD3WUC;
6497 wr32(E1000_CTRL, ctrl);
6498
6499 /* Allow time for pending master requests to run */
6500 igb_disable_pcie_master(hw);
6501
6502 wr32(E1000_WUC, E1000_WUC_PME_EN);
6503 wr32(E1000_WUFC, wufc);
6504 } else {
6505 wr32(E1000_WUC, 0);
6506 wr32(E1000_WUFC, 0);
6507 }
6508
6509 *enable_wake = wufc || adapter->en_mng_pt;
6510 if (!*enable_wake)
6511 igb_power_down_link(adapter);
6512 else
6513 igb_power_up_link(adapter);
6514
6515 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6516 * would have already happened in close and is redundant. */
6517 igb_release_hw_control(adapter);
6518
6519 pci_disable_device(pdev);
6520
6521 return 0;
6522 }
6523
6524 #ifdef CONFIG_PM
6525 #ifdef CONFIG_PM_SLEEP
6526 static int igb_suspend(struct device *dev)
6527 {
6528 int retval;
6529 bool wake;
6530 struct pci_dev *pdev = to_pci_dev(dev);
6531
6532 retval = __igb_shutdown(pdev, &wake, 0);
6533 if (retval)
6534 return retval;
6535
6536 if (wake) {
6537 pci_prepare_to_sleep(pdev);
6538 } else {
6539 pci_wake_from_d3(pdev, false);
6540 pci_set_power_state(pdev, PCI_D3hot);
6541 }
6542
6543 return 0;
6544 }
6545 #endif /* CONFIG_PM_SLEEP */
6546
6547 static int igb_resume(struct device *dev)
6548 {
6549 struct pci_dev *pdev = to_pci_dev(dev);
6550 struct net_device *netdev = pci_get_drvdata(pdev);
6551 struct igb_adapter *adapter = netdev_priv(netdev);
6552 struct e1000_hw *hw = &adapter->hw;
6553 u32 err;
6554
6555 pci_set_power_state(pdev, PCI_D0);
6556 pci_restore_state(pdev);
6557 pci_save_state(pdev);
6558
6559 err = pci_enable_device_mem(pdev);
6560 if (err) {
6561 dev_err(&pdev->dev,
6562 "igb: Cannot enable PCI device from suspend\n");
6563 return err;
6564 }
6565 pci_set_master(pdev);
6566
6567 pci_enable_wake(pdev, PCI_D3hot, 0);
6568 pci_enable_wake(pdev, PCI_D3cold, 0);
6569
6570 if (igb_init_interrupt_scheme(adapter)) {
6571 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6572 return -ENOMEM;
6573 }
6574
6575 igb_reset(adapter);
6576
6577 /* let the f/w know that the h/w is now under the control of the
6578 * driver. */
6579 igb_get_hw_control(adapter);
6580
6581 wr32(E1000_WUS, ~0);
6582
6583 if (netdev->flags & IFF_UP) {
6584 err = __igb_open(netdev, true);
6585 if (err)
6586 return err;
6587 }
6588
6589 netif_device_attach(netdev);
6590 return 0;
6591 }
6592
6593 #ifdef CONFIG_PM_RUNTIME
6594 static int igb_runtime_idle(struct device *dev)
6595 {
6596 struct pci_dev *pdev = to_pci_dev(dev);
6597 struct net_device *netdev = pci_get_drvdata(pdev);
6598 struct igb_adapter *adapter = netdev_priv(netdev);
6599
6600 if (!igb_has_link(adapter))
6601 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6602
6603 return -EBUSY;
6604 }
6605
6606 static int igb_runtime_suspend(struct device *dev)
6607 {
6608 struct pci_dev *pdev = to_pci_dev(dev);
6609 int retval;
6610 bool wake;
6611
6612 retval = __igb_shutdown(pdev, &wake, 1);
6613 if (retval)
6614 return retval;
6615
6616 if (wake) {
6617 pci_prepare_to_sleep(pdev);
6618 } else {
6619 pci_wake_from_d3(pdev, false);
6620 pci_set_power_state(pdev, PCI_D3hot);
6621 }
6622
6623 return 0;
6624 }
6625
6626 static int igb_runtime_resume(struct device *dev)
6627 {
6628 return igb_resume(dev);
6629 }
6630 #endif /* CONFIG_PM_RUNTIME */
6631 #endif
6632
6633 static void igb_shutdown(struct pci_dev *pdev)
6634 {
6635 bool wake;
6636
6637 __igb_shutdown(pdev, &wake, 0);
6638
6639 if (system_state == SYSTEM_POWER_OFF) {
6640 pci_wake_from_d3(pdev, wake);
6641 pci_set_power_state(pdev, PCI_D3hot);
6642 }
6643 }
6644
6645 #ifdef CONFIG_NET_POLL_CONTROLLER
6646 /*
6647 * Polling 'interrupt' - used by things like netconsole to send skbs
6648 * without having to re-enable interrupts. It's not called while
6649 * the interrupt routine is executing.
6650 */
6651 static void igb_netpoll(struct net_device *netdev)
6652 {
6653 struct igb_adapter *adapter = netdev_priv(netdev);
6654 struct e1000_hw *hw = &adapter->hw;
6655 struct igb_q_vector *q_vector;
6656 int i;
6657
6658 for (i = 0; i < adapter->num_q_vectors; i++) {
6659 q_vector = adapter->q_vector[i];
6660 if (adapter->msix_entries)
6661 wr32(E1000_EIMC, q_vector->eims_value);
6662 else
6663 igb_irq_disable(adapter);
6664 napi_schedule(&q_vector->napi);
6665 }
6666 }
6667 #endif /* CONFIG_NET_POLL_CONTROLLER */
6668
6669 /**
6670 * igb_io_error_detected - called when PCI error is detected
6671 * @pdev: Pointer to PCI device
6672 * @state: The current pci connection state
6673 *
6674 * This function is called after a PCI bus error affecting
6675 * this device has been detected.
6676 */
6677 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6678 pci_channel_state_t state)
6679 {
6680 struct net_device *netdev = pci_get_drvdata(pdev);
6681 struct igb_adapter *adapter = netdev_priv(netdev);
6682
6683 netif_device_detach(netdev);
6684
6685 if (state == pci_channel_io_perm_failure)
6686 return PCI_ERS_RESULT_DISCONNECT;
6687
6688 if (netif_running(netdev))
6689 igb_down(adapter);
6690 pci_disable_device(pdev);
6691
6692 /* Request a slot slot reset. */
6693 return PCI_ERS_RESULT_NEED_RESET;
6694 }
6695
6696 /**
6697 * igb_io_slot_reset - called after the pci bus has been reset.
6698 * @pdev: Pointer to PCI device
6699 *
6700 * Restart the card from scratch, as if from a cold-boot. Implementation
6701 * resembles the first-half of the igb_resume routine.
6702 */
6703 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6704 {
6705 struct net_device *netdev = pci_get_drvdata(pdev);
6706 struct igb_adapter *adapter = netdev_priv(netdev);
6707 struct e1000_hw *hw = &adapter->hw;
6708 pci_ers_result_t result;
6709 int err;
6710
6711 if (pci_enable_device_mem(pdev)) {
6712 dev_err(&pdev->dev,
6713 "Cannot re-enable PCI device after reset.\n");
6714 result = PCI_ERS_RESULT_DISCONNECT;
6715 } else {
6716 pci_set_master(pdev);
6717 pci_restore_state(pdev);
6718 pci_save_state(pdev);
6719
6720 pci_enable_wake(pdev, PCI_D3hot, 0);
6721 pci_enable_wake(pdev, PCI_D3cold, 0);
6722
6723 igb_reset(adapter);
6724 wr32(E1000_WUS, ~0);
6725 result = PCI_ERS_RESULT_RECOVERED;
6726 }
6727
6728 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6729 if (err) {
6730 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6731 "failed 0x%0x\n", err);
6732 /* non-fatal, continue */
6733 }
6734
6735 return result;
6736 }
6737
6738 /**
6739 * igb_io_resume - called when traffic can start flowing again.
6740 * @pdev: Pointer to PCI device
6741 *
6742 * This callback is called when the error recovery driver tells us that
6743 * its OK to resume normal operation. Implementation resembles the
6744 * second-half of the igb_resume routine.
6745 */
6746 static void igb_io_resume(struct pci_dev *pdev)
6747 {
6748 struct net_device *netdev = pci_get_drvdata(pdev);
6749 struct igb_adapter *adapter = netdev_priv(netdev);
6750
6751 if (netif_running(netdev)) {
6752 if (igb_up(adapter)) {
6753 dev_err(&pdev->dev, "igb_up failed after reset\n");
6754 return;
6755 }
6756 }
6757
6758 netif_device_attach(netdev);
6759
6760 /* let the f/w know that the h/w is now under the control of the
6761 * driver. */
6762 igb_get_hw_control(adapter);
6763 }
6764
6765 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6766 u8 qsel)
6767 {
6768 u32 rar_low, rar_high;
6769 struct e1000_hw *hw = &adapter->hw;
6770
6771 /* HW expects these in little endian so we reverse the byte order
6772 * from network order (big endian) to little endian
6773 */
6774 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6775 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6776 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6777
6778 /* Indicate to hardware the Address is Valid. */
6779 rar_high |= E1000_RAH_AV;
6780
6781 if (hw->mac.type == e1000_82575)
6782 rar_high |= E1000_RAH_POOL_1 * qsel;
6783 else
6784 rar_high |= E1000_RAH_POOL_1 << qsel;
6785
6786 wr32(E1000_RAL(index), rar_low);
6787 wrfl();
6788 wr32(E1000_RAH(index), rar_high);
6789 wrfl();
6790 }
6791
6792 static int igb_set_vf_mac(struct igb_adapter *adapter,
6793 int vf, unsigned char *mac_addr)
6794 {
6795 struct e1000_hw *hw = &adapter->hw;
6796 /* VF MAC addresses start at end of receive addresses and moves
6797 * torwards the first, as a result a collision should not be possible */
6798 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6799
6800 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6801
6802 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6803
6804 return 0;
6805 }
6806
6807 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6808 {
6809 struct igb_adapter *adapter = netdev_priv(netdev);
6810 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6811 return -EINVAL;
6812 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6813 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6814 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6815 " change effective.");
6816 if (test_bit(__IGB_DOWN, &adapter->state)) {
6817 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6818 " but the PF device is not up.\n");
6819 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6820 " attempting to use the VF device.\n");
6821 }
6822 return igb_set_vf_mac(adapter, vf, mac);
6823 }
6824
6825 static int igb_link_mbps(int internal_link_speed)
6826 {
6827 switch (internal_link_speed) {
6828 case SPEED_100:
6829 return 100;
6830 case SPEED_1000:
6831 return 1000;
6832 default:
6833 return 0;
6834 }
6835 }
6836
6837 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6838 int link_speed)
6839 {
6840 int rf_dec, rf_int;
6841 u32 bcnrc_val;
6842
6843 if (tx_rate != 0) {
6844 /* Calculate the rate factor values to set */
6845 rf_int = link_speed / tx_rate;
6846 rf_dec = (link_speed - (rf_int * tx_rate));
6847 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6848
6849 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6850 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6851 E1000_RTTBCNRC_RF_INT_MASK);
6852 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6853 } else {
6854 bcnrc_val = 0;
6855 }
6856
6857 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
6858 /*
6859 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
6860 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
6861 */
6862 wr32(E1000_RTTBCNRM, 0x14);
6863 wr32(E1000_RTTBCNRC, bcnrc_val);
6864 }
6865
6866 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
6867 {
6868 int actual_link_speed, i;
6869 bool reset_rate = false;
6870
6871 /* VF TX rate limit was not set or not supported */
6872 if ((adapter->vf_rate_link_speed == 0) ||
6873 (adapter->hw.mac.type != e1000_82576))
6874 return;
6875
6876 actual_link_speed = igb_link_mbps(adapter->link_speed);
6877 if (actual_link_speed != adapter->vf_rate_link_speed) {
6878 reset_rate = true;
6879 adapter->vf_rate_link_speed = 0;
6880 dev_info(&adapter->pdev->dev,
6881 "Link speed has been changed. VF Transmit "
6882 "rate is disabled\n");
6883 }
6884
6885 for (i = 0; i < adapter->vfs_allocated_count; i++) {
6886 if (reset_rate)
6887 adapter->vf_data[i].tx_rate = 0;
6888
6889 igb_set_vf_rate_limit(&adapter->hw, i,
6890 adapter->vf_data[i].tx_rate,
6891 actual_link_speed);
6892 }
6893 }
6894
6895 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6896 {
6897 struct igb_adapter *adapter = netdev_priv(netdev);
6898 struct e1000_hw *hw = &adapter->hw;
6899 int actual_link_speed;
6900
6901 if (hw->mac.type != e1000_82576)
6902 return -EOPNOTSUPP;
6903
6904 actual_link_speed = igb_link_mbps(adapter->link_speed);
6905 if ((vf >= adapter->vfs_allocated_count) ||
6906 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
6907 (tx_rate < 0) || (tx_rate > actual_link_speed))
6908 return -EINVAL;
6909
6910 adapter->vf_rate_link_speed = actual_link_speed;
6911 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
6912 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
6913
6914 return 0;
6915 }
6916
6917 static int igb_ndo_get_vf_config(struct net_device *netdev,
6918 int vf, struct ifla_vf_info *ivi)
6919 {
6920 struct igb_adapter *adapter = netdev_priv(netdev);
6921 if (vf >= adapter->vfs_allocated_count)
6922 return -EINVAL;
6923 ivi->vf = vf;
6924 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6925 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
6926 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6927 ivi->qos = adapter->vf_data[vf].pf_qos;
6928 return 0;
6929 }
6930
6931 static void igb_vmm_control(struct igb_adapter *adapter)
6932 {
6933 struct e1000_hw *hw = &adapter->hw;
6934 u32 reg;
6935
6936 switch (hw->mac.type) {
6937 case e1000_82575:
6938 case e1000_i210:
6939 case e1000_i211:
6940 default:
6941 /* replication is not supported for 82575 */
6942 return;
6943 case e1000_82576:
6944 /* notify HW that the MAC is adding vlan tags */
6945 reg = rd32(E1000_DTXCTL);
6946 reg |= E1000_DTXCTL_VLAN_ADDED;
6947 wr32(E1000_DTXCTL, reg);
6948 case e1000_82580:
6949 /* enable replication vlan tag stripping */
6950 reg = rd32(E1000_RPLOLR);
6951 reg |= E1000_RPLOLR_STRVLAN;
6952 wr32(E1000_RPLOLR, reg);
6953 case e1000_i350:
6954 /* none of the above registers are supported by i350 */
6955 break;
6956 }
6957
6958 if (adapter->vfs_allocated_count) {
6959 igb_vmdq_set_loopback_pf(hw, true);
6960 igb_vmdq_set_replication_pf(hw, true);
6961 igb_vmdq_set_anti_spoofing_pf(hw, true,
6962 adapter->vfs_allocated_count);
6963 } else {
6964 igb_vmdq_set_loopback_pf(hw, false);
6965 igb_vmdq_set_replication_pf(hw, false);
6966 }
6967 }
6968
6969 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
6970 {
6971 struct e1000_hw *hw = &adapter->hw;
6972 u32 dmac_thr;
6973 u16 hwm;
6974
6975 if (hw->mac.type > e1000_82580) {
6976 if (adapter->flags & IGB_FLAG_DMAC) {
6977 u32 reg;
6978
6979 /* force threshold to 0. */
6980 wr32(E1000_DMCTXTH, 0);
6981
6982 /*
6983 * DMA Coalescing high water mark needs to be greater
6984 * than the Rx threshold. Set hwm to PBA - max frame
6985 * size in 16B units, capping it at PBA - 6KB.
6986 */
6987 hwm = 64 * pba - adapter->max_frame_size / 16;
6988 if (hwm < 64 * (pba - 6))
6989 hwm = 64 * (pba - 6);
6990 reg = rd32(E1000_FCRTC);
6991 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
6992 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
6993 & E1000_FCRTC_RTH_COAL_MASK);
6994 wr32(E1000_FCRTC, reg);
6995
6996 /*
6997 * Set the DMA Coalescing Rx threshold to PBA - 2 * max
6998 * frame size, capping it at PBA - 10KB.
6999 */
7000 dmac_thr = pba - adapter->max_frame_size / 512;
7001 if (dmac_thr < pba - 10)
7002 dmac_thr = pba - 10;
7003 reg = rd32(E1000_DMACR);
7004 reg &= ~E1000_DMACR_DMACTHR_MASK;
7005 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
7006 & E1000_DMACR_DMACTHR_MASK);
7007
7008 /* transition to L0x or L1 if available..*/
7009 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
7010
7011 /* watchdog timer= +-1000 usec in 32usec intervals */
7012 reg |= (1000 >> 5);
7013
7014 /* Disable BMC-to-OS Watchdog Enable */
7015 reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
7016 wr32(E1000_DMACR, reg);
7017
7018 /*
7019 * no lower threshold to disable
7020 * coalescing(smart fifb)-UTRESH=0
7021 */
7022 wr32(E1000_DMCRTRH, 0);
7023
7024 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
7025
7026 wr32(E1000_DMCTLX, reg);
7027
7028 /*
7029 * free space in tx packet buffer to wake from
7030 * DMA coal
7031 */
7032 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
7033 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
7034
7035 /*
7036 * make low power state decision controlled
7037 * by DMA coal
7038 */
7039 reg = rd32(E1000_PCIEMISC);
7040 reg &= ~E1000_PCIEMISC_LX_DECISION;
7041 wr32(E1000_PCIEMISC, reg);
7042 } /* endif adapter->dmac is not disabled */
7043 } else if (hw->mac.type == e1000_82580) {
7044 u32 reg = rd32(E1000_PCIEMISC);
7045 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
7046 wr32(E1000_DMACR, 0);
7047 }
7048 }
7049
7050 /* igb_main.c */
Linux kernel - Deliverables
This page took 0.188932 seconds and 5 git commands to generate.