projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net
[deliverable/linux.git] / drivers / net / ethernet / intel / ixgbe / ixgbe_main.c
1 /*******************************************************************************
2
3 Intel 10 Gigabit PCI Express Linux driver
4 Copyright(c) 1999 - 2016 Intel Corporation.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 Linux NICS <linux.nics@intel.com>
24 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
25 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26
27 *******************************************************************************/
28
29 #include <linux/types.h>
30 #include <linux/module.h>
31 #include <linux/pci.h>
32 #include <linux/netdevice.h>
33 #include <linux/vmalloc.h>
34 #include <linux/string.h>
35 #include <linux/in.h>
36 #include <linux/interrupt.h>
37 #include <linux/ip.h>
38 #include <linux/tcp.h>
39 #include <linux/sctp.h>
40 #include <linux/pkt_sched.h>
41 #include <linux/ipv6.h>
42 #include <linux/slab.h>
43 #include <net/checksum.h>
44 #include <net/ip6_checksum.h>
45 #include <linux/etherdevice.h>
46 #include <linux/ethtool.h>
47 #include <linux/if.h>
48 #include <linux/if_vlan.h>
49 #include <linux/if_macvlan.h>
50 #include <linux/if_bridge.h>
51 #include <linux/prefetch.h>
52 #include <scsi/fc/fc_fcoe.h>
53 #include <net/vxlan.h>
54 #include <net/pkt_cls.h>
55 #include <net/tc_act/tc_gact.h>
56
57 #include "ixgbe.h"
58 #include "ixgbe_common.h"
59 #include "ixgbe_dcb_82599.h"
60 #include "ixgbe_sriov.h"
61 #include "ixgbe_model.h"
62
63 char ixgbe_driver_name[] = "ixgbe";
64 static const char ixgbe_driver_string[] =
65 "Intel(R) 10 Gigabit PCI Express Network Driver";
66 #ifdef IXGBE_FCOE
67 char ixgbe_default_device_descr[] =
68 "Intel(R) 10 Gigabit Network Connection";
69 #else
70 static char ixgbe_default_device_descr[] =
71 "Intel(R) 10 Gigabit Network Connection";
72 #endif
73 #define DRV_VERSION "4.4.0-k"
74 const char ixgbe_driver_version[] = DRV_VERSION;
75 static const char ixgbe_copyright[] =
76 "Copyright (c) 1999-2016 Intel Corporation.";
77
78 static const char ixgbe_overheat_msg[] = "Network adapter has been stopped because it has over heated. Restart the computer. If the problem persists, power off the system and replace the adapter";
79
80 static const struct ixgbe_info *ixgbe_info_tbl[] = {
81 [board_82598] = &ixgbe_82598_info,
82 [board_82599] = &ixgbe_82599_info,
83 [board_X540] = &ixgbe_X540_info,
84 [board_X550] = &ixgbe_X550_info,
85 [board_X550EM_x] = &ixgbe_X550EM_x_info,
86 [board_x550em_a] = &ixgbe_x550em_a_info,
87 };
88
89 /* ixgbe_pci_tbl - PCI Device ID Table
90 *
91 * Wildcard entries (PCI_ANY_ID) should come last
92 * Last entry must be all 0s
93 *
94 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
95 * Class, Class Mask, private data (not used) }
96 */
97 static const struct pci_device_id ixgbe_pci_tbl[] = {
98 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
99 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
100 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
101 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
102 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
103 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
104 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
105 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
106 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
107 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
108 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
109 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
110 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
111 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
112 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
113 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
114 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
115 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
116 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
117 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
118 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
119 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
120 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
121 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
122 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
123 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
124 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP), board_82599 },
125 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
126 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
127 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },
128 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T), board_X550},
129 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T1), board_X550},
130 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KX4), board_X550EM_x},
131 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KR), board_X550EM_x},
132 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_10G_T), board_X550EM_x},
133 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_SFP), board_X550EM_x},
134 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR), board_x550em_a },
135 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR_L), board_x550em_a },
136 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N), board_x550em_a },
137 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII), board_x550em_a },
138 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L), board_x550em_a },
139 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP), board_x550em_a },
140 /* required last entry */
141 {0, }
142 };
143 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
144
145 #ifdef CONFIG_IXGBE_DCA
146 static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
147 void *p);
148 static struct notifier_block dca_notifier = {
149 .notifier_call = ixgbe_notify_dca,
150 .next = NULL,
151 .priority = 0
152 };
153 #endif
154
155 #ifdef CONFIG_PCI_IOV
156 static unsigned int max_vfs;
157 module_param(max_vfs, uint, 0);
158 MODULE_PARM_DESC(max_vfs,
159 "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63. (Deprecated)");
160 #endif /* CONFIG_PCI_IOV */
161
162 static unsigned int allow_unsupported_sfp;
163 module_param(allow_unsupported_sfp, uint, 0);
164 MODULE_PARM_DESC(allow_unsupported_sfp,
165 "Allow unsupported and untested SFP+ modules on 82599-based adapters");
166
167 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
168 static int debug = -1;
169 module_param(debug, int, 0);
170 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
171
172 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
173 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
174 MODULE_LICENSE("GPL");
175 MODULE_VERSION(DRV_VERSION);
176
177 static struct workqueue_struct *ixgbe_wq;
178
179 static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev);
180
181 static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter,
182 u32 reg, u16 *value)
183 {
184 struct pci_dev *parent_dev;
185 struct pci_bus *parent_bus;
186
187 parent_bus = adapter->pdev->bus->parent;
188 if (!parent_bus)
189 return -1;
190
191 parent_dev = parent_bus->self;
192 if (!parent_dev)
193 return -1;
194
195 if (!pci_is_pcie(parent_dev))
196 return -1;
197
198 pcie_capability_read_word(parent_dev, reg, value);
199 if (*value == IXGBE_FAILED_READ_CFG_WORD &&
200 ixgbe_check_cfg_remove(&adapter->hw, parent_dev))
201 return -1;
202 return 0;
203 }
204
205 static s32 ixgbe_get_parent_bus_info(struct ixgbe_adapter *adapter)
206 {
207 struct ixgbe_hw *hw = &adapter->hw;
208 u16 link_status = 0;
209 int err;
210
211 hw->bus.type = ixgbe_bus_type_pci_express;
212
213 /* Get the negotiated link width and speed from PCI config space of the
214 * parent, as this device is behind a switch
215 */
216 err = ixgbe_read_pci_cfg_word_parent(adapter, 18, &link_status);
217
218 /* assume caller will handle error case */
219 if (err)
220 return err;
221
222 hw->bus.width = ixgbe_convert_bus_width(link_status);
223 hw->bus.speed = ixgbe_convert_bus_speed(link_status);
224
225 return 0;
226 }
227
228 /**
229 * ixgbe_check_from_parent - Determine whether PCIe info should come from parent
230 * @hw: hw specific details
231 *
232 * This function is used by probe to determine whether a device's PCI-Express
233 * bandwidth details should be gathered from the parent bus instead of from the
234 * device. Used to ensure that various locations all have the correct device ID
235 * checks.
236 */
237 static inline bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw)
238 {
239 switch (hw->device_id) {
240 case IXGBE_DEV_ID_82599_SFP_SF_QP:
241 case IXGBE_DEV_ID_82599_QSFP_SF_QP:
242 return true;
243 default:
244 return false;
245 }
246 }
247
248 static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter,
249 int expected_gts)
250 {
251 struct ixgbe_hw *hw = &adapter->hw;
252 int max_gts = 0;
253 enum pci_bus_speed speed = PCI_SPEED_UNKNOWN;
254 enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
255 struct pci_dev *pdev;
256
257 /* Some devices are not connected over PCIe and thus do not negotiate
258 * speed. These devices do not have valid bus info, and thus any report
259 * we generate may not be correct.
260 */
261 if (hw->bus.type == ixgbe_bus_type_internal)
262 return;
263
264 /* determine whether to use the parent device */
265 if (ixgbe_pcie_from_parent(&adapter->hw))
266 pdev = adapter->pdev->bus->parent->self;
267 else
268 pdev = adapter->pdev;
269
270 if (pcie_get_minimum_link(pdev, &speed, &width) ||
271 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) {
272 e_dev_warn("Unable to determine PCI Express bandwidth.\n");
273 return;
274 }
275
276 switch (speed) {
277 case PCIE_SPEED_2_5GT:
278 /* 8b/10b encoding reduces max throughput by 20% */
279 max_gts = 2 * width;
280 break;
281 case PCIE_SPEED_5_0GT:
282 /* 8b/10b encoding reduces max throughput by 20% */
283 max_gts = 4 * width;
284 break;
285 case PCIE_SPEED_8_0GT:
286 /* 128b/130b encoding reduces throughput by less than 2% */
287 max_gts = 8 * width;
288 break;
289 default:
290 e_dev_warn("Unable to determine PCI Express bandwidth.\n");
291 return;
292 }
293
294 e_dev_info("PCI Express bandwidth of %dGT/s available\n",
295 max_gts);
296 e_dev_info("(Speed:%s, Width: x%d, Encoding Loss:%s)\n",
297 (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" :
298 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" :
299 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" :
300 "Unknown"),
301 width,
302 (speed == PCIE_SPEED_2_5GT ? "20%" :
303 speed == PCIE_SPEED_5_0GT ? "20%" :
304 speed == PCIE_SPEED_8_0GT ? "<2%" :
305 "Unknown"));
306
307 if (max_gts < expected_gts) {
308 e_dev_warn("This is not sufficient for optimal performance of this card.\n");
309 e_dev_warn("For optimal performance, at least %dGT/s of bandwidth is required.\n",
310 expected_gts);
311 e_dev_warn("A slot with more lanes and/or higher speed is suggested.\n");
312 }
313 }
314
315 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
316 {
317 if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
318 !test_bit(__IXGBE_REMOVING, &adapter->state) &&
319 !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
320 queue_work(ixgbe_wq, &adapter->service_task);
321 }
322
323 static void ixgbe_remove_adapter(struct ixgbe_hw *hw)
324 {
325 struct ixgbe_adapter *adapter = hw->back;
326
327 if (!hw->hw_addr)
328 return;
329 hw->hw_addr = NULL;
330 e_dev_err("Adapter removed\n");
331 if (test_bit(__IXGBE_SERVICE_INITED, &adapter->state))
332 ixgbe_service_event_schedule(adapter);
333 }
334
335 static void ixgbe_check_remove(struct ixgbe_hw *hw, u32 reg)
336 {
337 u32 value;
338
339 /* The following check not only optimizes a bit by not
340 * performing a read on the status register when the
341 * register just read was a status register read that
342 * returned IXGBE_FAILED_READ_REG. It also blocks any
343 * potential recursion.
344 */
345 if (reg == IXGBE_STATUS) {
346 ixgbe_remove_adapter(hw);
347 return;
348 }
349 value = ixgbe_read_reg(hw, IXGBE_STATUS);
350 if (value == IXGBE_FAILED_READ_REG)
351 ixgbe_remove_adapter(hw);
352 }
353
354 /**
355 * ixgbe_read_reg - Read from device register
356 * @hw: hw specific details
357 * @reg: offset of register to read
358 *
359 * Returns : value read or IXGBE_FAILED_READ_REG if removed
360 *
361 * This function is used to read device registers. It checks for device
362 * removal by confirming any read that returns all ones by checking the
363 * status register value for all ones. This function avoids reading from
364 * the hardware if a removal was previously detected in which case it
365 * returns IXGBE_FAILED_READ_REG (all ones).
366 */
367 u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg)
368 {
369 u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr);
370 u32 value;
371
372 if (ixgbe_removed(reg_addr))
373 return IXGBE_FAILED_READ_REG;
374 value = readl(reg_addr + reg);
375 if (unlikely(value == IXGBE_FAILED_READ_REG))
376 ixgbe_check_remove(hw, reg);
377 return value;
378 }
379
380 static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev)
381 {
382 u16 value;
383
384 pci_read_config_word(pdev, PCI_VENDOR_ID, &value);
385 if (value == IXGBE_FAILED_READ_CFG_WORD) {
386 ixgbe_remove_adapter(hw);
387 return true;
388 }
389 return false;
390 }
391
392 u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg)
393 {
394 struct ixgbe_adapter *adapter = hw->back;
395 u16 value;
396
397 if (ixgbe_removed(hw->hw_addr))
398 return IXGBE_FAILED_READ_CFG_WORD;
399 pci_read_config_word(adapter->pdev, reg, &value);
400 if (value == IXGBE_FAILED_READ_CFG_WORD &&
401 ixgbe_check_cfg_remove(hw, adapter->pdev))
402 return IXGBE_FAILED_READ_CFG_WORD;
403 return value;
404 }
405
406 #ifdef CONFIG_PCI_IOV
407 static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg)
408 {
409 struct ixgbe_adapter *adapter = hw->back;
410 u32 value;
411
412 if (ixgbe_removed(hw->hw_addr))
413 return IXGBE_FAILED_READ_CFG_DWORD;
414 pci_read_config_dword(adapter->pdev, reg, &value);
415 if (value == IXGBE_FAILED_READ_CFG_DWORD &&
416 ixgbe_check_cfg_remove(hw, adapter->pdev))
417 return IXGBE_FAILED_READ_CFG_DWORD;
418 return value;
419 }
420 #endif /* CONFIG_PCI_IOV */
421
422 void ixgbe_write_pci_cfg_word(struct ixgbe_hw *hw, u32 reg, u16 value)
423 {
424 struct ixgbe_adapter *adapter = hw->back;
425
426 if (ixgbe_removed(hw->hw_addr))
427 return;
428 pci_write_config_word(adapter->pdev, reg, value);
429 }
430
431 static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
432 {
433 BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));
434
435 /* flush memory to make sure state is correct before next watchdog */
436 smp_mb__before_atomic();
437 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
438 }
439
440 struct ixgbe_reg_info {
441 u32 ofs;
442 char *name;
443 };
444
445 static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
446
447 /* General Registers */
448 {IXGBE_CTRL, "CTRL"},
449 {IXGBE_STATUS, "STATUS"},
450 {IXGBE_CTRL_EXT, "CTRL_EXT"},
451
452 /* Interrupt Registers */
453 {IXGBE_EICR, "EICR"},
454
455 /* RX Registers */
456 {IXGBE_SRRCTL(0), "SRRCTL"},
457 {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
458 {IXGBE_RDLEN(0), "RDLEN"},
459 {IXGBE_RDH(0), "RDH"},
460 {IXGBE_RDT(0), "RDT"},
461 {IXGBE_RXDCTL(0), "RXDCTL"},
462 {IXGBE_RDBAL(0), "RDBAL"},
463 {IXGBE_RDBAH(0), "RDBAH"},
464
465 /* TX Registers */
466 {IXGBE_TDBAL(0), "TDBAL"},
467 {IXGBE_TDBAH(0), "TDBAH"},
468 {IXGBE_TDLEN(0), "TDLEN"},
469 {IXGBE_TDH(0), "TDH"},
470 {IXGBE_TDT(0), "TDT"},
471 {IXGBE_TXDCTL(0), "TXDCTL"},
472
473 /* List Terminator */
474 { .name = NULL }
475 };
476
477
478 /*
479 * ixgbe_regdump - register printout routine
480 */
481 static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
482 {
483 int i = 0, j = 0;
484 char rname[16];
485 u32 regs[64];
486
487 switch (reginfo->ofs) {
488 case IXGBE_SRRCTL(0):
489 for (i = 0; i < 64; i++)
490 regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
491 break;
492 case IXGBE_DCA_RXCTRL(0):
493 for (i = 0; i < 64; i++)
494 regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
495 break;
496 case IXGBE_RDLEN(0):
497 for (i = 0; i < 64; i++)
498 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
499 break;
500 case IXGBE_RDH(0):
501 for (i = 0; i < 64; i++)
502 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
503 break;
504 case IXGBE_RDT(0):
505 for (i = 0; i < 64; i++)
506 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
507 break;
508 case IXGBE_RXDCTL(0):
509 for (i = 0; i < 64; i++)
510 regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
511 break;
512 case IXGBE_RDBAL(0):
513 for (i = 0; i < 64; i++)
514 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
515 break;
516 case IXGBE_RDBAH(0):
517 for (i = 0; i < 64; i++)
518 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
519 break;
520 case IXGBE_TDBAL(0):
521 for (i = 0; i < 64; i++)
522 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
523 break;
524 case IXGBE_TDBAH(0):
525 for (i = 0; i < 64; i++)
526 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
527 break;
528 case IXGBE_TDLEN(0):
529 for (i = 0; i < 64; i++)
530 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
531 break;
532 case IXGBE_TDH(0):
533 for (i = 0; i < 64; i++)
534 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
535 break;
536 case IXGBE_TDT(0):
537 for (i = 0; i < 64; i++)
538 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
539 break;
540 case IXGBE_TXDCTL(0):
541 for (i = 0; i < 64; i++)
542 regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
543 break;
544 default:
545 pr_info("%-15s %08x\n", reginfo->name,
546 IXGBE_READ_REG(hw, reginfo->ofs));
547 return;
548 }
549
550 for (i = 0; i < 8; i++) {
551 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
552 pr_err("%-15s", rname);
553 for (j = 0; j < 8; j++)
554 pr_cont(" %08x", regs[i*8+j]);
555 pr_cont("\n");
556 }
557
558 }
559
560 /*
561 * ixgbe_dump - Print registers, tx-rings and rx-rings
562 */
563 static void ixgbe_dump(struct ixgbe_adapter *adapter)
564 {
565 struct net_device *netdev = adapter->netdev;
566 struct ixgbe_hw *hw = &adapter->hw;
567 struct ixgbe_reg_info *reginfo;
568 int n = 0;
569 struct ixgbe_ring *tx_ring;
570 struct ixgbe_tx_buffer *tx_buffer;
571 union ixgbe_adv_tx_desc *tx_desc;
572 struct my_u0 { u64 a; u64 b; } *u0;
573 struct ixgbe_ring *rx_ring;
574 union ixgbe_adv_rx_desc *rx_desc;
575 struct ixgbe_rx_buffer *rx_buffer_info;
576 u32 staterr;
577 int i = 0;
578
579 if (!netif_msg_hw(adapter))
580 return;
581
582 /* Print netdevice Info */
583 if (netdev) {
584 dev_info(&adapter->pdev->dev, "Net device Info\n");
585 pr_info("Device Name state "
586 "trans_start last_rx\n");
587 pr_info("%-15s %016lX %016lX %016lX\n",
588 netdev->name,
589 netdev->state,
590 netdev->trans_start,
591 netdev->last_rx);
592 }
593
594 /* Print Registers */
595 dev_info(&adapter->pdev->dev, "Register Dump\n");
596 pr_info(" Register Name Value\n");
597 for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
598 reginfo->name; reginfo++) {
599 ixgbe_regdump(hw, reginfo);
600 }
601
602 /* Print TX Ring Summary */
603 if (!netdev || !netif_running(netdev))
604 return;
605
606 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
607 pr_info(" %s %s %s %s\n",
608 "Queue [NTU] [NTC] [bi(ntc)->dma ]",
609 "leng", "ntw", "timestamp");
610 for (n = 0; n < adapter->num_tx_queues; n++) {
611 tx_ring = adapter->tx_ring[n];
612 tx_buffer = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
613 pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n",
614 n, tx_ring->next_to_use, tx_ring->next_to_clean,
615 (u64)dma_unmap_addr(tx_buffer, dma),
616 dma_unmap_len(tx_buffer, len),
617 tx_buffer->next_to_watch,
618 (u64)tx_buffer->time_stamp);
619 }
620
621 /* Print TX Rings */
622 if (!netif_msg_tx_done(adapter))
623 goto rx_ring_summary;
624
625 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
626
627 /* Transmit Descriptor Formats
628 *
629 * 82598 Advanced Transmit Descriptor
630 * +--------------------------------------------------------------+
631 * 0 | Buffer Address [63:0] |
632 * +--------------------------------------------------------------+
633 * 8 | PAYLEN | POPTS | IDX | STA | DCMD |DTYP | RSV | DTALEN |
634 * +--------------------------------------------------------------+
635 * 63 46 45 40 39 36 35 32 31 24 23 20 19 0
636 *
637 * 82598 Advanced Transmit Descriptor (Write-Back Format)
638 * +--------------------------------------------------------------+
639 * 0 | RSV [63:0] |
640 * +--------------------------------------------------------------+
641 * 8 | RSV | STA | NXTSEQ |
642 * +--------------------------------------------------------------+
643 * 63 36 35 32 31 0
644 *
645 * 82599+ Advanced Transmit Descriptor
646 * +--------------------------------------------------------------+
647 * 0 | Buffer Address [63:0] |
648 * +--------------------------------------------------------------+
649 * 8 |PAYLEN |POPTS|CC|IDX |STA |DCMD |DTYP |MAC |RSV |DTALEN |
650 * +--------------------------------------------------------------+
651 * 63 46 45 40 39 38 36 35 32 31 24 23 20 19 18 17 16 15 0
652 *
653 * 82599+ Advanced Transmit Descriptor (Write-Back Format)
654 * +--------------------------------------------------------------+
655 * 0 | RSV [63:0] |
656 * +--------------------------------------------------------------+
657 * 8 | RSV | STA | RSV |
658 * +--------------------------------------------------------------+
659 * 63 36 35 32 31 0
660 */
661
662 for (n = 0; n < adapter->num_tx_queues; n++) {
663 tx_ring = adapter->tx_ring[n];
664 pr_info("------------------------------------\n");
665 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
666 pr_info("------------------------------------\n");
667 pr_info("%s%s %s %s %s %s\n",
668 "T [desc] [address 63:0 ] ",
669 "[PlPOIdStDDt Ln] [bi->dma ] ",
670 "leng", "ntw", "timestamp", "bi->skb");
671
672 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
673 tx_desc = IXGBE_TX_DESC(tx_ring, i);
674 tx_buffer = &tx_ring->tx_buffer_info[i];
675 u0 = (struct my_u0 *)tx_desc;
676 if (dma_unmap_len(tx_buffer, len) > 0) {
677 pr_info("T [0x%03X] %016llX %016llX %016llX %08X %p %016llX %p",
678 i,
679 le64_to_cpu(u0->a),
680 le64_to_cpu(u0->b),
681 (u64)dma_unmap_addr(tx_buffer, dma),
682 dma_unmap_len(tx_buffer, len),
683 tx_buffer->next_to_watch,
684 (u64)tx_buffer->time_stamp,
685 tx_buffer->skb);
686 if (i == tx_ring->next_to_use &&
687 i == tx_ring->next_to_clean)
688 pr_cont(" NTC/U\n");
689 else if (i == tx_ring->next_to_use)
690 pr_cont(" NTU\n");
691 else if (i == tx_ring->next_to_clean)
692 pr_cont(" NTC\n");
693 else
694 pr_cont("\n");
695
696 if (netif_msg_pktdata(adapter) &&
697 tx_buffer->skb)
698 print_hex_dump(KERN_INFO, "",
699 DUMP_PREFIX_ADDRESS, 16, 1,
700 tx_buffer->skb->data,
701 dma_unmap_len(tx_buffer, len),
702 true);
703 }
704 }
705 }
706
707 /* Print RX Rings Summary */
708 rx_ring_summary:
709 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
710 pr_info("Queue [NTU] [NTC]\n");
711 for (n = 0; n < adapter->num_rx_queues; n++) {
712 rx_ring = adapter->rx_ring[n];
713 pr_info("%5d %5X %5X\n",
714 n, rx_ring->next_to_use, rx_ring->next_to_clean);
715 }
716
717 /* Print RX Rings */
718 if (!netif_msg_rx_status(adapter))
719 return;
720
721 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
722
723 /* Receive Descriptor Formats
724 *
725 * 82598 Advanced Receive Descriptor (Read) Format
726 * 63 1 0
727 * +-----------------------------------------------------+
728 * 0 | Packet Buffer Address [63:1] |A0/NSE|
729 * +----------------------------------------------+------+
730 * 8 | Header Buffer Address [63:1] | DD |
731 * +-----------------------------------------------------+
732 *
733 *
734 * 82598 Advanced Receive Descriptor (Write-Back) Format
735 *
736 * 63 48 47 32 31 30 21 20 16 15 4 3 0
737 * +------------------------------------------------------+
738 * 0 | RSS Hash / |SPH| HDR_LEN | RSV |Packet| RSS |
739 * | Packet | IP | | | | Type | Type |
740 * | Checksum | Ident | | | | | |
741 * +------------------------------------------------------+
742 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
743 * +------------------------------------------------------+
744 * 63 48 47 32 31 20 19 0
745 *
746 * 82599+ Advanced Receive Descriptor (Read) Format
747 * 63 1 0
748 * +-----------------------------------------------------+
749 * 0 | Packet Buffer Address [63:1] |A0/NSE|
750 * +----------------------------------------------+------+
751 * 8 | Header Buffer Address [63:1] | DD |
752 * +-----------------------------------------------------+
753 *
754 *
755 * 82599+ Advanced Receive Descriptor (Write-Back) Format
756 *
757 * 63 48 47 32 31 30 21 20 17 16 4 3 0
758 * +------------------------------------------------------+
759 * 0 |RSS / Frag Checksum|SPH| HDR_LEN |RSC- |Packet| RSS |
760 * |/ RTT / PCoE_PARAM | | | CNT | Type | Type |
761 * |/ Flow Dir Flt ID | | | | | |
762 * +------------------------------------------------------+
763 * 8 | VLAN Tag | Length |Extended Error| Xtnd Status/NEXTP |
764 * +------------------------------------------------------+
765 * 63 48 47 32 31 20 19 0
766 */
767
768 for (n = 0; n < adapter->num_rx_queues; n++) {
769 rx_ring = adapter->rx_ring[n];
770 pr_info("------------------------------------\n");
771 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
772 pr_info("------------------------------------\n");
773 pr_info("%s%s%s",
774 "R [desc] [ PktBuf A0] ",
775 "[ HeadBuf DD] [bi->dma ] [bi->skb ] ",
776 "<-- Adv Rx Read format\n");
777 pr_info("%s%s%s",
778 "RWB[desc] [PcsmIpSHl PtRs] ",
779 "[vl er S cks ln] ---------------- [bi->skb ] ",
780 "<-- Adv Rx Write-Back format\n");
781
782 for (i = 0; i < rx_ring->count; i++) {
783 rx_buffer_info = &rx_ring->rx_buffer_info[i];
784 rx_desc = IXGBE_RX_DESC(rx_ring, i);
785 u0 = (struct my_u0 *)rx_desc;
786 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
787 if (staterr & IXGBE_RXD_STAT_DD) {
788 /* Descriptor Done */
789 pr_info("RWB[0x%03X] %016llX "
790 "%016llX ---------------- %p", i,
791 le64_to_cpu(u0->a),
792 le64_to_cpu(u0->b),
793 rx_buffer_info->skb);
794 } else {
795 pr_info("R [0x%03X] %016llX "
796 "%016llX %016llX %p", i,
797 le64_to_cpu(u0->a),
798 le64_to_cpu(u0->b),
799 (u64)rx_buffer_info->dma,
800 rx_buffer_info->skb);
801
802 if (netif_msg_pktdata(adapter) &&
803 rx_buffer_info->dma) {
804 print_hex_dump(KERN_INFO, "",
805 DUMP_PREFIX_ADDRESS, 16, 1,
806 page_address(rx_buffer_info->page) +
807 rx_buffer_info->page_offset,
808 ixgbe_rx_bufsz(rx_ring), true);
809 }
810 }
811
812 if (i == rx_ring->next_to_use)
813 pr_cont(" NTU\n");
814 else if (i == rx_ring->next_to_clean)
815 pr_cont(" NTC\n");
816 else
817 pr_cont("\n");
818
819 }
820 }
821 }
822
823 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
824 {
825 u32 ctrl_ext;
826
827 /* Let firmware take over control of h/w */
828 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
829 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
830 ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
831 }
832
833 static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
834 {
835 u32 ctrl_ext;
836
837 /* Let firmware know the driver has taken over */
838 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
839 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
840 ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
841 }
842
843 /**
844 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
845 * @adapter: pointer to adapter struct
846 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
847 * @queue: queue to map the corresponding interrupt to
848 * @msix_vector: the vector to map to the corresponding queue
849 *
850 */
851 static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
852 u8 queue, u8 msix_vector)
853 {
854 u32 ivar, index;
855 struct ixgbe_hw *hw = &adapter->hw;
856 switch (hw->mac.type) {
857 case ixgbe_mac_82598EB:
858 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
859 if (direction == -1)
860 direction = 0;
861 index = (((direction * 64) + queue) >> 2) & 0x1F;
862 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
863 ivar &= ~(0xFF << (8 * (queue & 0x3)));
864 ivar |= (msix_vector << (8 * (queue & 0x3)));
865 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
866 break;
867 case ixgbe_mac_82599EB:
868 case ixgbe_mac_X540:
869 case ixgbe_mac_X550:
870 case ixgbe_mac_X550EM_x:
871 case ixgbe_mac_x550em_a:
872 if (direction == -1) {
873 /* other causes */
874 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
875 index = ((queue & 1) * 8);
876 ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
877 ivar &= ~(0xFF << index);
878 ivar |= (msix_vector << index);
879 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
880 break;
881 } else {
882 /* tx or rx causes */
883 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
884 index = ((16 * (queue & 1)) + (8 * direction));
885 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
886 ivar &= ~(0xFF << index);
887 ivar |= (msix_vector << index);
888 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
889 break;
890 }
891 default:
892 break;
893 }
894 }
895
896 static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
897 u64 qmask)
898 {
899 u32 mask;
900
901 switch (adapter->hw.mac.type) {
902 case ixgbe_mac_82598EB:
903 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
904 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
905 break;
906 case ixgbe_mac_82599EB:
907 case ixgbe_mac_X540:
908 case ixgbe_mac_X550:
909 case ixgbe_mac_X550EM_x:
910 case ixgbe_mac_x550em_a:
911 mask = (qmask & 0xFFFFFFFF);
912 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
913 mask = (qmask >> 32);
914 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
915 break;
916 default:
917 break;
918 }
919 }
920
921 void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *ring,
922 struct ixgbe_tx_buffer *tx_buffer)
923 {
924 if (tx_buffer->skb) {
925 dev_kfree_skb_any(tx_buffer->skb);
926 if (dma_unmap_len(tx_buffer, len))
927 dma_unmap_single(ring->dev,
928 dma_unmap_addr(tx_buffer, dma),
929 dma_unmap_len(tx_buffer, len),
930 DMA_TO_DEVICE);
931 } else if (dma_unmap_len(tx_buffer, len)) {
932 dma_unmap_page(ring->dev,
933 dma_unmap_addr(tx_buffer, dma),
934 dma_unmap_len(tx_buffer, len),
935 DMA_TO_DEVICE);
936 }
937 tx_buffer->next_to_watch = NULL;
938 tx_buffer->skb = NULL;
939 dma_unmap_len_set(tx_buffer, len, 0);
940 /* tx_buffer must be completely set up in the transmit path */
941 }
942
943 static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)
944 {
945 struct ixgbe_hw *hw = &adapter->hw;
946 struct ixgbe_hw_stats *hwstats = &adapter->stats;
947 int i;
948 u32 data;
949
950 if ((hw->fc.current_mode != ixgbe_fc_full) &&
951 (hw->fc.current_mode != ixgbe_fc_rx_pause))
952 return;
953
954 switch (hw->mac.type) {
955 case ixgbe_mac_82598EB:
956 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
957 break;
958 default:
959 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
960 }
961 hwstats->lxoffrxc += data;
962
963 /* refill credits (no tx hang) if we received xoff */
964 if (!data)
965 return;
966
967 for (i = 0; i < adapter->num_tx_queues; i++)
968 clear_bit(__IXGBE_HANG_CHECK_ARMED,
969 &adapter->tx_ring[i]->state);
970 }
971
972 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
973 {
974 struct ixgbe_hw *hw = &adapter->hw;
975 struct ixgbe_hw_stats *hwstats = &adapter->stats;
976 u32 xoff[8] = {0};
977 u8 tc;
978 int i;
979 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
980
981 if (adapter->ixgbe_ieee_pfc)
982 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
983
984 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) {
985 ixgbe_update_xoff_rx_lfc(adapter);
986 return;
987 }
988
989 /* update stats for each tc, only valid with PFC enabled */
990 for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
991 u32 pxoffrxc;
992
993 switch (hw->mac.type) {
994 case ixgbe_mac_82598EB:
995 pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
996 break;
997 default:
998 pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
999 }
1000 hwstats->pxoffrxc[i] += pxoffrxc;
1001 /* Get the TC for given UP */
1002 tc = netdev_get_prio_tc_map(adapter->netdev, i);
1003 xoff[tc] += pxoffrxc;
1004 }
1005
1006 /* disarm tx queues that have received xoff frames */
1007 for (i = 0; i < adapter->num_tx_queues; i++) {
1008 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
1009
1010 tc = tx_ring->dcb_tc;
1011 if (xoff[tc])
1012 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
1013 }
1014 }
1015
1016 static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
1017 {
1018 return ring->stats.packets;
1019 }
1020
1021 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
1022 {
1023 struct ixgbe_adapter *adapter;
1024 struct ixgbe_hw *hw;
1025 u32 head, tail;
1026
1027 if (ring->l2_accel_priv)
1028 adapter = ring->l2_accel_priv->real_adapter;
1029 else
1030 adapter = netdev_priv(ring->netdev);
1031
1032 hw = &adapter->hw;
1033 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
1034 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
1035
1036 if (head != tail)
1037 return (head < tail) ?
1038 tail - head : (tail + ring->count - head);
1039
1040 return 0;
1041 }
1042
1043 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
1044 {
1045 u32 tx_done = ixgbe_get_tx_completed(tx_ring);
1046 u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
1047 u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
1048
1049 clear_check_for_tx_hang(tx_ring);
1050
1051 /*
1052 * Check for a hung queue, but be thorough. This verifies
1053 * that a transmit has been completed since the previous
1054 * check AND there is at least one packet pending. The
1055 * ARMED bit is set to indicate a potential hang. The
1056 * bit is cleared if a pause frame is received to remove
1057 * false hang detection due to PFC or 802.3x frames. By
1058 * requiring this to fail twice we avoid races with
1059 * pfc clearing the ARMED bit and conditions where we
1060 * run the check_tx_hang logic with a transmit completion
1061 * pending but without time to complete it yet.
1062 */
1063 if (tx_done_old == tx_done && tx_pending)
1064 /* make sure it is true for two checks in a row */
1065 return test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
1066 &tx_ring->state);
1067 /* update completed stats and continue */
1068 tx_ring->tx_stats.tx_done_old = tx_done;
1069 /* reset the countdown */
1070 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
1071
1072 return false;
1073 }
1074
1075 /**
1076 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
1077 * @adapter: driver private struct
1078 **/
1079 static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
1080 {
1081
1082 /* Do the reset outside of interrupt context */
1083 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1084 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
1085 e_warn(drv, "initiating reset due to tx timeout\n");
1086 ixgbe_service_event_schedule(adapter);
1087 }
1088 }
1089
1090 /**
1091 * ixgbe_tx_maxrate - callback to set the maximum per-queue bitrate
1092 **/
1093 static int ixgbe_tx_maxrate(struct net_device *netdev,
1094 int queue_index, u32 maxrate)
1095 {
1096 struct ixgbe_adapter *adapter = netdev_priv(netdev);
1097 struct ixgbe_hw *hw = &adapter->hw;
1098 u32 bcnrc_val = ixgbe_link_mbps(adapter);
1099
1100 if (!maxrate)
1101 return 0;
1102
1103 /* Calculate the rate factor values to set */
1104 bcnrc_val <<= IXGBE_RTTBCNRC_RF_INT_SHIFT;
1105 bcnrc_val /= maxrate;
1106
1107 /* clear everything but the rate factor */
1108 bcnrc_val &= IXGBE_RTTBCNRC_RF_INT_MASK |
1109 IXGBE_RTTBCNRC_RF_DEC_MASK;
1110
1111 /* enable the rate scheduler */
1112 bcnrc_val |= IXGBE_RTTBCNRC_RS_ENA;
1113
1114 IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_index);
1115 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);
1116
1117 return 0;
1118 }
1119
1120 /**
1121 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
1122 * @q_vector: structure containing interrupt and ring information
1123 * @tx_ring: tx ring to clean
1124 * @napi_budget: Used to determine if we are in netpoll
1125 **/
1126 static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
1127 struct ixgbe_ring *tx_ring, int napi_budget)
1128 {
1129 struct ixgbe_adapter *adapter = q_vector->adapter;
1130 struct ixgbe_tx_buffer *tx_buffer;
1131 union ixgbe_adv_tx_desc *tx_desc;
1132 unsigned int total_bytes = 0, total_packets = 0;
1133 unsigned int budget = q_vector->tx.work_limit;
1134 unsigned int i = tx_ring->next_to_clean;
1135
1136 if (test_bit(__IXGBE_DOWN, &adapter->state))
1137 return true;
1138
1139 tx_buffer = &tx_ring->tx_buffer_info[i];
1140 tx_desc = IXGBE_TX_DESC(tx_ring, i);
1141 i -= tx_ring->count;
1142
1143 do {
1144 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
1145
1146 /* if next_to_watch is not set then there is no work pending */
1147 if (!eop_desc)
1148 break;
1149
1150 /* prevent any other reads prior to eop_desc */
1151 read_barrier_depends();
1152
1153 /* if DD is not set pending work has not been completed */
1154 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
1155 break;
1156
1157 /* clear next_to_watch to prevent false hangs */
1158 tx_buffer->next_to_watch = NULL;
1159
1160 /* update the statistics for this packet */
1161 total_bytes += tx_buffer->bytecount;
1162 total_packets += tx_buffer->gso_segs;
1163
1164 /* free the skb */
1165 napi_consume_skb(tx_buffer->skb, napi_budget);
1166
1167 /* unmap skb header data */
1168 dma_unmap_single(tx_ring->dev,
1169 dma_unmap_addr(tx_buffer, dma),
1170 dma_unmap_len(tx_buffer, len),
1171 DMA_TO_DEVICE);
1172
1173 /* clear tx_buffer data */
1174 tx_buffer->skb = NULL;
1175 dma_unmap_len_set(tx_buffer, len, 0);
1176
1177 /* unmap remaining buffers */
1178 while (tx_desc != eop_desc) {
1179 tx_buffer++;
1180 tx_desc++;
1181 i++;
1182 if (unlikely(!i)) {
1183 i -= tx_ring->count;
1184 tx_buffer = tx_ring->tx_buffer_info;
1185 tx_desc = IXGBE_TX_DESC(tx_ring, 0);
1186 }
1187
1188 /* unmap any remaining paged data */
1189 if (dma_unmap_len(tx_buffer, len)) {
1190 dma_unmap_page(tx_ring->dev,
1191 dma_unmap_addr(tx_buffer, dma),
1192 dma_unmap_len(tx_buffer, len),
1193 DMA_TO_DEVICE);
1194 dma_unmap_len_set(tx_buffer, len, 0);
1195 }
1196 }
1197
1198 /* move us one more past the eop_desc for start of next pkt */
1199 tx_buffer++;
1200 tx_desc++;
1201 i++;
1202 if (unlikely(!i)) {
1203 i -= tx_ring->count;
1204 tx_buffer = tx_ring->tx_buffer_info;
1205 tx_desc = IXGBE_TX_DESC(tx_ring, 0);
1206 }
1207
1208 /* issue prefetch for next Tx descriptor */
1209 prefetch(tx_desc);
1210
1211 /* update budget accounting */
1212 budget--;
1213 } while (likely(budget));
1214
1215 i += tx_ring->count;
1216 tx_ring->next_to_clean = i;
1217 u64_stats_update_begin(&tx_ring->syncp);
1218 tx_ring->stats.bytes += total_bytes;
1219 tx_ring->stats.packets += total_packets;
1220 u64_stats_update_end(&tx_ring->syncp);
1221 q_vector->tx.total_bytes += total_bytes;
1222 q_vector->tx.total_packets += total_packets;
1223
1224 if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
1225 /* schedule immediate reset if we believe we hung */
1226 struct ixgbe_hw *hw = &adapter->hw;
1227 e_err(drv, "Detected Tx Unit Hang\n"
1228 " Tx Queue <%d>\n"
1229 " TDH, TDT <%x>, <%x>\n"
1230 " next_to_use <%x>\n"
1231 " next_to_clean <%x>\n"
1232 "tx_buffer_info[next_to_clean]\n"
1233 " time_stamp <%lx>\n"
1234 " jiffies <%lx>\n",
1235 tx_ring->queue_index,
1236 IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
1237 IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
1238 tx_ring->next_to_use, i,
1239 tx_ring->tx_buffer_info[i].time_stamp, jiffies);
1240
1241 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
1242
1243 e_info(probe,
1244 "tx hang %d detected on queue %d, resetting adapter\n",
1245 adapter->tx_timeout_count + 1, tx_ring->queue_index);
1246
1247 /* schedule immediate reset if we believe we hung */
1248 ixgbe_tx_timeout_reset(adapter);
1249
1250 /* the adapter is about to reset, no point in enabling stuff */
1251 return true;
1252 }
1253
1254 netdev_tx_completed_queue(txring_txq(tx_ring),
1255 total_packets, total_bytes);
1256
1257 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
1258 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
1259 (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
1260 /* Make sure that anybody stopping the queue after this
1261 * sees the new next_to_clean.
1262 */
1263 smp_mb();
1264 if (__netif_subqueue_stopped(tx_ring->netdev,
1265 tx_ring->queue_index)
1266 && !test_bit(__IXGBE_DOWN, &adapter->state)) {
1267 netif_wake_subqueue(tx_ring->netdev,
1268 tx_ring->queue_index);
1269 ++tx_ring->tx_stats.restart_queue;
1270 }
1271 }
1272
1273 return !!budget;
1274 }
1275
1276 #ifdef CONFIG_IXGBE_DCA
1277 static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
1278 struct ixgbe_ring *tx_ring,
1279 int cpu)
1280 {
1281 struct ixgbe_hw *hw = &adapter->hw;
1282 u32 txctrl = 0;
1283 u16 reg_offset;
1284
1285 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1286 txctrl = dca3_get_tag(tx_ring->dev, cpu);
1287
1288 switch (hw->mac.type) {
1289 case ixgbe_mac_82598EB:
1290 reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx);
1291 break;
1292 case ixgbe_mac_82599EB:
1293 case ixgbe_mac_X540:
1294 reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx);
1295 txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599;
1296 break;
1297 default:
1298 /* for unknown hardware do not write register */
1299 return;
1300 }
1301
1302 /*
1303 * We can enable relaxed ordering for reads, but not writes when
1304 * DCA is enabled. This is due to a known issue in some chipsets
1305 * which will cause the DCA tag to be cleared.
1306 */
1307 txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1308 IXGBE_DCA_TXCTRL_DATA_RRO_EN |
1309 IXGBE_DCA_TXCTRL_DESC_DCA_EN;
1310
1311 IXGBE_WRITE_REG(hw, reg_offset, txctrl);
1312 }
1313
1314 static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
1315 struct ixgbe_ring *rx_ring,
1316 int cpu)
1317 {
1318 struct ixgbe_hw *hw = &adapter->hw;
1319 u32 rxctrl = 0;
1320 u8 reg_idx = rx_ring->reg_idx;
1321
1322 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1323 rxctrl = dca3_get_tag(rx_ring->dev, cpu);
1324
1325 switch (hw->mac.type) {
1326 case ixgbe_mac_82599EB:
1327 case ixgbe_mac_X540:
1328 rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599;
1329 break;
1330 default:
1331 break;
1332 }
1333
1334 /*
1335 * We can enable relaxed ordering for reads, but not writes when
1336 * DCA is enabled. This is due to a known issue in some chipsets
1337 * which will cause the DCA tag to be cleared.
1338 */
1339 rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN |
1340 IXGBE_DCA_RXCTRL_DATA_DCA_EN |
1341 IXGBE_DCA_RXCTRL_DESC_DCA_EN;
1342
1343 IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
1344 }
1345
1346 static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
1347 {
1348 struct ixgbe_adapter *adapter = q_vector->adapter;
1349 struct ixgbe_ring *ring;
1350 int cpu = get_cpu();
1351
1352 if (q_vector->cpu == cpu)
1353 goto out_no_update;
1354
1355 ixgbe_for_each_ring(ring, q_vector->tx)
1356 ixgbe_update_tx_dca(adapter, ring, cpu);
1357
1358 ixgbe_for_each_ring(ring, q_vector->rx)
1359 ixgbe_update_rx_dca(adapter, ring, cpu);
1360
1361 q_vector->cpu = cpu;
1362 out_no_update:
1363 put_cpu();
1364 }
1365
1366 static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
1367 {
1368 int i;
1369
1370 /* always use CB2 mode, difference is masked in the CB driver */
1371 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1372 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1373 IXGBE_DCA_CTRL_DCA_MODE_CB2);
1374 else
1375 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1376 IXGBE_DCA_CTRL_DCA_DISABLE);
1377
1378 for (i = 0; i < adapter->num_q_vectors; i++) {
1379 adapter->q_vector[i]->cpu = -1;
1380 ixgbe_update_dca(adapter->q_vector[i]);
1381 }
1382 }
1383
1384 static int __ixgbe_notify_dca(struct device *dev, void *data)
1385 {
1386 struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
1387 unsigned long event = *(unsigned long *)data;
1388
1389 if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
1390 return 0;
1391
1392 switch (event) {
1393 case DCA_PROVIDER_ADD:
1394 /* if we're already enabled, don't do it again */
1395 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1396 break;
1397 if (dca_add_requester(dev) == 0) {
1398 adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
1399 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1400 IXGBE_DCA_CTRL_DCA_MODE_CB2);
1401 break;
1402 }
1403 /* Fall Through since DCA is disabled. */
1404 case DCA_PROVIDER_REMOVE:
1405 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
1406 dca_remove_requester(dev);
1407 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
1408 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1409 IXGBE_DCA_CTRL_DCA_DISABLE);
1410 }
1411 break;
1412 }
1413
1414 return 0;
1415 }
1416
1417 #endif /* CONFIG_IXGBE_DCA */
1418
1419 #define IXGBE_RSS_L4_TYPES_MASK \
1420 ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \
1421 (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \
1422 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \
1423 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP))
1424
1425 static inline void ixgbe_rx_hash(struct ixgbe_ring *ring,
1426 union ixgbe_adv_rx_desc *rx_desc,
1427 struct sk_buff *skb)
1428 {
1429 u16 rss_type;
1430
1431 if (!(ring->netdev->features & NETIF_F_RXHASH))
1432 return;
1433
1434 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
1435 IXGBE_RXDADV_RSSTYPE_MASK;
1436
1437 if (!rss_type)
1438 return;
1439
1440 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
1441 (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
1442 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
1443 }
1444
1445 #ifdef IXGBE_FCOE
1446 /**
1447 * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
1448 * @ring: structure containing ring specific data
1449 * @rx_desc: advanced rx descriptor
1450 *
1451 * Returns : true if it is FCoE pkt
1452 */
1453 static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring,
1454 union ixgbe_adv_rx_desc *rx_desc)
1455 {
1456 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1457
1458 return test_bit(__IXGBE_RX_FCOE, &ring->state) &&
1459 ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
1460 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
1461 IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
1462 }
1463
1464 #endif /* IXGBE_FCOE */
1465 /**
1466 * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
1467 * @ring: structure containing ring specific data
1468 * @rx_desc: current Rx descriptor being processed
1469 * @skb: skb currently being received and modified
1470 **/
1471 static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
1472 union ixgbe_adv_rx_desc *rx_desc,
1473 struct sk_buff *skb)
1474 {
1475 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1476 __le16 hdr_info = rx_desc->wb.lower.lo_dword.hs_rss.hdr_info;
1477 bool encap_pkt = false;
1478
1479 skb_checksum_none_assert(skb);
1480
1481 /* Rx csum disabled */
1482 if (!(ring->netdev->features & NETIF_F_RXCSUM))
1483 return;
1484
1485 if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_VXLAN)) &&
1486 (hdr_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_TUNNEL >> 16))) {
1487 encap_pkt = true;
1488 skb->encapsulation = 1;
1489 }
1490
1491 /* if IP and error */
1492 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
1493 ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
1494 ring->rx_stats.csum_err++;
1495 return;
1496 }
1497
1498 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
1499 return;
1500
1501 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
1502 /*
1503 * 82599 errata, UDP frames with a 0 checksum can be marked as
1504 * checksum errors.
1505 */
1506 if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) &&
1507 test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state))
1508 return;
1509
1510 ring->rx_stats.csum_err++;
1511 return;
1512 }
1513
1514 /* It must be a TCP or UDP packet with a valid checksum */
1515 skb->ip_summed = CHECKSUM_UNNECESSARY;
1516 if (encap_pkt) {
1517 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_OUTERIPCS))
1518 return;
1519
1520 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_OUTERIPER)) {
1521 skb->ip_summed = CHECKSUM_NONE;
1522 return;
1523 }
1524 /* If we checked the outer header let the stack know */
1525 skb->csum_level = 1;
1526 }
1527 }
1528
1529 static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
1530 struct ixgbe_rx_buffer *bi)
1531 {
1532 struct page *page = bi->page;
1533 dma_addr_t dma;
1534
1535 /* since we are recycling buffers we should seldom need to alloc */
1536 if (likely(page))
1537 return true;
1538
1539 /* alloc new page for storage */
1540 page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring));
1541 if (unlikely(!page)) {
1542 rx_ring->rx_stats.alloc_rx_page_failed++;
1543 return false;
1544 }
1545
1546 /* map page for use */
1547 dma = dma_map_page(rx_ring->dev, page, 0,
1548 ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE);
1549
1550 /*
1551 * if mapping failed free memory back to system since
1552 * there isn't much point in holding memory we can't use
1553 */
1554 if (dma_mapping_error(rx_ring->dev, dma)) {
1555 __free_pages(page, ixgbe_rx_pg_order(rx_ring));
1556
1557 rx_ring->rx_stats.alloc_rx_page_failed++;
1558 return false;
1559 }
1560
1561 bi->dma = dma;
1562 bi->page = page;
1563 bi->page_offset = 0;
1564
1565 return true;
1566 }
1567
1568 /**
1569 * ixgbe_alloc_rx_buffers - Replace used receive buffers
1570 * @rx_ring: ring to place buffers on
1571 * @cleaned_count: number of buffers to replace
1572 **/
1573 void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
1574 {
1575 union ixgbe_adv_rx_desc *rx_desc;
1576 struct ixgbe_rx_buffer *bi;
1577 u16 i = rx_ring->next_to_use;
1578
1579 /* nothing to do */
1580 if (!cleaned_count)
1581 return;
1582
1583 rx_desc = IXGBE_RX_DESC(rx_ring, i);
1584 bi = &rx_ring->rx_buffer_info[i];
1585 i -= rx_ring->count;
1586
1587 do {
1588 if (!ixgbe_alloc_mapped_page(rx_ring, bi))
1589 break;
1590
1591 /*
1592 * Refresh the desc even if buffer_addrs didn't change
1593 * because each write-back erases this info.
1594 */
1595 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
1596
1597 rx_desc++;
1598 bi++;
1599 i++;
1600 if (unlikely(!i)) {
1601 rx_desc = IXGBE_RX_DESC(rx_ring, 0);
1602 bi = rx_ring->rx_buffer_info;
1603 i -= rx_ring->count;
1604 }
1605
1606 /* clear the status bits for the next_to_use descriptor */
1607 rx_desc->wb.upper.status_error = 0;
1608
1609 cleaned_count--;
1610 } while (cleaned_count);
1611
1612 i += rx_ring->count;
1613
1614 if (rx_ring->next_to_use != i) {
1615 rx_ring->next_to_use = i;
1616
1617 /* update next to alloc since we have filled the ring */
1618 rx_ring->next_to_alloc = i;
1619
1620 /* Force memory writes to complete before letting h/w
1621 * know there are new descriptors to fetch. (Only
1622 * applicable for weak-ordered memory model archs,
1623 * such as IA-64).
1624 */
1625 wmb();
1626 writel(i, rx_ring->tail);
1627 }
1628 }
1629
1630 static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring,
1631 struct sk_buff *skb)
1632 {
1633 u16 hdr_len = skb_headlen(skb);
1634
1635 /* set gso_size to avoid messing up TCP MSS */
1636 skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
1637 IXGBE_CB(skb)->append_cnt);
1638 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1639 }
1640
1641 static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring,
1642 struct sk_buff *skb)
1643 {
1644 /* if append_cnt is 0 then frame is not RSC */
1645 if (!IXGBE_CB(skb)->append_cnt)
1646 return;
1647
1648 rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt;
1649 rx_ring->rx_stats.rsc_flush++;
1650
1651 ixgbe_set_rsc_gso_size(rx_ring, skb);
1652
1653 /* gso_size is computed using append_cnt so always clear it last */
1654 IXGBE_CB(skb)->append_cnt = 0;
1655 }
1656
1657 /**
1658 * ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor
1659 * @rx_ring: rx descriptor ring packet is being transacted on
1660 * @rx_desc: pointer to the EOP Rx descriptor
1661 * @skb: pointer to current skb being populated
1662 *
1663 * This function checks the ring, descriptor, and packet information in
1664 * order to populate the hash, checksum, VLAN, timestamp, protocol, and
1665 * other fields within the skb.
1666 **/
1667 static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,
1668 union ixgbe_adv_rx_desc *rx_desc,
1669 struct sk_buff *skb)
1670 {
1671 struct net_device *dev = rx_ring->netdev;
1672 u32 flags = rx_ring->q_vector->adapter->flags;
1673
1674 ixgbe_update_rsc_stats(rx_ring, skb);
1675
1676 ixgbe_rx_hash(rx_ring, rx_desc, skb);
1677
1678 ixgbe_rx_checksum(rx_ring, rx_desc, skb);
1679
1680 if (unlikely(flags & IXGBE_FLAG_RX_HWTSTAMP_ENABLED))
1681 ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb);
1682
1683 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
1684 ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
1685 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
1686 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1687 }
1688
1689 skb_record_rx_queue(skb, rx_ring->queue_index);
1690
1691 skb->protocol = eth_type_trans(skb, dev);
1692 }
1693
1694 static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,
1695 struct sk_buff *skb)
1696 {
1697 skb_mark_napi_id(skb, &q_vector->napi);
1698 if (ixgbe_qv_busy_polling(q_vector))
1699 netif_receive_skb(skb);
1700 else
1701 napi_gro_receive(&q_vector->napi, skb);
1702 }
1703
1704 /**
1705 * ixgbe_is_non_eop - process handling of non-EOP buffers
1706 * @rx_ring: Rx ring being processed
1707 * @rx_desc: Rx descriptor for current buffer
1708 * @skb: Current socket buffer containing buffer in progress
1709 *
1710 * This function updates next to clean. If the buffer is an EOP buffer
1711 * this function exits returning false, otherwise it will place the
1712 * sk_buff in the next buffer to be chained and return true indicating
1713 * that this is in fact a non-EOP buffer.
1714 **/
1715 static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring,
1716 union ixgbe_adv_rx_desc *rx_desc,
1717 struct sk_buff *skb)
1718 {
1719 u32 ntc = rx_ring->next_to_clean + 1;
1720
1721 /* fetch, update, and store next to clean */
1722 ntc = (ntc < rx_ring->count) ? ntc : 0;
1723 rx_ring->next_to_clean = ntc;
1724
1725 prefetch(IXGBE_RX_DESC(rx_ring, ntc));
1726
1727 /* update RSC append count if present */
1728 if (ring_is_rsc_enabled(rx_ring)) {
1729 __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
1730 cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK);
1731
1732 if (unlikely(rsc_enabled)) {
1733 u32 rsc_cnt = le32_to_cpu(rsc_enabled);
1734
1735 rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;
1736 IXGBE_CB(skb)->append_cnt += rsc_cnt - 1;
1737
1738 /* update ntc based on RSC value */
1739 ntc = le32_to_cpu(rx_desc->wb.upper.status_error);
1740 ntc &= IXGBE_RXDADV_NEXTP_MASK;
1741 ntc >>= IXGBE_RXDADV_NEXTP_SHIFT;
1742 }
1743 }
1744
1745 /* if we are the last buffer then there is nothing else to do */
1746 if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
1747 return false;
1748
1749 /* place skb in next buffer to be received */
1750 rx_ring->rx_buffer_info[ntc].skb = skb;
1751 rx_ring->rx_stats.non_eop_descs++;
1752
1753 return true;
1754 }
1755
1756 /**
1757 * ixgbe_pull_tail - ixgbe specific version of skb_pull_tail
1758 * @rx_ring: rx descriptor ring packet is being transacted on
1759 * @skb: pointer to current skb being adjusted
1760 *
1761 * This function is an ixgbe specific version of __pskb_pull_tail. The
1762 * main difference between this version and the original function is that
1763 * this function can make several assumptions about the state of things
1764 * that allow for significant optimizations versus the standard function.
1765 * As a result we can do things like drop a frag and maintain an accurate
1766 * truesize for the skb.
1767 */
1768 static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring,
1769 struct sk_buff *skb)
1770 {
1771 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
1772 unsigned char *va;
1773 unsigned int pull_len;
1774
1775 /*
1776 * it is valid to use page_address instead of kmap since we are
1777 * working with pages allocated out of the lomem pool per
1778 * alloc_page(GFP_ATOMIC)
1779 */
1780 va = skb_frag_address(frag);
1781
1782 /*
1783 * we need the header to contain the greater of either ETH_HLEN or
1784 * 60 bytes if the skb->len is less than 60 for skb_pad.
1785 */
1786 pull_len = eth_get_headlen(va, IXGBE_RX_HDR_SIZE);
1787
1788 /* align pull length to size of long to optimize memcpy performance */
1789 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
1790
1791 /* update all of the pointers */
1792 skb_frag_size_sub(frag, pull_len);
1793 frag->page_offset += pull_len;
1794 skb->data_len -= pull_len;
1795 skb->tail += pull_len;
1796 }
1797
1798 /**
1799 * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB
1800 * @rx_ring: rx descriptor ring packet is being transacted on
1801 * @skb: pointer to current skb being updated
1802 *
1803 * This function provides a basic DMA sync up for the first fragment of an
1804 * skb. The reason for doing this is that the first fragment cannot be
1805 * unmapped until we have reached the end of packet descriptor for a buffer
1806 * chain.
1807 */
1808 static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
1809 struct sk_buff *skb)
1810 {
1811 /* if the page was released unmap it, else just sync our portion */
1812 if (unlikely(IXGBE_CB(skb)->page_released)) {
1813 dma_unmap_page(rx_ring->dev, IXGBE_CB(skb)->dma,
1814 ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE);
1815 IXGBE_CB(skb)->page_released = false;
1816 } else {
1817 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
1818
1819 dma_sync_single_range_for_cpu(rx_ring->dev,
1820 IXGBE_CB(skb)->dma,
1821 frag->page_offset,
1822 ixgbe_rx_bufsz(rx_ring),
1823 DMA_FROM_DEVICE);
1824 }
1825 IXGBE_CB(skb)->dma = 0;
1826 }
1827
1828 /**
1829 * ixgbe_cleanup_headers - Correct corrupted or empty headers
1830 * @rx_ring: rx descriptor ring packet is being transacted on
1831 * @rx_desc: pointer to the EOP Rx descriptor
1832 * @skb: pointer to current skb being fixed
1833 *
1834 * Check for corrupted packet headers caused by senders on the local L2
1835 * embedded NIC switch not setting up their Tx Descriptors right. These
1836 * should be very rare.
1837 *
1838 * Also address the case where we are pulling data in on pages only
1839 * and as such no data is present in the skb header.
1840 *
1841 * In addition if skb is not at least 60 bytes we need to pad it so that
1842 * it is large enough to qualify as a valid Ethernet frame.
1843 *
1844 * Returns true if an error was encountered and skb was freed.
1845 **/
1846 static bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring,
1847 union ixgbe_adv_rx_desc *rx_desc,
1848 struct sk_buff *skb)
1849 {
1850 struct net_device *netdev = rx_ring->netdev;
1851
1852 /* verify that the packet does not have any known errors */
1853 if (unlikely(ixgbe_test_staterr(rx_desc,
1854 IXGBE_RXDADV_ERR_FRAME_ERR_MASK) &&
1855 !(netdev->features & NETIF_F_RXALL))) {
1856 dev_kfree_skb_any(skb);
1857 return true;
1858 }
1859
1860 /* place header in linear portion of buffer */
1861 if (skb_is_nonlinear(skb))
1862 ixgbe_pull_tail(rx_ring, skb);
1863
1864 #ifdef IXGBE_FCOE
1865 /* do not attempt to pad FCoE Frames as this will disrupt DDP */
1866 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc))
1867 return false;
1868
1869 #endif
1870 /* if eth_skb_pad returns an error the skb was freed */
1871 if (eth_skb_pad(skb))
1872 return true;
1873
1874 return false;
1875 }
1876
1877 /**
1878 * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring
1879 * @rx_ring: rx descriptor ring to store buffers on
1880 * @old_buff: donor buffer to have page reused
1881 *
1882 * Synchronizes page for reuse by the adapter
1883 **/
1884 static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring,
1885 struct ixgbe_rx_buffer *old_buff)
1886 {
1887 struct ixgbe_rx_buffer *new_buff;
1888 u16 nta = rx_ring->next_to_alloc;
1889
1890 new_buff = &rx_ring->rx_buffer_info[nta];
1891
1892 /* update, and store next to alloc */
1893 nta++;
1894 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1895
1896 /* transfer page from old buffer to new buffer */
1897 *new_buff = *old_buff;
1898
1899 /* sync the buffer for use by the device */
1900 dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma,
1901 new_buff->page_offset,
1902 ixgbe_rx_bufsz(rx_ring),
1903 DMA_FROM_DEVICE);
1904 }
1905
1906 static inline bool ixgbe_page_is_reserved(struct page *page)
1907 {
1908 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
1909 }
1910
1911 /**
1912 * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff
1913 * @rx_ring: rx descriptor ring to transact packets on
1914 * @rx_buffer: buffer containing page to add
1915 * @rx_desc: descriptor containing length of buffer written by hardware
1916 * @skb: sk_buff to place the data into
1917 *
1918 * This function will add the data contained in rx_buffer->page to the skb.
1919 * This is done either through a direct copy if the data in the buffer is
1920 * less than the skb header size, otherwise it will just attach the page as
1921 * a frag to the skb.
1922 *
1923 * The function will then update the page offset if necessary and return
1924 * true if the buffer can be reused by the adapter.
1925 **/
1926 static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
1927 struct ixgbe_rx_buffer *rx_buffer,
1928 union ixgbe_adv_rx_desc *rx_desc,
1929 struct sk_buff *skb)
1930 {
1931 struct page *page = rx_buffer->page;
1932 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
1933 #if (PAGE_SIZE < 8192)
1934 unsigned int truesize = ixgbe_rx_bufsz(rx_ring);
1935 #else
1936 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
1937 unsigned int last_offset = ixgbe_rx_pg_size(rx_ring) -
1938 ixgbe_rx_bufsz(rx_ring);
1939 #endif
1940
1941 if ((size <= IXGBE_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) {
1942 unsigned char *va = page_address(page) + rx_buffer->page_offset;
1943
1944 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
1945
1946 /* page is not reserved, we can reuse buffer as-is */
1947 if (likely(!ixgbe_page_is_reserved(page)))
1948 return true;
1949
1950 /* this page cannot be reused so discard it */
1951 __free_pages(page, ixgbe_rx_pg_order(rx_ring));
1952 return false;
1953 }
1954
1955 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
1956 rx_buffer->page_offset, size, truesize);
1957
1958 /* avoid re-using remote pages */
1959 if (unlikely(ixgbe_page_is_reserved(page)))
1960 return false;
1961
1962 #if (PAGE_SIZE < 8192)
1963 /* if we are only owner of page we can reuse it */
1964 if (unlikely(page_count(page) != 1))
1965 return false;
1966
1967 /* flip page offset to other buffer */
1968 rx_buffer->page_offset ^= truesize;
1969 #else
1970 /* move offset up to the next cache line */
1971 rx_buffer->page_offset += truesize;
1972
1973 if (rx_buffer->page_offset > last_offset)
1974 return false;
1975 #endif
1976
1977 /* Even if we own the page, we are not allowed to use atomic_set()
1978 * This would break get_page_unless_zero() users.
1979 */
1980 page_ref_inc(page);
1981
1982 return true;
1983 }
1984
1985 static struct sk_buff *ixgbe_fetch_rx_buffer(struct ixgbe_ring *rx_ring,
1986 union ixgbe_adv_rx_desc *rx_desc)
1987 {
1988 struct ixgbe_rx_buffer *rx_buffer;
1989 struct sk_buff *skb;
1990 struct page *page;
1991
1992 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
1993 page = rx_buffer->page;
1994 prefetchw(page);
1995
1996 skb = rx_buffer->skb;
1997
1998 if (likely(!skb)) {
1999 void *page_addr = page_address(page) +
2000 rx_buffer->page_offset;
2001
2002 /* prefetch first cache line of first page */
2003 prefetch(page_addr);
2004 #if L1_CACHE_BYTES < 128
2005 prefetch(page_addr + L1_CACHE_BYTES);
2006 #endif
2007
2008 /* allocate a skb to store the frags */
2009 skb = napi_alloc_skb(&rx_ring->q_vector->napi,
2010 IXGBE_RX_HDR_SIZE);
2011 if (unlikely(!skb)) {
2012 rx_ring->rx_stats.alloc_rx_buff_failed++;
2013 return NULL;
2014 }
2015
2016 /*
2017 * we will be copying header into skb->data in
2018 * pskb_may_pull so it is in our interest to prefetch
2019 * it now to avoid a possible cache miss
2020 */
2021 prefetchw(skb->data);
2022
2023 /*
2024 * Delay unmapping of the first packet. It carries the
2025 * header information, HW may still access the header
2026 * after the writeback. Only unmap it when EOP is
2027 * reached
2028 */
2029 if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
2030 goto dma_sync;
2031
2032 IXGBE_CB(skb)->dma = rx_buffer->dma;
2033 } else {
2034 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
2035 ixgbe_dma_sync_frag(rx_ring, skb);
2036
2037 dma_sync:
2038 /* we are reusing so sync this buffer for CPU use */
2039 dma_sync_single_range_for_cpu(rx_ring->dev,
2040 rx_buffer->dma,
2041 rx_buffer->page_offset,
2042 ixgbe_rx_bufsz(rx_ring),
2043 DMA_FROM_DEVICE);
2044
2045 rx_buffer->skb = NULL;
2046 }
2047
2048 /* pull page into skb */
2049 if (ixgbe_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
2050 /* hand second half of page back to the ring */
2051 ixgbe_reuse_rx_page(rx_ring, rx_buffer);
2052 } else if (IXGBE_CB(skb)->dma == rx_buffer->dma) {
2053 /* the page has been released from the ring */
2054 IXGBE_CB(skb)->page_released = true;
2055 } else {
2056 /* we are not reusing the buffer so unmap it */
2057 dma_unmap_page(rx_ring->dev, rx_buffer->dma,
2058 ixgbe_rx_pg_size(rx_ring),
2059 DMA_FROM_DEVICE);
2060 }
2061
2062 /* clear contents of buffer_info */
2063 rx_buffer->page = NULL;
2064
2065 return skb;
2066 }
2067
2068 /**
2069 * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
2070 * @q_vector: structure containing interrupt and ring information
2071 * @rx_ring: rx descriptor ring to transact packets on
2072 * @budget: Total limit on number of packets to process
2073 *
2074 * This function provides a "bounce buffer" approach to Rx interrupt
2075 * processing. The advantage to this is that on systems that have
2076 * expensive overhead for IOMMU access this provides a means of avoiding
2077 * it by maintaining the mapping of the page to the syste.
2078 *
2079 * Returns amount of work completed
2080 **/
2081 static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
2082 struct ixgbe_ring *rx_ring,
2083 const int budget)
2084 {
2085 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
2086 #ifdef IXGBE_FCOE
2087 struct ixgbe_adapter *adapter = q_vector->adapter;
2088 int ddp_bytes;
2089 unsigned int mss = 0;
2090 #endif /* IXGBE_FCOE */
2091 u16 cleaned_count = ixgbe_desc_unused(rx_ring);
2092
2093 while (likely(total_rx_packets < budget)) {
2094 union ixgbe_adv_rx_desc *rx_desc;
2095 struct sk_buff *skb;
2096
2097 /* return some buffers to hardware, one at a time is too slow */
2098 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
2099 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
2100 cleaned_count = 0;
2101 }
2102
2103 rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);
2104
2105 if (!rx_desc->wb.upper.status_error)
2106 break;
2107
2108 /* This memory barrier is needed to keep us from reading
2109 * any other fields out of the rx_desc until we know the
2110 * descriptor has been written back
2111 */
2112 dma_rmb();
2113
2114 /* retrieve a buffer from the ring */
2115 skb = ixgbe_fetch_rx_buffer(rx_ring, rx_desc);
2116
2117 /* exit if we failed to retrieve a buffer */
2118 if (!skb)
2119 break;
2120
2121 cleaned_count++;
2122
2123 /* place incomplete frames back on ring for completion */
2124 if (ixgbe_is_non_eop(rx_ring, rx_desc, skb))
2125 continue;
2126
2127 /* verify the packet layout is correct */
2128 if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb))
2129 continue;
2130
2131 /* probably a little skewed due to removing CRC */
2132 total_rx_bytes += skb->len;
2133
2134 /* populate checksum, timestamp, VLAN, and protocol */
2135 ixgbe_process_skb_fields(rx_ring, rx_desc, skb);
2136
2137 #ifdef IXGBE_FCOE
2138 /* if ddp, not passing to ULD unless for FCP_RSP or error */
2139 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) {
2140 ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb);
2141 /* include DDPed FCoE data */
2142 if (ddp_bytes > 0) {
2143 if (!mss) {
2144 mss = rx_ring->netdev->mtu -
2145 sizeof(struct fcoe_hdr) -
2146 sizeof(struct fc_frame_header) -
2147 sizeof(struct fcoe_crc_eof);
2148 if (mss > 512)
2149 mss &= ~511;
2150 }
2151 total_rx_bytes += ddp_bytes;
2152 total_rx_packets += DIV_ROUND_UP(ddp_bytes,
2153 mss);
2154 }
2155 if (!ddp_bytes) {
2156 dev_kfree_skb_any(skb);
2157 continue;
2158 }
2159 }
2160
2161 #endif /* IXGBE_FCOE */
2162 ixgbe_rx_skb(q_vector, skb);
2163
2164 /* update budget accounting */
2165 total_rx_packets++;
2166 }
2167
2168 u64_stats_update_begin(&rx_ring->syncp);
2169 rx_ring->stats.packets += total_rx_packets;
2170 rx_ring->stats.bytes += total_rx_bytes;
2171 u64_stats_update_end(&rx_ring->syncp);
2172 q_vector->rx.total_packets += total_rx_packets;
2173 q_vector->rx.total_bytes += total_rx_bytes;
2174
2175 return total_rx_packets;
2176 }
2177
2178 #ifdef CONFIG_NET_RX_BUSY_POLL
2179 /* must be called with local_bh_disable()d */
2180 static int ixgbe_low_latency_recv(struct napi_struct *napi)
2181 {
2182 struct ixgbe_q_vector *q_vector =
2183 container_of(napi, struct ixgbe_q_vector, napi);
2184 struct ixgbe_adapter *adapter = q_vector->adapter;
2185 struct ixgbe_ring *ring;
2186 int found = 0;
2187
2188 if (test_bit(__IXGBE_DOWN, &adapter->state))
2189 return LL_FLUSH_FAILED;
2190
2191 if (!ixgbe_qv_lock_poll(q_vector))
2192 return LL_FLUSH_BUSY;
2193
2194 ixgbe_for_each_ring(ring, q_vector->rx) {
2195 found = ixgbe_clean_rx_irq(q_vector, ring, 4);
2196 #ifdef BP_EXTENDED_STATS
2197 if (found)
2198 ring->stats.cleaned += found;
2199 else
2200 ring->stats.misses++;
2201 #endif
2202 if (found)
2203 break;
2204 }
2205
2206 ixgbe_qv_unlock_poll(q_vector);
2207
2208 return found;
2209 }
2210 #endif /* CONFIG_NET_RX_BUSY_POLL */
2211
2212 /**
2213 * ixgbe_configure_msix - Configure MSI-X hardware
2214 * @adapter: board private structure
2215 *
2216 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
2217 * interrupts.
2218 **/
2219 static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
2220 {
2221 struct ixgbe_q_vector *q_vector;
2222 int v_idx;
2223 u32 mask;
2224
2225 /* Populate MSIX to EITR Select */
2226 if (adapter->num_vfs > 32) {
2227 u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
2228 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
2229 }
2230
2231 /*
2232 * Populate the IVAR table and set the ITR values to the
2233 * corresponding register.
2234 */
2235 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
2236 struct ixgbe_ring *ring;
2237 q_vector = adapter->q_vector[v_idx];
2238
2239 ixgbe_for_each_ring(ring, q_vector->rx)
2240 ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);
2241
2242 ixgbe_for_each_ring(ring, q_vector->tx)
2243 ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);
2244
2245 ixgbe_write_eitr(q_vector);
2246 }
2247
2248 switch (adapter->hw.mac.type) {
2249 case ixgbe_mac_82598EB:
2250 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
2251 v_idx);
2252 break;
2253 case ixgbe_mac_82599EB:
2254 case ixgbe_mac_X540:
2255 case ixgbe_mac_X550:
2256 case ixgbe_mac_X550EM_x:
2257 case ixgbe_mac_x550em_a:
2258 ixgbe_set_ivar(adapter, -1, 1, v_idx);
2259 break;
2260 default:
2261 break;
2262 }
2263 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
2264
2265 /* set up to autoclear timer, and the vectors */
2266 mask = IXGBE_EIMS_ENABLE_MASK;
2267 mask &= ~(IXGBE_EIMS_OTHER |
2268 IXGBE_EIMS_MAILBOX |
2269 IXGBE_EIMS_LSC);
2270
2271 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
2272 }
2273
2274 enum latency_range {
2275 lowest_latency = 0,
2276 low_latency = 1,
2277 bulk_latency = 2,
2278 latency_invalid = 255
2279 };
2280
2281 /**
2282 * ixgbe_update_itr - update the dynamic ITR value based on statistics
2283 * @q_vector: structure containing interrupt and ring information
2284 * @ring_container: structure containing ring performance data
2285 *
2286 * Stores a new ITR value based on packets and byte
2287 * counts during the last interrupt. The advantage of per interrupt
2288 * computation is faster updates and more accurate ITR for the current
2289 * traffic pattern. Constants in this function were computed
2290 * based on theoretical maximum wire speed and thresholds were set based
2291 * on testing data as well as attempting to minimize response time
2292 * while increasing bulk throughput.
2293 * this functionality is controlled by the InterruptThrottleRate module
2294 * parameter (see ixgbe_param.c)
2295 **/
2296 static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
2297 struct ixgbe_ring_container *ring_container)
2298 {
2299 int bytes = ring_container->total_bytes;
2300 int packets = ring_container->total_packets;
2301 u32 timepassed_us;
2302 u64 bytes_perint;
2303 u8 itr_setting = ring_container->itr;
2304
2305 if (packets == 0)
2306 return;
2307
2308 /* simple throttlerate management
2309 * 0-10MB/s lowest (100000 ints/s)
2310 * 10-20MB/s low (20000 ints/s)
2311 * 20-1249MB/s bulk (12000 ints/s)
2312 */
2313 /* what was last interrupt timeslice? */
2314 timepassed_us = q_vector->itr >> 2;
2315 if (timepassed_us == 0)
2316 return;
2317
2318 bytes_perint = bytes / timepassed_us; /* bytes/usec */
2319
2320 switch (itr_setting) {
2321 case lowest_latency:
2322 if (bytes_perint > 10)
2323 itr_setting = low_latency;
2324 break;
2325 case low_latency:
2326 if (bytes_perint > 20)
2327 itr_setting = bulk_latency;
2328 else if (bytes_perint <= 10)
2329 itr_setting = lowest_latency;
2330 break;
2331 case bulk_latency:
2332 if (bytes_perint <= 20)
2333 itr_setting = low_latency;
2334 break;
2335 }
2336
2337 /* clear work counters since we have the values we need */
2338 ring_container->total_bytes = 0;
2339 ring_container->total_packets = 0;
2340
2341 /* write updated itr to ring container */
2342 ring_container->itr = itr_setting;
2343 }
2344
2345 /**
2346 * ixgbe_write_eitr - write EITR register in hardware specific way
2347 * @q_vector: structure containing interrupt and ring information
2348 *
2349 * This function is made to be called by ethtool and by the driver
2350 * when it needs to update EITR registers at runtime. Hardware
2351 * specific quirks/differences are taken care of here.
2352 */
2353 void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
2354 {
2355 struct ixgbe_adapter *adapter = q_vector->adapter;
2356 struct ixgbe_hw *hw = &adapter->hw;
2357 int v_idx = q_vector->v_idx;
2358 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
2359
2360 switch (adapter->hw.mac.type) {
2361 case ixgbe_mac_82598EB:
2362 /* must write high and low 16 bits to reset counter */
2363 itr_reg |= (itr_reg << 16);
2364 break;
2365 case ixgbe_mac_82599EB:
2366 case ixgbe_mac_X540:
2367 case ixgbe_mac_X550:
2368 case ixgbe_mac_X550EM_x:
2369 case ixgbe_mac_x550em_a:
2370 /*
2371 * set the WDIS bit to not clear the timer bits and cause an
2372 * immediate assertion of the interrupt
2373 */
2374 itr_reg |= IXGBE_EITR_CNT_WDIS;
2375 break;
2376 default:
2377 break;
2378 }
2379 IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
2380 }
2381
2382 static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
2383 {
2384 u32 new_itr = q_vector->itr;
2385 u8 current_itr;
2386
2387 ixgbe_update_itr(q_vector, &q_vector->tx);
2388 ixgbe_update_itr(q_vector, &q_vector->rx);
2389
2390 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
2391
2392 switch (current_itr) {
2393 /* counts and packets in update_itr are dependent on these numbers */
2394 case lowest_latency:
2395 new_itr = IXGBE_100K_ITR;
2396 break;
2397 case low_latency:
2398 new_itr = IXGBE_20K_ITR;
2399 break;
2400 case bulk_latency:
2401 new_itr = IXGBE_12K_ITR;
2402 break;
2403 default:
2404 break;
2405 }
2406
2407 if (new_itr != q_vector->itr) {
2408 /* do an exponential smoothing */
2409 new_itr = (10 * new_itr * q_vector->itr) /
2410 ((9 * new_itr) + q_vector->itr);
2411
2412 /* save the algorithm value here */
2413 q_vector->itr = new_itr;
2414
2415 ixgbe_write_eitr(q_vector);
2416 }
2417 }
2418
2419 /**
2420 * ixgbe_check_overtemp_subtask - check for over temperature
2421 * @adapter: pointer to adapter
2422 **/
2423 static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
2424 {
2425 struct ixgbe_hw *hw = &adapter->hw;
2426 u32 eicr = adapter->interrupt_event;
2427
2428 if (test_bit(__IXGBE_DOWN, &adapter->state))
2429 return;
2430
2431 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
2432 !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
2433 return;
2434
2435 adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2436
2437 switch (hw->device_id) {
2438 case IXGBE_DEV_ID_82599_T3_LOM:
2439 /*
2440 * Since the warning interrupt is for both ports
2441 * we don't have to check if:
2442 * - This interrupt wasn't for our port.
2443 * - We may have missed the interrupt so always have to
2444 * check if we got a LSC
2445 */
2446 if (!(eicr & IXGBE_EICR_GPI_SDP0_8259X) &&
2447 !(eicr & IXGBE_EICR_LSC))
2448 return;
2449
2450 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
2451 u32 speed;
2452 bool link_up = false;
2453
2454 hw->mac.ops.check_link(hw, &speed, &link_up, false);
2455
2456 if (link_up)
2457 return;
2458 }
2459
2460 /* Check if this is not due to overtemp */
2461 if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
2462 return;
2463
2464 break;
2465 default:
2466 if (adapter->hw.mac.type >= ixgbe_mac_X540)
2467 return;
2468 if (!(eicr & IXGBE_EICR_GPI_SDP0(hw)))
2469 return;
2470 break;
2471 }
2472 e_crit(drv, "%s\n", ixgbe_overheat_msg);
2473
2474 adapter->interrupt_event = 0;
2475 }
2476
2477 static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
2478 {
2479 struct ixgbe_hw *hw = &adapter->hw;
2480
2481 if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
2482 (eicr & IXGBE_EICR_GPI_SDP1(hw))) {
2483 e_crit(probe, "Fan has stopped, replace the adapter\n");
2484 /* write to clear the interrupt */
2485 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));
2486 }
2487 }
2488
2489 static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
2490 {
2491 struct ixgbe_hw *hw = &adapter->hw;
2492
2493 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
2494 return;
2495
2496 switch (adapter->hw.mac.type) {
2497 case ixgbe_mac_82599EB:
2498 /*
2499 * Need to check link state so complete overtemp check
2500 * on service task
2501 */
2502 if (((eicr & IXGBE_EICR_GPI_SDP0(hw)) ||
2503 (eicr & IXGBE_EICR_LSC)) &&
2504 (!test_bit(__IXGBE_DOWN, &adapter->state))) {
2505 adapter->interrupt_event = eicr;
2506 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2507 ixgbe_service_event_schedule(adapter);
2508 return;
2509 }
2510 return;
2511 case ixgbe_mac_X540:
2512 if (!(eicr & IXGBE_EICR_TS))
2513 return;
2514 break;
2515 default:
2516 return;
2517 }
2518
2519 e_crit(drv, "%s\n", ixgbe_overheat_msg);
2520 }
2521
2522 static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
2523 {
2524 switch (hw->mac.type) {
2525 case ixgbe_mac_82598EB:
2526 if (hw->phy.type == ixgbe_phy_nl)
2527 return true;
2528 return false;
2529 case ixgbe_mac_82599EB:
2530 case ixgbe_mac_X550EM_x:
2531 case ixgbe_mac_x550em_a:
2532 switch (hw->mac.ops.get_media_type(hw)) {
2533 case ixgbe_media_type_fiber:
2534 case ixgbe_media_type_fiber_qsfp:
2535 return true;
2536 default:
2537 return false;
2538 }
2539 default:
2540 return false;
2541 }
2542 }
2543
2544 static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
2545 {
2546 struct ixgbe_hw *hw = &adapter->hw;
2547 u32 eicr_mask = IXGBE_EICR_GPI_SDP2(hw);
2548
2549 if (!ixgbe_is_sfp(hw))
2550 return;
2551
2552 /* Later MAC's use different SDP */
2553 if (hw->mac.type >= ixgbe_mac_X540)
2554 eicr_mask = IXGBE_EICR_GPI_SDP0_X540;
2555
2556 if (eicr & eicr_mask) {
2557 /* Clear the interrupt */
2558 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);
2559 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2560 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
2561 adapter->sfp_poll_time = 0;
2562 ixgbe_service_event_schedule(adapter);
2563 }
2564 }
2565
2566 if (adapter->hw.mac.type == ixgbe_mac_82599EB &&
2567 (eicr & IXGBE_EICR_GPI_SDP1(hw))) {
2568 /* Clear the interrupt */
2569 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));
2570 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2571 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
2572 ixgbe_service_event_schedule(adapter);
2573 }
2574 }
2575 }
2576
2577 static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
2578 {
2579 struct ixgbe_hw *hw = &adapter->hw;
2580
2581 adapter->lsc_int++;
2582 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2583 adapter->link_check_timeout = jiffies;
2584 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2585 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
2586 IXGBE_WRITE_FLUSH(hw);
2587 ixgbe_service_event_schedule(adapter);
2588 }
2589 }
2590
2591 static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
2592 u64 qmask)
2593 {
2594 u32 mask;
2595 struct ixgbe_hw *hw = &adapter->hw;
2596
2597 switch (hw->mac.type) {
2598 case ixgbe_mac_82598EB:
2599 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
2600 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
2601 break;
2602 case ixgbe_mac_82599EB:
2603 case ixgbe_mac_X540:
2604 case ixgbe_mac_X550:
2605 case ixgbe_mac_X550EM_x:
2606 case ixgbe_mac_x550em_a:
2607 mask = (qmask & 0xFFFFFFFF);
2608 if (mask)
2609 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
2610 mask = (qmask >> 32);
2611 if (mask)
2612 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
2613 break;
2614 default:
2615 break;
2616 }
2617 /* skip the flush */
2618 }
2619
2620 static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
2621 u64 qmask)
2622 {
2623 u32 mask;
2624 struct ixgbe_hw *hw = &adapter->hw;
2625
2626 switch (hw->mac.type) {
2627 case ixgbe_mac_82598EB:
2628 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
2629 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
2630 break;
2631 case ixgbe_mac_82599EB:
2632 case ixgbe_mac_X540:
2633 case ixgbe_mac_X550:
2634 case ixgbe_mac_X550EM_x:
2635 case ixgbe_mac_x550em_a:
2636 mask = (qmask & 0xFFFFFFFF);
2637 if (mask)
2638 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
2639 mask = (qmask >> 32);
2640 if (mask)
2641 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
2642 break;
2643 default:
2644 break;
2645 }
2646 /* skip the flush */
2647 }
2648
2649 /**
2650 * ixgbe_irq_enable - Enable default interrupt generation settings
2651 * @adapter: board private structure
2652 **/
2653 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
2654 bool flush)
2655 {
2656 struct ixgbe_hw *hw = &adapter->hw;
2657 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
2658
2659 /* don't reenable LSC while waiting for link */
2660 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
2661 mask &= ~IXGBE_EIMS_LSC;
2662
2663 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
2664 switch (adapter->hw.mac.type) {
2665 case ixgbe_mac_82599EB:
2666 mask |= IXGBE_EIMS_GPI_SDP0(hw);
2667 break;
2668 case ixgbe_mac_X540:
2669 case ixgbe_mac_X550:
2670 case ixgbe_mac_X550EM_x:
2671 case ixgbe_mac_x550em_a:
2672 mask |= IXGBE_EIMS_TS;
2673 break;
2674 default:
2675 break;
2676 }
2677 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
2678 mask |= IXGBE_EIMS_GPI_SDP1(hw);
2679 switch (adapter->hw.mac.type) {
2680 case ixgbe_mac_82599EB:
2681 mask |= IXGBE_EIMS_GPI_SDP1(hw);
2682 mask |= IXGBE_EIMS_GPI_SDP2(hw);
2683 /* fall through */
2684 case ixgbe_mac_X540:
2685 case ixgbe_mac_X550:
2686 case ixgbe_mac_X550EM_x:
2687 case ixgbe_mac_x550em_a:
2688 if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
2689 adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
2690 adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N)
2691 mask |= IXGBE_EIMS_GPI_SDP0(&adapter->hw);
2692 if (adapter->hw.phy.type == ixgbe_phy_x550em_ext_t)
2693 mask |= IXGBE_EICR_GPI_SDP0_X540;
2694 mask |= IXGBE_EIMS_ECC;
2695 mask |= IXGBE_EIMS_MAILBOX;
2696 break;
2697 default:
2698 break;
2699 }
2700
2701 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
2702 !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
2703 mask |= IXGBE_EIMS_FLOW_DIR;
2704
2705 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
2706 if (queues)
2707 ixgbe_irq_enable_queues(adapter, ~0);
2708 if (flush)
2709 IXGBE_WRITE_FLUSH(&adapter->hw);
2710 }
2711
2712 static irqreturn_t ixgbe_msix_other(int irq, void *data)
2713 {
2714 struct ixgbe_adapter *adapter = data;
2715 struct ixgbe_hw *hw = &adapter->hw;
2716 u32 eicr;
2717
2718 /*
2719 * Workaround for Silicon errata. Use clear-by-write instead
2720 * of clear-by-read. Reading with EICS will return the
2721 * interrupt causes without clearing, which later be done
2722 * with the write to EICR.
2723 */
2724 eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
2725
2726 /* The lower 16bits of the EICR register are for the queue interrupts
2727 * which should be masked here in order to not accidentally clear them if
2728 * the bits are high when ixgbe_msix_other is called. There is a race
2729 * condition otherwise which results in possible performance loss
2730 * especially if the ixgbe_msix_other interrupt is triggering
2731 * consistently (as it would when PPS is turned on for the X540 device)
2732 */
2733 eicr &= 0xFFFF0000;
2734
2735 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
2736
2737 if (eicr & IXGBE_EICR_LSC)
2738 ixgbe_check_lsc(adapter);
2739
2740 if (eicr & IXGBE_EICR_MAILBOX)
2741 ixgbe_msg_task(adapter);
2742
2743 switch (hw->mac.type) {
2744 case ixgbe_mac_82599EB:
2745 case ixgbe_mac_X540:
2746 case ixgbe_mac_X550:
2747 case ixgbe_mac_X550EM_x:
2748 case ixgbe_mac_x550em_a:
2749 if (hw->phy.type == ixgbe_phy_x550em_ext_t &&
2750 (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
2751 adapter->flags2 |= IXGBE_FLAG2_PHY_INTERRUPT;
2752 ixgbe_service_event_schedule(adapter);
2753 IXGBE_WRITE_REG(hw, IXGBE_EICR,
2754 IXGBE_EICR_GPI_SDP0_X540);
2755 }
2756 if (eicr & IXGBE_EICR_ECC) {
2757 e_info(link, "Received ECC Err, initiating reset\n");
2758 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
2759 ixgbe_service_event_schedule(adapter);
2760 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
2761 }
2762 /* Handle Flow Director Full threshold interrupt */
2763 if (eicr & IXGBE_EICR_FLOW_DIR) {
2764 int reinit_count = 0;
2765 int i;
2766 for (i = 0; i < adapter->num_tx_queues; i++) {
2767 struct ixgbe_ring *ring = adapter->tx_ring[i];
2768 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
2769 &ring->state))
2770 reinit_count++;
2771 }
2772 if (reinit_count) {
2773 /* no more flow director interrupts until after init */
2774 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
2775 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
2776 ixgbe_service_event_schedule(adapter);
2777 }
2778 }
2779 ixgbe_check_sfp_event(adapter, eicr);
2780 ixgbe_check_overtemp_event(adapter, eicr);
2781 break;
2782 default:
2783 break;
2784 }
2785
2786 ixgbe_check_fan_failure(adapter, eicr);
2787
2788 if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
2789 ixgbe_ptp_check_pps_event(adapter);
2790
2791 /* re-enable the original interrupt state, no lsc, no queues */
2792 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2793 ixgbe_irq_enable(adapter, false, false);
2794
2795 return IRQ_HANDLED;
2796 }
2797
2798 static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
2799 {
2800 struct ixgbe_q_vector *q_vector = data;
2801
2802 /* EIAM disabled interrupts (on this vector) for us */
2803
2804 if (q_vector->rx.ring || q_vector->tx.ring)
2805 napi_schedule_irqoff(&q_vector->napi);
2806
2807 return IRQ_HANDLED;
2808 }
2809
2810 /**
2811 * ixgbe_poll - NAPI Rx polling callback
2812 * @napi: structure for representing this polling device
2813 * @budget: how many packets driver is allowed to clean
2814 *
2815 * This function is used for legacy and MSI, NAPI mode
2816 **/
2817 int ixgbe_poll(struct napi_struct *napi, int budget)
2818 {
2819 struct ixgbe_q_vector *q_vector =
2820 container_of(napi, struct ixgbe_q_vector, napi);
2821 struct ixgbe_adapter *adapter = q_vector->adapter;
2822 struct ixgbe_ring *ring;
2823 int per_ring_budget, work_done = 0;
2824 bool clean_complete = true;
2825
2826 #ifdef CONFIG_IXGBE_DCA
2827 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2828 ixgbe_update_dca(q_vector);
2829 #endif
2830
2831 ixgbe_for_each_ring(ring, q_vector->tx) {
2832 if (!ixgbe_clean_tx_irq(q_vector, ring, budget))
2833 clean_complete = false;
2834 }
2835
2836 /* Exit if we are called by netpoll or busy polling is active */
2837 if ((budget <= 0) || !ixgbe_qv_lock_napi(q_vector))
2838 return budget;
2839
2840 /* attempt to distribute budget to each queue fairly, but don't allow
2841 * the budget to go below 1 because we'll exit polling */
2842 if (q_vector->rx.count > 1)
2843 per_ring_budget = max(budget/q_vector->rx.count, 1);
2844 else
2845 per_ring_budget = budget;
2846
2847 ixgbe_for_each_ring(ring, q_vector->rx) {
2848 int cleaned = ixgbe_clean_rx_irq(q_vector, ring,
2849 per_ring_budget);
2850
2851 work_done += cleaned;
2852 if (cleaned >= per_ring_budget)
2853 clean_complete = false;
2854 }
2855
2856 ixgbe_qv_unlock_napi(q_vector);
2857 /* If all work not completed, return budget and keep polling */
2858 if (!clean_complete)
2859 return budget;
2860
2861 /* all work done, exit the polling mode */
2862 napi_complete_done(napi, work_done);
2863 if (adapter->rx_itr_setting & 1)
2864 ixgbe_set_itr(q_vector);
2865 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2866 ixgbe_irq_enable_queues(adapter, ((u64)1 << q_vector->v_idx));
2867
2868 return 0;
2869 }
2870
2871 /**
2872 * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
2873 * @adapter: board private structure
2874 *
2875 * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
2876 * interrupts from the kernel.
2877 **/
2878 static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
2879 {
2880 struct net_device *netdev = adapter->netdev;
2881 int vector, err;
2882 int ri = 0, ti = 0;
2883
2884 for (vector = 0; vector < adapter->num_q_vectors; vector++) {
2885 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
2886 struct msix_entry *entry = &adapter->msix_entries[vector];
2887
2888 if (q_vector->tx.ring && q_vector->rx.ring) {
2889 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2890 "%s-%s-%d", netdev->name, "TxRx", ri++);
2891 ti++;
2892 } else if (q_vector->rx.ring) {
2893 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2894 "%s-%s-%d", netdev->name, "rx", ri++);
2895 } else if (q_vector->tx.ring) {
2896 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2897 "%s-%s-%d", netdev->name, "tx", ti++);
2898 } else {
2899 /* skip this unused q_vector */
2900 continue;
2901 }
2902 err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
2903 q_vector->name, q_vector);
2904 if (err) {
2905 e_err(probe, "request_irq failed for MSIX interrupt "
2906 "Error: %d\n", err);
2907 goto free_queue_irqs;
2908 }
2909 /* If Flow Director is enabled, set interrupt affinity */
2910 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
2911 /* assign the mask for this irq */
2912 irq_set_affinity_hint(entry->vector,
2913 &q_vector->affinity_mask);
2914 }
2915 }
2916
2917 err = request_irq(adapter->msix_entries[vector].vector,
2918 ixgbe_msix_other, 0, netdev->name, adapter);
2919 if (err) {
2920 e_err(probe, "request_irq for msix_other failed: %d\n", err);
2921 goto free_queue_irqs;
2922 }
2923
2924 return 0;
2925
2926 free_queue_irqs:
2927 while (vector) {
2928 vector--;
2929 irq_set_affinity_hint(adapter->msix_entries[vector].vector,
2930 NULL);
2931 free_irq(adapter->msix_entries[vector].vector,
2932 adapter->q_vector[vector]);
2933 }
2934 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
2935 pci_disable_msix(adapter->pdev);
2936 kfree(adapter->msix_entries);
2937 adapter->msix_entries = NULL;
2938 return err;
2939 }
2940
2941 /**
2942 * ixgbe_intr - legacy mode Interrupt Handler
2943 * @irq: interrupt number
2944 * @data: pointer to a network interface device structure
2945 **/
2946 static irqreturn_t ixgbe_intr(int irq, void *data)
2947 {
2948 struct ixgbe_adapter *adapter = data;
2949 struct ixgbe_hw *hw = &adapter->hw;
2950 struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2951 u32 eicr;
2952
2953 /*
2954 * Workaround for silicon errata #26 on 82598. Mask the interrupt
2955 * before the read of EICR.
2956 */
2957 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
2958
2959 /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
2960 * therefore no explicit interrupt disable is necessary */
2961 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2962 if (!eicr) {
2963 /*
2964 * shared interrupt alert!
2965 * make sure interrupts are enabled because the read will
2966 * have disabled interrupts due to EIAM
2967 * finish the workaround of silicon errata on 82598. Unmask
2968 * the interrupt that we masked before the EICR read.
2969 */
2970 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2971 ixgbe_irq_enable(adapter, true, true);
2972 return IRQ_NONE; /* Not our interrupt */
2973 }
2974
2975 if (eicr & IXGBE_EICR_LSC)
2976 ixgbe_check_lsc(adapter);
2977
2978 switch (hw->mac.type) {
2979 case ixgbe_mac_82599EB:
2980 ixgbe_check_sfp_event(adapter, eicr);
2981 /* Fall through */
2982 case ixgbe_mac_X540:
2983 case ixgbe_mac_X550:
2984 case ixgbe_mac_X550EM_x:
2985 case ixgbe_mac_x550em_a:
2986 if (eicr & IXGBE_EICR_ECC) {
2987 e_info(link, "Received ECC Err, initiating reset\n");
2988 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
2989 ixgbe_service_event_schedule(adapter);
2990 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
2991 }
2992 ixgbe_check_overtemp_event(adapter, eicr);
2993 break;
2994 default:
2995 break;
2996 }
2997
2998 ixgbe_check_fan_failure(adapter, eicr);
2999 if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
3000 ixgbe_ptp_check_pps_event(adapter);
3001
3002 /* would disable interrupts here but EIAM disabled it */
3003 napi_schedule_irqoff(&q_vector->napi);
3004
3005 /*
3006 * re-enable link(maybe) and non-queue interrupts, no flush.
3007 * ixgbe_poll will re-enable the queue interrupts
3008 */
3009 if (!test_bit(__IXGBE_DOWN, &adapter->state))
3010 ixgbe_irq_enable(adapter, false, false);
3011
3012 return IRQ_HANDLED;
3013 }
3014
3015 /**
3016 * ixgbe_request_irq - initialize interrupts
3017 * @adapter: board private structure
3018 *
3019 * Attempts to configure interrupts using the best available
3020 * capabilities of the hardware and kernel.
3021 **/
3022 static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
3023 {
3024 struct net_device *netdev = adapter->netdev;
3025 int err;
3026
3027 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
3028 err = ixgbe_request_msix_irqs(adapter);
3029 else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
3030 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
3031 netdev->name, adapter);
3032 else
3033 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
3034 netdev->name, adapter);
3035
3036 if (err)
3037 e_err(probe, "request_irq failed, Error %d\n", err);
3038
3039 return err;
3040 }
3041
3042 static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
3043 {
3044 int vector;
3045
3046 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
3047 free_irq(adapter->pdev->irq, adapter);
3048 return;
3049 }
3050
3051 for (vector = 0; vector < adapter->num_q_vectors; vector++) {
3052 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
3053 struct msix_entry *entry = &adapter->msix_entries[vector];
3054
3055 /* free only the irqs that were actually requested */
3056 if (!q_vector->rx.ring && !q_vector->tx.ring)
3057 continue;
3058
3059 /* clear the affinity_mask in the IRQ descriptor */
3060 irq_set_affinity_hint(entry->vector, NULL);
3061
3062 free_irq(entry->vector, q_vector);
3063 }
3064
3065 free_irq(adapter->msix_entries[vector++].vector, adapter);
3066 }
3067
3068 /**
3069 * ixgbe_irq_disable - Mask off interrupt generation on the NIC
3070 * @adapter: board private structure
3071 **/
3072 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
3073 {
3074 switch (adapter->hw.mac.type) {
3075 case ixgbe_mac_82598EB:
3076 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
3077 break;
3078 case ixgbe_mac_82599EB:
3079 case ixgbe_mac_X540:
3080 case ixgbe_mac_X550:
3081 case ixgbe_mac_X550EM_x:
3082 case ixgbe_mac_x550em_a:
3083 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
3084 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
3085 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
3086 break;
3087 default:
3088 break;
3089 }
3090 IXGBE_WRITE_FLUSH(&adapter->hw);
3091 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
3092 int vector;
3093
3094 for (vector = 0; vector < adapter->num_q_vectors; vector++)
3095 synchronize_irq(adapter->msix_entries[vector].vector);
3096
3097 synchronize_irq(adapter->msix_entries[vector++].vector);
3098 } else {
3099 synchronize_irq(adapter->pdev->irq);
3100 }
3101 }
3102
3103 /**
3104 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
3105 *
3106 **/
3107 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
3108 {
3109 struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
3110
3111 ixgbe_write_eitr(q_vector);
3112
3113 ixgbe_set_ivar(adapter, 0, 0, 0);
3114 ixgbe_set_ivar(adapter, 1, 0, 0);
3115
3116 e_info(hw, "Legacy interrupt IVAR setup done\n");
3117 }
3118
3119 /**
3120 * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
3121 * @adapter: board private structure
3122 * @ring: structure containing ring specific data
3123 *
3124 * Configure the Tx descriptor ring after a reset.
3125 **/
3126 void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
3127 struct ixgbe_ring *ring)
3128 {
3129 struct ixgbe_hw *hw = &adapter->hw;
3130 u64 tdba = ring->dma;
3131 int wait_loop = 10;
3132 u32 txdctl = IXGBE_TXDCTL_ENABLE;
3133 u8 reg_idx = ring->reg_idx;
3134
3135 /* disable queue to avoid issues while updating state */
3136 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
3137 IXGBE_WRITE_FLUSH(hw);
3138
3139 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
3140 (tdba & DMA_BIT_MASK(32)));
3141 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
3142 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
3143 ring->count * sizeof(union ixgbe_adv_tx_desc));
3144 IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
3145 IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
3146 ring->tail = adapter->io_addr + IXGBE_TDT(reg_idx);
3147
3148 /*
3149 * set WTHRESH to encourage burst writeback, it should not be set
3150 * higher than 1 when:
3151 * - ITR is 0 as it could cause false TX hangs
3152 * - ITR is set to > 100k int/sec and BQL is enabled
3153 *
3154 * In order to avoid issues WTHRESH + PTHRESH should always be equal
3155 * to or less than the number of on chip descriptors, which is
3156 * currently 40.
3157 */
3158 if (!ring->q_vector || (ring->q_vector->itr < IXGBE_100K_ITR))
3159 txdctl |= (1 << 16); /* WTHRESH = 1 */
3160 else
3161 txdctl |= (8 << 16); /* WTHRESH = 8 */
3162
3163 /*
3164 * Setting PTHRESH to 32 both improves performance
3165 * and avoids a TX hang with DFP enabled
3166 */
3167 txdctl |= (1 << 8) | /* HTHRESH = 1 */
3168 32; /* PTHRESH = 32 */
3169
3170 /* reinitialize flowdirector state */
3171 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
3172 ring->atr_sample_rate = adapter->atr_sample_rate;
3173 ring->atr_count = 0;
3174 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
3175 } else {
3176 ring->atr_sample_rate = 0;
3177 }
3178
3179 /* initialize XPS */
3180 if (!test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, &ring->state)) {
3181 struct ixgbe_q_vector *q_vector = ring->q_vector;
3182
3183 if (q_vector)
3184 netif_set_xps_queue(ring->netdev,
3185 &q_vector->affinity_mask,
3186 ring->queue_index);
3187 }
3188
3189 clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);
3190
3191 /* enable queue */
3192 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
3193
3194 /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
3195 if (hw->mac.type == ixgbe_mac_82598EB &&
3196 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
3197 return;
3198
3199 /* poll to verify queue is enabled */
3200 do {
3201 usleep_range(1000, 2000);
3202 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
3203 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
3204 if (!wait_loop)
3205 e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
3206 }
3207
3208 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
3209 {
3210 struct ixgbe_hw *hw = &adapter->hw;
3211 u32 rttdcs, mtqc;
3212 u8 tcs = netdev_get_num_tc(adapter->netdev);
3213
3214 if (hw->mac.type == ixgbe_mac_82598EB)
3215 return;
3216
3217 /* disable the arbiter while setting MTQC */
3218 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3219 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3220 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3221
3222 /* set transmit pool layout */
3223 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
3224 mtqc = IXGBE_MTQC_VT_ENA;
3225 if (tcs > 4)
3226 mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
3227 else if (tcs > 1)
3228 mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
3229 else if (adapter->ring_feature[RING_F_RSS].indices == 4)
3230 mtqc |= IXGBE_MTQC_32VF;
3231 else
3232 mtqc |= IXGBE_MTQC_64VF;
3233 } else {
3234 if (tcs > 4)
3235 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
3236 else if (tcs > 1)
3237 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
3238 else
3239 mtqc = IXGBE_MTQC_64Q_1PB;
3240 }
3241
3242 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
3243
3244 /* Enable Security TX Buffer IFG for multiple pb */
3245 if (tcs) {
3246 u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
3247 sectx |= IXGBE_SECTX_DCB;
3248 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx);
3249 }
3250
3251 /* re-enable the arbiter */
3252 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3253 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3254 }
3255
3256 /**
3257 * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
3258 * @adapter: board private structure
3259 *
3260 * Configure the Tx unit of the MAC after a reset.
3261 **/
3262 static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
3263 {
3264 struct ixgbe_hw *hw = &adapter->hw;
3265 u32 dmatxctl;
3266 u32 i;
3267
3268 ixgbe_setup_mtqc(adapter);
3269
3270 if (hw->mac.type != ixgbe_mac_82598EB) {
3271 /* DMATXCTL.EN must be before Tx queues are enabled */
3272 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3273 dmatxctl |= IXGBE_DMATXCTL_TE;
3274 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3275 }
3276
3277 /* Setup the HW Tx Head and Tail descriptor pointers */
3278 for (i = 0; i < adapter->num_tx_queues; i++)
3279 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
3280 }
3281
3282 static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter,
3283 struct ixgbe_ring *ring)
3284 {
3285 struct ixgbe_hw *hw = &adapter->hw;
3286 u8 reg_idx = ring->reg_idx;
3287 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
3288
3289 srrctl |= IXGBE_SRRCTL_DROP_EN;
3290
3291 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
3292 }
3293
3294 static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter,
3295 struct ixgbe_ring *ring)
3296 {
3297 struct ixgbe_hw *hw = &adapter->hw;
3298 u8 reg_idx = ring->reg_idx;
3299 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
3300
3301 srrctl &= ~IXGBE_SRRCTL_DROP_EN;
3302
3303 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
3304 }
3305
3306 #ifdef CONFIG_IXGBE_DCB
3307 void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
3308 #else
3309 static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
3310 #endif
3311 {
3312 int i;
3313 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
3314
3315 if (adapter->ixgbe_ieee_pfc)
3316 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
3317
3318 /*
3319 * We should set the drop enable bit if:
3320 * SR-IOV is enabled
3321 * or
3322 * Number of Rx queues > 1 and flow control is disabled
3323 *
3324 * This allows us to avoid head of line blocking for security
3325 * and performance reasons.
3326 */
3327 if (adapter->num_vfs || (adapter->num_rx_queues > 1 &&
3328 !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) {
3329 for (i = 0; i < adapter->num_rx_queues; i++)
3330 ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]);
3331 } else {
3332 for (i = 0; i < adapter->num_rx_queues; i++)
3333 ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]);
3334 }
3335 }
3336
3337 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3338
3339 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
3340 struct ixgbe_ring *rx_ring)
3341 {
3342 struct ixgbe_hw *hw = &adapter->hw;
3343 u32 srrctl;
3344 u8 reg_idx = rx_ring->reg_idx;
3345
3346 if (hw->mac.type == ixgbe_mac_82598EB) {
3347 u16 mask = adapter->ring_feature[RING_F_RSS].mask;
3348
3349 /*
3350 * if VMDq is not active we must program one srrctl register
3351 * per RSS queue since we have enabled RDRXCTL.MVMEN
3352 */
3353 reg_idx &= mask;
3354 }
3355
3356 /* configure header buffer length, needed for RSC */
3357 srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;
3358
3359 /* configure the packet buffer length */
3360 srrctl |= ixgbe_rx_bufsz(rx_ring) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
3361
3362 /* configure descriptor type */
3363 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
3364
3365 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
3366 }
3367
3368 /**
3369 * ixgbe_rss_indir_tbl_entries - Return RSS indirection table entries
3370 * @adapter: device handle
3371 *
3372 * - 82598/82599/X540: 128
3373 * - X550(non-SRIOV mode): 512
3374 * - X550(SRIOV mode): 64
3375 */
3376 u32 ixgbe_rss_indir_tbl_entries(struct ixgbe_adapter *adapter)
3377 {
3378 if (adapter->hw.mac.type < ixgbe_mac_X550)
3379 return 128;
3380 else if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
3381 return 64;
3382 else
3383 return 512;
3384 }
3385
3386 /**
3387 * ixgbe_store_reta - Write the RETA table to HW
3388 * @adapter: device handle
3389 *
3390 * Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.
3391 */
3392 void ixgbe_store_reta(struct ixgbe_adapter *adapter)
3393 {
3394 u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
3395 struct ixgbe_hw *hw = &adapter->hw;
3396 u32 reta = 0;
3397 u32 indices_multi;
3398 u8 *indir_tbl = adapter->rss_indir_tbl;
3399
3400 /* Fill out the redirection table as follows:
3401 * - 82598: 8 bit wide entries containing pair of 4 bit RSS
3402 * indices.
3403 * - 82599/X540: 8 bit wide entries containing 4 bit RSS index
3404 * - X550: 8 bit wide entries containing 6 bit RSS index
3405 */
3406 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
3407 indices_multi = 0x11;
3408 else
3409 indices_multi = 0x1;
3410
3411 /* Write redirection table to HW */
3412 for (i = 0; i < reta_entries; i++) {
3413 reta |= indices_multi * indir_tbl[i] << (i & 0x3) * 8;
3414 if ((i & 3) == 3) {
3415 if (i < 128)
3416 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3417 else
3418 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
3419 reta);
3420 reta = 0;
3421 }
3422 }
3423 }
3424
3425 /**
3426 * ixgbe_store_vfreta - Write the RETA table to HW (x550 devices in SRIOV mode)
3427 * @adapter: device handle
3428 *
3429 * Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.
3430 */
3431 static void ixgbe_store_vfreta(struct ixgbe_adapter *adapter)
3432 {
3433 u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
3434 struct ixgbe_hw *hw = &adapter->hw;
3435 u32 vfreta = 0;
3436 unsigned int pf_pool = adapter->num_vfs;
3437
3438 /* Write redirection table to HW */
3439 for (i = 0; i < reta_entries; i++) {
3440 vfreta |= (u32)adapter->rss_indir_tbl[i] << (i & 0x3) * 8;
3441 if ((i & 3) == 3) {
3442 IXGBE_WRITE_REG(hw, IXGBE_PFVFRETA(i >> 2, pf_pool),
3443 vfreta);
3444 vfreta = 0;
3445 }
3446 }
3447 }
3448
3449 static void ixgbe_setup_reta(struct ixgbe_adapter *adapter)
3450 {
3451 struct ixgbe_hw *hw = &adapter->hw;
3452 u32 i, j;
3453 u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
3454 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
3455
3456 /* Program table for at least 2 queues w/ SR-IOV so that VFs can
3457 * make full use of any rings they may have. We will use the
3458 * PSRTYPE register to control how many rings we use within the PF.
3459 */
3460 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 2))
3461 rss_i = 2;
3462
3463 /* Fill out hash function seeds */
3464 for (i = 0; i < 10; i++)
3465 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), adapter->rss_key[i]);
3466
3467 /* Fill out redirection table */
3468 memset(adapter->rss_indir_tbl, 0, sizeof(adapter->rss_indir_tbl));
3469
3470 for (i = 0, j = 0; i < reta_entries; i++, j++) {
3471 if (j == rss_i)
3472 j = 0;
3473
3474 adapter->rss_indir_tbl[i] = j;
3475 }
3476
3477 ixgbe_store_reta(adapter);
3478 }
3479
3480 static void ixgbe_setup_vfreta(struct ixgbe_adapter *adapter)
3481 {
3482 struct ixgbe_hw *hw = &adapter->hw;
3483 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
3484 unsigned int pf_pool = adapter->num_vfs;
3485 int i, j;
3486
3487 /* Fill out hash function seeds */
3488 for (i = 0; i < 10; i++)
3489 IXGBE_WRITE_REG(hw, IXGBE_PFVFRSSRK(i, pf_pool),
3490 adapter->rss_key[i]);
3491
3492 /* Fill out the redirection table */
3493 for (i = 0, j = 0; i < 64; i++, j++) {
3494 if (j == rss_i)
3495 j = 0;
3496
3497 adapter->rss_indir_tbl[i] = j;
3498 }
3499
3500 ixgbe_store_vfreta(adapter);
3501 }
3502
3503 static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
3504 {
3505 struct ixgbe_hw *hw = &adapter->hw;
3506 u32 mrqc = 0, rss_field = 0, vfmrqc = 0;
3507 u32 rxcsum;
3508
3509 /* Disable indicating checksum in descriptor, enables RSS hash */
3510 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3511 rxcsum |= IXGBE_RXCSUM_PCSD;
3512 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3513
3514 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
3515 if (adapter->ring_feature[RING_F_RSS].mask)
3516 mrqc = IXGBE_MRQC_RSSEN;
3517 } else {
3518 u8 tcs = netdev_get_num_tc(adapter->netdev);
3519
3520 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
3521 if (tcs > 4)
3522 mrqc = IXGBE_MRQC_VMDQRT8TCEN; /* 8 TCs */
3523 else if (tcs > 1)
3524 mrqc = IXGBE_MRQC_VMDQRT4TCEN; /* 4 TCs */
3525 else if (adapter->ring_feature[RING_F_RSS].indices == 4)
3526 mrqc = IXGBE_MRQC_VMDQRSS32EN;
3527 else
3528 mrqc = IXGBE_MRQC_VMDQRSS64EN;
3529 } else {
3530 if (tcs > 4)
3531 mrqc = IXGBE_MRQC_RTRSS8TCEN;
3532 else if (tcs > 1)
3533 mrqc = IXGBE_MRQC_RTRSS4TCEN;
3534 else
3535 mrqc = IXGBE_MRQC_RSSEN;
3536 }
3537 }
3538
3539 /* Perform hash on these packet types */
3540 rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4 |
3541 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
3542 IXGBE_MRQC_RSS_FIELD_IPV6 |
3543 IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
3544
3545 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP)
3546 rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
3547 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP)
3548 rss_field |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
3549
3550 netdev_rss_key_fill(adapter->rss_key, sizeof(adapter->rss_key));
3551 if ((hw->mac.type >= ixgbe_mac_X550) &&
3552 (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) {
3553 unsigned int pf_pool = adapter->num_vfs;
3554
3555 /* Enable VF RSS mode */
3556 mrqc |= IXGBE_MRQC_MULTIPLE_RSS;
3557 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3558
3559 /* Setup RSS through the VF registers */
3560 ixgbe_setup_vfreta(adapter);
3561 vfmrqc = IXGBE_MRQC_RSSEN;
3562 vfmrqc |= rss_field;
3563 IXGBE_WRITE_REG(hw, IXGBE_PFVFMRQC(pf_pool), vfmrqc);
3564 } else {
3565 ixgbe_setup_reta(adapter);
3566 mrqc |= rss_field;
3567 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3568 }
3569 }
3570
3571 /**
3572 * ixgbe_configure_rscctl - enable RSC for the indicated ring
3573 * @adapter: address of board private structure
3574 * @index: index of ring to set
3575 **/
3576 static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
3577 struct ixgbe_ring *ring)
3578 {
3579 struct ixgbe_hw *hw = &adapter->hw;
3580 u32 rscctrl;
3581 u8 reg_idx = ring->reg_idx;
3582
3583 if (!ring_is_rsc_enabled(ring))
3584 return;
3585
3586 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
3587 rscctrl |= IXGBE_RSCCTL_RSCEN;
3588 /*
3589 * we must limit the number of descriptors so that the
3590 * total size of max desc * buf_len is not greater
3591 * than 65536
3592 */
3593 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3594 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
3595 }
3596
3597 #define IXGBE_MAX_RX_DESC_POLL 10
3598 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
3599 struct ixgbe_ring *ring)
3600 {
3601 struct ixgbe_hw *hw = &adapter->hw;
3602 int wait_loop = IXGBE_MAX_RX_DESC_POLL;
3603 u32 rxdctl;
3604 u8 reg_idx = ring->reg_idx;
3605
3606 if (ixgbe_removed(hw->hw_addr))
3607 return;
3608 /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
3609 if (hw->mac.type == ixgbe_mac_82598EB &&
3610 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
3611 return;
3612
3613 do {
3614 usleep_range(1000, 2000);
3615 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3616 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
3617
3618 if (!wait_loop) {
3619 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
3620 "the polling period\n", reg_idx);
3621 }
3622 }
3623
3624 void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter,
3625 struct ixgbe_ring *ring)
3626 {
3627 struct ixgbe_hw *hw = &adapter->hw;
3628 int wait_loop = IXGBE_MAX_RX_DESC_POLL;
3629 u32 rxdctl;
3630 u8 reg_idx = ring->reg_idx;
3631
3632 if (ixgbe_removed(hw->hw_addr))
3633 return;
3634 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3635 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
3636
3637 /* write value back with RXDCTL.ENABLE bit cleared */
3638 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
3639
3640 if (hw->mac.type == ixgbe_mac_82598EB &&
3641 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
3642 return;
3643
3644 /* the hardware may take up to 100us to really disable the rx queue */
3645 do {
3646 udelay(10);
3647 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3648 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
3649
3650 if (!wait_loop) {
3651 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
3652 "the polling period\n", reg_idx);
3653 }
3654 }
3655
3656 void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
3657 struct ixgbe_ring *ring)
3658 {
3659 struct ixgbe_hw *hw = &adapter->hw;
3660 u64 rdba = ring->dma;
3661 u32 rxdctl;
3662 u8 reg_idx = ring->reg_idx;
3663
3664 /* disable queue to avoid issues while updating state */
3665 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3666 ixgbe_disable_rx_queue(adapter, ring);
3667
3668 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
3669 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
3670 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
3671 ring->count * sizeof(union ixgbe_adv_rx_desc));
3672 /* Force flushing of IXGBE_RDLEN to prevent MDD */
3673 IXGBE_WRITE_FLUSH(hw);
3674
3675 IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
3676 IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
3677 ring->tail = adapter->io_addr + IXGBE_RDT(reg_idx);
3678
3679 ixgbe_configure_srrctl(adapter, ring);
3680 ixgbe_configure_rscctl(adapter, ring);
3681
3682 if (hw->mac.type == ixgbe_mac_82598EB) {
3683 /*
3684 * enable cache line friendly hardware writes:
3685 * PTHRESH=32 descriptors (half the internal cache),
3686 * this also removes ugly rx_no_buffer_count increment
3687 * HTHRESH=4 descriptors (to minimize latency on fetch)
3688 * WTHRESH=8 burst writeback up to two cache lines
3689 */
3690 rxdctl &= ~0x3FFFFF;
3691 rxdctl |= 0x080420;
3692 }
3693
3694 /* enable receive descriptor ring */
3695 rxdctl |= IXGBE_RXDCTL_ENABLE;
3696 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
3697
3698 ixgbe_rx_desc_queue_enable(adapter, ring);
3699 ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
3700 }
3701
3702 static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
3703 {
3704 struct ixgbe_hw *hw = &adapter->hw;
3705 int rss_i = adapter->ring_feature[RING_F_RSS].indices;
3706 u16 pool;
3707
3708 /* PSRTYPE must be initialized in non 82598 adapters */
3709 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
3710 IXGBE_PSRTYPE_UDPHDR |
3711 IXGBE_PSRTYPE_IPV4HDR |
3712 IXGBE_PSRTYPE_L2HDR |
3713 IXGBE_PSRTYPE_IPV6HDR;
3714
3715 if (hw->mac.type == ixgbe_mac_82598EB)
3716 return;
3717
3718 if (rss_i > 3)
3719 psrtype |= 2 << 29;
3720 else if (rss_i > 1)
3721 psrtype |= 1 << 29;
3722
3723 for_each_set_bit(pool, &adapter->fwd_bitmask, 32)
3724 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(pool)), psrtype);
3725 }
3726
3727 static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
3728 {
3729 struct ixgbe_hw *hw = &adapter->hw;
3730 u32 reg_offset, vf_shift;
3731 u32 gcr_ext, vmdctl;
3732 int i;
3733
3734 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
3735 return;
3736
3737 vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
3738 vmdctl |= IXGBE_VMD_CTL_VMDQ_EN;
3739 vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
3740 vmdctl |= VMDQ_P(0) << IXGBE_VT_CTL_POOL_SHIFT;
3741 vmdctl |= IXGBE_VT_CTL_REPLEN;
3742 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
3743
3744 vf_shift = VMDQ_P(0) % 32;
3745 reg_offset = (VMDQ_P(0) >= 32) ? 1 : 0;
3746
3747 /* Enable only the PF's pool for Tx/Rx */
3748 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (~0) << vf_shift);
3749 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), reg_offset - 1);
3750 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (~0) << vf_shift);
3751 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), reg_offset - 1);
3752 if (adapter->bridge_mode == BRIDGE_MODE_VEB)
3753 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
3754
3755 /* Map PF MAC address in RAR Entry 0 to first pool following VFs */
3756 hw->mac.ops.set_vmdq(hw, 0, VMDQ_P(0));
3757
3758 /* clear VLAN promisc flag so VFTA will be updated if necessary */
3759 adapter->flags2 &= ~IXGBE_FLAG2_VLAN_PROMISC;
3760
3761 /*
3762 * Set up VF register offsets for selected VT Mode,
3763 * i.e. 32 or 64 VFs for SR-IOV
3764 */
3765 switch (adapter->ring_feature[RING_F_VMDQ].mask) {
3766 case IXGBE_82599_VMDQ_8Q_MASK:
3767 gcr_ext = IXGBE_GCR_EXT_VT_MODE_16;
3768 break;
3769 case IXGBE_82599_VMDQ_4Q_MASK:
3770 gcr_ext = IXGBE_GCR_EXT_VT_MODE_32;
3771 break;
3772 default:
3773 gcr_ext = IXGBE_GCR_EXT_VT_MODE_64;
3774 break;
3775 }
3776
3777 IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
3778
3779
3780 /* Enable MAC Anti-Spoofing */
3781 hw->mac.ops.set_mac_anti_spoofing(hw, (adapter->num_vfs != 0),
3782 adapter->num_vfs);
3783
3784 /* Ensure LLDP and FC is set for Ethertype Antispoofing if we will be
3785 * calling set_ethertype_anti_spoofing for each VF in loop below
3786 */
3787 if (hw->mac.ops.set_ethertype_anti_spoofing) {
3788 IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_LLDP),
3789 (IXGBE_ETQF_FILTER_EN |
3790 IXGBE_ETQF_TX_ANTISPOOF |
3791 IXGBE_ETH_P_LLDP));
3792
3793 IXGBE_WRITE_REG(hw, IXGBE_ETQF(IXGBE_ETQF_FILTER_FC),
3794 (IXGBE_ETQF_FILTER_EN |
3795 IXGBE_ETQF_TX_ANTISPOOF |
3796 ETH_P_PAUSE));
3797 }
3798
3799 /* For VFs that have spoof checking turned off */
3800 for (i = 0; i < adapter->num_vfs; i++) {
3801 if (!adapter->vfinfo[i].spoofchk_enabled)
3802 ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i, false);
3803
3804 /* enable ethertype anti spoofing if hw supports it */
3805 if (hw->mac.ops.set_ethertype_anti_spoofing)
3806 hw->mac.ops.set_ethertype_anti_spoofing(hw, true, i);
3807
3808 /* Enable/Disable RSS query feature */
3809 ixgbe_ndo_set_vf_rss_query_en(adapter->netdev, i,
3810 adapter->vfinfo[i].rss_query_enabled);
3811 }
3812 }
3813
3814 static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
3815 {
3816 struct ixgbe_hw *hw = &adapter->hw;
3817 struct net_device *netdev = adapter->netdev;
3818 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
3819 struct ixgbe_ring *rx_ring;
3820 int i;
3821 u32 mhadd, hlreg0;
3822
3823 #ifdef IXGBE_FCOE
3824 /* adjust max frame to be able to do baby jumbo for FCoE */
3825 if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
3826 (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
3827 max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;
3828
3829 #endif /* IXGBE_FCOE */
3830
3831 /* adjust max frame to be at least the size of a standard frame */
3832 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
3833 max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN);
3834
3835 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
3836 if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
3837 mhadd &= ~IXGBE_MHADD_MFS_MASK;
3838 mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;
3839
3840 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
3841 }
3842
3843 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3844 /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
3845 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
3846 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
3847
3848 /*
3849 * Setup the HW Rx Head and Tail Descriptor Pointers and
3850 * the Base and Length of the Rx Descriptor Ring
3851 */
3852 for (i = 0; i < adapter->num_rx_queues; i++) {
3853 rx_ring = adapter->rx_ring[i];
3854 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
3855 set_ring_rsc_enabled(rx_ring);
3856 else
3857 clear_ring_rsc_enabled(rx_ring);
3858 }
3859 }
3860
3861 static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)
3862 {
3863 struct ixgbe_hw *hw = &adapter->hw;
3864 u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3865
3866 switch (hw->mac.type) {
3867 case ixgbe_mac_82598EB:
3868 /*
3869 * For VMDq support of different descriptor types or
3870 * buffer sizes through the use of multiple SRRCTL
3871 * registers, RDRXCTL.MVMEN must be set to 1
3872 *
3873 * also, the manual doesn't mention it clearly but DCA hints
3874 * will only use queue 0's tags unless this bit is set. Side
3875 * effects of setting this bit are only that SRRCTL must be
3876 * fully programmed [0..15]
3877 */
3878 rdrxctl |= IXGBE_RDRXCTL_MVMEN;
3879 break;
3880 case ixgbe_mac_X550:
3881 case ixgbe_mac_X550EM_x:
3882 case ixgbe_mac_x550em_a:
3883 if (adapter->num_vfs)
3884 rdrxctl |= IXGBE_RDRXCTL_PSP;
3885 /* fall through for older HW */
3886 case ixgbe_mac_82599EB:
3887 case ixgbe_mac_X540:
3888 /* Disable RSC for ACK packets */
3889 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3890 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3891 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3892 /* hardware requires some bits to be set by default */
3893 rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX);
3894 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3895 break;
3896 default:
3897 /* We should do nothing since we don't know this hardware */
3898 return;
3899 }
3900
3901 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3902 }
3903
3904 /**
3905 * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset
3906 * @adapter: board private structure
3907 *
3908 * Configure the Rx unit of the MAC after a reset.
3909 **/
3910 static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
3911 {
3912 struct ixgbe_hw *hw = &adapter->hw;
3913 int i;
3914 u32 rxctrl, rfctl;
3915
3916 /* disable receives while setting up the descriptors */
3917 hw->mac.ops.disable_rx(hw);
3918
3919 ixgbe_setup_psrtype(adapter);
3920 ixgbe_setup_rdrxctl(adapter);
3921
3922 /* RSC Setup */
3923 rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL);
3924 rfctl &= ~IXGBE_RFCTL_RSC_DIS;
3925 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED))
3926 rfctl |= IXGBE_RFCTL_RSC_DIS;
3927 IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl);
3928
3929 /* Program registers for the distribution of queues */
3930 ixgbe_setup_mrqc(adapter);
3931
3932 /* set_rx_buffer_len must be called before ring initialization */
3933 ixgbe_set_rx_buffer_len(adapter);
3934
3935 /*
3936 * Setup the HW Rx Head and Tail Descriptor Pointers and
3937 * the Base and Length of the Rx Descriptor Ring
3938 */
3939 for (i = 0; i < adapter->num_rx_queues; i++)
3940 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]);
3941
3942 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3943 /* disable drop enable for 82598 parts */
3944 if (hw->mac.type == ixgbe_mac_82598EB)
3945 rxctrl |= IXGBE_RXCTRL_DMBYPS;
3946
3947 /* enable all receives */
3948 rxctrl |= IXGBE_RXCTRL_RXEN;
3949 hw->mac.ops.enable_rx_dma(hw, rxctrl);
3950 }
3951
3952 static int ixgbe_vlan_rx_add_vid(struct net_device *netdev,
3953 __be16 proto, u16 vid)
3954 {
3955 struct ixgbe_adapter *adapter = netdev_priv(netdev);
3956 struct ixgbe_hw *hw = &adapter->hw;
3957
3958 /* add VID to filter table */
3959 if (!vid || !(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
3960 hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), true, !!vid);
3961
3962 set_bit(vid, adapter->active_vlans);
3963
3964 return 0;
3965 }
3966
3967 static int ixgbe_find_vlvf_entry(struct ixgbe_hw *hw, u32 vlan)
3968 {
3969 u32 vlvf;
3970 int idx;
3971
3972 /* short cut the special case */
3973 if (vlan == 0)
3974 return 0;
3975
3976 /* Search for the vlan id in the VLVF entries */
3977 for (idx = IXGBE_VLVF_ENTRIES; --idx;) {
3978 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(idx));
3979 if ((vlvf & VLAN_VID_MASK) == vlan)
3980 break;
3981 }
3982
3983 return idx;
3984 }
3985
3986 void ixgbe_update_pf_promisc_vlvf(struct ixgbe_adapter *adapter, u32 vid)
3987 {
3988 struct ixgbe_hw *hw = &adapter->hw;
3989 u32 bits, word;
3990 int idx;
3991
3992 idx = ixgbe_find_vlvf_entry(hw, vid);
3993 if (!idx)
3994 return;
3995
3996 /* See if any other pools are set for this VLAN filter
3997 * entry other than the PF.
3998 */
3999 word = idx * 2 + (VMDQ_P(0) / 32);
4000 bits = ~(1 << (VMDQ_P(0)) % 32);
4001 bits &= IXGBE_READ_REG(hw, IXGBE_VLVFB(word));
4002
4003 /* Disable the filter so this falls into the default pool. */
4004 if (!bits && !IXGBE_READ_REG(hw, IXGBE_VLVFB(word ^ 1))) {
4005 if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4006 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(word), 0);
4007 IXGBE_WRITE_REG(hw, IXGBE_VLVF(idx), 0);
4008 }
4009 }
4010
4011 static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev,
4012 __be16 proto, u16 vid)
4013 {
4014 struct ixgbe_adapter *adapter = netdev_priv(netdev);
4015 struct ixgbe_hw *hw = &adapter->hw;
4016
4017 /* remove VID from filter table */
4018 if (vid && !(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4019 hw->mac.ops.set_vfta(hw, vid, VMDQ_P(0), false, true);
4020
4021 clear_bit(vid, adapter->active_vlans);
4022
4023 return 0;
4024 }
4025
4026 /**
4027 * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping
4028 * @adapter: driver data
4029 */
4030 static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
4031 {
4032 struct ixgbe_hw *hw = &adapter->hw;
4033 u32 vlnctrl;
4034 int i, j;
4035
4036 switch (hw->mac.type) {
4037 case ixgbe_mac_82598EB:
4038 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4039 vlnctrl &= ~IXGBE_VLNCTRL_VME;
4040 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4041 break;
4042 case ixgbe_mac_82599EB:
4043 case ixgbe_mac_X540:
4044 case ixgbe_mac_X550:
4045 case ixgbe_mac_X550EM_x:
4046 case ixgbe_mac_x550em_a:
4047 for (i = 0; i < adapter->num_rx_queues; i++) {
4048 struct ixgbe_ring *ring = adapter->rx_ring[i];
4049
4050 if (ring->l2_accel_priv)
4051 continue;
4052 j = ring->reg_idx;
4053 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
4054 vlnctrl &= ~IXGBE_RXDCTL_VME;
4055 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
4056 }
4057 break;
4058 default:
4059 break;
4060 }
4061 }
4062
4063 /**
4064 * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping
4065 * @adapter: driver data
4066 */
4067 static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
4068 {
4069 struct ixgbe_hw *hw = &adapter->hw;
4070 u32 vlnctrl;
4071 int i, j;
4072
4073 switch (hw->mac.type) {
4074 case ixgbe_mac_82598EB:
4075 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4076 vlnctrl |= IXGBE_VLNCTRL_VME;
4077 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4078 break;
4079 case ixgbe_mac_82599EB:
4080 case ixgbe_mac_X540:
4081 case ixgbe_mac_X550:
4082 case ixgbe_mac_X550EM_x:
4083 case ixgbe_mac_x550em_a:
4084 for (i = 0; i < adapter->num_rx_queues; i++) {
4085 struct ixgbe_ring *ring = adapter->rx_ring[i];
4086
4087 if (ring->l2_accel_priv)
4088 continue;
4089 j = ring->reg_idx;
4090 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
4091 vlnctrl |= IXGBE_RXDCTL_VME;
4092 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
4093 }
4094 break;
4095 default:
4096 break;
4097 }
4098 }
4099
4100 static void ixgbe_vlan_promisc_enable(struct ixgbe_adapter *adapter)
4101 {
4102 struct ixgbe_hw *hw = &adapter->hw;
4103 u32 vlnctrl, i;
4104
4105 switch (hw->mac.type) {
4106 case ixgbe_mac_82599EB:
4107 case ixgbe_mac_X540:
4108 case ixgbe_mac_X550:
4109 case ixgbe_mac_X550EM_x:
4110 case ixgbe_mac_x550em_a:
4111 default:
4112 if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED)
4113 break;
4114 /* fall through */
4115 case ixgbe_mac_82598EB:
4116 /* legacy case, we can just disable VLAN filtering */
4117 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4118 vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN);
4119 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4120 return;
4121 }
4122
4123 /* We are already in VLAN promisc, nothing to do */
4124 if (adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC)
4125 return;
4126
4127 /* Set flag so we don't redo unnecessary work */
4128 adapter->flags2 |= IXGBE_FLAG2_VLAN_PROMISC;
4129
4130 /* Add PF to all active pools */
4131 for (i = IXGBE_VLVF_ENTRIES; --i;) {
4132 u32 reg_offset = IXGBE_VLVFB(i * 2 + VMDQ_P(0) / 32);
4133 u32 vlvfb = IXGBE_READ_REG(hw, reg_offset);
4134
4135 vlvfb |= 1 << (VMDQ_P(0) % 32);
4136 IXGBE_WRITE_REG(hw, reg_offset, vlvfb);
4137 }
4138
4139 /* Set all bits in the VLAN filter table array */
4140 for (i = hw->mac.vft_size; i--;)
4141 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), ~0U);
4142 }
4143
4144 #define VFTA_BLOCK_SIZE 8
4145 static void ixgbe_scrub_vfta(struct ixgbe_adapter *adapter, u32 vfta_offset)
4146 {
4147 struct ixgbe_hw *hw = &adapter->hw;
4148 u32 vfta[VFTA_BLOCK_SIZE] = { 0 };
4149 u32 vid_start = vfta_offset * 32;
4150 u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32);
4151 u32 i, vid, word, bits;
4152
4153 for (i = IXGBE_VLVF_ENTRIES; --i;) {
4154 u32 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(i));
4155
4156 /* pull VLAN ID from VLVF */
4157 vid = vlvf & VLAN_VID_MASK;
4158
4159 /* only concern outselves with a certain range */
4160 if (vid < vid_start || vid >= vid_end)
4161 continue;
4162
4163 if (vlvf) {
4164 /* record VLAN ID in VFTA */
4165 vfta[(vid - vid_start) / 32] |= 1 << (vid % 32);
4166
4167 /* if PF is part of this then continue */
4168 if (test_bit(vid, adapter->active_vlans))
4169 continue;
4170 }
4171
4172 /* remove PF from the pool */
4173 word = i * 2 + VMDQ_P(0) / 32;
4174 bits = ~(1 << (VMDQ_P(0) % 32));
4175 bits &= IXGBE_READ_REG(hw, IXGBE_VLVFB(word));
4176 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(word), bits);
4177 }
4178
4179 /* extract values from active_vlans and write back to VFTA */
4180 for (i = VFTA_BLOCK_SIZE; i--;) {
4181 vid = (vfta_offset + i) * 32;
4182 word = vid / BITS_PER_LONG;
4183 bits = vid % BITS_PER_LONG;
4184
4185 vfta[i] |= adapter->active_vlans[word] >> bits;
4186
4187 IXGBE_WRITE_REG(hw, IXGBE_VFTA(vfta_offset + i), vfta[i]);
4188 }
4189 }
4190
4191 static void ixgbe_vlan_promisc_disable(struct ixgbe_adapter *adapter)
4192 {
4193 struct ixgbe_hw *hw = &adapter->hw;
4194 u32 vlnctrl, i;
4195
4196 switch (hw->mac.type) {
4197 case ixgbe_mac_82599EB:
4198 case ixgbe_mac_X540:
4199 case ixgbe_mac_X550:
4200 case ixgbe_mac_X550EM_x:
4201 case ixgbe_mac_x550em_a:
4202 default:
4203 if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED)
4204 break;
4205 /* fall through */
4206 case ixgbe_mac_82598EB:
4207 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4208 vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
4209 vlnctrl |= IXGBE_VLNCTRL_VFE;
4210 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4211 return;
4212 }
4213
4214 /* We are not in VLAN promisc, nothing to do */
4215 if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4216 return;
4217
4218 /* Set flag so we don't redo unnecessary work */
4219 adapter->flags2 &= ~IXGBE_FLAG2_VLAN_PROMISC;
4220
4221 for (i = 0; i < hw->mac.vft_size; i += VFTA_BLOCK_SIZE)
4222 ixgbe_scrub_vfta(adapter, i);
4223 }
4224
4225 static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)
4226 {
4227 u16 vid = 1;
4228
4229 ixgbe_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0);
4230
4231 for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID)
4232 ixgbe_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
4233 }
4234
4235 /**
4236 * ixgbe_write_mc_addr_list - write multicast addresses to MTA
4237 * @netdev: network interface device structure
4238 *
4239 * Writes multicast address list to the MTA hash table.
4240 * Returns: -ENOMEM on failure
4241 * 0 on no addresses written
4242 * X on writing X addresses to MTA
4243 **/
4244 static int ixgbe_write_mc_addr_list(struct net_device *netdev)
4245 {
4246 struct ixgbe_adapter *adapter = netdev_priv(netdev);
4247 struct ixgbe_hw *hw = &adapter->hw;
4248
4249 if (!netif_running(netdev))
4250 return 0;
4251
4252 if (hw->mac.ops.update_mc_addr_list)
4253 hw->mac.ops.update_mc_addr_list(hw, netdev);
4254 else
4255 return -ENOMEM;
4256
4257 #ifdef CONFIG_PCI_IOV
4258 ixgbe_restore_vf_multicasts(adapter);
4259 #endif
4260
4261 return netdev_mc_count(netdev);
4262 }
4263
4264 #ifdef CONFIG_PCI_IOV
4265 void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter)
4266 {
4267 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
4268 struct ixgbe_hw *hw = &adapter->hw;
4269 int i;
4270
4271 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4272 mac_table->state &= ~IXGBE_MAC_STATE_MODIFIED;
4273
4274 if (mac_table->state & IXGBE_MAC_STATE_IN_USE)
4275 hw->mac.ops.set_rar(hw, i,
4276 mac_table->addr,
4277 mac_table->pool,
4278 IXGBE_RAH_AV);
4279 else
4280 hw->mac.ops.clear_rar(hw, i);
4281 }
4282 }
4283
4284 #endif
4285 static void ixgbe_sync_mac_table(struct ixgbe_adapter *adapter)
4286 {
4287 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
4288 struct ixgbe_hw *hw = &adapter->hw;
4289 int i;
4290
4291 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4292 if (!(mac_table->state & IXGBE_MAC_STATE_MODIFIED))
4293 continue;
4294
4295 mac_table->state &= ~IXGBE_MAC_STATE_MODIFIED;
4296
4297 if (mac_table->state & IXGBE_MAC_STATE_IN_USE)
4298 hw->mac.ops.set_rar(hw, i,
4299 mac_table->addr,
4300 mac_table->pool,
4301 IXGBE_RAH_AV);
4302 else
4303 hw->mac.ops.clear_rar(hw, i);
4304 }
4305 }
4306
4307 static void ixgbe_flush_sw_mac_table(struct ixgbe_adapter *adapter)
4308 {
4309 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
4310 struct ixgbe_hw *hw = &adapter->hw;
4311 int i;
4312
4313 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4314 mac_table->state |= IXGBE_MAC_STATE_MODIFIED;
4315 mac_table->state &= ~IXGBE_MAC_STATE_IN_USE;
4316 }
4317
4318 ixgbe_sync_mac_table(adapter);
4319 }
4320
4321 static int ixgbe_available_rars(struct ixgbe_adapter *adapter, u16 pool)
4322 {
4323 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
4324 struct ixgbe_hw *hw = &adapter->hw;
4325 int i, count = 0;
4326
4327 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4328 /* do not count default RAR as available */
4329 if (mac_table->state & IXGBE_MAC_STATE_DEFAULT)
4330 continue;
4331
4332 /* only count unused and addresses that belong to us */
4333 if (mac_table->state & IXGBE_MAC_STATE_IN_USE) {
4334 if (mac_table->pool != pool)
4335 continue;
4336 }
4337
4338 count++;
4339 }
4340
4341 return count;
4342 }
4343
4344 /* this function destroys the first RAR entry */
4345 static void ixgbe_mac_set_default_filter(struct ixgbe_adapter *adapter)
4346 {
4347 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
4348 struct ixgbe_hw *hw = &adapter->hw;
4349
4350 memcpy(&mac_table->addr, hw->mac.addr, ETH_ALEN);
4351 mac_table->pool = VMDQ_P(0);
4352
4353 mac_table->state = IXGBE_MAC_STATE_DEFAULT | IXGBE_MAC_STATE_IN_USE;
4354
4355 hw->mac.ops.set_rar(hw, 0, mac_table->addr, mac_table->pool,
4356 IXGBE_RAH_AV);
4357 }
4358
4359 int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter,
4360 const u8 *addr, u16 pool)
4361 {
4362 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
4363 struct ixgbe_hw *hw = &adapter->hw;
4364 int i;
4365
4366 if (is_zero_ether_addr(addr))
4367 return -EINVAL;
4368
4369 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4370 if (mac_table->state & IXGBE_MAC_STATE_IN_USE)
4371 continue;
4372
4373 ether_addr_copy(mac_table->addr, addr);
4374 mac_table->pool = pool;
4375
4376 mac_table->state |= IXGBE_MAC_STATE_MODIFIED |
4377 IXGBE_MAC_STATE_IN_USE;
4378
4379 ixgbe_sync_mac_table(adapter);
4380
4381 return i;
4382 }
4383
4384 return -ENOMEM;
4385 }
4386
4387 int ixgbe_del_mac_filter(struct ixgbe_adapter *adapter,
4388 const u8 *addr, u16 pool)
4389 {
4390 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
4391 struct ixgbe_hw *hw = &adapter->hw;
4392 int i;
4393
4394 if (is_zero_ether_addr(addr))
4395 return -EINVAL;
4396
4397 /* search table for addr, if found clear IN_USE flag and sync */
4398 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4399 /* we can only delete an entry if it is in use */
4400 if (!(mac_table->state & IXGBE_MAC_STATE_IN_USE))
4401 continue;
4402 /* we only care about entries that belong to the given pool */
4403 if (mac_table->pool != pool)
4404 continue;
4405 /* we only care about a specific MAC address */
4406 if (!ether_addr_equal(addr, mac_table->addr))
4407 continue;
4408
4409 mac_table->state |= IXGBE_MAC_STATE_MODIFIED;
4410 mac_table->state &= ~IXGBE_MAC_STATE_IN_USE;
4411
4412 ixgbe_sync_mac_table(adapter);
4413
4414 return 0;
4415 }
4416
4417 return -ENOMEM;
4418 }
4419 /**
4420 * ixgbe_write_uc_addr_list - write unicast addresses to RAR table
4421 * @netdev: network interface device structure
4422 *
4423 * Writes unicast address list to the RAR table.
4424 * Returns: -ENOMEM on failure/insufficient address space
4425 * 0 on no addresses written
4426 * X on writing X addresses to the RAR table
4427 **/
4428 static int ixgbe_write_uc_addr_list(struct net_device *netdev, int vfn)
4429 {
4430 struct ixgbe_adapter *adapter = netdev_priv(netdev);
4431 int count = 0;
4432
4433 /* return ENOMEM indicating insufficient memory for addresses */
4434 if (netdev_uc_count(netdev) > ixgbe_available_rars(adapter, vfn))
4435 return -ENOMEM;
4436
4437 if (!netdev_uc_empty(netdev)) {
4438 struct netdev_hw_addr *ha;
4439 netdev_for_each_uc_addr(ha, netdev) {
4440 ixgbe_del_mac_filter(adapter, ha->addr, vfn);
4441 ixgbe_add_mac_filter(adapter, ha->addr, vfn);
4442 count++;
4443 }
4444 }
4445 return count;
4446 }
4447
4448 static int ixgbe_uc_sync(struct net_device *netdev, const unsigned char *addr)
4449 {
4450 struct ixgbe_adapter *adapter = netdev_priv(netdev);
4451 int ret;
4452
4453 ret = ixgbe_add_mac_filter(adapter, addr, VMDQ_P(0));
4454
4455 return min_t(int, ret, 0);
4456 }
4457
4458 static int ixgbe_uc_unsync(struct net_device *netdev, const unsigned char *addr)
4459 {
4460 struct ixgbe_adapter *adapter = netdev_priv(netdev);
4461
4462 ixgbe_del_mac_filter(adapter, addr, VMDQ_P(0));
4463
4464 return 0;
4465 }
4466
4467 /**
4468 * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set
4469 * @netdev: network interface device structure
4470 *
4471 * The set_rx_method entry point is called whenever the unicast/multicast
4472 * address list or the network interface flags are updated. This routine is
4473 * responsible for configuring the hardware for proper unicast, multicast and
4474 * promiscuous mode.
4475 **/
4476 void ixgbe_set_rx_mode(struct net_device *netdev)
4477 {
4478 struct ixgbe_adapter *adapter = netdev_priv(netdev);
4479 struct ixgbe_hw *hw = &adapter->hw;
4480 u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;
4481 netdev_features_t features = netdev->features;
4482 int count;
4483
4484 /* Check for Promiscuous and All Multicast modes */
4485 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4486
4487 /* set all bits that we expect to always be set */
4488 fctrl &= ~IXGBE_FCTRL_SBP; /* disable store-bad-packets */
4489 fctrl |= IXGBE_FCTRL_BAM;
4490 fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */
4491 fctrl |= IXGBE_FCTRL_PMCF;
4492
4493 /* clear the bits we are changing the status of */
4494 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
4495 if (netdev->flags & IFF_PROMISC) {
4496 hw->addr_ctrl.user_set_promisc = true;
4497 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
4498 vmolr |= IXGBE_VMOLR_MPE;
4499 features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
4500 } else {
4501 if (netdev->flags & IFF_ALLMULTI) {
4502 fctrl |= IXGBE_FCTRL_MPE;
4503 vmolr |= IXGBE_VMOLR_MPE;
4504 }
4505 hw->addr_ctrl.user_set_promisc = false;
4506 }
4507
4508 /*
4509 * Write addresses to available RAR registers, if there is not
4510 * sufficient space to store all the addresses then enable
4511 * unicast promiscuous mode
4512 */
4513 if (__dev_uc_sync(netdev, ixgbe_uc_sync, ixgbe_uc_unsync)) {
4514 fctrl |= IXGBE_FCTRL_UPE;
4515 vmolr |= IXGBE_VMOLR_ROPE;
4516 }
4517
4518 /* Write addresses to the MTA, if the attempt fails
4519 * then we should just turn on promiscuous mode so
4520 * that we can at least receive multicast traffic
4521 */
4522 count = ixgbe_write_mc_addr_list(netdev);
4523 if (count < 0) {
4524 fctrl |= IXGBE_FCTRL_MPE;
4525 vmolr |= IXGBE_VMOLR_MPE;
4526 } else if (count) {
4527 vmolr |= IXGBE_VMOLR_ROMPE;
4528 }
4529
4530 if (hw->mac.type != ixgbe_mac_82598EB) {
4531 vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(VMDQ_P(0))) &
4532 ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE |
4533 IXGBE_VMOLR_ROPE);
4534 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(0)), vmolr);
4535 }
4536
4537 /* This is useful for sniffing bad packets. */
4538 if (features & NETIF_F_RXALL) {
4539 /* UPE and MPE will be handled by normal PROMISC logic
4540 * in e1000e_set_rx_mode */
4541 fctrl |= (IXGBE_FCTRL_SBP | /* Receive bad packets */
4542 IXGBE_FCTRL_BAM | /* RX All Bcast Pkts */
4543 IXGBE_FCTRL_PMCF); /* RX All MAC Ctrl Pkts */
4544
4545 fctrl &= ~(IXGBE_FCTRL_DPF);
4546 /* NOTE: VLAN filtering is disabled by setting PROMISC */
4547 }
4548
4549 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4550
4551 if (features & NETIF_F_HW_VLAN_CTAG_RX)
4552 ixgbe_vlan_strip_enable(adapter);
4553 else
4554 ixgbe_vlan_strip_disable(adapter);
4555
4556 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
4557 ixgbe_vlan_promisc_disable(adapter);
4558 else
4559 ixgbe_vlan_promisc_enable(adapter);
4560 }
4561
4562 static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)
4563 {
4564 int q_idx;
4565
4566 for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++) {
4567 ixgbe_qv_init_lock(adapter->q_vector[q_idx]);
4568 napi_enable(&adapter->q_vector[q_idx]->napi);
4569 }
4570 }
4571
4572 static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
4573 {
4574 int q_idx;
4575
4576 for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++) {
4577 napi_disable(&adapter->q_vector[q_idx]->napi);
4578 while (!ixgbe_qv_disable(adapter->q_vector[q_idx])) {
4579 pr_info("QV %d locked\n", q_idx);
4580 usleep_range(1000, 20000);
4581 }
4582 }
4583 }
4584
4585 static void ixgbe_clear_vxlan_port(struct ixgbe_adapter *adapter)
4586 {
4587 switch (adapter->hw.mac.type) {
4588 case ixgbe_mac_X550:
4589 case ixgbe_mac_X550EM_x:
4590 case ixgbe_mac_x550em_a:
4591 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VXLANCTRL, 0);
4592 adapter->vxlan_port = 0;
4593 break;
4594 default:
4595 break;
4596 }
4597 }
4598
4599 #ifdef CONFIG_IXGBE_DCB
4600 /**
4601 * ixgbe_configure_dcb - Configure DCB hardware
4602 * @adapter: ixgbe adapter struct
4603 *
4604 * This is called by the driver on open to configure the DCB hardware.
4605 * This is also called by the gennetlink interface when reconfiguring
4606 * the DCB state.
4607 */
4608 static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)
4609 {
4610 struct ixgbe_hw *hw = &adapter->hw;
4611 int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
4612
4613 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {
4614 if (hw->mac.type == ixgbe_mac_82598EB)
4615 netif_set_gso_max_size(adapter->netdev, 65536);
4616 return;
4617 }
4618
4619 if (hw->mac.type == ixgbe_mac_82598EB)
4620 netif_set_gso_max_size(adapter->netdev, 32768);
4621
4622 #ifdef IXGBE_FCOE
4623 if (adapter->netdev->features & NETIF_F_FCOE_MTU)
4624 max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);
4625 #endif
4626
4627 /* reconfigure the hardware */
4628 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) {
4629 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
4630 DCB_TX_CONFIG);
4631 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
4632 DCB_RX_CONFIG);
4633 ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);
4634 } else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) {
4635 ixgbe_dcb_hw_ets(&adapter->hw,
4636 adapter->ixgbe_ieee_ets,
4637 max_frame);
4638 ixgbe_dcb_hw_pfc_config(&adapter->hw,
4639 adapter->ixgbe_ieee_pfc->pfc_en,
4640 adapter->ixgbe_ieee_ets->prio_tc);
4641 }
4642
4643 /* Enable RSS Hash per TC */
4644 if (hw->mac.type != ixgbe_mac_82598EB) {
4645 u32 msb = 0;
4646 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices - 1;
4647
4648 while (rss_i) {
4649 msb++;
4650 rss_i >>= 1;
4651 }
4652
4653 /* write msb to all 8 TCs in one write */
4654 IXGBE_WRITE_REG(hw, IXGBE_RQTC, msb * 0x11111111);
4655 }
4656 }
4657 #endif
4658
4659 /* Additional bittime to account for IXGBE framing */
4660 #define IXGBE_ETH_FRAMING 20
4661
4662 /**
4663 * ixgbe_hpbthresh - calculate high water mark for flow control
4664 *
4665 * @adapter: board private structure to calculate for
4666 * @pb: packet buffer to calculate
4667 */
4668 static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb)
4669 {
4670 struct ixgbe_hw *hw = &adapter->hw;
4671 struct net_device *dev = adapter->netdev;
4672 int link, tc, kb, marker;
4673 u32 dv_id, rx_pba;
4674
4675 /* Calculate max LAN frame size */
4676 tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING;
4677
4678 #ifdef IXGBE_FCOE
4679 /* FCoE traffic class uses FCOE jumbo frames */
4680 if ((dev->features & NETIF_F_FCOE_MTU) &&
4681 (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) &&
4682 (pb == ixgbe_fcoe_get_tc(adapter)))
4683 tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;
4684 #endif
4685
4686 /* Calculate delay value for device */
4687 switch (hw->mac.type) {
4688 case ixgbe_mac_X540:
4689 case ixgbe_mac_X550:
4690 case ixgbe_mac_X550EM_x:
4691 case ixgbe_mac_x550em_a:
4692 dv_id = IXGBE_DV_X540(link, tc);
4693 break;
4694 default:
4695 dv_id = IXGBE_DV(link, tc);
4696 break;
4697 }
4698
4699 /* Loopback switch introduces additional latency */
4700 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
4701 dv_id += IXGBE_B2BT(tc);
4702
4703 /* Delay value is calculated in bit times convert to KB */
4704 kb = IXGBE_BT2KB(dv_id);
4705 rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10;
4706
4707 marker = rx_pba - kb;
4708
4709 /* It is possible that the packet buffer is not large enough
4710 * to provide required headroom. In this case throw an error
4711 * to user and a do the best we can.
4712 */
4713 if (marker < 0) {
4714 e_warn(drv, "Packet Buffer(%i) can not provide enough"
4715 "headroom to support flow control."
4716 "Decrease MTU or number of traffic classes\n", pb);
4717 marker = tc + 1;
4718 }
4719
4720 return marker;
4721 }
4722
4723 /**
4724 * ixgbe_lpbthresh - calculate low water mark for for flow control
4725 *
4726 * @adapter: board private structure to calculate for
4727 * @pb: packet buffer to calculate
4728 */
4729 static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb)
4730 {
4731 struct ixgbe_hw *hw = &adapter->hw;
4732 struct net_device *dev = adapter->netdev;
4733 int tc;
4734 u32 dv_id;
4735
4736 /* Calculate max LAN frame size */
4737 tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN;
4738
4739 #ifdef IXGBE_FCOE
4740 /* FCoE traffic class uses FCOE jumbo frames */
4741 if ((dev->features & NETIF_F_FCOE_MTU) &&
4742 (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) &&
4743 (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up)))
4744 tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;
4745 #endif
4746
4747 /* Calculate delay value for device */
4748 switch (hw->mac.type) {
4749 case ixgbe_mac_X540:
4750 case ixgbe_mac_X550:
4751 case ixgbe_mac_X550EM_x:
4752 case ixgbe_mac_x550em_a:
4753 dv_id = IXGBE_LOW_DV_X540(tc);
4754 break;
4755 default:
4756 dv_id = IXGBE_LOW_DV(tc);
4757 break;
4758 }
4759
4760 /* Delay value is calculated in bit times convert to KB */
4761 return IXGBE_BT2KB(dv_id);
4762 }
4763
4764 /*
4765 * ixgbe_pbthresh_setup - calculate and setup high low water marks
4766 */
4767 static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter)
4768 {
4769 struct ixgbe_hw *hw = &adapter->hw;
4770 int num_tc = netdev_get_num_tc(adapter->netdev);
4771 int i;
4772
4773 if (!num_tc)
4774 num_tc = 1;
4775
4776 for (i = 0; i < num_tc; i++) {
4777 hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i);
4778 hw->fc.low_water[i] = ixgbe_lpbthresh(adapter, i);
4779
4780 /* Low water marks must not be larger than high water marks */
4781 if (hw->fc.low_water[i] > hw->fc.high_water[i])
4782 hw->fc.low_water[i] = 0;
4783 }
4784
4785 for (; i < MAX_TRAFFIC_CLASS; i++)
4786 hw->fc.high_water[i] = 0;
4787 }
4788
4789 static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)
4790 {
4791 struct ixgbe_hw *hw = &adapter->hw;
4792 int hdrm;
4793 u8 tc = netdev_get_num_tc(adapter->netdev);
4794
4795 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
4796 adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
4797 hdrm = 32 << adapter->fdir_pballoc;
4798 else
4799 hdrm = 0;
4800
4801 hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
4802 ixgbe_pbthresh_setup(adapter);
4803 }
4804
4805 static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)
4806 {
4807 struct ixgbe_hw *hw = &adapter->hw;
4808 struct hlist_node *node2;
4809 struct ixgbe_fdir_filter *filter;
4810
4811 spin_lock(&adapter->fdir_perfect_lock);
4812
4813 if (!hlist_empty(&adapter->fdir_filter_list))
4814 ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask);
4815
4816 hlist_for_each_entry_safe(filter, node2,
4817 &adapter->fdir_filter_list, fdir_node) {
4818 ixgbe_fdir_write_perfect_filter_82599(hw,
4819 &filter->filter,
4820 filter->sw_idx,
4821 (filter->action == IXGBE_FDIR_DROP_QUEUE) ?
4822 IXGBE_FDIR_DROP_QUEUE :
4823 adapter->rx_ring[filter->action]->reg_idx);
4824 }
4825
4826 spin_unlock(&adapter->fdir_perfect_lock);
4827 }
4828
4829 static void ixgbe_macvlan_set_rx_mode(struct net_device *dev, unsigned int pool,
4830 struct ixgbe_adapter *adapter)
4831 {
4832 struct ixgbe_hw *hw = &adapter->hw;
4833 u32 vmolr;
4834
4835 /* No unicast promiscuous support for VMDQ devices. */
4836 vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(pool));
4837 vmolr |= (IXGBE_VMOLR_ROMPE | IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE);
4838
4839 /* clear the affected bit */
4840 vmolr &= ~IXGBE_VMOLR_MPE;
4841
4842 if (dev->flags & IFF_ALLMULTI) {
4843 vmolr |= IXGBE_VMOLR_MPE;
4844 } else {
4845 vmolr |= IXGBE_VMOLR_ROMPE;
4846 hw->mac.ops.update_mc_addr_list(hw, dev);
4847 }
4848 ixgbe_write_uc_addr_list(adapter->netdev, pool);
4849 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
4850 }
4851
4852 static void ixgbe_fwd_psrtype(struct ixgbe_fwd_adapter *vadapter)
4853 {
4854 struct ixgbe_adapter *adapter = vadapter->real_adapter;
4855 int rss_i = adapter->num_rx_queues_per_pool;
4856 struct ixgbe_hw *hw = &adapter->hw;
4857 u16 pool = vadapter->pool;
4858 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4859 IXGBE_PSRTYPE_UDPHDR |
4860 IXGBE_PSRTYPE_IPV4HDR |
4861 IXGBE_PSRTYPE_L2HDR |
4862 IXGBE_PSRTYPE_IPV6HDR;
4863
4864 if (hw->mac.type == ixgbe_mac_82598EB)
4865 return;
4866
4867 if (rss_i > 3)
4868 psrtype |= 2 << 29;
4869 else if (rss_i > 1)
4870 psrtype |= 1 << 29;
4871
4872 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(pool)), psrtype);
4873 }
4874
4875 /**
4876 * ixgbe_clean_rx_ring - Free Rx Buffers per Queue
4877 * @rx_ring: ring to free buffers from
4878 **/
4879 static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
4880 {
4881 struct device *dev = rx_ring->dev;
4882 unsigned long size;
4883 u16 i;
4884
4885 /* ring already cleared, nothing to do */
4886 if (!rx_ring->rx_buffer_info)
4887 return;
4888
4889 /* Free all the Rx ring sk_buffs */
4890 for (i = 0; i < rx_ring->count; i++) {
4891 struct ixgbe_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i];
4892
4893 if (rx_buffer->skb) {
4894 struct sk_buff *skb = rx_buffer->skb;
4895 if (IXGBE_CB(skb)->page_released)
4896 dma_unmap_page(dev,
4897 IXGBE_CB(skb)->dma,
4898 ixgbe_rx_bufsz(rx_ring),
4899 DMA_FROM_DEVICE);
4900 dev_kfree_skb(skb);
4901 rx_buffer->skb = NULL;
4902 }
4903
4904 if (!rx_buffer->page)
4905 continue;
4906
4907 dma_unmap_page(dev, rx_buffer->dma,
4908 ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE);
4909 __free_pages(rx_buffer->page, ixgbe_rx_pg_order(rx_ring));
4910
4911 rx_buffer->page = NULL;
4912 }
4913
4914 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
4915 memset(rx_ring->rx_buffer_info, 0, size);
4916
4917 /* Zero out the descriptor ring */
4918 memset(rx_ring->desc, 0, rx_ring->size);
4919
4920 rx_ring->next_to_alloc = 0;
4921 rx_ring->next_to_clean = 0;
4922 rx_ring->next_to_use = 0;
4923 }
4924
4925 static void ixgbe_disable_fwd_ring(struct ixgbe_fwd_adapter *vadapter,
4926 struct ixgbe_ring *rx_ring)
4927 {
4928 struct ixgbe_adapter *adapter = vadapter->real_adapter;
4929 int index = rx_ring->queue_index + vadapter->rx_base_queue;
4930
4931 /* shutdown specific queue receive and wait for dma to settle */
4932 ixgbe_disable_rx_queue(adapter, rx_ring);
4933 usleep_range(10000, 20000);
4934 ixgbe_irq_disable_queues(adapter, ((u64)1 << index));
4935 ixgbe_clean_rx_ring(rx_ring);
4936 rx_ring->l2_accel_priv = NULL;
4937 }
4938
4939 static int ixgbe_fwd_ring_down(struct net_device *vdev,
4940 struct ixgbe_fwd_adapter *accel)
4941 {
4942 struct ixgbe_adapter *adapter = accel->real_adapter;
4943 unsigned int rxbase = accel->rx_base_queue;
4944 unsigned int txbase = accel->tx_base_queue;
4945 int i;
4946
4947 netif_tx_stop_all_queues(vdev);
4948
4949 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
4950 ixgbe_disable_fwd_ring(accel, adapter->rx_ring[rxbase + i]);
4951 adapter->rx_ring[rxbase + i]->netdev = adapter->netdev;
4952 }
4953
4954 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
4955 adapter->tx_ring[txbase + i]->l2_accel_priv = NULL;
4956 adapter->tx_ring[txbase + i]->netdev = adapter->netdev;
4957 }
4958
4959
4960 return 0;
4961 }
4962
4963 static int ixgbe_fwd_ring_up(struct net_device *vdev,
4964 struct ixgbe_fwd_adapter *accel)
4965 {
4966 struct ixgbe_adapter *adapter = accel->real_adapter;
4967 unsigned int rxbase, txbase, queues;
4968 int i, baseq, err = 0;
4969
4970 if (!test_bit(accel->pool, &adapter->fwd_bitmask))
4971 return 0;
4972
4973 baseq = accel->pool * adapter->num_rx_queues_per_pool;
4974 netdev_dbg(vdev, "pool %i:%i queues %i:%i VSI bitmask %lx\n",
4975 accel->pool, adapter->num_rx_pools,
4976 baseq, baseq + adapter->num_rx_queues_per_pool,
4977 adapter->fwd_bitmask);
4978
4979 accel->netdev = vdev;
4980 accel->rx_base_queue = rxbase = baseq;
4981 accel->tx_base_queue = txbase = baseq;
4982
4983 for (i = 0; i < adapter->num_rx_queues_per_pool; i++)
4984 ixgbe_disable_fwd_ring(accel, adapter->rx_ring[rxbase + i]);
4985
4986 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
4987 adapter->rx_ring[rxbase + i]->netdev = vdev;
4988 adapter->rx_ring[rxbase + i]->l2_accel_priv = accel;
4989 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[rxbase + i]);
4990 }
4991
4992 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
4993 adapter->tx_ring[txbase + i]->netdev = vdev;
4994 adapter->tx_ring[txbase + i]->l2_accel_priv = accel;
4995 }
4996
4997 queues = min_t(unsigned int,
4998 adapter->num_rx_queues_per_pool, vdev->num_tx_queues);
4999 err = netif_set_real_num_tx_queues(vdev, queues);
5000 if (err)
5001 goto fwd_queue_err;
5002
5003 err = netif_set_real_num_rx_queues(vdev, queues);
5004 if (err)
5005 goto fwd_queue_err;
5006
5007 if (is_valid_ether_addr(vdev->dev_addr))
5008 ixgbe_add_mac_filter(adapter, vdev->dev_addr, accel->pool);
5009
5010 ixgbe_fwd_psrtype(accel);
5011 ixgbe_macvlan_set_rx_mode(vdev, accel->pool, adapter);
5012 return err;
5013 fwd_queue_err:
5014 ixgbe_fwd_ring_down(vdev, accel);
5015 return err;
5016 }
5017
5018 static void ixgbe_configure_dfwd(struct ixgbe_adapter *adapter)
5019 {
5020 struct net_device *upper;
5021 struct list_head *iter;
5022 int err;
5023
5024 netdev_for_each_all_upper_dev_rcu(adapter->netdev, upper, iter) {
5025 if (netif_is_macvlan(upper)) {
5026 struct macvlan_dev *dfwd = netdev_priv(upper);
5027 struct ixgbe_fwd_adapter *vadapter = dfwd->fwd_priv;
5028
5029 if (dfwd->fwd_priv) {
5030 err = ixgbe_fwd_ring_up(upper, vadapter);
5031 if (err)
5032 continue;
5033 }
5034 }
5035 }
5036 }
5037
5038 static void ixgbe_configure(struct ixgbe_adapter *adapter)
5039 {
5040 struct ixgbe_hw *hw = &adapter->hw;
5041
5042 ixgbe_configure_pb(adapter);
5043 #ifdef CONFIG_IXGBE_DCB
5044 ixgbe_configure_dcb(adapter);
5045 #endif
5046 /*
5047 * We must restore virtualization before VLANs or else
5048 * the VLVF registers will not be populated
5049 */
5050 ixgbe_configure_virtualization(adapter);
5051
5052 ixgbe_set_rx_mode(adapter->netdev);
5053 ixgbe_restore_vlan(adapter);
5054
5055 switch (hw->mac.type) {
5056 case ixgbe_mac_82599EB:
5057 case ixgbe_mac_X540:
5058 hw->mac.ops.disable_rx_buff(hw);
5059 break;
5060 default:
5061 break;
5062 }
5063
5064 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
5065 ixgbe_init_fdir_signature_82599(&adapter->hw,
5066 adapter->fdir_pballoc);
5067 } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {
5068 ixgbe_init_fdir_perfect_82599(&adapter->hw,
5069 adapter->fdir_pballoc);
5070 ixgbe_fdir_filter_restore(adapter);
5071 }
5072
5073 switch (hw->mac.type) {
5074 case ixgbe_mac_82599EB:
5075 case ixgbe_mac_X540:
5076 hw->mac.ops.enable_rx_buff(hw);
5077 break;
5078 default:
5079 break;
5080 }
5081
5082 #ifdef CONFIG_IXGBE_DCA
5083 /* configure DCA */
5084 if (adapter->flags & IXGBE_FLAG_DCA_CAPABLE)
5085 ixgbe_setup_dca(adapter);
5086 #endif /* CONFIG_IXGBE_DCA */
5087
5088 #ifdef IXGBE_FCOE
5089 /* configure FCoE L2 filters, redirection table, and Rx control */
5090 ixgbe_configure_fcoe(adapter);
5091
5092 #endif /* IXGBE_FCOE */
5093 ixgbe_configure_tx(adapter);
5094 ixgbe_configure_rx(adapter);
5095 ixgbe_configure_dfwd(adapter);
5096 }
5097
5098 /**
5099 * ixgbe_sfp_link_config - set up SFP+ link
5100 * @adapter: pointer to private adapter struct
5101 **/
5102 static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)
5103 {
5104 /*
5105 * We are assuming the worst case scenario here, and that
5106 * is that an SFP was inserted/removed after the reset
5107 * but before SFP detection was enabled. As such the best
5108 * solution is to just start searching as soon as we start
5109 */
5110 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
5111 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
5112
5113 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
5114 adapter->sfp_poll_time = 0;
5115 }
5116
5117 /**
5118 * ixgbe_non_sfp_link_config - set up non-SFP+ link
5119 * @hw: pointer to private hardware struct
5120 *
5121 * Returns 0 on success, negative on failure
5122 **/
5123 static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
5124 {
5125 u32 speed;
5126 bool autoneg, link_up = false;
5127 int ret = IXGBE_ERR_LINK_SETUP;
5128
5129 if (hw->mac.ops.check_link)
5130 ret = hw->mac.ops.check_link(hw, &speed, &link_up, false);
5131
5132 if (ret)
5133 return ret;
5134
5135 speed = hw->phy.autoneg_advertised;
5136 if ((!speed) && (hw->mac.ops.get_link_capabilities))
5137 ret = hw->mac.ops.get_link_capabilities(hw, &speed,
5138 &autoneg);
5139 if (ret)
5140 return ret;
5141
5142 if (hw->mac.ops.setup_link)
5143 ret = hw->mac.ops.setup_link(hw, speed, link_up);
5144
5145 return ret;
5146 }
5147
5148 static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)
5149 {
5150 struct ixgbe_hw *hw = &adapter->hw;
5151 u32 gpie = 0;
5152
5153 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
5154 gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
5155 IXGBE_GPIE_OCD;
5156 gpie |= IXGBE_GPIE_EIAME;
5157 /*
5158 * use EIAM to auto-mask when MSI-X interrupt is asserted
5159 * this saves a register write for every interrupt
5160 */
5161 switch (hw->mac.type) {
5162 case ixgbe_mac_82598EB:
5163 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
5164 break;
5165 case ixgbe_mac_82599EB:
5166 case ixgbe_mac_X540:
5167 case ixgbe_mac_X550:
5168 case ixgbe_mac_X550EM_x:
5169 case ixgbe_mac_x550em_a:
5170 default:
5171 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
5172 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
5173 break;
5174 }
5175 } else {
5176 /* legacy interrupts, use EIAM to auto-mask when reading EICR,
5177 * specifically only auto mask tx and rx interrupts */
5178 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
5179 }
5180
5181 /* XXX: to interrupt immediately for EICS writes, enable this */
5182 /* gpie |= IXGBE_GPIE_EIMEN; */
5183
5184 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
5185 gpie &= ~IXGBE_GPIE_VTMODE_MASK;
5186
5187 switch (adapter->ring_feature[RING_F_VMDQ].mask) {
5188 case IXGBE_82599_VMDQ_8Q_MASK:
5189 gpie |= IXGBE_GPIE_VTMODE_16;
5190 break;
5191 case IXGBE_82599_VMDQ_4Q_MASK:
5192 gpie |= IXGBE_GPIE_VTMODE_32;
5193 break;
5194 default:
5195 gpie |= IXGBE_GPIE_VTMODE_64;
5196 break;
5197 }
5198 }
5199
5200 /* Enable Thermal over heat sensor interrupt */
5201 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) {
5202 switch (adapter->hw.mac.type) {
5203 case ixgbe_mac_82599EB:
5204 gpie |= IXGBE_SDP0_GPIEN_8259X;
5205 break;
5206 default:
5207 break;
5208 }
5209 }
5210
5211 /* Enable fan failure interrupt */
5212 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
5213 gpie |= IXGBE_SDP1_GPIEN(hw);
5214
5215 switch (hw->mac.type) {
5216 case ixgbe_mac_82599EB:
5217 gpie |= IXGBE_SDP1_GPIEN_8259X | IXGBE_SDP2_GPIEN_8259X;
5218 break;
5219 case ixgbe_mac_X550EM_x:
5220 case ixgbe_mac_x550em_a:
5221 gpie |= IXGBE_SDP0_GPIEN_X540;
5222 break;
5223 default:
5224 break;
5225 }
5226
5227 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
5228 }
5229
5230 static void ixgbe_up_complete(struct ixgbe_adapter *adapter)
5231 {
5232 struct ixgbe_hw *hw = &adapter->hw;
5233 int err;
5234 u32 ctrl_ext;
5235
5236 ixgbe_get_hw_control(adapter);
5237 ixgbe_setup_gpie(adapter);
5238
5239 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
5240 ixgbe_configure_msix(adapter);
5241 else
5242 ixgbe_configure_msi_and_legacy(adapter);
5243
5244 /* enable the optics for 82599 SFP+ fiber */
5245 if (hw->mac.ops.enable_tx_laser)
5246 hw->mac.ops.enable_tx_laser(hw);
5247
5248 if (hw->phy.ops.set_phy_power)
5249 hw->phy.ops.set_phy_power(hw, true);
5250
5251 smp_mb__before_atomic();
5252 clear_bit(__IXGBE_DOWN, &adapter->state);
5253 ixgbe_napi_enable_all(adapter);
5254
5255 if (ixgbe_is_sfp(hw)) {
5256 ixgbe_sfp_link_config(adapter);
5257 } else {
5258 err = ixgbe_non_sfp_link_config(hw);
5259 if (err)
5260 e_err(probe, "link_config FAILED %d\n", err);
5261 }
5262
5263 /* clear any pending interrupts, may auto mask */
5264 IXGBE_READ_REG(hw, IXGBE_EICR);
5265 ixgbe_irq_enable(adapter, true, true);
5266
5267 /*
5268 * If this adapter has a fan, check to see if we had a failure
5269 * before we enabled the interrupt.
5270 */
5271 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
5272 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
5273 if (esdp & IXGBE_ESDP_SDP1)
5274 e_crit(drv, "Fan has stopped, replace the adapter\n");
5275 }
5276
5277 /* bring the link up in the watchdog, this could race with our first
5278 * link up interrupt but shouldn't be a problem */
5279 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
5280 adapter->link_check_timeout = jiffies;
5281 mod_timer(&adapter->service_timer, jiffies);
5282
5283 /* Set PF Reset Done bit so PF/VF Mail Ops can work */
5284 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5285 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
5286 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5287 }
5288
5289 void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
5290 {
5291 WARN_ON(in_interrupt());
5292 /* put off any impending NetWatchDogTimeout */
5293 adapter->netdev->trans_start = jiffies;
5294
5295 while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state))
5296 usleep_range(1000, 2000);
5297 ixgbe_down(adapter);
5298 /*
5299 * If SR-IOV enabled then wait a bit before bringing the adapter
5300 * back up to give the VFs time to respond to the reset. The
5301 * two second wait is based upon the watchdog timer cycle in
5302 * the VF driver.
5303 */
5304 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
5305 msleep(2000);
5306 ixgbe_up(adapter);
5307 clear_bit(__IXGBE_RESETTING, &adapter->state);
5308 }
5309
5310 void ixgbe_up(struct ixgbe_adapter *adapter)
5311 {
5312 /* hardware has been reset, we need to reload some things */
5313 ixgbe_configure(adapter);
5314
5315 ixgbe_up_complete(adapter);
5316 }
5317
5318 void ixgbe_reset(struct ixgbe_adapter *adapter)
5319 {
5320 struct ixgbe_hw *hw = &adapter->hw;
5321 struct net_device *netdev = adapter->netdev;
5322 int err;
5323
5324 if (ixgbe_removed(hw->hw_addr))
5325 return;
5326 /* lock SFP init bit to prevent race conditions with the watchdog */
5327 while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
5328 usleep_range(1000, 2000);
5329
5330 /* clear all SFP and link config related flags while holding SFP_INIT */
5331 adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP |
5332 IXGBE_FLAG2_SFP_NEEDS_RESET);
5333 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
5334
5335 err = hw->mac.ops.init_hw(hw);
5336 switch (err) {
5337 case 0:
5338 case IXGBE_ERR_SFP_NOT_PRESENT:
5339 case IXGBE_ERR_SFP_NOT_SUPPORTED:
5340 break;
5341 case IXGBE_ERR_MASTER_REQUESTS_PENDING:
5342 e_dev_err("master disable timed out\n");
5343 break;
5344 case IXGBE_ERR_EEPROM_VERSION:
5345 /* We are running on a pre-production device, log a warning */
5346 e_dev_warn("This device is a pre-production adapter/LOM. "
5347 "Please be aware there may be issues associated with "
5348 "your hardware. If you are experiencing problems "
5349 "please contact your Intel or hardware "
5350 "representative who provided you with this "
5351 "hardware.\n");
5352 break;
5353 default:
5354 e_dev_err("Hardware Error: %d\n", err);
5355 }
5356
5357 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
5358
5359 /* flush entries out of MAC table */
5360 ixgbe_flush_sw_mac_table(adapter);
5361 __dev_uc_unsync(netdev, NULL);
5362
5363 /* do not flush user set addresses */
5364 ixgbe_mac_set_default_filter(adapter);
5365
5366 /* update SAN MAC vmdq pool selection */
5367 if (hw->mac.san_mac_rar_index)
5368 hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));
5369
5370 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
5371 ixgbe_ptp_reset(adapter);
5372
5373 if (hw->phy.ops.set_phy_power) {
5374 if (!netif_running(adapter->netdev) && !adapter->wol)
5375 hw->phy.ops.set_phy_power(hw, false);
5376 else
5377 hw->phy.ops.set_phy_power(hw, true);
5378 }
5379 }
5380
5381 /**
5382 * ixgbe_clean_tx_ring - Free Tx Buffers
5383 * @tx_ring: ring to be cleaned
5384 **/
5385 static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
5386 {
5387 struct ixgbe_tx_buffer *tx_buffer_info;
5388 unsigned long size;
5389 u16 i;
5390
5391 /* ring already cleared, nothing to do */
5392 if (!tx_ring->tx_buffer_info)
5393 return;
5394
5395 /* Free all the Tx ring sk_buffs */
5396 for (i = 0; i < tx_ring->count; i++) {
5397 tx_buffer_info = &tx_ring->tx_buffer_info[i];
5398 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
5399 }
5400
5401 netdev_tx_reset_queue(txring_txq(tx_ring));
5402
5403 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
5404 memset(tx_ring->tx_buffer_info, 0, size);
5405
5406 /* Zero out the descriptor ring */
5407 memset(tx_ring->desc, 0, tx_ring->size);
5408
5409 tx_ring->next_to_use = 0;
5410 tx_ring->next_to_clean = 0;
5411 }
5412
5413 /**
5414 * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues
5415 * @adapter: board private structure
5416 **/
5417 static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)
5418 {
5419 int i;
5420
5421 for (i = 0; i < adapter->num_rx_queues; i++)
5422 ixgbe_clean_rx_ring(adapter->rx_ring[i]);
5423 }
5424
5425 /**
5426 * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues
5427 * @adapter: board private structure
5428 **/
5429 static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)
5430 {
5431 int i;
5432
5433 for (i = 0; i < adapter->num_tx_queues; i++)
5434 ixgbe_clean_tx_ring(adapter->tx_ring[i]);
5435 }
5436
5437 static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)
5438 {
5439 struct hlist_node *node2;
5440 struct ixgbe_fdir_filter *filter;
5441
5442 spin_lock(&adapter->fdir_perfect_lock);
5443
5444 hlist_for_each_entry_safe(filter, node2,
5445 &adapter->fdir_filter_list, fdir_node) {
5446 hlist_del(&filter->fdir_node);
5447 kfree(filter);
5448 }
5449 adapter->fdir_filter_count = 0;
5450
5451 spin_unlock(&adapter->fdir_perfect_lock);
5452 }
5453
5454 void ixgbe_down(struct ixgbe_adapter *adapter)
5455 {
5456 struct net_device *netdev = adapter->netdev;
5457 struct ixgbe_hw *hw = &adapter->hw;
5458 struct net_device *upper;
5459 struct list_head *iter;
5460 int i;
5461
5462 /* signal that we are down to the interrupt handler */
5463 if (test_and_set_bit(__IXGBE_DOWN, &adapter->state))
5464 return; /* do nothing if already down */
5465
5466 /* disable receives */
5467 hw->mac.ops.disable_rx(hw);
5468
5469 /* disable all enabled rx queues */
5470 for (i = 0; i < adapter->num_rx_queues; i++)
5471 /* this call also flushes the previous write */
5472 ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]);
5473
5474 usleep_range(10000, 20000);
5475
5476 netif_tx_stop_all_queues(netdev);
5477
5478 /* call carrier off first to avoid false dev_watchdog timeouts */
5479 netif_carrier_off(netdev);
5480 netif_tx_disable(netdev);
5481
5482 /* disable any upper devices */
5483 netdev_for_each_all_upper_dev_rcu(adapter->netdev, upper, iter) {
5484 if (netif_is_macvlan(upper)) {
5485 struct macvlan_dev *vlan = netdev_priv(upper);
5486
5487 if (vlan->fwd_priv) {
5488 netif_tx_stop_all_queues(upper);
5489 netif_carrier_off(upper);
5490 netif_tx_disable(upper);
5491 }
5492 }
5493 }
5494
5495 ixgbe_irq_disable(adapter);
5496
5497 ixgbe_napi_disable_all(adapter);
5498
5499 adapter->flags2 &= ~(IXGBE_FLAG2_FDIR_REQUIRES_REINIT |
5500 IXGBE_FLAG2_RESET_REQUESTED);
5501 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
5502
5503 del_timer_sync(&adapter->service_timer);
5504
5505 if (adapter->num_vfs) {
5506 /* Clear EITR Select mapping */
5507 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);
5508
5509 /* Mark all the VFs as inactive */
5510 for (i = 0 ; i < adapter->num_vfs; i++)
5511 adapter->vfinfo[i].clear_to_send = false;
5512
5513 /* ping all the active vfs to let them know we are going down */
5514 ixgbe_ping_all_vfs(adapter);
5515
5516 /* Disable all VFTE/VFRE TX/RX */
5517 ixgbe_disable_tx_rx(adapter);
5518 }
5519
5520 /* disable transmits in the hardware now that interrupts are off */
5521 for (i = 0; i < adapter->num_tx_queues; i++) {
5522 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
5523 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
5524 }
5525
5526 /* Disable the Tx DMA engine on 82599 and later MAC */
5527 switch (hw->mac.type) {
5528 case ixgbe_mac_82599EB:
5529 case ixgbe_mac_X540:
5530 case ixgbe_mac_X550:
5531 case ixgbe_mac_X550EM_x:
5532 case ixgbe_mac_x550em_a:
5533 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL,
5534 (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) &
5535 ~IXGBE_DMATXCTL_TE));
5536 break;
5537 default:
5538 break;
5539 }
5540
5541 if (!pci_channel_offline(adapter->pdev))
5542 ixgbe_reset(adapter);
5543
5544 /* power down the optics for 82599 SFP+ fiber */
5545 if (hw->mac.ops.disable_tx_laser)
5546 hw->mac.ops.disable_tx_laser(hw);
5547
5548 ixgbe_clean_all_tx_rings(adapter);
5549 ixgbe_clean_all_rx_rings(adapter);
5550 }
5551
5552 /**
5553 * ixgbe_tx_timeout - Respond to a Tx Hang
5554 * @netdev: network interface device structure
5555 **/
5556 static void ixgbe_tx_timeout(struct net_device *netdev)
5557 {
5558 struct ixgbe_adapter *adapter = netdev_priv(netdev);
5559
5560 /* Do the reset outside of interrupt context */
5561 ixgbe_tx_timeout_reset(adapter);
5562 }
5563
5564 /**
5565 * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)
5566 * @adapter: board private structure to initialize
5567 *
5568 * ixgbe_sw_init initializes the Adapter private data structure.
5569 * Fields are initialized based on PCI device information and
5570 * OS network device settings (MTU size).
5571 **/
5572 static int ixgbe_sw_init(struct ixgbe_adapter *adapter)
5573 {
5574 struct ixgbe_hw *hw = &adapter->hw;
5575 struct pci_dev *pdev = adapter->pdev;
5576 unsigned int rss, fdir;
5577 u32 fwsm;
5578 #ifdef CONFIG_IXGBE_DCB
5579 int j;
5580 struct tc_configuration *tc;
5581 #endif
5582
5583 /* PCI config space info */
5584
5585 hw->vendor_id = pdev->vendor;
5586 hw->device_id = pdev->device;
5587 hw->revision_id = pdev->revision;
5588 hw->subsystem_vendor_id = pdev->subsystem_vendor;
5589 hw->subsystem_device_id = pdev->subsystem_device;
5590
5591 /* Set common capability flags and settings */
5592 rss = min_t(int, ixgbe_max_rss_indices(adapter), num_online_cpus());
5593 adapter->ring_feature[RING_F_RSS].limit = rss;
5594 adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;
5595 adapter->max_q_vectors = MAX_Q_VECTORS_82599;
5596 adapter->atr_sample_rate = 20;
5597 fdir = min_t(int, IXGBE_MAX_FDIR_INDICES, num_online_cpus());
5598 adapter->ring_feature[RING_F_FDIR].limit = fdir;
5599 adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K;
5600 #ifdef CONFIG_IXGBE_DCA
5601 adapter->flags |= IXGBE_FLAG_DCA_CAPABLE;
5602 #endif
5603 #ifdef IXGBE_FCOE
5604 adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;
5605 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
5606 #ifdef CONFIG_IXGBE_DCB
5607 /* Default traffic class to use for FCoE */
5608 adapter->fcoe.up = IXGBE_FCOE_DEFTC;
5609 #endif /* CONFIG_IXGBE_DCB */
5610 #endif /* IXGBE_FCOE */
5611
5612 /* initialize static ixgbe jump table entries */
5613 adapter->jump_tables[0] = ixgbe_ipv4_fields;
5614
5615 adapter->mac_table = kzalloc(sizeof(struct ixgbe_mac_addr) *
5616 hw->mac.num_rar_entries,
5617 GFP_ATOMIC);
5618 if (!adapter->mac_table)
5619 return -ENOMEM;
5620
5621 /* Set MAC specific capability flags and exceptions */
5622 switch (hw->mac.type) {
5623 case ixgbe_mac_82598EB:
5624 adapter->flags2 &= ~IXGBE_FLAG2_RSC_CAPABLE;
5625
5626 if (hw->device_id == IXGBE_DEV_ID_82598AT)
5627 adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
5628
5629 adapter->max_q_vectors = MAX_Q_VECTORS_82598;
5630 adapter->ring_feature[RING_F_FDIR].limit = 0;
5631 adapter->atr_sample_rate = 0;
5632 adapter->fdir_pballoc = 0;
5633 #ifdef IXGBE_FCOE
5634 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
5635 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
5636 #ifdef CONFIG_IXGBE_DCB
5637 adapter->fcoe.up = 0;
5638 #endif /* IXGBE_DCB */
5639 #endif /* IXGBE_FCOE */
5640 break;
5641 case ixgbe_mac_82599EB:
5642 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM)
5643 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
5644 break;
5645 case ixgbe_mac_X540:
5646 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM(hw));
5647 if (fwsm & IXGBE_FWSM_TS_ENABLED)
5648 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
5649 break;
5650 case ixgbe_mac_X550EM_x:
5651 case ixgbe_mac_x550em_a:
5652 case ixgbe_mac_X550:
5653 #ifdef CONFIG_IXGBE_DCA
5654 adapter->flags &= ~IXGBE_FLAG_DCA_CAPABLE;
5655 #endif
5656 #ifdef CONFIG_IXGBE_VXLAN
5657 adapter->flags |= IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE;
5658 #endif
5659 break;
5660 default:
5661 break;
5662 }
5663
5664 #ifdef IXGBE_FCOE
5665 /* FCoE support exists, always init the FCoE lock */
5666 spin_lock_init(&adapter->fcoe.lock);
5667
5668 #endif
5669 /* n-tuple support exists, always init our spinlock */
5670 spin_lock_init(&adapter->fdir_perfect_lock);
5671
5672 #ifdef CONFIG_IXGBE_DCB
5673 switch (hw->mac.type) {
5674 case ixgbe_mac_X540:
5675 case ixgbe_mac_X550:
5676 case ixgbe_mac_X550EM_x:
5677 case ixgbe_mac_x550em_a:
5678 adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS;
5679 adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS;
5680 break;
5681 default:
5682 adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS;
5683 adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS;
5684 break;
5685 }
5686
5687 /* Configure DCB traffic classes */
5688 for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {
5689 tc = &adapter->dcb_cfg.tc_config[j];
5690 tc->path[DCB_TX_CONFIG].bwg_id = 0;
5691 tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);
5692 tc->path[DCB_RX_CONFIG].bwg_id = 0;
5693 tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);
5694 tc->dcb_pfc = pfc_disabled;
5695 }
5696
5697 /* Initialize default user to priority mapping, UPx->TC0 */
5698 tc = &adapter->dcb_cfg.tc_config[0];
5699 tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
5700 tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
5701
5702 adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;
5703 adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;
5704 adapter->dcb_cfg.pfc_mode_enable = false;
5705 adapter->dcb_set_bitmap = 0x00;
5706 adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE;
5707 memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg,
5708 sizeof(adapter->temp_dcb_cfg));
5709
5710 #endif
5711
5712 /* default flow control settings */
5713 hw->fc.requested_mode = ixgbe_fc_full;
5714 hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */
5715 ixgbe_pbthresh_setup(adapter);
5716 hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;
5717 hw->fc.send_xon = true;
5718 hw->fc.disable_fc_autoneg = ixgbe_device_supports_autoneg_fc(hw);
5719
5720 #ifdef CONFIG_PCI_IOV
5721 if (max_vfs > 0)
5722 e_dev_warn("Enabling SR-IOV VFs using the max_vfs module parameter is deprecated - please use the pci sysfs interface instead.\n");
5723
5724 /* assign number of SR-IOV VFs */
5725 if (hw->mac.type != ixgbe_mac_82598EB) {
5726 if (max_vfs > IXGBE_MAX_VFS_DRV_LIMIT) {
5727 adapter->num_vfs = 0;
5728 e_dev_warn("max_vfs parameter out of range. Not assigning any SR-IOV VFs\n");
5729 } else {
5730 adapter->num_vfs = max_vfs;
5731 }
5732 }
5733 #endif /* CONFIG_PCI_IOV */
5734
5735 /* enable itr by default in dynamic mode */
5736 adapter->rx_itr_setting = 1;
5737 adapter->tx_itr_setting = 1;
5738
5739 /* set default ring sizes */
5740 adapter->tx_ring_count = IXGBE_DEFAULT_TXD;
5741 adapter->rx_ring_count = IXGBE_DEFAULT_RXD;
5742
5743 /* set default work limits */
5744 adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK;
5745
5746 /* initialize eeprom parameters */
5747 if (ixgbe_init_eeprom_params_generic(hw)) {
5748 e_dev_err("EEPROM initialization failed\n");
5749 return -EIO;
5750 }
5751
5752 /* PF holds first pool slot */
5753 set_bit(0, &adapter->fwd_bitmask);
5754 set_bit(__IXGBE_DOWN, &adapter->state);
5755
5756 return 0;
5757 }
5758
5759 /**
5760 * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors)
5761 * @tx_ring: tx descriptor ring (for a specific queue) to setup
5762 *
5763 * Return 0 on success, negative on failure
5764 **/
5765 int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)
5766 {
5767 struct device *dev = tx_ring->dev;
5768 int orig_node = dev_to_node(dev);
5769 int ring_node = -1;
5770 int size;
5771
5772 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
5773
5774 if (tx_ring->q_vector)
5775 ring_node = tx_ring->q_vector->numa_node;
5776
5777 tx_ring->tx_buffer_info = vzalloc_node(size, ring_node);
5778 if (!tx_ring->tx_buffer_info)
5779 tx_ring->tx_buffer_info = vzalloc(size);
5780 if (!tx_ring->tx_buffer_info)
5781 goto err;
5782
5783 u64_stats_init(&tx_ring->syncp);
5784
5785 /* round up to nearest 4K */
5786 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
5787 tx_ring->size = ALIGN(tx_ring->size, 4096);
5788
5789 set_dev_node(dev, ring_node);
5790 tx_ring->desc = dma_alloc_coherent(dev,
5791 tx_ring->size,
5792 &tx_ring->dma,
5793 GFP_KERNEL);
5794 set_dev_node(dev, orig_node);
5795 if (!tx_ring->desc)
5796 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
5797 &tx_ring->dma, GFP_KERNEL);
5798 if (!tx_ring->desc)
5799 goto err;
5800
5801 tx_ring->next_to_use = 0;
5802 tx_ring->next_to_clean = 0;
5803 return 0;
5804
5805 err:
5806 vfree(tx_ring->tx_buffer_info);
5807 tx_ring->tx_buffer_info = NULL;
5808 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
5809 return -ENOMEM;
5810 }
5811
5812 /**
5813 * ixgbe_setup_all_tx_resources - allocate all queues Tx resources
5814 * @adapter: board private structure
5815 *
5816 * If this function returns with an error, then it's possible one or
5817 * more of the rings is populated (while the rest are not). It is the
5818 * callers duty to clean those orphaned rings.
5819 *
5820 * Return 0 on success, negative on failure
5821 **/
5822 static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
5823 {
5824 int i, err = 0;
5825
5826 for (i = 0; i < adapter->num_tx_queues; i++) {
5827 err = ixgbe_setup_tx_resources(adapter->tx_ring[i]);
5828 if (!err)
5829 continue;
5830
5831 e_err(probe, "Allocation for Tx Queue %u failed\n", i);
5832 goto err_setup_tx;
5833 }
5834
5835 return 0;
5836 err_setup_tx:
5837 /* rewind the index freeing the rings as we go */
5838 while (i--)
5839 ixgbe_free_tx_resources(adapter->tx_ring[i]);
5840 return err;
5841 }
5842
5843 /**
5844 * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)
5845 * @rx_ring: rx descriptor ring (for a specific queue) to setup
5846 *
5847 * Returns 0 on success, negative on failure
5848 **/
5849 int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring)
5850 {
5851 struct device *dev = rx_ring->dev;
5852 int orig_node = dev_to_node(dev);
5853 int ring_node = -1;
5854 int size;
5855
5856 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
5857
5858 if (rx_ring->q_vector)
5859 ring_node = rx_ring->q_vector->numa_node;
5860
5861 rx_ring->rx_buffer_info = vzalloc_node(size, ring_node);
5862 if (!rx_ring->rx_buffer_info)
5863 rx_ring->rx_buffer_info = vzalloc(size);
5864 if (!rx_ring->rx_buffer_info)
5865 goto err;
5866
5867 u64_stats_init(&rx_ring->syncp);
5868
5869 /* Round up to nearest 4K */
5870 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
5871 rx_ring->size = ALIGN(rx_ring->size, 4096);
5872
5873 set_dev_node(dev, ring_node);
5874 rx_ring->desc = dma_alloc_coherent(dev,
5875 rx_ring->size,
5876 &rx_ring->dma,
5877 GFP_KERNEL);
5878 set_dev_node(dev, orig_node);
5879 if (!rx_ring->desc)
5880 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
5881 &rx_ring->dma, GFP_KERNEL);
5882 if (!rx_ring->desc)
5883 goto err;
5884
5885 rx_ring->next_to_clean = 0;
5886 rx_ring->next_to_use = 0;
5887
5888 return 0;
5889 err:
5890 vfree(rx_ring->rx_buffer_info);
5891 rx_ring->rx_buffer_info = NULL;
5892 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
5893 return -ENOMEM;
5894 }
5895
5896 /**
5897 * ixgbe_setup_all_rx_resources - allocate all queues Rx resources
5898 * @adapter: board private structure
5899 *
5900 * If this function returns with an error, then it's possible one or
5901 * more of the rings is populated (while the rest are not). It is the
5902 * callers duty to clean those orphaned rings.
5903 *
5904 * Return 0 on success, negative on failure
5905 **/
5906 static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)
5907 {
5908 int i, err = 0;
5909
5910 for (i = 0; i < adapter->num_rx_queues; i++) {
5911 err = ixgbe_setup_rx_resources(adapter->rx_ring[i]);
5912 if (!err)
5913 continue;
5914
5915 e_err(probe, "Allocation for Rx Queue %u failed\n", i);
5916 goto err_setup_rx;
5917 }
5918
5919 #ifdef IXGBE_FCOE
5920 err = ixgbe_setup_fcoe_ddp_resources(adapter);
5921 if (!err)
5922 #endif
5923 return 0;
5924 err_setup_rx:
5925 /* rewind the index freeing the rings as we go */
5926 while (i--)
5927 ixgbe_free_rx_resources(adapter->rx_ring[i]);
5928 return err;
5929 }
5930
5931 /**
5932 * ixgbe_free_tx_resources - Free Tx Resources per Queue
5933 * @tx_ring: Tx descriptor ring for a specific queue
5934 *
5935 * Free all transmit software resources
5936 **/
5937 void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring)
5938 {
5939 ixgbe_clean_tx_ring(tx_ring);
5940
5941 vfree(tx_ring->tx_buffer_info);
5942 tx_ring->tx_buffer_info = NULL;
5943
5944 /* if not set, then don't free */
5945 if (!tx_ring->desc)
5946 return;
5947
5948 dma_free_coherent(tx_ring->dev, tx_ring->size,
5949 tx_ring->desc, tx_ring->dma);
5950
5951 tx_ring->desc = NULL;
5952 }
5953
5954 /**
5955 * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues
5956 * @adapter: board private structure
5957 *
5958 * Free all transmit software resources
5959 **/
5960 static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)
5961 {
5962 int i;
5963
5964 for (i = 0; i < adapter->num_tx_queues; i++)
5965 if (adapter->tx_ring[i]->desc)
5966 ixgbe_free_tx_resources(adapter->tx_ring[i]);
5967 }
5968
5969 /**
5970 * ixgbe_free_rx_resources - Free Rx Resources
5971 * @rx_ring: ring to clean the resources from
5972 *
5973 * Free all receive software resources
5974 **/
5975 void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)
5976 {
5977 ixgbe_clean_rx_ring(rx_ring);
5978
5979 vfree(rx_ring->rx_buffer_info);
5980 rx_ring->rx_buffer_info = NULL;
5981
5982 /* if not set, then don't free */
5983 if (!rx_ring->desc)
5984 return;
5985
5986 dma_free_coherent(rx_ring->dev, rx_ring->size,
5987 rx_ring->desc, rx_ring->dma);
5988
5989 rx_ring->desc = NULL;
5990 }
5991
5992 /**
5993 * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues
5994 * @adapter: board private structure
5995 *
5996 * Free all receive software resources
5997 **/
5998 static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)
5999 {
6000 int i;
6001
6002 #ifdef IXGBE_FCOE
6003 ixgbe_free_fcoe_ddp_resources(adapter);
6004
6005 #endif
6006 for (i = 0; i < adapter->num_rx_queues; i++)
6007 if (adapter->rx_ring[i]->desc)
6008 ixgbe_free_rx_resources(adapter->rx_ring[i]);
6009 }
6010
6011 /**
6012 * ixgbe_change_mtu - Change the Maximum Transfer Unit
6013 * @netdev: network interface device structure
6014 * @new_mtu: new value for maximum frame size
6015 *
6016 * Returns 0 on success, negative on failure
6017 **/
6018 static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
6019 {
6020 struct ixgbe_adapter *adapter = netdev_priv(netdev);
6021 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
6022
6023 /* MTU < 68 is an error and causes problems on some kernels */
6024 if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE))
6025 return -EINVAL;
6026
6027 /*
6028 * For 82599EB we cannot allow legacy VFs to enable their receive
6029 * paths when MTU greater than 1500 is configured. So display a
6030 * warning that legacy VFs will be disabled.
6031 */
6032 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
6033 (adapter->hw.mac.type == ixgbe_mac_82599EB) &&
6034 (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)))
6035 e_warn(probe, "Setting MTU > 1500 will disable legacy VFs\n");
6036
6037 e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
6038
6039 /* must set new MTU before calling down or up */
6040 netdev->mtu = new_mtu;
6041
6042 if (netif_running(netdev))
6043 ixgbe_reinit_locked(adapter);
6044
6045 return 0;
6046 }
6047
6048 /**
6049 * ixgbe_open - Called when a network interface is made active
6050 * @netdev: network interface device structure
6051 *
6052 * Returns 0 on success, negative value on failure
6053 *
6054 * The open entry point is called when a network interface is made
6055 * active by the system (IFF_UP). At this point all resources needed
6056 * for transmit and receive operations are allocated, the interrupt
6057 * handler is registered with the OS, the watchdog timer is started,
6058 * and the stack is notified that the interface is ready.
6059 **/
6060 int ixgbe_open(struct net_device *netdev)
6061 {
6062 struct ixgbe_adapter *adapter = netdev_priv(netdev);
6063 struct ixgbe_hw *hw = &adapter->hw;
6064 int err, queues;
6065
6066 /* disallow open during test */
6067 if (test_bit(__IXGBE_TESTING, &adapter->state))
6068 return -EBUSY;
6069
6070 netif_carrier_off(netdev);
6071
6072 /* allocate transmit descriptors */
6073 err = ixgbe_setup_all_tx_resources(adapter);
6074 if (err)
6075 goto err_setup_tx;
6076
6077 /* allocate receive descriptors */
6078 err = ixgbe_setup_all_rx_resources(adapter);
6079 if (err)
6080 goto err_setup_rx;
6081
6082 ixgbe_configure(adapter);
6083
6084 err = ixgbe_request_irq(adapter);
6085 if (err)
6086 goto err_req_irq;
6087
6088 /* Notify the stack of the actual queue counts. */
6089 if (adapter->num_rx_pools > 1)
6090 queues = adapter->num_rx_queues_per_pool;
6091 else
6092 queues = adapter->num_tx_queues;
6093
6094 err = netif_set_real_num_tx_queues(netdev, queues);
6095 if (err)
6096 goto err_set_queues;
6097
6098 if (adapter->num_rx_pools > 1 &&
6099 adapter->num_rx_queues > IXGBE_MAX_L2A_QUEUES)
6100 queues = IXGBE_MAX_L2A_QUEUES;
6101 else
6102 queues = adapter->num_rx_queues;
6103 err = netif_set_real_num_rx_queues(netdev, queues);
6104 if (err)
6105 goto err_set_queues;
6106
6107 ixgbe_ptp_init(adapter);
6108
6109 ixgbe_up_complete(adapter);
6110
6111 ixgbe_clear_vxlan_port(adapter);
6112 #ifdef CONFIG_IXGBE_VXLAN
6113 vxlan_get_rx_port(netdev);
6114 #endif
6115
6116 return 0;
6117
6118 err_set_queues:
6119 ixgbe_free_irq(adapter);
6120 err_req_irq:
6121 ixgbe_free_all_rx_resources(adapter);
6122 if (hw->phy.ops.set_phy_power && !adapter->wol)
6123 hw->phy.ops.set_phy_power(&adapter->hw, false);
6124 err_setup_rx:
6125 ixgbe_free_all_tx_resources(adapter);
6126 err_setup_tx:
6127 ixgbe_reset(adapter);
6128
6129 return err;
6130 }
6131
6132 static void ixgbe_close_suspend(struct ixgbe_adapter *adapter)
6133 {
6134 ixgbe_ptp_suspend(adapter);
6135
6136 if (adapter->hw.phy.ops.enter_lplu) {
6137 adapter->hw.phy.reset_disable = true;
6138 ixgbe_down(adapter);
6139 adapter->hw.phy.ops.enter_lplu(&adapter->hw);
6140 adapter->hw.phy.reset_disable = false;
6141 } else {
6142 ixgbe_down(adapter);
6143 }
6144
6145 ixgbe_free_irq(adapter);
6146
6147 ixgbe_free_all_tx_resources(adapter);
6148 ixgbe_free_all_rx_resources(adapter);
6149 }
6150
6151 /**
6152 * ixgbe_close - Disables a network interface
6153 * @netdev: network interface device structure
6154 *
6155 * Returns 0, this is not allowed to fail
6156 *
6157 * The close entry point is called when an interface is de-activated
6158 * by the OS. The hardware is still under the drivers control, but
6159 * needs to be disabled. A global MAC reset is issued to stop the
6160 * hardware, and all transmit and receive resources are freed.
6161 **/
6162 int ixgbe_close(struct net_device *netdev)
6163 {
6164 struct ixgbe_adapter *adapter = netdev_priv(netdev);
6165
6166 ixgbe_ptp_stop(adapter);
6167
6168 ixgbe_close_suspend(adapter);
6169
6170 ixgbe_fdir_filter_exit(adapter);
6171
6172 ixgbe_release_hw_control(adapter);
6173
6174 return 0;
6175 }
6176
6177 #ifdef CONFIG_PM
6178 static int ixgbe_resume(struct pci_dev *pdev)
6179 {
6180 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
6181 struct net_device *netdev = adapter->netdev;
6182 u32 err;
6183
6184 adapter->hw.hw_addr = adapter->io_addr;
6185 pci_set_power_state(pdev, PCI_D0);
6186 pci_restore_state(pdev);
6187 /*
6188 * pci_restore_state clears dev->state_saved so call
6189 * pci_save_state to restore it.
6190 */
6191 pci_save_state(pdev);
6192
6193 err = pci_enable_device_mem(pdev);
6194 if (err) {
6195 e_dev_err("Cannot enable PCI device from suspend\n");
6196 return err;
6197 }
6198 smp_mb__before_atomic();
6199 clear_bit(__IXGBE_DISABLED, &adapter->state);
6200 pci_set_master(pdev);
6201
6202 pci_wake_from_d3(pdev, false);
6203
6204 ixgbe_reset(adapter);
6205
6206 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
6207
6208 rtnl_lock();
6209 err = ixgbe_init_interrupt_scheme(adapter);
6210 if (!err && netif_running(netdev))
6211 err = ixgbe_open(netdev);
6212
6213 rtnl_unlock();
6214
6215 if (err)
6216 return err;
6217
6218 netif_device_attach(netdev);
6219
6220 return 0;
6221 }
6222 #endif /* CONFIG_PM */
6223
6224 static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
6225 {
6226 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
6227 struct net_device *netdev = adapter->netdev;
6228 struct ixgbe_hw *hw = &adapter->hw;
6229 u32 ctrl, fctrl;
6230 u32 wufc = adapter->wol;
6231 #ifdef CONFIG_PM
6232 int retval = 0;
6233 #endif
6234
6235 netif_device_detach(netdev);
6236
6237 rtnl_lock();
6238 if (netif_running(netdev))
6239 ixgbe_close_suspend(adapter);
6240 rtnl_unlock();
6241
6242 ixgbe_clear_interrupt_scheme(adapter);
6243
6244 #ifdef CONFIG_PM
6245 retval = pci_save_state(pdev);
6246 if (retval)
6247 return retval;
6248
6249 #endif
6250 if (hw->mac.ops.stop_link_on_d3)
6251 hw->mac.ops.stop_link_on_d3(hw);
6252
6253 if (wufc) {
6254 ixgbe_set_rx_mode(netdev);
6255
6256 /* enable the optics for 82599 SFP+ fiber as we can WoL */
6257 if (hw->mac.ops.enable_tx_laser)
6258 hw->mac.ops.enable_tx_laser(hw);
6259
6260 /* turn on all-multi mode if wake on multicast is enabled */
6261 if (wufc & IXGBE_WUFC_MC) {
6262 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
6263 fctrl |= IXGBE_FCTRL_MPE;
6264 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
6265 }
6266
6267 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
6268 ctrl |= IXGBE_CTRL_GIO_DIS;
6269 IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);
6270
6271 IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);
6272 } else {
6273 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
6274 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
6275 }
6276
6277 switch (hw->mac.type) {
6278 case ixgbe_mac_82598EB:
6279 pci_wake_from_d3(pdev, false);
6280 break;
6281 case ixgbe_mac_82599EB:
6282 case ixgbe_mac_X540:
6283 case ixgbe_mac_X550:
6284 case ixgbe_mac_X550EM_x:
6285 case ixgbe_mac_x550em_a:
6286 pci_wake_from_d3(pdev, !!wufc);
6287 break;
6288 default:
6289 break;
6290 }
6291
6292 *enable_wake = !!wufc;
6293 if (hw->phy.ops.set_phy_power && !*enable_wake)
6294 hw->phy.ops.set_phy_power(hw, false);
6295
6296 ixgbe_release_hw_control(adapter);
6297
6298 if (!test_and_set_bit(__IXGBE_DISABLED, &adapter->state))
6299 pci_disable_device(pdev);
6300
6301 return 0;
6302 }
6303
6304 #ifdef CONFIG_PM
6305 static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state)
6306 {
6307 int retval;
6308 bool wake;
6309
6310 retval = __ixgbe_shutdown(pdev, &wake);
6311 if (retval)
6312 return retval;
6313
6314 if (wake) {
6315 pci_prepare_to_sleep(pdev);
6316 } else {
6317 pci_wake_from_d3(pdev, false);
6318 pci_set_power_state(pdev, PCI_D3hot);
6319 }
6320
6321 return 0;
6322 }
6323 #endif /* CONFIG_PM */
6324
6325 static void ixgbe_shutdown(struct pci_dev *pdev)
6326 {
6327 bool wake;
6328
6329 __ixgbe_shutdown(pdev, &wake);
6330
6331 if (system_state == SYSTEM_POWER_OFF) {
6332 pci_wake_from_d3(pdev, wake);
6333 pci_set_power_state(pdev, PCI_D3hot);
6334 }
6335 }
6336
6337 /**
6338 * ixgbe_update_stats - Update the board statistics counters.
6339 * @adapter: board private structure
6340 **/
6341 void ixgbe_update_stats(struct ixgbe_adapter *adapter)
6342 {
6343 struct net_device *netdev = adapter->netdev;
6344 struct ixgbe_hw *hw = &adapter->hw;
6345 struct ixgbe_hw_stats *hwstats = &adapter->stats;
6346 u64 total_mpc = 0;
6347 u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
6348 u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;
6349 u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
6350 u64 bytes = 0, packets = 0, hw_csum_rx_error = 0;
6351
6352 if (test_bit(__IXGBE_DOWN, &adapter->state) ||
6353 test_bit(__IXGBE_RESETTING, &adapter->state))
6354 return;
6355
6356 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {
6357 u64 rsc_count = 0;
6358 u64 rsc_flush = 0;
6359 for (i = 0; i < adapter->num_rx_queues; i++) {
6360 rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;
6361 rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;
6362 }
6363 adapter->rsc_total_count = rsc_count;
6364 adapter->rsc_total_flush = rsc_flush;
6365 }
6366
6367 for (i = 0; i < adapter->num_rx_queues; i++) {
6368 struct ixgbe_ring *rx_ring = adapter->rx_ring[i];
6369 non_eop_descs += rx_ring->rx_stats.non_eop_descs;
6370 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
6371 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
6372 hw_csum_rx_error += rx_ring->rx_stats.csum_err;
6373 bytes += rx_ring->stats.bytes;
6374 packets += rx_ring->stats.packets;
6375 }
6376 adapter->non_eop_descs = non_eop_descs;
6377 adapter->alloc_rx_page_failed = alloc_rx_page_failed;
6378 adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
6379 adapter->hw_csum_rx_error = hw_csum_rx_error;
6380 netdev->stats.rx_bytes = bytes;
6381 netdev->stats.rx_packets = packets;
6382
6383 bytes = 0;
6384 packets = 0;
6385 /* gather some stats to the adapter struct that are per queue */
6386 for (i = 0; i < adapter->num_tx_queues; i++) {
6387 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
6388 restart_queue += tx_ring->tx_stats.restart_queue;
6389 tx_busy += tx_ring->tx_stats.tx_busy;
6390 bytes += tx_ring->stats.bytes;
6391 packets += tx_ring->stats.packets;
6392 }
6393 adapter->restart_queue = restart_queue;
6394 adapter->tx_busy = tx_busy;
6395 netdev->stats.tx_bytes = bytes;
6396 netdev->stats.tx_packets = packets;
6397
6398 hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
6399
6400 /* 8 register reads */
6401 for (i = 0; i < 8; i++) {
6402 /* for packet buffers not used, the register should read 0 */
6403 mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
6404 missed_rx += mpc;
6405 hwstats->mpc[i] += mpc;
6406 total_mpc += hwstats->mpc[i];
6407 hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
6408 hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
6409 switch (hw->mac.type) {
6410 case ixgbe_mac_82598EB:
6411 hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
6412 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
6413 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
6414 hwstats->pxonrxc[i] +=
6415 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
6416 break;
6417 case ixgbe_mac_82599EB:
6418 case ixgbe_mac_X540:
6419 case ixgbe_mac_X550:
6420 case ixgbe_mac_X550EM_x:
6421 case ixgbe_mac_x550em_a:
6422 hwstats->pxonrxc[i] +=
6423 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
6424 break;
6425 default:
6426 break;
6427 }
6428 }
6429
6430 /*16 register reads */
6431 for (i = 0; i < 16; i++) {
6432 hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
6433 hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
6434 if ((hw->mac.type == ixgbe_mac_82599EB) ||
6435 (hw->mac.type == ixgbe_mac_X540) ||
6436 (hw->mac.type == ixgbe_mac_X550) ||
6437 (hw->mac.type == ixgbe_mac_X550EM_x) ||
6438 (hw->mac.type == ixgbe_mac_x550em_a)) {
6439 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
6440 IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */
6441 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
6442 IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */
6443 }
6444 }
6445
6446 hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
6447 /* work around hardware counting issue */
6448 hwstats->gprc -= missed_rx;
6449
6450 ixgbe_update_xoff_received(adapter);
6451
6452 /* 82598 hardware only has a 32 bit counter in the high register */
6453 switch (hw->mac.type) {
6454 case ixgbe_mac_82598EB:
6455 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
6456 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
6457 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
6458 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
6459 break;
6460 case ixgbe_mac_X540:
6461 case ixgbe_mac_X550:
6462 case ixgbe_mac_X550EM_x:
6463 case ixgbe_mac_x550em_a:
6464 /* OS2BMC stats are X540 and later */
6465 hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC);
6466 hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC);
6467 hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC);
6468 hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC);
6469 case ixgbe_mac_82599EB:
6470 for (i = 0; i < 16; i++)
6471 adapter->hw_rx_no_dma_resources +=
6472 IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
6473 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
6474 IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */
6475 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
6476 IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */
6477 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
6478 IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */
6479 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
6480 hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
6481 hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
6482 #ifdef IXGBE_FCOE
6483 hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
6484 hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
6485 hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
6486 hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
6487 hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
6488 hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
6489 /* Add up per cpu counters for total ddp aloc fail */
6490 if (adapter->fcoe.ddp_pool) {
6491 struct ixgbe_fcoe *fcoe = &adapter->fcoe;
6492 struct ixgbe_fcoe_ddp_pool *ddp_pool;
6493 unsigned int cpu;
6494 u64 noddp = 0, noddp_ext_buff = 0;
6495 for_each_possible_cpu(cpu) {
6496 ddp_pool = per_cpu_ptr(fcoe->ddp_pool, cpu);
6497 noddp += ddp_pool->noddp;
6498 noddp_ext_buff += ddp_pool->noddp_ext_buff;
6499 }
6500 hwstats->fcoe_noddp = noddp;
6501 hwstats->fcoe_noddp_ext_buff = noddp_ext_buff;
6502 }
6503 #endif /* IXGBE_FCOE */
6504 break;
6505 default:
6506 break;
6507 }
6508 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
6509 hwstats->bprc += bprc;
6510 hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
6511 if (hw->mac.type == ixgbe_mac_82598EB)
6512 hwstats->mprc -= bprc;
6513 hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
6514 hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
6515 hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
6516 hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
6517 hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
6518 hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
6519 hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
6520 hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
6521 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
6522 hwstats->lxontxc += lxon;
6523 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
6524 hwstats->lxofftxc += lxoff;
6525 hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
6526 hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
6527 /*
6528 * 82598 errata - tx of flow control packets is included in tx counters
6529 */
6530 xon_off_tot = lxon + lxoff;
6531 hwstats->gptc -= xon_off_tot;
6532 hwstats->mptc -= xon_off_tot;
6533 hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
6534 hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
6535 hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
6536 hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
6537 hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
6538 hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
6539 hwstats->ptc64 -= xon_off_tot;
6540 hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
6541 hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
6542 hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
6543 hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
6544 hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
6545 hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
6546
6547 /* Fill out the OS statistics structure */
6548 netdev->stats.multicast = hwstats->mprc;
6549
6550 /* Rx Errors */
6551 netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;
6552 netdev->stats.rx_dropped = 0;
6553 netdev->stats.rx_length_errors = hwstats->rlec;
6554 netdev->stats.rx_crc_errors = hwstats->crcerrs;
6555 netdev->stats.rx_missed_errors = total_mpc;
6556 }
6557
6558 /**
6559 * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table
6560 * @adapter: pointer to the device adapter structure
6561 **/
6562 static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)
6563 {
6564 struct ixgbe_hw *hw = &adapter->hw;
6565 int i;
6566
6567 if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
6568 return;
6569
6570 adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
6571
6572 /* if interface is down do nothing */
6573 if (test_bit(__IXGBE_DOWN, &adapter->state))
6574 return;
6575
6576 /* do nothing if we are not using signature filters */
6577 if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE))
6578 return;
6579
6580 adapter->fdir_overflow++;
6581
6582 if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
6583 for (i = 0; i < adapter->num_tx_queues; i++)
6584 set_bit(__IXGBE_TX_FDIR_INIT_DONE,
6585 &(adapter->tx_ring[i]->state));
6586 /* re-enable flow director interrupts */
6587 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
6588 } else {
6589 e_err(probe, "failed to finish FDIR re-initialization, "
6590 "ignored adding FDIR ATR filters\n");
6591 }
6592 }
6593
6594 /**
6595 * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts
6596 * @adapter: pointer to the device adapter structure
6597 *
6598 * This function serves two purposes. First it strobes the interrupt lines
6599 * in order to make certain interrupts are occurring. Secondly it sets the
6600 * bits needed to check for TX hangs. As a result we should immediately
6601 * determine if a hang has occurred.
6602 */
6603 static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)
6604 {
6605 struct ixgbe_hw *hw = &adapter->hw;
6606 u64 eics = 0;
6607 int i;
6608
6609 /* If we're down, removing or resetting, just bail */
6610 if (test_bit(__IXGBE_DOWN, &adapter->state) ||
6611 test_bit(__IXGBE_REMOVING, &adapter->state) ||
6612 test_bit(__IXGBE_RESETTING, &adapter->state))
6613 return;
6614
6615 /* Force detection of hung controller */
6616 if (netif_carrier_ok(adapter->netdev)) {
6617 for (i = 0; i < adapter->num_tx_queues; i++)
6618 set_check_for_tx_hang(adapter->tx_ring[i]);
6619 }
6620
6621 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
6622 /*
6623 * for legacy and MSI interrupts don't set any bits
6624 * that are enabled for EIAM, because this operation
6625 * would set *both* EIMS and EICS for any bit in EIAM
6626 */
6627 IXGBE_WRITE_REG(hw, IXGBE_EICS,
6628 (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER));
6629 } else {
6630 /* get one bit for every active tx/rx interrupt vector */
6631 for (i = 0; i < adapter->num_q_vectors; i++) {
6632 struct ixgbe_q_vector *qv = adapter->q_vector[i];
6633 if (qv->rx.ring || qv->tx.ring)
6634 eics |= ((u64)1 << i);
6635 }
6636 }
6637
6638 /* Cause software interrupt to ensure rings are cleaned */
6639 ixgbe_irq_rearm_queues(adapter, eics);
6640 }
6641
6642 /**
6643 * ixgbe_watchdog_update_link - update the link status
6644 * @adapter: pointer to the device adapter structure
6645 * @link_speed: pointer to a u32 to store the link_speed
6646 **/
6647 static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)
6648 {
6649 struct ixgbe_hw *hw = &adapter->hw;
6650 u32 link_speed = adapter->link_speed;
6651 bool link_up = adapter->link_up;
6652 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
6653
6654 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
6655 return;
6656
6657 if (hw->mac.ops.check_link) {
6658 hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
6659 } else {
6660 /* always assume link is up, if no check link function */
6661 link_speed = IXGBE_LINK_SPEED_10GB_FULL;
6662 link_up = true;
6663 }
6664
6665 if (adapter->ixgbe_ieee_pfc)
6666 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
6667
6668 if (link_up && !((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && pfc_en)) {
6669 hw->mac.ops.fc_enable(hw);
6670 ixgbe_set_rx_drop_en(adapter);
6671 }
6672
6673 if (link_up ||
6674 time_after(jiffies, (adapter->link_check_timeout +
6675 IXGBE_TRY_LINK_TIMEOUT))) {
6676 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
6677 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC);
6678 IXGBE_WRITE_FLUSH(hw);
6679 }
6680
6681 adapter->link_up = link_up;
6682 adapter->link_speed = link_speed;
6683 }
6684
6685 static void ixgbe_update_default_up(struct ixgbe_adapter *adapter)
6686 {
6687 #ifdef CONFIG_IXGBE_DCB
6688 struct net_device *netdev = adapter->netdev;
6689 struct dcb_app app = {
6690 .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE,
6691 .protocol = 0,
6692 };
6693 u8 up = 0;
6694
6695 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE)
6696 up = dcb_ieee_getapp_mask(netdev, &app);
6697
6698 adapter->default_up = (up > 1) ? (ffs(up) - 1) : 0;
6699 #endif
6700 }
6701
6702 /**
6703 * ixgbe_watchdog_link_is_up - update netif_carrier status and
6704 * print link up message
6705 * @adapter: pointer to the device adapter structure
6706 **/
6707 static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
6708 {
6709 struct net_device *netdev = adapter->netdev;
6710 struct ixgbe_hw *hw = &adapter->hw;
6711 struct net_device *upper;
6712 struct list_head *iter;
6713 u32 link_speed = adapter->link_speed;
6714 const char *speed_str;
6715 bool flow_rx, flow_tx;
6716
6717 /* only continue if link was previously down */
6718 if (netif_carrier_ok(netdev))
6719 return;
6720
6721 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
6722
6723 switch (hw->mac.type) {
6724 case ixgbe_mac_82598EB: {
6725 u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
6726 u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);
6727 flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);
6728 flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);
6729 }
6730 break;
6731 case ixgbe_mac_X540:
6732 case ixgbe_mac_X550:
6733 case ixgbe_mac_X550EM_x:
6734 case ixgbe_mac_x550em_a:
6735 case ixgbe_mac_82599EB: {
6736 u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
6737 u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
6738 flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);
6739 flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);
6740 }
6741 break;
6742 default:
6743 flow_tx = false;
6744 flow_rx = false;
6745 break;
6746 }
6747
6748 adapter->last_rx_ptp_check = jiffies;
6749
6750 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
6751 ixgbe_ptp_start_cyclecounter(adapter);
6752
6753 switch (link_speed) {
6754 case IXGBE_LINK_SPEED_10GB_FULL:
6755 speed_str = "10 Gbps";
6756 break;
6757 case IXGBE_LINK_SPEED_2_5GB_FULL:
6758 speed_str = "2.5 Gbps";
6759 break;
6760 case IXGBE_LINK_SPEED_1GB_FULL:
6761 speed_str = "1 Gbps";
6762 break;
6763 case IXGBE_LINK_SPEED_100_FULL:
6764 speed_str = "100 Mbps";
6765 break;
6766 default:
6767 speed_str = "unknown speed";
6768 break;
6769 }
6770 e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", speed_str,
6771 ((flow_rx && flow_tx) ? "RX/TX" :
6772 (flow_rx ? "RX" :
6773 (flow_tx ? "TX" : "None"))));
6774
6775 netif_carrier_on(netdev);
6776 ixgbe_check_vf_rate_limit(adapter);
6777
6778 /* enable transmits */
6779 netif_tx_wake_all_queues(adapter->netdev);
6780
6781 /* enable any upper devices */
6782 rtnl_lock();
6783 netdev_for_each_all_upper_dev_rcu(adapter->netdev, upper, iter) {
6784 if (netif_is_macvlan(upper)) {
6785 struct macvlan_dev *vlan = netdev_priv(upper);
6786
6787 if (vlan->fwd_priv)
6788 netif_tx_wake_all_queues(upper);
6789 }
6790 }
6791 rtnl_unlock();
6792
6793 /* update the default user priority for VFs */
6794 ixgbe_update_default_up(adapter);
6795
6796 /* ping all the active vfs to let them know link has changed */
6797 ixgbe_ping_all_vfs(adapter);
6798 }
6799
6800 /**
6801 * ixgbe_watchdog_link_is_down - update netif_carrier status and
6802 * print link down message
6803 * @adapter: pointer to the adapter structure
6804 **/
6805 static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *adapter)
6806 {
6807 struct net_device *netdev = adapter->netdev;
6808 struct ixgbe_hw *hw = &adapter->hw;
6809
6810 adapter->link_up = false;
6811 adapter->link_speed = 0;
6812
6813 /* only continue if link was up previously */
6814 if (!netif_carrier_ok(netdev))
6815 return;
6816
6817 /* poll for SFP+ cable when link is down */
6818 if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB)
6819 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
6820
6821 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
6822 ixgbe_ptp_start_cyclecounter(adapter);
6823
6824 e_info(drv, "NIC Link is Down\n");
6825 netif_carrier_off(netdev);
6826
6827 /* ping all the active vfs to let them know link has changed */
6828 ixgbe_ping_all_vfs(adapter);
6829 }
6830
6831 static bool ixgbe_ring_tx_pending(struct ixgbe_adapter *adapter)
6832 {
6833 int i;
6834
6835 for (i = 0; i < adapter->num_tx_queues; i++) {
6836 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
6837
6838 if (tx_ring->next_to_use != tx_ring->next_to_clean)
6839 return true;
6840 }
6841
6842 return false;
6843 }
6844
6845 static bool ixgbe_vf_tx_pending(struct ixgbe_adapter *adapter)
6846 {
6847 struct ixgbe_hw *hw = &adapter->hw;
6848 struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];
6849 u32 q_per_pool = __ALIGN_MASK(1, ~vmdq->mask);
6850
6851 int i, j;
6852
6853 if (!adapter->num_vfs)
6854 return false;
6855
6856 /* resetting the PF is only needed for MAC before X550 */
6857 if (hw->mac.type >= ixgbe_mac_X550)
6858 return false;
6859
6860 for (i = 0; i < adapter->num_vfs; i++) {
6861 for (j = 0; j < q_per_pool; j++) {
6862 u32 h, t;
6863
6864 h = IXGBE_READ_REG(hw, IXGBE_PVFTDHN(q_per_pool, i, j));
6865 t = IXGBE_READ_REG(hw, IXGBE_PVFTDTN(q_per_pool, i, j));
6866
6867 if (h != t)
6868 return true;
6869 }
6870 }
6871
6872 return false;
6873 }
6874
6875 /**
6876 * ixgbe_watchdog_flush_tx - flush queues on link down
6877 * @adapter: pointer to the device adapter structure
6878 **/
6879 static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
6880 {
6881 if (!netif_carrier_ok(adapter->netdev)) {
6882 if (ixgbe_ring_tx_pending(adapter) ||
6883 ixgbe_vf_tx_pending(adapter)) {
6884 /* We've lost link, so the controller stops DMA,
6885 * but we've got queued Tx work that's never going
6886 * to get done, so reset controller to flush Tx.
6887 * (Do the reset outside of interrupt context).
6888 */
6889 e_warn(drv, "initiating reset to clear Tx work after link loss\n");
6890 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
6891 }
6892 }
6893 }
6894
6895 #ifdef CONFIG_PCI_IOV
6896 static inline void ixgbe_issue_vf_flr(struct ixgbe_adapter *adapter,
6897 struct pci_dev *vfdev)
6898 {
6899 if (!pci_wait_for_pending_transaction(vfdev))
6900 e_dev_warn("Issuing VFLR with pending transactions\n");
6901
6902 e_dev_err("Issuing VFLR for VF %s\n", pci_name(vfdev));
6903 pcie_capability_set_word(vfdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
6904
6905 msleep(100);
6906 }
6907
6908 static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter)
6909 {
6910 struct ixgbe_hw *hw = &adapter->hw;
6911 struct pci_dev *pdev = adapter->pdev;
6912 unsigned int vf;
6913 u32 gpc;
6914
6915 if (!(netif_carrier_ok(adapter->netdev)))
6916 return;
6917
6918 gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC);
6919 if (gpc) /* If incrementing then no need for the check below */
6920 return;
6921 /* Check to see if a bad DMA write target from an errant or
6922 * malicious VF has caused a PCIe error. If so then we can
6923 * issue a VFLR to the offending VF(s) and then resume without
6924 * requesting a full slot reset.
6925 */
6926
6927 if (!pdev)
6928 return;
6929
6930 /* check status reg for all VFs owned by this PF */
6931 for (vf = 0; vf < adapter->num_vfs; ++vf) {
6932 struct pci_dev *vfdev = adapter->vfinfo[vf].vfdev;
6933 u16 status_reg;
6934
6935 if (!vfdev)
6936 continue;
6937 pci_read_config_word(vfdev, PCI_STATUS, &status_reg);
6938 if (status_reg != IXGBE_FAILED_READ_CFG_WORD &&
6939 status_reg & PCI_STATUS_REC_MASTER_ABORT)
6940 ixgbe_issue_vf_flr(adapter, vfdev);
6941 }
6942 }
6943
6944 static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)
6945 {
6946 u32 ssvpc;
6947
6948 /* Do not perform spoof check for 82598 or if not in IOV mode */
6949 if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
6950 adapter->num_vfs == 0)
6951 return;
6952
6953 ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);
6954
6955 /*
6956 * ssvpc register is cleared on read, if zero then no
6957 * spoofed packets in the last interval.
6958 */
6959 if (!ssvpc)
6960 return;
6961
6962 e_warn(drv, "%u Spoofed packets detected\n", ssvpc);
6963 }
6964 #else
6965 static void ixgbe_spoof_check(struct ixgbe_adapter __always_unused *adapter)
6966 {
6967 }
6968
6969 static void
6970 ixgbe_check_for_bad_vf(struct ixgbe_adapter __always_unused *adapter)
6971 {
6972 }
6973 #endif /* CONFIG_PCI_IOV */
6974
6975
6976 /**
6977 * ixgbe_watchdog_subtask - check and bring link up
6978 * @adapter: pointer to the device adapter structure
6979 **/
6980 static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter)
6981 {
6982 /* if interface is down, removing or resetting, do nothing */
6983 if (test_bit(__IXGBE_DOWN, &adapter->state) ||
6984 test_bit(__IXGBE_REMOVING, &adapter->state) ||
6985 test_bit(__IXGBE_RESETTING, &adapter->state))
6986 return;
6987
6988 ixgbe_watchdog_update_link(adapter);
6989
6990 if (adapter->link_up)
6991 ixgbe_watchdog_link_is_up(adapter);
6992 else
6993 ixgbe_watchdog_link_is_down(adapter);
6994
6995 ixgbe_check_for_bad_vf(adapter);
6996 ixgbe_spoof_check(adapter);
6997 ixgbe_update_stats(adapter);
6998
6999 ixgbe_watchdog_flush_tx(adapter);
7000 }
7001
7002 /**
7003 * ixgbe_sfp_detection_subtask - poll for SFP+ cable
7004 * @adapter: the ixgbe adapter structure
7005 **/
7006 static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)
7007 {
7008 struct ixgbe_hw *hw = &adapter->hw;
7009 s32 err;
7010
7011 /* not searching for SFP so there is nothing to do here */
7012 if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) &&
7013 !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
7014 return;
7015
7016 if (adapter->sfp_poll_time &&
7017 time_after(adapter->sfp_poll_time, jiffies))
7018 return; /* If not yet time to poll for SFP */
7019
7020 /* someone else is in init, wait until next service event */
7021 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
7022 return;
7023
7024 adapter->sfp_poll_time = jiffies + IXGBE_SFP_POLL_JIFFIES - 1;
7025
7026 err = hw->phy.ops.identify_sfp(hw);
7027 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
7028 goto sfp_out;
7029
7030 if (err == IXGBE_ERR_SFP_NOT_PRESENT) {
7031 /* If no cable is present, then we need to reset
7032 * the next time we find a good cable. */
7033 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
7034 }
7035
7036 /* exit on error */
7037 if (err)
7038 goto sfp_out;
7039
7040 /* exit if reset not needed */
7041 if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
7042 goto sfp_out;
7043
7044 adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET;
7045
7046 /*
7047 * A module may be identified correctly, but the EEPROM may not have
7048 * support for that module. setup_sfp() will fail in that case, so
7049 * we should not allow that module to load.
7050 */
7051 if (hw->mac.type == ixgbe_mac_82598EB)
7052 err = hw->phy.ops.reset(hw);
7053 else
7054 err = hw->mac.ops.setup_sfp(hw);
7055
7056 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
7057 goto sfp_out;
7058
7059 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
7060 e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);
7061
7062 sfp_out:
7063 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
7064
7065 if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) &&
7066 (adapter->netdev->reg_state == NETREG_REGISTERED)) {
7067 e_dev_err("failed to initialize because an unsupported "
7068 "SFP+ module type was detected.\n");
7069 e_dev_err("Reload the driver after installing a "
7070 "supported module.\n");
7071 unregister_netdev(adapter->netdev);
7072 }
7073 }
7074
7075 /**
7076 * ixgbe_sfp_link_config_subtask - set up link SFP after module install
7077 * @adapter: the ixgbe adapter structure
7078 **/
7079 static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
7080 {
7081 struct ixgbe_hw *hw = &adapter->hw;
7082 u32 speed;
7083 bool autoneg = false;
7084
7085 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG))
7086 return;
7087
7088 /* someone else is in init, wait until next service event */
7089 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
7090 return;
7091
7092 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
7093
7094 speed = hw->phy.autoneg_advertised;
7095 if ((!speed) && (hw->mac.ops.get_link_capabilities)) {
7096 hw->mac.ops.get_link_capabilities(hw, &speed, &autoneg);
7097
7098 /* setup the highest link when no autoneg */
7099 if (!autoneg) {
7100 if (speed & IXGBE_LINK_SPEED_10GB_FULL)
7101 speed = IXGBE_LINK_SPEED_10GB_FULL;
7102 }
7103 }
7104
7105 if (hw->mac.ops.setup_link)
7106 hw->mac.ops.setup_link(hw, speed, true);
7107
7108 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
7109 adapter->link_check_timeout = jiffies;
7110 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
7111 }
7112
7113 /**
7114 * ixgbe_service_timer - Timer Call-back
7115 * @data: pointer to adapter cast into an unsigned long
7116 **/
7117 static void ixgbe_service_timer(unsigned long data)
7118 {
7119 struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data;
7120 unsigned long next_event_offset;
7121
7122 /* poll faster when waiting for link */
7123 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
7124 next_event_offset = HZ / 10;
7125 else
7126 next_event_offset = HZ * 2;
7127
7128 /* Reset the timer */
7129 mod_timer(&adapter->service_timer, next_event_offset + jiffies);
7130
7131 ixgbe_service_event_schedule(adapter);
7132 }
7133
7134 static void ixgbe_phy_interrupt_subtask(struct ixgbe_adapter *adapter)
7135 {
7136 struct ixgbe_hw *hw = &adapter->hw;
7137 u32 status;
7138
7139 if (!(adapter->flags2 & IXGBE_FLAG2_PHY_INTERRUPT))
7140 return;
7141
7142 adapter->flags2 &= ~IXGBE_FLAG2_PHY_INTERRUPT;
7143
7144 if (!hw->phy.ops.handle_lasi)
7145 return;
7146
7147 status = hw->phy.ops.handle_lasi(&adapter->hw);
7148 if (status != IXGBE_ERR_OVERTEMP)
7149 return;
7150
7151 e_crit(drv, "%s\n", ixgbe_overheat_msg);
7152 }
7153
7154 static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)
7155 {
7156 if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED))
7157 return;
7158
7159 adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED;
7160
7161 /* If we're already down, removing or resetting, just bail */
7162 if (test_bit(__IXGBE_DOWN, &adapter->state) ||
7163 test_bit(__IXGBE_REMOVING, &adapter->state) ||
7164 test_bit(__IXGBE_RESETTING, &adapter->state))
7165 return;
7166
7167 ixgbe_dump(adapter);
7168 netdev_err(adapter->netdev, "Reset adapter\n");
7169 adapter->tx_timeout_count++;
7170
7171 rtnl_lock();
7172 ixgbe_reinit_locked(adapter);
7173 rtnl_unlock();
7174 }
7175
7176 /**
7177 * ixgbe_service_task - manages and runs subtasks
7178 * @work: pointer to work_struct containing our data
7179 **/
7180 static void ixgbe_service_task(struct work_struct *work)
7181 {
7182 struct ixgbe_adapter *adapter = container_of(work,
7183 struct ixgbe_adapter,
7184 service_task);
7185 if (ixgbe_removed(adapter->hw.hw_addr)) {
7186 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
7187 rtnl_lock();
7188 ixgbe_down(adapter);
7189 rtnl_unlock();
7190 }
7191 ixgbe_service_event_complete(adapter);
7192 return;
7193 }
7194 #ifdef CONFIG_IXGBE_VXLAN
7195 rtnl_lock();
7196 if (adapter->flags2 & IXGBE_FLAG2_VXLAN_REREG_NEEDED) {
7197 adapter->flags2 &= ~IXGBE_FLAG2_VXLAN_REREG_NEEDED;
7198 vxlan_get_rx_port(adapter->netdev);
7199 }
7200 rtnl_unlock();
7201 #endif /* CONFIG_IXGBE_VXLAN */
7202 ixgbe_reset_subtask(adapter);
7203 ixgbe_phy_interrupt_subtask(adapter);
7204 ixgbe_sfp_detection_subtask(adapter);
7205 ixgbe_sfp_link_config_subtask(adapter);
7206 ixgbe_check_overtemp_subtask(adapter);
7207 ixgbe_watchdog_subtask(adapter);
7208 ixgbe_fdir_reinit_subtask(adapter);
7209 ixgbe_check_hang_subtask(adapter);
7210
7211 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) {
7212 ixgbe_ptp_overflow_check(adapter);
7213 ixgbe_ptp_rx_hang(adapter);
7214 }
7215
7216 ixgbe_service_event_complete(adapter);
7217 }
7218
7219 static int ixgbe_tso(struct ixgbe_ring *tx_ring,
7220 struct ixgbe_tx_buffer *first,
7221 u8 *hdr_len)
7222 {
7223 struct sk_buff *skb = first->skb;
7224 u32 vlan_macip_lens, type_tucmd;
7225 u32 mss_l4len_idx, l4len;
7226 int err;
7227
7228 if (skb->ip_summed != CHECKSUM_PARTIAL)
7229 return 0;
7230
7231 if (!skb_is_gso(skb))
7232 return 0;
7233
7234 err = skb_cow_head(skb, 0);
7235 if (err < 0)
7236 return err;
7237
7238 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
7239 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
7240
7241 if (first->protocol == htons(ETH_P_IP)) {
7242 struct iphdr *iph = ip_hdr(skb);
7243 iph->tot_len = 0;
7244 iph->check = 0;
7245 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
7246 iph->daddr, 0,
7247 IPPROTO_TCP,
7248 0);
7249 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
7250 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
7251 IXGBE_TX_FLAGS_CSUM |
7252 IXGBE_TX_FLAGS_IPV4;
7253 } else if (skb_is_gso_v6(skb)) {
7254 ipv6_hdr(skb)->payload_len = 0;
7255 tcp_hdr(skb)->check =
7256 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
7257 &ipv6_hdr(skb)->daddr,
7258 0, IPPROTO_TCP, 0);
7259 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
7260 IXGBE_TX_FLAGS_CSUM;
7261 }
7262
7263 /* compute header lengths */
7264 l4len = tcp_hdrlen(skb);
7265 *hdr_len = skb_transport_offset(skb) + l4len;
7266
7267 /* update gso size and bytecount with header size */
7268 first->gso_segs = skb_shinfo(skb)->gso_segs;
7269 first->bytecount += (first->gso_segs - 1) * *hdr_len;
7270
7271 /* mss_l4len_id: use 0 as index for TSO */
7272 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
7273 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
7274
7275 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
7276 vlan_macip_lens = skb_network_header_len(skb);
7277 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
7278 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
7279
7280 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd,
7281 mss_l4len_idx);
7282
7283 return 1;
7284 }
7285
7286 static inline bool ixgbe_ipv6_csum_is_sctp(struct sk_buff *skb)
7287 {
7288 unsigned int offset = 0;
7289
7290 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL);
7291
7292 return offset == skb_checksum_start_offset(skb);
7293 }
7294
7295 static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring,
7296 struct ixgbe_tx_buffer *first)
7297 {
7298 struct sk_buff *skb = first->skb;
7299 u32 vlan_macip_lens = 0;
7300 u32 type_tucmd = 0;
7301
7302 if (skb->ip_summed != CHECKSUM_PARTIAL) {
7303 csum_failed:
7304 if (!(first->tx_flags & (IXGBE_TX_FLAGS_HW_VLAN |
7305 IXGBE_TX_FLAGS_CC)))
7306 return;
7307 goto no_csum;
7308 }
7309
7310 switch (skb->csum_offset) {
7311 case offsetof(struct tcphdr, check):
7312 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
7313 /* fall through */
7314 case offsetof(struct udphdr, check):
7315 break;
7316 case offsetof(struct sctphdr, checksum):
7317 /* validate that this is actually an SCTP request */
7318 if (((first->protocol == htons(ETH_P_IP)) &&
7319 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) ||
7320 ((first->protocol == htons(ETH_P_IPV6)) &&
7321 ixgbe_ipv6_csum_is_sctp(skb))) {
7322 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_SCTP;
7323 break;
7324 }
7325 /* fall through */
7326 default:
7327 skb_checksum_help(skb);
7328 goto csum_failed;
7329 }
7330
7331 /* update TX checksum flag */
7332 first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
7333 vlan_macip_lens = skb_checksum_start_offset(skb) -
7334 skb_network_offset(skb);
7335 no_csum:
7336 /* vlan_macip_lens: MACLEN, VLAN tag */
7337 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
7338 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
7339
7340 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd, 0);
7341 }
7342
7343 #define IXGBE_SET_FLAG(_input, _flag, _result) \
7344 ((_flag <= _result) ? \
7345 ((u32)(_input & _flag) * (_result / _flag)) : \
7346 ((u32)(_input & _flag) / (_flag / _result)))
7347
7348 static u32 ixgbe_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
7349 {
7350 /* set type for advanced descriptor with frame checksum insertion */
7351 u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA |
7352 IXGBE_ADVTXD_DCMD_DEXT |
7353 IXGBE_ADVTXD_DCMD_IFCS;
7354
7355 /* set HW vlan bit if vlan is present */
7356 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_HW_VLAN,
7357 IXGBE_ADVTXD_DCMD_VLE);
7358
7359 /* set segmentation enable bits for TSO/FSO */
7360 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSO,
7361 IXGBE_ADVTXD_DCMD_TSE);
7362
7363 /* set timestamp bit if present */
7364 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSTAMP,
7365 IXGBE_ADVTXD_MAC_TSTAMP);
7366
7367 /* insert frame checksum */
7368 cmd_type ^= IXGBE_SET_FLAG(skb->no_fcs, 1, IXGBE_ADVTXD_DCMD_IFCS);
7369
7370 return cmd_type;
7371 }
7372
7373 static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
7374 u32 tx_flags, unsigned int paylen)
7375 {
7376 u32 olinfo_status = paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
7377
7378 /* enable L4 checksum for TSO and TX checksum offload */
7379 olinfo_status |= IXGBE_SET_FLAG(tx_flags,
7380 IXGBE_TX_FLAGS_CSUM,
7381 IXGBE_ADVTXD_POPTS_TXSM);
7382
7383 /* enble IPv4 checksum for TSO */
7384 olinfo_status |= IXGBE_SET_FLAG(tx_flags,
7385 IXGBE_TX_FLAGS_IPV4,
7386 IXGBE_ADVTXD_POPTS_IXSM);
7387
7388 /*
7389 * Check Context must be set if Tx switch is enabled, which it
7390 * always is for case where virtual functions are running
7391 */
7392 olinfo_status |= IXGBE_SET_FLAG(tx_flags,
7393 IXGBE_TX_FLAGS_CC,
7394 IXGBE_ADVTXD_CC);
7395
7396 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
7397 }
7398
7399 static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
7400 {
7401 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
7402
7403 /* Herbert's original patch had:
7404 * smp_mb__after_netif_stop_queue();
7405 * but since that doesn't exist yet, just open code it.
7406 */
7407 smp_mb();
7408
7409 /* We need to check again in a case another CPU has just
7410 * made room available.
7411 */
7412 if (likely(ixgbe_desc_unused(tx_ring) < size))
7413 return -EBUSY;
7414
7415 /* A reprieve! - use start_queue because it doesn't call schedule */
7416 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
7417 ++tx_ring->tx_stats.restart_queue;
7418 return 0;
7419 }
7420
7421 static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
7422 {
7423 if (likely(ixgbe_desc_unused(tx_ring) >= size))
7424 return 0;
7425
7426 return __ixgbe_maybe_stop_tx(tx_ring, size);
7427 }
7428
7429 #define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \
7430 IXGBE_TXD_CMD_RS)
7431
7432 static void ixgbe_tx_map(struct ixgbe_ring *tx_ring,
7433 struct ixgbe_tx_buffer *first,
7434 const u8 hdr_len)
7435 {
7436 struct sk_buff *skb = first->skb;
7437 struct ixgbe_tx_buffer *tx_buffer;
7438 union ixgbe_adv_tx_desc *tx_desc;
7439 struct skb_frag_struct *frag;
7440 dma_addr_t dma;
7441 unsigned int data_len, size;
7442 u32 tx_flags = first->tx_flags;
7443 u32 cmd_type = ixgbe_tx_cmd_type(skb, tx_flags);
7444 u16 i = tx_ring->next_to_use;
7445
7446 tx_desc = IXGBE_TX_DESC(tx_ring, i);
7447
7448 ixgbe_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
7449
7450 size = skb_headlen(skb);
7451 data_len = skb->data_len;
7452
7453 #ifdef IXGBE_FCOE
7454 if (tx_flags & IXGBE_TX_FLAGS_FCOE) {
7455 if (data_len < sizeof(struct fcoe_crc_eof)) {
7456 size -= sizeof(struct fcoe_crc_eof) - data_len;
7457 data_len = 0;
7458 } else {
7459 data_len -= sizeof(struct fcoe_crc_eof);
7460 }
7461 }
7462
7463 #endif
7464 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
7465
7466 tx_buffer = first;
7467
7468 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
7469 if (dma_mapping_error(tx_ring->dev, dma))
7470 goto dma_error;
7471
7472 /* record length, and DMA address */
7473 dma_unmap_len_set(tx_buffer, len, size);
7474 dma_unmap_addr_set(tx_buffer, dma, dma);
7475
7476 tx_desc->read.buffer_addr = cpu_to_le64(dma);
7477
7478 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
7479 tx_desc->read.cmd_type_len =
7480 cpu_to_le32(cmd_type ^ IXGBE_MAX_DATA_PER_TXD);
7481
7482 i++;
7483 tx_desc++;
7484 if (i == tx_ring->count) {
7485 tx_desc = IXGBE_TX_DESC(tx_ring, 0);
7486 i = 0;
7487 }
7488 tx_desc->read.olinfo_status = 0;
7489
7490 dma += IXGBE_MAX_DATA_PER_TXD;
7491 size -= IXGBE_MAX_DATA_PER_TXD;
7492
7493 tx_desc->read.buffer_addr = cpu_to_le64(dma);
7494 }
7495
7496 if (likely(!data_len))
7497 break;
7498
7499 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
7500
7501 i++;
7502 tx_desc++;
7503 if (i == tx_ring->count) {
7504 tx_desc = IXGBE_TX_DESC(tx_ring, 0);
7505 i = 0;
7506 }
7507 tx_desc->read.olinfo_status = 0;
7508
7509 #ifdef IXGBE_FCOE
7510 size = min_t(unsigned int, data_len, skb_frag_size(frag));
7511 #else
7512 size = skb_frag_size(frag);
7513 #endif
7514 data_len -= size;
7515
7516 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
7517 DMA_TO_DEVICE);
7518
7519 tx_buffer = &tx_ring->tx_buffer_info[i];
7520 }
7521
7522 /* write last descriptor with RS and EOP bits */
7523 cmd_type |= size | IXGBE_TXD_CMD;
7524 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
7525
7526 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
7527
7528 /* set the timestamp */
7529 first->time_stamp = jiffies;
7530
7531 /*
7532 * Force memory writes to complete before letting h/w know there
7533 * are new descriptors to fetch. (Only applicable for weak-ordered
7534 * memory model archs, such as IA-64).
7535 *
7536 * We also need this memory barrier to make certain all of the
7537 * status bits have been updated before next_to_watch is written.
7538 */
7539 wmb();
7540
7541 /* set next_to_watch value indicating a packet is present */
7542 first->next_to_watch = tx_desc;
7543
7544 i++;
7545 if (i == tx_ring->count)
7546 i = 0;
7547
7548 tx_ring->next_to_use = i;
7549
7550 ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED);
7551
7552 if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) {
7553 writel(i, tx_ring->tail);
7554
7555 /* we need this if more than one processor can write to our tail
7556 * at a time, it synchronizes IO on IA64/Altix systems
7557 */
7558 mmiowb();
7559 }
7560
7561 return;
7562 dma_error:
7563 dev_err(tx_ring->dev, "TX DMA map failed\n");
7564
7565 /* clear dma mappings for failed tx_buffer_info map */
7566 for (;;) {
7567 tx_buffer = &tx_ring->tx_buffer_info[i];
7568 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer);
7569 if (tx_buffer == first)
7570 break;
7571 if (i == 0)
7572 i = tx_ring->count;
7573 i--;
7574 }
7575
7576 tx_ring->next_to_use = i;
7577 }
7578
7579 static void ixgbe_atr(struct ixgbe_ring *ring,
7580 struct ixgbe_tx_buffer *first)
7581 {
7582 struct ixgbe_q_vector *q_vector = ring->q_vector;
7583 union ixgbe_atr_hash_dword input = { .dword = 0 };
7584 union ixgbe_atr_hash_dword common = { .dword = 0 };
7585 union {
7586 unsigned char *network;
7587 struct iphdr *ipv4;
7588 struct ipv6hdr *ipv6;
7589 } hdr;
7590 struct tcphdr *th;
7591 unsigned int hlen;
7592 struct sk_buff *skb;
7593 __be16 vlan_id;
7594 int l4_proto;
7595
7596 /* if ring doesn't have a interrupt vector, cannot perform ATR */
7597 if (!q_vector)
7598 return;
7599
7600 /* do nothing if sampling is disabled */
7601 if (!ring->atr_sample_rate)
7602 return;
7603
7604 ring->atr_count++;
7605
7606 /* currently only IPv4/IPv6 with TCP is supported */
7607 if ((first->protocol != htons(ETH_P_IP)) &&
7608 (first->protocol != htons(ETH_P_IPV6)))
7609 return;
7610
7611 /* snag network header to get L4 type and address */
7612 skb = first->skb;
7613 hdr.network = skb_network_header(skb);
7614 #ifdef CONFIG_IXGBE_VXLAN
7615 if (skb->encapsulation &&
7616 first->protocol == htons(ETH_P_IP) &&
7617 hdr.ipv4->protocol != IPPROTO_UDP) {
7618 struct ixgbe_adapter *adapter = q_vector->adapter;
7619
7620 /* verify the port is recognized as VXLAN */
7621 if (adapter->vxlan_port &&
7622 udp_hdr(skb)->dest == adapter->vxlan_port)
7623 hdr.network = skb_inner_network_header(skb);
7624 }
7625 #endif /* CONFIG_IXGBE_VXLAN */
7626
7627 /* Currently only IPv4/IPv6 with TCP is supported */
7628 switch (hdr.ipv4->version) {
7629 case IPVERSION:
7630 /* access ihl as u8 to avoid unaligned access on ia64 */
7631 hlen = (hdr.network[0] & 0x0F) << 2;
7632 l4_proto = hdr.ipv4->protocol;
7633 break;
7634 case 6:
7635 hlen = hdr.network - skb->data;
7636 l4_proto = ipv6_find_hdr(skb, &hlen, IPPROTO_TCP, NULL, NULL);
7637 hlen -= hdr.network - skb->data;
7638 break;
7639 default:
7640 return;
7641 }
7642
7643 if (l4_proto != IPPROTO_TCP)
7644 return;
7645
7646 th = (struct tcphdr *)(hdr.network + hlen);
7647
7648 /* skip this packet since the socket is closing */
7649 if (th->fin)
7650 return;
7651
7652 /* sample on all syn packets or once every atr sample count */
7653 if (!th->syn && (ring->atr_count < ring->atr_sample_rate))
7654 return;
7655
7656 /* reset sample count */
7657 ring->atr_count = 0;
7658
7659 vlan_id = htons(first->tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT);
7660
7661 /*
7662 * src and dst are inverted, think how the receiver sees them
7663 *
7664 * The input is broken into two sections, a non-compressed section
7665 * containing vm_pool, vlan_id, and flow_type. The rest of the data
7666 * is XORed together and stored in the compressed dword.
7667 */
7668 input.formatted.vlan_id = vlan_id;
7669
7670 /*
7671 * since src port and flex bytes occupy the same word XOR them together
7672 * and write the value to source port portion of compressed dword
7673 */
7674 if (first->tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN))
7675 common.port.src ^= th->dest ^ htons(ETH_P_8021Q);
7676 else
7677 common.port.src ^= th->dest ^ first->protocol;
7678 common.port.dst ^= th->source;
7679
7680 switch (hdr.ipv4->version) {
7681 case IPVERSION:
7682 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4;
7683 common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr;
7684 break;
7685 case 6:
7686 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6;
7687 common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^
7688 hdr.ipv6->saddr.s6_addr32[1] ^
7689 hdr.ipv6->saddr.s6_addr32[2] ^
7690 hdr.ipv6->saddr.s6_addr32[3] ^
7691 hdr.ipv6->daddr.s6_addr32[0] ^
7692 hdr.ipv6->daddr.s6_addr32[1] ^
7693 hdr.ipv6->daddr.s6_addr32[2] ^
7694 hdr.ipv6->daddr.s6_addr32[3];
7695 break;
7696 default:
7697 break;
7698 }
7699
7700 if (hdr.network != skb_network_header(skb))
7701 input.formatted.flow_type |= IXGBE_ATR_L4TYPE_TUNNEL_MASK;
7702
7703 /* This assumes the Rx queue and Tx queue are bound to the same CPU */
7704 ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw,
7705 input, common, ring->queue_index);
7706 }
7707
7708 static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb,
7709 void *accel_priv, select_queue_fallback_t fallback)
7710 {
7711 struct ixgbe_fwd_adapter *fwd_adapter = accel_priv;
7712 #ifdef IXGBE_FCOE
7713 struct ixgbe_adapter *adapter;
7714 struct ixgbe_ring_feature *f;
7715 int txq;
7716 #endif
7717
7718 if (fwd_adapter)
7719 return skb->queue_mapping + fwd_adapter->tx_base_queue;
7720
7721 #ifdef IXGBE_FCOE
7722
7723 /*
7724 * only execute the code below if protocol is FCoE
7725 * or FIP and we have FCoE enabled on the adapter
7726 */
7727 switch (vlan_get_protocol(skb)) {
7728 case htons(ETH_P_FCOE):
7729 case htons(ETH_P_FIP):
7730 adapter = netdev_priv(dev);
7731
7732 if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)
7733 break;
7734 default:
7735 return fallback(dev, skb);
7736 }
7737
7738 f = &adapter->ring_feature[RING_F_FCOE];
7739
7740 txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) :
7741 smp_processor_id();
7742
7743 while (txq >= f->indices)
7744 txq -= f->indices;
7745
7746 return txq + f->offset;
7747 #else
7748 return fallback(dev, skb);
7749 #endif
7750 }
7751
7752 netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
7753 struct ixgbe_adapter *adapter,
7754 struct ixgbe_ring *tx_ring)
7755 {
7756 struct ixgbe_tx_buffer *first;
7757 int tso;
7758 u32 tx_flags = 0;
7759 unsigned short f;
7760 u16 count = TXD_USE_COUNT(skb_headlen(skb));
7761 __be16 protocol = skb->protocol;
7762 u8 hdr_len = 0;
7763
7764 /*
7765 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
7766 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
7767 * + 2 desc gap to keep tail from touching head,
7768 * + 1 desc for context descriptor,
7769 * otherwise try next time
7770 */
7771 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
7772 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
7773
7774 if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) {
7775 tx_ring->tx_stats.tx_busy++;
7776 return NETDEV_TX_BUSY;
7777 }
7778
7779 /* record the location of the first descriptor for this packet */
7780 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
7781 first->skb = skb;
7782 first->bytecount = skb->len;
7783 first->gso_segs = 1;
7784
7785 /* if we have a HW VLAN tag being added default to the HW one */
7786 if (skb_vlan_tag_present(skb)) {
7787 tx_flags |= skb_vlan_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT;
7788 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
7789 /* else if it is a SW VLAN check the next protocol and store the tag */
7790 } else if (protocol == htons(ETH_P_8021Q)) {
7791 struct vlan_hdr *vhdr, _vhdr;
7792 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
7793 if (!vhdr)
7794 goto out_drop;
7795
7796 tx_flags |= ntohs(vhdr->h_vlan_TCI) <<
7797 IXGBE_TX_FLAGS_VLAN_SHIFT;
7798 tx_flags |= IXGBE_TX_FLAGS_SW_VLAN;
7799 }
7800 protocol = vlan_get_protocol(skb);
7801
7802 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
7803 adapter->ptp_clock &&
7804 !test_and_set_bit_lock(__IXGBE_PTP_TX_IN_PROGRESS,
7805 &adapter->state)) {
7806 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
7807 tx_flags |= IXGBE_TX_FLAGS_TSTAMP;
7808
7809 /* schedule check for Tx timestamp */
7810 adapter->ptp_tx_skb = skb_get(skb);
7811 adapter->ptp_tx_start = jiffies;
7812 schedule_work(&adapter->ptp_tx_work);
7813 }
7814
7815 skb_tx_timestamp(skb);
7816
7817 #ifdef CONFIG_PCI_IOV
7818 /*
7819 * Use the l2switch_enable flag - would be false if the DMA
7820 * Tx switch had been disabled.
7821 */
7822 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
7823 tx_flags |= IXGBE_TX_FLAGS_CC;
7824
7825 #endif
7826 /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */
7827 if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
7828 ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) ||
7829 (skb->priority != TC_PRIO_CONTROL))) {
7830 tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK;
7831 tx_flags |= (skb->priority & 0x7) <<
7832 IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT;
7833 if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) {
7834 struct vlan_ethhdr *vhdr;
7835
7836 if (skb_cow_head(skb, 0))
7837 goto out_drop;
7838 vhdr = (struct vlan_ethhdr *)skb->data;
7839 vhdr->h_vlan_TCI = htons(tx_flags >>
7840 IXGBE_TX_FLAGS_VLAN_SHIFT);
7841 } else {
7842 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
7843 }
7844 }
7845
7846 /* record initial flags and protocol */
7847 first->tx_flags = tx_flags;
7848 first->protocol = protocol;
7849
7850 #ifdef IXGBE_FCOE
7851 /* setup tx offload for FCoE */
7852 if ((protocol == htons(ETH_P_FCOE)) &&
7853 (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) {
7854 tso = ixgbe_fso(tx_ring, first, &hdr_len);
7855 if (tso < 0)
7856 goto out_drop;
7857
7858 goto xmit_fcoe;
7859 }
7860
7861 #endif /* IXGBE_FCOE */
7862 tso = ixgbe_tso(tx_ring, first, &hdr_len);
7863 if (tso < 0)
7864 goto out_drop;
7865 else if (!tso)
7866 ixgbe_tx_csum(tx_ring, first);
7867
7868 /* add the ATR filter if ATR is on */
7869 if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))
7870 ixgbe_atr(tx_ring, first);
7871
7872 #ifdef IXGBE_FCOE
7873 xmit_fcoe:
7874 #endif /* IXGBE_FCOE */
7875 ixgbe_tx_map(tx_ring, first, hdr_len);
7876
7877 return NETDEV_TX_OK;
7878
7879 out_drop:
7880 dev_kfree_skb_any(first->skb);
7881 first->skb = NULL;
7882
7883 return NETDEV_TX_OK;
7884 }
7885
7886 static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb,
7887 struct net_device *netdev,
7888 struct ixgbe_ring *ring)
7889 {
7890 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7891 struct ixgbe_ring *tx_ring;
7892
7893 /*
7894 * The minimum packet size for olinfo paylen is 17 so pad the skb
7895 * in order to meet this minimum size requirement.
7896 */
7897 if (skb_put_padto(skb, 17))
7898 return NETDEV_TX_OK;
7899
7900 tx_ring = ring ? ring : adapter->tx_ring[skb->queue_mapping];
7901
7902 return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);
7903 }
7904
7905 static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb,
7906 struct net_device *netdev)
7907 {
7908 return __ixgbe_xmit_frame(skb, netdev, NULL);
7909 }
7910
7911 /**
7912 * ixgbe_set_mac - Change the Ethernet Address of the NIC
7913 * @netdev: network interface device structure
7914 * @p: pointer to an address structure
7915 *
7916 * Returns 0 on success, negative on failure
7917 **/
7918 static int ixgbe_set_mac(struct net_device *netdev, void *p)
7919 {
7920 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7921 struct ixgbe_hw *hw = &adapter->hw;
7922 struct sockaddr *addr = p;
7923
7924 if (!is_valid_ether_addr(addr->sa_data))
7925 return -EADDRNOTAVAIL;
7926
7927 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
7928 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
7929
7930 ixgbe_mac_set_default_filter(adapter);
7931
7932 return 0;
7933 }
7934
7935 static int
7936 ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr)
7937 {
7938 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7939 struct ixgbe_hw *hw = &adapter->hw;
7940 u16 value;
7941 int rc;
7942
7943 if (prtad != hw->phy.mdio.prtad)
7944 return -EINVAL;
7945 rc = hw->phy.ops.read_reg(hw, addr, devad, &value);
7946 if (!rc)
7947 rc = value;
7948 return rc;
7949 }
7950
7951 static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad,
7952 u16 addr, u16 value)
7953 {
7954 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7955 struct ixgbe_hw *hw = &adapter->hw;
7956
7957 if (prtad != hw->phy.mdio.prtad)
7958 return -EINVAL;
7959 return hw->phy.ops.write_reg(hw, addr, devad, value);
7960 }
7961
7962 static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
7963 {
7964 struct ixgbe_adapter *adapter = netdev_priv(netdev);
7965
7966 switch (cmd) {
7967 case SIOCSHWTSTAMP:
7968 return ixgbe_ptp_set_ts_config(adapter, req);
7969 case SIOCGHWTSTAMP:
7970 return ixgbe_ptp_get_ts_config(adapter, req);
7971 default:
7972 return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd);
7973 }
7974 }
7975
7976 /**
7977 * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding
7978 * netdev->dev_addrs
7979 * @netdev: network interface device structure
7980 *
7981 * Returns non-zero on failure
7982 **/
7983 static int ixgbe_add_sanmac_netdev(struct net_device *dev)
7984 {
7985 int err = 0;
7986 struct ixgbe_adapter *adapter = netdev_priv(dev);
7987 struct ixgbe_hw *hw = &adapter->hw;
7988
7989 if (is_valid_ether_addr(hw->mac.san_addr)) {
7990 rtnl_lock();
7991 err = dev_addr_add(dev, hw->mac.san_addr, NETDEV_HW_ADDR_T_SAN);
7992 rtnl_unlock();
7993
7994 /* update SAN MAC vmdq pool selection */
7995 hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));
7996 }
7997 return err;
7998 }
7999
8000 /**
8001 * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding
8002 * netdev->dev_addrs
8003 * @netdev: network interface device structure
8004 *
8005 * Returns non-zero on failure
8006 **/
8007 static int ixgbe_del_sanmac_netdev(struct net_device *dev)
8008 {
8009 int err = 0;
8010 struct ixgbe_adapter *adapter = netdev_priv(dev);
8011 struct ixgbe_mac_info *mac = &adapter->hw.mac;
8012
8013 if (is_valid_ether_addr(mac->san_addr)) {
8014 rtnl_lock();
8015 err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
8016 rtnl_unlock();
8017 }
8018 return err;
8019 }
8020
8021 #ifdef CONFIG_NET_POLL_CONTROLLER
8022 /*
8023 * Polling 'interrupt' - used by things like netconsole to send skbs
8024 * without having to re-enable interrupts. It's not called while
8025 * the interrupt routine is executing.
8026 */
8027 static void ixgbe_netpoll(struct net_device *netdev)
8028 {
8029 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8030 int i;
8031
8032 /* if interface is down do nothing */
8033 if (test_bit(__IXGBE_DOWN, &adapter->state))
8034 return;
8035
8036 /* loop through and schedule all active queues */
8037 for (i = 0; i < adapter->num_q_vectors; i++)
8038 ixgbe_msix_clean_rings(0, adapter->q_vector[i]);
8039 }
8040
8041 #endif
8042 static struct rtnl_link_stats64 *ixgbe_get_stats64(struct net_device *netdev,
8043 struct rtnl_link_stats64 *stats)
8044 {
8045 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8046 int i;
8047
8048 rcu_read_lock();
8049 for (i = 0; i < adapter->num_rx_queues; i++) {
8050 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]);
8051 u64 bytes, packets;
8052 unsigned int start;
8053
8054 if (ring) {
8055 do {
8056 start = u64_stats_fetch_begin_irq(&ring->syncp);
8057 packets = ring->stats.packets;
8058 bytes = ring->stats.bytes;
8059 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
8060 stats->rx_packets += packets;
8061 stats->rx_bytes += bytes;
8062 }
8063 }
8064
8065 for (i = 0; i < adapter->num_tx_queues; i++) {
8066 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->tx_ring[i]);
8067 u64 bytes, packets;
8068 unsigned int start;
8069
8070 if (ring) {
8071 do {
8072 start = u64_stats_fetch_begin_irq(&ring->syncp);
8073 packets = ring->stats.packets;
8074 bytes = ring->stats.bytes;
8075 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
8076 stats->tx_packets += packets;
8077 stats->tx_bytes += bytes;
8078 }
8079 }
8080 rcu_read_unlock();
8081 /* following stats updated by ixgbe_watchdog_task() */
8082 stats->multicast = netdev->stats.multicast;
8083 stats->rx_errors = netdev->stats.rx_errors;
8084 stats->rx_length_errors = netdev->stats.rx_length_errors;
8085 stats->rx_crc_errors = netdev->stats.rx_crc_errors;
8086 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
8087 return stats;
8088 }
8089
8090 #ifdef CONFIG_IXGBE_DCB
8091 /**
8092 * ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid.
8093 * @adapter: pointer to ixgbe_adapter
8094 * @tc: number of traffic classes currently enabled
8095 *
8096 * Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm
8097 * 802.1Q priority maps to a packet buffer that exists.
8098 */
8099 static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc)
8100 {
8101 struct ixgbe_hw *hw = &adapter->hw;
8102 u32 reg, rsave;
8103 int i;
8104
8105 /* 82598 have a static priority to TC mapping that can not
8106 * be changed so no validation is needed.
8107 */
8108 if (hw->mac.type == ixgbe_mac_82598EB)
8109 return;
8110
8111 reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC);
8112 rsave = reg;
8113
8114 for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
8115 u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT);
8116
8117 /* If up2tc is out of bounds default to zero */
8118 if (up2tc > tc)
8119 reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT);
8120 }
8121
8122 if (reg != rsave)
8123 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg);
8124
8125 return;
8126 }
8127
8128 /**
8129 * ixgbe_set_prio_tc_map - Configure netdev prio tc map
8130 * @adapter: Pointer to adapter struct
8131 *
8132 * Populate the netdev user priority to tc map
8133 */
8134 static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter)
8135 {
8136 struct net_device *dev = adapter->netdev;
8137 struct ixgbe_dcb_config *dcb_cfg = &adapter->dcb_cfg;
8138 struct ieee_ets *ets = adapter->ixgbe_ieee_ets;
8139 u8 prio;
8140
8141 for (prio = 0; prio < MAX_USER_PRIORITY; prio++) {
8142 u8 tc = 0;
8143
8144 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE)
8145 tc = ixgbe_dcb_get_tc_from_up(dcb_cfg, 0, prio);
8146 else if (ets)
8147 tc = ets->prio_tc[prio];
8148
8149 netdev_set_prio_tc_map(dev, prio, tc);
8150 }
8151 }
8152
8153 #endif /* CONFIG_IXGBE_DCB */
8154 /**
8155 * ixgbe_setup_tc - configure net_device for multiple traffic classes
8156 *
8157 * @netdev: net device to configure
8158 * @tc: number of traffic classes to enable
8159 */
8160 int ixgbe_setup_tc(struct net_device *dev, u8 tc)
8161 {
8162 struct ixgbe_adapter *adapter = netdev_priv(dev);
8163 struct ixgbe_hw *hw = &adapter->hw;
8164 bool pools;
8165
8166 /* Hardware supports up to 8 traffic classes */
8167 if (tc > adapter->dcb_cfg.num_tcs.pg_tcs)
8168 return -EINVAL;
8169
8170 if (hw->mac.type == ixgbe_mac_82598EB && tc && tc < MAX_TRAFFIC_CLASS)
8171 return -EINVAL;
8172
8173 pools = (find_first_zero_bit(&adapter->fwd_bitmask, 32) > 1);
8174 if (tc && pools && adapter->num_rx_pools > IXGBE_MAX_DCBMACVLANS)
8175 return -EBUSY;
8176
8177 /* Hardware has to reinitialize queues and interrupts to
8178 * match packet buffer alignment. Unfortunately, the
8179 * hardware is not flexible enough to do this dynamically.
8180 */
8181 if (netif_running(dev))
8182 ixgbe_close(dev);
8183 else
8184 ixgbe_reset(adapter);
8185
8186 ixgbe_clear_interrupt_scheme(adapter);
8187
8188 #ifdef CONFIG_IXGBE_DCB
8189 if (tc) {
8190 netdev_set_num_tc(dev, tc);
8191 ixgbe_set_prio_tc_map(adapter);
8192
8193 adapter->flags |= IXGBE_FLAG_DCB_ENABLED;
8194
8195 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
8196 adapter->last_lfc_mode = adapter->hw.fc.requested_mode;
8197 adapter->hw.fc.requested_mode = ixgbe_fc_none;
8198 }
8199 } else {
8200 netdev_reset_tc(dev);
8201
8202 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
8203 adapter->hw.fc.requested_mode = adapter->last_lfc_mode;
8204
8205 adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
8206
8207 adapter->temp_dcb_cfg.pfc_mode_enable = false;
8208 adapter->dcb_cfg.pfc_mode_enable = false;
8209 }
8210
8211 ixgbe_validate_rtr(adapter, tc);
8212
8213 #endif /* CONFIG_IXGBE_DCB */
8214 ixgbe_init_interrupt_scheme(adapter);
8215
8216 if (netif_running(dev))
8217 return ixgbe_open(dev);
8218
8219 return 0;
8220 }
8221
8222 static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter,
8223 struct tc_cls_u32_offload *cls)
8224 {
8225 u32 uhtid = TC_U32_USERHTID(cls->knode.handle);
8226 u32 loc;
8227 int err;
8228
8229 if ((uhtid != 0x800) && (uhtid >= IXGBE_MAX_LINK_HANDLE))
8230 return -EINVAL;
8231
8232 loc = cls->knode.handle & 0xfffff;
8233
8234 spin_lock(&adapter->fdir_perfect_lock);
8235 err = ixgbe_update_ethtool_fdir_entry(adapter, NULL, loc);
8236 spin_unlock(&adapter->fdir_perfect_lock);
8237 return err;
8238 }
8239
8240 static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter,
8241 __be16 protocol,
8242 struct tc_cls_u32_offload *cls)
8243 {
8244 u32 uhtid = TC_U32_USERHTID(cls->hnode.handle);
8245
8246 if (uhtid >= IXGBE_MAX_LINK_HANDLE)
8247 return -EINVAL;
8248
8249 /* This ixgbe devices do not support hash tables at the moment
8250 * so abort when given hash tables.
8251 */
8252 if (cls->hnode.divisor > 0)
8253 return -EINVAL;
8254
8255 set_bit(uhtid - 1, &adapter->tables);
8256 return 0;
8257 }
8258
8259 static int ixgbe_configure_clsu32_del_hnode(struct ixgbe_adapter *adapter,
8260 struct tc_cls_u32_offload *cls)
8261 {
8262 u32 uhtid = TC_U32_USERHTID(cls->hnode.handle);
8263
8264 if (uhtid >= IXGBE_MAX_LINK_HANDLE)
8265 return -EINVAL;
8266
8267 clear_bit(uhtid - 1, &adapter->tables);
8268 return 0;
8269 }
8270
8271 static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter,
8272 __be16 protocol,
8273 struct tc_cls_u32_offload *cls)
8274 {
8275 u32 loc = cls->knode.handle & 0xfffff;
8276 struct ixgbe_hw *hw = &adapter->hw;
8277 struct ixgbe_mat_field *field_ptr;
8278 struct ixgbe_fdir_filter *input;
8279 union ixgbe_atr_input mask;
8280 #ifdef CONFIG_NET_CLS_ACT
8281 const struct tc_action *a;
8282 #endif
8283 int i, err = 0;
8284 u8 queue;
8285 u32 uhtid, link_uhtid;
8286
8287 memset(&mask, 0, sizeof(union ixgbe_atr_input));
8288 uhtid = TC_U32_USERHTID(cls->knode.handle);
8289 link_uhtid = TC_U32_USERHTID(cls->knode.link_handle);
8290
8291 /* At the moment cls_u32 jumps to network layer and skips past
8292 * L2 headers. The canonical method to match L2 frames is to use
8293 * negative values. However this is error prone at best but really
8294 * just broken because there is no way to "know" what sort of hdr
8295 * is in front of the network layer. Fix cls_u32 to support L2
8296 * headers when needed.
8297 */
8298 if (protocol != htons(ETH_P_IP))
8299 return -EINVAL;
8300
8301 if (link_uhtid) {
8302 struct ixgbe_nexthdr *nexthdr = ixgbe_ipv4_jumps;
8303
8304 if (link_uhtid >= IXGBE_MAX_LINK_HANDLE)
8305 return -EINVAL;
8306
8307 if (!test_bit(link_uhtid - 1, &adapter->tables))
8308 return -EINVAL;
8309
8310 for (i = 0; nexthdr[i].jump; i++) {
8311 if (nexthdr[i].o != cls->knode.sel->offoff ||
8312 nexthdr[i].s != cls->knode.sel->offshift ||
8313 nexthdr[i].m != cls->knode.sel->offmask ||
8314 /* do not support multiple key jumps its just mad */
8315 cls->knode.sel->nkeys > 1)
8316 return -EINVAL;
8317
8318 if (nexthdr[i].off == cls->knode.sel->keys[0].off &&
8319 nexthdr[i].val == cls->knode.sel->keys[0].val &&
8320 nexthdr[i].mask == cls->knode.sel->keys[0].mask) {
8321 adapter->jump_tables[link_uhtid] =
8322 nexthdr[i].jump;
8323 break;
8324 }
8325 }
8326 return 0;
8327 }
8328
8329 if (loc >= ((1024 << adapter->fdir_pballoc) - 2)) {
8330 e_err(drv, "Location out of range\n");
8331 return -EINVAL;
8332 }
8333
8334 /* cls u32 is a graph starting at root node 0x800. The driver tracks
8335 * links and also the fields used to advance the parser across each
8336 * link (e.g. nexthdr/eat parameters from 'tc'). This way we can map
8337 * the u32 graph onto the hardware parse graph denoted in ixgbe_model.h
8338 * To add support for new nodes update ixgbe_model.h parse structures
8339 * this function _should_ be generic try not to hardcode values here.
8340 */
8341 if (uhtid == 0x800) {
8342 field_ptr = adapter->jump_tables[0];
8343 } else {
8344 if (uhtid >= IXGBE_MAX_LINK_HANDLE)
8345 return -EINVAL;
8346
8347 field_ptr = adapter->jump_tables[uhtid];
8348 }
8349
8350 if (!field_ptr)
8351 return -EINVAL;
8352
8353 input = kzalloc(sizeof(*input), GFP_KERNEL);
8354 if (!input)
8355 return -ENOMEM;
8356
8357 for (i = 0; i < cls->knode.sel->nkeys; i++) {
8358 int off = cls->knode.sel->keys[i].off;
8359 __be32 val = cls->knode.sel->keys[i].val;
8360 __be32 m = cls->knode.sel->keys[i].mask;
8361 bool found_entry = false;
8362 int j;
8363
8364 for (j = 0; field_ptr[j].val; j++) {
8365 if (field_ptr[j].off == off) {
8366 field_ptr[j].val(input, &mask, val, m);
8367 input->filter.formatted.flow_type |=
8368 field_ptr[j].type;
8369 found_entry = true;
8370 break;
8371 }
8372 }
8373
8374 if (!found_entry)
8375 goto err_out;
8376 }
8377
8378 mask.formatted.flow_type = IXGBE_ATR_L4TYPE_IPV6_MASK |
8379 IXGBE_ATR_L4TYPE_MASK;
8380
8381 if (input->filter.formatted.flow_type == IXGBE_ATR_FLOW_TYPE_IPV4)
8382 mask.formatted.flow_type &= IXGBE_ATR_L4TYPE_IPV6_MASK;
8383
8384 #ifdef CONFIG_NET_CLS_ACT
8385 if (list_empty(&cls->knode.exts->actions))
8386 goto err_out;
8387
8388 list_for_each_entry(a, &cls->knode.exts->actions, list) {
8389 if (!is_tcf_gact_shot(a))
8390 goto err_out;
8391 }
8392 #endif
8393
8394 input->action = IXGBE_FDIR_DROP_QUEUE;
8395 queue = IXGBE_FDIR_DROP_QUEUE;
8396 input->sw_idx = loc;
8397
8398 spin_lock(&adapter->fdir_perfect_lock);
8399
8400 if (hlist_empty(&adapter->fdir_filter_list)) {
8401 memcpy(&adapter->fdir_mask, &mask, sizeof(mask));
8402 err = ixgbe_fdir_set_input_mask_82599(hw, &mask);
8403 if (err)
8404 goto err_out_w_lock;
8405 } else if (memcmp(&adapter->fdir_mask, &mask, sizeof(mask))) {
8406 err = -EINVAL;
8407 goto err_out_w_lock;
8408 }
8409
8410 ixgbe_atr_compute_perfect_hash_82599(&input->filter, &mask);
8411 err = ixgbe_fdir_write_perfect_filter_82599(hw, &input->filter,
8412 input->sw_idx, queue);
8413 if (!err)
8414 ixgbe_update_ethtool_fdir_entry(adapter, input, input->sw_idx);
8415 spin_unlock(&adapter->fdir_perfect_lock);
8416
8417 return err;
8418 err_out_w_lock:
8419 spin_unlock(&adapter->fdir_perfect_lock);
8420 err_out:
8421 kfree(input);
8422 return -EINVAL;
8423 }
8424
8425 static int __ixgbe_setup_tc(struct net_device *dev, u32 handle, __be16 proto,
8426 struct tc_to_netdev *tc)
8427 {
8428 struct ixgbe_adapter *adapter = netdev_priv(dev);
8429
8430 if (TC_H_MAJ(handle) == TC_H_MAJ(TC_H_INGRESS) &&
8431 tc->type == TC_SETUP_CLSU32) {
8432 switch (tc->cls_u32->command) {
8433 case TC_CLSU32_NEW_KNODE:
8434 case TC_CLSU32_REPLACE_KNODE:
8435 return ixgbe_configure_clsu32(adapter,
8436 proto, tc->cls_u32);
8437 case TC_CLSU32_DELETE_KNODE:
8438 return ixgbe_delete_clsu32(adapter, tc->cls_u32);
8439 case TC_CLSU32_NEW_HNODE:
8440 case TC_CLSU32_REPLACE_HNODE:
8441 return ixgbe_configure_clsu32_add_hnode(adapter, proto,
8442 tc->cls_u32);
8443 case TC_CLSU32_DELETE_HNODE:
8444 return ixgbe_configure_clsu32_del_hnode(adapter,
8445 tc->cls_u32);
8446 default:
8447 return -EINVAL;
8448 }
8449 }
8450
8451 if (tc->type != TC_SETUP_MQPRIO)
8452 return -EINVAL;
8453
8454 return ixgbe_setup_tc(dev, tc->tc);
8455 }
8456
8457 #ifdef CONFIG_PCI_IOV
8458 void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter)
8459 {
8460 struct net_device *netdev = adapter->netdev;
8461
8462 rtnl_lock();
8463 ixgbe_setup_tc(netdev, netdev_get_num_tc(netdev));
8464 rtnl_unlock();
8465 }
8466
8467 #endif
8468 void ixgbe_do_reset(struct net_device *netdev)
8469 {
8470 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8471
8472 if (netif_running(netdev))
8473 ixgbe_reinit_locked(adapter);
8474 else
8475 ixgbe_reset(adapter);
8476 }
8477
8478 static netdev_features_t ixgbe_fix_features(struct net_device *netdev,
8479 netdev_features_t features)
8480 {
8481 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8482
8483 /* If Rx checksum is disabled, then RSC/LRO should also be disabled */
8484 if (!(features & NETIF_F_RXCSUM))
8485 features &= ~NETIF_F_LRO;
8486
8487 /* Turn off LRO if not RSC capable */
8488 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE))
8489 features &= ~NETIF_F_LRO;
8490
8491 return features;
8492 }
8493
8494 static int ixgbe_set_features(struct net_device *netdev,
8495 netdev_features_t features)
8496 {
8497 struct ixgbe_adapter *adapter = netdev_priv(netdev);
8498 netdev_features_t changed = netdev->features ^ features;
8499 bool need_reset = false;
8500
8501 /* Make sure RSC matches LRO, reset if change */
8502 if (!(features & NETIF_F_LRO)) {
8503 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
8504 need_reset = true;
8505 adapter->flags2 &= ~IXGBE_FLAG2_RSC_ENABLED;
8506 } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) &&
8507 !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) {
8508 if (adapter->rx_itr_setting == 1 ||
8509 adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) {
8510 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
8511 need_reset = true;
8512 } else if ((changed ^ features) & NETIF_F_LRO) {
8513 e_info(probe, "rx-usecs set too low, "
8514 "disabling RSC\n");
8515 }
8516 }
8517
8518 /*
8519 * Check if Flow Director n-tuple support or hw_tc support was
8520 * enabled or disabled. If the state changed, we need to reset.
8521 */
8522 if ((features & NETIF_F_NTUPLE) || (features & NETIF_F_HW_TC)) {
8523 /* turn off ATR, enable perfect filters and reset */
8524 if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE))
8525 need_reset = true;
8526
8527 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
8528 adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
8529 } else {
8530 /* turn off perfect filters, enable ATR and reset */
8531 if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
8532 need_reset = true;
8533
8534 adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
8535
8536 /* We cannot enable ATR if SR-IOV is enabled */
8537 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED ||
8538 /* We cannot enable ATR if we have 2 or more tcs */
8539 (netdev_get_num_tc(netdev) > 1) ||
8540 /* We cannot enable ATR if RSS is disabled */
8541 (adapter->ring_feature[RING_F_RSS].limit <= 1) ||
8542 /* A sample rate of 0 indicates ATR disabled */
8543 (!adapter->atr_sample_rate))
8544 ; /* do nothing not supported */
8545 else /* otherwise supported and set the flag */
8546 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
8547 }
8548
8549 if (changed & NETIF_F_RXALL)
8550 need_reset = true;
8551
8552 netdev->features = features;
8553
8554 #ifdef CONFIG_IXGBE_VXLAN
8555 if ((adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE)) {
8556 if (features & NETIF_F_RXCSUM)
8557 adapter->flags2 |= IXGBE_FLAG2_VXLAN_REREG_NEEDED;
8558 else
8559 ixgbe_clear_vxlan_port(adapter);
8560 }
8561 #endif /* CONFIG_IXGBE_VXLAN */
8562
8563 if (need_reset)
8564 ixgbe_do_reset(netdev);
8565 else if (changed & (NETIF_F_HW_VLAN_CTAG_RX |
8566 NETIF_F_HW_VLAN_CTAG_FILTER))
8567 ixgbe_set_rx_mode(netdev);
8568
8569 return 0;
8570 }
8571
8572 #ifdef CONFIG_IXGBE_VXLAN
8573 /**
8574 * ixgbe_add_vxlan_port - Get notifications about VXLAN ports that come up
8575 * @dev: The port's netdev
8576 * @sa_family: Socket Family that VXLAN is notifiying us about
8577 * @port: New UDP port number that VXLAN started listening to
8578 **/
8579 static void ixgbe_add_vxlan_port(struct net_device *dev, sa_family_t sa_family,
8580 __be16 port)
8581 {
8582 struct ixgbe_adapter *adapter = netdev_priv(dev);
8583 struct ixgbe_hw *hw = &adapter->hw;
8584
8585 if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE))
8586 return;
8587
8588 if (sa_family == AF_INET6)
8589 return;
8590
8591 if (adapter->vxlan_port == port)
8592 return;
8593
8594 if (adapter->vxlan_port) {
8595 netdev_info(dev,
8596 "Hit Max num of VXLAN ports, not adding port %d\n",
8597 ntohs(port));
8598 return;
8599 }
8600
8601 adapter->vxlan_port = port;
8602 IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL, ntohs(port));
8603 }
8604
8605 /**
8606 * ixgbe_del_vxlan_port - Get notifications about VXLAN ports that go away
8607 * @dev: The port's netdev
8608 * @sa_family: Socket Family that VXLAN is notifying us about
8609 * @port: UDP port number that VXLAN stopped listening to
8610 **/
8611 static void ixgbe_del_vxlan_port(struct net_device *dev, sa_family_t sa_family,
8612 __be16 port)
8613 {
8614 struct ixgbe_adapter *adapter = netdev_priv(dev);
8615
8616 if (!(adapter->flags & IXGBE_FLAG_VXLAN_OFFLOAD_CAPABLE))
8617 return;
8618
8619 if (sa_family == AF_INET6)
8620 return;
8621
8622 if (adapter->vxlan_port != port) {
8623 netdev_info(dev, "Port %d was not found, not deleting\n",
8624 ntohs(port));
8625 return;
8626 }
8627
8628 ixgbe_clear_vxlan_port(adapter);
8629 adapter->flags2 |= IXGBE_FLAG2_VXLAN_REREG_NEEDED;
8630 }
8631 #endif /* CONFIG_IXGBE_VXLAN */
8632
8633 static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
8634 struct net_device *dev,
8635 const unsigned char *addr, u16 vid,
8636 u16 flags)
8637 {
8638 /* guarantee we can provide a unique filter for the unicast address */
8639 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
8640 struct ixgbe_adapter *adapter = netdev_priv(dev);
8641 u16 pool = VMDQ_P(0);
8642
8643 if (netdev_uc_count(dev) >= ixgbe_available_rars(adapter, pool))
8644 return -ENOMEM;
8645 }
8646
8647 return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags);
8648 }
8649
8650 /**
8651 * ixgbe_configure_bridge_mode - set various bridge modes
8652 * @adapter - the private structure
8653 * @mode - requested bridge mode
8654 *
8655 * Configure some settings require for various bridge modes.
8656 **/
8657 static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter,
8658 __u16 mode)
8659 {
8660 struct ixgbe_hw *hw = &adapter->hw;
8661 unsigned int p, num_pools;
8662 u32 vmdctl;
8663
8664 switch (mode) {
8665 case BRIDGE_MODE_VEPA:
8666 /* disable Tx loopback, rely on switch hairpin mode */
8667 IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC, 0);
8668
8669 /* must enable Rx switching replication to allow multicast
8670 * packet reception on all VFs, and to enable source address
8671 * pruning.
8672 */
8673 vmdctl = IXGBE_READ_REG(hw, IXGBE_VMD_CTL);
8674 vmdctl |= IXGBE_VT_CTL_REPLEN;
8675 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
8676
8677 /* enable Rx source address pruning. Note, this requires
8678 * replication to be enabled or else it does nothing.
8679 */
8680 num_pools = adapter->num_vfs + adapter->num_rx_pools;
8681 for (p = 0; p < num_pools; p++) {
8682 if (hw->mac.ops.set_source_address_pruning)
8683 hw->mac.ops.set_source_address_pruning(hw,
8684 true,
8685 p);
8686 }
8687 break;
8688 case BRIDGE_MODE_VEB:
8689 /* enable Tx loopback for internal VF/PF communication */
8690 IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC,
8691 IXGBE_PFDTXGSWC_VT_LBEN);
8692
8693 /* disable Rx switching replication unless we have SR-IOV
8694 * virtual functions
8695 */
8696 vmdctl = IXGBE_READ_REG(hw, IXGBE_VMD_CTL);
8697 if (!adapter->num_vfs)
8698 vmdctl &= ~IXGBE_VT_CTL_REPLEN;
8699 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
8700
8701 /* disable Rx source address pruning, since we don't expect to
8702 * be receiving external loopback of our transmitted frames.
8703 */
8704 num_pools = adapter->num_vfs + adapter->num_rx_pools;
8705 for (p = 0; p < num_pools; p++) {
8706 if (hw->mac.ops.set_source_address_pruning)
8707 hw->mac.ops.set_source_address_pruning(hw,
8708 false,
8709 p);
8710 }
8711 break;
8712 default:
8713 return -EINVAL;
8714 }
8715
8716 adapter->bridge_mode = mode;
8717
8718 e_info(drv, "enabling bridge mode: %s\n",
8719 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
8720
8721 return 0;
8722 }
8723
8724 static int ixgbe_ndo_bridge_setlink(struct net_device *dev,
8725 struct nlmsghdr *nlh, u16 flags)
8726 {
8727 struct ixgbe_adapter *adapter = netdev_priv(dev);
8728 struct nlattr *attr, *br_spec;
8729 int rem;
8730
8731 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
8732 return -EOPNOTSUPP;
8733
8734 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
8735 if (!br_spec)
8736 return -EINVAL;
8737
8738 nla_for_each_nested(attr, br_spec, rem) {
8739 int status;
8740 __u16 mode;
8741
8742 if (nla_type(attr) != IFLA_BRIDGE_MODE)
8743 continue;
8744
8745 if (nla_len(attr) < sizeof(mode))
8746 return -EINVAL;
8747
8748 mode = nla_get_u16(attr);
8749 status = ixgbe_configure_bridge_mode(adapter, mode);
8750 if (status)
8751 return status;
8752
8753 break;
8754 }
8755
8756 return 0;
8757 }
8758
8759 static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
8760 struct net_device *dev,
8761 u32 filter_mask, int nlflags)
8762 {
8763 struct ixgbe_adapter *adapter = netdev_priv(dev);
8764
8765 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
8766 return 0;
8767
8768 return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
8769 adapter->bridge_mode, 0, 0, nlflags,
8770 filter_mask, NULL);
8771 }
8772
8773 static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev)
8774 {
8775 struct ixgbe_fwd_adapter *fwd_adapter = NULL;
8776 struct ixgbe_adapter *adapter = netdev_priv(pdev);
8777 int used_pools = adapter->num_vfs + adapter->num_rx_pools;
8778 unsigned int limit;
8779 int pool, err;
8780
8781 /* Hardware has a limited number of available pools. Each VF, and the
8782 * PF require a pool. Check to ensure we don't attempt to use more
8783 * then the available number of pools.
8784 */
8785 if (used_pools >= IXGBE_MAX_VF_FUNCTIONS)
8786 return ERR_PTR(-EINVAL);
8787
8788 #ifdef CONFIG_RPS
8789 if (vdev->num_rx_queues != vdev->num_tx_queues) {
8790 netdev_info(pdev, "%s: Only supports a single queue count for TX and RX\n",
8791 vdev->name);
8792 return ERR_PTR(-EINVAL);
8793 }
8794 #endif
8795 /* Check for hardware restriction on number of rx/tx queues */
8796 if (vdev->num_tx_queues > IXGBE_MAX_L2A_QUEUES ||
8797 vdev->num_tx_queues == IXGBE_BAD_L2A_QUEUE) {
8798 netdev_info(pdev,
8799 "%s: Supports RX/TX Queue counts 1,2, and 4\n",
8800 pdev->name);
8801 return ERR_PTR(-EINVAL);
8802 }
8803
8804 if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
8805 adapter->num_rx_pools > IXGBE_MAX_DCBMACVLANS - 1) ||
8806 (adapter->num_rx_pools > IXGBE_MAX_MACVLANS))
8807 return ERR_PTR(-EBUSY);
8808
8809 fwd_adapter = kzalloc(sizeof(*fwd_adapter), GFP_KERNEL);
8810 if (!fwd_adapter)
8811 return ERR_PTR(-ENOMEM);
8812
8813 pool = find_first_zero_bit(&adapter->fwd_bitmask, 32);
8814 adapter->num_rx_pools++;
8815 set_bit(pool, &adapter->fwd_bitmask);
8816 limit = find_last_bit(&adapter->fwd_bitmask, 32);
8817
8818 /* Enable VMDq flag so device will be set in VM mode */
8819 adapter->flags |= IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_SRIOV_ENABLED;
8820 adapter->ring_feature[RING_F_VMDQ].limit = limit + 1;
8821 adapter->ring_feature[RING_F_RSS].limit = vdev->num_tx_queues;
8822
8823 /* Force reinit of ring allocation with VMDQ enabled */
8824 err = ixgbe_setup_tc(pdev, netdev_get_num_tc(pdev));
8825 if (err)
8826 goto fwd_add_err;
8827 fwd_adapter->pool = pool;
8828 fwd_adapter->real_adapter = adapter;
8829 err = ixgbe_fwd_ring_up(vdev, fwd_adapter);
8830 if (err)
8831 goto fwd_add_err;
8832 netif_tx_start_all_queues(vdev);
8833 return fwd_adapter;
8834 fwd_add_err:
8835 /* unwind counter and free adapter struct */
8836 netdev_info(pdev,
8837 "%s: dfwd hardware acceleration failed\n", vdev->name);
8838 clear_bit(pool, &adapter->fwd_bitmask);
8839 adapter->num_rx_pools--;
8840 kfree(fwd_adapter);
8841 return ERR_PTR(err);
8842 }
8843
8844 static void ixgbe_fwd_del(struct net_device *pdev, void *priv)
8845 {
8846 struct ixgbe_fwd_adapter *fwd_adapter = priv;
8847 struct ixgbe_adapter *adapter = fwd_adapter->real_adapter;
8848 unsigned int limit;
8849
8850 clear_bit(fwd_adapter->pool, &adapter->fwd_bitmask);
8851 adapter->num_rx_pools--;
8852
8853 limit = find_last_bit(&adapter->fwd_bitmask, 32);
8854 adapter->ring_feature[RING_F_VMDQ].limit = limit + 1;
8855 ixgbe_fwd_ring_down(fwd_adapter->netdev, fwd_adapter);
8856 ixgbe_setup_tc(pdev, netdev_get_num_tc(pdev));
8857 netdev_dbg(pdev, "pool %i:%i queues %i:%i VSI bitmask %lx\n",
8858 fwd_adapter->pool, adapter->num_rx_pools,
8859 fwd_adapter->rx_base_queue,
8860 fwd_adapter->rx_base_queue + adapter->num_rx_queues_per_pool,
8861 adapter->fwd_bitmask);
8862 kfree(fwd_adapter);
8863 }
8864
8865 #define IXGBE_MAX_TUNNEL_HDR_LEN 80
8866 static netdev_features_t
8867 ixgbe_features_check(struct sk_buff *skb, struct net_device *dev,
8868 netdev_features_t features)
8869 {
8870 if (!skb->encapsulation)
8871 return features;
8872
8873 if (unlikely(skb_inner_mac_header(skb) - skb_transport_header(skb) >
8874 IXGBE_MAX_TUNNEL_HDR_LEN))
8875 return features & ~NETIF_F_CSUM_MASK;
8876
8877 return features;
8878 }
8879
8880 static const struct net_device_ops ixgbe_netdev_ops = {
8881 .ndo_open = ixgbe_open,
8882 .ndo_stop = ixgbe_close,
8883 .ndo_start_xmit = ixgbe_xmit_frame,
8884 .ndo_select_queue = ixgbe_select_queue,
8885 .ndo_set_rx_mode = ixgbe_set_rx_mode,
8886 .ndo_validate_addr = eth_validate_addr,
8887 .ndo_set_mac_address = ixgbe_set_mac,
8888 .ndo_change_mtu = ixgbe_change_mtu,
8889 .ndo_tx_timeout = ixgbe_tx_timeout,
8890 .ndo_set_tx_maxrate = ixgbe_tx_maxrate,
8891 .ndo_vlan_rx_add_vid = ixgbe_vlan_rx_add_vid,
8892 .ndo_vlan_rx_kill_vid = ixgbe_vlan_rx_kill_vid,
8893 .ndo_do_ioctl = ixgbe_ioctl,
8894 .ndo_set_vf_mac = ixgbe_ndo_set_vf_mac,
8895 .ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan,
8896 .ndo_set_vf_rate = ixgbe_ndo_set_vf_bw,
8897 .ndo_set_vf_spoofchk = ixgbe_ndo_set_vf_spoofchk,
8898 .ndo_set_vf_rss_query_en = ixgbe_ndo_set_vf_rss_query_en,
8899 .ndo_set_vf_trust = ixgbe_ndo_set_vf_trust,
8900 .ndo_get_vf_config = ixgbe_ndo_get_vf_config,
8901 .ndo_get_stats64 = ixgbe_get_stats64,
8902 .ndo_setup_tc = __ixgbe_setup_tc,
8903 #ifdef CONFIG_NET_POLL_CONTROLLER
8904 .ndo_poll_controller = ixgbe_netpoll,
8905 #endif
8906 #ifdef CONFIG_NET_RX_BUSY_POLL
8907 .ndo_busy_poll = ixgbe_low_latency_recv,
8908 #endif
8909 #ifdef IXGBE_FCOE
8910 .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get,
8911 .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target,
8912 .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put,
8913 .ndo_fcoe_enable = ixgbe_fcoe_enable,
8914 .ndo_fcoe_disable = ixgbe_fcoe_disable,
8915 .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn,
8916 .ndo_fcoe_get_hbainfo = ixgbe_fcoe_get_hbainfo,
8917 #endif /* IXGBE_FCOE */
8918 .ndo_set_features = ixgbe_set_features,
8919 .ndo_fix_features = ixgbe_fix_features,
8920 .ndo_fdb_add = ixgbe_ndo_fdb_add,
8921 .ndo_bridge_setlink = ixgbe_ndo_bridge_setlink,
8922 .ndo_bridge_getlink = ixgbe_ndo_bridge_getlink,
8923 .ndo_dfwd_add_station = ixgbe_fwd_add,
8924 .ndo_dfwd_del_station = ixgbe_fwd_del,
8925 #ifdef CONFIG_IXGBE_VXLAN
8926 .ndo_add_vxlan_port = ixgbe_add_vxlan_port,
8927 .ndo_del_vxlan_port = ixgbe_del_vxlan_port,
8928 #endif /* CONFIG_IXGBE_VXLAN */
8929 .ndo_features_check = ixgbe_features_check,
8930 };
8931
8932 /**
8933 * ixgbe_enumerate_functions - Get the number of ports this device has
8934 * @adapter: adapter structure
8935 *
8936 * This function enumerates the phsyical functions co-located on a single slot,
8937 * in order to determine how many ports a device has. This is most useful in
8938 * determining the required GT/s of PCIe bandwidth necessary for optimal
8939 * performance.
8940 **/
8941 static inline int ixgbe_enumerate_functions(struct ixgbe_adapter *adapter)
8942 {
8943 struct pci_dev *entry, *pdev = adapter->pdev;
8944 int physfns = 0;
8945
8946 /* Some cards can not use the generic count PCIe functions method,
8947 * because they are behind a parent switch, so we hardcode these with
8948 * the correct number of functions.
8949 */
8950 if (ixgbe_pcie_from_parent(&adapter->hw))
8951 physfns = 4;
8952
8953 list_for_each_entry(entry, &adapter->pdev->bus->devices, bus_list) {
8954 /* don't count virtual functions */
8955 if (entry->is_virtfn)
8956 continue;
8957
8958 /* When the devices on the bus don't all match our device ID,
8959 * we can't reliably determine the correct number of
8960 * functions. This can occur if a function has been direct
8961 * attached to a virtual machine using VT-d, for example. In
8962 * this case, simply return -1 to indicate this.
8963 */
8964 if ((entry->vendor != pdev->vendor) ||
8965 (entry->device != pdev->device))
8966 return -1;
8967
8968 physfns++;
8969 }
8970
8971 return physfns;
8972 }
8973
8974 /**
8975 * ixgbe_wol_supported - Check whether device supports WoL
8976 * @hw: hw specific details
8977 * @device_id: the device ID
8978 * @subdev_id: the subsystem device ID
8979 *
8980 * This function is used by probe and ethtool to determine
8981 * which devices have WoL support
8982 *
8983 **/
8984 int ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id,
8985 u16 subdevice_id)
8986 {
8987 struct ixgbe_hw *hw = &adapter->hw;
8988 u16 wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK;
8989 int is_wol_supported = 0;
8990
8991 switch (device_id) {
8992 case IXGBE_DEV_ID_82599_SFP:
8993 /* Only these subdevices could supports WOL */
8994 switch (subdevice_id) {
8995 case IXGBE_SUBDEV_ID_82599_SFP_WOL0:
8996 case IXGBE_SUBDEV_ID_82599_560FLR:
8997 /* only support first port */
8998 if (hw->bus.func != 0)
8999 break;
9000 case IXGBE_SUBDEV_ID_82599_SP_560FLR:
9001 case IXGBE_SUBDEV_ID_82599_SFP:
9002 case IXGBE_SUBDEV_ID_82599_RNDC:
9003 case IXGBE_SUBDEV_ID_82599_ECNA_DP:
9004 case IXGBE_SUBDEV_ID_82599_LOM_SFP:
9005 is_wol_supported = 1;
9006 break;
9007 }
9008 break;
9009 case IXGBE_DEV_ID_82599EN_SFP:
9010 /* Only this subdevice supports WOL */
9011 switch (subdevice_id) {
9012 case IXGBE_SUBDEV_ID_82599EN_SFP_OCP1:
9013 is_wol_supported = 1;
9014 break;
9015 }
9016 break;
9017 case IXGBE_DEV_ID_82599_COMBO_BACKPLANE:
9018 /* All except this subdevice support WOL */
9019 if (subdevice_id != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ)
9020 is_wol_supported = 1;
9021 break;
9022 case IXGBE_DEV_ID_82599_KX4:
9023 is_wol_supported = 1;
9024 break;
9025 case IXGBE_DEV_ID_X540T:
9026 case IXGBE_DEV_ID_X540T1:
9027 case IXGBE_DEV_ID_X550T:
9028 case IXGBE_DEV_ID_X550T1:
9029 case IXGBE_DEV_ID_X550EM_X_KX4:
9030 case IXGBE_DEV_ID_X550EM_X_KR:
9031 case IXGBE_DEV_ID_X550EM_X_10G_T:
9032 /* check eeprom to see if enabled wol */
9033 if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) ||
9034 ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) &&
9035 (hw->bus.func == 0))) {
9036 is_wol_supported = 1;
9037 }
9038 break;
9039 }
9040
9041 return is_wol_supported;
9042 }
9043
9044 /**
9045 * ixgbe_probe - Device Initialization Routine
9046 * @pdev: PCI device information struct
9047 * @ent: entry in ixgbe_pci_tbl
9048 *
9049 * Returns 0 on success, negative on failure
9050 *
9051 * ixgbe_probe initializes an adapter identified by a pci_dev structure.
9052 * The OS initialization, configuring of the adapter private structure,
9053 * and a hardware reset occur.
9054 **/
9055 static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
9056 {
9057 struct net_device *netdev;
9058 struct ixgbe_adapter *adapter = NULL;
9059 struct ixgbe_hw *hw;
9060 const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data];
9061 int i, err, pci_using_dac, expected_gts;
9062 unsigned int indices = MAX_TX_QUEUES;
9063 u8 part_str[IXGBE_PBANUM_LENGTH];
9064 bool disable_dev = false;
9065 #ifdef IXGBE_FCOE
9066 u16 device_caps;
9067 #endif
9068 u32 eec;
9069
9070 /* Catch broken hardware that put the wrong VF device ID in
9071 * the PCIe SR-IOV capability.
9072 */
9073 if (pdev->is_virtfn) {
9074 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
9075 pci_name(pdev), pdev->vendor, pdev->device);
9076 return -EINVAL;
9077 }
9078
9079 err = pci_enable_device_mem(pdev);
9080 if (err)
9081 return err;
9082
9083 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
9084 pci_using_dac = 1;
9085 } else {
9086 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
9087 if (err) {
9088 dev_err(&pdev->dev,
9089 "No usable DMA configuration, aborting\n");
9090 goto err_dma;
9091 }
9092 pci_using_dac = 0;
9093 }
9094
9095 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
9096 IORESOURCE_MEM), ixgbe_driver_name);
9097 if (err) {
9098 dev_err(&pdev->dev,
9099 "pci_request_selected_regions failed 0x%x\n", err);
9100 goto err_pci_reg;
9101 }
9102
9103 pci_enable_pcie_error_reporting(pdev);
9104
9105 pci_set_master(pdev);
9106 pci_save_state(pdev);
9107
9108 if (ii->mac == ixgbe_mac_82598EB) {
9109 #ifdef CONFIG_IXGBE_DCB
9110 /* 8 TC w/ 4 queues per TC */
9111 indices = 4 * MAX_TRAFFIC_CLASS;
9112 #else
9113 indices = IXGBE_MAX_RSS_INDICES;
9114 #endif
9115 }
9116
9117 netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices);
9118 if (!netdev) {
9119 err = -ENOMEM;
9120 goto err_alloc_etherdev;
9121 }
9122
9123 SET_NETDEV_DEV(netdev, &pdev->dev);
9124
9125 adapter = netdev_priv(netdev);
9126
9127 adapter->netdev = netdev;
9128 adapter->pdev = pdev;
9129 hw = &adapter->hw;
9130 hw->back = adapter;
9131 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
9132
9133 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
9134 pci_resource_len(pdev, 0));
9135 adapter->io_addr = hw->hw_addr;
9136 if (!hw->hw_addr) {
9137 err = -EIO;
9138 goto err_ioremap;
9139 }
9140
9141 netdev->netdev_ops = &ixgbe_netdev_ops;
9142 ixgbe_set_ethtool_ops(netdev);
9143 netdev->watchdog_timeo = 5 * HZ;
9144 strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
9145
9146 /* Setup hw api */
9147 hw->mac.ops = *ii->mac_ops;
9148 hw->mac.type = ii->mac;
9149 hw->mvals = ii->mvals;
9150
9151 /* EEPROM */
9152 hw->eeprom.ops = *ii->eeprom_ops;
9153 eec = IXGBE_READ_REG(hw, IXGBE_EEC(hw));
9154 if (ixgbe_removed(hw->hw_addr)) {
9155 err = -EIO;
9156 goto err_ioremap;
9157 }
9158 /* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */
9159 if (!(eec & (1 << 8)))
9160 hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic;
9161
9162 /* PHY */
9163 hw->phy.ops = *ii->phy_ops;
9164 hw->phy.sfp_type = ixgbe_sfp_type_unknown;
9165 /* ixgbe_identify_phy_generic will set prtad and mmds properly */
9166 hw->phy.mdio.prtad = MDIO_PRTAD_NONE;
9167 hw->phy.mdio.mmds = 0;
9168 hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
9169 hw->phy.mdio.dev = netdev;
9170 hw->phy.mdio.mdio_read = ixgbe_mdio_read;
9171 hw->phy.mdio.mdio_write = ixgbe_mdio_write;
9172
9173 ii->get_invariants(hw);
9174
9175 /* setup the private structure */
9176 err = ixgbe_sw_init(adapter);
9177 if (err)
9178 goto err_sw_init;
9179
9180 /* Make sure the SWFW semaphore is in a valid state */
9181 if (hw->mac.ops.init_swfw_sync)
9182 hw->mac.ops.init_swfw_sync(hw);
9183
9184 /* Make it possible the adapter to be woken up via WOL */
9185 switch (adapter->hw.mac.type) {
9186 case ixgbe_mac_82599EB:
9187 case ixgbe_mac_X540:
9188 case ixgbe_mac_X550:
9189 case ixgbe_mac_X550EM_x:
9190 case ixgbe_mac_x550em_a:
9191 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
9192 break;
9193 default:
9194 break;
9195 }
9196
9197 /*
9198 * If there is a fan on this device and it has failed log the
9199 * failure.
9200 */
9201 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
9202 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
9203 if (esdp & IXGBE_ESDP_SDP1)
9204 e_crit(probe, "Fan has stopped, replace the adapter\n");
9205 }
9206
9207 if (allow_unsupported_sfp)
9208 hw->allow_unsupported_sfp = allow_unsupported_sfp;
9209
9210 /* reset_hw fills in the perm_addr as well */
9211 hw->phy.reset_if_overtemp = true;
9212 err = hw->mac.ops.reset_hw(hw);
9213 hw->phy.reset_if_overtemp = false;
9214 if (err == IXGBE_ERR_SFP_NOT_PRESENT) {
9215 err = 0;
9216 } else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
9217 e_dev_err("failed to load because an unsupported SFP+ or QSFP module type was detected.\n");
9218 e_dev_err("Reload the driver after installing a supported module.\n");
9219 goto err_sw_init;
9220 } else if (err) {
9221 e_dev_err("HW Init failed: %d\n", err);
9222 goto err_sw_init;
9223 }
9224
9225 #ifdef CONFIG_PCI_IOV
9226 /* SR-IOV not supported on the 82598 */
9227 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
9228 goto skip_sriov;
9229 /* Mailbox */
9230 ixgbe_init_mbx_params_pf(hw);
9231 hw->mbx.ops = ii->mbx_ops;
9232 pci_sriov_set_totalvfs(pdev, IXGBE_MAX_VFS_DRV_LIMIT);
9233 ixgbe_enable_sriov(adapter);
9234 skip_sriov:
9235
9236 #endif
9237 netdev->features = NETIF_F_SG |
9238 NETIF_F_TSO |
9239 NETIF_F_TSO6 |
9240 NETIF_F_RXHASH |
9241 NETIF_F_RXCSUM |
9242 NETIF_F_HW_CSUM |
9243 NETIF_F_HW_VLAN_CTAG_TX |
9244 NETIF_F_HW_VLAN_CTAG_RX |
9245 NETIF_F_HW_VLAN_CTAG_FILTER;
9246
9247 if (hw->mac.type >= ixgbe_mac_82599EB)
9248 netdev->features |= NETIF_F_SCTP_CRC;
9249
9250 /* copy netdev features into list of user selectable features */
9251 netdev->hw_features |= netdev->features;
9252 netdev->hw_features |= NETIF_F_RXALL |
9253 NETIF_F_HW_L2FW_DOFFLOAD;
9254
9255 if (hw->mac.type >= ixgbe_mac_82599EB)
9256 netdev->hw_features |= NETIF_F_NTUPLE |
9257 NETIF_F_HW_TC;
9258
9259 netdev->vlan_features |= NETIF_F_SG |
9260 NETIF_F_TSO |
9261 NETIF_F_TSO6 |
9262 NETIF_F_HW_CSUM |
9263 NETIF_F_SCTP_CRC;
9264
9265 netdev->mpls_features |= NETIF_F_HW_CSUM;
9266 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
9267
9268 netdev->priv_flags |= IFF_UNICAST_FLT;
9269 netdev->priv_flags |= IFF_SUPP_NOFCS;
9270
9271 #ifdef CONFIG_IXGBE_DCB
9272 netdev->dcbnl_ops = &dcbnl_ops;
9273 #endif
9274
9275 #ifdef IXGBE_FCOE
9276 if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
9277 unsigned int fcoe_l;
9278
9279 if (hw->mac.ops.get_device_caps) {
9280 hw->mac.ops.get_device_caps(hw, &device_caps);
9281 if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS)
9282 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
9283 }
9284
9285
9286 fcoe_l = min_t(int, IXGBE_FCRETA_SIZE, num_online_cpus());
9287 adapter->ring_feature[RING_F_FCOE].limit = fcoe_l;
9288
9289 netdev->features |= NETIF_F_FSO |
9290 NETIF_F_FCOE_CRC;
9291
9292 netdev->vlan_features |= NETIF_F_FSO |
9293 NETIF_F_FCOE_CRC |
9294 NETIF_F_FCOE_MTU;
9295 }
9296 #endif /* IXGBE_FCOE */
9297 if (pci_using_dac) {
9298 netdev->features |= NETIF_F_HIGHDMA;
9299 netdev->vlan_features |= NETIF_F_HIGHDMA;
9300 }
9301
9302 if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)
9303 netdev->hw_features |= NETIF_F_LRO;
9304 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
9305 netdev->features |= NETIF_F_LRO;
9306
9307 /* make sure the EEPROM is good */
9308 if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) {
9309 e_dev_err("The EEPROM Checksum Is Not Valid\n");
9310 err = -EIO;
9311 goto err_sw_init;
9312 }
9313
9314 eth_platform_get_mac_address(&adapter->pdev->dev,
9315 adapter->hw.mac.perm_addr);
9316
9317 memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len);
9318
9319 if (!is_valid_ether_addr(netdev->dev_addr)) {
9320 e_dev_err("invalid MAC address\n");
9321 err = -EIO;
9322 goto err_sw_init;
9323 }
9324
9325 /* Set hw->mac.addr to permanent MAC address */
9326 ether_addr_copy(hw->mac.addr, hw->mac.perm_addr);
9327 ixgbe_mac_set_default_filter(adapter);
9328
9329 setup_timer(&adapter->service_timer, &ixgbe_service_timer,
9330 (unsigned long) adapter);
9331
9332 if (ixgbe_removed(hw->hw_addr)) {
9333 err = -EIO;
9334 goto err_sw_init;
9335 }
9336 INIT_WORK(&adapter->service_task, ixgbe_service_task);
9337 set_bit(__IXGBE_SERVICE_INITED, &adapter->state);
9338 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
9339
9340 err = ixgbe_init_interrupt_scheme(adapter);
9341 if (err)
9342 goto err_sw_init;
9343
9344 /* WOL not supported for all devices */
9345 adapter->wol = 0;
9346 hw->eeprom.ops.read(hw, 0x2c, &adapter->eeprom_cap);
9347 hw->wol_enabled = ixgbe_wol_supported(adapter, pdev->device,
9348 pdev->subsystem_device);
9349 if (hw->wol_enabled)
9350 adapter->wol = IXGBE_WUFC_MAG;
9351
9352 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
9353
9354 /* save off EEPROM version number */
9355 hw->eeprom.ops.read(hw, 0x2e, &adapter->eeprom_verh);
9356 hw->eeprom.ops.read(hw, 0x2d, &adapter->eeprom_verl);
9357
9358 /* pick up the PCI bus settings for reporting later */
9359 if (ixgbe_pcie_from_parent(hw))
9360 ixgbe_get_parent_bus_info(adapter);
9361 else
9362 hw->mac.ops.get_bus_info(hw);
9363
9364 /* calculate the expected PCIe bandwidth required for optimal
9365 * performance. Note that some older parts will never have enough
9366 * bandwidth due to being older generation PCIe parts. We clamp these
9367 * parts to ensure no warning is displayed if it can't be fixed.
9368 */
9369 switch (hw->mac.type) {
9370 case ixgbe_mac_82598EB:
9371 expected_gts = min(ixgbe_enumerate_functions(adapter) * 10, 16);
9372 break;
9373 default:
9374 expected_gts = ixgbe_enumerate_functions(adapter) * 10;
9375 break;
9376 }
9377
9378 /* don't check link if we failed to enumerate functions */
9379 if (expected_gts > 0)
9380 ixgbe_check_minimum_link(adapter, expected_gts);
9381
9382 err = ixgbe_read_pba_string_generic(hw, part_str, sizeof(part_str));
9383 if (err)
9384 strlcpy(part_str, "Unknown", sizeof(part_str));
9385 if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
9386 e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n",
9387 hw->mac.type, hw->phy.type, hw->phy.sfp_type,
9388 part_str);
9389 else
9390 e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n",
9391 hw->mac.type, hw->phy.type, part_str);
9392
9393 e_dev_info("%pM\n", netdev->dev_addr);
9394
9395 /* reset the hardware with the new settings */
9396 err = hw->mac.ops.start_hw(hw);
9397 if (err == IXGBE_ERR_EEPROM_VERSION) {
9398 /* We are running on a pre-production device, log a warning */
9399 e_dev_warn("This device is a pre-production adapter/LOM. "
9400 "Please be aware there may be issues associated "
9401 "with your hardware. If you are experiencing "
9402 "problems please contact your Intel or hardware "
9403 "representative who provided you with this "
9404 "hardware.\n");
9405 }
9406 strcpy(netdev->name, "eth%d");
9407 err = register_netdev(netdev);
9408 if (err)
9409 goto err_register;
9410
9411 pci_set_drvdata(pdev, adapter);
9412
9413 /* power down the optics for 82599 SFP+ fiber */
9414 if (hw->mac.ops.disable_tx_laser)
9415 hw->mac.ops.disable_tx_laser(hw);
9416
9417 /* carrier off reporting is important to ethtool even BEFORE open */
9418 netif_carrier_off(netdev);
9419
9420 #ifdef CONFIG_IXGBE_DCA
9421 if (dca_add_requester(&pdev->dev) == 0) {
9422 adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
9423 ixgbe_setup_dca(adapter);
9424 }
9425 #endif
9426 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
9427 e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs);
9428 for (i = 0; i < adapter->num_vfs; i++)
9429 ixgbe_vf_configuration(pdev, (i | 0x10000000));
9430 }
9431
9432 /* firmware requires driver version to be 0xFFFFFFFF
9433 * since os does not support feature
9434 */
9435 if (hw->mac.ops.set_fw_drv_ver)
9436 hw->mac.ops.set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF,
9437 0xFF);
9438
9439 /* add san mac addr to netdev */
9440 ixgbe_add_sanmac_netdev(netdev);
9441
9442 e_dev_info("%s\n", ixgbe_default_device_descr);
9443
9444 #ifdef CONFIG_IXGBE_HWMON
9445 if (ixgbe_sysfs_init(adapter))
9446 e_err(probe, "failed to allocate sysfs resources\n");
9447 #endif /* CONFIG_IXGBE_HWMON */
9448
9449 ixgbe_dbg_adapter_init(adapter);
9450
9451 /* setup link for SFP devices with MNG FW, else wait for IXGBE_UP */
9452 if (ixgbe_mng_enabled(hw) && ixgbe_is_sfp(hw) && hw->mac.ops.setup_link)
9453 hw->mac.ops.setup_link(hw,
9454 IXGBE_LINK_SPEED_10GB_FULL | IXGBE_LINK_SPEED_1GB_FULL,
9455 true);
9456
9457 return 0;
9458
9459 err_register:
9460 ixgbe_release_hw_control(adapter);
9461 ixgbe_clear_interrupt_scheme(adapter);
9462 err_sw_init:
9463 ixgbe_disable_sriov(adapter);
9464 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
9465 iounmap(adapter->io_addr);
9466 kfree(adapter->mac_table);
9467 err_ioremap:
9468 disable_dev = !test_and_set_bit(__IXGBE_DISABLED, &adapter->state);
9469 free_netdev(netdev);
9470 err_alloc_etherdev:
9471 pci_release_selected_regions(pdev,
9472 pci_select_bars(pdev, IORESOURCE_MEM));
9473 err_pci_reg:
9474 err_dma:
9475 if (!adapter || disable_dev)
9476 pci_disable_device(pdev);
9477 return err;
9478 }
9479
9480 /**
9481 * ixgbe_remove - Device Removal Routine
9482 * @pdev: PCI device information struct
9483 *
9484 * ixgbe_remove is called by the PCI subsystem to alert the driver
9485 * that it should release a PCI device. The could be caused by a
9486 * Hot-Plug event, or because the driver is going to be removed from
9487 * memory.
9488 **/
9489 static void ixgbe_remove(struct pci_dev *pdev)
9490 {
9491 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
9492 struct net_device *netdev;
9493 bool disable_dev;
9494
9495 /* if !adapter then we already cleaned up in probe */
9496 if (!adapter)
9497 return;
9498
9499 netdev = adapter->netdev;
9500 ixgbe_dbg_adapter_exit(adapter);
9501
9502 set_bit(__IXGBE_REMOVING, &adapter->state);
9503 cancel_work_sync(&adapter->service_task);
9504
9505
9506 #ifdef CONFIG_IXGBE_DCA
9507 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
9508 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
9509 dca_remove_requester(&pdev->dev);
9510 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
9511 IXGBE_DCA_CTRL_DCA_DISABLE);
9512 }
9513
9514 #endif
9515 #ifdef CONFIG_IXGBE_HWMON
9516 ixgbe_sysfs_exit(adapter);
9517 #endif /* CONFIG_IXGBE_HWMON */
9518
9519 /* remove the added san mac */
9520 ixgbe_del_sanmac_netdev(netdev);
9521
9522 #ifdef CONFIG_PCI_IOV
9523 ixgbe_disable_sriov(adapter);
9524 #endif
9525 if (netdev->reg_state == NETREG_REGISTERED)
9526 unregister_netdev(netdev);
9527
9528 ixgbe_clear_interrupt_scheme(adapter);
9529
9530 ixgbe_release_hw_control(adapter);
9531
9532 #ifdef CONFIG_DCB
9533 kfree(adapter->ixgbe_ieee_pfc);
9534 kfree(adapter->ixgbe_ieee_ets);
9535
9536 #endif
9537 iounmap(adapter->io_addr);
9538 pci_release_selected_regions(pdev, pci_select_bars(pdev,
9539 IORESOURCE_MEM));
9540
9541 e_dev_info("complete\n");
9542
9543 kfree(adapter->mac_table);
9544 disable_dev = !test_and_set_bit(__IXGBE_DISABLED, &adapter->state);
9545 free_netdev(netdev);
9546
9547 pci_disable_pcie_error_reporting(pdev);
9548
9549 if (disable_dev)
9550 pci_disable_device(pdev);
9551 }
9552
9553 /**
9554 * ixgbe_io_error_detected - called when PCI error is detected
9555 * @pdev: Pointer to PCI device
9556 * @state: The current pci connection state
9557 *
9558 * This function is called after a PCI bus error affecting
9559 * this device has been detected.
9560 */
9561 static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,
9562 pci_channel_state_t state)
9563 {
9564 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
9565 struct net_device *netdev = adapter->netdev;
9566
9567 #ifdef CONFIG_PCI_IOV
9568 struct ixgbe_hw *hw = &adapter->hw;
9569 struct pci_dev *bdev, *vfdev;
9570 u32 dw0, dw1, dw2, dw3;
9571 int vf, pos;
9572 u16 req_id, pf_func;
9573
9574 if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
9575 adapter->num_vfs == 0)
9576 goto skip_bad_vf_detection;
9577
9578 bdev = pdev->bus->self;
9579 while (bdev && (pci_pcie_type(bdev) != PCI_EXP_TYPE_ROOT_PORT))
9580 bdev = bdev->bus->self;
9581
9582 if (!bdev)
9583 goto skip_bad_vf_detection;
9584
9585 pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR);
9586 if (!pos)
9587 goto skip_bad_vf_detection;
9588
9589 dw0 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG);
9590 dw1 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG + 4);
9591 dw2 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG + 8);
9592 dw3 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG + 12);
9593 if (ixgbe_removed(hw->hw_addr))
9594 goto skip_bad_vf_detection;
9595
9596 req_id = dw1 >> 16;
9597 /* On the 82599 if bit 7 of the requestor ID is set then it's a VF */
9598 if (!(req_id & 0x0080))
9599 goto skip_bad_vf_detection;
9600
9601 pf_func = req_id & 0x01;
9602 if ((pf_func & 1) == (pdev->devfn & 1)) {
9603 unsigned int device_id;
9604
9605 vf = (req_id & 0x7F) >> 1;
9606 e_dev_err("VF %d has caused a PCIe error\n", vf);
9607 e_dev_err("TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "
9608 "%8.8x\tdw3: %8.8x\n",
9609 dw0, dw1, dw2, dw3);
9610 switch (adapter->hw.mac.type) {
9611 case ixgbe_mac_82599EB:
9612 device_id = IXGBE_82599_VF_DEVICE_ID;
9613 break;
9614 case ixgbe_mac_X540:
9615 device_id = IXGBE_X540_VF_DEVICE_ID;
9616 break;
9617 case ixgbe_mac_X550:
9618 device_id = IXGBE_DEV_ID_X550_VF;
9619 break;
9620 case ixgbe_mac_X550EM_x:
9621 device_id = IXGBE_DEV_ID_X550EM_X_VF;
9622 break;
9623 case ixgbe_mac_x550em_a:
9624 device_id = IXGBE_DEV_ID_X550EM_A_VF;
9625 break;
9626 default:
9627 device_id = 0;
9628 break;
9629 }
9630
9631 /* Find the pci device of the offending VF */
9632 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, device_id, NULL);
9633 while (vfdev) {
9634 if (vfdev->devfn == (req_id & 0xFF))
9635 break;
9636 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
9637 device_id, vfdev);
9638 }
9639 /*
9640 * There's a slim chance the VF could have been hot plugged,
9641 * so if it is no longer present we don't need to issue the
9642 * VFLR. Just clean up the AER in that case.
9643 */
9644 if (vfdev) {
9645 ixgbe_issue_vf_flr(adapter, vfdev);
9646 /* Free device reference count */
9647 pci_dev_put(vfdev);
9648 }
9649
9650 pci_cleanup_aer_uncorrect_error_status(pdev);
9651 }
9652
9653 /*
9654 * Even though the error may have occurred on the other port
9655 * we still need to increment the vf error reference count for
9656 * both ports because the I/O resume function will be called
9657 * for both of them.
9658 */
9659 adapter->vferr_refcount++;
9660
9661 return PCI_ERS_RESULT_RECOVERED;
9662
9663 skip_bad_vf_detection:
9664 #endif /* CONFIG_PCI_IOV */
9665 if (!test_bit(__IXGBE_SERVICE_INITED, &adapter->state))
9666 return PCI_ERS_RESULT_DISCONNECT;
9667
9668 rtnl_lock();
9669 netif_device_detach(netdev);
9670
9671 if (state == pci_channel_io_perm_failure) {
9672 rtnl_unlock();
9673 return PCI_ERS_RESULT_DISCONNECT;
9674 }
9675
9676 if (netif_running(netdev))
9677 ixgbe_down(adapter);
9678
9679 if (!test_and_set_bit(__IXGBE_DISABLED, &adapter->state))
9680 pci_disable_device(pdev);
9681 rtnl_unlock();
9682
9683 /* Request a slot reset. */
9684 return PCI_ERS_RESULT_NEED_RESET;
9685 }
9686
9687 /**
9688 * ixgbe_io_slot_reset - called after the pci bus has been reset.
9689 * @pdev: Pointer to PCI device
9690 *
9691 * Restart the card from scratch, as if from a cold-boot.
9692 */
9693 static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev)
9694 {
9695 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
9696 pci_ers_result_t result;
9697 int err;
9698
9699 if (pci_enable_device_mem(pdev)) {
9700 e_err(probe, "Cannot re-enable PCI device after reset.\n");
9701 result = PCI_ERS_RESULT_DISCONNECT;
9702 } else {
9703 smp_mb__before_atomic();
9704 clear_bit(__IXGBE_DISABLED, &adapter->state);
9705 adapter->hw.hw_addr = adapter->io_addr;
9706 pci_set_master(pdev);
9707 pci_restore_state(pdev);
9708 pci_save_state(pdev);
9709
9710 pci_wake_from_d3(pdev, false);
9711
9712 ixgbe_reset(adapter);
9713 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
9714 result = PCI_ERS_RESULT_RECOVERED;
9715 }
9716
9717 err = pci_cleanup_aer_uncorrect_error_status(pdev);
9718 if (err) {
9719 e_dev_err("pci_cleanup_aer_uncorrect_error_status "
9720 "failed 0x%0x\n", err);
9721 /* non-fatal, continue */
9722 }
9723
9724 return result;
9725 }
9726
9727 /**
9728 * ixgbe_io_resume - called when traffic can start flowing again.
9729 * @pdev: Pointer to PCI device
9730 *
9731 * This callback is called when the error recovery driver tells us that
9732 * its OK to resume normal operation.
9733 */
9734 static void ixgbe_io_resume(struct pci_dev *pdev)
9735 {
9736 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
9737 struct net_device *netdev = adapter->netdev;
9738
9739 #ifdef CONFIG_PCI_IOV
9740 if (adapter->vferr_refcount) {
9741 e_info(drv, "Resuming after VF err\n");
9742 adapter->vferr_refcount--;
9743 return;
9744 }
9745
9746 #endif
9747 if (netif_running(netdev))
9748 ixgbe_up(adapter);
9749
9750 netif_device_attach(netdev);
9751 }
9752
9753 static const struct pci_error_handlers ixgbe_err_handler = {
9754 .error_detected = ixgbe_io_error_detected,
9755 .slot_reset = ixgbe_io_slot_reset,
9756 .resume = ixgbe_io_resume,
9757 };
9758
9759 static struct pci_driver ixgbe_driver = {
9760 .name = ixgbe_driver_name,
9761 .id_table = ixgbe_pci_tbl,
9762 .probe = ixgbe_probe,
9763 .remove = ixgbe_remove,
9764 #ifdef CONFIG_PM
9765 .suspend = ixgbe_suspend,
9766 .resume = ixgbe_resume,
9767 #endif
9768 .shutdown = ixgbe_shutdown,
9769 .sriov_configure = ixgbe_pci_sriov_configure,
9770 .err_handler = &ixgbe_err_handler
9771 };
9772
9773 /**
9774 * ixgbe_init_module - Driver Registration Routine
9775 *
9776 * ixgbe_init_module is the first routine called when the driver is
9777 * loaded. All it does is register with the PCI subsystem.
9778 **/
9779 static int __init ixgbe_init_module(void)
9780 {
9781 int ret;
9782 pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version);
9783 pr_info("%s\n", ixgbe_copyright);
9784
9785 ixgbe_wq = create_singlethread_workqueue(ixgbe_driver_name);
9786 if (!ixgbe_wq) {
9787 pr_err("%s: Failed to create workqueue\n", ixgbe_driver_name);
9788 return -ENOMEM;
9789 }
9790
9791 ixgbe_dbg_init();
9792
9793 ret = pci_register_driver(&ixgbe_driver);
9794 if (ret) {
9795 ixgbe_dbg_exit();
9796 return ret;
9797 }
9798
9799 #ifdef CONFIG_IXGBE_DCA
9800 dca_register_notify(&dca_notifier);
9801 #endif
9802
9803 return 0;
9804 }
9805
9806 module_init(ixgbe_init_module);
9807
9808 /**
9809 * ixgbe_exit_module - Driver Exit Cleanup Routine
9810 *
9811 * ixgbe_exit_module is called just before the driver is removed
9812 * from memory.
9813 **/
9814 static void __exit ixgbe_exit_module(void)
9815 {
9816 #ifdef CONFIG_IXGBE_DCA
9817 dca_unregister_notify(&dca_notifier);
9818 #endif
9819 pci_unregister_driver(&ixgbe_driver);
9820
9821 ixgbe_dbg_exit();
9822 if (ixgbe_wq) {
9823 destroy_workqueue(ixgbe_wq);
9824 ixgbe_wq = NULL;
9825 }
9826 }
9827
9828 #ifdef CONFIG_IXGBE_DCA
9829 static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event,
9830 void *p)
9831 {
9832 int ret_val;
9833
9834 ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event,
9835 __ixgbe_notify_dca);
9836
9837 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
9838 }
9839
9840 #endif /* CONFIG_IXGBE_DCA */
9841
9842 module_exit(ixgbe_exit_module);
9843
9844 /* ixgbe_main.c */
Linux kernel - Deliverables
This page took 0.42314 seconds and 6 git commands to generate.