projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ixgbevf: Cleanup variable usage, improve stack performance
[deliverable/linux.git] / drivers / net / ethernet / intel / ixgbevf / ixgbevf_main.c
CommitLineData
92915f71
GR		 1/*******************************************************************************
2
3 Intel 82599 Virtual Function driver
2e7cfbdd 4 Copyright(c) 1999 - 2014 Intel Corporation.
92915f71
GR		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28
29/******************************************************************************
30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
31******************************************************************************/
dbd9636e
JK		 32
33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34
92915f71 35#include <linux/types.h>
dadcd65f 36#include <linux/bitops.h>
92915f71
GR		 37#include <linux/module.h>
38#include <linux/pci.h>
39#include <linux/netdevice.h>
40#include <linux/vmalloc.h>
41#include <linux/string.h>
42#include <linux/in.h>
43#include <linux/ip.h>
44#include <linux/tcp.h>
70a10e25 45#include <linux/sctp.h>
92915f71 46#include <linux/ipv6.h>
5a0e3ad6 47#include <linux/slab.h>
92915f71
GR		 48#include <net/checksum.h>
49#include <net/ip6_checksum.h>
50#include <linux/ethtool.h>
01789349 51#include <linux/if.h>
92915f71 52#include <linux/if_vlan.h>
70c71606 53#include <linux/prefetch.h>
92915f71
GR		 54
55#include "ixgbevf.h"
56
3d8fe98f 57const char ixgbevf_driver_name[] = "ixgbevf";
92915f71 58static const char ixgbevf_driver_string[] =
422e05d1 59 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
92915f71 60
86f359f6 61#define DRV_VERSION "2.12.1-k"
92915f71 62const char ixgbevf_driver_version[] = DRV_VERSION;
66c87bd5 63static char ixgbevf_copyright[] =
5c47a2b6 64 "Copyright (c) 2009 - 2012 Intel Corporation.";
92915f71
GR		 65
66static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
2316aa2a
GR		 67 [board_82599_vf] = &ixgbevf_82599_vf_info,
68 [board_X540_vf] = &ixgbevf_X540_vf_info,
92915f71
GR		 69};
70
71/* ixgbevf_pci_tbl - PCI Device ID Table
72 *
73 * Wildcard entries (PCI_ANY_ID) should come last
74 * Last entry must be all 0s
75 *
76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
77 * Class, Class Mask, private data (not used) }
78 */
9baa3c34 79static const struct pci_device_id ixgbevf_pci_tbl[] = {
39ba22b4
SH		 80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
81 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
92915f71
GR		 82 /* required last entry */
83 {0, }
84};
85MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
86
87MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
b8ce18cd 88MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver");
92915f71
GR		 89MODULE_LICENSE("GPL");
90MODULE_VERSION(DRV_VERSION);
91
b3f4d599 92#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
93static int debug = -1;
94module_param(debug, int, 0);
95MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
92915f71
GR		 96
97/* forward decls */
220fe050 98static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter);
fa71ae27 99static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
56e94095 100static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
92915f71 101
dbf8b0d8
MR		 102static void ixgbevf_remove_adapter(struct ixgbe_hw *hw)
103{
104 struct ixgbevf_adapter *adapter = hw->back;
105
106 if (!hw->hw_addr)
107 return;
108 hw->hw_addr = NULL;
109 dev_err(&adapter->pdev->dev, "Adapter removed\n");
ea699569
MR		 110 if (test_bit(__IXGBEVF_WORK_INIT, &adapter->state))
111 schedule_work(&adapter->watchdog_task);
dbf8b0d8
MR		 112}
113
114static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg)
115{
116 u32 value;
117
118 /* The following check not only optimizes a bit by not
119 * performing a read on the status register when the
120 * register just read was a status register read that
121 * returned IXGBE_FAILED_READ_REG. It also blocks any
122 * potential recursion.
123 */
124 if (reg == IXGBE_VFSTATUS) {
125 ixgbevf_remove_adapter(hw);
126 return;
127 }
32c74949 128 value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS);
dbf8b0d8
MR		 129 if (value == IXGBE_FAILED_READ_REG)
130 ixgbevf_remove_adapter(hw);
131}
132
32c74949 133u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg)
dbf8b0d8
MR		 134{
135 u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr);
136 u32 value;
137
138 if (IXGBE_REMOVED(reg_addr))
139 return IXGBE_FAILED_READ_REG;
140 value = readl(reg_addr + reg);
141 if (unlikely(value == IXGBE_FAILED_READ_REG))
142 ixgbevf_check_remove(hw, reg);
143 return value;
144}
145
49ce9c2c 146/**
65d676c8 147 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
92915f71
GR		 148 * @adapter: pointer to adapter struct
149 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
150 * @queue: queue to map the corresponding interrupt to
151 * @msix_vector: the vector to map to the corresponding queue
92915f71
GR		 152 */
153static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
154 u8 queue, u8 msix_vector)
155{
156 u32 ivar, index;
157 struct ixgbe_hw *hw = &adapter->hw;
158 if (direction == -1) {
159 /* other causes */
160 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
161 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
162 ivar &= ~0xFF;
163 ivar |= msix_vector;
164 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
165 } else {
166 /* tx or rx causes */
167 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
168 index = ((16 * (queue & 1)) + (8 * direction));
169 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
170 ivar &= ~(0xFF << index);
171 ivar |= (msix_vector << index);
172 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
173 }
174}
175
70a10e25 176static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
9bdfefd2
ET		 177 struct ixgbevf_tx_buffer *tx_buffer)
178{
179 if (tx_buffer->skb) {
180 dev_kfree_skb_any(tx_buffer->skb);
181 if (dma_unmap_len(tx_buffer, len))
70a10e25 182 dma_unmap_single(tx_ring->dev,
9bdfefd2
ET		 183 dma_unmap_addr(tx_buffer, dma),
184 dma_unmap_len(tx_buffer, len),
2a1f8794 185 DMA_TO_DEVICE);
9bdfefd2
ET		 186 } else if (dma_unmap_len(tx_buffer, len)) {
187 dma_unmap_page(tx_ring->dev,
188 dma_unmap_addr(tx_buffer, dma),
189 dma_unmap_len(tx_buffer, len),
190 DMA_TO_DEVICE);
92915f71 191 }
9bdfefd2
ET		 192 tx_buffer->next_to_watch = NULL;
193 tx_buffer->skb = NULL;
194 dma_unmap_len_set(tx_buffer, len, 0);
195 /* tx_buffer must be completely set up in the transmit path */
92915f71
GR		 196}
197
92915f71
GR		 198#define IXGBE_MAX_TXD_PWR 14
199#define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR)
200
201/* Tx Descriptors needed, worst case */
3595990a
AD		 202#define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
203#define DESC_NEEDED (MAX_SKB_FRAGS + 4)
92915f71
GR		 204
205static void ixgbevf_tx_timeout(struct net_device *netdev);
206
207/**
208 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
fa71ae27 209 * @q_vector: board private structure
92915f71
GR		 210 * @tx_ring: tx ring to clean
211 **/
fa71ae27 212static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
92915f71
GR		 213 struct ixgbevf_ring *tx_ring)
214{
fa71ae27 215 struct ixgbevf_adapter *adapter = q_vector->adapter;
7ad1a093
ET		 216 struct ixgbevf_tx_buffer *tx_buffer;
217 union ixgbe_adv_tx_desc *tx_desc;
92915f71 218 unsigned int total_bytes = 0, total_packets = 0;
7ad1a093
ET		 219 unsigned int budget = tx_ring->count / 2;
220 unsigned int i = tx_ring->next_to_clean;
92915f71 221
10cc1bdd
AD		 222 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
223 return true;
224
7ad1a093
ET		 225 tx_buffer = &tx_ring->tx_buffer_info[i];
226 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
227 i -= tx_ring->count;
92915f71 228
e757e3e1 229 do {
7ad1a093 230 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
e757e3e1
AD		 231
232 /* if next_to_watch is not set then there is no work pending */
233 if (!eop_desc)
234 break;
235
236 /* prevent any other reads prior to eop_desc */
237 read_barrier_depends();
238
239 /* if DD is not set pending work has not been completed */
240 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
241 break;
242
243 /* clear next_to_watch to prevent false hangs */
7ad1a093 244 tx_buffer->next_to_watch = NULL;
e757e3e1 245
7ad1a093
ET		 246 /* update the statistics for this packet */
247 total_bytes += tx_buffer->bytecount;
248 total_packets += tx_buffer->gso_segs;
92915f71 249
9bdfefd2
ET		 250 /* free the skb */
251 dev_kfree_skb_any(tx_buffer->skb);
252
253 /* unmap skb header data */
254 dma_unmap_single(tx_ring->dev,
255 dma_unmap_addr(tx_buffer, dma),
256 dma_unmap_len(tx_buffer, len),
257 DMA_TO_DEVICE);
258
7ad1a093 259 /* clear tx_buffer data */
9bdfefd2
ET		 260 tx_buffer->skb = NULL;
261 dma_unmap_len_set(tx_buffer, len, 0);
92915f71 262
7ad1a093
ET		 263 /* unmap remaining buffers */
264 while (tx_desc != eop_desc) {
7ad1a093
ET		 265 tx_buffer++;
266 tx_desc++;
92915f71 267 i++;
7ad1a093
ET		 268 if (unlikely(!i)) {
269 i -= tx_ring->count;
270 tx_buffer = tx_ring->tx_buffer_info;
271 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
272 }
e757e3e1 273
9bdfefd2
ET		 274 /* unmap any remaining paged data */
275 if (dma_unmap_len(tx_buffer, len)) {
276 dma_unmap_page(tx_ring->dev,
277 dma_unmap_addr(tx_buffer, dma),
278 dma_unmap_len(tx_buffer, len),
279 DMA_TO_DEVICE);
280 dma_unmap_len_set(tx_buffer, len, 0);
281 }
92915f71
GR		 282 }
283
7ad1a093
ET		 284 /* move us one more past the eop_desc for start of next pkt */
285 tx_buffer++;
286 tx_desc++;
287 i++;
288 if (unlikely(!i)) {
289 i -= tx_ring->count;
290 tx_buffer = tx_ring->tx_buffer_info;
291 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
292 }
293
294 /* issue prefetch for next Tx descriptor */
295 prefetch(tx_desc);
296
297 /* update budget accounting */
298 budget--;
299 } while (likely(budget));
300
301 i += tx_ring->count;
92915f71 302 tx_ring->next_to_clean = i;
7ad1a093
ET		 303 u64_stats_update_begin(&tx_ring->syncp);
304 tx_ring->stats.bytes += total_bytes;
305 tx_ring->stats.packets += total_packets;
306 u64_stats_update_end(&tx_ring->syncp);
307 q_vector->tx.total_bytes += total_bytes;
308 q_vector->tx.total_packets += total_packets;
92915f71
GR		 309
310#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
7ad1a093 311 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
f880d07b 312 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
92915f71
GR		 313 /* Make sure that anybody stopping the queue after this
314 * sees the new next_to_clean.
315 */
316 smp_mb();
7ad1a093 317
fb40195c
AD		 318 if (__netif_subqueue_stopped(tx_ring->netdev,
319 tx_ring->queue_index) &&
92915f71 320 !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
fb40195c
AD		 321 netif_wake_subqueue(tx_ring->netdev,
322 tx_ring->queue_index);
7ad1a093 323 ++tx_ring->tx_stats.restart_queue;
92915f71 324 }
92915f71
GR		 325 }
326
7ad1a093 327 return !!budget;
92915f71
GR		 328}
329
08681618
JK		 330/**
331 * ixgbevf_rx_skb - Helper function to determine proper Rx method
332 * @q_vector: structure containing interrupt and ring information
333 * @skb: packet to send up
08681618
JK		 334 **/
335static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
dff80520 336 struct sk_buff *skb)
08681618 337{
c777cdfa
JK		 338#ifdef CONFIG_NET_RX_BUSY_POLL
339 skb_mark_napi_id(skb, &q_vector->napi);
340
341 if (ixgbevf_qv_busy_polling(q_vector)) {
342 netif_receive_skb(skb);
343 /* exit early if we busy polled */
344 return;
345 }
346#endif /* CONFIG_NET_RX_BUSY_POLL */
dff80520
ET		 347 if (!(q_vector->adapter->flags & IXGBE_FLAG_IN_NETPOLL))
348 napi_gro_receive(&q_vector->napi, skb);
349 else
350 netif_rx(skb);
08681618
JK		 351}
352
ec62fe26
ET		 353/* ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
354 * @ring: structure containig ring specific data
355 * @rx_desc: current Rx descriptor being processed
92915f71 356 * @skb: skb currently being received and modified
ec62fe26 357 */
55fb277c 358static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
ec62fe26
ET		 359 union ixgbe_adv_rx_desc *rx_desc,
360 struct sk_buff *skb)
92915f71 361{
bc8acf2c 362 skb_checksum_none_assert(skb);
92915f71
GR		 363
364 /* Rx csum disabled */
fb40195c 365 if (!(ring->netdev->features & NETIF_F_RXCSUM))
92915f71
GR		 366 return;
367
368 /* if IP and error */
ec62fe26
ET		 369 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
370 ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
095e2617 371 ring->rx_stats.csum_err++;
92915f71
GR		 372 return;
373 }
374
ec62fe26 375 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
92915f71
GR		 376 return;
377
ec62fe26 378 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
095e2617 379 ring->rx_stats.csum_err++;
92915f71
GR		 380 return;
381 }
382
383 /* It must be a TCP or UDP packet with a valid checksum */
384 skb->ip_summed = CHECKSUM_UNNECESSARY;
92915f71
GR		 385}
386
dff80520
ET		 387/* ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor
388 * @rx_ring: rx descriptor ring packet is being transacted on
389 * @rx_desc: pointer to the EOP Rx descriptor
390 * @skb: pointer to current skb being populated
391 *
392 * This function checks the ring, descriptor, and packet information in
393 * order to populate the checksum, VLAN, protocol, and other fields within
394 * the skb.
395 */
396static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring,
397 union ixgbe_adv_rx_desc *rx_desc,
398 struct sk_buff *skb)
399{
400 ixgbevf_rx_checksum(rx_ring, rx_desc, skb);
401
402 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
403 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
404 unsigned long *active_vlans = netdev_priv(rx_ring->netdev);
405
406 if (test_bit(vid & VLAN_VID_MASK, active_vlans))
407 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
408 }
409
410 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
411}
412
bafa578f
ET		 413static bool ixgbevf_alloc_mapped_skb(struct ixgbevf_ring *rx_ring,
414 struct ixgbevf_rx_buffer *bi)
415{
416 struct sk_buff *skb = bi->skb;
417 dma_addr_t dma = bi->dma;
418
419 if (unlikely(skb))
420 return true;
421
422 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
423 rx_ring->rx_buf_len);
424 if (unlikely(!skb)) {
425 rx_ring->rx_stats.alloc_rx_buff_failed++;
426 return false;
427 }
428
429 dma = dma_map_single(rx_ring->dev, skb->data,
430 rx_ring->rx_buf_len, DMA_FROM_DEVICE);
431
432 /* if mapping failed free memory back to system since
433 * there isn't much point in holding memory we can't use
434 */
435 if (dma_mapping_error(rx_ring->dev, dma)) {
436 dev_kfree_skb_any(skb);
437
438 rx_ring->rx_stats.alloc_rx_buff_failed++;
439 return false;
440 }
441
442 bi->skb = skb;
443 bi->dma = dma;
444
445 return true;
446}
447
92915f71
GR		 448/**
449 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
095e2617 450 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
bafa578f 451 * @cleaned_count: number of buffers to replace
92915f71 452 **/
095e2617 453static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
bafa578f 454 u16 cleaned_count)
92915f71 455{
92915f71
GR		 456 union ixgbe_adv_rx_desc *rx_desc;
457 struct ixgbevf_rx_buffer *bi;
fb40195c 458 unsigned int i = rx_ring->next_to_use;
92915f71 459
bafa578f
ET		 460 /* nothing to do or no valid netdev defined */
461 if (!cleaned_count || !rx_ring->netdev)
462 return;
b9dd245b 463
bafa578f
ET		 464 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
465 bi = &rx_ring->rx_buffer_info[i];
466 i -= rx_ring->count;
05d063aa 467
bafa578f
ET		 468 do {
469 if (!ixgbevf_alloc_mapped_skb(rx_ring, bi))
470 break;
b9dd245b 471
bafa578f
ET		 472 /* Refresh the desc even if pkt_addr didn't change
473 * because each write-back erases this info.
474 */
77d5dfca 475 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
92915f71 476
bafa578f
ET		 477 rx_desc++;
478 bi++;
92915f71 479 i++;
bafa578f
ET		 480 if (unlikely(!i)) {
481 rx_desc = IXGBEVF_RX_DESC(rx_ring, 0);
482 bi = rx_ring->rx_buffer_info;
483 i -= rx_ring->count;
484 }
485
486 /* clear the hdr_addr for the next_to_use descriptor */
487 rx_desc->read.hdr_addr = 0;
488
489 cleaned_count--;
490 } while (cleaned_count);
491
492 i += rx_ring->count;
92915f71 493
bafa578f
ET		 494 if (rx_ring->next_to_use != i) {
495 /* record the next descriptor to use */
496 rx_ring->next_to_use = i;
497
498 /* Force memory writes to complete before letting h/w
499 * know there are new descriptors to fetch. (Only
500 * applicable for weak-ordered memory model archs,
501 * such as IA-64).
502 */
503 wmb();
504 ixgbevf_write_tail(rx_ring, i);
505 }
92915f71
GR		 506}
507
508static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
5f3600eb 509 u32 qmask)
92915f71 510{
92915f71
GR		 511 struct ixgbe_hw *hw = &adapter->hw;
512
5f3600eb 513 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
92915f71
GR		 514}
515
08e50a20
JK		 516static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
517 struct ixgbevf_ring *rx_ring,
518 int budget)
92915f71 519{
b97fe3b1 520 union ixgbe_adv_rx_desc *rx_desc;
92915f71 521 unsigned int i;
92915f71 522 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
bafa578f 523 u16 cleaned_count = ixgbevf_desc_unused(rx_ring);
92915f71
GR		 524
525 i = rx_ring->next_to_clean;
908421f6 526 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
92915f71 527
ec62fe26 528 while (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD)) {
b97fe3b1
ET		 529 union ixgbe_adv_rx_desc *next_rxd;
530 struct ixgbevf_rx_buffer *rx_buffer_info;
531 struct sk_buff *skb;
532
fa71ae27 533 if (!budget)
92915f71 534 break;
fa71ae27 535 budget--;
92915f71 536
2d0bb1c1 537 rmb(); /* read descriptor and rx_buffer_info after status DD */
ec62fe26 538
b97fe3b1 539 rx_buffer_info = &rx_ring->rx_buffer_info[i];
92915f71 540 skb = rx_buffer_info->skb;
b97fe3b1 541 prefetch(skb->data);
92915f71
GR		 542 rx_buffer_info->skb = NULL;
543
ec62fe26
ET		 544 dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
545 rx_ring->rx_buf_len,
546 DMA_FROM_DEVICE);
547 rx_buffer_info->dma = 0;
548 skb_put(skb, le16_to_cpu(rx_desc->wb.upper.length));
92915f71 549
b97fe3b1
ET		 550 cleaned_count++;
551
92915f71
GR		 552 i++;
553 if (i == rx_ring->count)
554 i = 0;
555
908421f6 556 next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
92915f71 557 prefetch(next_rxd);
92915f71 558
ec62fe26 559 if (!(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) {
b97fe3b1 560 skb->next = rx_ring->rx_buffer_info[i].skb;
5c60f81a 561 IXGBE_CB(skb->next)->prev = skb;
095e2617 562 rx_ring->rx_stats.non_eop_descs++;
92915f71
GR		 563 goto next_desc;
564 }
565
5c60f81a
AD		 566 /* we should not be chaining buffers, if we did drop the skb */
567 if (IXGBE_CB(skb)->prev) {
568 do {
569 struct sk_buff *this = skb;
570 skb = IXGBE_CB(skb)->prev;
571 dev_kfree_skb(this);
572 } while (skb);
573 goto next_desc;
574 }
575
92915f71 576 /* ERR_MASK will only have valid bits if EOP set */
ec62fe26
ET		 577 if (unlikely(ixgbevf_test_staterr(rx_desc,
578 IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) {
92915f71
GR		 579 dev_kfree_skb_irq(skb);
580 goto next_desc;
581 }
582
92915f71
GR		 583 /* probably a little skewed due to removing CRC */
584 total_rx_bytes += skb->len;
585 total_rx_packets++;
586
815cccbf
JF		 587 /* Workaround hardware that can't do proper VEPA multicast
588 * source pruning.
589 */
bd9d5592
FF		 590 if ((skb->pkt_type == PACKET_BROADCAST ||
591 skb->pkt_type == PACKET_MULTICAST) &&
095e2617 592 ether_addr_equal(rx_ring->netdev->dev_addr,
7367d0b5 593 eth_hdr(skb)->h_source)) {
815cccbf
JF		 594 dev_kfree_skb_irq(skb);
595 goto next_desc;
596 }
597
dff80520
ET		 598 /* populate checksum, VLAN, and protocol */
599 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb);
600
601 ixgbevf_rx_skb(q_vector, skb);
92915f71
GR		 602
603next_desc:
92915f71
GR		 604 /* return some buffers to hardware, one at a time is too slow */
605 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
095e2617 606 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
92915f71
GR		 607 cleaned_count = 0;
608 }
609
610 /* use prefetched values */
611 rx_desc = next_rxd;
612 rx_buffer_info = &rx_ring->rx_buffer_info[i];
b97fe3b1 613 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
92915f71
GR		 614 }
615
616 rx_ring->next_to_clean = i;
4197aa7b 617 u64_stats_update_begin(&rx_ring->syncp);
095e2617
ET		 618 rx_ring->stats.packets += total_rx_packets;
619 rx_ring->stats.bytes += total_rx_bytes;
4197aa7b 620 u64_stats_update_end(&rx_ring->syncp);
ac6ed8f0
GR		 621 q_vector->rx.total_packets += total_rx_packets;
622 q_vector->rx.total_bytes += total_rx_bytes;
92915f71 623
bafa578f
ET		 624 if (cleaned_count)
625 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
626
08e50a20 627 return total_rx_packets;
92915f71
GR		 628}
629
630/**
fa71ae27 631 * ixgbevf_poll - NAPI polling calback
92915f71
GR		 632 * @napi: napi struct with our devices info in it
633 * @budget: amount of work driver is allowed to do this pass, in packets
634 *
fa71ae27 635 * This function will clean more than one or more rings associated with a
92915f71
GR		 636 * q_vector.
637 **/
fa71ae27 638static int ixgbevf_poll(struct napi_struct *napi, int budget)
92915f71
GR		 639{
640 struct ixgbevf_q_vector *q_vector =
641 container_of(napi, struct ixgbevf_q_vector, napi);
642 struct ixgbevf_adapter *adapter = q_vector->adapter;
fa71ae27
AD		 643 struct ixgbevf_ring *ring;
644 int per_ring_budget;
645 bool clean_complete = true;
646
647 ixgbevf_for_each_ring(ring, q_vector->tx)
648 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
92915f71 649
c777cdfa
JK		 650#ifdef CONFIG_NET_RX_BUSY_POLL
651 if (!ixgbevf_qv_lock_napi(q_vector))
652 return budget;
653#endif
654
92915f71
GR		 655 /* attempt to distribute budget to each queue fairly, but don't allow
656 * the budget to go below 1 because we'll exit polling */
fa71ae27
AD		 657 if (q_vector->rx.count > 1)
658 per_ring_budget = max(budget/q_vector->rx.count, 1);
659 else
660 per_ring_budget = budget;
661
366c1099 662 adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
fa71ae27 663 ixgbevf_for_each_ring(ring, q_vector->rx)
08e50a20
JK		 664 clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring,
665 per_ring_budget)
666 < per_ring_budget);
366c1099 667 adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
fa71ae27 668
c777cdfa
JK		 669#ifdef CONFIG_NET_RX_BUSY_POLL
670 ixgbevf_qv_unlock_napi(q_vector);
671#endif
672
fa71ae27
AD		 673 /* If all work not completed, return budget and keep polling */
674 if (!clean_complete)
675 return budget;
676 /* all work done, exit the polling mode */
677 napi_complete(napi);
678 if (adapter->rx_itr_setting & 1)
679 ixgbevf_set_itr(q_vector);
2e7cfbdd
MR		 680 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
681 !test_bit(__IXGBEVF_REMOVING, &adapter->state))
fa71ae27
AD		 682 ixgbevf_irq_enable_queues(adapter,
683 1 << q_vector->v_idx);
92915f71 684
fa71ae27 685 return 0;
92915f71
GR		 686}
687
ce422606
GR		 688/**
689 * ixgbevf_write_eitr - write VTEITR register in hardware specific way
690 * @q_vector: structure containing interrupt and ring information
691 */
3849623e 692void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
ce422606
GR		 693{
694 struct ixgbevf_adapter *adapter = q_vector->adapter;
695 struct ixgbe_hw *hw = &adapter->hw;
696 int v_idx = q_vector->v_idx;
697 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
698
699 /*
700 * set the WDIS bit to not clear the timer bits and cause an
701 * immediate assertion of the interrupt
702 */
703 itr_reg |= IXGBE_EITR_CNT_WDIS;
704
705 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
706}
92915f71 707
c777cdfa
JK		 708#ifdef CONFIG_NET_RX_BUSY_POLL
709/* must be called with local_bh_disable()d */
710static int ixgbevf_busy_poll_recv(struct napi_struct *napi)
711{
712 struct ixgbevf_q_vector *q_vector =
713 container_of(napi, struct ixgbevf_q_vector, napi);
714 struct ixgbevf_adapter *adapter = q_vector->adapter;
715 struct ixgbevf_ring *ring;
716 int found = 0;
717
718 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
719 return LL_FLUSH_FAILED;
720
721 if (!ixgbevf_qv_lock_poll(q_vector))
722 return LL_FLUSH_BUSY;
723
724 ixgbevf_for_each_ring(ring, q_vector->rx) {
725 found = ixgbevf_clean_rx_irq(q_vector, ring, 4);
3b5dca26
JK		 726#ifdef BP_EXTENDED_STATS
727 if (found)
095e2617 728 ring->stats.cleaned += found;
3b5dca26 729 else
095e2617 730 ring->stats.misses++;
3b5dca26 731#endif
c777cdfa
JK		 732 if (found)
733 break;
734 }
735
736 ixgbevf_qv_unlock_poll(q_vector);
737
738 return found;
739}
740#endif /* CONFIG_NET_RX_BUSY_POLL */
741
92915f71
GR		 742/**
743 * ixgbevf_configure_msix - Configure MSI-X hardware
744 * @adapter: board private structure
745 *
746 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
747 * interrupts.
748 **/
749static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
750{
751 struct ixgbevf_q_vector *q_vector;
6b43c446 752 int q_vectors, v_idx;
92915f71
GR		 753
754 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
5f3600eb 755 adapter->eims_enable_mask = 0;
92915f71
GR		 756
757 /*
758 * Populate the IVAR table and set the ITR values to the
759 * corresponding register.
760 */
761 for (v_idx = 0; v_idx < q_vectors; v_idx++) {
6b43c446 762 struct ixgbevf_ring *ring;
92915f71 763 q_vector = adapter->q_vector[v_idx];
6b43c446
AD		 764
765 ixgbevf_for_each_ring(ring, q_vector->rx)
766 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
767
768 ixgbevf_for_each_ring(ring, q_vector->tx)
769 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
92915f71 770
5f3600eb
AD		 771 if (q_vector->tx.ring && !q_vector->rx.ring) {
772 /* tx only vector */
773 if (adapter->tx_itr_setting == 1)
774 q_vector->itr = IXGBE_10K_ITR;
775 else
776 q_vector->itr = adapter->tx_itr_setting;
777 } else {
778 /* rx or rx/tx vector */
779 if (adapter->rx_itr_setting == 1)
780 q_vector->itr = IXGBE_20K_ITR;
781 else
782 q_vector->itr = adapter->rx_itr_setting;
783 }
784
785 /* add q_vector eims value to global eims_enable_mask */
786 adapter->eims_enable_mask |= 1 << v_idx;
92915f71 787
5f3600eb 788 ixgbevf_write_eitr(q_vector);
92915f71
GR		 789 }
790
791 ixgbevf_set_ivar(adapter, -1, 1, v_idx);
5f3600eb
AD		 792 /* setup eims_other and add value to global eims_enable_mask */
793 adapter->eims_other = 1 << v_idx;
794 adapter->eims_enable_mask |= adapter->eims_other;
92915f71
GR		 795}
796
797enum latency_range {
798 lowest_latency = 0,
799 low_latency = 1,
800 bulk_latency = 2,
801 latency_invalid = 255
802};
803
804/**
805 * ixgbevf_update_itr - update the dynamic ITR value based on statistics
5f3600eb
AD		 806 * @q_vector: structure containing interrupt and ring information
807 * @ring_container: structure containing ring performance data
92915f71
GR		 808 *
809 * Stores a new ITR value based on packets and byte
810 * counts during the last interrupt. The advantage of per interrupt
811 * computation is faster updates and more accurate ITR for the current
812 * traffic pattern. Constants in this function were computed
813 * based on theoretical maximum wire speed and thresholds were set based
814 * on testing data as well as attempting to minimize response time
815 * while increasing bulk throughput.
816 **/
5f3600eb
AD		 817static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
818 struct ixgbevf_ring_container *ring_container)
92915f71 819{
5f3600eb
AD		 820 int bytes = ring_container->total_bytes;
821 int packets = ring_container->total_packets;
92915f71
GR		 822 u32 timepassed_us;
823 u64 bytes_perint;
5f3600eb 824 u8 itr_setting = ring_container->itr;
92915f71
GR		 825
826 if (packets == 0)
5f3600eb 827 return;
92915f71
GR		 828
829 /* simple throttlerate management
830 * 0-20MB/s lowest (100000 ints/s)
831 * 20-100MB/s low (20000 ints/s)
832 * 100-1249MB/s bulk (8000 ints/s)
833 */
834 /* what was last interrupt timeslice? */
5f3600eb 835 timepassed_us = q_vector->itr >> 2;
92915f71
GR		 836 bytes_perint = bytes / timepassed_us; /* bytes/usec */
837
838 switch (itr_setting) {
839 case lowest_latency:
e2c28ce7 840 if (bytes_perint > 10)
5f3600eb 841 itr_setting = low_latency;
92915f71
GR		 842 break;
843 case low_latency:
e2c28ce7 844 if (bytes_perint > 20)
5f3600eb 845 itr_setting = bulk_latency;
e2c28ce7 846 else if (bytes_perint <= 10)
5f3600eb 847 itr_setting = lowest_latency;
92915f71
GR		 848 break;
849 case bulk_latency:
e2c28ce7 850 if (bytes_perint <= 20)
5f3600eb 851 itr_setting = low_latency;
92915f71
GR		 852 break;
853 }
854
5f3600eb
AD		 855 /* clear work counters since we have the values we need */
856 ring_container->total_bytes = 0;
857 ring_container->total_packets = 0;
858
859 /* write updated itr to ring container */
860 ring_container->itr = itr_setting;
92915f71
GR		 861}
862
fa71ae27 863static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
92915f71 864{
5f3600eb
AD		 865 u32 new_itr = q_vector->itr;
866 u8 current_itr;
92915f71 867
5f3600eb
AD		 868 ixgbevf_update_itr(q_vector, &q_vector->tx);
869 ixgbevf_update_itr(q_vector, &q_vector->rx);
92915f71 870
6b43c446 871 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
92915f71
GR		 872
873 switch (current_itr) {
874 /* counts and packets in update_itr are dependent on these numbers */
875 case lowest_latency:
5f3600eb 876 new_itr = IXGBE_100K_ITR;
92915f71
GR		 877 break;
878 case low_latency:
5f3600eb 879 new_itr = IXGBE_20K_ITR;
92915f71
GR		 880 break;
881 case bulk_latency:
882 default:
5f3600eb 883 new_itr = IXGBE_8K_ITR;
92915f71
GR		 884 break;
885 }
886
5f3600eb 887 if (new_itr != q_vector->itr) {
92915f71 888 /* do an exponential smoothing */
5f3600eb
AD		 889 new_itr = (10 * new_itr * q_vector->itr) /
890 ((9 * new_itr) + q_vector->itr);
891
892 /* save the algorithm value here */
893 q_vector->itr = new_itr;
894
895 ixgbevf_write_eitr(q_vector);
92915f71 896 }
92915f71
GR		 897}
898
4b2cd27f 899static irqreturn_t ixgbevf_msix_other(int irq, void *data)
92915f71 900{
fa71ae27 901 struct ixgbevf_adapter *adapter = data;
92915f71 902 struct ixgbe_hw *hw = &adapter->hw;
08259594 903
4b2cd27f 904 hw->mac.get_link_status = 1;
1e72bfc3 905
2e7cfbdd
MR		 906 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
907 !test_bit(__IXGBEVF_REMOVING, &adapter->state))
c7bb417d 908 mod_timer(&adapter->watchdog_timer, jiffies);
3a2c4033 909
5f3600eb
AD		 910 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
911
92915f71
GR		 912 return IRQ_HANDLED;
913}
914
92915f71 915/**
fa71ae27 916 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
92915f71
GR		 917 * @irq: unused
918 * @data: pointer to our q_vector struct for this interrupt vector
919 **/
fa71ae27 920static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
92915f71
GR		 921{
922 struct ixgbevf_q_vector *q_vector = data;
92915f71 923
5f3600eb 924 /* EIAM disabled interrupts (on this vector) for us */
fa71ae27
AD		 925 if (q_vector->rx.ring || q_vector->tx.ring)
926 napi_schedule(&q_vector->napi);
92915f71
GR		 927
928 return IRQ_HANDLED;
929}
930
931static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
932 int r_idx)
933{
934 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
935
87e70ab9
DS		 936 a->rx_ring[r_idx]->next = q_vector->rx.ring;
937 q_vector->rx.ring = a->rx_ring[r_idx];
6b43c446 938 q_vector->rx.count++;
92915f71
GR		 939}
940
941static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
942 int t_idx)
943{
944 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
945
87e70ab9
DS		 946 a->tx_ring[t_idx]->next = q_vector->tx.ring;
947 q_vector->tx.ring = a->tx_ring[t_idx];
6b43c446 948 q_vector->tx.count++;
92915f71
GR		 949}
950
951/**
952 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
953 * @adapter: board private structure to initialize
954 *
955 * This function maps descriptor rings to the queue-specific vectors
956 * we were allotted through the MSI-X enabling code. Ideally, we'd have
957 * one vector per ring/queue, but on a constrained vector budget, we
958 * group the rings as "efficiently" as possible. You would add new
959 * mapping configurations in here.
960 **/
961static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
962{
963 int q_vectors;
964 int v_start = 0;
965 int rxr_idx = 0, txr_idx = 0;
966 int rxr_remaining = adapter->num_rx_queues;
967 int txr_remaining = adapter->num_tx_queues;
968 int i, j;
969 int rqpv, tqpv;
970 int err = 0;
971
972 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
973
974 /*
975 * The ideal configuration...
976 * We have enough vectors to map one per queue.
977 */
978 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
979 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
980 map_vector_to_rxq(adapter, v_start, rxr_idx);
981
982 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
983 map_vector_to_txq(adapter, v_start, txr_idx);
984 goto out;
985 }
986
987 /*
988 * If we don't have enough vectors for a 1-to-1
989 * mapping, we'll have to group them so there are
990 * multiple queues per vector.
991 */
992 /* Re-adjusting *qpv takes care of the remainder. */
993 for (i = v_start; i < q_vectors; i++) {
994 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
995 for (j = 0; j < rqpv; j++) {
996 map_vector_to_rxq(adapter, i, rxr_idx);
997 rxr_idx++;
998 rxr_remaining--;
999 }
1000 }
1001 for (i = v_start; i < q_vectors; i++) {
1002 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
1003 for (j = 0; j < tqpv; j++) {
1004 map_vector_to_txq(adapter, i, txr_idx);
1005 txr_idx++;
1006 txr_remaining--;
1007 }
1008 }
1009
1010out:
1011 return err;
1012}
1013
1014/**
1015 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
1016 * @adapter: board private structure
1017 *
1018 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
1019 * interrupts from the kernel.
1020 **/
1021static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
1022{
1023 struct net_device *netdev = adapter->netdev;
fa71ae27
AD		 1024 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1025 int vector, err;
92915f71
GR		 1026 int ri = 0, ti = 0;
1027
92915f71 1028 for (vector = 0; vector < q_vectors; vector++) {
fa71ae27
AD		 1029 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
1030 struct msix_entry *entry = &adapter->msix_entries[vector];
1031
1032 if (q_vector->tx.ring && q_vector->rx.ring) {
1033 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1034 "%s-%s-%d", netdev->name, "TxRx", ri++);
1035 ti++;
1036 } else if (q_vector->rx.ring) {
1037 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1038 "%s-%s-%d", netdev->name, "rx", ri++);
1039 } else if (q_vector->tx.ring) {
1040 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1041 "%s-%s-%d", netdev->name, "tx", ti++);
92915f71
GR		 1042 } else {
1043 /* skip this unused q_vector */
1044 continue;
1045 }
fa71ae27
AD		 1046 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
1047 q_vector->name, q_vector);
92915f71
GR		 1048 if (err) {
1049 hw_dbg(&adapter->hw,
1050 "request_irq failed for MSIX interrupt "
1051 "Error: %d\n", err);
1052 goto free_queue_irqs;
1053 }
1054 }
1055
92915f71 1056 err = request_irq(adapter->msix_entries[vector].vector,
4b2cd27f 1057 &ixgbevf_msix_other, 0, netdev->name, adapter);
92915f71
GR		 1058 if (err) {
1059 hw_dbg(&adapter->hw,
4b2cd27f 1060 "request_irq for msix_other failed: %d\n", err);
92915f71
GR		 1061 goto free_queue_irqs;
1062 }
1063
1064 return 0;
1065
1066free_queue_irqs:
fa71ae27
AD		 1067 while (vector) {
1068 vector--;
1069 free_irq(adapter->msix_entries[vector].vector,
1070 adapter->q_vector[vector]);
1071 }
a1f6c6b1 1072 /* This failure is non-recoverable - it indicates the system is
1073 * out of MSIX vector resources and the VF driver cannot run
1074 * without them. Set the number of msix vectors to zero
1075 * indicating that not enough can be allocated. The error
1076 * will be returned to the user indicating device open failed.
1077 * Any further attempts to force the driver to open will also
1078 * fail. The only way to recover is to unload the driver and
1079 * reload it again. If the system has recovered some MSIX
1080 * vectors then it may succeed.
1081 */
1082 adapter->num_msix_vectors = 0;
92915f71
GR		 1083 return err;
1084}
1085
1086static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
1087{
1088 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1089
1090 for (i = 0; i < q_vectors; i++) {
1091 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
6b43c446
AD		 1092 q_vector->rx.ring = NULL;
1093 q_vector->tx.ring = NULL;
1094 q_vector->rx.count = 0;
1095 q_vector->tx.count = 0;
92915f71
GR		 1096 }
1097}
1098
1099/**
1100 * ixgbevf_request_irq - initialize interrupts
1101 * @adapter: board private structure
1102 *
1103 * Attempts to configure interrupts using the best available
1104 * capabilities of the hardware and kernel.
1105 **/
1106static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1107{
1108 int err = 0;
1109
1110 err = ixgbevf_request_msix_irqs(adapter);
1111
1112 if (err)
1113 hw_dbg(&adapter->hw,
1114 "request_irq failed, Error %d\n", err);
1115
1116 return err;
1117}
1118
1119static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1120{
92915f71
GR		 1121 int i, q_vectors;
1122
1123 q_vectors = adapter->num_msix_vectors;
92915f71
GR		 1124 i = q_vectors - 1;
1125
fa71ae27 1126 free_irq(adapter->msix_entries[i].vector, adapter);
92915f71
GR		 1127 i--;
1128
1129 for (; i >= 0; i--) {
fa71ae27
AD		 1130 /* free only the irqs that were actually requested */
1131 if (!adapter->q_vector[i]->rx.ring &&
1132 !adapter->q_vector[i]->tx.ring)
1133 continue;
1134
92915f71
GR		 1135 free_irq(adapter->msix_entries[i].vector,
1136 adapter->q_vector[i]);
1137 }
1138
1139 ixgbevf_reset_q_vectors(adapter);
1140}
1141
1142/**
1143 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1144 * @adapter: board private structure
1145 **/
1146static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1147{
92915f71 1148 struct ixgbe_hw *hw = &adapter->hw;
5f3600eb 1149 int i;
92915f71 1150
5f3600eb 1151 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
92915f71 1152 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
5f3600eb 1153 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
92915f71
GR		 1154
1155 IXGBE_WRITE_FLUSH(hw);
1156
1157 for (i = 0; i < adapter->num_msix_vectors; i++)
1158 synchronize_irq(adapter->msix_entries[i].vector);
1159}
1160
1161/**
1162 * ixgbevf_irq_enable - Enable default interrupt generation settings
1163 * @adapter: board private structure
1164 **/
5f3600eb 1165static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
92915f71
GR		 1166{
1167 struct ixgbe_hw *hw = &adapter->hw;
92915f71 1168
5f3600eb
AD		 1169 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1170 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1171 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
92915f71
GR		 1172}
1173
de02decb
DS		 1174/**
1175 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset
1176 * @adapter: board private structure
1177 * @ring: structure containing ring specific data
1178 *
1179 * Configure the Tx descriptor ring after a reset.
1180 **/
1181static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
1182 struct ixgbevf_ring *ring)
1183{
1184 struct ixgbe_hw *hw = &adapter->hw;
1185 u64 tdba = ring->dma;
1186 int wait_loop = 10;
1187 u32 txdctl = IXGBE_TXDCTL_ENABLE;
1188 u8 reg_idx = ring->reg_idx;
1189
1190 /* disable queue to avoid issues while updating state */
1191 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
1192 IXGBE_WRITE_FLUSH(hw);
1193
1194 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32));
1195 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32);
1196 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx),
1197 ring->count * sizeof(union ixgbe_adv_tx_desc));
1198
1199 /* disable head writeback */
1200 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0);
1201 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0);
1202
1203 /* enable relaxed ordering */
1204 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx),
1205 (IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1206 IXGBE_DCA_TXCTRL_DATA_RRO_EN));
1207
1208 /* reset head and tail pointers */
1209 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0);
1210 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0);
dbf8b0d8 1211 ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx);
de02decb
DS		 1212
1213 /* reset ntu and ntc to place SW in sync with hardwdare */
1214 ring->next_to_clean = 0;
1215 ring->next_to_use = 0;
1216
1217 /* In order to avoid issues WTHRESH + PTHRESH should always be equal
1218 * to or less than the number of on chip descriptors, which is
1219 * currently 40.
1220 */
1221 txdctl |= (8 << 16); /* WTHRESH = 8 */
1222
1223 /* Setting PTHRESH to 32 both improves performance */
1224 txdctl |= (1 << 8) | /* HTHRESH = 1 */
1225 32; /* PTHRESH = 32 */
1226
1227 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl);
1228
1229 /* poll to verify queue is enabled */
1230 do {
1231 usleep_range(1000, 2000);
1232 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx));
1233 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
1234 if (!wait_loop)
1235 pr_err("Could not enable Tx Queue %d\n", reg_idx);
1236}
1237
92915f71
GR		 1238/**
1239 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1240 * @adapter: board private structure
1241 *
1242 * Configure the Tx unit of the MAC after a reset.
1243 **/
1244static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1245{
de02decb 1246 u32 i;
92915f71
GR		 1247
1248 /* Setup the HW Tx Head and Tail descriptor pointers */
de02decb
DS		 1249 for (i = 0; i < adapter->num_tx_queues; i++)
1250 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]);
92915f71
GR		 1251}
1252
1253#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1254
1255static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1256{
1257 struct ixgbevf_ring *rx_ring;
1258 struct ixgbe_hw *hw = &adapter->hw;
1259 u32 srrctl;
1260
87e70ab9 1261 rx_ring = adapter->rx_ring[index];
92915f71
GR		 1262
1263 srrctl = IXGBE_SRRCTL_DROP_EN;
1264
77d5dfca 1265 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
92915f71 1266
dd1fe113
AD		 1267 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
1268 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1269
92915f71
GR		 1270 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1271}
1272
1bb9c639
DS		 1273static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter)
1274{
1275 struct ixgbe_hw *hw = &adapter->hw;
1276
1277 /* PSRTYPE must be initialized in 82599 */
1278 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
1279 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR |
1280 IXGBE_PSRTYPE_L2HDR;
1281
1282 if (adapter->num_rx_queues > 1)
1283 psrtype |= 1 << 29;
1284
1285 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1286}
1287
dd1fe113
AD		 1288static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter)
1289{
1290 struct ixgbe_hw *hw = &adapter->hw;
1291 struct net_device *netdev = adapter->netdev;
1292 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1293 int i;
1294 u16 rx_buf_len;
1295
1296 /* notify the PF of our intent to use this size of frame */
1297 ixgbevf_rlpml_set_vf(hw, max_frame);
1298
1299 /* PF will allow an extra 4 bytes past for vlan tagged frames */
1300 max_frame += VLAN_HLEN;
1301
1302 /*
85624caf
GR		 1303 * Allocate buffer sizes that fit well into 32K and
1304 * take into account max frame size of 9.5K
dd1fe113
AD		 1305 */
1306 if ((hw->mac.type == ixgbe_mac_X540_vf) &&
1307 (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE))
1308 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85624caf
GR		 1309 else if (max_frame <= IXGBEVF_RXBUFFER_2K)
1310 rx_buf_len = IXGBEVF_RXBUFFER_2K;
1311 else if (max_frame <= IXGBEVF_RXBUFFER_4K)
1312 rx_buf_len = IXGBEVF_RXBUFFER_4K;
1313 else if (max_frame <= IXGBEVF_RXBUFFER_8K)
1314 rx_buf_len = IXGBEVF_RXBUFFER_8K;
dd1fe113 1315 else
85624caf 1316 rx_buf_len = IXGBEVF_RXBUFFER_10K;
dd1fe113
AD		 1317
1318 for (i = 0; i < adapter->num_rx_queues; i++)
87e70ab9 1319 adapter->rx_ring[i]->rx_buf_len = rx_buf_len;
dd1fe113
AD		 1320}
1321
de02decb
DS		 1322#define IXGBEVF_MAX_RX_DESC_POLL 10
1323static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
1324 struct ixgbevf_ring *ring)
1325{
1326 struct ixgbe_hw *hw = &adapter->hw;
1327 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1328 u32 rxdctl;
1329 u8 reg_idx = ring->reg_idx;
1330
26597802
MR		 1331 if (IXGBE_REMOVED(hw->hw_addr))
1332 return;
de02decb
DS		 1333 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1334 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1335
1336 /* write value back with RXDCTL.ENABLE bit cleared */
1337 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1338
1339 /* the hardware may take up to 100us to really disable the rx queue */
1340 do {
1341 udelay(10);
1342 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1343 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
1344
1345 if (!wait_loop)
1346 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n",
1347 reg_idx);
1348}
1349
1350static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1351 struct ixgbevf_ring *ring)
1352{
1353 struct ixgbe_hw *hw = &adapter->hw;
1354 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1355 u32 rxdctl;
1356 u8 reg_idx = ring->reg_idx;
1357
26597802
MR		 1358 if (IXGBE_REMOVED(hw->hw_addr))
1359 return;
de02decb
DS		 1360 do {
1361 usleep_range(1000, 2000);
1362 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1363 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
1364
1365 if (!wait_loop)
1366 pr_err("RXDCTL.ENABLE queue %d not set while polling\n",
1367 reg_idx);
1368}
1369
1370static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
1371 struct ixgbevf_ring *ring)
1372{
1373 struct ixgbe_hw *hw = &adapter->hw;
1374 u64 rdba = ring->dma;
1375 u32 rxdctl;
1376 u8 reg_idx = ring->reg_idx;
1377
1378 /* disable queue to avoid issues while updating state */
1379 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1380 ixgbevf_disable_rx_queue(adapter, ring);
1381
1382 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32));
1383 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32);
1384 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx),
1385 ring->count * sizeof(union ixgbe_adv_rx_desc));
1386
1387 /* enable relaxed ordering */
1388 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1389 IXGBE_DCA_RXCTRL_DESC_RRO_EN);
1390
1391 /* reset head and tail pointers */
1392 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0);
1393 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0);
dbf8b0d8 1394 ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx);
de02decb
DS		 1395
1396 /* reset ntu and ntc to place SW in sync with hardwdare */
1397 ring->next_to_clean = 0;
1398 ring->next_to_use = 0;
1399
1400 ixgbevf_configure_srrctl(adapter, reg_idx);
1401
1402 /* prevent DMA from exceeding buffer space available */
1403 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1404 rxdctl |= ring->rx_buf_len | IXGBE_RXDCTL_RLPML_EN;
1405 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1406 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1407
1408 ixgbevf_rx_desc_queue_enable(adapter, ring);
095e2617 1409 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring));
de02decb
DS		 1410}
1411
92915f71
GR		 1412/**
1413 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1414 * @adapter: board private structure
1415 *
1416 * Configure the Rx unit of the MAC after a reset.
1417 **/
1418static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1419{
de02decb 1420 int i;
92915f71 1421
1bb9c639 1422 ixgbevf_setup_psrtype(adapter);
dd1fe113
AD		 1423
1424 /* set_rx_buffer_len must be called before ring initialization */
1425 ixgbevf_set_rx_buffer_len(adapter);
92915f71 1426
92915f71
GR		 1427 /* Setup the HW Rx Head and Tail Descriptor Pointers and
1428 * the Base and Length of the Rx Descriptor Ring */
de02decb
DS		 1429 for (i = 0; i < adapter->num_rx_queues; i++)
1430 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]);
92915f71
GR		 1431}
1432
80d5c368
PM		 1433static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev,
1434 __be16 proto, u16 vid)
92915f71
GR		 1435{
1436 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1437 struct ixgbe_hw *hw = &adapter->hw;
2ddc7fe1
AD		 1438 int err;
1439
55fdd45b 1440 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1441
92915f71 1442 /* add VID to filter table */
2ddc7fe1 1443 err = hw->mac.ops.set_vfta(hw, vid, 0, true);
1c55ed76 1444
55fdd45b 1445 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1446
2ddc7fe1
AD		 1447 /* translate error return types so error makes sense */
1448 if (err == IXGBE_ERR_MBX)
1449 return -EIO;
1450
1451 if (err == IXGBE_ERR_INVALID_ARGUMENT)
1452 return -EACCES;
1453
dadcd65f 1454 set_bit(vid, adapter->active_vlans);
8e586137 1455
2ddc7fe1 1456 return err;
92915f71
GR		 1457}
1458
80d5c368
PM		 1459static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev,
1460 __be16 proto, u16 vid)
92915f71
GR		 1461{
1462 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1463 struct ixgbe_hw *hw = &adapter->hw;
2ddc7fe1 1464 int err = -EOPNOTSUPP;
92915f71 1465
55fdd45b 1466 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1467
92915f71 1468 /* remove VID from filter table */
92fe0bf7 1469 err = hw->mac.ops.set_vfta(hw, vid, 0, false);
1c55ed76 1470
55fdd45b 1471 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1472
dadcd65f 1473 clear_bit(vid, adapter->active_vlans);
8e586137 1474
2ddc7fe1 1475 return err;
92915f71
GR		 1476}
1477
1478static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1479{
dadcd65f 1480 u16 vid;
92915f71 1481
dadcd65f 1482 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
80d5c368
PM		 1483 ixgbevf_vlan_rx_add_vid(adapter->netdev,
1484 htons(ETH_P_8021Q), vid);
92915f71
GR		 1485}
1486
46ec20ff
GR		 1487static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1488{
1489 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1490 struct ixgbe_hw *hw = &adapter->hw;
1491 int count = 0;
1492
1493 if ((netdev_uc_count(netdev)) > 10) {
dbd9636e 1494 pr_err("Too many unicast filters - No Space\n");
46ec20ff
GR		 1495 return -ENOSPC;
1496 }
1497
1498 if (!netdev_uc_empty(netdev)) {
1499 struct netdev_hw_addr *ha;
1500 netdev_for_each_uc_addr(ha, netdev) {
1501 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1502 udelay(200);
1503 }
1504 } else {
1505 /*
1506 * If the list is empty then send message to PF driver to
1507 * clear all macvlans on this VF.
1508 */
1509 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1510 }
1511
1512 return count;
1513}
1514
92915f71 1515/**
dee847f5 1516 * ixgbevf_set_rx_mode - Multicast and unicast set
92915f71
GR		 1517 * @netdev: network interface device structure
1518 *
1519 * The set_rx_method entry point is called whenever the multicast address
dee847f5
GR		 1520 * list, unicast address list or the network interface flags are updated.
1521 * This routine is responsible for configuring the hardware for proper
1522 * multicast mode and configuring requested unicast filters.
92915f71
GR		 1523 **/
1524static void ixgbevf_set_rx_mode(struct net_device *netdev)
1525{
1526 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1527 struct ixgbe_hw *hw = &adapter->hw;
92915f71 1528
55fdd45b 1529 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1530
92915f71 1531 /* reprogram multicast list */
92fe0bf7 1532 hw->mac.ops.update_mc_addr_list(hw, netdev);
46ec20ff
GR		 1533
1534 ixgbevf_write_uc_addr_list(netdev);
1c55ed76 1535
55fdd45b 1536 spin_unlock_bh(&adapter->mbx_lock);
92915f71
GR		 1537}
1538
1539static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1540{
1541 int q_idx;
1542 struct ixgbevf_q_vector *q_vector;
1543 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1544
1545 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
92915f71 1546 q_vector = adapter->q_vector[q_idx];
c777cdfa
JK		 1547#ifdef CONFIG_NET_RX_BUSY_POLL
1548 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]);
1549#endif
fa71ae27 1550 napi_enable(&q_vector->napi);
92915f71
GR		 1551 }
1552}
1553
1554static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1555{
1556 int q_idx;
1557 struct ixgbevf_q_vector *q_vector;
1558 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1559
1560 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1561 q_vector = adapter->q_vector[q_idx];
92915f71 1562 napi_disable(&q_vector->napi);
c777cdfa
JK		 1563#ifdef CONFIG_NET_RX_BUSY_POLL
1564 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) {
1565 pr_info("QV %d locked\n", q_idx);
1566 usleep_range(1000, 20000);
1567 }
1568#endif /* CONFIG_NET_RX_BUSY_POLL */
92915f71
GR		 1569 }
1570}
1571
220fe050
DS		 1572static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter)
1573{
1574 struct ixgbe_hw *hw = &adapter->hw;
1575 unsigned int def_q = 0;
1576 unsigned int num_tcs = 0;
1577 unsigned int num_rx_queues = 1;
1578 int err;
1579
1580 spin_lock_bh(&adapter->mbx_lock);
1581
1582 /* fetch queue configuration from the PF */
1583 err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1584
1585 spin_unlock_bh(&adapter->mbx_lock);
1586
1587 if (err)
1588 return err;
1589
1590 if (num_tcs > 1) {
1591 /* update default Tx ring register index */
87e70ab9 1592 adapter->tx_ring[0]->reg_idx = def_q;
220fe050
DS		 1593
1594 /* we need as many queues as traffic classes */
1595 num_rx_queues = num_tcs;
1596 }
1597
1598 /* if we have a bad config abort request queue reset */
1599 if (adapter->num_rx_queues != num_rx_queues) {
1600 /* force mailbox timeout to prevent further messages */
1601 hw->mbx.timeout = 0;
1602
1603 /* wait for watchdog to come around and bail us out */
1604 adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
1605 }
1606
1607 return 0;
1608}
1609
92915f71
GR		 1610static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1611{
220fe050
DS		 1612 ixgbevf_configure_dcb(adapter);
1613
de02decb 1614 ixgbevf_set_rx_mode(adapter->netdev);
92915f71
GR		 1615
1616 ixgbevf_restore_vlan(adapter);
1617
1618 ixgbevf_configure_tx(adapter);
1619 ixgbevf_configure_rx(adapter);
92915f71
GR		 1620}
1621
33bd9f60
GR		 1622static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1623{
1624 /* Only save pre-reset stats if there are some */
1625 if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1626 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1627 adapter->stats.base_vfgprc;
1628 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1629 adapter->stats.base_vfgptc;
1630 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1631 adapter->stats.base_vfgorc;
1632 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1633 adapter->stats.base_vfgotc;
1634 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1635 adapter->stats.base_vfmprc;
1636 }
1637}
1638
1639static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1640{
1641 struct ixgbe_hw *hw = &adapter->hw;
1642
1643 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1644 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1645 adapter->stats.last_vfgorc |=
1646 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1647 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1648 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1649 adapter->stats.last_vfgotc |=
1650 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1651 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1652
1653 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1654 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1655 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1656 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1657 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1658}
1659
31186785
AD		 1660static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
1661{
1662 struct ixgbe_hw *hw = &adapter->hw;
56e94095
AD		 1663 int api[] = { ixgbe_mbox_api_11,
1664 ixgbe_mbox_api_10,
31186785
AD		 1665 ixgbe_mbox_api_unknown };
1666 int err = 0, idx = 0;
1667
55fdd45b 1668 spin_lock_bh(&adapter->mbx_lock);
31186785
AD		 1669
1670 while (api[idx] != ixgbe_mbox_api_unknown) {
1671 err = ixgbevf_negotiate_api_version(hw, api[idx]);
1672 if (!err)
1673 break;
1674 idx++;
1675 }
1676
55fdd45b 1677 spin_unlock_bh(&adapter->mbx_lock);
31186785
AD		 1678}
1679
795180d8 1680static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
92915f71
GR		 1681{
1682 struct net_device *netdev = adapter->netdev;
1683 struct ixgbe_hw *hw = &adapter->hw;
92915f71
GR		 1684
1685 ixgbevf_configure_msix(adapter);
1686
55fdd45b 1687 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1688
92fe0bf7
GR		 1689 if (is_valid_ether_addr(hw->mac.addr))
1690 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1691 else
1692 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
92915f71 1693
55fdd45b 1694 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1695
4e857c58 1696 smp_mb__before_atomic();
92915f71
GR		 1697 clear_bit(__IXGBEVF_DOWN, &adapter->state);
1698 ixgbevf_napi_enable_all(adapter);
1699
1700 /* enable transmits */
1701 netif_tx_start_all_queues(netdev);
1702
33bd9f60
GR		 1703 ixgbevf_save_reset_stats(adapter);
1704 ixgbevf_init_last_counter_stats(adapter);
1705
4b2cd27f 1706 hw->mac.get_link_status = 1;
92915f71 1707 mod_timer(&adapter->watchdog_timer, jiffies);
92915f71
GR		 1708}
1709
795180d8 1710void ixgbevf_up(struct ixgbevf_adapter *adapter)
92915f71 1711{
92915f71
GR		 1712 struct ixgbe_hw *hw = &adapter->hw;
1713
1714 ixgbevf_configure(adapter);
1715
795180d8 1716 ixgbevf_up_complete(adapter);
92915f71
GR		 1717
1718 /* clear any pending interrupts, may auto mask */
1719 IXGBE_READ_REG(hw, IXGBE_VTEICR);
1720
5f3600eb 1721 ixgbevf_irq_enable(adapter);
92915f71
GR		 1722}
1723
1724/**
1725 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
92915f71
GR		 1726 * @rx_ring: ring to free buffers from
1727 **/
05d063aa 1728static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
92915f71 1729{
92915f71
GR		 1730 unsigned long size;
1731 unsigned int i;
1732
c0456c23
GR		 1733 if (!rx_ring->rx_buffer_info)
1734 return;
92915f71 1735
c0456c23 1736 /* Free all the Rx ring sk_buffs */
92915f71
GR		 1737 for (i = 0; i < rx_ring->count; i++) {
1738 struct ixgbevf_rx_buffer *rx_buffer_info;
1739
1740 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1741 if (rx_buffer_info->dma) {
05d063aa 1742 dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
92915f71 1743 rx_ring->rx_buf_len,
2a1f8794 1744 DMA_FROM_DEVICE);
92915f71
GR		 1745 rx_buffer_info->dma = 0;
1746 }
1747 if (rx_buffer_info->skb) {
1748 struct sk_buff *skb = rx_buffer_info->skb;
1749 rx_buffer_info->skb = NULL;
1750 do {
1751 struct sk_buff *this = skb;
5c60f81a 1752 skb = IXGBE_CB(skb)->prev;
92915f71
GR		 1753 dev_kfree_skb(this);
1754 } while (skb);
1755 }
92915f71
GR		 1756 }
1757
1758 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1759 memset(rx_ring->rx_buffer_info, 0, size);
1760
1761 /* Zero out the descriptor ring */
1762 memset(rx_ring->desc, 0, rx_ring->size);
92915f71
GR		 1763}
1764
1765/**
1766 * ixgbevf_clean_tx_ring - Free Tx Buffers
92915f71
GR		 1767 * @tx_ring: ring to be cleaned
1768 **/
05d063aa 1769static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
92915f71
GR		 1770{
1771 struct ixgbevf_tx_buffer *tx_buffer_info;
1772 unsigned long size;
1773 unsigned int i;
1774
c0456c23
GR		 1775 if (!tx_ring->tx_buffer_info)
1776 return;
1777
92915f71 1778 /* Free all the Tx ring sk_buffs */
92915f71
GR		 1779 for (i = 0; i < tx_ring->count; i++) {
1780 tx_buffer_info = &tx_ring->tx_buffer_info[i];
70a10e25 1781 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
92915f71
GR		 1782 }
1783
1784 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1785 memset(tx_ring->tx_buffer_info, 0, size);
1786
1787 memset(tx_ring->desc, 0, tx_ring->size);
92915f71
GR		 1788}
1789
1790/**
1791 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1792 * @adapter: board private structure
1793 **/
1794static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1795{
1796 int i;
1797
1798 for (i = 0; i < adapter->num_rx_queues; i++)
05d063aa 1799 ixgbevf_clean_rx_ring(adapter->rx_ring[i]);
92915f71
GR		 1800}
1801
1802/**
1803 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1804 * @adapter: board private structure
1805 **/
1806static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1807{
1808 int i;
1809
1810 for (i = 0; i < adapter->num_tx_queues; i++)
05d063aa 1811 ixgbevf_clean_tx_ring(adapter->tx_ring[i]);
92915f71
GR		 1812}
1813
1814void ixgbevf_down(struct ixgbevf_adapter *adapter)
1815{
1816 struct net_device *netdev = adapter->netdev;
1817 struct ixgbe_hw *hw = &adapter->hw;
de02decb 1818 int i;
92915f71
GR		 1819
1820 /* signal that we are down to the interrupt handler */
5b346dc9
MR		 1821 if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state))
1822 return; /* do nothing if already down */
858c3dda
DS		 1823
1824 /* disable all enabled rx queues */
1825 for (i = 0; i < adapter->num_rx_queues; i++)
87e70ab9 1826 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]);
92915f71
GR		 1827
1828 netif_tx_disable(netdev);
1829
1830 msleep(10);
1831
1832 netif_tx_stop_all_queues(netdev);
1833
1834 ixgbevf_irq_disable(adapter);
1835
1836 ixgbevf_napi_disable_all(adapter);
1837
1838 del_timer_sync(&adapter->watchdog_timer);
1839 /* can't call flush scheduled work here because it can deadlock
1840 * if linkwatch_event tries to acquire the rtnl_lock which we are
1841 * holding */
1842 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1843 msleep(1);
1844
1845 /* disable transmits in the hardware now that interrupts are off */
1846 for (i = 0; i < adapter->num_tx_queues; i++) {
de02decb
DS		 1847 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
1848
1849 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
1850 IXGBE_TXDCTL_SWFLSH);
92915f71
GR		 1851 }
1852
1853 netif_carrier_off(netdev);
1854
1855 if (!pci_channel_offline(adapter->pdev))
1856 ixgbevf_reset(adapter);
1857
1858 ixgbevf_clean_all_tx_rings(adapter);
1859 ixgbevf_clean_all_rx_rings(adapter);
1860}
1861
1862void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1863{
1864 WARN_ON(in_interrupt());
c0456c23 1865
92915f71
GR		 1866 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1867 msleep(1);
1868
4b2cd27f
AD		 1869 ixgbevf_down(adapter);
1870 ixgbevf_up(adapter);
92915f71
GR		 1871
1872 clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1873}
1874
1875void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1876{
1877 struct ixgbe_hw *hw = &adapter->hw;
1878 struct net_device *netdev = adapter->netdev;
1879
798e381a 1880 if (hw->mac.ops.reset_hw(hw)) {
92915f71 1881 hw_dbg(hw, "PF still resetting\n");
798e381a 1882 } else {
92915f71 1883 hw->mac.ops.init_hw(hw);
798e381a
DS		 1884 ixgbevf_negotiate_api(adapter);
1885 }
92915f71
GR		 1886
1887 if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1888 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1889 netdev->addr_len);
1890 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1891 netdev->addr_len);
1892 }
1893}
1894
e45dd5fe
JK		 1895static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1896 int vectors)
92915f71 1897{
a5f9337b 1898 int vector_threshold;
92915f71 1899
fa71ae27
AD		 1900 /* We'll want at least 2 (vector_threshold):
1901 * 1) TxQ[0] + RxQ[0] handler
1902 * 2) Other (Link Status Change, etc.)
92915f71
GR		 1903 */
1904 vector_threshold = MIN_MSIX_COUNT;
1905
1906 /* The more we get, the more we will assign to Tx/Rx Cleanup
1907 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1908 * Right now, we simply care about how many we'll get; we'll
1909 * set them up later while requesting irq's.
1910 */
5c1e3588
AG		 1911 vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1912 vector_threshold, vectors);
92915f71 1913
5c1e3588 1914 if (vectors < 0) {
e45dd5fe
JK		 1915 dev_err(&adapter->pdev->dev,
1916 "Unable to allocate MSI-X interrupts\n");
92915f71
GR		 1917 kfree(adapter->msix_entries);
1918 adapter->msix_entries = NULL;
5c1e3588 1919 return vectors;
92915f71 1920 }
dee847f5 1921
5c1e3588
AG		 1922 /* Adjust for only the vectors we'll use, which is minimum
1923 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1924 * vectors we were allocated.
1925 */
1926 adapter->num_msix_vectors = vectors;
1927
1928 return 0;
92915f71
GR		 1929}
1930
49ce9c2c
BH		 1931/**
1932 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
92915f71
GR		 1933 * @adapter: board private structure to initialize
1934 *
1935 * This is the top level queue allocation routine. The order here is very
1936 * important, starting with the "most" number of features turned on at once,
1937 * and ending with the smallest set of features. This way large combinations
1938 * can be allocated if they're turned on, and smaller combinations are the
1939 * fallthrough conditions.
1940 *
1941 **/
1942static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1943{
220fe050
DS		 1944 struct ixgbe_hw *hw = &adapter->hw;
1945 unsigned int def_q = 0;
1946 unsigned int num_tcs = 0;
1947 int err;
1948
92915f71
GR		 1949 /* Start with base case */
1950 adapter->num_rx_queues = 1;
1951 adapter->num_tx_queues = 1;
220fe050
DS		 1952
1953 spin_lock_bh(&adapter->mbx_lock);
1954
1955 /* fetch queue configuration from the PF */
1956 err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1957
1958 spin_unlock_bh(&adapter->mbx_lock);
1959
1960 if (err)
1961 return;
1962
1963 /* we need as many queues as traffic classes */
1964 if (num_tcs > 1)
1965 adapter->num_rx_queues = num_tcs;
92915f71
GR		 1966}
1967
1968/**
1969 * ixgbevf_alloc_queues - Allocate memory for all rings
1970 * @adapter: board private structure to initialize
1971 *
1972 * We allocate one ring per queue at run-time since we don't know the
1973 * number of queues at compile-time. The polling_netdev array is
1974 * intended for Multiqueue, but should work fine with a single queue.
1975 **/
1976static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1977{
87e70ab9
DS		 1978 struct ixgbevf_ring *ring;
1979 int rx = 0, tx = 0;
92915f71 1980
87e70ab9
DS		 1981 for (; tx < adapter->num_tx_queues; tx++) {
1982 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1983 if (!ring)
1984 goto err_allocation;
92915f71 1985
87e70ab9
DS		 1986 ring->dev = &adapter->pdev->dev;
1987 ring->netdev = adapter->netdev;
1988 ring->count = adapter->tx_ring_count;
1989 ring->queue_index = tx;
1990 ring->reg_idx = tx;
92915f71 1991
87e70ab9 1992 adapter->tx_ring[tx] = ring;
92915f71
GR		 1993 }
1994
87e70ab9
DS		 1995 for (; rx < adapter->num_rx_queues; rx++) {
1996 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1997 if (!ring)
1998 goto err_allocation;
1999
2000 ring->dev = &adapter->pdev->dev;
2001 ring->netdev = adapter->netdev;
2002
2003 ring->count = adapter->rx_ring_count;
2004 ring->queue_index = rx;
2005 ring->reg_idx = rx;
2006
2007 adapter->rx_ring[rx] = ring;
92915f71
GR		 2008 }
2009
2010 return 0;
2011
87e70ab9
DS		 2012err_allocation:
2013 while (tx) {
2014 kfree(adapter->tx_ring[--tx]);
2015 adapter->tx_ring[tx] = NULL;
2016 }
2017
2018 while (rx) {
2019 kfree(adapter->rx_ring[--rx]);
2020 adapter->rx_ring[rx] = NULL;
2021 }
92915f71
GR		 2022 return -ENOMEM;
2023}
2024
2025/**
2026 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
2027 * @adapter: board private structure to initialize
2028 *
2029 * Attempt to configure the interrupts using the best available
2030 * capabilities of the hardware and the kernel.
2031 **/
2032static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
2033{
91e2b89b 2034 struct net_device *netdev = adapter->netdev;
92915f71
GR		 2035 int err = 0;
2036 int vector, v_budget;
2037
2038 /*
2039 * It's easy to be greedy for MSI-X vectors, but it really
2040 * doesn't do us much good if we have a lot more vectors
2041 * than CPU's. So let's be conservative and only ask for
fa71ae27
AD		 2042 * (roughly) the same number of vectors as there are CPU's.
2043 * The default is to use pairs of vectors.
92915f71 2044 */
fa71ae27
AD		 2045 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
2046 v_budget = min_t(int, v_budget, num_online_cpus());
2047 v_budget += NON_Q_VECTORS;
92915f71
GR		 2048
2049 /* A failure in MSI-X entry allocation isn't fatal, but it does
2050 * mean we disable MSI-X capabilities of the adapter. */
2051 adapter->msix_entries = kcalloc(v_budget,
2052 sizeof(struct msix_entry), GFP_KERNEL);
2053 if (!adapter->msix_entries) {
2054 err = -ENOMEM;
2055 goto out;
2056 }
2057
2058 for (vector = 0; vector < v_budget; vector++)
2059 adapter->msix_entries[vector].entry = vector;
2060
e45dd5fe
JK		 2061 err = ixgbevf_acquire_msix_vectors(adapter, v_budget);
2062 if (err)
2063 goto out;
92915f71 2064
91e2b89b
GR		 2065 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
2066 if (err)
2067 goto out;
2068
2069 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
2070
92915f71
GR		 2071out:
2072 return err;
2073}
2074
2075/**
2076 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
2077 * @adapter: board private structure to initialize
2078 *
2079 * We allocate one q_vector per queue interrupt. If allocation fails we
2080 * return -ENOMEM.
2081 **/
2082static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
2083{
2084 int q_idx, num_q_vectors;
2085 struct ixgbevf_q_vector *q_vector;
92915f71
GR		 2086
2087 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
92915f71
GR		 2088
2089 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2090 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
2091 if (!q_vector)
2092 goto err_out;
2093 q_vector->adapter = adapter;
2094 q_vector->v_idx = q_idx;
fa71ae27
AD		 2095 netif_napi_add(adapter->netdev, &q_vector->napi,
2096 ixgbevf_poll, 64);
c777cdfa
JK		 2097#ifdef CONFIG_NET_RX_BUSY_POLL
2098 napi_hash_add(&q_vector->napi);
2099#endif
92915f71
GR		 2100 adapter->q_vector[q_idx] = q_vector;
2101 }
2102
2103 return 0;
2104
2105err_out:
2106 while (q_idx) {
2107 q_idx--;
2108 q_vector = adapter->q_vector[q_idx];
c777cdfa
JK		 2109#ifdef CONFIG_NET_RX_BUSY_POLL
2110 napi_hash_del(&q_vector->napi);
2111#endif
92915f71
GR		 2112 netif_napi_del(&q_vector->napi);
2113 kfree(q_vector);
2114 adapter->q_vector[q_idx] = NULL;
2115 }
2116 return -ENOMEM;
2117}
2118
2119/**
2120 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2121 * @adapter: board private structure to initialize
2122 *
2123 * This function frees the memory allocated to the q_vectors. In addition if
2124 * NAPI is enabled it will delete any references to the NAPI struct prior
2125 * to freeing the q_vector.
2126 **/
2127static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2128{
f4477702 2129 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
92915f71
GR		 2130
2131 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2132 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
2133
2134 adapter->q_vector[q_idx] = NULL;
c777cdfa
JK		 2135#ifdef CONFIG_NET_RX_BUSY_POLL
2136 napi_hash_del(&q_vector->napi);
2137#endif
f4477702 2138 netif_napi_del(&q_vector->napi);
92915f71
GR		 2139 kfree(q_vector);
2140 }
2141}
2142
2143/**
2144 * ixgbevf_reset_interrupt_capability - Reset MSIX setup
2145 * @adapter: board private structure
2146 *
2147 **/
2148static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2149{
2150 pci_disable_msix(adapter->pdev);
2151 kfree(adapter->msix_entries);
2152 adapter->msix_entries = NULL;
92915f71
GR		 2153}
2154
2155/**
2156 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2157 * @adapter: board private structure to initialize
2158 *
2159 **/
2160static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2161{
2162 int err;
2163
2164 /* Number of supported queues */
2165 ixgbevf_set_num_queues(adapter);
2166
2167 err = ixgbevf_set_interrupt_capability(adapter);
2168 if (err) {
2169 hw_dbg(&adapter->hw,
2170 "Unable to setup interrupt capabilities\n");
2171 goto err_set_interrupt;
2172 }
2173
2174 err = ixgbevf_alloc_q_vectors(adapter);
2175 if (err) {
2176 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
2177 "vectors\n");
2178 goto err_alloc_q_vectors;
2179 }
2180
2181 err = ixgbevf_alloc_queues(adapter);
2182 if (err) {
dbd9636e 2183 pr_err("Unable to allocate memory for queues\n");
92915f71
GR		 2184 goto err_alloc_queues;
2185 }
2186
2187 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
2188 "Tx Queue count = %u\n",
2189 (adapter->num_rx_queues > 1) ? "Enabled" :
2190 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
2191
2192 set_bit(__IXGBEVF_DOWN, &adapter->state);
2193
2194 return 0;
2195err_alloc_queues:
2196 ixgbevf_free_q_vectors(adapter);
2197err_alloc_q_vectors:
2198 ixgbevf_reset_interrupt_capability(adapter);
2199err_set_interrupt:
2200 return err;
2201}
2202
0ac1e8ce
AD		 2203/**
2204 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
2205 * @adapter: board private structure to clear interrupt scheme on
2206 *
2207 * We go through and clear interrupt specific resources and reset the structure
2208 * to pre-load conditions
2209 **/
2210static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
2211{
87e70ab9
DS		 2212 int i;
2213
2214 for (i = 0; i < adapter->num_tx_queues; i++) {
2215 kfree(adapter->tx_ring[i]);
2216 adapter->tx_ring[i] = NULL;
2217 }
2218 for (i = 0; i < adapter->num_rx_queues; i++) {
2219 kfree(adapter->rx_ring[i]);
2220 adapter->rx_ring[i] = NULL;
2221 }
2222
0ac1e8ce
AD		 2223 adapter->num_tx_queues = 0;
2224 adapter->num_rx_queues = 0;
2225
2226 ixgbevf_free_q_vectors(adapter);
2227 ixgbevf_reset_interrupt_capability(adapter);
2228}
2229
92915f71
GR		 2230/**
2231 * ixgbevf_sw_init - Initialize general software structures
2232 * (struct ixgbevf_adapter)
2233 * @adapter: board private structure to initialize
2234 *
2235 * ixgbevf_sw_init initializes the Adapter private data structure.
2236 * Fields are initialized based on PCI device information and
2237 * OS network device settings (MTU size).
2238 **/
9f9a12f8 2239static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
92915f71
GR		 2240{
2241 struct ixgbe_hw *hw = &adapter->hw;
2242 struct pci_dev *pdev = adapter->pdev;
e1941a74 2243 struct net_device *netdev = adapter->netdev;
92915f71
GR		 2244 int err;
2245
2246 /* PCI config space info */
2247
2248 hw->vendor_id = pdev->vendor;
2249 hw->device_id = pdev->device;
ff938e43 2250 hw->revision_id = pdev->revision;
92915f71
GR		 2251 hw->subsystem_vendor_id = pdev->subsystem_vendor;
2252 hw->subsystem_device_id = pdev->subsystem_device;
2253
2254 hw->mbx.ops.init_params(hw);
56e94095
AD		 2255
2256 /* assume legacy case in which PF would only give VF 2 queues */
2257 hw->mac.max_tx_queues = 2;
2258 hw->mac.max_rx_queues = 2;
2259
798e381a
DS		 2260 /* lock to protect mailbox accesses */
2261 spin_lock_init(&adapter->mbx_lock);
2262
92915f71
GR		 2263 err = hw->mac.ops.reset_hw(hw);
2264 if (err) {
2265 dev_info(&pdev->dev,
e1941a74 2266 "PF still in reset state. Is the PF interface up?\n");
92915f71
GR		 2267 } else {
2268 err = hw->mac.ops.init_hw(hw);
2269 if (err) {
dbd9636e 2270 pr_err("init_shared_code failed: %d\n", err);
92915f71
GR		 2271 goto out;
2272 }
798e381a 2273 ixgbevf_negotiate_api(adapter);
e1941a74
GR		 2274 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
2275 if (err)
2276 dev_info(&pdev->dev, "Error reading MAC address\n");
2277 else if (is_zero_ether_addr(adapter->hw.mac.addr))
2278 dev_info(&pdev->dev,
2279 "MAC address not assigned by administrator.\n");
2280 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2281 }
2282
2283 if (!is_valid_ether_addr(netdev->dev_addr)) {
2284 dev_info(&pdev->dev, "Assigning random MAC address\n");
2285 eth_hw_addr_random(netdev);
2286 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len);
92915f71
GR		 2287 }
2288
2289 /* Enable dynamic interrupt throttling rates */
5f3600eb
AD		 2290 adapter->rx_itr_setting = 1;
2291 adapter->tx_itr_setting = 1;
92915f71 2292
92915f71
GR		 2293 /* set default ring sizes */
2294 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
2295 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
2296
92915f71 2297 set_bit(__IXGBEVF_DOWN, &adapter->state);
1a0d6ae5 2298 return 0;
92915f71
GR		 2299
2300out:
2301 return err;
2302}
2303
92915f71
GR		 2304#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \
2305 { \
2306 u32 current_counter = IXGBE_READ_REG(hw, reg); \
2307 if (current_counter < last_counter) \
2308 counter += 0x100000000LL; \
2309 last_counter = current_counter; \
2310 counter &= 0xFFFFFFFF00000000LL; \
2311 counter |= current_counter; \
2312 }
2313
2314#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2315 { \
2316 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \
2317 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \
2318 u64 current_counter = (current_counter_msb << 32) | \
2319 current_counter_lsb; \
2320 if (current_counter < last_counter) \
2321 counter += 0x1000000000LL; \
2322 last_counter = current_counter; \
2323 counter &= 0xFFFFFFF000000000LL; \
2324 counter |= current_counter; \
2325 }
2326/**
2327 * ixgbevf_update_stats - Update the board statistics counters.
2328 * @adapter: board private structure
2329 **/
2330void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2331{
2332 struct ixgbe_hw *hw = &adapter->hw;
55fb277c 2333 int i;
92915f71 2334
088245a3
GR		 2335 if (!adapter->link_up)
2336 return;
2337
92915f71
GR		 2338 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2339 adapter->stats.vfgprc);
2340 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2341 adapter->stats.vfgptc);
2342 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2343 adapter->stats.last_vfgorc,
2344 adapter->stats.vfgorc);
2345 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2346 adapter->stats.last_vfgotc,
2347 adapter->stats.vfgotc);
2348 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2349 adapter->stats.vfmprc);
55fb277c
GR		 2350
2351 for (i = 0; i < adapter->num_rx_queues; i++) {
2352 adapter->hw_csum_rx_error +=
87e70ab9 2353 adapter->rx_ring[i]->hw_csum_rx_error;
87e70ab9 2354 adapter->rx_ring[i]->hw_csum_rx_error = 0;
55fb277c 2355 }
92915f71
GR		 2356}
2357
2358/**
2359 * ixgbevf_watchdog - Timer Call-back
2360 * @data: pointer to adapter cast into an unsigned long
2361 **/
2362static void ixgbevf_watchdog(unsigned long data)
2363{
2364 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2365 struct ixgbe_hw *hw = &adapter->hw;
5f3600eb 2366 u32 eics = 0;
92915f71
GR		 2367 int i;
2368
2369 /*
2370 * Do the watchdog outside of interrupt context due to the lovely
2371 * delays that some of the newer hardware requires
2372 */
2373
2374 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2375 goto watchdog_short_circuit;
2376
2377 /* get one bit for every active tx/rx interrupt vector */
2378 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2379 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
6b43c446 2380 if (qv->rx.ring || qv->tx.ring)
5f3600eb 2381 eics |= 1 << i;
92915f71
GR		 2382 }
2383
5f3600eb 2384 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
92915f71
GR		 2385
2386watchdog_short_circuit:
2387 schedule_work(&adapter->watchdog_task);
2388}
2389
2390/**
2391 * ixgbevf_tx_timeout - Respond to a Tx Hang
2392 * @netdev: network interface device structure
2393 **/
2394static void ixgbevf_tx_timeout(struct net_device *netdev)
2395{
2396 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2397
2398 /* Do the reset outside of interrupt context */
2399 schedule_work(&adapter->reset_task);
2400}
2401
2402static void ixgbevf_reset_task(struct work_struct *work)
2403{
2404 struct ixgbevf_adapter *adapter;
2405 adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2406
2407 /* If we're already down or resetting, just bail */
2408 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2e7cfbdd 2409 test_bit(__IXGBEVF_REMOVING, &adapter->state) ||
92915f71
GR		 2410 test_bit(__IXGBEVF_RESETTING, &adapter->state))
2411 return;
2412
2413 adapter->tx_timeout_count++;
2414
2415 ixgbevf_reinit_locked(adapter);
2416}
2417
2418/**
2419 * ixgbevf_watchdog_task - worker thread to bring link up
2420 * @work: pointer to work_struct containing our data
2421 **/
2422static void ixgbevf_watchdog_task(struct work_struct *work)
2423{
2424 struct ixgbevf_adapter *adapter = container_of(work,
2425 struct ixgbevf_adapter,
2426 watchdog_task);
2427 struct net_device *netdev = adapter->netdev;
2428 struct ixgbe_hw *hw = &adapter->hw;
2429 u32 link_speed = adapter->link_speed;
2430 bool link_up = adapter->link_up;
92fe0bf7 2431 s32 need_reset;
92915f71 2432
26597802
MR		 2433 if (IXGBE_REMOVED(hw->hw_addr)) {
2434 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) {
2435 rtnl_lock();
2436 ixgbevf_down(adapter);
2437 rtnl_unlock();
2438 }
2439 return;
2440 }
220fe050
DS		 2441 ixgbevf_queue_reset_subtask(adapter);
2442
92915f71
GR		 2443 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2444
2445 /*
2446 * Always check the link on the watchdog because we have
2447 * no LSC interrupt
2448 */
92fe0bf7 2449 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 2450
92fe0bf7 2451 need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
1c55ed76 2452
92fe0bf7 2453 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 2454
92fe0bf7
GR		 2455 if (need_reset) {
2456 adapter->link_up = link_up;
2457 adapter->link_speed = link_speed;
2458 netif_carrier_off(netdev);
2459 netif_tx_stop_all_queues(netdev);
2460 schedule_work(&adapter->reset_task);
2461 goto pf_has_reset;
92915f71
GR		 2462 }
2463 adapter->link_up = link_up;
2464 adapter->link_speed = link_speed;
2465
2466 if (link_up) {
2467 if (!netif_carrier_ok(netdev)) {
b876a744
GR		 2468 char *link_speed_string;
2469 switch (link_speed) {
2470 case IXGBE_LINK_SPEED_10GB_FULL:
2471 link_speed_string = "10 Gbps";
2472 break;
2473 case IXGBE_LINK_SPEED_1GB_FULL:
2474 link_speed_string = "1 Gbps";
2475 break;
2476 case IXGBE_LINK_SPEED_100_FULL:
2477 link_speed_string = "100 Mbps";
2478 break;
2479 default:
2480 link_speed_string = "unknown speed";
2481 break;
2482 }
6fe59675 2483 dev_info(&adapter->pdev->dev,
b876a744 2484 "NIC Link is Up, %s\n", link_speed_string);
92915f71
GR		 2485 netif_carrier_on(netdev);
2486 netif_tx_wake_all_queues(netdev);
92915f71
GR		 2487 }
2488 } else {
2489 adapter->link_up = false;
2490 adapter->link_speed = 0;
2491 if (netif_carrier_ok(netdev)) {
6fe59675 2492 dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
92915f71
GR		 2493 netif_carrier_off(netdev);
2494 netif_tx_stop_all_queues(netdev);
2495 }
2496 }
2497
92915f71
GR		 2498 ixgbevf_update_stats(adapter);
2499
33bd9f60 2500pf_has_reset:
92915f71 2501 /* Reset the timer */
2e7cfbdd
MR		 2502 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
2503 !test_bit(__IXGBEVF_REMOVING, &adapter->state))
92915f71
GR		 2504 mod_timer(&adapter->watchdog_timer,
2505 round_jiffies(jiffies + (2 * HZ)));
2506
2507 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2508}
2509
2510/**
2511 * ixgbevf_free_tx_resources - Free Tx Resources per Queue
92915f71
GR		 2512 * @tx_ring: Tx descriptor ring for a specific queue
2513 *
2514 * Free all transmit software resources
2515 **/
05d063aa 2516void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring)
92915f71 2517{
05d063aa 2518 ixgbevf_clean_tx_ring(tx_ring);
92915f71
GR		 2519
2520 vfree(tx_ring->tx_buffer_info);
2521 tx_ring->tx_buffer_info = NULL;
2522
de02decb
DS		 2523 /* if not set, then don't free */
2524 if (!tx_ring->desc)
2525 return;
2526
05d063aa 2527 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc,
2a1f8794 2528 tx_ring->dma);
92915f71
GR		 2529
2530 tx_ring->desc = NULL;
2531}
2532
2533/**
2534 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2535 * @adapter: board private structure
2536 *
2537 * Free all transmit software resources
2538 **/
2539static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2540{
2541 int i;
2542
2543 for (i = 0; i < adapter->num_tx_queues; i++)
87e70ab9 2544 if (adapter->tx_ring[i]->desc)
05d063aa 2545 ixgbevf_free_tx_resources(adapter->tx_ring[i]);
92915f71
GR		 2546}
2547
2548/**
2549 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
92915f71
GR		 2550 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2551 *
2552 * Return 0 on success, negative on failure
2553 **/
05d063aa 2554int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
92915f71 2555{
92915f71
GR		 2556 int size;
2557
2558 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
89bf67f1 2559 tx_ring->tx_buffer_info = vzalloc(size);
92915f71
GR		 2560 if (!tx_ring->tx_buffer_info)
2561 goto err;
92915f71
GR		 2562
2563 /* round up to nearest 4K */
2564 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2565 tx_ring->size = ALIGN(tx_ring->size, 4096);
2566
05d063aa 2567 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size,
2a1f8794 2568 &tx_ring->dma, GFP_KERNEL);
92915f71
GR		 2569 if (!tx_ring->desc)
2570 goto err;
2571
92915f71
GR		 2572 return 0;
2573
2574err:
2575 vfree(tx_ring->tx_buffer_info);
2576 tx_ring->tx_buffer_info = NULL;
2577 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2578 "descriptor ring\n");
2579 return -ENOMEM;
2580}
2581
2582/**
2583 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2584 * @adapter: board private structure
2585 *
2586 * If this function returns with an error, then it's possible one or
2587 * more of the rings is populated (while the rest are not). It is the
2588 * callers duty to clean those orphaned rings.
2589 *
2590 * Return 0 on success, negative on failure
2591 **/
2592static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2593{
2594 int i, err = 0;
2595
2596 for (i = 0; i < adapter->num_tx_queues; i++) {
05d063aa 2597 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]);
92915f71
GR		 2598 if (!err)
2599 continue;
2600 hw_dbg(&adapter->hw,
2601 "Allocation for Tx Queue %u failed\n", i);
2602 break;
2603 }
2604
2605 return err;
2606}
2607
2608/**
2609 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
92915f71
GR		 2610 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2611 *
2612 * Returns 0 on success, negative on failure
2613 **/
05d063aa 2614int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring)
92915f71 2615{
92915f71
GR		 2616 int size;
2617
2618 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
89bf67f1 2619 rx_ring->rx_buffer_info = vzalloc(size);
e404decb 2620 if (!rx_ring->rx_buffer_info)
05d063aa 2621 goto err;
92915f71
GR		 2622
2623 /* Round up to nearest 4K */
2624 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2625 rx_ring->size = ALIGN(rx_ring->size, 4096);
2626
05d063aa 2627 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size,
2a1f8794 2628 &rx_ring->dma, GFP_KERNEL);
92915f71 2629
05d063aa
ET		 2630 if (!rx_ring->desc)
2631 goto err;
92915f71 2632
92915f71 2633 return 0;
05d063aa
ET		 2634err:
2635 vfree(rx_ring->rx_buffer_info);
2636 rx_ring->rx_buffer_info = NULL;
2637 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n");
92915f71
GR		 2638 return -ENOMEM;
2639}
2640
2641/**
2642 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2643 * @adapter: board private structure
2644 *
2645 * If this function returns with an error, then it's possible one or
2646 * more of the rings is populated (while the rest are not). It is the
2647 * callers duty to clean those orphaned rings.
2648 *
2649 * Return 0 on success, negative on failure
2650 **/
2651static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2652{
2653 int i, err = 0;
2654
2655 for (i = 0; i < adapter->num_rx_queues; i++) {
05d063aa 2656 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]);
92915f71
GR		 2657 if (!err)
2658 continue;
2659 hw_dbg(&adapter->hw,
2660 "Allocation for Rx Queue %u failed\n", i);
2661 break;
2662 }
2663 return err;
2664}
2665
2666/**
2667 * ixgbevf_free_rx_resources - Free Rx Resources
92915f71
GR		 2668 * @rx_ring: ring to clean the resources from
2669 *
2670 * Free all receive software resources
2671 **/
05d063aa 2672void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring)
92915f71 2673{
05d063aa 2674 ixgbevf_clean_rx_ring(rx_ring);
92915f71
GR		 2675
2676 vfree(rx_ring->rx_buffer_info);
2677 rx_ring->rx_buffer_info = NULL;
2678
05d063aa 2679 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc,
2a1f8794 2680 rx_ring->dma);
92915f71
GR		 2681
2682 rx_ring->desc = NULL;
2683}
2684
2685/**
2686 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2687 * @adapter: board private structure
2688 *
2689 * Free all receive software resources
2690 **/
2691static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2692{
2693 int i;
2694
2695 for (i = 0; i < adapter->num_rx_queues; i++)
87e70ab9 2696 if (adapter->rx_ring[i]->desc)
05d063aa 2697 ixgbevf_free_rx_resources(adapter->rx_ring[i]);
92915f71
GR		 2698}
2699
2700/**
2701 * ixgbevf_open - Called when a network interface is made active
2702 * @netdev: network interface device structure
2703 *
2704 * Returns 0 on success, negative value on failure
2705 *
2706 * The open entry point is called when a network interface is made
2707 * active by the system (IFF_UP). At this point all resources needed
2708 * for transmit and receive operations are allocated, the interrupt
2709 * handler is registered with the OS, the watchdog timer is started,
2710 * and the stack is notified that the interface is ready.
2711 **/
2712static int ixgbevf_open(struct net_device *netdev)
2713{
2714 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2715 struct ixgbe_hw *hw = &adapter->hw;
2716 int err;
2717
a1f6c6b1 2718 /* A previous failure to open the device because of a lack of
2719 * available MSIX vector resources may have reset the number
2720 * of msix vectors variable to zero. The only way to recover
2721 * is to unload/reload the driver and hope that the system has
2722 * been able to recover some MSIX vector resources.
2723 */
2724 if (!adapter->num_msix_vectors)
2725 return -ENOMEM;
2726
92915f71
GR		 2727 /* disallow open during test */
2728 if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2729 return -EBUSY;
2730
2731 if (hw->adapter_stopped) {
2732 ixgbevf_reset(adapter);
2733 /* if adapter is still stopped then PF isn't up and
2734 * the vf can't start. */
2735 if (hw->adapter_stopped) {
2736 err = IXGBE_ERR_MBX;
dbd9636e
JK		 2737 pr_err("Unable to start - perhaps the PF Driver isn't "
2738 "up yet\n");
92915f71
GR		 2739 goto err_setup_reset;
2740 }
2741 }
2742
2743 /* allocate transmit descriptors */
2744 err = ixgbevf_setup_all_tx_resources(adapter);
2745 if (err)
2746 goto err_setup_tx;
2747
2748 /* allocate receive descriptors */
2749 err = ixgbevf_setup_all_rx_resources(adapter);
2750 if (err)
2751 goto err_setup_rx;
2752
2753 ixgbevf_configure(adapter);
2754
2755 /*
2756 * Map the Tx/Rx rings to the vectors we were allotted.
2757 * if request_irq will be called in this function map_rings
2758 * must be called *before* up_complete
2759 */
2760 ixgbevf_map_rings_to_vectors(adapter);
2761
795180d8 2762 ixgbevf_up_complete(adapter);
92915f71
GR		 2763
2764 /* clear any pending interrupts, may auto mask */
2765 IXGBE_READ_REG(hw, IXGBE_VTEICR);
2766 err = ixgbevf_request_irq(adapter);
2767 if (err)
2768 goto err_req_irq;
2769
5f3600eb 2770 ixgbevf_irq_enable(adapter);
92915f71
GR		 2771
2772 return 0;
2773
2774err_req_irq:
2775 ixgbevf_down(adapter);
92915f71
GR		 2776err_setup_rx:
2777 ixgbevf_free_all_rx_resources(adapter);
2778err_setup_tx:
2779 ixgbevf_free_all_tx_resources(adapter);
2780 ixgbevf_reset(adapter);
2781
2782err_setup_reset:
2783
2784 return err;
2785}
2786
2787/**
2788 * ixgbevf_close - Disables a network interface
2789 * @netdev: network interface device structure
2790 *
2791 * Returns 0, this is not allowed to fail
2792 *
2793 * The close entry point is called when an interface is de-activated
2794 * by the OS. The hardware is still under the drivers control, but
2795 * needs to be disabled. A global MAC reset is issued to stop the
2796 * hardware, and all transmit and receive resources are freed.
2797 **/
2798static int ixgbevf_close(struct net_device *netdev)
2799{
2800 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2801
2802 ixgbevf_down(adapter);
2803 ixgbevf_free_irq(adapter);
2804
2805 ixgbevf_free_all_tx_resources(adapter);
2806 ixgbevf_free_all_rx_resources(adapter);
2807
2808 return 0;
2809}
2810
220fe050
DS		 2811static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter)
2812{
2813 struct net_device *dev = adapter->netdev;
2814
2815 if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED))
2816 return;
2817
2818 adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
2819
2820 /* if interface is down do nothing */
2821 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2822 test_bit(__IXGBEVF_RESETTING, &adapter->state))
2823 return;
2824
2825 /* Hardware has to reinitialize queues and interrupts to
2826 * match packet buffer alignment. Unfortunately, the
2827 * hardware is not flexible enough to do this dynamically.
2828 */
2829 if (netif_running(dev))
2830 ixgbevf_close(dev);
2831
2832 ixgbevf_clear_interrupt_scheme(adapter);
2833 ixgbevf_init_interrupt_scheme(adapter);
2834
2835 if (netif_running(dev))
2836 ixgbevf_open(dev);
2837}
2838
70a10e25
AD		 2839static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
2840 u32 vlan_macip_lens, u32 type_tucmd,
2841 u32 mss_l4len_idx)
92915f71
GR		 2842{
2843 struct ixgbe_adv_tx_context_desc *context_desc;
70a10e25 2844 u16 i = tx_ring->next_to_use;
92915f71 2845
70a10e25 2846 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
92915f71 2847
70a10e25
AD		 2848 i++;
2849 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
92915f71 2850
70a10e25
AD		 2851 /* set bits to identify this as an advanced context descriptor */
2852 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
92915f71 2853
70a10e25
AD		 2854 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
2855 context_desc->seqnum_seed = 0;
2856 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
2857 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
2858}
2859
2860static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
7ad1a093
ET		 2861 struct ixgbevf_tx_buffer *first,
2862 u8 *hdr_len)
70a10e25 2863{
7ad1a093 2864 struct sk_buff *skb = first->skb;
70a10e25
AD		 2865 u32 vlan_macip_lens, type_tucmd;
2866 u32 mss_l4len_idx, l4len;
8f12c034 2867 int err;
70a10e25 2868
01a545cf
ET		 2869 if (skb->ip_summed != CHECKSUM_PARTIAL)
2870 return 0;
2871
70a10e25
AD		 2872 if (!skb_is_gso(skb))
2873 return 0;
92915f71 2874
8f12c034
FR		 2875 err = skb_cow_head(skb, 0);
2876 if (err < 0)
2877 return err;
92915f71 2878
70a10e25
AD		 2879 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2880 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
2881
2882 if (skb->protocol == htons(ETH_P_IP)) {
2883 struct iphdr *iph = ip_hdr(skb);
2884 iph->tot_len = 0;
2885 iph->check = 0;
2886 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2887 iph->daddr, 0,
2888 IPPROTO_TCP,
2889 0);
2890 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
7ad1a093
ET		 2891 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
2892 IXGBE_TX_FLAGS_CSUM |
2893 IXGBE_TX_FLAGS_IPV4;
70a10e25
AD		 2894 } else if (skb_is_gso_v6(skb)) {
2895 ipv6_hdr(skb)->payload_len = 0;
2896 tcp_hdr(skb)->check =
2897 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2898 &ipv6_hdr(skb)->daddr,
2899 0, IPPROTO_TCP, 0);
7ad1a093
ET		 2900 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
2901 IXGBE_TX_FLAGS_CSUM;
70a10e25
AD		 2902 }
2903
2904 /* compute header lengths */
2905 l4len = tcp_hdrlen(skb);
2906 *hdr_len += l4len;
2907 *hdr_len = skb_transport_offset(skb) + l4len;
2908
7ad1a093
ET		 2909 /* update gso size and bytecount with header size */
2910 first->gso_segs = skb_shinfo(skb)->gso_segs;
2911 first->bytecount += (first->gso_segs - 1) * *hdr_len;
2912
70a10e25
AD		 2913 /* mss_l4len_id: use 1 as index for TSO */
2914 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
2915 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
2916 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
2917
2918 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
2919 vlan_macip_lens = skb_network_header_len(skb);
2920 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
7ad1a093 2921 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
70a10e25
AD		 2922
2923 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2924 type_tucmd, mss_l4len_idx);
2925
2926 return 1;
92915f71
GR		 2927}
2928
7ad1a093
ET		 2929static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
2930 struct ixgbevf_tx_buffer *first)
92915f71 2931{
7ad1a093 2932 struct sk_buff *skb = first->skb;
70a10e25
AD		 2933 u32 vlan_macip_lens = 0;
2934 u32 mss_l4len_idx = 0;
2935 u32 type_tucmd = 0;
92915f71 2936
70a10e25
AD		 2937 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2938 u8 l4_hdr = 0;
2939 switch (skb->protocol) {
0933ce4a 2940 case htons(ETH_P_IP):
70a10e25
AD		 2941 vlan_macip_lens |= skb_network_header_len(skb);
2942 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2943 l4_hdr = ip_hdr(skb)->protocol;
2944 break;
0933ce4a 2945 case htons(ETH_P_IPV6):
70a10e25
AD		 2946 vlan_macip_lens |= skb_network_header_len(skb);
2947 l4_hdr = ipv6_hdr(skb)->nexthdr;
2948 break;
2949 default:
2950 if (unlikely(net_ratelimit())) {
2951 dev_warn(tx_ring->dev,
2952 "partial checksum but proto=%x!\n",
7ad1a093 2953 first->protocol);
70a10e25
AD		 2954 }
2955 break;
2956 }
92915f71 2957
70a10e25
AD		 2958 switch (l4_hdr) {
2959 case IPPROTO_TCP:
2960 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2961 mss_l4len_idx = tcp_hdrlen(skb) <<
2962 IXGBE_ADVTXD_L4LEN_SHIFT;
2963 break;
2964 case IPPROTO_SCTP:
2965 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2966 mss_l4len_idx = sizeof(struct sctphdr) <<
2967 IXGBE_ADVTXD_L4LEN_SHIFT;
2968 break;
2969 case IPPROTO_UDP:
2970 mss_l4len_idx = sizeof(struct udphdr) <<
2971 IXGBE_ADVTXD_L4LEN_SHIFT;
2972 break;
2973 default:
2974 if (unlikely(net_ratelimit())) {
2975 dev_warn(tx_ring->dev,
2976 "partial checksum but l4 proto=%x!\n",
2977 l4_hdr);
2978 }
2979 break;
2980 }
7ad1a093
ET		 2981
2982 /* update TX checksum flag */
2983 first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
92915f71
GR		 2984 }
2985
70a10e25
AD		 2986 /* vlan_macip_lens: MACLEN, VLAN tag */
2987 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
7ad1a093 2988 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
70a10e25
AD		 2989
2990 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2991 type_tucmd, mss_l4len_idx);
92915f71
GR		 2992}
2993
29d37fa1 2994static __le32 ixgbevf_tx_cmd_type(u32 tx_flags)
92915f71 2995{
29d37fa1
ET		 2996 /* set type for advanced descriptor with frame checksum insertion */
2997 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
2998 IXGBE_ADVTXD_DCMD_IFCS |
2999 IXGBE_ADVTXD_DCMD_DEXT);
92915f71 3000
29d37fa1
ET		 3001 /* set HW vlan bit if vlan is present */
3002 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
3003 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
92915f71 3004
29d37fa1
ET		 3005 /* set segmentation enable bits for TSO/FSO */
3006 if (tx_flags & IXGBE_TX_FLAGS_TSO)
3007 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
92915f71 3008
29d37fa1
ET		 3009 return cmd_type;
3010}
92915f71 3011
29d37fa1
ET		 3012static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
3013 u32 tx_flags, unsigned int paylen)
3014{
3015 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
9bdfefd2 3016
29d37fa1
ET		 3017 /* enable L4 checksum for TSO and TX checksum offload */
3018 if (tx_flags & IXGBE_TX_FLAGS_CSUM)
3019 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
92915f71 3020
29d37fa1
ET		 3021 /* enble IPv4 checksum for TSO */
3022 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
3023 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
92915f71 3024
29d37fa1
ET		 3025 /* use index 1 context for TSO/FSO/FCOE */
3026 if (tx_flags & IXGBE_TX_FLAGS_TSO)
3027 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT);
92915f71 3028
29d37fa1
ET		 3029 /* Check Context must be set if Tx switch is enabled, which it
3030 * always is for case where virtual functions are running
3031 */
3032 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC);
92915f71 3033
29d37fa1
ET		 3034 tx_desc->read.olinfo_status = olinfo_status;
3035}
92915f71 3036
29d37fa1
ET		 3037static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
3038 struct ixgbevf_tx_buffer *first,
3039 const u8 hdr_len)
3040{
3041 dma_addr_t dma;
3042 struct sk_buff *skb = first->skb;
3043 struct ixgbevf_tx_buffer *tx_buffer;
3044 union ixgbe_adv_tx_desc *tx_desc;
3045 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
3046 unsigned int data_len = skb->data_len;
3047 unsigned int size = skb_headlen(skb);
3048 unsigned int paylen = skb->len - hdr_len;
3049 u32 tx_flags = first->tx_flags;
3050 __le32 cmd_type;
3051 u16 i = tx_ring->next_to_use;
9bdfefd2 3052
29d37fa1 3053 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
92915f71 3054
29d37fa1
ET		 3055 ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen);
3056 cmd_type = ixgbevf_tx_cmd_type(tx_flags);
7ad1a093 3057
29d37fa1
ET		 3058 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
3059 if (dma_mapping_error(tx_ring->dev, dma))
3060 goto dma_error;
92915f71 3061
29d37fa1
ET		 3062 /* record length, and DMA address */
3063 dma_unmap_len_set(first, len, size);
3064 dma_unmap_addr_set(first, dma, dma);
92915f71 3065
29d37fa1 3066 tx_desc->read.buffer_addr = cpu_to_le64(dma);
92915f71 3067
29d37fa1
ET		 3068 for (;;) {
3069 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
3070 tx_desc->read.cmd_type_len =
3071 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
92915f71 3072
29d37fa1
ET		 3073 i++;
3074 tx_desc++;
3075 if (i == tx_ring->count) {
3076 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
3077 i = 0;
3078 }
92915f71 3079
29d37fa1
ET		 3080 dma += IXGBE_MAX_DATA_PER_TXD;
3081 size -= IXGBE_MAX_DATA_PER_TXD;
92915f71 3082
29d37fa1
ET		 3083 tx_desc->read.buffer_addr = cpu_to_le64(dma);
3084 tx_desc->read.olinfo_status = 0;
3085 }
92915f71 3086
29d37fa1
ET		 3087 if (likely(!data_len))
3088 break;
92915f71 3089
29d37fa1 3090 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
92915f71 3091
29d37fa1
ET		 3092 i++;
3093 tx_desc++;
3094 if (i == tx_ring->count) {
3095 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
3096 i = 0;
3097 }
92915f71 3098
29d37fa1
ET		 3099 size = skb_frag_size(frag);
3100 data_len -= size;
92915f71 3101
29d37fa1
ET		 3102 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
3103 DMA_TO_DEVICE);
3104 if (dma_mapping_error(tx_ring->dev, dma))
3105 goto dma_error;
70a10e25 3106
29d37fa1
ET		 3107 tx_buffer = &tx_ring->tx_buffer_info[i];
3108 dma_unmap_len_set(tx_buffer, len, size);
3109 dma_unmap_addr_set(tx_buffer, dma, dma);
92915f71 3110
29d37fa1
ET		 3111 tx_desc->read.buffer_addr = cpu_to_le64(dma);
3112 tx_desc->read.olinfo_status = 0;
3113
3114 frag++;
70a10e25 3115 }
92915f71 3116
29d37fa1
ET		 3117 /* write last descriptor with RS and EOP bits */
3118 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD);
3119 tx_desc->read.cmd_type_len = cmd_type;
3120
3121 /* set the timestamp */
3122 first->time_stamp = jiffies;
3123
3124 /* Force memory writes to complete before letting h/w know there
3125 * are new descriptors to fetch. (Only applicable for weak-ordered
3126 * memory model archs, such as IA-64).
3127 *
3128 * We also need this memory barrier (wmb) to make certain all of the
3129 * status bits have been updated before next_to_watch is written.
70a10e25 3130 */
29d37fa1 3131 wmb();
92915f71 3132
29d37fa1
ET		 3133 /* set next_to_watch value indicating a packet is present */
3134 first->next_to_watch = tx_desc;
92915f71 3135
29d37fa1
ET		 3136 i++;
3137 if (i == tx_ring->count)
3138 i = 0;
9bdfefd2 3139
29d37fa1 3140 tx_ring->next_to_use = i;
92915f71 3141
29d37fa1 3142 /* notify HW of packet */
06380db6 3143 ixgbevf_write_tail(tx_ring, i);
29d37fa1
ET		 3144
3145 return;
3146dma_error:
3147 dev_err(tx_ring->dev, "TX DMA map failed\n");
3148
3149 /* clear dma mappings for failed tx_buffer_info map */
3150 for (;;) {
3151 tx_buffer = &tx_ring->tx_buffer_info[i];
3152 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer);
3153 if (tx_buffer == first)
3154 break;
3155 if (i == 0)
3156 i = tx_ring->count;
3157 i--;
3158 }
92915f71 3159
92915f71 3160 tx_ring->next_to_use = i;
92915f71
GR		 3161}
3162
fb40195c 3163static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
92915f71 3164{
fb40195c 3165 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
92915f71
GR		 3166 /* Herbert's original patch had:
3167 * smp_mb__after_netif_stop_queue();
3168 * but since that doesn't exist yet, just open code it. */
3169 smp_mb();
3170
3171 /* We need to check again in a case another CPU has just
3172 * made room available. */
f880d07b 3173 if (likely(ixgbevf_desc_unused(tx_ring) < size))
92915f71
GR		 3174 return -EBUSY;
3175
3176 /* A reprieve! - use start_queue because it doesn't call schedule */
fb40195c 3177 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
095e2617
ET		 3178 ++tx_ring->tx_stats.restart_queue;
3179
92915f71
GR		 3180 return 0;
3181}
3182
fb40195c 3183static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
92915f71 3184{
f880d07b 3185 if (likely(ixgbevf_desc_unused(tx_ring) >= size))
92915f71 3186 return 0;
fb40195c 3187 return __ixgbevf_maybe_stop_tx(tx_ring, size);
92915f71
GR		 3188}
3189
3190static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
3191{
3192 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
7ad1a093 3193 struct ixgbevf_tx_buffer *first;
92915f71 3194 struct ixgbevf_ring *tx_ring;
7ad1a093
ET		 3195 int tso;
3196 u32 tx_flags = 0;
3595990a
AD		 3197 u16 count = TXD_USE_COUNT(skb_headlen(skb));
3198#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3199 unsigned short f;
3200#endif
7ad1a093 3201 u8 hdr_len = 0;
f9d08f16 3202 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL);
7ad1a093 3203
46acc460 3204 if (!dst_mac || is_link_local_ether_addr(dst_mac)) {
f9d08f16
GR		 3205 dev_kfree_skb(skb);
3206 return NETDEV_TX_OK;
3207 }
92915f71 3208
7ad1a093 3209 tx_ring = adapter->tx_ring[skb->queue_mapping];
92915f71 3210
3595990a
AD		 3211 /*
3212 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
3213 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
3214 * + 2 desc gap to keep tail from touching head,
3215 * + 1 desc for context descriptor,
3216 * otherwise try next time
3217 */
3218#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3219 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
3220 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
3221#else
3222 count += skb_shinfo(skb)->nr_frags;
3223#endif
fb40195c 3224 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
095e2617 3225 tx_ring->tx_stats.tx_busy++;
3595990a
AD		 3226 return NETDEV_TX_BUSY;
3227 }
3228
7ad1a093
ET		 3229 /* record the location of the first descriptor for this packet */
3230 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
3231 first->skb = skb;
3232 first->bytecount = skb->len;
3233 first->gso_segs = 1;
3234
eab6d18d 3235 if (vlan_tx_tag_present(skb)) {
92915f71
GR		 3236 tx_flags |= vlan_tx_tag_get(skb);
3237 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
3238 tx_flags |= IXGBE_TX_FLAGS_VLAN;
3239 }
3240
7ad1a093
ET		 3241 /* record initial flags and protocol */
3242 first->tx_flags = tx_flags;
3243 first->protocol = vlan_get_protocol(skb);
92915f71 3244
7ad1a093
ET		 3245 tso = ixgbevf_tso(tx_ring, first, &hdr_len);
3246 if (tso < 0)
3247 goto out_drop;
b5d217f3 3248 else if (!tso)
7ad1a093 3249 ixgbevf_tx_csum(tx_ring, first);
92915f71 3250
29d37fa1 3251 ixgbevf_tx_map(tx_ring, first, hdr_len);
70a10e25 3252
fb40195c 3253 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
92915f71 3254
7ad1a093
ET		 3255 return NETDEV_TX_OK;
3256
3257out_drop:
3258 dev_kfree_skb_any(first->skb);
3259 first->skb = NULL;
3260
92915f71
GR		 3261 return NETDEV_TX_OK;
3262}
3263
92915f71
GR		 3264/**
3265 * ixgbevf_set_mac - Change the Ethernet Address of the NIC
3266 * @netdev: network interface device structure
3267 * @p: pointer to an address structure
3268 *
3269 * Returns 0 on success, negative on failure
3270 **/
3271static int ixgbevf_set_mac(struct net_device *netdev, void *p)
3272{
3273 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3274 struct ixgbe_hw *hw = &adapter->hw;
3275 struct sockaddr *addr = p;
3276
3277 if (!is_valid_ether_addr(addr->sa_data))
3278 return -EADDRNOTAVAIL;
3279
3280 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3281 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3282
55fdd45b 3283 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 3284
92fe0bf7 3285 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
92915f71 3286
55fdd45b 3287 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 3288
92915f71
GR		 3289 return 0;
3290}
3291
3292/**
3293 * ixgbevf_change_mtu - Change the Maximum Transfer Unit
3294 * @netdev: network interface device structure
3295 * @new_mtu: new value for maximum frame size
3296 *
3297 * Returns 0 on success, negative on failure
3298 **/
3299static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
3300{
3301 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3302 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
69bfbec4 3303 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
69bfbec4 3304
56e94095
AD		 3305 switch (adapter->hw.api_version) {
3306 case ixgbe_mbox_api_11:
69bfbec4 3307 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
56e94095
AD		 3308 break;
3309 default:
3310 if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
3311 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3312 break;
3313 }
92915f71
GR		 3314
3315 /* MTU < 68 is an error and causes problems on some kernels */
69bfbec4 3316 if ((new_mtu < 68) || (max_frame > max_possible_frame))
92915f71
GR		 3317 return -EINVAL;
3318
3319 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
3320 netdev->mtu, new_mtu);
3321 /* must set new MTU before calling down or up */
3322 netdev->mtu = new_mtu;
3323
3324 if (netif_running(netdev))
3325 ixgbevf_reinit_locked(adapter);
3326
3327 return 0;
3328}
3329
0ac1e8ce 3330static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state)
92915f71
GR		 3331{
3332 struct net_device *netdev = pci_get_drvdata(pdev);
3333 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
0ac1e8ce
AD		 3334#ifdef CONFIG_PM
3335 int retval = 0;
3336#endif
92915f71
GR		 3337
3338 netif_device_detach(netdev);
3339
3340 if (netif_running(netdev)) {
0ac1e8ce 3341 rtnl_lock();
92915f71
GR		 3342 ixgbevf_down(adapter);
3343 ixgbevf_free_irq(adapter);
3344 ixgbevf_free_all_tx_resources(adapter);
3345 ixgbevf_free_all_rx_resources(adapter);
0ac1e8ce 3346 rtnl_unlock();
92915f71
GR		 3347 }
3348
0ac1e8ce 3349 ixgbevf_clear_interrupt_scheme(adapter);
92915f71 3350
0ac1e8ce
AD		 3351#ifdef CONFIG_PM
3352 retval = pci_save_state(pdev);
3353 if (retval)
3354 return retval;
92915f71 3355
0ac1e8ce 3356#endif
bc0c7151
MR		 3357 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state))
3358 pci_disable_device(pdev);
0ac1e8ce
AD		 3359
3360 return 0;
3361}
3362
3363#ifdef CONFIG_PM
3364static int ixgbevf_resume(struct pci_dev *pdev)
3365{
27ae2967
WY		 3366 struct net_device *netdev = pci_get_drvdata(pdev);
3367 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
0ac1e8ce
AD		 3368 u32 err;
3369
0ac1e8ce
AD		 3370 pci_restore_state(pdev);
3371 /*
3372 * pci_restore_state clears dev->state_saved so call
3373 * pci_save_state to restore it.
3374 */
3375 pci_save_state(pdev);
3376
3377 err = pci_enable_device_mem(pdev);
3378 if (err) {
3379 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
3380 return err;
3381 }
4e857c58 3382 smp_mb__before_atomic();
bc0c7151 3383 clear_bit(__IXGBEVF_DISABLED, &adapter->state);
0ac1e8ce
AD		 3384 pci_set_master(pdev);
3385
798e381a
DS		 3386 ixgbevf_reset(adapter);
3387
0ac1e8ce
AD		 3388 rtnl_lock();
3389 err = ixgbevf_init_interrupt_scheme(adapter);
3390 rtnl_unlock();
3391 if (err) {
3392 dev_err(&pdev->dev, "Cannot initialize interrupts\n");
3393 return err;
3394 }
3395
0ac1e8ce
AD		 3396 if (netif_running(netdev)) {
3397 err = ixgbevf_open(netdev);
3398 if (err)
3399 return err;
3400 }
3401
3402 netif_device_attach(netdev);
3403
3404 return err;
3405}
3406
3407#endif /* CONFIG_PM */
3408static void ixgbevf_shutdown(struct pci_dev *pdev)
3409{
3410 ixgbevf_suspend(pdev, PMSG_SUSPEND);
92915f71
GR		 3411}
3412
4197aa7b
ED		 3413static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3414 struct rtnl_link_stats64 *stats)
3415{
3416 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3417 unsigned int start;
3418 u64 bytes, packets;
3419 const struct ixgbevf_ring *ring;
3420 int i;
3421
3422 ixgbevf_update_stats(adapter);
3423
3424 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3425
3426 for (i = 0; i < adapter->num_rx_queues; i++) {
87e70ab9 3427 ring = adapter->rx_ring[i];
4197aa7b 3428 do {
57a7744e 3429 start = u64_stats_fetch_begin_irq(&ring->syncp);
095e2617
ET		 3430 bytes = ring->stats.bytes;
3431 packets = ring->stats.packets;
57a7744e 3432 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
4197aa7b
ED		 3433 stats->rx_bytes += bytes;
3434 stats->rx_packets += packets;
3435 }
3436
3437 for (i = 0; i < adapter->num_tx_queues; i++) {
87e70ab9 3438 ring = adapter->tx_ring[i];
4197aa7b 3439 do {
57a7744e 3440 start = u64_stats_fetch_begin_irq(&ring->syncp);
095e2617
ET		 3441 bytes = ring->stats.bytes;
3442 packets = ring->stats.packets;
57a7744e 3443 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
4197aa7b
ED		 3444 stats->tx_bytes += bytes;
3445 stats->tx_packets += packets;
3446 }
3447
3448 return stats;
3449}
3450
0ac1e8ce 3451static const struct net_device_ops ixgbevf_netdev_ops = {
c12db769
SH		 3452 .ndo_open = ixgbevf_open,
3453 .ndo_stop = ixgbevf_close,
3454 .ndo_start_xmit = ixgbevf_xmit_frame,
3455 .ndo_set_rx_mode = ixgbevf_set_rx_mode,
4197aa7b 3456 .ndo_get_stats64 = ixgbevf_get_stats,
92915f71 3457 .ndo_validate_addr = eth_validate_addr,
c12db769
SH		 3458 .ndo_set_mac_address = ixgbevf_set_mac,
3459 .ndo_change_mtu = ixgbevf_change_mtu,
3460 .ndo_tx_timeout = ixgbevf_tx_timeout,
c12db769
SH		 3461 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid,
3462 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid,
c777cdfa
JK		 3463#ifdef CONFIG_NET_RX_BUSY_POLL
3464 .ndo_busy_poll = ixgbevf_busy_poll_recv,
3465#endif
92915f71 3466};
92915f71
GR		 3467
3468static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3469{
0ac1e8ce 3470 dev->netdev_ops = &ixgbevf_netdev_ops;
92915f71
GR		 3471 ixgbevf_set_ethtool_ops(dev);
3472 dev->watchdog_timeo = 5 * HZ;
3473}
3474
3475/**
3476 * ixgbevf_probe - Device Initialization Routine
3477 * @pdev: PCI device information struct
3478 * @ent: entry in ixgbevf_pci_tbl
3479 *
3480 * Returns 0 on success, negative on failure
3481 *
3482 * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3483 * The OS initialization, configuring of the adapter private structure,
3484 * and a hardware reset occur.
3485 **/
1dd06ae8 3486static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
92915f71
GR		 3487{
3488 struct net_device *netdev;
3489 struct ixgbevf_adapter *adapter = NULL;
3490 struct ixgbe_hw *hw = NULL;
3491 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
92915f71
GR		 3492 int err, pci_using_dac;
3493
3494 err = pci_enable_device(pdev);
3495 if (err)
3496 return err;
3497
53567aa4 3498 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
92915f71
GR		 3499 pci_using_dac = 1;
3500 } else {
53567aa4 3501 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
92915f71 3502 if (err) {
53567aa4
RK		 3503 dev_err(&pdev->dev, "No usable DMA "
3504 "configuration, aborting\n");
3505 goto err_dma;
92915f71
GR		 3506 }
3507 pci_using_dac = 0;
3508 }
3509
3510 err = pci_request_regions(pdev, ixgbevf_driver_name);
3511 if (err) {
3512 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3513 goto err_pci_reg;
3514 }
3515
3516 pci_set_master(pdev);
3517
92915f71
GR		 3518 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3519 MAX_TX_QUEUES);
92915f71
GR		 3520 if (!netdev) {
3521 err = -ENOMEM;
3522 goto err_alloc_etherdev;
3523 }
3524
3525 SET_NETDEV_DEV(netdev, &pdev->dev);
3526
3527 pci_set_drvdata(pdev, netdev);
3528 adapter = netdev_priv(netdev);
3529
3530 adapter->netdev = netdev;
3531 adapter->pdev = pdev;
3532 hw = &adapter->hw;
3533 hw->back = adapter;
b3f4d599 3534 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
92915f71
GR		 3535
3536 /*
3537 * call save state here in standalone driver because it relies on
3538 * adapter struct to exist, and needs to call netdev_priv
3539 */
3540 pci_save_state(pdev);
3541
3542 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3543 pci_resource_len(pdev, 0));
dbf8b0d8 3544 adapter->io_addr = hw->hw_addr;
92915f71
GR		 3545 if (!hw->hw_addr) {
3546 err = -EIO;
3547 goto err_ioremap;
3548 }
3549
3550 ixgbevf_assign_netdev_ops(netdev);
3551
92915f71
GR		 3552 /* Setup hw api */
3553 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3554 hw->mac.type = ii->mac;
3555
3556 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
f416dfc0 3557 sizeof(struct ixgbe_mbx_operations));
92915f71 3558
92915f71
GR		 3559 /* setup the private structure */
3560 err = ixgbevf_sw_init(adapter);
1a0d6ae5
DK		 3561 if (err)
3562 goto err_sw_init;
3563
3564 /* The HW MAC address was set and/or determined in sw_init */
1a0d6ae5
DK		 3565 if (!is_valid_ether_addr(netdev->dev_addr)) {
3566 pr_err("invalid MAC address\n");
3567 err = -EIO;
3568 goto err_sw_init;
3569 }
92915f71 3570
471a76de 3571 netdev->hw_features = NETIF_F_SG |
92915f71 3572 NETIF_F_IP_CSUM |
471a76de
MM		 3573 NETIF_F_IPV6_CSUM |
3574 NETIF_F_TSO |
3575 NETIF_F_TSO6 |
3576 NETIF_F_RXCSUM;
3577
3578 netdev->features = netdev->hw_features |
f646968f
PM		 3579 NETIF_F_HW_VLAN_CTAG_TX |
3580 NETIF_F_HW_VLAN_CTAG_RX |
3581 NETIF_F_HW_VLAN_CTAG_FILTER;
92915f71 3582
92915f71
GR		 3583 netdev->vlan_features |= NETIF_F_TSO;
3584 netdev->vlan_features |= NETIF_F_TSO6;
3585 netdev->vlan_features |= NETIF_F_IP_CSUM;
3bfacf96 3586 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
92915f71
GR		 3587 netdev->vlan_features |= NETIF_F_SG;
3588
3589 if (pci_using_dac)
3590 netdev->features |= NETIF_F_HIGHDMA;
3591
01789349
JP		 3592 netdev->priv_flags |= IFF_UNICAST_FLT;
3593
92915f71 3594 init_timer(&adapter->watchdog_timer);
c061b18d 3595 adapter->watchdog_timer.function = ixgbevf_watchdog;
92915f71
GR		 3596 adapter->watchdog_timer.data = (unsigned long)adapter;
3597
ea699569
MR		 3598 if (IXGBE_REMOVED(hw->hw_addr)) {
3599 err = -EIO;
3600 goto err_sw_init;
3601 }
92915f71
GR		 3602 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3603 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
ea699569 3604 set_bit(__IXGBEVF_WORK_INIT, &adapter->state);
92915f71
GR		 3605
3606 err = ixgbevf_init_interrupt_scheme(adapter);
3607 if (err)
3608 goto err_sw_init;
3609
92915f71
GR		 3610 strcpy(netdev->name, "eth%d");
3611
3612 err = register_netdev(netdev);
3613 if (err)
3614 goto err_register;
3615
5d426ad1
GR		 3616 netif_carrier_off(netdev);
3617
33bd9f60
GR		 3618 ixgbevf_init_last_counter_stats(adapter);
3619
92915f71 3620 /* print the MAC address */
f794e7ef 3621 hw_dbg(hw, "%pM\n", netdev->dev_addr);
92915f71
GR		 3622
3623 hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3624
92915f71 3625 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
92915f71
GR		 3626 return 0;
3627
3628err_register:
0ac1e8ce 3629 ixgbevf_clear_interrupt_scheme(adapter);
92915f71
GR		 3630err_sw_init:
3631 ixgbevf_reset_interrupt_capability(adapter);
dbf8b0d8 3632 iounmap(adapter->io_addr);
92915f71
GR		 3633err_ioremap:
3634 free_netdev(netdev);
3635err_alloc_etherdev:
3636 pci_release_regions(pdev);
3637err_pci_reg:
3638err_dma:
bc0c7151
MR		 3639 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state))
3640 pci_disable_device(pdev);
92915f71
GR		 3641 return err;
3642}
3643
3644/**
3645 * ixgbevf_remove - Device Removal Routine
3646 * @pdev: PCI device information struct
3647 *
3648 * ixgbevf_remove is called by the PCI subsystem to alert the driver
3649 * that it should release a PCI device. The could be caused by a
3650 * Hot-Plug event, or because the driver is going to be removed from
3651 * memory.
3652 **/
9f9a12f8 3653static void ixgbevf_remove(struct pci_dev *pdev)
92915f71
GR		 3654{
3655 struct net_device *netdev = pci_get_drvdata(pdev);
3656 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3657
2e7cfbdd 3658 set_bit(__IXGBEVF_REMOVING, &adapter->state);
92915f71
GR		 3659
3660 del_timer_sync(&adapter->watchdog_timer);
3661
23f333a2 3662 cancel_work_sync(&adapter->reset_task);
92915f71
GR		 3663 cancel_work_sync(&adapter->watchdog_task);
3664
fd13a9ab 3665 if (netdev->reg_state == NETREG_REGISTERED)
92915f71 3666 unregister_netdev(netdev);
92915f71 3667
0ac1e8ce 3668 ixgbevf_clear_interrupt_scheme(adapter);
92915f71
GR		 3669 ixgbevf_reset_interrupt_capability(adapter);
3670
dbf8b0d8 3671 iounmap(adapter->io_addr);
92915f71
GR		 3672 pci_release_regions(pdev);
3673
3674 hw_dbg(&adapter->hw, "Remove complete\n");
3675
92915f71
GR		 3676 free_netdev(netdev);
3677
bc0c7151
MR		 3678 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state))
3679 pci_disable_device(pdev);
92915f71
GR		 3680}
3681
9f19f31d
AD		 3682/**
3683 * ixgbevf_io_error_detected - called when PCI error is detected
3684 * @pdev: Pointer to PCI device
3685 * @state: The current pci connection state
3686 *
3687 * This function is called after a PCI bus error affecting
3688 * this device has been detected.
3689 */
3690static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
3691 pci_channel_state_t state)
3692{
3693 struct net_device *netdev = pci_get_drvdata(pdev);
3694 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3695
ea699569
MR		 3696 if (!test_bit(__IXGBEVF_WORK_INIT, &adapter->state))
3697 return PCI_ERS_RESULT_DISCONNECT;
3698
bc0c7151 3699 rtnl_lock();
9f19f31d
AD		 3700 netif_device_detach(netdev);
3701
bc0c7151
MR		 3702 if (state == pci_channel_io_perm_failure) {
3703 rtnl_unlock();
9f19f31d 3704 return PCI_ERS_RESULT_DISCONNECT;
bc0c7151 3705 }
9f19f31d
AD		 3706
3707 if (netif_running(netdev))
3708 ixgbevf_down(adapter);
3709
bc0c7151
MR		 3710 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state))
3711 pci_disable_device(pdev);
3712 rtnl_unlock();
9f19f31d
AD		 3713
3714 /* Request a slot slot reset. */
3715 return PCI_ERS_RESULT_NEED_RESET;
3716}
3717
3718/**
3719 * ixgbevf_io_slot_reset - called after the pci bus has been reset.
3720 * @pdev: Pointer to PCI device
3721 *
3722 * Restart the card from scratch, as if from a cold-boot. Implementation
3723 * resembles the first-half of the ixgbevf_resume routine.
3724 */
3725static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
3726{
3727 struct net_device *netdev = pci_get_drvdata(pdev);
3728 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3729
3730 if (pci_enable_device_mem(pdev)) {
3731 dev_err(&pdev->dev,
3732 "Cannot re-enable PCI device after reset.\n");
3733 return PCI_ERS_RESULT_DISCONNECT;
3734 }
3735
4e857c58 3736 smp_mb__before_atomic();
bc0c7151 3737 clear_bit(__IXGBEVF_DISABLED, &adapter->state);
9f19f31d
AD		 3738 pci_set_master(pdev);
3739
3740 ixgbevf_reset(adapter);
3741
3742 return PCI_ERS_RESULT_RECOVERED;
3743}
3744
3745/**
3746 * ixgbevf_io_resume - called when traffic can start flowing again.
3747 * @pdev: Pointer to PCI device
3748 *
3749 * This callback is called when the error recovery driver tells us that
3750 * its OK to resume normal operation. Implementation resembles the
3751 * second-half of the ixgbevf_resume routine.
3752 */
3753static void ixgbevf_io_resume(struct pci_dev *pdev)
3754{
3755 struct net_device *netdev = pci_get_drvdata(pdev);
3756 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3757
3758 if (netif_running(netdev))
3759 ixgbevf_up(adapter);
3760
3761 netif_device_attach(netdev);
3762}
3763
3764/* PCI Error Recovery (ERS) */
3646f0e5 3765static const struct pci_error_handlers ixgbevf_err_handler = {
9f19f31d
AD		 3766 .error_detected = ixgbevf_io_error_detected,
3767 .slot_reset = ixgbevf_io_slot_reset,
3768 .resume = ixgbevf_io_resume,
3769};
3770
92915f71
GR		 3771static struct pci_driver ixgbevf_driver = {
3772 .name = ixgbevf_driver_name,
3773 .id_table = ixgbevf_pci_tbl,
3774 .probe = ixgbevf_probe,
9f9a12f8 3775 .remove = ixgbevf_remove,
0ac1e8ce
AD		 3776#ifdef CONFIG_PM
3777 /* Power Management Hooks */
3778 .suspend = ixgbevf_suspend,
3779 .resume = ixgbevf_resume,
3780#endif
92915f71 3781 .shutdown = ixgbevf_shutdown,
9f19f31d 3782 .err_handler = &ixgbevf_err_handler
92915f71
GR		 3783};
3784
3785/**
65d676c8 3786 * ixgbevf_init_module - Driver Registration Routine
92915f71 3787 *
65d676c8 3788 * ixgbevf_init_module is the first routine called when the driver is
92915f71
GR		 3789 * loaded. All it does is register with the PCI subsystem.
3790 **/
3791static int __init ixgbevf_init_module(void)
3792{
3793 int ret;
dbd9636e
JK		 3794 pr_info("%s - version %s\n", ixgbevf_driver_string,
3795 ixgbevf_driver_version);
92915f71 3796
dbd9636e 3797 pr_info("%s\n", ixgbevf_copyright);
92915f71
GR		 3798
3799 ret = pci_register_driver(&ixgbevf_driver);
3800 return ret;
3801}
3802
3803module_init(ixgbevf_init_module);
3804
3805/**
65d676c8 3806 * ixgbevf_exit_module - Driver Exit Cleanup Routine
92915f71 3807 *
65d676c8 3808 * ixgbevf_exit_module is called just before the driver is removed
92915f71
GR		 3809 * from memory.
3810 **/
3811static void __exit ixgbevf_exit_module(void)
3812{
3813 pci_unregister_driver(&ixgbevf_driver);
3814}
3815
3816#ifdef DEBUG
3817/**
65d676c8 3818 * ixgbevf_get_hw_dev_name - return device name string
92915f71
GR		 3819 * used by hardware layer to print debugging information
3820 **/
3821char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3822{
3823 struct ixgbevf_adapter *adapter = hw->back;
3824 return adapter->netdev->name;
3825}
3826
3827#endif
3828module_exit(ixgbevf_exit_module);
3829
3830/* ixgbevf_main.c */
Linux kernel - Deliverables
This page took 1.261816 seconds and 5 git commands to generate.