projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ixgbevf: use PCI_DEVICE_TABLE macro
[deliverable/linux.git] / drivers / net / ethernet / intel / ixgbevf / ixgbevf_main.c
CommitLineData
92915f71
GR		 1/*******************************************************************************
2
3 Intel 82599 Virtual Function driver
5c47a2b6 4 Copyright(c) 1999 - 2012 Intel Corporation.
92915f71
GR		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28
29/******************************************************************************
30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
31******************************************************************************/
dbd9636e
JK		 32
33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34
92915f71 35#include <linux/types.h>
dadcd65f 36#include <linux/bitops.h>
92915f71
GR		 37#include <linux/module.h>
38#include <linux/pci.h>
39#include <linux/netdevice.h>
40#include <linux/vmalloc.h>
41#include <linux/string.h>
42#include <linux/in.h>
43#include <linux/ip.h>
44#include <linux/tcp.h>
70a10e25 45#include <linux/sctp.h>
92915f71 46#include <linux/ipv6.h>
5a0e3ad6 47#include <linux/slab.h>
92915f71
GR		 48#include <net/checksum.h>
49#include <net/ip6_checksum.h>
50#include <linux/ethtool.h>
01789349 51#include <linux/if.h>
92915f71 52#include <linux/if_vlan.h>
70c71606 53#include <linux/prefetch.h>
92915f71
GR		 54
55#include "ixgbevf.h"
56
3d8fe98f 57const char ixgbevf_driver_name[] = "ixgbevf";
92915f71 58static const char ixgbevf_driver_string[] =
422e05d1 59 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
92915f71 60
1b3d2d77 61#define DRV_VERSION "2.7.12-k"
92915f71 62const char ixgbevf_driver_version[] = DRV_VERSION;
66c87bd5 63static char ixgbevf_copyright[] =
5c47a2b6 64 "Copyright (c) 2009 - 2012 Intel Corporation.";
92915f71
GR		 65
66static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
2316aa2a
GR		 67 [board_82599_vf] = &ixgbevf_82599_vf_info,
68 [board_X540_vf] = &ixgbevf_X540_vf_info,
92915f71
GR		 69};
70
71/* ixgbevf_pci_tbl - PCI Device ID Table
72 *
73 * Wildcard entries (PCI_ANY_ID) should come last
74 * Last entry must be all 0s
75 *
76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
77 * Class, Class Mask, private data (not used) }
78 */
39ba22b4
SH		 79static DEFINE_PCI_DEVICE_TABLE(ixgbevf_pci_tbl) = {
80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
81 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
92915f71
GR		 82 /* required last entry */
83 {0, }
84};
85MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
86
87MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
88MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
89MODULE_LICENSE("GPL");
90MODULE_VERSION(DRV_VERSION);
91
b3f4d599 92#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
93static int debug = -1;
94module_param(debug, int, 0);
95MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
92915f71
GR		 96
97/* forward decls */
fa71ae27 98static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
56e94095 99static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
92915f71
GR		 100
101static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw,
102 struct ixgbevf_ring *rx_ring,
103 u32 val)
104{
105 /*
106 * Force memory writes to complete before letting h/w
107 * know there are new descriptors to fetch. (Only
108 * applicable for weak-ordered memory model archs,
109 * such as IA-64).
110 */
111 wmb();
112 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val);
113}
114
49ce9c2c 115/**
65d676c8 116 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
92915f71
GR		 117 * @adapter: pointer to adapter struct
118 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
119 * @queue: queue to map the corresponding interrupt to
120 * @msix_vector: the vector to map to the corresponding queue
92915f71
GR		 121 */
122static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
123 u8 queue, u8 msix_vector)
124{
125 u32 ivar, index;
126 struct ixgbe_hw *hw = &adapter->hw;
127 if (direction == -1) {
128 /* other causes */
129 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
130 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
131 ivar &= ~0xFF;
132 ivar |= msix_vector;
133 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
134 } else {
135 /* tx or rx causes */
136 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
137 index = ((16 * (queue & 1)) + (8 * direction));
138 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
139 ivar &= ~(0xFF << index);
140 ivar |= (msix_vector << index);
141 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
142 }
143}
144
70a10e25 145static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
92915f71
GR		 146 struct ixgbevf_tx_buffer
147 *tx_buffer_info)
148{
149 if (tx_buffer_info->dma) {
150 if (tx_buffer_info->mapped_as_page)
70a10e25 151 dma_unmap_page(tx_ring->dev,
92915f71
GR		 152 tx_buffer_info->dma,
153 tx_buffer_info->length,
2a1f8794 154 DMA_TO_DEVICE);
92915f71 155 else
70a10e25 156 dma_unmap_single(tx_ring->dev,
92915f71
GR		 157 tx_buffer_info->dma,
158 tx_buffer_info->length,
2a1f8794 159 DMA_TO_DEVICE);
92915f71
GR		 160 tx_buffer_info->dma = 0;
161 }
162 if (tx_buffer_info->skb) {
163 dev_kfree_skb_any(tx_buffer_info->skb);
164 tx_buffer_info->skb = NULL;
165 }
166 tx_buffer_info->time_stamp = 0;
167 /* tx_buffer_info must be completely set up in the transmit path */
168}
169
92915f71
GR		 170#define IXGBE_MAX_TXD_PWR 14
171#define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR)
172
173/* Tx Descriptors needed, worst case */
3595990a
AD		 174#define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
175#define DESC_NEEDED (MAX_SKB_FRAGS + 4)
92915f71
GR		 176
177static void ixgbevf_tx_timeout(struct net_device *netdev);
178
179/**
180 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
fa71ae27 181 * @q_vector: board private structure
92915f71
GR		 182 * @tx_ring: tx ring to clean
183 **/
fa71ae27 184static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
92915f71
GR		 185 struct ixgbevf_ring *tx_ring)
186{
fa71ae27 187 struct ixgbevf_adapter *adapter = q_vector->adapter;
92915f71
GR		 188 union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
189 struct ixgbevf_tx_buffer *tx_buffer_info;
e757e3e1 190 unsigned int i, count = 0;
92915f71
GR		 191 unsigned int total_bytes = 0, total_packets = 0;
192
10cc1bdd
AD		 193 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
194 return true;
195
92915f71 196 i = tx_ring->next_to_clean;
e757e3e1
AD		 197 tx_buffer_info = &tx_ring->tx_buffer_info[i];
198 eop_desc = tx_buffer_info->next_to_watch;
92915f71 199
e757e3e1 200 do {
92915f71 201 bool cleaned = false;
e757e3e1
AD		 202
203 /* if next_to_watch is not set then there is no work pending */
204 if (!eop_desc)
205 break;
206
207 /* prevent any other reads prior to eop_desc */
208 read_barrier_depends();
209
210 /* if DD is not set pending work has not been completed */
211 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
212 break;
213
214 /* clear next_to_watch to prevent false hangs */
215 tx_buffer_info->next_to_watch = NULL;
216
92915f71
GR		 217 for ( ; !cleaned; count++) {
218 struct sk_buff *skb;
908421f6 219 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
e757e3e1 220 cleaned = (tx_desc == eop_desc);
92915f71
GR		 221 skb = tx_buffer_info->skb;
222
223 if (cleaned && skb) {
224 unsigned int segs, bytecount;
225
226 /* gso_segs is currently only valid for tcp */
227 segs = skb_shinfo(skb)->gso_segs ?: 1;
228 /* multiply data chunks by size of headers */
229 bytecount = ((segs - 1) * skb_headlen(skb)) +
230 skb->len;
231 total_packets += segs;
232 total_bytes += bytecount;
233 }
234
70a10e25 235 ixgbevf_unmap_and_free_tx_resource(tx_ring,
92915f71
GR		 236 tx_buffer_info);
237
238 tx_desc->wb.status = 0;
239
240 i++;
241 if (i == tx_ring->count)
242 i = 0;
e757e3e1
AD		 243
244 tx_buffer_info = &tx_ring->tx_buffer_info[i];
92915f71
GR		 245 }
246
e757e3e1
AD		 247 eop_desc = tx_buffer_info->next_to_watch;
248 } while (count < tx_ring->count);
92915f71
GR		 249
250 tx_ring->next_to_clean = i;
251
252#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
fb40195c 253 if (unlikely(count && netif_carrier_ok(tx_ring->netdev) &&
92915f71
GR		 254 (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
255 /* Make sure that anybody stopping the queue after this
256 * sees the new next_to_clean.
257 */
258 smp_mb();
fb40195c
AD		 259 if (__netif_subqueue_stopped(tx_ring->netdev,
260 tx_ring->queue_index) &&
92915f71 261 !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
fb40195c
AD		 262 netif_wake_subqueue(tx_ring->netdev,
263 tx_ring->queue_index);
92915f71
GR		 264 ++adapter->restart_queue;
265 }
92915f71
GR		 266 }
267
4197aa7b 268 u64_stats_update_begin(&tx_ring->syncp);
92915f71
GR		 269 tx_ring->total_bytes += total_bytes;
270 tx_ring->total_packets += total_packets;
4197aa7b 271 u64_stats_update_end(&tx_ring->syncp);
ac6ed8f0
GR		 272 q_vector->tx.total_bytes += total_bytes;
273 q_vector->tx.total_packets += total_packets;
92915f71 274
fa71ae27 275 return count < tx_ring->count;
92915f71
GR		 276}
277
278/**
279 * ixgbevf_receive_skb - Send a completed packet up the stack
280 * @q_vector: structure containing interrupt and ring information
281 * @skb: packet to send up
282 * @status: hardware indication of status of receive
92915f71
GR		 283 * @rx_desc: rx descriptor
284 **/
285static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
286 struct sk_buff *skb, u8 status,
92915f71
GR		 287 union ixgbe_adv_rx_desc *rx_desc)
288{
289 struct ixgbevf_adapter *adapter = q_vector->adapter;
290 bool is_vlan = (status & IXGBE_RXD_STAT_VP);
dd1ed3b7 291 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
92915f71 292
5d9a533b 293 if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans))
dadcd65f 294 __vlan_hwaccel_put_tag(skb, tag);
dadcd65f 295
366c1099
GR		 296 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
297 napi_gro_receive(&q_vector->napi, skb);
298 else
299 netif_rx(skb);
92915f71
GR		 300}
301
302/**
303 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
55fb277c 304 * @ring: pointer to Rx descriptor ring structure
92915f71
GR		 305 * @status_err: hardware indication of status of receive
306 * @skb: skb currently being received and modified
307 **/
55fb277c 308static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
92915f71
GR		 309 u32 status_err, struct sk_buff *skb)
310{
bc8acf2c 311 skb_checksum_none_assert(skb);
92915f71
GR		 312
313 /* Rx csum disabled */
fb40195c 314 if (!(ring->netdev->features & NETIF_F_RXCSUM))
92915f71
GR		 315 return;
316
317 /* if IP and error */
318 if ((status_err & IXGBE_RXD_STAT_IPCS) &&
319 (status_err & IXGBE_RXDADV_ERR_IPE)) {
55fb277c 320 ring->hw_csum_rx_error++;
92915f71
GR		 321 return;
322 }
323
324 if (!(status_err & IXGBE_RXD_STAT_L4CS))
325 return;
326
327 if (status_err & IXGBE_RXDADV_ERR_TCPE) {
55fb277c 328 ring->hw_csum_rx_error++;
92915f71
GR		 329 return;
330 }
331
332 /* It must be a TCP or UDP packet with a valid checksum */
333 skb->ip_summed = CHECKSUM_UNNECESSARY;
55fb277c 334 ring->hw_csum_rx_good++;
92915f71
GR		 335}
336
337/**
338 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
339 * @adapter: address of board private structure
340 **/
341static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter,
342 struct ixgbevf_ring *rx_ring,
343 int cleaned_count)
344{
345 struct pci_dev *pdev = adapter->pdev;
346 union ixgbe_adv_rx_desc *rx_desc;
347 struct ixgbevf_rx_buffer *bi;
fb40195c 348 unsigned int i = rx_ring->next_to_use;
92915f71 349
92915f71
GR		 350 bi = &rx_ring->rx_buffer_info[i];
351
352 while (cleaned_count--) {
908421f6 353 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
b9dd245b
GR		 354
355 if (!bi->skb) {
356 struct sk_buff *skb;
357
fb40195c
AD		 358 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
359 rx_ring->rx_buf_len);
92915f71
GR		 360 if (!skb) {
361 adapter->alloc_rx_buff_failed++;
362 goto no_buffers;
363 }
92915f71 364 bi->skb = skb;
b9dd245b 365
2a1f8794 366 bi->dma = dma_map_single(&pdev->dev, skb->data,
92915f71 367 rx_ring->rx_buf_len,
2a1f8794 368 DMA_FROM_DEVICE);
6132ee8a
GR		 369 if (dma_mapping_error(&pdev->dev, bi->dma)) {
370 dev_kfree_skb(skb);
371 bi->skb = NULL;
372 dev_err(&pdev->dev, "RX DMA map failed\n");
373 break;
374 }
92915f71 375 }
77d5dfca 376 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
92915f71
GR		 377
378 i++;
379 if (i == rx_ring->count)
380 i = 0;
381 bi = &rx_ring->rx_buffer_info[i];
382 }
383
384no_buffers:
385 if (rx_ring->next_to_use != i) {
386 rx_ring->next_to_use = i;
92915f71
GR		 387 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i);
388 }
389}
390
391static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
5f3600eb 392 u32 qmask)
92915f71 393{
92915f71
GR		 394 struct ixgbe_hw *hw = &adapter->hw;
395
5f3600eb 396 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
92915f71
GR		 397}
398
92915f71
GR		 399static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
400 struct ixgbevf_ring *rx_ring,
fa71ae27 401 int budget)
92915f71
GR		 402{
403 struct ixgbevf_adapter *adapter = q_vector->adapter;
404 struct pci_dev *pdev = adapter->pdev;
405 union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
406 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
407 struct sk_buff *skb;
408 unsigned int i;
409 u32 len, staterr;
92915f71
GR		 410 int cleaned_count = 0;
411 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
412
413 i = rx_ring->next_to_clean;
908421f6 414 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
92915f71
GR		 415 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
416 rx_buffer_info = &rx_ring->rx_buffer_info[i];
417
418 while (staterr & IXGBE_RXD_STAT_DD) {
fa71ae27 419 if (!budget)
92915f71 420 break;
fa71ae27 421 budget--;
92915f71 422
2d0bb1c1 423 rmb(); /* read descriptor and rx_buffer_info after status DD */
77d5dfca 424 len = le16_to_cpu(rx_desc->wb.upper.length);
92915f71
GR		 425 skb = rx_buffer_info->skb;
426 prefetch(skb->data - NET_IP_ALIGN);
427 rx_buffer_info->skb = NULL;
428
429 if (rx_buffer_info->dma) {
2a1f8794 430 dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
92915f71 431 rx_ring->rx_buf_len,
2a1f8794 432 DMA_FROM_DEVICE);
92915f71
GR		 433 rx_buffer_info->dma = 0;
434 skb_put(skb, len);
435 }
436
92915f71
GR		 437 i++;
438 if (i == rx_ring->count)
439 i = 0;
440
908421f6 441 next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
92915f71
GR		 442 prefetch(next_rxd);
443 cleaned_count++;
444
445 next_buffer = &rx_ring->rx_buffer_info[i];
446
447 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
77d5dfca 448 skb->next = next_buffer->skb;
5c60f81a 449 IXGBE_CB(skb->next)->prev = skb;
92915f71
GR		 450 adapter->non_eop_descs++;
451 goto next_desc;
452 }
453
5c60f81a
AD		 454 /* we should not be chaining buffers, if we did drop the skb */
455 if (IXGBE_CB(skb)->prev) {
456 do {
457 struct sk_buff *this = skb;
458 skb = IXGBE_CB(skb)->prev;
459 dev_kfree_skb(this);
460 } while (skb);
461 goto next_desc;
462 }
463
92915f71
GR		 464 /* ERR_MASK will only have valid bits if EOP set */
465 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
466 dev_kfree_skb_irq(skb);
467 goto next_desc;
468 }
469
55fb277c 470 ixgbevf_rx_checksum(rx_ring, staterr, skb);
92915f71
GR		 471
472 /* probably a little skewed due to removing CRC */
473 total_rx_bytes += skb->len;
474 total_rx_packets++;
475
476 /*
477 * Work around issue of some types of VM to VM loop back
478 * packets not getting split correctly
479 */
480 if (staterr & IXGBE_RXD_STAT_LB) {
e743d313 481 u32 header_fixup_len = skb_headlen(skb);
92915f71
GR		 482 if (header_fixup_len < 14)
483 skb_push(skb, header_fixup_len);
484 }
fb40195c 485 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
92915f71 486
815cccbf
JF		 487 /* Workaround hardware that can't do proper VEPA multicast
488 * source pruning.
489 */
490 if ((skb->pkt_type & (PACKET_BROADCAST | PACKET_MULTICAST)) &&
491 !(compare_ether_addr(adapter->netdev->dev_addr,
492 eth_hdr(skb)->h_source))) {
493 dev_kfree_skb_irq(skb);
494 goto next_desc;
495 }
496
b3d58a8f 497 ixgbevf_receive_skb(q_vector, skb, staterr, rx_desc);
92915f71
GR		 498
499next_desc:
500 rx_desc->wb.upper.status_error = 0;
501
502 /* return some buffers to hardware, one at a time is too slow */
503 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
504 ixgbevf_alloc_rx_buffers(adapter, rx_ring,
505 cleaned_count);
506 cleaned_count = 0;
507 }
508
509 /* use prefetched values */
510 rx_desc = next_rxd;
511 rx_buffer_info = &rx_ring->rx_buffer_info[i];
512
513 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
514 }
515
516 rx_ring->next_to_clean = i;
517 cleaned_count = IXGBE_DESC_UNUSED(rx_ring);
518
519 if (cleaned_count)
520 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count);
521
4197aa7b 522 u64_stats_update_begin(&rx_ring->syncp);
92915f71
GR		 523 rx_ring->total_packets += total_rx_packets;
524 rx_ring->total_bytes += total_rx_bytes;
4197aa7b 525 u64_stats_update_end(&rx_ring->syncp);
ac6ed8f0
GR		 526 q_vector->rx.total_packets += total_rx_packets;
527 q_vector->rx.total_bytes += total_rx_bytes;
92915f71 528
fa71ae27 529 return !!budget;
92915f71
GR		 530}
531
532/**
fa71ae27 533 * ixgbevf_poll - NAPI polling calback
92915f71
GR		 534 * @napi: napi struct with our devices info in it
535 * @budget: amount of work driver is allowed to do this pass, in packets
536 *
fa71ae27 537 * This function will clean more than one or more rings associated with a
92915f71
GR		 538 * q_vector.
539 **/
fa71ae27 540static int ixgbevf_poll(struct napi_struct *napi, int budget)
92915f71
GR		 541{
542 struct ixgbevf_q_vector *q_vector =
543 container_of(napi, struct ixgbevf_q_vector, napi);
544 struct ixgbevf_adapter *adapter = q_vector->adapter;
fa71ae27
AD		 545 struct ixgbevf_ring *ring;
546 int per_ring_budget;
547 bool clean_complete = true;
548
549 ixgbevf_for_each_ring(ring, q_vector->tx)
550 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
92915f71
GR		 551
552 /* attempt to distribute budget to each queue fairly, but don't allow
553 * the budget to go below 1 because we'll exit polling */
fa71ae27
AD		 554 if (q_vector->rx.count > 1)
555 per_ring_budget = max(budget/q_vector->rx.count, 1);
556 else
557 per_ring_budget = budget;
558
366c1099 559 adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
fa71ae27
AD		 560 ixgbevf_for_each_ring(ring, q_vector->rx)
561 clean_complete &= ixgbevf_clean_rx_irq(q_vector, ring,
562 per_ring_budget);
366c1099 563 adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
fa71ae27
AD		 564
565 /* If all work not completed, return budget and keep polling */
566 if (!clean_complete)
567 return budget;
568 /* all work done, exit the polling mode */
569 napi_complete(napi);
570 if (adapter->rx_itr_setting & 1)
571 ixgbevf_set_itr(q_vector);
572 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
573 ixgbevf_irq_enable_queues(adapter,
574 1 << q_vector->v_idx);
92915f71 575
fa71ae27 576 return 0;
92915f71
GR		 577}
578
ce422606
GR		 579/**
580 * ixgbevf_write_eitr - write VTEITR register in hardware specific way
581 * @q_vector: structure containing interrupt and ring information
582 */
583static void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
584{
585 struct ixgbevf_adapter *adapter = q_vector->adapter;
586 struct ixgbe_hw *hw = &adapter->hw;
587 int v_idx = q_vector->v_idx;
588 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
589
590 /*
591 * set the WDIS bit to not clear the timer bits and cause an
592 * immediate assertion of the interrupt
593 */
594 itr_reg |= IXGBE_EITR_CNT_WDIS;
595
596 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
597}
92915f71
GR		 598
599/**
600 * ixgbevf_configure_msix - Configure MSI-X hardware
601 * @adapter: board private structure
602 *
603 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
604 * interrupts.
605 **/
606static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
607{
608 struct ixgbevf_q_vector *q_vector;
6b43c446 609 int q_vectors, v_idx;
92915f71
GR		 610
611 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
5f3600eb 612 adapter->eims_enable_mask = 0;
92915f71
GR		 613
614 /*
615 * Populate the IVAR table and set the ITR values to the
616 * corresponding register.
617 */
618 for (v_idx = 0; v_idx < q_vectors; v_idx++) {
6b43c446 619 struct ixgbevf_ring *ring;
92915f71 620 q_vector = adapter->q_vector[v_idx];
6b43c446
AD		 621
622 ixgbevf_for_each_ring(ring, q_vector->rx)
623 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
624
625 ixgbevf_for_each_ring(ring, q_vector->tx)
626 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
92915f71 627
5f3600eb
AD		 628 if (q_vector->tx.ring && !q_vector->rx.ring) {
629 /* tx only vector */
630 if (adapter->tx_itr_setting == 1)
631 q_vector->itr = IXGBE_10K_ITR;
632 else
633 q_vector->itr = adapter->tx_itr_setting;
634 } else {
635 /* rx or rx/tx vector */
636 if (adapter->rx_itr_setting == 1)
637 q_vector->itr = IXGBE_20K_ITR;
638 else
639 q_vector->itr = adapter->rx_itr_setting;
640 }
641
642 /* add q_vector eims value to global eims_enable_mask */
643 adapter->eims_enable_mask |= 1 << v_idx;
92915f71 644
5f3600eb 645 ixgbevf_write_eitr(q_vector);
92915f71
GR		 646 }
647
648 ixgbevf_set_ivar(adapter, -1, 1, v_idx);
5f3600eb
AD		 649 /* setup eims_other and add value to global eims_enable_mask */
650 adapter->eims_other = 1 << v_idx;
651 adapter->eims_enable_mask |= adapter->eims_other;
92915f71
GR		 652}
653
654enum latency_range {
655 lowest_latency = 0,
656 low_latency = 1,
657 bulk_latency = 2,
658 latency_invalid = 255
659};
660
661/**
662 * ixgbevf_update_itr - update the dynamic ITR value based on statistics
5f3600eb
AD		 663 * @q_vector: structure containing interrupt and ring information
664 * @ring_container: structure containing ring performance data
92915f71
GR		 665 *
666 * Stores a new ITR value based on packets and byte
667 * counts during the last interrupt. The advantage of per interrupt
668 * computation is faster updates and more accurate ITR for the current
669 * traffic pattern. Constants in this function were computed
670 * based on theoretical maximum wire speed and thresholds were set based
671 * on testing data as well as attempting to minimize response time
672 * while increasing bulk throughput.
673 **/
5f3600eb
AD		 674static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
675 struct ixgbevf_ring_container *ring_container)
92915f71 676{
5f3600eb
AD		 677 int bytes = ring_container->total_bytes;
678 int packets = ring_container->total_packets;
92915f71
GR		 679 u32 timepassed_us;
680 u64 bytes_perint;
5f3600eb 681 u8 itr_setting = ring_container->itr;
92915f71
GR		 682
683 if (packets == 0)
5f3600eb 684 return;
92915f71
GR		 685
686 /* simple throttlerate management
687 * 0-20MB/s lowest (100000 ints/s)
688 * 20-100MB/s low (20000 ints/s)
689 * 100-1249MB/s bulk (8000 ints/s)
690 */
691 /* what was last interrupt timeslice? */
5f3600eb 692 timepassed_us = q_vector->itr >> 2;
92915f71
GR		 693 bytes_perint = bytes / timepassed_us; /* bytes/usec */
694
695 switch (itr_setting) {
696 case lowest_latency:
e2c28ce7 697 if (bytes_perint > 10)
5f3600eb 698 itr_setting = low_latency;
92915f71
GR		 699 break;
700 case low_latency:
e2c28ce7 701 if (bytes_perint > 20)
5f3600eb 702 itr_setting = bulk_latency;
e2c28ce7 703 else if (bytes_perint <= 10)
5f3600eb 704 itr_setting = lowest_latency;
92915f71
GR		 705 break;
706 case bulk_latency:
e2c28ce7 707 if (bytes_perint <= 20)
5f3600eb 708 itr_setting = low_latency;
92915f71
GR		 709 break;
710 }
711
5f3600eb
AD		 712 /* clear work counters since we have the values we need */
713 ring_container->total_bytes = 0;
714 ring_container->total_packets = 0;
715
716 /* write updated itr to ring container */
717 ring_container->itr = itr_setting;
92915f71
GR		 718}
719
fa71ae27 720static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
92915f71 721{
5f3600eb
AD		 722 u32 new_itr = q_vector->itr;
723 u8 current_itr;
92915f71 724
5f3600eb
AD		 725 ixgbevf_update_itr(q_vector, &q_vector->tx);
726 ixgbevf_update_itr(q_vector, &q_vector->rx);
92915f71 727
6b43c446 728 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
92915f71
GR		 729
730 switch (current_itr) {
731 /* counts and packets in update_itr are dependent on these numbers */
732 case lowest_latency:
5f3600eb 733 new_itr = IXGBE_100K_ITR;
92915f71
GR		 734 break;
735 case low_latency:
5f3600eb 736 new_itr = IXGBE_20K_ITR;
92915f71
GR		 737 break;
738 case bulk_latency:
739 default:
5f3600eb 740 new_itr = IXGBE_8K_ITR;
92915f71
GR		 741 break;
742 }
743
5f3600eb 744 if (new_itr != q_vector->itr) {
92915f71 745 /* do an exponential smoothing */
5f3600eb
AD		 746 new_itr = (10 * new_itr * q_vector->itr) /
747 ((9 * new_itr) + q_vector->itr);
748
749 /* save the algorithm value here */
750 q_vector->itr = new_itr;
751
752 ixgbevf_write_eitr(q_vector);
92915f71 753 }
92915f71
GR		 754}
755
4b2cd27f 756static irqreturn_t ixgbevf_msix_other(int irq, void *data)
92915f71 757{
fa71ae27 758 struct ixgbevf_adapter *adapter = data;
1e72bfc3 759 struct pci_dev *pdev = adapter->pdev;
92915f71 760 struct ixgbe_hw *hw = &adapter->hw;
1e72bfc3
GR		 761 u32 msg;
762 bool got_ack = false;
08259594 763
4b2cd27f 764 hw->mac.get_link_status = 1;
1e72bfc3
GR		 765 if (!hw->mbx.ops.check_for_ack(hw))
766 got_ack = true;
a9ee25a2 767
1e72bfc3
GR		 768 if (!hw->mbx.ops.check_for_msg(hw)) {
769 hw->mbx.ops.read(hw, &msg, 1);
770
088245a3 771 if ((msg & IXGBE_MBVFICR_VFREQ_MASK) == IXGBE_PF_CONTROL_MSG) {
1e72bfc3
GR		 772 mod_timer(&adapter->watchdog_timer,
773 round_jiffies(jiffies + 1));
088245a3
GR		 774 adapter->link_up = false;
775 }
1e72bfc3
GR		 776
777 if (msg & IXGBE_VT_MSGTYPE_NACK)
778 dev_info(&pdev->dev,
779 "Last Request of type %2.2x to PF Nacked\n",
780 msg & 0xFF);
781 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFSTS;
782 }
783
784 /* checking for the ack clears the PFACK bit. Place
785 * it back in the v2p_mailbox cache so that anyone
786 * polling for an ack will not miss it
787 */
788 if (got_ack)
789 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFACK;
3a2c4033 790
5f3600eb
AD		 791 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
792
92915f71
GR		 793 return IRQ_HANDLED;
794}
795
92915f71 796/**
fa71ae27 797 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
92915f71
GR		 798 * @irq: unused
799 * @data: pointer to our q_vector struct for this interrupt vector
800 **/
fa71ae27 801static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
92915f71
GR		 802{
803 struct ixgbevf_q_vector *q_vector = data;
92915f71 804
5f3600eb 805 /* EIAM disabled interrupts (on this vector) for us */
fa71ae27
AD		 806 if (q_vector->rx.ring || q_vector->tx.ring)
807 napi_schedule(&q_vector->napi);
92915f71
GR		 808
809 return IRQ_HANDLED;
810}
811
812static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
813 int r_idx)
814{
815 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
816
6b43c446
AD		 817 a->rx_ring[r_idx].next = q_vector->rx.ring;
818 q_vector->rx.ring = &a->rx_ring[r_idx];
819 q_vector->rx.count++;
92915f71
GR		 820}
821
822static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
823 int t_idx)
824{
825 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
826
6b43c446
AD		 827 a->tx_ring[t_idx].next = q_vector->tx.ring;
828 q_vector->tx.ring = &a->tx_ring[t_idx];
829 q_vector->tx.count++;
92915f71
GR		 830}
831
832/**
833 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
834 * @adapter: board private structure to initialize
835 *
836 * This function maps descriptor rings to the queue-specific vectors
837 * we were allotted through the MSI-X enabling code. Ideally, we'd have
838 * one vector per ring/queue, but on a constrained vector budget, we
839 * group the rings as "efficiently" as possible. You would add new
840 * mapping configurations in here.
841 **/
842static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
843{
844 int q_vectors;
845 int v_start = 0;
846 int rxr_idx = 0, txr_idx = 0;
847 int rxr_remaining = adapter->num_rx_queues;
848 int txr_remaining = adapter->num_tx_queues;
849 int i, j;
850 int rqpv, tqpv;
851 int err = 0;
852
853 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
854
855 /*
856 * The ideal configuration...
857 * We have enough vectors to map one per queue.
858 */
859 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
860 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
861 map_vector_to_rxq(adapter, v_start, rxr_idx);
862
863 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
864 map_vector_to_txq(adapter, v_start, txr_idx);
865 goto out;
866 }
867
868 /*
869 * If we don't have enough vectors for a 1-to-1
870 * mapping, we'll have to group them so there are
871 * multiple queues per vector.
872 */
873 /* Re-adjusting *qpv takes care of the remainder. */
874 for (i = v_start; i < q_vectors; i++) {
875 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
876 for (j = 0; j < rqpv; j++) {
877 map_vector_to_rxq(adapter, i, rxr_idx);
878 rxr_idx++;
879 rxr_remaining--;
880 }
881 }
882 for (i = v_start; i < q_vectors; i++) {
883 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
884 for (j = 0; j < tqpv; j++) {
885 map_vector_to_txq(adapter, i, txr_idx);
886 txr_idx++;
887 txr_remaining--;
888 }
889 }
890
891out:
892 return err;
893}
894
895/**
896 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
897 * @adapter: board private structure
898 *
899 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
900 * interrupts from the kernel.
901 **/
902static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
903{
904 struct net_device *netdev = adapter->netdev;
fa71ae27
AD		 905 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
906 int vector, err;
92915f71
GR		 907 int ri = 0, ti = 0;
908
92915f71 909 for (vector = 0; vector < q_vectors; vector++) {
fa71ae27
AD		 910 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
911 struct msix_entry *entry = &adapter->msix_entries[vector];
912
913 if (q_vector->tx.ring && q_vector->rx.ring) {
914 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
915 "%s-%s-%d", netdev->name, "TxRx", ri++);
916 ti++;
917 } else if (q_vector->rx.ring) {
918 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
919 "%s-%s-%d", netdev->name, "rx", ri++);
920 } else if (q_vector->tx.ring) {
921 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
922 "%s-%s-%d", netdev->name, "tx", ti++);
92915f71
GR		 923 } else {
924 /* skip this unused q_vector */
925 continue;
926 }
fa71ae27
AD		 927 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
928 q_vector->name, q_vector);
92915f71
GR		 929 if (err) {
930 hw_dbg(&adapter->hw,
931 "request_irq failed for MSIX interrupt "
932 "Error: %d\n", err);
933 goto free_queue_irqs;
934 }
935 }
936
92915f71 937 err = request_irq(adapter->msix_entries[vector].vector,
4b2cd27f 938 &ixgbevf_msix_other, 0, netdev->name, adapter);
92915f71
GR		 939 if (err) {
940 hw_dbg(&adapter->hw,
4b2cd27f 941 "request_irq for msix_other failed: %d\n", err);
92915f71
GR		 942 goto free_queue_irqs;
943 }
944
945 return 0;
946
947free_queue_irqs:
fa71ae27
AD		 948 while (vector) {
949 vector--;
950 free_irq(adapter->msix_entries[vector].vector,
951 adapter->q_vector[vector]);
952 }
92915f71
GR		 953 pci_disable_msix(adapter->pdev);
954 kfree(adapter->msix_entries);
955 adapter->msix_entries = NULL;
956 return err;
957}
958
959static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
960{
961 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
962
963 for (i = 0; i < q_vectors; i++) {
964 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
6b43c446
AD		 965 q_vector->rx.ring = NULL;
966 q_vector->tx.ring = NULL;
967 q_vector->rx.count = 0;
968 q_vector->tx.count = 0;
92915f71
GR		 969 }
970}
971
972/**
973 * ixgbevf_request_irq - initialize interrupts
974 * @adapter: board private structure
975 *
976 * Attempts to configure interrupts using the best available
977 * capabilities of the hardware and kernel.
978 **/
979static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
980{
981 int err = 0;
982
983 err = ixgbevf_request_msix_irqs(adapter);
984
985 if (err)
986 hw_dbg(&adapter->hw,
987 "request_irq failed, Error %d\n", err);
988
989 return err;
990}
991
992static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
993{
92915f71
GR		 994 int i, q_vectors;
995
996 q_vectors = adapter->num_msix_vectors;
92915f71
GR		 997 i = q_vectors - 1;
998
fa71ae27 999 free_irq(adapter->msix_entries[i].vector, adapter);
92915f71
GR		 1000 i--;
1001
1002 for (; i >= 0; i--) {
fa71ae27
AD		 1003 /* free only the irqs that were actually requested */
1004 if (!adapter->q_vector[i]->rx.ring &&
1005 !adapter->q_vector[i]->tx.ring)
1006 continue;
1007
92915f71
GR		 1008 free_irq(adapter->msix_entries[i].vector,
1009 adapter->q_vector[i]);
1010 }
1011
1012 ixgbevf_reset_q_vectors(adapter);
1013}
1014
1015/**
1016 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1017 * @adapter: board private structure
1018 **/
1019static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1020{
92915f71 1021 struct ixgbe_hw *hw = &adapter->hw;
5f3600eb 1022 int i;
92915f71 1023
5f3600eb 1024 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
92915f71 1025 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
5f3600eb 1026 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
92915f71
GR		 1027
1028 IXGBE_WRITE_FLUSH(hw);
1029
1030 for (i = 0; i < adapter->num_msix_vectors; i++)
1031 synchronize_irq(adapter->msix_entries[i].vector);
1032}
1033
1034/**
1035 * ixgbevf_irq_enable - Enable default interrupt generation settings
1036 * @adapter: board private structure
1037 **/
5f3600eb 1038static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
92915f71
GR		 1039{
1040 struct ixgbe_hw *hw = &adapter->hw;
92915f71 1041
5f3600eb
AD		 1042 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1043 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1044 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
92915f71
GR		 1045}
1046
1047/**
1048 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1049 * @adapter: board private structure
1050 *
1051 * Configure the Tx unit of the MAC after a reset.
1052 **/
1053static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1054{
1055 u64 tdba;
1056 struct ixgbe_hw *hw = &adapter->hw;
1057 u32 i, j, tdlen, txctrl;
1058
1059 /* Setup the HW Tx Head and Tail descriptor pointers */
1060 for (i = 0; i < adapter->num_tx_queues; i++) {
1061 struct ixgbevf_ring *ring = &adapter->tx_ring[i];
1062 j = ring->reg_idx;
1063 tdba = ring->dma;
1064 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc);
1065 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1066 (tdba & DMA_BIT_MASK(32)));
1067 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1068 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen);
1069 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0);
1070 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0);
1071 adapter->tx_ring[i].head = IXGBE_VFTDH(j);
1072 adapter->tx_ring[i].tail = IXGBE_VFTDT(j);
1073 /* Disable Tx Head Writeback RO bit, since this hoses
1074 * bookkeeping if things aren't delivered in order.
1075 */
1076 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1077 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
1078 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1079 }
1080}
1081
1082#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1083
1084static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1085{
1086 struct ixgbevf_ring *rx_ring;
1087 struct ixgbe_hw *hw = &adapter->hw;
1088 u32 srrctl;
1089
1090 rx_ring = &adapter->rx_ring[index];
1091
1092 srrctl = IXGBE_SRRCTL_DROP_EN;
1093
77d5dfca 1094 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
92915f71 1095
dd1fe113
AD		 1096 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
1097 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1098
92915f71
GR		 1099 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1100}
1101
dd1fe113
AD		 1102static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter)
1103{
1104 struct ixgbe_hw *hw = &adapter->hw;
1105 struct net_device *netdev = adapter->netdev;
1106 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1107 int i;
1108 u16 rx_buf_len;
1109
1110 /* notify the PF of our intent to use this size of frame */
1111 ixgbevf_rlpml_set_vf(hw, max_frame);
1112
1113 /* PF will allow an extra 4 bytes past for vlan tagged frames */
1114 max_frame += VLAN_HLEN;
1115
1116 /*
85624caf
GR		 1117 * Allocate buffer sizes that fit well into 32K and
1118 * take into account max frame size of 9.5K
dd1fe113
AD		 1119 */
1120 if ((hw->mac.type == ixgbe_mac_X540_vf) &&
1121 (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE))
1122 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85624caf
GR		 1123 else if (max_frame <= IXGBEVF_RXBUFFER_2K)
1124 rx_buf_len = IXGBEVF_RXBUFFER_2K;
1125 else if (max_frame <= IXGBEVF_RXBUFFER_4K)
1126 rx_buf_len = IXGBEVF_RXBUFFER_4K;
1127 else if (max_frame <= IXGBEVF_RXBUFFER_8K)
1128 rx_buf_len = IXGBEVF_RXBUFFER_8K;
dd1fe113 1129 else
85624caf 1130 rx_buf_len = IXGBEVF_RXBUFFER_10K;
dd1fe113
AD		 1131
1132 for (i = 0; i < adapter->num_rx_queues; i++)
1133 adapter->rx_ring[i].rx_buf_len = rx_buf_len;
1134}
1135
92915f71
GR		 1136/**
1137 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1138 * @adapter: board private structure
1139 *
1140 * Configure the Rx unit of the MAC after a reset.
1141 **/
1142static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1143{
1144 u64 rdba;
1145 struct ixgbe_hw *hw = &adapter->hw;
92915f71
GR		 1146 int i, j;
1147 u32 rdlen;
92915f71 1148
77d5dfca
AD		 1149 /* PSRTYPE must be initialized in 82599 */
1150 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0);
dd1fe113
AD		 1151
1152 /* set_rx_buffer_len must be called before ring initialization */
1153 ixgbevf_set_rx_buffer_len(adapter);
92915f71
GR		 1154
1155 rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc);
1156 /* Setup the HW Rx Head and Tail Descriptor Pointers and
1157 * the Base and Length of the Rx Descriptor Ring */
1158 for (i = 0; i < adapter->num_rx_queues; i++) {
1159 rdba = adapter->rx_ring[i].dma;
1160 j = adapter->rx_ring[i].reg_idx;
1161 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1162 (rdba & DMA_BIT_MASK(32)));
1163 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
1164 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen);
1165 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0);
1166 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0);
1167 adapter->rx_ring[i].head = IXGBE_VFRDH(j);
1168 adapter->rx_ring[i].tail = IXGBE_VFRDT(j);
92915f71
GR		 1169
1170 ixgbevf_configure_srrctl(adapter, j);
1171 }
1172}
1173
8e586137 1174static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
92915f71
GR		 1175{
1176 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1177 struct ixgbe_hw *hw = &adapter->hw;
2ddc7fe1
AD		 1178 int err;
1179
55fdd45b 1180 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1181
92915f71 1182 /* add VID to filter table */
2ddc7fe1 1183 err = hw->mac.ops.set_vfta(hw, vid, 0, true);
1c55ed76 1184
55fdd45b 1185 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1186
2ddc7fe1
AD		 1187 /* translate error return types so error makes sense */
1188 if (err == IXGBE_ERR_MBX)
1189 return -EIO;
1190
1191 if (err == IXGBE_ERR_INVALID_ARGUMENT)
1192 return -EACCES;
1193
dadcd65f 1194 set_bit(vid, adapter->active_vlans);
8e586137 1195
2ddc7fe1 1196 return err;
92915f71
GR		 1197}
1198
8e586137 1199static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
92915f71
GR		 1200{
1201 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1202 struct ixgbe_hw *hw = &adapter->hw;
2ddc7fe1 1203 int err = -EOPNOTSUPP;
92915f71 1204
55fdd45b 1205 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1206
92915f71 1207 /* remove VID from filter table */
92fe0bf7 1208 err = hw->mac.ops.set_vfta(hw, vid, 0, false);
1c55ed76 1209
55fdd45b 1210 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1211
dadcd65f 1212 clear_bit(vid, adapter->active_vlans);
8e586137 1213
2ddc7fe1 1214 return err;
92915f71
GR		 1215}
1216
1217static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1218{
dadcd65f 1219 u16 vid;
92915f71 1220
dadcd65f
JP		 1221 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1222 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid);
92915f71
GR		 1223}
1224
46ec20ff
GR		 1225static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1226{
1227 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1228 struct ixgbe_hw *hw = &adapter->hw;
1229 int count = 0;
1230
1231 if ((netdev_uc_count(netdev)) > 10) {
dbd9636e 1232 pr_err("Too many unicast filters - No Space\n");
46ec20ff
GR		 1233 return -ENOSPC;
1234 }
1235
1236 if (!netdev_uc_empty(netdev)) {
1237 struct netdev_hw_addr *ha;
1238 netdev_for_each_uc_addr(ha, netdev) {
1239 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1240 udelay(200);
1241 }
1242 } else {
1243 /*
1244 * If the list is empty then send message to PF driver to
1245 * clear all macvlans on this VF.
1246 */
1247 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1248 }
1249
1250 return count;
1251}
1252
92915f71 1253/**
dee847f5 1254 * ixgbevf_set_rx_mode - Multicast and unicast set
92915f71
GR		 1255 * @netdev: network interface device structure
1256 *
1257 * The set_rx_method entry point is called whenever the multicast address
dee847f5
GR		 1258 * list, unicast address list or the network interface flags are updated.
1259 * This routine is responsible for configuring the hardware for proper
1260 * multicast mode and configuring requested unicast filters.
92915f71
GR		 1261 **/
1262static void ixgbevf_set_rx_mode(struct net_device *netdev)
1263{
1264 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1265 struct ixgbe_hw *hw = &adapter->hw;
92915f71 1266
55fdd45b 1267 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1268
92915f71 1269 /* reprogram multicast list */
92fe0bf7 1270 hw->mac.ops.update_mc_addr_list(hw, netdev);
46ec20ff
GR		 1271
1272 ixgbevf_write_uc_addr_list(netdev);
1c55ed76 1273
55fdd45b 1274 spin_unlock_bh(&adapter->mbx_lock);
92915f71
GR		 1275}
1276
1277static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1278{
1279 int q_idx;
1280 struct ixgbevf_q_vector *q_vector;
1281 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1282
1283 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
92915f71 1284 q_vector = adapter->q_vector[q_idx];
fa71ae27 1285 napi_enable(&q_vector->napi);
92915f71
GR		 1286 }
1287}
1288
1289static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1290{
1291 int q_idx;
1292 struct ixgbevf_q_vector *q_vector;
1293 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1294
1295 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1296 q_vector = adapter->q_vector[q_idx];
92915f71
GR		 1297 napi_disable(&q_vector->napi);
1298 }
1299}
1300
1301static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1302{
1303 struct net_device *netdev = adapter->netdev;
1304 int i;
1305
1306 ixgbevf_set_rx_mode(netdev);
1307
1308 ixgbevf_restore_vlan(adapter);
1309
1310 ixgbevf_configure_tx(adapter);
1311 ixgbevf_configure_rx(adapter);
1312 for (i = 0; i < adapter->num_rx_queues; i++) {
1313 struct ixgbevf_ring *ring = &adapter->rx_ring[i];
18c63089
AD		 1314 ixgbevf_alloc_rx_buffers(adapter, ring,
1315 IXGBE_DESC_UNUSED(ring));
92915f71
GR		 1316 }
1317}
1318
1319#define IXGBE_MAX_RX_DESC_POLL 10
1320static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1321 int rxr)
1322{
1323 struct ixgbe_hw *hw = &adapter->hw;
1324 int j = adapter->rx_ring[rxr].reg_idx;
1325 int k;
1326
1327 for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) {
1328 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE)
1329 break;
1330 else
1331 msleep(1);
1332 }
1333 if (k >= IXGBE_MAX_RX_DESC_POLL) {
1334 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d "
1335 "not set within the polling period\n", rxr);
1336 }
1337
6259a01f
GR		 1338 ixgbevf_release_rx_desc(hw, &adapter->rx_ring[rxr],
1339 adapter->rx_ring[rxr].count - 1);
92915f71
GR		 1340}
1341
33bd9f60
GR		 1342static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1343{
1344 /* Only save pre-reset stats if there are some */
1345 if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1346 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1347 adapter->stats.base_vfgprc;
1348 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1349 adapter->stats.base_vfgptc;
1350 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1351 adapter->stats.base_vfgorc;
1352 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1353 adapter->stats.base_vfgotc;
1354 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1355 adapter->stats.base_vfmprc;
1356 }
1357}
1358
1359static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1360{
1361 struct ixgbe_hw *hw = &adapter->hw;
1362
1363 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1364 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1365 adapter->stats.last_vfgorc |=
1366 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1367 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1368 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1369 adapter->stats.last_vfgotc |=
1370 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1371 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1372
1373 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1374 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1375 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1376 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1377 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1378}
1379
31186785
AD		 1380static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
1381{
1382 struct ixgbe_hw *hw = &adapter->hw;
56e94095
AD		 1383 int api[] = { ixgbe_mbox_api_11,
1384 ixgbe_mbox_api_10,
31186785
AD		 1385 ixgbe_mbox_api_unknown };
1386 int err = 0, idx = 0;
1387
55fdd45b 1388 spin_lock_bh(&adapter->mbx_lock);
31186785
AD		 1389
1390 while (api[idx] != ixgbe_mbox_api_unknown) {
1391 err = ixgbevf_negotiate_api_version(hw, api[idx]);
1392 if (!err)
1393 break;
1394 idx++;
1395 }
1396
55fdd45b 1397 spin_unlock_bh(&adapter->mbx_lock);
31186785
AD		 1398}
1399
795180d8 1400static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
92915f71
GR		 1401{
1402 struct net_device *netdev = adapter->netdev;
1403 struct ixgbe_hw *hw = &adapter->hw;
1404 int i, j = 0;
1405 int num_rx_rings = adapter->num_rx_queues;
1406 u32 txdctl, rxdctl;
1407
1408 for (i = 0; i < adapter->num_tx_queues; i++) {
1409 j = adapter->tx_ring[i].reg_idx;
1410 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1411 /* enable WTHRESH=8 descriptors, to encourage burst writeback */
1412 txdctl |= (8 << 16);
1413 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1414 }
1415
1416 for (i = 0; i < adapter->num_tx_queues; i++) {
1417 j = adapter->tx_ring[i].reg_idx;
1418 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1419 txdctl |= IXGBE_TXDCTL_ENABLE;
1420 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1421 }
1422
1423 for (i = 0; i < num_rx_rings; i++) {
1424 j = adapter->rx_ring[i].reg_idx;
1425 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
dadcd65f 1426 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
69bfbec4
GR		 1427 if (hw->mac.type == ixgbe_mac_X540_vf) {
1428 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1429 rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) |
1430 IXGBE_RXDCTL_RLPML_EN);
1431 }
92915f71
GR		 1432 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1433 ixgbevf_rx_desc_queue_enable(adapter, i);
1434 }
1435
1436 ixgbevf_configure_msix(adapter);
1437
55fdd45b 1438 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1439
92fe0bf7
GR		 1440 if (is_valid_ether_addr(hw->mac.addr))
1441 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1442 else
1443 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
92915f71 1444
55fdd45b 1445 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1446
92915f71
GR		 1447 clear_bit(__IXGBEVF_DOWN, &adapter->state);
1448 ixgbevf_napi_enable_all(adapter);
1449
1450 /* enable transmits */
1451 netif_tx_start_all_queues(netdev);
1452
33bd9f60
GR		 1453 ixgbevf_save_reset_stats(adapter);
1454 ixgbevf_init_last_counter_stats(adapter);
1455
4b2cd27f 1456 hw->mac.get_link_status = 1;
92915f71 1457 mod_timer(&adapter->watchdog_timer, jiffies);
92915f71
GR		 1458}
1459
56e94095
AD		 1460static int ixgbevf_reset_queues(struct ixgbevf_adapter *adapter)
1461{
1462 struct ixgbe_hw *hw = &adapter->hw;
1463 struct ixgbevf_ring *rx_ring;
1464 unsigned int def_q = 0;
1465 unsigned int num_tcs = 0;
1466 unsigned int num_rx_queues = 1;
1467 int err, i;
1468
55fdd45b 1469 spin_lock_bh(&adapter->mbx_lock);
56e94095
AD		 1470
1471 /* fetch queue configuration from the PF */
1472 err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1473
55fdd45b 1474 spin_unlock_bh(&adapter->mbx_lock);
56e94095
AD		 1475
1476 if (err)
1477 return err;
1478
1479 if (num_tcs > 1) {
1480 /* update default Tx ring register index */
1481 adapter->tx_ring[0].reg_idx = def_q;
1482
1483 /* we need as many queues as traffic classes */
1484 num_rx_queues = num_tcs;
1485 }
1486
1487 /* nothing to do if we have the correct number of queues */
1488 if (adapter->num_rx_queues == num_rx_queues)
1489 return 0;
1490
1491 /* allocate new rings */
1492 rx_ring = kcalloc(num_rx_queues,
1493 sizeof(struct ixgbevf_ring), GFP_KERNEL);
1494 if (!rx_ring)
1495 return -ENOMEM;
1496
1497 /* setup ring fields */
1498 for (i = 0; i < num_rx_queues; i++) {
1499 rx_ring[i].count = adapter->rx_ring_count;
1500 rx_ring[i].queue_index = i;
1501 rx_ring[i].reg_idx = i;
1502 rx_ring[i].dev = &adapter->pdev->dev;
1503 rx_ring[i].netdev = adapter->netdev;
1504
1505 /* allocate resources on the ring */
1506 err = ixgbevf_setup_rx_resources(adapter, &rx_ring[i]);
1507 if (err) {
1508 while (i) {
1509 i--;
1510 ixgbevf_free_rx_resources(adapter, &rx_ring[i]);
1511 }
1512 kfree(rx_ring);
1513 return err;
1514 }
1515 }
1516
1517 /* free the existing rings and queues */
1518 ixgbevf_free_all_rx_resources(adapter);
1519 adapter->num_rx_queues = 0;
1520 kfree(adapter->rx_ring);
1521
1522 /* move new rings into position on the adapter struct */
1523 adapter->rx_ring = rx_ring;
1524 adapter->num_rx_queues = num_rx_queues;
1525
1526 /* reset ring to vector mapping */
1527 ixgbevf_reset_q_vectors(adapter);
1528 ixgbevf_map_rings_to_vectors(adapter);
1529
1530 return 0;
1531}
1532
795180d8 1533void ixgbevf_up(struct ixgbevf_adapter *adapter)
92915f71 1534{
92915f71
GR		 1535 struct ixgbe_hw *hw = &adapter->hw;
1536
31186785
AD		 1537 ixgbevf_negotiate_api(adapter);
1538
56e94095
AD		 1539 ixgbevf_reset_queues(adapter);
1540
92915f71
GR		 1541 ixgbevf_configure(adapter);
1542
795180d8 1543 ixgbevf_up_complete(adapter);
92915f71
GR		 1544
1545 /* clear any pending interrupts, may auto mask */
1546 IXGBE_READ_REG(hw, IXGBE_VTEICR);
1547
5f3600eb 1548 ixgbevf_irq_enable(adapter);
92915f71
GR		 1549}
1550
1551/**
1552 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
1553 * @adapter: board private structure
1554 * @rx_ring: ring to free buffers from
1555 **/
1556static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter,
1557 struct ixgbevf_ring *rx_ring)
1558{
1559 struct pci_dev *pdev = adapter->pdev;
1560 unsigned long size;
1561 unsigned int i;
1562
c0456c23
GR		 1563 if (!rx_ring->rx_buffer_info)
1564 return;
92915f71 1565
c0456c23 1566 /* Free all the Rx ring sk_buffs */
92915f71
GR		 1567 for (i = 0; i < rx_ring->count; i++) {
1568 struct ixgbevf_rx_buffer *rx_buffer_info;
1569
1570 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1571 if (rx_buffer_info->dma) {
2a1f8794 1572 dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
92915f71 1573 rx_ring->rx_buf_len,
2a1f8794 1574 DMA_FROM_DEVICE);
92915f71
GR		 1575 rx_buffer_info->dma = 0;
1576 }
1577 if (rx_buffer_info->skb) {
1578 struct sk_buff *skb = rx_buffer_info->skb;
1579 rx_buffer_info->skb = NULL;
1580 do {
1581 struct sk_buff *this = skb;
5c60f81a 1582 skb = IXGBE_CB(skb)->prev;
92915f71
GR		 1583 dev_kfree_skb(this);
1584 } while (skb);
1585 }
92915f71
GR		 1586 }
1587
1588 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1589 memset(rx_ring->rx_buffer_info, 0, size);
1590
1591 /* Zero out the descriptor ring */
1592 memset(rx_ring->desc, 0, rx_ring->size);
1593
1594 rx_ring->next_to_clean = 0;
1595 rx_ring->next_to_use = 0;
1596
1597 if (rx_ring->head)
1598 writel(0, adapter->hw.hw_addr + rx_ring->head);
1599 if (rx_ring->tail)
1600 writel(0, adapter->hw.hw_addr + rx_ring->tail);
1601}
1602
1603/**
1604 * ixgbevf_clean_tx_ring - Free Tx Buffers
1605 * @adapter: board private structure
1606 * @tx_ring: ring to be cleaned
1607 **/
1608static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter,
1609 struct ixgbevf_ring *tx_ring)
1610{
1611 struct ixgbevf_tx_buffer *tx_buffer_info;
1612 unsigned long size;
1613 unsigned int i;
1614
c0456c23
GR		 1615 if (!tx_ring->tx_buffer_info)
1616 return;
1617
92915f71 1618 /* Free all the Tx ring sk_buffs */
92915f71
GR		 1619 for (i = 0; i < tx_ring->count; i++) {
1620 tx_buffer_info = &tx_ring->tx_buffer_info[i];
70a10e25 1621 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
92915f71
GR		 1622 }
1623
1624 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1625 memset(tx_ring->tx_buffer_info, 0, size);
1626
1627 memset(tx_ring->desc, 0, tx_ring->size);
1628
1629 tx_ring->next_to_use = 0;
1630 tx_ring->next_to_clean = 0;
1631
1632 if (tx_ring->head)
1633 writel(0, adapter->hw.hw_addr + tx_ring->head);
1634 if (tx_ring->tail)
1635 writel(0, adapter->hw.hw_addr + tx_ring->tail);
1636}
1637
1638/**
1639 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1640 * @adapter: board private structure
1641 **/
1642static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1643{
1644 int i;
1645
1646 for (i = 0; i < adapter->num_rx_queues; i++)
1647 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]);
1648}
1649
1650/**
1651 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1652 * @adapter: board private structure
1653 **/
1654static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1655{
1656 int i;
1657
1658 for (i = 0; i < adapter->num_tx_queues; i++)
1659 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]);
1660}
1661
1662void ixgbevf_down(struct ixgbevf_adapter *adapter)
1663{
1664 struct net_device *netdev = adapter->netdev;
1665 struct ixgbe_hw *hw = &adapter->hw;
1666 u32 txdctl;
1667 int i, j;
1668
1669 /* signal that we are down to the interrupt handler */
1670 set_bit(__IXGBEVF_DOWN, &adapter->state);
1671 /* disable receives */
1672
1673 netif_tx_disable(netdev);
1674
1675 msleep(10);
1676
1677 netif_tx_stop_all_queues(netdev);
1678
1679 ixgbevf_irq_disable(adapter);
1680
1681 ixgbevf_napi_disable_all(adapter);
1682
1683 del_timer_sync(&adapter->watchdog_timer);
1684 /* can't call flush scheduled work here because it can deadlock
1685 * if linkwatch_event tries to acquire the rtnl_lock which we are
1686 * holding */
1687 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1688 msleep(1);
1689
1690 /* disable transmits in the hardware now that interrupts are off */
1691 for (i = 0; i < adapter->num_tx_queues; i++) {
1692 j = adapter->tx_ring[i].reg_idx;
1693 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1694 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j),
1695 (txdctl & ~IXGBE_TXDCTL_ENABLE));
1696 }
1697
1698 netif_carrier_off(netdev);
1699
1700 if (!pci_channel_offline(adapter->pdev))
1701 ixgbevf_reset(adapter);
1702
1703 ixgbevf_clean_all_tx_rings(adapter);
1704 ixgbevf_clean_all_rx_rings(adapter);
1705}
1706
1707void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1708{
1709 WARN_ON(in_interrupt());
c0456c23 1710
92915f71
GR		 1711 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1712 msleep(1);
1713
4b2cd27f
AD		 1714 ixgbevf_down(adapter);
1715 ixgbevf_up(adapter);
92915f71
GR		 1716
1717 clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1718}
1719
1720void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1721{
1722 struct ixgbe_hw *hw = &adapter->hw;
1723 struct net_device *netdev = adapter->netdev;
1724
1725 if (hw->mac.ops.reset_hw(hw))
1726 hw_dbg(hw, "PF still resetting\n");
1727 else
1728 hw->mac.ops.init_hw(hw);
1729
1730 if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1731 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1732 netdev->addr_len);
1733 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1734 netdev->addr_len);
1735 }
1736}
1737
e45dd5fe
JK		 1738static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1739 int vectors)
92915f71 1740{
a5f9337b
ET		 1741 int err = 0;
1742 int vector_threshold;
92915f71 1743
fa71ae27
AD		 1744 /* We'll want at least 2 (vector_threshold):
1745 * 1) TxQ[0] + RxQ[0] handler
1746 * 2) Other (Link Status Change, etc.)
92915f71
GR		 1747 */
1748 vector_threshold = MIN_MSIX_COUNT;
1749
1750 /* The more we get, the more we will assign to Tx/Rx Cleanup
1751 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1752 * Right now, we simply care about how many we'll get; we'll
1753 * set them up later while requesting irq's.
1754 */
1755 while (vectors >= vector_threshold) {
1756 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1757 vectors);
e45dd5fe 1758 if (!err || err < 0) /* Success or a nasty failure. */
92915f71 1759 break;
92915f71
GR		 1760 else /* err == number of vectors we should try again with */
1761 vectors = err;
1762 }
1763
e45dd5fe
JK		 1764 if (vectors < vector_threshold)
1765 err = -ENOMEM;
1766
1767 if (err) {
1768 dev_err(&adapter->pdev->dev,
1769 "Unable to allocate MSI-X interrupts\n");
92915f71
GR		 1770 kfree(adapter->msix_entries);
1771 adapter->msix_entries = NULL;
1772 } else {
1773 /*
1774 * Adjust for only the vectors we'll use, which is minimum
1775 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1776 * vectors we were allocated.
1777 */
1778 adapter->num_msix_vectors = vectors;
1779 }
dee847f5 1780
e45dd5fe 1781 return err;
92915f71
GR		 1782}
1783
49ce9c2c
BH		 1784/**
1785 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
92915f71
GR		 1786 * @adapter: board private structure to initialize
1787 *
1788 * This is the top level queue allocation routine. The order here is very
1789 * important, starting with the "most" number of features turned on at once,
1790 * and ending with the smallest set of features. This way large combinations
1791 * can be allocated if they're turned on, and smaller combinations are the
1792 * fallthrough conditions.
1793 *
1794 **/
1795static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1796{
1797 /* Start with base case */
1798 adapter->num_rx_queues = 1;
1799 adapter->num_tx_queues = 1;
92915f71
GR		 1800}
1801
1802/**
1803 * ixgbevf_alloc_queues - Allocate memory for all rings
1804 * @adapter: board private structure to initialize
1805 *
1806 * We allocate one ring per queue at run-time since we don't know the
1807 * number of queues at compile-time. The polling_netdev array is
1808 * intended for Multiqueue, but should work fine with a single queue.
1809 **/
1810static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1811{
1812 int i;
1813
1814 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1815 sizeof(struct ixgbevf_ring), GFP_KERNEL);
1816 if (!adapter->tx_ring)
1817 goto err_tx_ring_allocation;
1818
1819 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1820 sizeof(struct ixgbevf_ring), GFP_KERNEL);
1821 if (!adapter->rx_ring)
1822 goto err_rx_ring_allocation;
1823
1824 for (i = 0; i < adapter->num_tx_queues; i++) {
1825 adapter->tx_ring[i].count = adapter->tx_ring_count;
1826 adapter->tx_ring[i].queue_index = i;
56e94095 1827 /* reg_idx may be remapped later by DCB config */
92915f71 1828 adapter->tx_ring[i].reg_idx = i;
fb40195c
AD		 1829 adapter->tx_ring[i].dev = &adapter->pdev->dev;
1830 adapter->tx_ring[i].netdev = adapter->netdev;
92915f71
GR		 1831 }
1832
1833 for (i = 0; i < adapter->num_rx_queues; i++) {
1834 adapter->rx_ring[i].count = adapter->rx_ring_count;
1835 adapter->rx_ring[i].queue_index = i;
1836 adapter->rx_ring[i].reg_idx = i;
fb40195c
AD		 1837 adapter->rx_ring[i].dev = &adapter->pdev->dev;
1838 adapter->rx_ring[i].netdev = adapter->netdev;
92915f71
GR		 1839 }
1840
1841 return 0;
1842
1843err_rx_ring_allocation:
1844 kfree(adapter->tx_ring);
1845err_tx_ring_allocation:
1846 return -ENOMEM;
1847}
1848
1849/**
1850 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
1851 * @adapter: board private structure to initialize
1852 *
1853 * Attempt to configure the interrupts using the best available
1854 * capabilities of the hardware and the kernel.
1855 **/
1856static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
1857{
91e2b89b 1858 struct net_device *netdev = adapter->netdev;
92915f71
GR		 1859 int err = 0;
1860 int vector, v_budget;
1861
1862 /*
1863 * It's easy to be greedy for MSI-X vectors, but it really
1864 * doesn't do us much good if we have a lot more vectors
1865 * than CPU's. So let's be conservative and only ask for
fa71ae27
AD		 1866 * (roughly) the same number of vectors as there are CPU's.
1867 * The default is to use pairs of vectors.
92915f71 1868 */
fa71ae27
AD		 1869 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
1870 v_budget = min_t(int, v_budget, num_online_cpus());
1871 v_budget += NON_Q_VECTORS;
92915f71
GR		 1872
1873 /* A failure in MSI-X entry allocation isn't fatal, but it does
1874 * mean we disable MSI-X capabilities of the adapter. */
1875 adapter->msix_entries = kcalloc(v_budget,
1876 sizeof(struct msix_entry), GFP_KERNEL);
1877 if (!adapter->msix_entries) {
1878 err = -ENOMEM;
1879 goto out;
1880 }
1881
1882 for (vector = 0; vector < v_budget; vector++)
1883 adapter->msix_entries[vector].entry = vector;
1884
e45dd5fe
JK		 1885 err = ixgbevf_acquire_msix_vectors(adapter, v_budget);
1886 if (err)
1887 goto out;
92915f71 1888
91e2b89b
GR		 1889 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
1890 if (err)
1891 goto out;
1892
1893 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
1894
92915f71
GR		 1895out:
1896 return err;
1897}
1898
1899/**
1900 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
1901 * @adapter: board private structure to initialize
1902 *
1903 * We allocate one q_vector per queue interrupt. If allocation fails we
1904 * return -ENOMEM.
1905 **/
1906static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
1907{
1908 int q_idx, num_q_vectors;
1909 struct ixgbevf_q_vector *q_vector;
92915f71
GR		 1910
1911 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
92915f71
GR		 1912
1913 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1914 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
1915 if (!q_vector)
1916 goto err_out;
1917 q_vector->adapter = adapter;
1918 q_vector->v_idx = q_idx;
fa71ae27
AD		 1919 netif_napi_add(adapter->netdev, &q_vector->napi,
1920 ixgbevf_poll, 64);
92915f71
GR		 1921 adapter->q_vector[q_idx] = q_vector;
1922 }
1923
1924 return 0;
1925
1926err_out:
1927 while (q_idx) {
1928 q_idx--;
1929 q_vector = adapter->q_vector[q_idx];
1930 netif_napi_del(&q_vector->napi);
1931 kfree(q_vector);
1932 adapter->q_vector[q_idx] = NULL;
1933 }
1934 return -ENOMEM;
1935}
1936
1937/**
1938 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
1939 * @adapter: board private structure to initialize
1940 *
1941 * This function frees the memory allocated to the q_vectors. In addition if
1942 * NAPI is enabled it will delete any references to the NAPI struct prior
1943 * to freeing the q_vector.
1944 **/
1945static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
1946{
f4477702 1947 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
92915f71
GR		 1948
1949 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1950 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
1951
1952 adapter->q_vector[q_idx] = NULL;
f4477702 1953 netif_napi_del(&q_vector->napi);
92915f71
GR		 1954 kfree(q_vector);
1955 }
1956}
1957
1958/**
1959 * ixgbevf_reset_interrupt_capability - Reset MSIX setup
1960 * @adapter: board private structure
1961 *
1962 **/
1963static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
1964{
1965 pci_disable_msix(adapter->pdev);
1966 kfree(adapter->msix_entries);
1967 adapter->msix_entries = NULL;
92915f71
GR		 1968}
1969
1970/**
1971 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
1972 * @adapter: board private structure to initialize
1973 *
1974 **/
1975static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
1976{
1977 int err;
1978
1979 /* Number of supported queues */
1980 ixgbevf_set_num_queues(adapter);
1981
1982 err = ixgbevf_set_interrupt_capability(adapter);
1983 if (err) {
1984 hw_dbg(&adapter->hw,
1985 "Unable to setup interrupt capabilities\n");
1986 goto err_set_interrupt;
1987 }
1988
1989 err = ixgbevf_alloc_q_vectors(adapter);
1990 if (err) {
1991 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
1992 "vectors\n");
1993 goto err_alloc_q_vectors;
1994 }
1995
1996 err = ixgbevf_alloc_queues(adapter);
1997 if (err) {
dbd9636e 1998 pr_err("Unable to allocate memory for queues\n");
92915f71
GR		 1999 goto err_alloc_queues;
2000 }
2001
2002 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
2003 "Tx Queue count = %u\n",
2004 (adapter->num_rx_queues > 1) ? "Enabled" :
2005 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
2006
2007 set_bit(__IXGBEVF_DOWN, &adapter->state);
2008
2009 return 0;
2010err_alloc_queues:
2011 ixgbevf_free_q_vectors(adapter);
2012err_alloc_q_vectors:
2013 ixgbevf_reset_interrupt_capability(adapter);
2014err_set_interrupt:
2015 return err;
2016}
2017
0ac1e8ce
AD		 2018/**
2019 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
2020 * @adapter: board private structure to clear interrupt scheme on
2021 *
2022 * We go through and clear interrupt specific resources and reset the structure
2023 * to pre-load conditions
2024 **/
2025static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
2026{
2027 adapter->num_tx_queues = 0;
2028 adapter->num_rx_queues = 0;
2029
2030 ixgbevf_free_q_vectors(adapter);
2031 ixgbevf_reset_interrupt_capability(adapter);
2032}
2033
92915f71
GR		 2034/**
2035 * ixgbevf_sw_init - Initialize general software structures
2036 * (struct ixgbevf_adapter)
2037 * @adapter: board private structure to initialize
2038 *
2039 * ixgbevf_sw_init initializes the Adapter private data structure.
2040 * Fields are initialized based on PCI device information and
2041 * OS network device settings (MTU size).
2042 **/
9f9a12f8 2043static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
92915f71
GR		 2044{
2045 struct ixgbe_hw *hw = &adapter->hw;
2046 struct pci_dev *pdev = adapter->pdev;
2047 int err;
2048
2049 /* PCI config space info */
2050
2051 hw->vendor_id = pdev->vendor;
2052 hw->device_id = pdev->device;
ff938e43 2053 hw->revision_id = pdev->revision;
92915f71
GR		 2054 hw->subsystem_vendor_id = pdev->subsystem_vendor;
2055 hw->subsystem_device_id = pdev->subsystem_device;
2056
2057 hw->mbx.ops.init_params(hw);
56e94095
AD		 2058
2059 /* assume legacy case in which PF would only give VF 2 queues */
2060 hw->mac.max_tx_queues = 2;
2061 hw->mac.max_rx_queues = 2;
2062
92915f71
GR		 2063 err = hw->mac.ops.reset_hw(hw);
2064 if (err) {
2065 dev_info(&pdev->dev,
2066 "PF still in reset state, assigning new address\n");
1a0d6ae5
DK		 2067 eth_hw_addr_random(adapter->netdev);
2068 memcpy(adapter->hw.mac.addr, adapter->netdev->dev_addr,
2069 adapter->netdev->addr_len);
92915f71
GR		 2070 } else {
2071 err = hw->mac.ops.init_hw(hw);
2072 if (err) {
dbd9636e 2073 pr_err("init_shared_code failed: %d\n", err);
92915f71
GR		 2074 goto out;
2075 }
1a0d6ae5 2076 memcpy(adapter->netdev->dev_addr, adapter->hw.mac.addr,
dee847f5 2077 adapter->netdev->addr_len);
92915f71
GR		 2078 }
2079
1c55ed76
AD		 2080 /* lock to protect mailbox accesses */
2081 spin_lock_init(&adapter->mbx_lock);
2082
92915f71 2083 /* Enable dynamic interrupt throttling rates */
5f3600eb
AD		 2084 adapter->rx_itr_setting = 1;
2085 adapter->tx_itr_setting = 1;
92915f71 2086
92915f71
GR		 2087 /* set default ring sizes */
2088 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
2089 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
2090
92915f71 2091 set_bit(__IXGBEVF_DOWN, &adapter->state);
1a0d6ae5 2092 return 0;
92915f71
GR		 2093
2094out:
2095 return err;
2096}
2097
92915f71
GR		 2098#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \
2099 { \
2100 u32 current_counter = IXGBE_READ_REG(hw, reg); \
2101 if (current_counter < last_counter) \
2102 counter += 0x100000000LL; \
2103 last_counter = current_counter; \
2104 counter &= 0xFFFFFFFF00000000LL; \
2105 counter |= current_counter; \
2106 }
2107
2108#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2109 { \
2110 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \
2111 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \
2112 u64 current_counter = (current_counter_msb << 32) | \
2113 current_counter_lsb; \
2114 if (current_counter < last_counter) \
2115 counter += 0x1000000000LL; \
2116 last_counter = current_counter; \
2117 counter &= 0xFFFFFFF000000000LL; \
2118 counter |= current_counter; \
2119 }
2120/**
2121 * ixgbevf_update_stats - Update the board statistics counters.
2122 * @adapter: board private structure
2123 **/
2124void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2125{
2126 struct ixgbe_hw *hw = &adapter->hw;
55fb277c 2127 int i;
92915f71 2128
088245a3
GR		 2129 if (!adapter->link_up)
2130 return;
2131
92915f71
GR		 2132 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2133 adapter->stats.vfgprc);
2134 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2135 adapter->stats.vfgptc);
2136 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2137 adapter->stats.last_vfgorc,
2138 adapter->stats.vfgorc);
2139 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2140 adapter->stats.last_vfgotc,
2141 adapter->stats.vfgotc);
2142 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2143 adapter->stats.vfmprc);
55fb277c
GR		 2144
2145 for (i = 0; i < adapter->num_rx_queues; i++) {
2146 adapter->hw_csum_rx_error +=
2147 adapter->rx_ring[i].hw_csum_rx_error;
2148 adapter->hw_csum_rx_good +=
2149 adapter->rx_ring[i].hw_csum_rx_good;
2150 adapter->rx_ring[i].hw_csum_rx_error = 0;
2151 adapter->rx_ring[i].hw_csum_rx_good = 0;
2152 }
92915f71
GR		 2153}
2154
2155/**
2156 * ixgbevf_watchdog - Timer Call-back
2157 * @data: pointer to adapter cast into an unsigned long
2158 **/
2159static void ixgbevf_watchdog(unsigned long data)
2160{
2161 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2162 struct ixgbe_hw *hw = &adapter->hw;
5f3600eb 2163 u32 eics = 0;
92915f71
GR		 2164 int i;
2165
2166 /*
2167 * Do the watchdog outside of interrupt context due to the lovely
2168 * delays that some of the newer hardware requires
2169 */
2170
2171 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2172 goto watchdog_short_circuit;
2173
2174 /* get one bit for every active tx/rx interrupt vector */
2175 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2176 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
6b43c446 2177 if (qv->rx.ring || qv->tx.ring)
5f3600eb 2178 eics |= 1 << i;
92915f71
GR		 2179 }
2180
5f3600eb 2181 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
92915f71
GR		 2182
2183watchdog_short_circuit:
2184 schedule_work(&adapter->watchdog_task);
2185}
2186
2187/**
2188 * ixgbevf_tx_timeout - Respond to a Tx Hang
2189 * @netdev: network interface device structure
2190 **/
2191static void ixgbevf_tx_timeout(struct net_device *netdev)
2192{
2193 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2194
2195 /* Do the reset outside of interrupt context */
2196 schedule_work(&adapter->reset_task);
2197}
2198
2199static void ixgbevf_reset_task(struct work_struct *work)
2200{
2201 struct ixgbevf_adapter *adapter;
2202 adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2203
2204 /* If we're already down or resetting, just bail */
2205 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2206 test_bit(__IXGBEVF_RESETTING, &adapter->state))
2207 return;
2208
2209 adapter->tx_timeout_count++;
2210
2211 ixgbevf_reinit_locked(adapter);
2212}
2213
2214/**
2215 * ixgbevf_watchdog_task - worker thread to bring link up
2216 * @work: pointer to work_struct containing our data
2217 **/
2218static void ixgbevf_watchdog_task(struct work_struct *work)
2219{
2220 struct ixgbevf_adapter *adapter = container_of(work,
2221 struct ixgbevf_adapter,
2222 watchdog_task);
2223 struct net_device *netdev = adapter->netdev;
2224 struct ixgbe_hw *hw = &adapter->hw;
2225 u32 link_speed = adapter->link_speed;
2226 bool link_up = adapter->link_up;
92fe0bf7 2227 s32 need_reset;
92915f71
GR		 2228
2229 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2230
2231 /*
2232 * Always check the link on the watchdog because we have
2233 * no LSC interrupt
2234 */
92fe0bf7 2235 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 2236
92fe0bf7 2237 need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
1c55ed76 2238
92fe0bf7 2239 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 2240
92fe0bf7
GR		 2241 if (need_reset) {
2242 adapter->link_up = link_up;
2243 adapter->link_speed = link_speed;
2244 netif_carrier_off(netdev);
2245 netif_tx_stop_all_queues(netdev);
2246 schedule_work(&adapter->reset_task);
2247 goto pf_has_reset;
92915f71
GR		 2248 }
2249 adapter->link_up = link_up;
2250 adapter->link_speed = link_speed;
2251
2252 if (link_up) {
2253 if (!netif_carrier_ok(netdev)) {
b876a744
GR		 2254 char *link_speed_string;
2255 switch (link_speed) {
2256 case IXGBE_LINK_SPEED_10GB_FULL:
2257 link_speed_string = "10 Gbps";
2258 break;
2259 case IXGBE_LINK_SPEED_1GB_FULL:
2260 link_speed_string = "1 Gbps";
2261 break;
2262 case IXGBE_LINK_SPEED_100_FULL:
2263 link_speed_string = "100 Mbps";
2264 break;
2265 default:
2266 link_speed_string = "unknown speed";
2267 break;
2268 }
6fe59675 2269 dev_info(&adapter->pdev->dev,
b876a744 2270 "NIC Link is Up, %s\n", link_speed_string);
92915f71
GR		 2271 netif_carrier_on(netdev);
2272 netif_tx_wake_all_queues(netdev);
92915f71
GR		 2273 }
2274 } else {
2275 adapter->link_up = false;
2276 adapter->link_speed = 0;
2277 if (netif_carrier_ok(netdev)) {
6fe59675 2278 dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
92915f71
GR		 2279 netif_carrier_off(netdev);
2280 netif_tx_stop_all_queues(netdev);
2281 }
2282 }
2283
92915f71
GR		 2284 ixgbevf_update_stats(adapter);
2285
33bd9f60 2286pf_has_reset:
92915f71
GR		 2287 /* Reset the timer */
2288 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
2289 mod_timer(&adapter->watchdog_timer,
2290 round_jiffies(jiffies + (2 * HZ)));
2291
2292 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2293}
2294
2295/**
2296 * ixgbevf_free_tx_resources - Free Tx Resources per Queue
2297 * @adapter: board private structure
2298 * @tx_ring: Tx descriptor ring for a specific queue
2299 *
2300 * Free all transmit software resources
2301 **/
2302void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter,
2303 struct ixgbevf_ring *tx_ring)
2304{
2305 struct pci_dev *pdev = adapter->pdev;
2306
92915f71
GR		 2307 ixgbevf_clean_tx_ring(adapter, tx_ring);
2308
2309 vfree(tx_ring->tx_buffer_info);
2310 tx_ring->tx_buffer_info = NULL;
2311
2a1f8794
NN		 2312 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2313 tx_ring->dma);
92915f71
GR		 2314
2315 tx_ring->desc = NULL;
2316}
2317
2318/**
2319 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2320 * @adapter: board private structure
2321 *
2322 * Free all transmit software resources
2323 **/
2324static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2325{
2326 int i;
2327
2328 for (i = 0; i < adapter->num_tx_queues; i++)
2329 if (adapter->tx_ring[i].desc)
2330 ixgbevf_free_tx_resources(adapter,
2331 &adapter->tx_ring[i]);
2332
2333}
2334
2335/**
2336 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
2337 * @adapter: board private structure
2338 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2339 *
2340 * Return 0 on success, negative on failure
2341 **/
2342int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter,
2343 struct ixgbevf_ring *tx_ring)
2344{
2345 struct pci_dev *pdev = adapter->pdev;
2346 int size;
2347
2348 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
89bf67f1 2349 tx_ring->tx_buffer_info = vzalloc(size);
92915f71
GR		 2350 if (!tx_ring->tx_buffer_info)
2351 goto err;
92915f71
GR		 2352
2353 /* round up to nearest 4K */
2354 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2355 tx_ring->size = ALIGN(tx_ring->size, 4096);
2356
2a1f8794
NN		 2357 tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
2358 &tx_ring->dma, GFP_KERNEL);
92915f71
GR		 2359 if (!tx_ring->desc)
2360 goto err;
2361
2362 tx_ring->next_to_use = 0;
2363 tx_ring->next_to_clean = 0;
92915f71
GR		 2364 return 0;
2365
2366err:
2367 vfree(tx_ring->tx_buffer_info);
2368 tx_ring->tx_buffer_info = NULL;
2369 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2370 "descriptor ring\n");
2371 return -ENOMEM;
2372}
2373
2374/**
2375 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2376 * @adapter: board private structure
2377 *
2378 * If this function returns with an error, then it's possible one or
2379 * more of the rings is populated (while the rest are not). It is the
2380 * callers duty to clean those orphaned rings.
2381 *
2382 * Return 0 on success, negative on failure
2383 **/
2384static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2385{
2386 int i, err = 0;
2387
2388 for (i = 0; i < adapter->num_tx_queues; i++) {
2389 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]);
2390 if (!err)
2391 continue;
2392 hw_dbg(&adapter->hw,
2393 "Allocation for Tx Queue %u failed\n", i);
2394 break;
2395 }
2396
2397 return err;
2398}
2399
2400/**
2401 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
2402 * @adapter: board private structure
2403 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2404 *
2405 * Returns 0 on success, negative on failure
2406 **/
2407int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
2408 struct ixgbevf_ring *rx_ring)
2409{
2410 struct pci_dev *pdev = adapter->pdev;
2411 int size;
2412
2413 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
89bf67f1 2414 rx_ring->rx_buffer_info = vzalloc(size);
e404decb 2415 if (!rx_ring->rx_buffer_info)
92915f71 2416 goto alloc_failed;
92915f71
GR		 2417
2418 /* Round up to nearest 4K */
2419 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2420 rx_ring->size = ALIGN(rx_ring->size, 4096);
2421
2a1f8794
NN		 2422 rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
2423 &rx_ring->dma, GFP_KERNEL);
92915f71
GR		 2424
2425 if (!rx_ring->desc) {
2426 hw_dbg(&adapter->hw,
2427 "Unable to allocate memory for "
2428 "the receive descriptor ring\n");
2429 vfree(rx_ring->rx_buffer_info);
2430 rx_ring->rx_buffer_info = NULL;
2431 goto alloc_failed;
2432 }
2433
2434 rx_ring->next_to_clean = 0;
2435 rx_ring->next_to_use = 0;
2436
2437 return 0;
2438alloc_failed:
2439 return -ENOMEM;
2440}
2441
2442/**
2443 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2444 * @adapter: board private structure
2445 *
2446 * If this function returns with an error, then it's possible one or
2447 * more of the rings is populated (while the rest are not). It is the
2448 * callers duty to clean those orphaned rings.
2449 *
2450 * Return 0 on success, negative on failure
2451 **/
2452static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2453{
2454 int i, err = 0;
2455
2456 for (i = 0; i < adapter->num_rx_queues; i++) {
2457 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]);
2458 if (!err)
2459 continue;
2460 hw_dbg(&adapter->hw,
2461 "Allocation for Rx Queue %u failed\n", i);
2462 break;
2463 }
2464 return err;
2465}
2466
2467/**
2468 * ixgbevf_free_rx_resources - Free Rx Resources
2469 * @adapter: board private structure
2470 * @rx_ring: ring to clean the resources from
2471 *
2472 * Free all receive software resources
2473 **/
2474void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter,
2475 struct ixgbevf_ring *rx_ring)
2476{
2477 struct pci_dev *pdev = adapter->pdev;
2478
2479 ixgbevf_clean_rx_ring(adapter, rx_ring);
2480
2481 vfree(rx_ring->rx_buffer_info);
2482 rx_ring->rx_buffer_info = NULL;
2483
2a1f8794
NN		 2484 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2485 rx_ring->dma);
92915f71
GR		 2486
2487 rx_ring->desc = NULL;
2488}
2489
2490/**
2491 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2492 * @adapter: board private structure
2493 *
2494 * Free all receive software resources
2495 **/
2496static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2497{
2498 int i;
2499
2500 for (i = 0; i < adapter->num_rx_queues; i++)
2501 if (adapter->rx_ring[i].desc)
2502 ixgbevf_free_rx_resources(adapter,
2503 &adapter->rx_ring[i]);
2504}
2505
56e94095
AD		 2506static int ixgbevf_setup_queues(struct ixgbevf_adapter *adapter)
2507{
2508 struct ixgbe_hw *hw = &adapter->hw;
2509 struct ixgbevf_ring *rx_ring;
2510 unsigned int def_q = 0;
2511 unsigned int num_tcs = 0;
2512 unsigned int num_rx_queues = 1;
2513 int err, i;
2514
55fdd45b 2515 spin_lock_bh(&adapter->mbx_lock);
56e94095
AD		 2516
2517 /* fetch queue configuration from the PF */
2518 err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
2519
55fdd45b 2520 spin_unlock_bh(&adapter->mbx_lock);
56e94095
AD		 2521
2522 if (err)
2523 return err;
2524
2525 if (num_tcs > 1) {
2526 /* update default Tx ring register index */
2527 adapter->tx_ring[0].reg_idx = def_q;
2528
2529 /* we need as many queues as traffic classes */
2530 num_rx_queues = num_tcs;
2531 }
2532
2533 /* nothing to do if we have the correct number of queues */
2534 if (adapter->num_rx_queues == num_rx_queues)
2535 return 0;
2536
2537 /* allocate new rings */
2538 rx_ring = kcalloc(num_rx_queues,
2539 sizeof(struct ixgbevf_ring), GFP_KERNEL);
2540 if (!rx_ring)
2541 return -ENOMEM;
2542
2543 /* setup ring fields */
2544 for (i = 0; i < num_rx_queues; i++) {
2545 rx_ring[i].count = adapter->rx_ring_count;
2546 rx_ring[i].queue_index = i;
2547 rx_ring[i].reg_idx = i;
2548 rx_ring[i].dev = &adapter->pdev->dev;
2549 rx_ring[i].netdev = adapter->netdev;
2550 }
2551
2552 /* free the existing ring and queues */
2553 adapter->num_rx_queues = 0;
2554 kfree(adapter->rx_ring);
2555
2556 /* move new rings into position on the adapter struct */
2557 adapter->rx_ring = rx_ring;
2558 adapter->num_rx_queues = num_rx_queues;
2559
2560 return 0;
2561}
2562
92915f71
GR		 2563/**
2564 * ixgbevf_open - Called when a network interface is made active
2565 * @netdev: network interface device structure
2566 *
2567 * Returns 0 on success, negative value on failure
2568 *
2569 * The open entry point is called when a network interface is made
2570 * active by the system (IFF_UP). At this point all resources needed
2571 * for transmit and receive operations are allocated, the interrupt
2572 * handler is registered with the OS, the watchdog timer is started,
2573 * and the stack is notified that the interface is ready.
2574 **/
2575static int ixgbevf_open(struct net_device *netdev)
2576{
2577 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2578 struct ixgbe_hw *hw = &adapter->hw;
2579 int err;
2580
2581 /* disallow open during test */
2582 if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2583 return -EBUSY;
2584
2585 if (hw->adapter_stopped) {
2586 ixgbevf_reset(adapter);
2587 /* if adapter is still stopped then PF isn't up and
2588 * the vf can't start. */
2589 if (hw->adapter_stopped) {
2590 err = IXGBE_ERR_MBX;
dbd9636e
JK		 2591 pr_err("Unable to start - perhaps the PF Driver isn't "
2592 "up yet\n");
92915f71
GR		 2593 goto err_setup_reset;
2594 }
2595 }
2596
31186785
AD		 2597 ixgbevf_negotiate_api(adapter);
2598
56e94095
AD		 2599 /* setup queue reg_idx and Rx queue count */
2600 err = ixgbevf_setup_queues(adapter);
2601 if (err)
2602 goto err_setup_queues;
2603
92915f71
GR		 2604 /* allocate transmit descriptors */
2605 err = ixgbevf_setup_all_tx_resources(adapter);
2606 if (err)
2607 goto err_setup_tx;
2608
2609 /* allocate receive descriptors */
2610 err = ixgbevf_setup_all_rx_resources(adapter);
2611 if (err)
2612 goto err_setup_rx;
2613
2614 ixgbevf_configure(adapter);
2615
2616 /*
2617 * Map the Tx/Rx rings to the vectors we were allotted.
2618 * if request_irq will be called in this function map_rings
2619 * must be called *before* up_complete
2620 */
2621 ixgbevf_map_rings_to_vectors(adapter);
2622
795180d8 2623 ixgbevf_up_complete(adapter);
92915f71
GR		 2624
2625 /* clear any pending interrupts, may auto mask */
2626 IXGBE_READ_REG(hw, IXGBE_VTEICR);
2627 err = ixgbevf_request_irq(adapter);
2628 if (err)
2629 goto err_req_irq;
2630
5f3600eb 2631 ixgbevf_irq_enable(adapter);
92915f71
GR		 2632
2633 return 0;
2634
2635err_req_irq:
2636 ixgbevf_down(adapter);
92915f71
GR		 2637 ixgbevf_free_irq(adapter);
2638err_setup_rx:
2639 ixgbevf_free_all_rx_resources(adapter);
2640err_setup_tx:
2641 ixgbevf_free_all_tx_resources(adapter);
56e94095 2642err_setup_queues:
92915f71
GR		 2643 ixgbevf_reset(adapter);
2644
2645err_setup_reset:
2646
2647 return err;
2648}
2649
2650/**
2651 * ixgbevf_close - Disables a network interface
2652 * @netdev: network interface device structure
2653 *
2654 * Returns 0, this is not allowed to fail
2655 *
2656 * The close entry point is called when an interface is de-activated
2657 * by the OS. The hardware is still under the drivers control, but
2658 * needs to be disabled. A global MAC reset is issued to stop the
2659 * hardware, and all transmit and receive resources are freed.
2660 **/
2661static int ixgbevf_close(struct net_device *netdev)
2662{
2663 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2664
2665 ixgbevf_down(adapter);
2666 ixgbevf_free_irq(adapter);
2667
2668 ixgbevf_free_all_tx_resources(adapter);
2669 ixgbevf_free_all_rx_resources(adapter);
2670
2671 return 0;
2672}
2673
70a10e25
AD		 2674static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
2675 u32 vlan_macip_lens, u32 type_tucmd,
2676 u32 mss_l4len_idx)
92915f71
GR		 2677{
2678 struct ixgbe_adv_tx_context_desc *context_desc;
70a10e25 2679 u16 i = tx_ring->next_to_use;
92915f71 2680
70a10e25 2681 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
92915f71 2682
70a10e25
AD		 2683 i++;
2684 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
92915f71 2685
70a10e25
AD		 2686 /* set bits to identify this as an advanced context descriptor */
2687 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
92915f71 2688
70a10e25
AD		 2689 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
2690 context_desc->seqnum_seed = 0;
2691 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
2692 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
2693}
2694
2695static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
2696 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2697{
2698 u32 vlan_macip_lens, type_tucmd;
2699 u32 mss_l4len_idx, l4len;
2700
2701 if (!skb_is_gso(skb))
2702 return 0;
92915f71 2703
70a10e25
AD		 2704 if (skb_header_cloned(skb)) {
2705 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2706 if (err)
2707 return err;
92915f71
GR		 2708 }
2709
70a10e25
AD		 2710 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2711 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
2712
2713 if (skb->protocol == htons(ETH_P_IP)) {
2714 struct iphdr *iph = ip_hdr(skb);
2715 iph->tot_len = 0;
2716 iph->check = 0;
2717 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2718 iph->daddr, 0,
2719 IPPROTO_TCP,
2720 0);
2721 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2722 } else if (skb_is_gso_v6(skb)) {
2723 ipv6_hdr(skb)->payload_len = 0;
2724 tcp_hdr(skb)->check =
2725 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2726 &ipv6_hdr(skb)->daddr,
2727 0, IPPROTO_TCP, 0);
2728 }
2729
2730 /* compute header lengths */
2731 l4len = tcp_hdrlen(skb);
2732 *hdr_len += l4len;
2733 *hdr_len = skb_transport_offset(skb) + l4len;
2734
2735 /* mss_l4len_id: use 1 as index for TSO */
2736 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
2737 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
2738 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
2739
2740 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
2741 vlan_macip_lens = skb_network_header_len(skb);
2742 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2743 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2744
2745 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2746 type_tucmd, mss_l4len_idx);
2747
2748 return 1;
92915f71
GR		 2749}
2750
70a10e25 2751static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
92915f71
GR		 2752 struct sk_buff *skb, u32 tx_flags)
2753{
70a10e25
AD		 2754 u32 vlan_macip_lens = 0;
2755 u32 mss_l4len_idx = 0;
2756 u32 type_tucmd = 0;
92915f71 2757
70a10e25
AD		 2758 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2759 u8 l4_hdr = 0;
2760 switch (skb->protocol) {
2761 case __constant_htons(ETH_P_IP):
2762 vlan_macip_lens |= skb_network_header_len(skb);
2763 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2764 l4_hdr = ip_hdr(skb)->protocol;
2765 break;
2766 case __constant_htons(ETH_P_IPV6):
2767 vlan_macip_lens |= skb_network_header_len(skb);
2768 l4_hdr = ipv6_hdr(skb)->nexthdr;
2769 break;
2770 default:
2771 if (unlikely(net_ratelimit())) {
2772 dev_warn(tx_ring->dev,
2773 "partial checksum but proto=%x!\n",
2774 skb->protocol);
2775 }
2776 break;
2777 }
92915f71 2778
70a10e25
AD		 2779 switch (l4_hdr) {
2780 case IPPROTO_TCP:
2781 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2782 mss_l4len_idx = tcp_hdrlen(skb) <<
2783 IXGBE_ADVTXD_L4LEN_SHIFT;
2784 break;
2785 case IPPROTO_SCTP:
2786 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2787 mss_l4len_idx = sizeof(struct sctphdr) <<
2788 IXGBE_ADVTXD_L4LEN_SHIFT;
2789 break;
2790 case IPPROTO_UDP:
2791 mss_l4len_idx = sizeof(struct udphdr) <<
2792 IXGBE_ADVTXD_L4LEN_SHIFT;
2793 break;
2794 default:
2795 if (unlikely(net_ratelimit())) {
2796 dev_warn(tx_ring->dev,
2797 "partial checksum but l4 proto=%x!\n",
2798 l4_hdr);
2799 }
2800 break;
2801 }
92915f71
GR		 2802 }
2803
70a10e25
AD		 2804 /* vlan_macip_lens: MACLEN, VLAN tag */
2805 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2806 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2807
2808 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2809 type_tucmd, mss_l4len_idx);
2810
2811 return (skb->ip_summed == CHECKSUM_PARTIAL);
92915f71
GR		 2812}
2813
70a10e25 2814static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
e757e3e1 2815 struct sk_buff *skb, u32 tx_flags)
92915f71 2816{
92915f71
GR		 2817 struct ixgbevf_tx_buffer *tx_buffer_info;
2818 unsigned int len;
2819 unsigned int total = skb->len;
2540ddb5
KV		 2820 unsigned int offset = 0, size;
2821 int count = 0;
92915f71
GR		 2822 unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
2823 unsigned int f;
65deeed7 2824 int i;
92915f71
GR		 2825
2826 i = tx_ring->next_to_use;
2827
2828 len = min(skb_headlen(skb), total);
2829 while (len) {
2830 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2831 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2832
2833 tx_buffer_info->length = size;
2834 tx_buffer_info->mapped_as_page = false;
70a10e25 2835 tx_buffer_info->dma = dma_map_single(tx_ring->dev,
92915f71 2836 skb->data + offset,
2a1f8794 2837 size, DMA_TO_DEVICE);
70a10e25 2838 if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma))
92915f71 2839 goto dma_error;
92915f71
GR		 2840
2841 len -= size;
2842 total -= size;
2843 offset += size;
2844 count++;
2845 i++;
2846 if (i == tx_ring->count)
2847 i = 0;
2848 }
2849
2850 for (f = 0; f < nr_frags; f++) {
9e903e08 2851 const struct skb_frag_struct *frag;
92915f71
GR		 2852
2853 frag = &skb_shinfo(skb)->frags[f];
9e903e08 2854 len = min((unsigned int)skb_frag_size(frag), total);
877749bf 2855 offset = 0;
92915f71
GR		 2856
2857 while (len) {
2858 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2859 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2860
2861 tx_buffer_info->length = size;
877749bf 2862 tx_buffer_info->dma =
70a10e25 2863 skb_frag_dma_map(tx_ring->dev, frag,
877749bf 2864 offset, size, DMA_TO_DEVICE);
70a10e25
AD		 2865 if (dma_mapping_error(tx_ring->dev,
2866 tx_buffer_info->dma))
92915f71 2867 goto dma_error;
6132ee8a 2868 tx_buffer_info->mapped_as_page = true;
92915f71
GR		 2869
2870 len -= size;
2871 total -= size;
2872 offset += size;
2873 count++;
2874 i++;
2875 if (i == tx_ring->count)
2876 i = 0;
2877 }
2878 if (total == 0)
2879 break;
2880 }
2881
2882 if (i == 0)
2883 i = tx_ring->count - 1;
2884 else
2885 i = i - 1;
2886 tx_ring->tx_buffer_info[i].skb = skb;
92915f71
GR		 2887
2888 return count;
2889
2890dma_error:
70a10e25 2891 dev_err(tx_ring->dev, "TX DMA map failed\n");
92915f71
GR		 2892
2893 /* clear timestamp and dma mappings for failed tx_buffer_info map */
2894 tx_buffer_info->dma = 0;
92915f71
GR		 2895 count--;
2896
2897 /* clear timestamp and dma mappings for remaining portion of packet */
2898 while (count >= 0) {
2899 count--;
2900 i--;
2901 if (i < 0)
2902 i += tx_ring->count;
2903 tx_buffer_info = &tx_ring->tx_buffer_info[i];
70a10e25 2904 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
92915f71
GR		 2905 }
2906
2907 return count;
2908}
2909
70a10e25 2910static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags,
e757e3e1
AD		 2911 int count, unsigned int first, u32 paylen,
2912 u8 hdr_len)
92915f71
GR		 2913{
2914 union ixgbe_adv_tx_desc *tx_desc = NULL;
2915 struct ixgbevf_tx_buffer *tx_buffer_info;
2916 u32 olinfo_status = 0, cmd_type_len = 0;
2917 unsigned int i;
2918
2919 u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
2920
2921 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
2922
2923 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
2924
2925 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2926 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
2927
70a10e25
AD		 2928 if (tx_flags & IXGBE_TX_FLAGS_CSUM)
2929 olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM;
2930
92915f71
GR		 2931 if (tx_flags & IXGBE_TX_FLAGS_TSO) {
2932 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
2933
92915f71
GR		 2934 /* use index 1 context for tso */
2935 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
2936 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
70a10e25 2937 olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM;
70a10e25 2938 }
92915f71 2939
70a10e25
AD		 2940 /*
2941 * Check Context must be set if Tx switch is enabled, which it
2942 * always is for case where virtual functions are running
2943 */
2944 olinfo_status |= IXGBE_ADVTXD_CC;
92915f71
GR		 2945
2946 olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
2947
2948 i = tx_ring->next_to_use;
2949 while (count--) {
2950 tx_buffer_info = &tx_ring->tx_buffer_info[i];
908421f6 2951 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
92915f71
GR		 2952 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
2953 tx_desc->read.cmd_type_len =
2954 cpu_to_le32(cmd_type_len | tx_buffer_info->length);
2955 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2956 i++;
2957 if (i == tx_ring->count)
2958 i = 0;
2959 }
2960
2961 tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
2962
e757e3e1
AD		 2963 tx_ring->tx_buffer_info[first].time_stamp = jiffies;
2964
2965 /* Force memory writes to complete before letting h/w
2966 * know there are new descriptors to fetch. (Only
2967 * applicable for weak-ordered memory model archs,
2968 * such as IA-64).
2969 */
2970 wmb();
2971
2972 tx_ring->tx_buffer_info[first].next_to_watch = tx_desc;
92915f71 2973 tx_ring->next_to_use = i;
92915f71
GR		 2974}
2975
fb40195c 2976static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
92915f71 2977{
fb40195c 2978 struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
92915f71 2979
fb40195c 2980 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
92915f71
GR		 2981 /* Herbert's original patch had:
2982 * smp_mb__after_netif_stop_queue();
2983 * but since that doesn't exist yet, just open code it. */
2984 smp_mb();
2985
2986 /* We need to check again in a case another CPU has just
2987 * made room available. */
2988 if (likely(IXGBE_DESC_UNUSED(tx_ring) < size))
2989 return -EBUSY;
2990
2991 /* A reprieve! - use start_queue because it doesn't call schedule */
fb40195c 2992 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
92915f71
GR		 2993 ++adapter->restart_queue;
2994 return 0;
2995}
2996
fb40195c 2997static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
92915f71
GR		 2998{
2999 if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size))
3000 return 0;
fb40195c 3001 return __ixgbevf_maybe_stop_tx(tx_ring, size);
92915f71
GR		 3002}
3003
3004static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
3005{
3006 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3007 struct ixgbevf_ring *tx_ring;
3008 unsigned int first;
3009 unsigned int tx_flags = 0;
3010 u8 hdr_len = 0;
3011 int r_idx = 0, tso;
3595990a
AD		 3012 u16 count = TXD_USE_COUNT(skb_headlen(skb));
3013#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3014 unsigned short f;
3015#endif
f9d08f16 3016 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL);
46acc460 3017 if (!dst_mac || is_link_local_ether_addr(dst_mac)) {
f9d08f16
GR		 3018 dev_kfree_skb(skb);
3019 return NETDEV_TX_OK;
3020 }
92915f71
GR		 3021
3022 tx_ring = &adapter->tx_ring[r_idx];
3023
3595990a
AD		 3024 /*
3025 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
3026 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
3027 * + 2 desc gap to keep tail from touching head,
3028 * + 1 desc for context descriptor,
3029 * otherwise try next time
3030 */
3031#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3032 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
3033 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
3034#else
3035 count += skb_shinfo(skb)->nr_frags;
3036#endif
fb40195c 3037 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
3595990a
AD		 3038 adapter->tx_busy++;
3039 return NETDEV_TX_BUSY;
3040 }
3041
eab6d18d 3042 if (vlan_tx_tag_present(skb)) {
92915f71
GR		 3043 tx_flags |= vlan_tx_tag_get(skb);
3044 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
3045 tx_flags |= IXGBE_TX_FLAGS_VLAN;
3046 }
3047
92915f71
GR		 3048 first = tx_ring->next_to_use;
3049
3050 if (skb->protocol == htons(ETH_P_IP))
3051 tx_flags |= IXGBE_TX_FLAGS_IPV4;
70a10e25 3052 tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len);
92915f71
GR		 3053 if (tso < 0) {
3054 dev_kfree_skb_any(skb);
3055 return NETDEV_TX_OK;
3056 }
3057
3058 if (tso)
70a10e25
AD		 3059 tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM;
3060 else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags))
92915f71
GR		 3061 tx_flags |= IXGBE_TX_FLAGS_CSUM;
3062
70a10e25 3063 ixgbevf_tx_queue(tx_ring, tx_flags,
e757e3e1
AD		 3064 ixgbevf_tx_map(tx_ring, skb, tx_flags),
3065 first, skb->len, hdr_len);
70a10e25
AD		 3066
3067 writel(tx_ring->next_to_use, adapter->hw.hw_addr + tx_ring->tail);
92915f71 3068
fb40195c 3069 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
92915f71
GR		 3070
3071 return NETDEV_TX_OK;
3072}
3073
92915f71
GR		 3074/**
3075 * ixgbevf_set_mac - Change the Ethernet Address of the NIC
3076 * @netdev: network interface device structure
3077 * @p: pointer to an address structure
3078 *
3079 * Returns 0 on success, negative on failure
3080 **/
3081static int ixgbevf_set_mac(struct net_device *netdev, void *p)
3082{
3083 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3084 struct ixgbe_hw *hw = &adapter->hw;
3085 struct sockaddr *addr = p;
3086
3087 if (!is_valid_ether_addr(addr->sa_data))
3088 return -EADDRNOTAVAIL;
3089
3090 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3091 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3092
55fdd45b 3093 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 3094
92fe0bf7 3095 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
92915f71 3096
55fdd45b 3097 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 3098
92915f71
GR		 3099 return 0;
3100}
3101
3102/**
3103 * ixgbevf_change_mtu - Change the Maximum Transfer Unit
3104 * @netdev: network interface device structure
3105 * @new_mtu: new value for maximum frame size
3106 *
3107 * Returns 0 on success, negative on failure
3108 **/
3109static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
3110{
3111 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3112 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
69bfbec4 3113 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
69bfbec4 3114
56e94095
AD		 3115 switch (adapter->hw.api_version) {
3116 case ixgbe_mbox_api_11:
69bfbec4 3117 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
56e94095
AD		 3118 break;
3119 default:
3120 if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
3121 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3122 break;
3123 }
92915f71
GR		 3124
3125 /* MTU < 68 is an error and causes problems on some kernels */
69bfbec4 3126 if ((new_mtu < 68) || (max_frame > max_possible_frame))
92915f71
GR		 3127 return -EINVAL;
3128
3129 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
3130 netdev->mtu, new_mtu);
3131 /* must set new MTU before calling down or up */
3132 netdev->mtu = new_mtu;
3133
3134 if (netif_running(netdev))
3135 ixgbevf_reinit_locked(adapter);
3136
3137 return 0;
3138}
3139
0ac1e8ce 3140static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state)
92915f71
GR		 3141{
3142 struct net_device *netdev = pci_get_drvdata(pdev);
3143 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
0ac1e8ce
AD		 3144#ifdef CONFIG_PM
3145 int retval = 0;
3146#endif
92915f71
GR		 3147
3148 netif_device_detach(netdev);
3149
3150 if (netif_running(netdev)) {
0ac1e8ce 3151 rtnl_lock();
92915f71
GR		 3152 ixgbevf_down(adapter);
3153 ixgbevf_free_irq(adapter);
3154 ixgbevf_free_all_tx_resources(adapter);
3155 ixgbevf_free_all_rx_resources(adapter);
0ac1e8ce 3156 rtnl_unlock();
92915f71
GR		 3157 }
3158
0ac1e8ce 3159 ixgbevf_clear_interrupt_scheme(adapter);
92915f71 3160
0ac1e8ce
AD		 3161#ifdef CONFIG_PM
3162 retval = pci_save_state(pdev);
3163 if (retval)
3164 return retval;
92915f71 3165
0ac1e8ce 3166#endif
92915f71 3167 pci_disable_device(pdev);
0ac1e8ce
AD		 3168
3169 return 0;
3170}
3171
3172#ifdef CONFIG_PM
3173static int ixgbevf_resume(struct pci_dev *pdev)
3174{
3175 struct ixgbevf_adapter *adapter = pci_get_drvdata(pdev);
3176 struct net_device *netdev = adapter->netdev;
3177 u32 err;
3178
3179 pci_set_power_state(pdev, PCI_D0);
3180 pci_restore_state(pdev);
3181 /*
3182 * pci_restore_state clears dev->state_saved so call
3183 * pci_save_state to restore it.
3184 */
3185 pci_save_state(pdev);
3186
3187 err = pci_enable_device_mem(pdev);
3188 if (err) {
3189 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
3190 return err;
3191 }
3192 pci_set_master(pdev);
3193
3194 rtnl_lock();
3195 err = ixgbevf_init_interrupt_scheme(adapter);
3196 rtnl_unlock();
3197 if (err) {
3198 dev_err(&pdev->dev, "Cannot initialize interrupts\n");
3199 return err;
3200 }
3201
3202 ixgbevf_reset(adapter);
3203
3204 if (netif_running(netdev)) {
3205 err = ixgbevf_open(netdev);
3206 if (err)
3207 return err;
3208 }
3209
3210 netif_device_attach(netdev);
3211
3212 return err;
3213}
3214
3215#endif /* CONFIG_PM */
3216static void ixgbevf_shutdown(struct pci_dev *pdev)
3217{
3218 ixgbevf_suspend(pdev, PMSG_SUSPEND);
92915f71
GR		 3219}
3220
4197aa7b
ED		 3221static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3222 struct rtnl_link_stats64 *stats)
3223{
3224 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3225 unsigned int start;
3226 u64 bytes, packets;
3227 const struct ixgbevf_ring *ring;
3228 int i;
3229
3230 ixgbevf_update_stats(adapter);
3231
3232 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3233
3234 for (i = 0; i < adapter->num_rx_queues; i++) {
3235 ring = &adapter->rx_ring[i];
3236 do {
3237 start = u64_stats_fetch_begin_bh(&ring->syncp);
3238 bytes = ring->total_bytes;
3239 packets = ring->total_packets;
3240 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3241 stats->rx_bytes += bytes;
3242 stats->rx_packets += packets;
3243 }
3244
3245 for (i = 0; i < adapter->num_tx_queues; i++) {
3246 ring = &adapter->tx_ring[i];
3247 do {
3248 start = u64_stats_fetch_begin_bh(&ring->syncp);
3249 bytes = ring->total_bytes;
3250 packets = ring->total_packets;
3251 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3252 stats->tx_bytes += bytes;
3253 stats->tx_packets += packets;
3254 }
3255
3256 return stats;
3257}
3258
0ac1e8ce 3259static const struct net_device_ops ixgbevf_netdev_ops = {
c12db769
SH		 3260 .ndo_open = ixgbevf_open,
3261 .ndo_stop = ixgbevf_close,
3262 .ndo_start_xmit = ixgbevf_xmit_frame,
3263 .ndo_set_rx_mode = ixgbevf_set_rx_mode,
4197aa7b 3264 .ndo_get_stats64 = ixgbevf_get_stats,
92915f71 3265 .ndo_validate_addr = eth_validate_addr,
c12db769
SH		 3266 .ndo_set_mac_address = ixgbevf_set_mac,
3267 .ndo_change_mtu = ixgbevf_change_mtu,
3268 .ndo_tx_timeout = ixgbevf_tx_timeout,
c12db769
SH		 3269 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid,
3270 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid,
92915f71 3271};
92915f71
GR		 3272
3273static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3274{
0ac1e8ce 3275 dev->netdev_ops = &ixgbevf_netdev_ops;
92915f71
GR		 3276 ixgbevf_set_ethtool_ops(dev);
3277 dev->watchdog_timeo = 5 * HZ;
3278}
3279
3280/**
3281 * ixgbevf_probe - Device Initialization Routine
3282 * @pdev: PCI device information struct
3283 * @ent: entry in ixgbevf_pci_tbl
3284 *
3285 * Returns 0 on success, negative on failure
3286 *
3287 * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3288 * The OS initialization, configuring of the adapter private structure,
3289 * and a hardware reset occur.
3290 **/
1dd06ae8 3291static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
92915f71
GR		 3292{
3293 struct net_device *netdev;
3294 struct ixgbevf_adapter *adapter = NULL;
3295 struct ixgbe_hw *hw = NULL;
3296 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3297 static int cards_found;
3298 int err, pci_using_dac;
3299
3300 err = pci_enable_device(pdev);
3301 if (err)
3302 return err;
3303
2a1f8794
NN		 3304 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
3305 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
92915f71
GR		 3306 pci_using_dac = 1;
3307 } else {
2a1f8794 3308 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
92915f71 3309 if (err) {
2a1f8794
NN		 3310 err = dma_set_coherent_mask(&pdev->dev,
3311 DMA_BIT_MASK(32));
92915f71
GR		 3312 if (err) {
3313 dev_err(&pdev->dev, "No usable DMA "
3314 "configuration, aborting\n");
3315 goto err_dma;
3316 }
3317 }
3318 pci_using_dac = 0;
3319 }
3320
3321 err = pci_request_regions(pdev, ixgbevf_driver_name);
3322 if (err) {
3323 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3324 goto err_pci_reg;
3325 }
3326
3327 pci_set_master(pdev);
3328
92915f71
GR		 3329 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3330 MAX_TX_QUEUES);
92915f71
GR		 3331 if (!netdev) {
3332 err = -ENOMEM;
3333 goto err_alloc_etherdev;
3334 }
3335
3336 SET_NETDEV_DEV(netdev, &pdev->dev);
3337
3338 pci_set_drvdata(pdev, netdev);
3339 adapter = netdev_priv(netdev);
3340
3341 adapter->netdev = netdev;
3342 adapter->pdev = pdev;
3343 hw = &adapter->hw;
3344 hw->back = adapter;
b3f4d599 3345 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
92915f71
GR		 3346
3347 /*
3348 * call save state here in standalone driver because it relies on
3349 * adapter struct to exist, and needs to call netdev_priv
3350 */
3351 pci_save_state(pdev);
3352
3353 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3354 pci_resource_len(pdev, 0));
3355 if (!hw->hw_addr) {
3356 err = -EIO;
3357 goto err_ioremap;
3358 }
3359
3360 ixgbevf_assign_netdev_ops(netdev);
3361
3362 adapter->bd_number = cards_found;
3363
3364 /* Setup hw api */
3365 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3366 hw->mac.type = ii->mac;
3367
3368 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
f416dfc0 3369 sizeof(struct ixgbe_mbx_operations));
92915f71 3370
92915f71
GR		 3371 /* setup the private structure */
3372 err = ixgbevf_sw_init(adapter);
1a0d6ae5
DK		 3373 if (err)
3374 goto err_sw_init;
3375
3376 /* The HW MAC address was set and/or determined in sw_init */
1a0d6ae5
DK		 3377 if (!is_valid_ether_addr(netdev->dev_addr)) {
3378 pr_err("invalid MAC address\n");
3379 err = -EIO;
3380 goto err_sw_init;
3381 }
92915f71 3382
471a76de 3383 netdev->hw_features = NETIF_F_SG |
92915f71 3384 NETIF_F_IP_CSUM |
471a76de
MM		 3385 NETIF_F_IPV6_CSUM |
3386 NETIF_F_TSO |
3387 NETIF_F_TSO6 |
3388 NETIF_F_RXCSUM;
3389
3390 netdev->features = netdev->hw_features |
92915f71
GR		 3391 NETIF_F_HW_VLAN_TX |
3392 NETIF_F_HW_VLAN_RX |
3393 NETIF_F_HW_VLAN_FILTER;
3394
92915f71
GR		 3395 netdev->vlan_features |= NETIF_F_TSO;
3396 netdev->vlan_features |= NETIF_F_TSO6;
3397 netdev->vlan_features |= NETIF_F_IP_CSUM;
3bfacf96 3398 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
92915f71
GR		 3399 netdev->vlan_features |= NETIF_F_SG;
3400
3401 if (pci_using_dac)
3402 netdev->features |= NETIF_F_HIGHDMA;
3403
01789349
JP		 3404 netdev->priv_flags |= IFF_UNICAST_FLT;
3405
92915f71 3406 init_timer(&adapter->watchdog_timer);
c061b18d 3407 adapter->watchdog_timer.function = ixgbevf_watchdog;
92915f71
GR		 3408 adapter->watchdog_timer.data = (unsigned long)adapter;
3409
3410 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3411 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3412
3413 err = ixgbevf_init_interrupt_scheme(adapter);
3414 if (err)
3415 goto err_sw_init;
3416
92915f71
GR		 3417 strcpy(netdev->name, "eth%d");
3418
3419 err = register_netdev(netdev);
3420 if (err)
3421 goto err_register;
3422
5d426ad1
GR		 3423 netif_carrier_off(netdev);
3424
33bd9f60
GR		 3425 ixgbevf_init_last_counter_stats(adapter);
3426
92915f71 3427 /* print the MAC address */
f794e7ef 3428 hw_dbg(hw, "%pM\n", netdev->dev_addr);
92915f71
GR		 3429
3430 hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3431
92915f71
GR		 3432 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3433 cards_found++;
3434 return 0;
3435
3436err_register:
0ac1e8ce 3437 ixgbevf_clear_interrupt_scheme(adapter);
92915f71
GR		 3438err_sw_init:
3439 ixgbevf_reset_interrupt_capability(adapter);
3440 iounmap(hw->hw_addr);
3441err_ioremap:
3442 free_netdev(netdev);
3443err_alloc_etherdev:
3444 pci_release_regions(pdev);
3445err_pci_reg:
3446err_dma:
3447 pci_disable_device(pdev);
3448 return err;
3449}
3450
3451/**
3452 * ixgbevf_remove - Device Removal Routine
3453 * @pdev: PCI device information struct
3454 *
3455 * ixgbevf_remove is called by the PCI subsystem to alert the driver
3456 * that it should release a PCI device. The could be caused by a
3457 * Hot-Plug event, or because the driver is going to be removed from
3458 * memory.
3459 **/
9f9a12f8 3460static void ixgbevf_remove(struct pci_dev *pdev)
92915f71
GR		 3461{
3462 struct net_device *netdev = pci_get_drvdata(pdev);
3463 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3464
3465 set_bit(__IXGBEVF_DOWN, &adapter->state);
3466
3467 del_timer_sync(&adapter->watchdog_timer);
3468
23f333a2 3469 cancel_work_sync(&adapter->reset_task);
92915f71
GR		 3470 cancel_work_sync(&adapter->watchdog_task);
3471
fd13a9ab 3472 if (netdev->reg_state == NETREG_REGISTERED)
92915f71 3473 unregister_netdev(netdev);
92915f71 3474
0ac1e8ce 3475 ixgbevf_clear_interrupt_scheme(adapter);
92915f71
GR		 3476 ixgbevf_reset_interrupt_capability(adapter);
3477
3478 iounmap(adapter->hw.hw_addr);
3479 pci_release_regions(pdev);
3480
3481 hw_dbg(&adapter->hw, "Remove complete\n");
3482
3483 kfree(adapter->tx_ring);
3484 kfree(adapter->rx_ring);
3485
3486 free_netdev(netdev);
3487
3488 pci_disable_device(pdev);
3489}
3490
9f19f31d
AD		 3491/**
3492 * ixgbevf_io_error_detected - called when PCI error is detected
3493 * @pdev: Pointer to PCI device
3494 * @state: The current pci connection state
3495 *
3496 * This function is called after a PCI bus error affecting
3497 * this device has been detected.
3498 */
3499static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
3500 pci_channel_state_t state)
3501{
3502 struct net_device *netdev = pci_get_drvdata(pdev);
3503 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3504
3505 netif_device_detach(netdev);
3506
3507 if (state == pci_channel_io_perm_failure)
3508 return PCI_ERS_RESULT_DISCONNECT;
3509
3510 if (netif_running(netdev))
3511 ixgbevf_down(adapter);
3512
3513 pci_disable_device(pdev);
3514
3515 /* Request a slot slot reset. */
3516 return PCI_ERS_RESULT_NEED_RESET;
3517}
3518
3519/**
3520 * ixgbevf_io_slot_reset - called after the pci bus has been reset.
3521 * @pdev: Pointer to PCI device
3522 *
3523 * Restart the card from scratch, as if from a cold-boot. Implementation
3524 * resembles the first-half of the ixgbevf_resume routine.
3525 */
3526static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
3527{
3528 struct net_device *netdev = pci_get_drvdata(pdev);
3529 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3530
3531 if (pci_enable_device_mem(pdev)) {
3532 dev_err(&pdev->dev,
3533 "Cannot re-enable PCI device after reset.\n");
3534 return PCI_ERS_RESULT_DISCONNECT;
3535 }
3536
3537 pci_set_master(pdev);
3538
3539 ixgbevf_reset(adapter);
3540
3541 return PCI_ERS_RESULT_RECOVERED;
3542}
3543
3544/**
3545 * ixgbevf_io_resume - called when traffic can start flowing again.
3546 * @pdev: Pointer to PCI device
3547 *
3548 * This callback is called when the error recovery driver tells us that
3549 * its OK to resume normal operation. Implementation resembles the
3550 * second-half of the ixgbevf_resume routine.
3551 */
3552static void ixgbevf_io_resume(struct pci_dev *pdev)
3553{
3554 struct net_device *netdev = pci_get_drvdata(pdev);
3555 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3556
3557 if (netif_running(netdev))
3558 ixgbevf_up(adapter);
3559
3560 netif_device_attach(netdev);
3561}
3562
3563/* PCI Error Recovery (ERS) */
3646f0e5 3564static const struct pci_error_handlers ixgbevf_err_handler = {
9f19f31d
AD		 3565 .error_detected = ixgbevf_io_error_detected,
3566 .slot_reset = ixgbevf_io_slot_reset,
3567 .resume = ixgbevf_io_resume,
3568};
3569
92915f71
GR		 3570static struct pci_driver ixgbevf_driver = {
3571 .name = ixgbevf_driver_name,
3572 .id_table = ixgbevf_pci_tbl,
3573 .probe = ixgbevf_probe,
9f9a12f8 3574 .remove = ixgbevf_remove,
0ac1e8ce
AD		 3575#ifdef CONFIG_PM
3576 /* Power Management Hooks */
3577 .suspend = ixgbevf_suspend,
3578 .resume = ixgbevf_resume,
3579#endif
92915f71 3580 .shutdown = ixgbevf_shutdown,
9f19f31d 3581 .err_handler = &ixgbevf_err_handler
92915f71
GR		 3582};
3583
3584/**
65d676c8 3585 * ixgbevf_init_module - Driver Registration Routine
92915f71 3586 *
65d676c8 3587 * ixgbevf_init_module is the first routine called when the driver is
92915f71
GR		 3588 * loaded. All it does is register with the PCI subsystem.
3589 **/
3590static int __init ixgbevf_init_module(void)
3591{
3592 int ret;
dbd9636e
JK		 3593 pr_info("%s - version %s\n", ixgbevf_driver_string,
3594 ixgbevf_driver_version);
92915f71 3595
dbd9636e 3596 pr_info("%s\n", ixgbevf_copyright);
92915f71
GR		 3597
3598 ret = pci_register_driver(&ixgbevf_driver);
3599 return ret;
3600}
3601
3602module_init(ixgbevf_init_module);
3603
3604/**
65d676c8 3605 * ixgbevf_exit_module - Driver Exit Cleanup Routine
92915f71 3606 *
65d676c8 3607 * ixgbevf_exit_module is called just before the driver is removed
92915f71
GR		 3608 * from memory.
3609 **/
3610static void __exit ixgbevf_exit_module(void)
3611{
3612 pci_unregister_driver(&ixgbevf_driver);
3613}
3614
3615#ifdef DEBUG
3616/**
65d676c8 3617 * ixgbevf_get_hw_dev_name - return device name string
92915f71
GR		 3618 * used by hardware layer to print debugging information
3619 **/
3620char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3621{
3622 struct ixgbevf_adapter *adapter = hw->back;
3623 return adapter->netdev->name;
3624}
3625
3626#endif
3627module_exit(ixgbevf_exit_module);
3628
3629/* ixgbevf_main.c */
Linux kernel - Deliverables
This page took 0.625631 seconds and 5 git commands to generate.