projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: vlan: rename NETIF_F_HW_VLAN_* feature flags to NETIF_F_HW_VLAN_CTAG_*
[deliverable/linux.git] / drivers / net / ethernet / intel / ixgbevf / ixgbevf_main.c
CommitLineData
92915f71
GR		 1/*******************************************************************************
2
3 Intel 82599 Virtual Function driver
5c47a2b6 4 Copyright(c) 1999 - 2012 Intel Corporation.
92915f71
GR		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28
29/******************************************************************************
30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
31******************************************************************************/
dbd9636e
JK		 32
33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34
92915f71 35#include <linux/types.h>
dadcd65f 36#include <linux/bitops.h>
92915f71
GR		 37#include <linux/module.h>
38#include <linux/pci.h>
39#include <linux/netdevice.h>
40#include <linux/vmalloc.h>
41#include <linux/string.h>
42#include <linux/in.h>
43#include <linux/ip.h>
44#include <linux/tcp.h>
70a10e25 45#include <linux/sctp.h>
92915f71 46#include <linux/ipv6.h>
5a0e3ad6 47#include <linux/slab.h>
92915f71
GR		 48#include <net/checksum.h>
49#include <net/ip6_checksum.h>
50#include <linux/ethtool.h>
01789349 51#include <linux/if.h>
92915f71 52#include <linux/if_vlan.h>
70c71606 53#include <linux/prefetch.h>
92915f71
GR		 54
55#include "ixgbevf.h"
56
3d8fe98f 57const char ixgbevf_driver_name[] = "ixgbevf";
92915f71 58static const char ixgbevf_driver_string[] =
422e05d1 59 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
92915f71 60
1b3d2d77 61#define DRV_VERSION "2.7.12-k"
92915f71 62const char ixgbevf_driver_version[] = DRV_VERSION;
66c87bd5 63static char ixgbevf_copyright[] =
5c47a2b6 64 "Copyright (c) 2009 - 2012 Intel Corporation.";
92915f71
GR		 65
66static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
2316aa2a
GR		 67 [board_82599_vf] = &ixgbevf_82599_vf_info,
68 [board_X540_vf] = &ixgbevf_X540_vf_info,
92915f71
GR		 69};
70
71/* ixgbevf_pci_tbl - PCI Device ID Table
72 *
73 * Wildcard entries (PCI_ANY_ID) should come last
74 * Last entry must be all 0s
75 *
76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
77 * Class, Class Mask, private data (not used) }
78 */
39ba22b4
SH		 79static DEFINE_PCI_DEVICE_TABLE(ixgbevf_pci_tbl) = {
80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
81 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
92915f71
GR		 82 /* required last entry */
83 {0, }
84};
85MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
86
87MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
88MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
89MODULE_LICENSE("GPL");
90MODULE_VERSION(DRV_VERSION);
91
b3f4d599 92#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
93static int debug = -1;
94module_param(debug, int, 0);
95MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
92915f71
GR		 96
97/* forward decls */
fa71ae27 98static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
56e94095 99static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
92915f71
GR		 100
101static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw,
102 struct ixgbevf_ring *rx_ring,
103 u32 val)
104{
105 /*
106 * Force memory writes to complete before letting h/w
107 * know there are new descriptors to fetch. (Only
108 * applicable for weak-ordered memory model archs,
109 * such as IA-64).
110 */
111 wmb();
112 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val);
113}
114
49ce9c2c 115/**
65d676c8 116 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
92915f71
GR		 117 * @adapter: pointer to adapter struct
118 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
119 * @queue: queue to map the corresponding interrupt to
120 * @msix_vector: the vector to map to the corresponding queue
92915f71
GR		 121 */
122static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
123 u8 queue, u8 msix_vector)
124{
125 u32 ivar, index;
126 struct ixgbe_hw *hw = &adapter->hw;
127 if (direction == -1) {
128 /* other causes */
129 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
130 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
131 ivar &= ~0xFF;
132 ivar |= msix_vector;
133 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
134 } else {
135 /* tx or rx causes */
136 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
137 index = ((16 * (queue & 1)) + (8 * direction));
138 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
139 ivar &= ~(0xFF << index);
140 ivar |= (msix_vector << index);
141 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
142 }
143}
144
70a10e25 145static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
92915f71
GR		 146 struct ixgbevf_tx_buffer
147 *tx_buffer_info)
148{
149 if (tx_buffer_info->dma) {
150 if (tx_buffer_info->mapped_as_page)
70a10e25 151 dma_unmap_page(tx_ring->dev,
92915f71
GR		 152 tx_buffer_info->dma,
153 tx_buffer_info->length,
2a1f8794 154 DMA_TO_DEVICE);
92915f71 155 else
70a10e25 156 dma_unmap_single(tx_ring->dev,
92915f71
GR		 157 tx_buffer_info->dma,
158 tx_buffer_info->length,
2a1f8794 159 DMA_TO_DEVICE);
92915f71
GR		 160 tx_buffer_info->dma = 0;
161 }
162 if (tx_buffer_info->skb) {
163 dev_kfree_skb_any(tx_buffer_info->skb);
164 tx_buffer_info->skb = NULL;
165 }
166 tx_buffer_info->time_stamp = 0;
167 /* tx_buffer_info must be completely set up in the transmit path */
168}
169
92915f71
GR		 170#define IXGBE_MAX_TXD_PWR 14
171#define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR)
172
173/* Tx Descriptors needed, worst case */
3595990a
AD		 174#define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
175#define DESC_NEEDED (MAX_SKB_FRAGS + 4)
92915f71
GR		 176
177static void ixgbevf_tx_timeout(struct net_device *netdev);
178
179/**
180 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
fa71ae27 181 * @q_vector: board private structure
92915f71
GR		 182 * @tx_ring: tx ring to clean
183 **/
fa71ae27 184static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
92915f71
GR		 185 struct ixgbevf_ring *tx_ring)
186{
fa71ae27 187 struct ixgbevf_adapter *adapter = q_vector->adapter;
92915f71
GR		 188 union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
189 struct ixgbevf_tx_buffer *tx_buffer_info;
e757e3e1 190 unsigned int i, count = 0;
92915f71
GR		 191 unsigned int total_bytes = 0, total_packets = 0;
192
10cc1bdd
AD		 193 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
194 return true;
195
92915f71 196 i = tx_ring->next_to_clean;
e757e3e1
AD		 197 tx_buffer_info = &tx_ring->tx_buffer_info[i];
198 eop_desc = tx_buffer_info->next_to_watch;
92915f71 199
e757e3e1 200 do {
92915f71 201 bool cleaned = false;
e757e3e1
AD		 202
203 /* if next_to_watch is not set then there is no work pending */
204 if (!eop_desc)
205 break;
206
207 /* prevent any other reads prior to eop_desc */
208 read_barrier_depends();
209
210 /* if DD is not set pending work has not been completed */
211 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
212 break;
213
214 /* clear next_to_watch to prevent false hangs */
215 tx_buffer_info->next_to_watch = NULL;
216
92915f71
GR		 217 for ( ; !cleaned; count++) {
218 struct sk_buff *skb;
908421f6 219 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
e757e3e1 220 cleaned = (tx_desc == eop_desc);
92915f71
GR		 221 skb = tx_buffer_info->skb;
222
223 if (cleaned && skb) {
224 unsigned int segs, bytecount;
225
226 /* gso_segs is currently only valid for tcp */
227 segs = skb_shinfo(skb)->gso_segs ?: 1;
228 /* multiply data chunks by size of headers */
229 bytecount = ((segs - 1) * skb_headlen(skb)) +
230 skb->len;
231 total_packets += segs;
232 total_bytes += bytecount;
233 }
234
70a10e25 235 ixgbevf_unmap_and_free_tx_resource(tx_ring,
92915f71
GR		 236 tx_buffer_info);
237
238 tx_desc->wb.status = 0;
239
240 i++;
241 if (i == tx_ring->count)
242 i = 0;
e757e3e1
AD		 243
244 tx_buffer_info = &tx_ring->tx_buffer_info[i];
92915f71
GR		 245 }
246
e757e3e1
AD		 247 eop_desc = tx_buffer_info->next_to_watch;
248 } while (count < tx_ring->count);
92915f71
GR		 249
250 tx_ring->next_to_clean = i;
251
252#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
fb40195c 253 if (unlikely(count && netif_carrier_ok(tx_ring->netdev) &&
92915f71
GR		 254 (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
255 /* Make sure that anybody stopping the queue after this
256 * sees the new next_to_clean.
257 */
258 smp_mb();
fb40195c
AD		 259 if (__netif_subqueue_stopped(tx_ring->netdev,
260 tx_ring->queue_index) &&
92915f71 261 !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
fb40195c
AD		 262 netif_wake_subqueue(tx_ring->netdev,
263 tx_ring->queue_index);
92915f71
GR		 264 ++adapter->restart_queue;
265 }
92915f71
GR		 266 }
267
4197aa7b 268 u64_stats_update_begin(&tx_ring->syncp);
92915f71
GR		 269 tx_ring->total_bytes += total_bytes;
270 tx_ring->total_packets += total_packets;
4197aa7b 271 u64_stats_update_end(&tx_ring->syncp);
ac6ed8f0
GR		 272 q_vector->tx.total_bytes += total_bytes;
273 q_vector->tx.total_packets += total_packets;
92915f71 274
fa71ae27 275 return count < tx_ring->count;
92915f71
GR		 276}
277
278/**
279 * ixgbevf_receive_skb - Send a completed packet up the stack
280 * @q_vector: structure containing interrupt and ring information
281 * @skb: packet to send up
282 * @status: hardware indication of status of receive
92915f71
GR		 283 * @rx_desc: rx descriptor
284 **/
285static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
286 struct sk_buff *skb, u8 status,
92915f71
GR		 287 union ixgbe_adv_rx_desc *rx_desc)
288{
289 struct ixgbevf_adapter *adapter = q_vector->adapter;
290 bool is_vlan = (status & IXGBE_RXD_STAT_VP);
dd1ed3b7 291 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
92915f71 292
5d9a533b 293 if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans))
dadcd65f 294 __vlan_hwaccel_put_tag(skb, tag);
dadcd65f 295
366c1099
GR		 296 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
297 napi_gro_receive(&q_vector->napi, skb);
298 else
299 netif_rx(skb);
92915f71
GR		 300}
301
302/**
303 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
55fb277c 304 * @ring: pointer to Rx descriptor ring structure
92915f71
GR		 305 * @status_err: hardware indication of status of receive
306 * @skb: skb currently being received and modified
307 **/
55fb277c 308static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
92915f71
GR		 309 u32 status_err, struct sk_buff *skb)
310{
bc8acf2c 311 skb_checksum_none_assert(skb);
92915f71
GR		 312
313 /* Rx csum disabled */
fb40195c 314 if (!(ring->netdev->features & NETIF_F_RXCSUM))
92915f71
GR		 315 return;
316
317 /* if IP and error */
318 if ((status_err & IXGBE_RXD_STAT_IPCS) &&
319 (status_err & IXGBE_RXDADV_ERR_IPE)) {
55fb277c 320 ring->hw_csum_rx_error++;
92915f71
GR		 321 return;
322 }
323
324 if (!(status_err & IXGBE_RXD_STAT_L4CS))
325 return;
326
327 if (status_err & IXGBE_RXDADV_ERR_TCPE) {
55fb277c 328 ring->hw_csum_rx_error++;
92915f71
GR		 329 return;
330 }
331
332 /* It must be a TCP or UDP packet with a valid checksum */
333 skb->ip_summed = CHECKSUM_UNNECESSARY;
55fb277c 334 ring->hw_csum_rx_good++;
92915f71
GR		 335}
336
337/**
338 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
339 * @adapter: address of board private structure
340 **/
341static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter,
342 struct ixgbevf_ring *rx_ring,
343 int cleaned_count)
344{
345 struct pci_dev *pdev = adapter->pdev;
346 union ixgbe_adv_rx_desc *rx_desc;
347 struct ixgbevf_rx_buffer *bi;
fb40195c 348 unsigned int i = rx_ring->next_to_use;
92915f71 349
92915f71
GR		 350 bi = &rx_ring->rx_buffer_info[i];
351
352 while (cleaned_count--) {
908421f6 353 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
b9dd245b
GR		 354
355 if (!bi->skb) {
356 struct sk_buff *skb;
357
fb40195c
AD		 358 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
359 rx_ring->rx_buf_len);
92915f71
GR		 360 if (!skb) {
361 adapter->alloc_rx_buff_failed++;
362 goto no_buffers;
363 }
92915f71 364 bi->skb = skb;
b9dd245b 365
2a1f8794 366 bi->dma = dma_map_single(&pdev->dev, skb->data,
92915f71 367 rx_ring->rx_buf_len,
2a1f8794 368 DMA_FROM_DEVICE);
6132ee8a
GR		 369 if (dma_mapping_error(&pdev->dev, bi->dma)) {
370 dev_kfree_skb(skb);
371 bi->skb = NULL;
372 dev_err(&pdev->dev, "RX DMA map failed\n");
373 break;
374 }
92915f71 375 }
77d5dfca 376 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
92915f71
GR		 377
378 i++;
379 if (i == rx_ring->count)
380 i = 0;
381 bi = &rx_ring->rx_buffer_info[i];
382 }
383
384no_buffers:
385 if (rx_ring->next_to_use != i) {
386 rx_ring->next_to_use = i;
92915f71
GR		 387 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i);
388 }
389}
390
391static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
5f3600eb 392 u32 qmask)
92915f71 393{
92915f71
GR		 394 struct ixgbe_hw *hw = &adapter->hw;
395
5f3600eb 396 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
92915f71
GR		 397}
398
92915f71
GR		 399static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
400 struct ixgbevf_ring *rx_ring,
fa71ae27 401 int budget)
92915f71
GR		 402{
403 struct ixgbevf_adapter *adapter = q_vector->adapter;
404 struct pci_dev *pdev = adapter->pdev;
405 union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
406 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
407 struct sk_buff *skb;
408 unsigned int i;
409 u32 len, staterr;
92915f71
GR		 410 int cleaned_count = 0;
411 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
412
413 i = rx_ring->next_to_clean;
908421f6 414 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
92915f71
GR		 415 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
416 rx_buffer_info = &rx_ring->rx_buffer_info[i];
417
418 while (staterr & IXGBE_RXD_STAT_DD) {
fa71ae27 419 if (!budget)
92915f71 420 break;
fa71ae27 421 budget--;
92915f71 422
2d0bb1c1 423 rmb(); /* read descriptor and rx_buffer_info after status DD */
77d5dfca 424 len = le16_to_cpu(rx_desc->wb.upper.length);
92915f71
GR		 425 skb = rx_buffer_info->skb;
426 prefetch(skb->data - NET_IP_ALIGN);
427 rx_buffer_info->skb = NULL;
428
429 if (rx_buffer_info->dma) {
2a1f8794 430 dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
92915f71 431 rx_ring->rx_buf_len,
2a1f8794 432 DMA_FROM_DEVICE);
92915f71
GR		 433 rx_buffer_info->dma = 0;
434 skb_put(skb, len);
435 }
436
92915f71
GR		 437 i++;
438 if (i == rx_ring->count)
439 i = 0;
440
908421f6 441 next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
92915f71
GR		 442 prefetch(next_rxd);
443 cleaned_count++;
444
445 next_buffer = &rx_ring->rx_buffer_info[i];
446
447 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
77d5dfca 448 skb->next = next_buffer->skb;
5c60f81a 449 IXGBE_CB(skb->next)->prev = skb;
92915f71
GR		 450 adapter->non_eop_descs++;
451 goto next_desc;
452 }
453
5c60f81a
AD		 454 /* we should not be chaining buffers, if we did drop the skb */
455 if (IXGBE_CB(skb)->prev) {
456 do {
457 struct sk_buff *this = skb;
458 skb = IXGBE_CB(skb)->prev;
459 dev_kfree_skb(this);
460 } while (skb);
461 goto next_desc;
462 }
463
92915f71
GR		 464 /* ERR_MASK will only have valid bits if EOP set */
465 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
466 dev_kfree_skb_irq(skb);
467 goto next_desc;
468 }
469
55fb277c 470 ixgbevf_rx_checksum(rx_ring, staterr, skb);
92915f71
GR		 471
472 /* probably a little skewed due to removing CRC */
473 total_rx_bytes += skb->len;
474 total_rx_packets++;
475
476 /*
477 * Work around issue of some types of VM to VM loop back
478 * packets not getting split correctly
479 */
480 if (staterr & IXGBE_RXD_STAT_LB) {
e743d313 481 u32 header_fixup_len = skb_headlen(skb);
92915f71
GR		 482 if (header_fixup_len < 14)
483 skb_push(skb, header_fixup_len);
484 }
fb40195c 485 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
92915f71 486
815cccbf
JF		 487 /* Workaround hardware that can't do proper VEPA multicast
488 * source pruning.
489 */
490 if ((skb->pkt_type & (PACKET_BROADCAST | PACKET_MULTICAST)) &&
491 !(compare_ether_addr(adapter->netdev->dev_addr,
492 eth_hdr(skb)->h_source))) {
493 dev_kfree_skb_irq(skb);
494 goto next_desc;
495 }
496
b3d58a8f 497 ixgbevf_receive_skb(q_vector, skb, staterr, rx_desc);
92915f71
GR		 498
499next_desc:
500 rx_desc->wb.upper.status_error = 0;
501
502 /* return some buffers to hardware, one at a time is too slow */
503 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
504 ixgbevf_alloc_rx_buffers(adapter, rx_ring,
505 cleaned_count);
506 cleaned_count = 0;
507 }
508
509 /* use prefetched values */
510 rx_desc = next_rxd;
511 rx_buffer_info = &rx_ring->rx_buffer_info[i];
512
513 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
514 }
515
516 rx_ring->next_to_clean = i;
517 cleaned_count = IXGBE_DESC_UNUSED(rx_ring);
518
519 if (cleaned_count)
520 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count);
521
4197aa7b 522 u64_stats_update_begin(&rx_ring->syncp);
92915f71
GR		 523 rx_ring->total_packets += total_rx_packets;
524 rx_ring->total_bytes += total_rx_bytes;
4197aa7b 525 u64_stats_update_end(&rx_ring->syncp);
ac6ed8f0
GR		 526 q_vector->rx.total_packets += total_rx_packets;
527 q_vector->rx.total_bytes += total_rx_bytes;
92915f71 528
fa71ae27 529 return !!budget;
92915f71
GR		 530}
531
532/**
fa71ae27 533 * ixgbevf_poll - NAPI polling calback
92915f71
GR		 534 * @napi: napi struct with our devices info in it
535 * @budget: amount of work driver is allowed to do this pass, in packets
536 *
fa71ae27 537 * This function will clean more than one or more rings associated with a
92915f71
GR		 538 * q_vector.
539 **/
fa71ae27 540static int ixgbevf_poll(struct napi_struct *napi, int budget)
92915f71
GR		 541{
542 struct ixgbevf_q_vector *q_vector =
543 container_of(napi, struct ixgbevf_q_vector, napi);
544 struct ixgbevf_adapter *adapter = q_vector->adapter;
fa71ae27
AD		 545 struct ixgbevf_ring *ring;
546 int per_ring_budget;
547 bool clean_complete = true;
548
549 ixgbevf_for_each_ring(ring, q_vector->tx)
550 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
92915f71
GR		 551
552 /* attempt to distribute budget to each queue fairly, but don't allow
553 * the budget to go below 1 because we'll exit polling */
fa71ae27
AD		 554 if (q_vector->rx.count > 1)
555 per_ring_budget = max(budget/q_vector->rx.count, 1);
556 else
557 per_ring_budget = budget;
558
366c1099 559 adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
fa71ae27
AD		 560 ixgbevf_for_each_ring(ring, q_vector->rx)
561 clean_complete &= ixgbevf_clean_rx_irq(q_vector, ring,
562 per_ring_budget);
366c1099 563 adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
fa71ae27
AD		 564
565 /* If all work not completed, return budget and keep polling */
566 if (!clean_complete)
567 return budget;
568 /* all work done, exit the polling mode */
569 napi_complete(napi);
570 if (adapter->rx_itr_setting & 1)
571 ixgbevf_set_itr(q_vector);
572 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
573 ixgbevf_irq_enable_queues(adapter,
574 1 << q_vector->v_idx);
92915f71 575
fa71ae27 576 return 0;
92915f71
GR		 577}
578
ce422606
GR		 579/**
580 * ixgbevf_write_eitr - write VTEITR register in hardware specific way
581 * @q_vector: structure containing interrupt and ring information
582 */
583static void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
584{
585 struct ixgbevf_adapter *adapter = q_vector->adapter;
586 struct ixgbe_hw *hw = &adapter->hw;
587 int v_idx = q_vector->v_idx;
588 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
589
590 /*
591 * set the WDIS bit to not clear the timer bits and cause an
592 * immediate assertion of the interrupt
593 */
594 itr_reg |= IXGBE_EITR_CNT_WDIS;
595
596 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
597}
92915f71
GR		 598
599/**
600 * ixgbevf_configure_msix - Configure MSI-X hardware
601 * @adapter: board private structure
602 *
603 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
604 * interrupts.
605 **/
606static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
607{
608 struct ixgbevf_q_vector *q_vector;
6b43c446 609 int q_vectors, v_idx;
92915f71
GR		 610
611 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
5f3600eb 612 adapter->eims_enable_mask = 0;
92915f71
GR		 613
614 /*
615 * Populate the IVAR table and set the ITR values to the
616 * corresponding register.
617 */
618 for (v_idx = 0; v_idx < q_vectors; v_idx++) {
6b43c446 619 struct ixgbevf_ring *ring;
92915f71 620 q_vector = adapter->q_vector[v_idx];
6b43c446
AD		 621
622 ixgbevf_for_each_ring(ring, q_vector->rx)
623 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
624
625 ixgbevf_for_each_ring(ring, q_vector->tx)
626 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
92915f71 627
5f3600eb
AD		 628 if (q_vector->tx.ring && !q_vector->rx.ring) {
629 /* tx only vector */
630 if (adapter->tx_itr_setting == 1)
631 q_vector->itr = IXGBE_10K_ITR;
632 else
633 q_vector->itr = adapter->tx_itr_setting;
634 } else {
635 /* rx or rx/tx vector */
636 if (adapter->rx_itr_setting == 1)
637 q_vector->itr = IXGBE_20K_ITR;
638 else
639 q_vector->itr = adapter->rx_itr_setting;
640 }
641
642 /* add q_vector eims value to global eims_enable_mask */
643 adapter->eims_enable_mask |= 1 << v_idx;
92915f71 644
5f3600eb 645 ixgbevf_write_eitr(q_vector);
92915f71
GR		 646 }
647
648 ixgbevf_set_ivar(adapter, -1, 1, v_idx);
5f3600eb
AD		 649 /* setup eims_other and add value to global eims_enable_mask */
650 adapter->eims_other = 1 << v_idx;
651 adapter->eims_enable_mask |= adapter->eims_other;
92915f71
GR		 652}
653
654enum latency_range {
655 lowest_latency = 0,
656 low_latency = 1,
657 bulk_latency = 2,
658 latency_invalid = 255
659};
660
661/**
662 * ixgbevf_update_itr - update the dynamic ITR value based on statistics
5f3600eb
AD		 663 * @q_vector: structure containing interrupt and ring information
664 * @ring_container: structure containing ring performance data
92915f71
GR		 665 *
666 * Stores a new ITR value based on packets and byte
667 * counts during the last interrupt. The advantage of per interrupt
668 * computation is faster updates and more accurate ITR for the current
669 * traffic pattern. Constants in this function were computed
670 * based on theoretical maximum wire speed and thresholds were set based
671 * on testing data as well as attempting to minimize response time
672 * while increasing bulk throughput.
673 **/
5f3600eb
AD		 674static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
675 struct ixgbevf_ring_container *ring_container)
92915f71 676{
5f3600eb
AD		 677 int bytes = ring_container->total_bytes;
678 int packets = ring_container->total_packets;
92915f71
GR		 679 u32 timepassed_us;
680 u64 bytes_perint;
5f3600eb 681 u8 itr_setting = ring_container->itr;
92915f71
GR		 682
683 if (packets == 0)
5f3600eb 684 return;
92915f71
GR		 685
686 /* simple throttlerate management
687 * 0-20MB/s lowest (100000 ints/s)
688 * 20-100MB/s low (20000 ints/s)
689 * 100-1249MB/s bulk (8000 ints/s)
690 */
691 /* what was last interrupt timeslice? */
5f3600eb 692 timepassed_us = q_vector->itr >> 2;
92915f71
GR		 693 bytes_perint = bytes / timepassed_us; /* bytes/usec */
694
695 switch (itr_setting) {
696 case lowest_latency:
e2c28ce7 697 if (bytes_perint > 10)
5f3600eb 698 itr_setting = low_latency;
92915f71
GR		 699 break;
700 case low_latency:
e2c28ce7 701 if (bytes_perint > 20)
5f3600eb 702 itr_setting = bulk_latency;
e2c28ce7 703 else if (bytes_perint <= 10)
5f3600eb 704 itr_setting = lowest_latency;
92915f71
GR		 705 break;
706 case bulk_latency:
e2c28ce7 707 if (bytes_perint <= 20)
5f3600eb 708 itr_setting = low_latency;
92915f71
GR		 709 break;
710 }
711
5f3600eb
AD		 712 /* clear work counters since we have the values we need */
713 ring_container->total_bytes = 0;
714 ring_container->total_packets = 0;
715
716 /* write updated itr to ring container */
717 ring_container->itr = itr_setting;
92915f71
GR		 718}
719
fa71ae27 720static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
92915f71 721{
5f3600eb
AD		 722 u32 new_itr = q_vector->itr;
723 u8 current_itr;
92915f71 724
5f3600eb
AD		 725 ixgbevf_update_itr(q_vector, &q_vector->tx);
726 ixgbevf_update_itr(q_vector, &q_vector->rx);
92915f71 727
6b43c446 728 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
92915f71
GR		 729
730 switch (current_itr) {
731 /* counts and packets in update_itr are dependent on these numbers */
732 case lowest_latency:
5f3600eb 733 new_itr = IXGBE_100K_ITR;
92915f71
GR		 734 break;
735 case low_latency:
5f3600eb 736 new_itr = IXGBE_20K_ITR;
92915f71
GR		 737 break;
738 case bulk_latency:
739 default:
5f3600eb 740 new_itr = IXGBE_8K_ITR;
92915f71
GR		 741 break;
742 }
743
5f3600eb 744 if (new_itr != q_vector->itr) {
92915f71 745 /* do an exponential smoothing */
5f3600eb
AD		 746 new_itr = (10 * new_itr * q_vector->itr) /
747 ((9 * new_itr) + q_vector->itr);
748
749 /* save the algorithm value here */
750 q_vector->itr = new_itr;
751
752 ixgbevf_write_eitr(q_vector);
92915f71 753 }
92915f71
GR		 754}
755
4b2cd27f 756static irqreturn_t ixgbevf_msix_other(int irq, void *data)
92915f71 757{
fa71ae27 758 struct ixgbevf_adapter *adapter = data;
1e72bfc3 759 struct pci_dev *pdev = adapter->pdev;
92915f71 760 struct ixgbe_hw *hw = &adapter->hw;
1e72bfc3
GR		 761 u32 msg;
762 bool got_ack = false;
08259594 763
4b2cd27f 764 hw->mac.get_link_status = 1;
1e72bfc3
GR		 765 if (!hw->mbx.ops.check_for_ack(hw))
766 got_ack = true;
a9ee25a2 767
1e72bfc3
GR		 768 if (!hw->mbx.ops.check_for_msg(hw)) {
769 hw->mbx.ops.read(hw, &msg, 1);
770
088245a3 771 if ((msg & IXGBE_MBVFICR_VFREQ_MASK) == IXGBE_PF_CONTROL_MSG) {
1e72bfc3
GR		 772 mod_timer(&adapter->watchdog_timer,
773 round_jiffies(jiffies + 1));
088245a3
GR		 774 adapter->link_up = false;
775 }
1e72bfc3
GR		 776
777 if (msg & IXGBE_VT_MSGTYPE_NACK)
778 dev_info(&pdev->dev,
779 "Last Request of type %2.2x to PF Nacked\n",
780 msg & 0xFF);
781 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFSTS;
782 }
783
784 /* checking for the ack clears the PFACK bit. Place
785 * it back in the v2p_mailbox cache so that anyone
786 * polling for an ack will not miss it
787 */
788 if (got_ack)
789 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFACK;
3a2c4033 790
5f3600eb
AD		 791 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
792
92915f71
GR		 793 return IRQ_HANDLED;
794}
795
92915f71 796/**
fa71ae27 797 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
92915f71
GR		 798 * @irq: unused
799 * @data: pointer to our q_vector struct for this interrupt vector
800 **/
fa71ae27 801static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
92915f71
GR		 802{
803 struct ixgbevf_q_vector *q_vector = data;
92915f71 804
5f3600eb 805 /* EIAM disabled interrupts (on this vector) for us */
fa71ae27
AD		 806 if (q_vector->rx.ring || q_vector->tx.ring)
807 napi_schedule(&q_vector->napi);
92915f71
GR		 808
809 return IRQ_HANDLED;
810}
811
812static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
813 int r_idx)
814{
815 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
816
6b43c446
AD		 817 a->rx_ring[r_idx].next = q_vector->rx.ring;
818 q_vector->rx.ring = &a->rx_ring[r_idx];
819 q_vector->rx.count++;
92915f71
GR		 820}
821
822static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
823 int t_idx)
824{
825 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
826
6b43c446
AD		 827 a->tx_ring[t_idx].next = q_vector->tx.ring;
828 q_vector->tx.ring = &a->tx_ring[t_idx];
829 q_vector->tx.count++;
92915f71
GR		 830}
831
832/**
833 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
834 * @adapter: board private structure to initialize
835 *
836 * This function maps descriptor rings to the queue-specific vectors
837 * we were allotted through the MSI-X enabling code. Ideally, we'd have
838 * one vector per ring/queue, but on a constrained vector budget, we
839 * group the rings as "efficiently" as possible. You would add new
840 * mapping configurations in here.
841 **/
842static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
843{
844 int q_vectors;
845 int v_start = 0;
846 int rxr_idx = 0, txr_idx = 0;
847 int rxr_remaining = adapter->num_rx_queues;
848 int txr_remaining = adapter->num_tx_queues;
849 int i, j;
850 int rqpv, tqpv;
851 int err = 0;
852
853 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
854
855 /*
856 * The ideal configuration...
857 * We have enough vectors to map one per queue.
858 */
859 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
860 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
861 map_vector_to_rxq(adapter, v_start, rxr_idx);
862
863 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
864 map_vector_to_txq(adapter, v_start, txr_idx);
865 goto out;
866 }
867
868 /*
869 * If we don't have enough vectors for a 1-to-1
870 * mapping, we'll have to group them so there are
871 * multiple queues per vector.
872 */
873 /* Re-adjusting *qpv takes care of the remainder. */
874 for (i = v_start; i < q_vectors; i++) {
875 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
876 for (j = 0; j < rqpv; j++) {
877 map_vector_to_rxq(adapter, i, rxr_idx);
878 rxr_idx++;
879 rxr_remaining--;
880 }
881 }
882 for (i = v_start; i < q_vectors; i++) {
883 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
884 for (j = 0; j < tqpv; j++) {
885 map_vector_to_txq(adapter, i, txr_idx);
886 txr_idx++;
887 txr_remaining--;
888 }
889 }
890
891out:
892 return err;
893}
894
895/**
896 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
897 * @adapter: board private structure
898 *
899 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
900 * interrupts from the kernel.
901 **/
902static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
903{
904 struct net_device *netdev = adapter->netdev;
fa71ae27
AD		 905 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
906 int vector, err;
92915f71
GR		 907 int ri = 0, ti = 0;
908
92915f71 909 for (vector = 0; vector < q_vectors; vector++) {
fa71ae27
AD		 910 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
911 struct msix_entry *entry = &adapter->msix_entries[vector];
912
913 if (q_vector->tx.ring && q_vector->rx.ring) {
914 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
915 "%s-%s-%d", netdev->name, "TxRx", ri++);
916 ti++;
917 } else if (q_vector->rx.ring) {
918 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
919 "%s-%s-%d", netdev->name, "rx", ri++);
920 } else if (q_vector->tx.ring) {
921 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
922 "%s-%s-%d", netdev->name, "tx", ti++);
92915f71
GR		 923 } else {
924 /* skip this unused q_vector */
925 continue;
926 }
fa71ae27
AD		 927 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
928 q_vector->name, q_vector);
92915f71
GR		 929 if (err) {
930 hw_dbg(&adapter->hw,
931 "request_irq failed for MSIX interrupt "
932 "Error: %d\n", err);
933 goto free_queue_irqs;
934 }
935 }
936
92915f71 937 err = request_irq(adapter->msix_entries[vector].vector,
4b2cd27f 938 &ixgbevf_msix_other, 0, netdev->name, adapter);
92915f71
GR		 939 if (err) {
940 hw_dbg(&adapter->hw,
4b2cd27f 941 "request_irq for msix_other failed: %d\n", err);
92915f71
GR		 942 goto free_queue_irqs;
943 }
944
945 return 0;
946
947free_queue_irqs:
fa71ae27
AD		 948 while (vector) {
949 vector--;
950 free_irq(adapter->msix_entries[vector].vector,
951 adapter->q_vector[vector]);
952 }
a1f6c6b1 953 /* This failure is non-recoverable - it indicates the system is
954 * out of MSIX vector resources and the VF driver cannot run
955 * without them. Set the number of msix vectors to zero
956 * indicating that not enough can be allocated. The error
957 * will be returned to the user indicating device open failed.
958 * Any further attempts to force the driver to open will also
959 * fail. The only way to recover is to unload the driver and
960 * reload it again. If the system has recovered some MSIX
961 * vectors then it may succeed.
962 */
963 adapter->num_msix_vectors = 0;
92915f71
GR		 964 return err;
965}
966
967static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
968{
969 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
970
971 for (i = 0; i < q_vectors; i++) {
972 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
6b43c446
AD		 973 q_vector->rx.ring = NULL;
974 q_vector->tx.ring = NULL;
975 q_vector->rx.count = 0;
976 q_vector->tx.count = 0;
92915f71
GR		 977 }
978}
979
980/**
981 * ixgbevf_request_irq - initialize interrupts
982 * @adapter: board private structure
983 *
984 * Attempts to configure interrupts using the best available
985 * capabilities of the hardware and kernel.
986 **/
987static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
988{
989 int err = 0;
990
991 err = ixgbevf_request_msix_irqs(adapter);
992
993 if (err)
994 hw_dbg(&adapter->hw,
995 "request_irq failed, Error %d\n", err);
996
997 return err;
998}
999
1000static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1001{
92915f71
GR		 1002 int i, q_vectors;
1003
1004 q_vectors = adapter->num_msix_vectors;
92915f71
GR		 1005 i = q_vectors - 1;
1006
fa71ae27 1007 free_irq(adapter->msix_entries[i].vector, adapter);
92915f71
GR		 1008 i--;
1009
1010 for (; i >= 0; i--) {
fa71ae27
AD		 1011 /* free only the irqs that were actually requested */
1012 if (!adapter->q_vector[i]->rx.ring &&
1013 !adapter->q_vector[i]->tx.ring)
1014 continue;
1015
92915f71
GR		 1016 free_irq(adapter->msix_entries[i].vector,
1017 adapter->q_vector[i]);
1018 }
1019
1020 ixgbevf_reset_q_vectors(adapter);
1021}
1022
1023/**
1024 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1025 * @adapter: board private structure
1026 **/
1027static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1028{
92915f71 1029 struct ixgbe_hw *hw = &adapter->hw;
5f3600eb 1030 int i;
92915f71 1031
5f3600eb 1032 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
92915f71 1033 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
5f3600eb 1034 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
92915f71
GR		 1035
1036 IXGBE_WRITE_FLUSH(hw);
1037
1038 for (i = 0; i < adapter->num_msix_vectors; i++)
1039 synchronize_irq(adapter->msix_entries[i].vector);
1040}
1041
1042/**
1043 * ixgbevf_irq_enable - Enable default interrupt generation settings
1044 * @adapter: board private structure
1045 **/
5f3600eb 1046static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
92915f71
GR		 1047{
1048 struct ixgbe_hw *hw = &adapter->hw;
92915f71 1049
5f3600eb
AD		 1050 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1051 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1052 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
92915f71
GR		 1053}
1054
1055/**
1056 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1057 * @adapter: board private structure
1058 *
1059 * Configure the Tx unit of the MAC after a reset.
1060 **/
1061static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1062{
1063 u64 tdba;
1064 struct ixgbe_hw *hw = &adapter->hw;
1065 u32 i, j, tdlen, txctrl;
1066
1067 /* Setup the HW Tx Head and Tail descriptor pointers */
1068 for (i = 0; i < adapter->num_tx_queues; i++) {
1069 struct ixgbevf_ring *ring = &adapter->tx_ring[i];
1070 j = ring->reg_idx;
1071 tdba = ring->dma;
1072 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc);
1073 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1074 (tdba & DMA_BIT_MASK(32)));
1075 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1076 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen);
1077 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0);
1078 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0);
1079 adapter->tx_ring[i].head = IXGBE_VFTDH(j);
1080 adapter->tx_ring[i].tail = IXGBE_VFTDT(j);
1081 /* Disable Tx Head Writeback RO bit, since this hoses
1082 * bookkeeping if things aren't delivered in order.
1083 */
1084 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1085 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
1086 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1087 }
1088}
1089
1090#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1091
1092static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1093{
1094 struct ixgbevf_ring *rx_ring;
1095 struct ixgbe_hw *hw = &adapter->hw;
1096 u32 srrctl;
1097
1098 rx_ring = &adapter->rx_ring[index];
1099
1100 srrctl = IXGBE_SRRCTL_DROP_EN;
1101
77d5dfca 1102 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
92915f71 1103
dd1fe113
AD		 1104 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
1105 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1106
92915f71
GR		 1107 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1108}
1109
dd1fe113
AD		 1110static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter)
1111{
1112 struct ixgbe_hw *hw = &adapter->hw;
1113 struct net_device *netdev = adapter->netdev;
1114 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1115 int i;
1116 u16 rx_buf_len;
1117
1118 /* notify the PF of our intent to use this size of frame */
1119 ixgbevf_rlpml_set_vf(hw, max_frame);
1120
1121 /* PF will allow an extra 4 bytes past for vlan tagged frames */
1122 max_frame += VLAN_HLEN;
1123
1124 /*
85624caf
GR		 1125 * Allocate buffer sizes that fit well into 32K and
1126 * take into account max frame size of 9.5K
dd1fe113
AD		 1127 */
1128 if ((hw->mac.type == ixgbe_mac_X540_vf) &&
1129 (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE))
1130 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
85624caf
GR		 1131 else if (max_frame <= IXGBEVF_RXBUFFER_2K)
1132 rx_buf_len = IXGBEVF_RXBUFFER_2K;
1133 else if (max_frame <= IXGBEVF_RXBUFFER_4K)
1134 rx_buf_len = IXGBEVF_RXBUFFER_4K;
1135 else if (max_frame <= IXGBEVF_RXBUFFER_8K)
1136 rx_buf_len = IXGBEVF_RXBUFFER_8K;
dd1fe113 1137 else
85624caf 1138 rx_buf_len = IXGBEVF_RXBUFFER_10K;
dd1fe113
AD		 1139
1140 for (i = 0; i < adapter->num_rx_queues; i++)
1141 adapter->rx_ring[i].rx_buf_len = rx_buf_len;
1142}
1143
92915f71
GR		 1144/**
1145 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1146 * @adapter: board private structure
1147 *
1148 * Configure the Rx unit of the MAC after a reset.
1149 **/
1150static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1151{
1152 u64 rdba;
1153 struct ixgbe_hw *hw = &adapter->hw;
92915f71
GR		 1154 int i, j;
1155 u32 rdlen;
92915f71 1156
77d5dfca
AD		 1157 /* PSRTYPE must be initialized in 82599 */
1158 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0);
dd1fe113
AD		 1159
1160 /* set_rx_buffer_len must be called before ring initialization */
1161 ixgbevf_set_rx_buffer_len(adapter);
92915f71
GR		 1162
1163 rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc);
1164 /* Setup the HW Rx Head and Tail Descriptor Pointers and
1165 * the Base and Length of the Rx Descriptor Ring */
1166 for (i = 0; i < adapter->num_rx_queues; i++) {
1167 rdba = adapter->rx_ring[i].dma;
1168 j = adapter->rx_ring[i].reg_idx;
1169 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1170 (rdba & DMA_BIT_MASK(32)));
1171 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
1172 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen);
1173 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0);
1174 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0);
1175 adapter->rx_ring[i].head = IXGBE_VFRDH(j);
1176 adapter->rx_ring[i].tail = IXGBE_VFRDT(j);
92915f71
GR		 1177
1178 ixgbevf_configure_srrctl(adapter, j);
1179 }
1180}
1181
8e586137 1182static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
92915f71
GR		 1183{
1184 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1185 struct ixgbe_hw *hw = &adapter->hw;
2ddc7fe1
AD		 1186 int err;
1187
55fdd45b 1188 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1189
92915f71 1190 /* add VID to filter table */
2ddc7fe1 1191 err = hw->mac.ops.set_vfta(hw, vid, 0, true);
1c55ed76 1192
55fdd45b 1193 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1194
2ddc7fe1
AD		 1195 /* translate error return types so error makes sense */
1196 if (err == IXGBE_ERR_MBX)
1197 return -EIO;
1198
1199 if (err == IXGBE_ERR_INVALID_ARGUMENT)
1200 return -EACCES;
1201
dadcd65f 1202 set_bit(vid, adapter->active_vlans);
8e586137 1203
2ddc7fe1 1204 return err;
92915f71
GR		 1205}
1206
8e586137 1207static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
92915f71
GR		 1208{
1209 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1210 struct ixgbe_hw *hw = &adapter->hw;
2ddc7fe1 1211 int err = -EOPNOTSUPP;
92915f71 1212
55fdd45b 1213 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1214
92915f71 1215 /* remove VID from filter table */
92fe0bf7 1216 err = hw->mac.ops.set_vfta(hw, vid, 0, false);
1c55ed76 1217
55fdd45b 1218 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1219
dadcd65f 1220 clear_bit(vid, adapter->active_vlans);
8e586137 1221
2ddc7fe1 1222 return err;
92915f71
GR		 1223}
1224
1225static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1226{
dadcd65f 1227 u16 vid;
92915f71 1228
dadcd65f
JP		 1229 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1230 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid);
92915f71
GR		 1231}
1232
46ec20ff
GR		 1233static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1234{
1235 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1236 struct ixgbe_hw *hw = &adapter->hw;
1237 int count = 0;
1238
1239 if ((netdev_uc_count(netdev)) > 10) {
dbd9636e 1240 pr_err("Too many unicast filters - No Space\n");
46ec20ff
GR		 1241 return -ENOSPC;
1242 }
1243
1244 if (!netdev_uc_empty(netdev)) {
1245 struct netdev_hw_addr *ha;
1246 netdev_for_each_uc_addr(ha, netdev) {
1247 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1248 udelay(200);
1249 }
1250 } else {
1251 /*
1252 * If the list is empty then send message to PF driver to
1253 * clear all macvlans on this VF.
1254 */
1255 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1256 }
1257
1258 return count;
1259}
1260
92915f71 1261/**
dee847f5 1262 * ixgbevf_set_rx_mode - Multicast and unicast set
92915f71
GR		 1263 * @netdev: network interface device structure
1264 *
1265 * The set_rx_method entry point is called whenever the multicast address
dee847f5
GR		 1266 * list, unicast address list or the network interface flags are updated.
1267 * This routine is responsible for configuring the hardware for proper
1268 * multicast mode and configuring requested unicast filters.
92915f71
GR		 1269 **/
1270static void ixgbevf_set_rx_mode(struct net_device *netdev)
1271{
1272 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1273 struct ixgbe_hw *hw = &adapter->hw;
92915f71 1274
55fdd45b 1275 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1276
92915f71 1277 /* reprogram multicast list */
92fe0bf7 1278 hw->mac.ops.update_mc_addr_list(hw, netdev);
46ec20ff
GR		 1279
1280 ixgbevf_write_uc_addr_list(netdev);
1c55ed76 1281
55fdd45b 1282 spin_unlock_bh(&adapter->mbx_lock);
92915f71
GR		 1283}
1284
1285static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1286{
1287 int q_idx;
1288 struct ixgbevf_q_vector *q_vector;
1289 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1290
1291 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
92915f71 1292 q_vector = adapter->q_vector[q_idx];
fa71ae27 1293 napi_enable(&q_vector->napi);
92915f71
GR		 1294 }
1295}
1296
1297static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1298{
1299 int q_idx;
1300 struct ixgbevf_q_vector *q_vector;
1301 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1302
1303 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1304 q_vector = adapter->q_vector[q_idx];
92915f71
GR		 1305 napi_disable(&q_vector->napi);
1306 }
1307}
1308
1309static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1310{
1311 struct net_device *netdev = adapter->netdev;
1312 int i;
1313
1314 ixgbevf_set_rx_mode(netdev);
1315
1316 ixgbevf_restore_vlan(adapter);
1317
1318 ixgbevf_configure_tx(adapter);
1319 ixgbevf_configure_rx(adapter);
1320 for (i = 0; i < adapter->num_rx_queues; i++) {
1321 struct ixgbevf_ring *ring = &adapter->rx_ring[i];
18c63089
AD		 1322 ixgbevf_alloc_rx_buffers(adapter, ring,
1323 IXGBE_DESC_UNUSED(ring));
92915f71
GR		 1324 }
1325}
1326
1327#define IXGBE_MAX_RX_DESC_POLL 10
1328static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1329 int rxr)
1330{
1331 struct ixgbe_hw *hw = &adapter->hw;
1332 int j = adapter->rx_ring[rxr].reg_idx;
1333 int k;
1334
1335 for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) {
1336 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE)
1337 break;
1338 else
1339 msleep(1);
1340 }
1341 if (k >= IXGBE_MAX_RX_DESC_POLL) {
1342 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d "
1343 "not set within the polling period\n", rxr);
1344 }
1345
6259a01f
GR		 1346 ixgbevf_release_rx_desc(hw, &adapter->rx_ring[rxr],
1347 adapter->rx_ring[rxr].count - 1);
92915f71
GR		 1348}
1349
33bd9f60
GR		 1350static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1351{
1352 /* Only save pre-reset stats if there are some */
1353 if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1354 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1355 adapter->stats.base_vfgprc;
1356 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1357 adapter->stats.base_vfgptc;
1358 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1359 adapter->stats.base_vfgorc;
1360 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1361 adapter->stats.base_vfgotc;
1362 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1363 adapter->stats.base_vfmprc;
1364 }
1365}
1366
1367static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1368{
1369 struct ixgbe_hw *hw = &adapter->hw;
1370
1371 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1372 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1373 adapter->stats.last_vfgorc |=
1374 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1375 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1376 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1377 adapter->stats.last_vfgotc |=
1378 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1379 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1380
1381 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1382 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1383 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1384 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1385 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1386}
1387
31186785
AD		 1388static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
1389{
1390 struct ixgbe_hw *hw = &adapter->hw;
56e94095
AD		 1391 int api[] = { ixgbe_mbox_api_11,
1392 ixgbe_mbox_api_10,
31186785
AD		 1393 ixgbe_mbox_api_unknown };
1394 int err = 0, idx = 0;
1395
55fdd45b 1396 spin_lock_bh(&adapter->mbx_lock);
31186785
AD		 1397
1398 while (api[idx] != ixgbe_mbox_api_unknown) {
1399 err = ixgbevf_negotiate_api_version(hw, api[idx]);
1400 if (!err)
1401 break;
1402 idx++;
1403 }
1404
55fdd45b 1405 spin_unlock_bh(&adapter->mbx_lock);
31186785
AD		 1406}
1407
795180d8 1408static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
92915f71
GR		 1409{
1410 struct net_device *netdev = adapter->netdev;
1411 struct ixgbe_hw *hw = &adapter->hw;
1412 int i, j = 0;
1413 int num_rx_rings = adapter->num_rx_queues;
1414 u32 txdctl, rxdctl;
1415
1416 for (i = 0; i < adapter->num_tx_queues; i++) {
1417 j = adapter->tx_ring[i].reg_idx;
1418 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1419 /* enable WTHRESH=8 descriptors, to encourage burst writeback */
1420 txdctl |= (8 << 16);
1421 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1422 }
1423
1424 for (i = 0; i < adapter->num_tx_queues; i++) {
1425 j = adapter->tx_ring[i].reg_idx;
1426 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1427 txdctl |= IXGBE_TXDCTL_ENABLE;
1428 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1429 }
1430
1431 for (i = 0; i < num_rx_rings; i++) {
1432 j = adapter->rx_ring[i].reg_idx;
1433 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
dadcd65f 1434 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
69bfbec4
GR		 1435 if (hw->mac.type == ixgbe_mac_X540_vf) {
1436 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1437 rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) |
1438 IXGBE_RXDCTL_RLPML_EN);
1439 }
92915f71
GR		 1440 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1441 ixgbevf_rx_desc_queue_enable(adapter, i);
1442 }
1443
1444 ixgbevf_configure_msix(adapter);
1445
55fdd45b 1446 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 1447
92fe0bf7
GR		 1448 if (is_valid_ether_addr(hw->mac.addr))
1449 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1450 else
1451 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
92915f71 1452
55fdd45b 1453 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 1454
92915f71
GR		 1455 clear_bit(__IXGBEVF_DOWN, &adapter->state);
1456 ixgbevf_napi_enable_all(adapter);
1457
1458 /* enable transmits */
1459 netif_tx_start_all_queues(netdev);
1460
33bd9f60
GR		 1461 ixgbevf_save_reset_stats(adapter);
1462 ixgbevf_init_last_counter_stats(adapter);
1463
4b2cd27f 1464 hw->mac.get_link_status = 1;
92915f71 1465 mod_timer(&adapter->watchdog_timer, jiffies);
92915f71
GR		 1466}
1467
56e94095
AD		 1468static int ixgbevf_reset_queues(struct ixgbevf_adapter *adapter)
1469{
1470 struct ixgbe_hw *hw = &adapter->hw;
1471 struct ixgbevf_ring *rx_ring;
1472 unsigned int def_q = 0;
1473 unsigned int num_tcs = 0;
1474 unsigned int num_rx_queues = 1;
1475 int err, i;
1476
55fdd45b 1477 spin_lock_bh(&adapter->mbx_lock);
56e94095
AD		 1478
1479 /* fetch queue configuration from the PF */
1480 err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1481
55fdd45b 1482 spin_unlock_bh(&adapter->mbx_lock);
56e94095
AD		 1483
1484 if (err)
1485 return err;
1486
1487 if (num_tcs > 1) {
1488 /* update default Tx ring register index */
1489 adapter->tx_ring[0].reg_idx = def_q;
1490
1491 /* we need as many queues as traffic classes */
1492 num_rx_queues = num_tcs;
1493 }
1494
1495 /* nothing to do if we have the correct number of queues */
1496 if (adapter->num_rx_queues == num_rx_queues)
1497 return 0;
1498
1499 /* allocate new rings */
1500 rx_ring = kcalloc(num_rx_queues,
1501 sizeof(struct ixgbevf_ring), GFP_KERNEL);
1502 if (!rx_ring)
1503 return -ENOMEM;
1504
1505 /* setup ring fields */
1506 for (i = 0; i < num_rx_queues; i++) {
1507 rx_ring[i].count = adapter->rx_ring_count;
1508 rx_ring[i].queue_index = i;
1509 rx_ring[i].reg_idx = i;
1510 rx_ring[i].dev = &adapter->pdev->dev;
1511 rx_ring[i].netdev = adapter->netdev;
1512
1513 /* allocate resources on the ring */
1514 err = ixgbevf_setup_rx_resources(adapter, &rx_ring[i]);
1515 if (err) {
1516 while (i) {
1517 i--;
1518 ixgbevf_free_rx_resources(adapter, &rx_ring[i]);
1519 }
1520 kfree(rx_ring);
1521 return err;
1522 }
1523 }
1524
1525 /* free the existing rings and queues */
1526 ixgbevf_free_all_rx_resources(adapter);
1527 adapter->num_rx_queues = 0;
1528 kfree(adapter->rx_ring);
1529
1530 /* move new rings into position on the adapter struct */
1531 adapter->rx_ring = rx_ring;
1532 adapter->num_rx_queues = num_rx_queues;
1533
1534 /* reset ring to vector mapping */
1535 ixgbevf_reset_q_vectors(adapter);
1536 ixgbevf_map_rings_to_vectors(adapter);
1537
1538 return 0;
1539}
1540
795180d8 1541void ixgbevf_up(struct ixgbevf_adapter *adapter)
92915f71 1542{
92915f71
GR		 1543 struct ixgbe_hw *hw = &adapter->hw;
1544
31186785
AD		 1545 ixgbevf_negotiate_api(adapter);
1546
56e94095
AD		 1547 ixgbevf_reset_queues(adapter);
1548
92915f71
GR		 1549 ixgbevf_configure(adapter);
1550
795180d8 1551 ixgbevf_up_complete(adapter);
92915f71
GR		 1552
1553 /* clear any pending interrupts, may auto mask */
1554 IXGBE_READ_REG(hw, IXGBE_VTEICR);
1555
5f3600eb 1556 ixgbevf_irq_enable(adapter);
92915f71
GR		 1557}
1558
1559/**
1560 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
1561 * @adapter: board private structure
1562 * @rx_ring: ring to free buffers from
1563 **/
1564static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter,
1565 struct ixgbevf_ring *rx_ring)
1566{
1567 struct pci_dev *pdev = adapter->pdev;
1568 unsigned long size;
1569 unsigned int i;
1570
c0456c23
GR		 1571 if (!rx_ring->rx_buffer_info)
1572 return;
92915f71 1573
c0456c23 1574 /* Free all the Rx ring sk_buffs */
92915f71
GR		 1575 for (i = 0; i < rx_ring->count; i++) {
1576 struct ixgbevf_rx_buffer *rx_buffer_info;
1577
1578 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1579 if (rx_buffer_info->dma) {
2a1f8794 1580 dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
92915f71 1581 rx_ring->rx_buf_len,
2a1f8794 1582 DMA_FROM_DEVICE);
92915f71
GR		 1583 rx_buffer_info->dma = 0;
1584 }
1585 if (rx_buffer_info->skb) {
1586 struct sk_buff *skb = rx_buffer_info->skb;
1587 rx_buffer_info->skb = NULL;
1588 do {
1589 struct sk_buff *this = skb;
5c60f81a 1590 skb = IXGBE_CB(skb)->prev;
92915f71
GR		 1591 dev_kfree_skb(this);
1592 } while (skb);
1593 }
92915f71
GR		 1594 }
1595
1596 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1597 memset(rx_ring->rx_buffer_info, 0, size);
1598
1599 /* Zero out the descriptor ring */
1600 memset(rx_ring->desc, 0, rx_ring->size);
1601
1602 rx_ring->next_to_clean = 0;
1603 rx_ring->next_to_use = 0;
1604
1605 if (rx_ring->head)
1606 writel(0, adapter->hw.hw_addr + rx_ring->head);
1607 if (rx_ring->tail)
1608 writel(0, adapter->hw.hw_addr + rx_ring->tail);
1609}
1610
1611/**
1612 * ixgbevf_clean_tx_ring - Free Tx Buffers
1613 * @adapter: board private structure
1614 * @tx_ring: ring to be cleaned
1615 **/
1616static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter,
1617 struct ixgbevf_ring *tx_ring)
1618{
1619 struct ixgbevf_tx_buffer *tx_buffer_info;
1620 unsigned long size;
1621 unsigned int i;
1622
c0456c23
GR		 1623 if (!tx_ring->tx_buffer_info)
1624 return;
1625
92915f71 1626 /* Free all the Tx ring sk_buffs */
92915f71
GR		 1627 for (i = 0; i < tx_ring->count; i++) {
1628 tx_buffer_info = &tx_ring->tx_buffer_info[i];
70a10e25 1629 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
92915f71
GR		 1630 }
1631
1632 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1633 memset(tx_ring->tx_buffer_info, 0, size);
1634
1635 memset(tx_ring->desc, 0, tx_ring->size);
1636
1637 tx_ring->next_to_use = 0;
1638 tx_ring->next_to_clean = 0;
1639
1640 if (tx_ring->head)
1641 writel(0, adapter->hw.hw_addr + tx_ring->head);
1642 if (tx_ring->tail)
1643 writel(0, adapter->hw.hw_addr + tx_ring->tail);
1644}
1645
1646/**
1647 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1648 * @adapter: board private structure
1649 **/
1650static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1651{
1652 int i;
1653
1654 for (i = 0; i < adapter->num_rx_queues; i++)
1655 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]);
1656}
1657
1658/**
1659 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1660 * @adapter: board private structure
1661 **/
1662static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1663{
1664 int i;
1665
1666 for (i = 0; i < adapter->num_tx_queues; i++)
1667 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]);
1668}
1669
1670void ixgbevf_down(struct ixgbevf_adapter *adapter)
1671{
1672 struct net_device *netdev = adapter->netdev;
1673 struct ixgbe_hw *hw = &adapter->hw;
1674 u32 txdctl;
1675 int i, j;
1676
1677 /* signal that we are down to the interrupt handler */
1678 set_bit(__IXGBEVF_DOWN, &adapter->state);
1679 /* disable receives */
1680
1681 netif_tx_disable(netdev);
1682
1683 msleep(10);
1684
1685 netif_tx_stop_all_queues(netdev);
1686
1687 ixgbevf_irq_disable(adapter);
1688
1689 ixgbevf_napi_disable_all(adapter);
1690
1691 del_timer_sync(&adapter->watchdog_timer);
1692 /* can't call flush scheduled work here because it can deadlock
1693 * if linkwatch_event tries to acquire the rtnl_lock which we are
1694 * holding */
1695 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1696 msleep(1);
1697
1698 /* disable transmits in the hardware now that interrupts are off */
1699 for (i = 0; i < adapter->num_tx_queues; i++) {
1700 j = adapter->tx_ring[i].reg_idx;
1701 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1702 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j),
1703 (txdctl & ~IXGBE_TXDCTL_ENABLE));
1704 }
1705
1706 netif_carrier_off(netdev);
1707
1708 if (!pci_channel_offline(adapter->pdev))
1709 ixgbevf_reset(adapter);
1710
1711 ixgbevf_clean_all_tx_rings(adapter);
1712 ixgbevf_clean_all_rx_rings(adapter);
1713}
1714
1715void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1716{
1717 WARN_ON(in_interrupt());
c0456c23 1718
92915f71
GR		 1719 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1720 msleep(1);
1721
4b2cd27f
AD		 1722 ixgbevf_down(adapter);
1723 ixgbevf_up(adapter);
92915f71
GR		 1724
1725 clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1726}
1727
1728void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1729{
1730 struct ixgbe_hw *hw = &adapter->hw;
1731 struct net_device *netdev = adapter->netdev;
1732
1733 if (hw->mac.ops.reset_hw(hw))
1734 hw_dbg(hw, "PF still resetting\n");
1735 else
1736 hw->mac.ops.init_hw(hw);
1737
1738 if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1739 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1740 netdev->addr_len);
1741 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1742 netdev->addr_len);
1743 }
1744}
1745
e45dd5fe
JK		 1746static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1747 int vectors)
92915f71 1748{
a5f9337b
ET		 1749 int err = 0;
1750 int vector_threshold;
92915f71 1751
fa71ae27
AD		 1752 /* We'll want at least 2 (vector_threshold):
1753 * 1) TxQ[0] + RxQ[0] handler
1754 * 2) Other (Link Status Change, etc.)
92915f71
GR		 1755 */
1756 vector_threshold = MIN_MSIX_COUNT;
1757
1758 /* The more we get, the more we will assign to Tx/Rx Cleanup
1759 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1760 * Right now, we simply care about how many we'll get; we'll
1761 * set them up later while requesting irq's.
1762 */
1763 while (vectors >= vector_threshold) {
1764 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1765 vectors);
e45dd5fe 1766 if (!err || err < 0) /* Success or a nasty failure. */
92915f71 1767 break;
92915f71
GR		 1768 else /* err == number of vectors we should try again with */
1769 vectors = err;
1770 }
1771
e45dd5fe
JK		 1772 if (vectors < vector_threshold)
1773 err = -ENOMEM;
1774
1775 if (err) {
1776 dev_err(&adapter->pdev->dev,
1777 "Unable to allocate MSI-X interrupts\n");
92915f71
GR		 1778 kfree(adapter->msix_entries);
1779 adapter->msix_entries = NULL;
1780 } else {
1781 /*
1782 * Adjust for only the vectors we'll use, which is minimum
1783 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1784 * vectors we were allocated.
1785 */
1786 adapter->num_msix_vectors = vectors;
1787 }
dee847f5 1788
e45dd5fe 1789 return err;
92915f71
GR		 1790}
1791
49ce9c2c
BH		 1792/**
1793 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
92915f71
GR		 1794 * @adapter: board private structure to initialize
1795 *
1796 * This is the top level queue allocation routine. The order here is very
1797 * important, starting with the "most" number of features turned on at once,
1798 * and ending with the smallest set of features. This way large combinations
1799 * can be allocated if they're turned on, and smaller combinations are the
1800 * fallthrough conditions.
1801 *
1802 **/
1803static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1804{
1805 /* Start with base case */
1806 adapter->num_rx_queues = 1;
1807 adapter->num_tx_queues = 1;
92915f71
GR		 1808}
1809
1810/**
1811 * ixgbevf_alloc_queues - Allocate memory for all rings
1812 * @adapter: board private structure to initialize
1813 *
1814 * We allocate one ring per queue at run-time since we don't know the
1815 * number of queues at compile-time. The polling_netdev array is
1816 * intended for Multiqueue, but should work fine with a single queue.
1817 **/
1818static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1819{
1820 int i;
1821
1822 adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1823 sizeof(struct ixgbevf_ring), GFP_KERNEL);
1824 if (!adapter->tx_ring)
1825 goto err_tx_ring_allocation;
1826
1827 adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1828 sizeof(struct ixgbevf_ring), GFP_KERNEL);
1829 if (!adapter->rx_ring)
1830 goto err_rx_ring_allocation;
1831
1832 for (i = 0; i < adapter->num_tx_queues; i++) {
1833 adapter->tx_ring[i].count = adapter->tx_ring_count;
1834 adapter->tx_ring[i].queue_index = i;
56e94095 1835 /* reg_idx may be remapped later by DCB config */
92915f71 1836 adapter->tx_ring[i].reg_idx = i;
fb40195c
AD		 1837 adapter->tx_ring[i].dev = &adapter->pdev->dev;
1838 adapter->tx_ring[i].netdev = adapter->netdev;
92915f71
GR		 1839 }
1840
1841 for (i = 0; i < adapter->num_rx_queues; i++) {
1842 adapter->rx_ring[i].count = adapter->rx_ring_count;
1843 adapter->rx_ring[i].queue_index = i;
1844 adapter->rx_ring[i].reg_idx = i;
fb40195c
AD		 1845 adapter->rx_ring[i].dev = &adapter->pdev->dev;
1846 adapter->rx_ring[i].netdev = adapter->netdev;
92915f71
GR		 1847 }
1848
1849 return 0;
1850
1851err_rx_ring_allocation:
1852 kfree(adapter->tx_ring);
1853err_tx_ring_allocation:
1854 return -ENOMEM;
1855}
1856
1857/**
1858 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
1859 * @adapter: board private structure to initialize
1860 *
1861 * Attempt to configure the interrupts using the best available
1862 * capabilities of the hardware and the kernel.
1863 **/
1864static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
1865{
91e2b89b 1866 struct net_device *netdev = adapter->netdev;
92915f71
GR		 1867 int err = 0;
1868 int vector, v_budget;
1869
1870 /*
1871 * It's easy to be greedy for MSI-X vectors, but it really
1872 * doesn't do us much good if we have a lot more vectors
1873 * than CPU's. So let's be conservative and only ask for
fa71ae27
AD		 1874 * (roughly) the same number of vectors as there are CPU's.
1875 * The default is to use pairs of vectors.
92915f71 1876 */
fa71ae27
AD		 1877 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
1878 v_budget = min_t(int, v_budget, num_online_cpus());
1879 v_budget += NON_Q_VECTORS;
92915f71
GR		 1880
1881 /* A failure in MSI-X entry allocation isn't fatal, but it does
1882 * mean we disable MSI-X capabilities of the adapter. */
1883 adapter->msix_entries = kcalloc(v_budget,
1884 sizeof(struct msix_entry), GFP_KERNEL);
1885 if (!adapter->msix_entries) {
1886 err = -ENOMEM;
1887 goto out;
1888 }
1889
1890 for (vector = 0; vector < v_budget; vector++)
1891 adapter->msix_entries[vector].entry = vector;
1892
e45dd5fe
JK		 1893 err = ixgbevf_acquire_msix_vectors(adapter, v_budget);
1894 if (err)
1895 goto out;
92915f71 1896
91e2b89b
GR		 1897 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
1898 if (err)
1899 goto out;
1900
1901 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
1902
92915f71
GR		 1903out:
1904 return err;
1905}
1906
1907/**
1908 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
1909 * @adapter: board private structure to initialize
1910 *
1911 * We allocate one q_vector per queue interrupt. If allocation fails we
1912 * return -ENOMEM.
1913 **/
1914static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
1915{
1916 int q_idx, num_q_vectors;
1917 struct ixgbevf_q_vector *q_vector;
92915f71
GR		 1918
1919 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
92915f71
GR		 1920
1921 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1922 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
1923 if (!q_vector)
1924 goto err_out;
1925 q_vector->adapter = adapter;
1926 q_vector->v_idx = q_idx;
fa71ae27
AD		 1927 netif_napi_add(adapter->netdev, &q_vector->napi,
1928 ixgbevf_poll, 64);
92915f71
GR		 1929 adapter->q_vector[q_idx] = q_vector;
1930 }
1931
1932 return 0;
1933
1934err_out:
1935 while (q_idx) {
1936 q_idx--;
1937 q_vector = adapter->q_vector[q_idx];
1938 netif_napi_del(&q_vector->napi);
1939 kfree(q_vector);
1940 adapter->q_vector[q_idx] = NULL;
1941 }
1942 return -ENOMEM;
1943}
1944
1945/**
1946 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
1947 * @adapter: board private structure to initialize
1948 *
1949 * This function frees the memory allocated to the q_vectors. In addition if
1950 * NAPI is enabled it will delete any references to the NAPI struct prior
1951 * to freeing the q_vector.
1952 **/
1953static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
1954{
f4477702 1955 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
92915f71
GR		 1956
1957 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1958 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
1959
1960 adapter->q_vector[q_idx] = NULL;
f4477702 1961 netif_napi_del(&q_vector->napi);
92915f71
GR		 1962 kfree(q_vector);
1963 }
1964}
1965
1966/**
1967 * ixgbevf_reset_interrupt_capability - Reset MSIX setup
1968 * @adapter: board private structure
1969 *
1970 **/
1971static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
1972{
1973 pci_disable_msix(adapter->pdev);
1974 kfree(adapter->msix_entries);
1975 adapter->msix_entries = NULL;
92915f71
GR		 1976}
1977
1978/**
1979 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
1980 * @adapter: board private structure to initialize
1981 *
1982 **/
1983static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
1984{
1985 int err;
1986
1987 /* Number of supported queues */
1988 ixgbevf_set_num_queues(adapter);
1989
1990 err = ixgbevf_set_interrupt_capability(adapter);
1991 if (err) {
1992 hw_dbg(&adapter->hw,
1993 "Unable to setup interrupt capabilities\n");
1994 goto err_set_interrupt;
1995 }
1996
1997 err = ixgbevf_alloc_q_vectors(adapter);
1998 if (err) {
1999 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
2000 "vectors\n");
2001 goto err_alloc_q_vectors;
2002 }
2003
2004 err = ixgbevf_alloc_queues(adapter);
2005 if (err) {
dbd9636e 2006 pr_err("Unable to allocate memory for queues\n");
92915f71
GR		 2007 goto err_alloc_queues;
2008 }
2009
2010 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
2011 "Tx Queue count = %u\n",
2012 (adapter->num_rx_queues > 1) ? "Enabled" :
2013 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
2014
2015 set_bit(__IXGBEVF_DOWN, &adapter->state);
2016
2017 return 0;
2018err_alloc_queues:
2019 ixgbevf_free_q_vectors(adapter);
2020err_alloc_q_vectors:
2021 ixgbevf_reset_interrupt_capability(adapter);
2022err_set_interrupt:
2023 return err;
2024}
2025
0ac1e8ce
AD		 2026/**
2027 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
2028 * @adapter: board private structure to clear interrupt scheme on
2029 *
2030 * We go through and clear interrupt specific resources and reset the structure
2031 * to pre-load conditions
2032 **/
2033static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
2034{
2035 adapter->num_tx_queues = 0;
2036 adapter->num_rx_queues = 0;
2037
2038 ixgbevf_free_q_vectors(adapter);
2039 ixgbevf_reset_interrupt_capability(adapter);
2040}
2041
92915f71
GR		 2042/**
2043 * ixgbevf_sw_init - Initialize general software structures
2044 * (struct ixgbevf_adapter)
2045 * @adapter: board private structure to initialize
2046 *
2047 * ixgbevf_sw_init initializes the Adapter private data structure.
2048 * Fields are initialized based on PCI device information and
2049 * OS network device settings (MTU size).
2050 **/
9f9a12f8 2051static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
92915f71
GR		 2052{
2053 struct ixgbe_hw *hw = &adapter->hw;
2054 struct pci_dev *pdev = adapter->pdev;
e1941a74 2055 struct net_device *netdev = adapter->netdev;
92915f71
GR		 2056 int err;
2057
2058 /* PCI config space info */
2059
2060 hw->vendor_id = pdev->vendor;
2061 hw->device_id = pdev->device;
ff938e43 2062 hw->revision_id = pdev->revision;
92915f71
GR		 2063 hw->subsystem_vendor_id = pdev->subsystem_vendor;
2064 hw->subsystem_device_id = pdev->subsystem_device;
2065
2066 hw->mbx.ops.init_params(hw);
56e94095
AD		 2067
2068 /* assume legacy case in which PF would only give VF 2 queues */
2069 hw->mac.max_tx_queues = 2;
2070 hw->mac.max_rx_queues = 2;
2071
92915f71
GR		 2072 err = hw->mac.ops.reset_hw(hw);
2073 if (err) {
2074 dev_info(&pdev->dev,
e1941a74 2075 "PF still in reset state. Is the PF interface up?\n");
92915f71
GR		 2076 } else {
2077 err = hw->mac.ops.init_hw(hw);
2078 if (err) {
dbd9636e 2079 pr_err("init_shared_code failed: %d\n", err);
92915f71
GR		 2080 goto out;
2081 }
e1941a74
GR		 2082 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
2083 if (err)
2084 dev_info(&pdev->dev, "Error reading MAC address\n");
2085 else if (is_zero_ether_addr(adapter->hw.mac.addr))
2086 dev_info(&pdev->dev,
2087 "MAC address not assigned by administrator.\n");
2088 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2089 }
2090
2091 if (!is_valid_ether_addr(netdev->dev_addr)) {
2092 dev_info(&pdev->dev, "Assigning random MAC address\n");
2093 eth_hw_addr_random(netdev);
2094 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len);
92915f71
GR		 2095 }
2096
1c55ed76
AD		 2097 /* lock to protect mailbox accesses */
2098 spin_lock_init(&adapter->mbx_lock);
2099
92915f71 2100 /* Enable dynamic interrupt throttling rates */
5f3600eb
AD		 2101 adapter->rx_itr_setting = 1;
2102 adapter->tx_itr_setting = 1;
92915f71 2103
92915f71
GR		 2104 /* set default ring sizes */
2105 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
2106 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
2107
92915f71 2108 set_bit(__IXGBEVF_DOWN, &adapter->state);
1a0d6ae5 2109 return 0;
92915f71
GR		 2110
2111out:
2112 return err;
2113}
2114
92915f71
GR		 2115#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \
2116 { \
2117 u32 current_counter = IXGBE_READ_REG(hw, reg); \
2118 if (current_counter < last_counter) \
2119 counter += 0x100000000LL; \
2120 last_counter = current_counter; \
2121 counter &= 0xFFFFFFFF00000000LL; \
2122 counter |= current_counter; \
2123 }
2124
2125#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2126 { \
2127 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \
2128 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \
2129 u64 current_counter = (current_counter_msb << 32) | \
2130 current_counter_lsb; \
2131 if (current_counter < last_counter) \
2132 counter += 0x1000000000LL; \
2133 last_counter = current_counter; \
2134 counter &= 0xFFFFFFF000000000LL; \
2135 counter |= current_counter; \
2136 }
2137/**
2138 * ixgbevf_update_stats - Update the board statistics counters.
2139 * @adapter: board private structure
2140 **/
2141void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2142{
2143 struct ixgbe_hw *hw = &adapter->hw;
55fb277c 2144 int i;
92915f71 2145
088245a3
GR		 2146 if (!adapter->link_up)
2147 return;
2148
92915f71
GR		 2149 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2150 adapter->stats.vfgprc);
2151 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2152 adapter->stats.vfgptc);
2153 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2154 adapter->stats.last_vfgorc,
2155 adapter->stats.vfgorc);
2156 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2157 adapter->stats.last_vfgotc,
2158 adapter->stats.vfgotc);
2159 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2160 adapter->stats.vfmprc);
55fb277c
GR		 2161
2162 for (i = 0; i < adapter->num_rx_queues; i++) {
2163 adapter->hw_csum_rx_error +=
2164 adapter->rx_ring[i].hw_csum_rx_error;
2165 adapter->hw_csum_rx_good +=
2166 adapter->rx_ring[i].hw_csum_rx_good;
2167 adapter->rx_ring[i].hw_csum_rx_error = 0;
2168 adapter->rx_ring[i].hw_csum_rx_good = 0;
2169 }
92915f71
GR		 2170}
2171
2172/**
2173 * ixgbevf_watchdog - Timer Call-back
2174 * @data: pointer to adapter cast into an unsigned long
2175 **/
2176static void ixgbevf_watchdog(unsigned long data)
2177{
2178 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2179 struct ixgbe_hw *hw = &adapter->hw;
5f3600eb 2180 u32 eics = 0;
92915f71
GR		 2181 int i;
2182
2183 /*
2184 * Do the watchdog outside of interrupt context due to the lovely
2185 * delays that some of the newer hardware requires
2186 */
2187
2188 if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2189 goto watchdog_short_circuit;
2190
2191 /* get one bit for every active tx/rx interrupt vector */
2192 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2193 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
6b43c446 2194 if (qv->rx.ring || qv->tx.ring)
5f3600eb 2195 eics |= 1 << i;
92915f71
GR		 2196 }
2197
5f3600eb 2198 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
92915f71
GR		 2199
2200watchdog_short_circuit:
2201 schedule_work(&adapter->watchdog_task);
2202}
2203
2204/**
2205 * ixgbevf_tx_timeout - Respond to a Tx Hang
2206 * @netdev: network interface device structure
2207 **/
2208static void ixgbevf_tx_timeout(struct net_device *netdev)
2209{
2210 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2211
2212 /* Do the reset outside of interrupt context */
2213 schedule_work(&adapter->reset_task);
2214}
2215
2216static void ixgbevf_reset_task(struct work_struct *work)
2217{
2218 struct ixgbevf_adapter *adapter;
2219 adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2220
2221 /* If we're already down or resetting, just bail */
2222 if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2223 test_bit(__IXGBEVF_RESETTING, &adapter->state))
2224 return;
2225
2226 adapter->tx_timeout_count++;
2227
2228 ixgbevf_reinit_locked(adapter);
2229}
2230
2231/**
2232 * ixgbevf_watchdog_task - worker thread to bring link up
2233 * @work: pointer to work_struct containing our data
2234 **/
2235static void ixgbevf_watchdog_task(struct work_struct *work)
2236{
2237 struct ixgbevf_adapter *adapter = container_of(work,
2238 struct ixgbevf_adapter,
2239 watchdog_task);
2240 struct net_device *netdev = adapter->netdev;
2241 struct ixgbe_hw *hw = &adapter->hw;
2242 u32 link_speed = adapter->link_speed;
2243 bool link_up = adapter->link_up;
92fe0bf7 2244 s32 need_reset;
92915f71
GR		 2245
2246 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2247
2248 /*
2249 * Always check the link on the watchdog because we have
2250 * no LSC interrupt
2251 */
92fe0bf7 2252 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 2253
92fe0bf7 2254 need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
1c55ed76 2255
92fe0bf7 2256 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 2257
92fe0bf7
GR		 2258 if (need_reset) {
2259 adapter->link_up = link_up;
2260 adapter->link_speed = link_speed;
2261 netif_carrier_off(netdev);
2262 netif_tx_stop_all_queues(netdev);
2263 schedule_work(&adapter->reset_task);
2264 goto pf_has_reset;
92915f71
GR		 2265 }
2266 adapter->link_up = link_up;
2267 adapter->link_speed = link_speed;
2268
2269 if (link_up) {
2270 if (!netif_carrier_ok(netdev)) {
b876a744
GR		 2271 char *link_speed_string;
2272 switch (link_speed) {
2273 case IXGBE_LINK_SPEED_10GB_FULL:
2274 link_speed_string = "10 Gbps";
2275 break;
2276 case IXGBE_LINK_SPEED_1GB_FULL:
2277 link_speed_string = "1 Gbps";
2278 break;
2279 case IXGBE_LINK_SPEED_100_FULL:
2280 link_speed_string = "100 Mbps";
2281 break;
2282 default:
2283 link_speed_string = "unknown speed";
2284 break;
2285 }
6fe59675 2286 dev_info(&adapter->pdev->dev,
b876a744 2287 "NIC Link is Up, %s\n", link_speed_string);
92915f71
GR		 2288 netif_carrier_on(netdev);
2289 netif_tx_wake_all_queues(netdev);
92915f71
GR		 2290 }
2291 } else {
2292 adapter->link_up = false;
2293 adapter->link_speed = 0;
2294 if (netif_carrier_ok(netdev)) {
6fe59675 2295 dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
92915f71
GR		 2296 netif_carrier_off(netdev);
2297 netif_tx_stop_all_queues(netdev);
2298 }
2299 }
2300
92915f71
GR		 2301 ixgbevf_update_stats(adapter);
2302
33bd9f60 2303pf_has_reset:
92915f71
GR		 2304 /* Reset the timer */
2305 if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
2306 mod_timer(&adapter->watchdog_timer,
2307 round_jiffies(jiffies + (2 * HZ)));
2308
2309 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2310}
2311
2312/**
2313 * ixgbevf_free_tx_resources - Free Tx Resources per Queue
2314 * @adapter: board private structure
2315 * @tx_ring: Tx descriptor ring for a specific queue
2316 *
2317 * Free all transmit software resources
2318 **/
2319void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter,
2320 struct ixgbevf_ring *tx_ring)
2321{
2322 struct pci_dev *pdev = adapter->pdev;
2323
92915f71
GR		 2324 ixgbevf_clean_tx_ring(adapter, tx_ring);
2325
2326 vfree(tx_ring->tx_buffer_info);
2327 tx_ring->tx_buffer_info = NULL;
2328
2a1f8794
NN		 2329 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2330 tx_ring->dma);
92915f71
GR		 2331
2332 tx_ring->desc = NULL;
2333}
2334
2335/**
2336 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2337 * @adapter: board private structure
2338 *
2339 * Free all transmit software resources
2340 **/
2341static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2342{
2343 int i;
2344
2345 for (i = 0; i < adapter->num_tx_queues; i++)
2346 if (adapter->tx_ring[i].desc)
2347 ixgbevf_free_tx_resources(adapter,
2348 &adapter->tx_ring[i]);
2349
2350}
2351
2352/**
2353 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
2354 * @adapter: board private structure
2355 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2356 *
2357 * Return 0 on success, negative on failure
2358 **/
2359int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter,
2360 struct ixgbevf_ring *tx_ring)
2361{
2362 struct pci_dev *pdev = adapter->pdev;
2363 int size;
2364
2365 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
89bf67f1 2366 tx_ring->tx_buffer_info = vzalloc(size);
92915f71
GR		 2367 if (!tx_ring->tx_buffer_info)
2368 goto err;
92915f71
GR		 2369
2370 /* round up to nearest 4K */
2371 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2372 tx_ring->size = ALIGN(tx_ring->size, 4096);
2373
2a1f8794
NN		 2374 tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
2375 &tx_ring->dma, GFP_KERNEL);
92915f71
GR		 2376 if (!tx_ring->desc)
2377 goto err;
2378
2379 tx_ring->next_to_use = 0;
2380 tx_ring->next_to_clean = 0;
92915f71
GR		 2381 return 0;
2382
2383err:
2384 vfree(tx_ring->tx_buffer_info);
2385 tx_ring->tx_buffer_info = NULL;
2386 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2387 "descriptor ring\n");
2388 return -ENOMEM;
2389}
2390
2391/**
2392 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2393 * @adapter: board private structure
2394 *
2395 * If this function returns with an error, then it's possible one or
2396 * more of the rings is populated (while the rest are not). It is the
2397 * callers duty to clean those orphaned rings.
2398 *
2399 * Return 0 on success, negative on failure
2400 **/
2401static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2402{
2403 int i, err = 0;
2404
2405 for (i = 0; i < adapter->num_tx_queues; i++) {
2406 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]);
2407 if (!err)
2408 continue;
2409 hw_dbg(&adapter->hw,
2410 "Allocation for Tx Queue %u failed\n", i);
2411 break;
2412 }
2413
2414 return err;
2415}
2416
2417/**
2418 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
2419 * @adapter: board private structure
2420 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2421 *
2422 * Returns 0 on success, negative on failure
2423 **/
2424int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
2425 struct ixgbevf_ring *rx_ring)
2426{
2427 struct pci_dev *pdev = adapter->pdev;
2428 int size;
2429
2430 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
89bf67f1 2431 rx_ring->rx_buffer_info = vzalloc(size);
e404decb 2432 if (!rx_ring->rx_buffer_info)
92915f71 2433 goto alloc_failed;
92915f71
GR		 2434
2435 /* Round up to nearest 4K */
2436 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2437 rx_ring->size = ALIGN(rx_ring->size, 4096);
2438
2a1f8794
NN		 2439 rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
2440 &rx_ring->dma, GFP_KERNEL);
92915f71
GR		 2441
2442 if (!rx_ring->desc) {
92915f71
GR		 2443 vfree(rx_ring->rx_buffer_info);
2444 rx_ring->rx_buffer_info = NULL;
2445 goto alloc_failed;
2446 }
2447
2448 rx_ring->next_to_clean = 0;
2449 rx_ring->next_to_use = 0;
2450
2451 return 0;
2452alloc_failed:
2453 return -ENOMEM;
2454}
2455
2456/**
2457 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2458 * @adapter: board private structure
2459 *
2460 * If this function returns with an error, then it's possible one or
2461 * more of the rings is populated (while the rest are not). It is the
2462 * callers duty to clean those orphaned rings.
2463 *
2464 * Return 0 on success, negative on failure
2465 **/
2466static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2467{
2468 int i, err = 0;
2469
2470 for (i = 0; i < adapter->num_rx_queues; i++) {
2471 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]);
2472 if (!err)
2473 continue;
2474 hw_dbg(&adapter->hw,
2475 "Allocation for Rx Queue %u failed\n", i);
2476 break;
2477 }
2478 return err;
2479}
2480
2481/**
2482 * ixgbevf_free_rx_resources - Free Rx Resources
2483 * @adapter: board private structure
2484 * @rx_ring: ring to clean the resources from
2485 *
2486 * Free all receive software resources
2487 **/
2488void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter,
2489 struct ixgbevf_ring *rx_ring)
2490{
2491 struct pci_dev *pdev = adapter->pdev;
2492
2493 ixgbevf_clean_rx_ring(adapter, rx_ring);
2494
2495 vfree(rx_ring->rx_buffer_info);
2496 rx_ring->rx_buffer_info = NULL;
2497
2a1f8794
NN		 2498 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2499 rx_ring->dma);
92915f71
GR		 2500
2501 rx_ring->desc = NULL;
2502}
2503
2504/**
2505 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2506 * @adapter: board private structure
2507 *
2508 * Free all receive software resources
2509 **/
2510static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2511{
2512 int i;
2513
2514 for (i = 0; i < adapter->num_rx_queues; i++)
2515 if (adapter->rx_ring[i].desc)
2516 ixgbevf_free_rx_resources(adapter,
2517 &adapter->rx_ring[i]);
2518}
2519
56e94095
AD		 2520static int ixgbevf_setup_queues(struct ixgbevf_adapter *adapter)
2521{
2522 struct ixgbe_hw *hw = &adapter->hw;
2523 struct ixgbevf_ring *rx_ring;
2524 unsigned int def_q = 0;
2525 unsigned int num_tcs = 0;
2526 unsigned int num_rx_queues = 1;
2527 int err, i;
2528
55fdd45b 2529 spin_lock_bh(&adapter->mbx_lock);
56e94095
AD		 2530
2531 /* fetch queue configuration from the PF */
2532 err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
2533
55fdd45b 2534 spin_unlock_bh(&adapter->mbx_lock);
56e94095
AD		 2535
2536 if (err)
2537 return err;
2538
2539 if (num_tcs > 1) {
2540 /* update default Tx ring register index */
2541 adapter->tx_ring[0].reg_idx = def_q;
2542
2543 /* we need as many queues as traffic classes */
2544 num_rx_queues = num_tcs;
2545 }
2546
2547 /* nothing to do if we have the correct number of queues */
2548 if (adapter->num_rx_queues == num_rx_queues)
2549 return 0;
2550
2551 /* allocate new rings */
2552 rx_ring = kcalloc(num_rx_queues,
2553 sizeof(struct ixgbevf_ring), GFP_KERNEL);
2554 if (!rx_ring)
2555 return -ENOMEM;
2556
2557 /* setup ring fields */
2558 for (i = 0; i < num_rx_queues; i++) {
2559 rx_ring[i].count = adapter->rx_ring_count;
2560 rx_ring[i].queue_index = i;
2561 rx_ring[i].reg_idx = i;
2562 rx_ring[i].dev = &adapter->pdev->dev;
2563 rx_ring[i].netdev = adapter->netdev;
2564 }
2565
2566 /* free the existing ring and queues */
2567 adapter->num_rx_queues = 0;
2568 kfree(adapter->rx_ring);
2569
2570 /* move new rings into position on the adapter struct */
2571 adapter->rx_ring = rx_ring;
2572 adapter->num_rx_queues = num_rx_queues;
2573
2574 return 0;
2575}
2576
92915f71
GR		 2577/**
2578 * ixgbevf_open - Called when a network interface is made active
2579 * @netdev: network interface device structure
2580 *
2581 * Returns 0 on success, negative value on failure
2582 *
2583 * The open entry point is called when a network interface is made
2584 * active by the system (IFF_UP). At this point all resources needed
2585 * for transmit and receive operations are allocated, the interrupt
2586 * handler is registered with the OS, the watchdog timer is started,
2587 * and the stack is notified that the interface is ready.
2588 **/
2589static int ixgbevf_open(struct net_device *netdev)
2590{
2591 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2592 struct ixgbe_hw *hw = &adapter->hw;
2593 int err;
2594
a1f6c6b1 2595 /* A previous failure to open the device because of a lack of
2596 * available MSIX vector resources may have reset the number
2597 * of msix vectors variable to zero. The only way to recover
2598 * is to unload/reload the driver and hope that the system has
2599 * been able to recover some MSIX vector resources.
2600 */
2601 if (!adapter->num_msix_vectors)
2602 return -ENOMEM;
2603
92915f71
GR		 2604 /* disallow open during test */
2605 if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2606 return -EBUSY;
2607
2608 if (hw->adapter_stopped) {
2609 ixgbevf_reset(adapter);
2610 /* if adapter is still stopped then PF isn't up and
2611 * the vf can't start. */
2612 if (hw->adapter_stopped) {
2613 err = IXGBE_ERR_MBX;
dbd9636e
JK		 2614 pr_err("Unable to start - perhaps the PF Driver isn't "
2615 "up yet\n");
92915f71
GR		 2616 goto err_setup_reset;
2617 }
2618 }
2619
31186785
AD		 2620 ixgbevf_negotiate_api(adapter);
2621
56e94095
AD		 2622 /* setup queue reg_idx and Rx queue count */
2623 err = ixgbevf_setup_queues(adapter);
2624 if (err)
2625 goto err_setup_queues;
2626
92915f71
GR		 2627 /* allocate transmit descriptors */
2628 err = ixgbevf_setup_all_tx_resources(adapter);
2629 if (err)
2630 goto err_setup_tx;
2631
2632 /* allocate receive descriptors */
2633 err = ixgbevf_setup_all_rx_resources(adapter);
2634 if (err)
2635 goto err_setup_rx;
2636
2637 ixgbevf_configure(adapter);
2638
2639 /*
2640 * Map the Tx/Rx rings to the vectors we were allotted.
2641 * if request_irq will be called in this function map_rings
2642 * must be called *before* up_complete
2643 */
2644 ixgbevf_map_rings_to_vectors(adapter);
2645
795180d8 2646 ixgbevf_up_complete(adapter);
92915f71
GR		 2647
2648 /* clear any pending interrupts, may auto mask */
2649 IXGBE_READ_REG(hw, IXGBE_VTEICR);
2650 err = ixgbevf_request_irq(adapter);
2651 if (err)
2652 goto err_req_irq;
2653
5f3600eb 2654 ixgbevf_irq_enable(adapter);
92915f71
GR		 2655
2656 return 0;
2657
2658err_req_irq:
2659 ixgbevf_down(adapter);
92915f71
GR		 2660err_setup_rx:
2661 ixgbevf_free_all_rx_resources(adapter);
2662err_setup_tx:
2663 ixgbevf_free_all_tx_resources(adapter);
56e94095 2664err_setup_queues:
92915f71
GR		 2665 ixgbevf_reset(adapter);
2666
2667err_setup_reset:
2668
2669 return err;
2670}
2671
2672/**
2673 * ixgbevf_close - Disables a network interface
2674 * @netdev: network interface device structure
2675 *
2676 * Returns 0, this is not allowed to fail
2677 *
2678 * The close entry point is called when an interface is de-activated
2679 * by the OS. The hardware is still under the drivers control, but
2680 * needs to be disabled. A global MAC reset is issued to stop the
2681 * hardware, and all transmit and receive resources are freed.
2682 **/
2683static int ixgbevf_close(struct net_device *netdev)
2684{
2685 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2686
2687 ixgbevf_down(adapter);
2688 ixgbevf_free_irq(adapter);
2689
2690 ixgbevf_free_all_tx_resources(adapter);
2691 ixgbevf_free_all_rx_resources(adapter);
2692
2693 return 0;
2694}
2695
70a10e25
AD		 2696static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
2697 u32 vlan_macip_lens, u32 type_tucmd,
2698 u32 mss_l4len_idx)
92915f71
GR		 2699{
2700 struct ixgbe_adv_tx_context_desc *context_desc;
70a10e25 2701 u16 i = tx_ring->next_to_use;
92915f71 2702
70a10e25 2703 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
92915f71 2704
70a10e25
AD		 2705 i++;
2706 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
92915f71 2707
70a10e25
AD		 2708 /* set bits to identify this as an advanced context descriptor */
2709 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
92915f71 2710
70a10e25
AD		 2711 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
2712 context_desc->seqnum_seed = 0;
2713 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
2714 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
2715}
2716
2717static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
2718 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2719{
2720 u32 vlan_macip_lens, type_tucmd;
2721 u32 mss_l4len_idx, l4len;
2722
2723 if (!skb_is_gso(skb))
2724 return 0;
92915f71 2725
70a10e25
AD		 2726 if (skb_header_cloned(skb)) {
2727 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2728 if (err)
2729 return err;
92915f71
GR		 2730 }
2731
70a10e25
AD		 2732 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2733 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
2734
2735 if (skb->protocol == htons(ETH_P_IP)) {
2736 struct iphdr *iph = ip_hdr(skb);
2737 iph->tot_len = 0;
2738 iph->check = 0;
2739 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2740 iph->daddr, 0,
2741 IPPROTO_TCP,
2742 0);
2743 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2744 } else if (skb_is_gso_v6(skb)) {
2745 ipv6_hdr(skb)->payload_len = 0;
2746 tcp_hdr(skb)->check =
2747 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2748 &ipv6_hdr(skb)->daddr,
2749 0, IPPROTO_TCP, 0);
2750 }
2751
2752 /* compute header lengths */
2753 l4len = tcp_hdrlen(skb);
2754 *hdr_len += l4len;
2755 *hdr_len = skb_transport_offset(skb) + l4len;
2756
2757 /* mss_l4len_id: use 1 as index for TSO */
2758 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
2759 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
2760 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
2761
2762 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
2763 vlan_macip_lens = skb_network_header_len(skb);
2764 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2765 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2766
2767 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2768 type_tucmd, mss_l4len_idx);
2769
2770 return 1;
92915f71
GR		 2771}
2772
70a10e25 2773static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
92915f71
GR		 2774 struct sk_buff *skb, u32 tx_flags)
2775{
70a10e25
AD		 2776 u32 vlan_macip_lens = 0;
2777 u32 mss_l4len_idx = 0;
2778 u32 type_tucmd = 0;
92915f71 2779
70a10e25
AD		 2780 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2781 u8 l4_hdr = 0;
2782 switch (skb->protocol) {
2783 case __constant_htons(ETH_P_IP):
2784 vlan_macip_lens |= skb_network_header_len(skb);
2785 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2786 l4_hdr = ip_hdr(skb)->protocol;
2787 break;
2788 case __constant_htons(ETH_P_IPV6):
2789 vlan_macip_lens |= skb_network_header_len(skb);
2790 l4_hdr = ipv6_hdr(skb)->nexthdr;
2791 break;
2792 default:
2793 if (unlikely(net_ratelimit())) {
2794 dev_warn(tx_ring->dev,
2795 "partial checksum but proto=%x!\n",
2796 skb->protocol);
2797 }
2798 break;
2799 }
92915f71 2800
70a10e25
AD		 2801 switch (l4_hdr) {
2802 case IPPROTO_TCP:
2803 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2804 mss_l4len_idx = tcp_hdrlen(skb) <<
2805 IXGBE_ADVTXD_L4LEN_SHIFT;
2806 break;
2807 case IPPROTO_SCTP:
2808 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2809 mss_l4len_idx = sizeof(struct sctphdr) <<
2810 IXGBE_ADVTXD_L4LEN_SHIFT;
2811 break;
2812 case IPPROTO_UDP:
2813 mss_l4len_idx = sizeof(struct udphdr) <<
2814 IXGBE_ADVTXD_L4LEN_SHIFT;
2815 break;
2816 default:
2817 if (unlikely(net_ratelimit())) {
2818 dev_warn(tx_ring->dev,
2819 "partial checksum but l4 proto=%x!\n",
2820 l4_hdr);
2821 }
2822 break;
2823 }
92915f71
GR		 2824 }
2825
70a10e25
AD		 2826 /* vlan_macip_lens: MACLEN, VLAN tag */
2827 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2828 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2829
2830 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2831 type_tucmd, mss_l4len_idx);
2832
2833 return (skb->ip_summed == CHECKSUM_PARTIAL);
92915f71
GR		 2834}
2835
70a10e25 2836static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
e757e3e1 2837 struct sk_buff *skb, u32 tx_flags)
92915f71 2838{
92915f71
GR		 2839 struct ixgbevf_tx_buffer *tx_buffer_info;
2840 unsigned int len;
2841 unsigned int total = skb->len;
2540ddb5
KV		 2842 unsigned int offset = 0, size;
2843 int count = 0;
92915f71
GR		 2844 unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
2845 unsigned int f;
65deeed7 2846 int i;
92915f71
GR		 2847
2848 i = tx_ring->next_to_use;
2849
2850 len = min(skb_headlen(skb), total);
2851 while (len) {
2852 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2853 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2854
2855 tx_buffer_info->length = size;
2856 tx_buffer_info->mapped_as_page = false;
70a10e25 2857 tx_buffer_info->dma = dma_map_single(tx_ring->dev,
92915f71 2858 skb->data + offset,
2a1f8794 2859 size, DMA_TO_DEVICE);
70a10e25 2860 if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma))
92915f71 2861 goto dma_error;
92915f71
GR		 2862
2863 len -= size;
2864 total -= size;
2865 offset += size;
2866 count++;
2867 i++;
2868 if (i == tx_ring->count)
2869 i = 0;
2870 }
2871
2872 for (f = 0; f < nr_frags; f++) {
9e903e08 2873 const struct skb_frag_struct *frag;
92915f71
GR		 2874
2875 frag = &skb_shinfo(skb)->frags[f];
9e903e08 2876 len = min((unsigned int)skb_frag_size(frag), total);
877749bf 2877 offset = 0;
92915f71
GR		 2878
2879 while (len) {
2880 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2881 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2882
2883 tx_buffer_info->length = size;
877749bf 2884 tx_buffer_info->dma =
70a10e25 2885 skb_frag_dma_map(tx_ring->dev, frag,
877749bf 2886 offset, size, DMA_TO_DEVICE);
70a10e25
AD		 2887 if (dma_mapping_error(tx_ring->dev,
2888 tx_buffer_info->dma))
92915f71 2889 goto dma_error;
6132ee8a 2890 tx_buffer_info->mapped_as_page = true;
92915f71
GR		 2891
2892 len -= size;
2893 total -= size;
2894 offset += size;
2895 count++;
2896 i++;
2897 if (i == tx_ring->count)
2898 i = 0;
2899 }
2900 if (total == 0)
2901 break;
2902 }
2903
2904 if (i == 0)
2905 i = tx_ring->count - 1;
2906 else
2907 i = i - 1;
2908 tx_ring->tx_buffer_info[i].skb = skb;
92915f71
GR		 2909
2910 return count;
2911
2912dma_error:
70a10e25 2913 dev_err(tx_ring->dev, "TX DMA map failed\n");
92915f71
GR		 2914
2915 /* clear timestamp and dma mappings for failed tx_buffer_info map */
2916 tx_buffer_info->dma = 0;
92915f71
GR		 2917 count--;
2918
2919 /* clear timestamp and dma mappings for remaining portion of packet */
2920 while (count >= 0) {
2921 count--;
2922 i--;
2923 if (i < 0)
2924 i += tx_ring->count;
2925 tx_buffer_info = &tx_ring->tx_buffer_info[i];
70a10e25 2926 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
92915f71
GR		 2927 }
2928
2929 return count;
2930}
2931
70a10e25 2932static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags,
e757e3e1
AD		 2933 int count, unsigned int first, u32 paylen,
2934 u8 hdr_len)
92915f71
GR		 2935{
2936 union ixgbe_adv_tx_desc *tx_desc = NULL;
2937 struct ixgbevf_tx_buffer *tx_buffer_info;
2938 u32 olinfo_status = 0, cmd_type_len = 0;
2939 unsigned int i;
2940
2941 u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
2942
2943 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
2944
2945 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
2946
2947 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2948 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
2949
70a10e25
AD		 2950 if (tx_flags & IXGBE_TX_FLAGS_CSUM)
2951 olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM;
2952
92915f71
GR		 2953 if (tx_flags & IXGBE_TX_FLAGS_TSO) {
2954 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
2955
92915f71
GR		 2956 /* use index 1 context for tso */
2957 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
2958 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
70a10e25 2959 olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM;
70a10e25 2960 }
92915f71 2961
70a10e25
AD		 2962 /*
2963 * Check Context must be set if Tx switch is enabled, which it
2964 * always is for case where virtual functions are running
2965 */
2966 olinfo_status |= IXGBE_ADVTXD_CC;
92915f71
GR		 2967
2968 olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
2969
2970 i = tx_ring->next_to_use;
2971 while (count--) {
2972 tx_buffer_info = &tx_ring->tx_buffer_info[i];
908421f6 2973 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
92915f71
GR		 2974 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
2975 tx_desc->read.cmd_type_len =
2976 cpu_to_le32(cmd_type_len | tx_buffer_info->length);
2977 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2978 i++;
2979 if (i == tx_ring->count)
2980 i = 0;
2981 }
2982
2983 tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
2984
e757e3e1
AD		 2985 tx_ring->tx_buffer_info[first].time_stamp = jiffies;
2986
2987 /* Force memory writes to complete before letting h/w
2988 * know there are new descriptors to fetch. (Only
2989 * applicable for weak-ordered memory model archs,
2990 * such as IA-64).
2991 */
2992 wmb();
2993
2994 tx_ring->tx_buffer_info[first].next_to_watch = tx_desc;
92915f71 2995 tx_ring->next_to_use = i;
92915f71
GR		 2996}
2997
fb40195c 2998static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
92915f71 2999{
fb40195c 3000 struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
92915f71 3001
fb40195c 3002 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
92915f71
GR		 3003 /* Herbert's original patch had:
3004 * smp_mb__after_netif_stop_queue();
3005 * but since that doesn't exist yet, just open code it. */
3006 smp_mb();
3007
3008 /* We need to check again in a case another CPU has just
3009 * made room available. */
3010 if (likely(IXGBE_DESC_UNUSED(tx_ring) < size))
3011 return -EBUSY;
3012
3013 /* A reprieve! - use start_queue because it doesn't call schedule */
fb40195c 3014 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
92915f71
GR		 3015 ++adapter->restart_queue;
3016 return 0;
3017}
3018
fb40195c 3019static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
92915f71
GR		 3020{
3021 if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size))
3022 return 0;
fb40195c 3023 return __ixgbevf_maybe_stop_tx(tx_ring, size);
92915f71
GR		 3024}
3025
3026static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
3027{
3028 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3029 struct ixgbevf_ring *tx_ring;
3030 unsigned int first;
3031 unsigned int tx_flags = 0;
3032 u8 hdr_len = 0;
3033 int r_idx = 0, tso;
3595990a
AD		 3034 u16 count = TXD_USE_COUNT(skb_headlen(skb));
3035#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3036 unsigned short f;
3037#endif
f9d08f16 3038 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL);
46acc460 3039 if (!dst_mac || is_link_local_ether_addr(dst_mac)) {
f9d08f16
GR		 3040 dev_kfree_skb(skb);
3041 return NETDEV_TX_OK;
3042 }
92915f71
GR		 3043
3044 tx_ring = &adapter->tx_ring[r_idx];
3045
3595990a
AD		 3046 /*
3047 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
3048 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
3049 * + 2 desc gap to keep tail from touching head,
3050 * + 1 desc for context descriptor,
3051 * otherwise try next time
3052 */
3053#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3054 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
3055 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
3056#else
3057 count += skb_shinfo(skb)->nr_frags;
3058#endif
fb40195c 3059 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
3595990a
AD		 3060 adapter->tx_busy++;
3061 return NETDEV_TX_BUSY;
3062 }
3063
eab6d18d 3064 if (vlan_tx_tag_present(skb)) {
92915f71
GR		 3065 tx_flags |= vlan_tx_tag_get(skb);
3066 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
3067 tx_flags |= IXGBE_TX_FLAGS_VLAN;
3068 }
3069
92915f71
GR		 3070 first = tx_ring->next_to_use;
3071
3072 if (skb->protocol == htons(ETH_P_IP))
3073 tx_flags |= IXGBE_TX_FLAGS_IPV4;
70a10e25 3074 tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len);
92915f71
GR		 3075 if (tso < 0) {
3076 dev_kfree_skb_any(skb);
3077 return NETDEV_TX_OK;
3078 }
3079
3080 if (tso)
70a10e25
AD		 3081 tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM;
3082 else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags))
92915f71
GR		 3083 tx_flags |= IXGBE_TX_FLAGS_CSUM;
3084
70a10e25 3085 ixgbevf_tx_queue(tx_ring, tx_flags,
e757e3e1
AD		 3086 ixgbevf_tx_map(tx_ring, skb, tx_flags),
3087 first, skb->len, hdr_len);
70a10e25
AD		 3088
3089 writel(tx_ring->next_to_use, adapter->hw.hw_addr + tx_ring->tail);
92915f71 3090
fb40195c 3091 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
92915f71
GR		 3092
3093 return NETDEV_TX_OK;
3094}
3095
92915f71
GR		 3096/**
3097 * ixgbevf_set_mac - Change the Ethernet Address of the NIC
3098 * @netdev: network interface device structure
3099 * @p: pointer to an address structure
3100 *
3101 * Returns 0 on success, negative on failure
3102 **/
3103static int ixgbevf_set_mac(struct net_device *netdev, void *p)
3104{
3105 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3106 struct ixgbe_hw *hw = &adapter->hw;
3107 struct sockaddr *addr = p;
3108
3109 if (!is_valid_ether_addr(addr->sa_data))
3110 return -EADDRNOTAVAIL;
3111
3112 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3113 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3114
55fdd45b 3115 spin_lock_bh(&adapter->mbx_lock);
1c55ed76 3116
92fe0bf7 3117 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
92915f71 3118
55fdd45b 3119 spin_unlock_bh(&adapter->mbx_lock);
1c55ed76 3120
92915f71
GR		 3121 return 0;
3122}
3123
3124/**
3125 * ixgbevf_change_mtu - Change the Maximum Transfer Unit
3126 * @netdev: network interface device structure
3127 * @new_mtu: new value for maximum frame size
3128 *
3129 * Returns 0 on success, negative on failure
3130 **/
3131static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
3132{
3133 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3134 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
69bfbec4 3135 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
69bfbec4 3136
56e94095
AD		 3137 switch (adapter->hw.api_version) {
3138 case ixgbe_mbox_api_11:
69bfbec4 3139 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
56e94095
AD		 3140 break;
3141 default:
3142 if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
3143 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3144 break;
3145 }
92915f71
GR		 3146
3147 /* MTU < 68 is an error and causes problems on some kernels */
69bfbec4 3148 if ((new_mtu < 68) || (max_frame > max_possible_frame))
92915f71
GR		 3149 return -EINVAL;
3150
3151 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
3152 netdev->mtu, new_mtu);
3153 /* must set new MTU before calling down or up */
3154 netdev->mtu = new_mtu;
3155
3156 if (netif_running(netdev))
3157 ixgbevf_reinit_locked(adapter);
3158
3159 return 0;
3160}
3161
0ac1e8ce 3162static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state)
92915f71
GR		 3163{
3164 struct net_device *netdev = pci_get_drvdata(pdev);
3165 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
0ac1e8ce
AD		 3166#ifdef CONFIG_PM
3167 int retval = 0;
3168#endif
92915f71
GR		 3169
3170 netif_device_detach(netdev);
3171
3172 if (netif_running(netdev)) {
0ac1e8ce 3173 rtnl_lock();
92915f71
GR		 3174 ixgbevf_down(adapter);
3175 ixgbevf_free_irq(adapter);
3176 ixgbevf_free_all_tx_resources(adapter);
3177 ixgbevf_free_all_rx_resources(adapter);
0ac1e8ce 3178 rtnl_unlock();
92915f71
GR		 3179 }
3180
0ac1e8ce 3181 ixgbevf_clear_interrupt_scheme(adapter);
92915f71 3182
0ac1e8ce
AD		 3183#ifdef CONFIG_PM
3184 retval = pci_save_state(pdev);
3185 if (retval)
3186 return retval;
92915f71 3187
0ac1e8ce 3188#endif
92915f71 3189 pci_disable_device(pdev);
0ac1e8ce
AD		 3190
3191 return 0;
3192}
3193
3194#ifdef CONFIG_PM
3195static int ixgbevf_resume(struct pci_dev *pdev)
3196{
3197 struct ixgbevf_adapter *adapter = pci_get_drvdata(pdev);
3198 struct net_device *netdev = adapter->netdev;
3199 u32 err;
3200
3201 pci_set_power_state(pdev, PCI_D0);
3202 pci_restore_state(pdev);
3203 /*
3204 * pci_restore_state clears dev->state_saved so call
3205 * pci_save_state to restore it.
3206 */
3207 pci_save_state(pdev);
3208
3209 err = pci_enable_device_mem(pdev);
3210 if (err) {
3211 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
3212 return err;
3213 }
3214 pci_set_master(pdev);
3215
3216 rtnl_lock();
3217 err = ixgbevf_init_interrupt_scheme(adapter);
3218 rtnl_unlock();
3219 if (err) {
3220 dev_err(&pdev->dev, "Cannot initialize interrupts\n");
3221 return err;
3222 }
3223
3224 ixgbevf_reset(adapter);
3225
3226 if (netif_running(netdev)) {
3227 err = ixgbevf_open(netdev);
3228 if (err)
3229 return err;
3230 }
3231
3232 netif_device_attach(netdev);
3233
3234 return err;
3235}
3236
3237#endif /* CONFIG_PM */
3238static void ixgbevf_shutdown(struct pci_dev *pdev)
3239{
3240 ixgbevf_suspend(pdev, PMSG_SUSPEND);
92915f71
GR		 3241}
3242
4197aa7b
ED		 3243static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3244 struct rtnl_link_stats64 *stats)
3245{
3246 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3247 unsigned int start;
3248 u64 bytes, packets;
3249 const struct ixgbevf_ring *ring;
3250 int i;
3251
3252 ixgbevf_update_stats(adapter);
3253
3254 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3255
3256 for (i = 0; i < adapter->num_rx_queues; i++) {
3257 ring = &adapter->rx_ring[i];
3258 do {
3259 start = u64_stats_fetch_begin_bh(&ring->syncp);
3260 bytes = ring->total_bytes;
3261 packets = ring->total_packets;
3262 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3263 stats->rx_bytes += bytes;
3264 stats->rx_packets += packets;
3265 }
3266
3267 for (i = 0; i < adapter->num_tx_queues; i++) {
3268 ring = &adapter->tx_ring[i];
3269 do {
3270 start = u64_stats_fetch_begin_bh(&ring->syncp);
3271 bytes = ring->total_bytes;
3272 packets = ring->total_packets;
3273 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3274 stats->tx_bytes += bytes;
3275 stats->tx_packets += packets;
3276 }
3277
3278 return stats;
3279}
3280
0ac1e8ce 3281static const struct net_device_ops ixgbevf_netdev_ops = {
c12db769
SH		 3282 .ndo_open = ixgbevf_open,
3283 .ndo_stop = ixgbevf_close,
3284 .ndo_start_xmit = ixgbevf_xmit_frame,
3285 .ndo_set_rx_mode = ixgbevf_set_rx_mode,
4197aa7b 3286 .ndo_get_stats64 = ixgbevf_get_stats,
92915f71 3287 .ndo_validate_addr = eth_validate_addr,
c12db769
SH		 3288 .ndo_set_mac_address = ixgbevf_set_mac,
3289 .ndo_change_mtu = ixgbevf_change_mtu,
3290 .ndo_tx_timeout = ixgbevf_tx_timeout,
c12db769
SH		 3291 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid,
3292 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid,
92915f71 3293};
92915f71
GR		 3294
3295static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3296{
0ac1e8ce 3297 dev->netdev_ops = &ixgbevf_netdev_ops;
92915f71
GR		 3298 ixgbevf_set_ethtool_ops(dev);
3299 dev->watchdog_timeo = 5 * HZ;
3300}
3301
3302/**
3303 * ixgbevf_probe - Device Initialization Routine
3304 * @pdev: PCI device information struct
3305 * @ent: entry in ixgbevf_pci_tbl
3306 *
3307 * Returns 0 on success, negative on failure
3308 *
3309 * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3310 * The OS initialization, configuring of the adapter private structure,
3311 * and a hardware reset occur.
3312 **/
1dd06ae8 3313static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
92915f71
GR		 3314{
3315 struct net_device *netdev;
3316 struct ixgbevf_adapter *adapter = NULL;
3317 struct ixgbe_hw *hw = NULL;
3318 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3319 static int cards_found;
3320 int err, pci_using_dac;
3321
3322 err = pci_enable_device(pdev);
3323 if (err)
3324 return err;
3325
2a1f8794
NN		 3326 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
3327 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
92915f71
GR		 3328 pci_using_dac = 1;
3329 } else {
2a1f8794 3330 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
92915f71 3331 if (err) {
2a1f8794
NN		 3332 err = dma_set_coherent_mask(&pdev->dev,
3333 DMA_BIT_MASK(32));
92915f71
GR		 3334 if (err) {
3335 dev_err(&pdev->dev, "No usable DMA "
3336 "configuration, aborting\n");
3337 goto err_dma;
3338 }
3339 }
3340 pci_using_dac = 0;
3341 }
3342
3343 err = pci_request_regions(pdev, ixgbevf_driver_name);
3344 if (err) {
3345 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3346 goto err_pci_reg;
3347 }
3348
3349 pci_set_master(pdev);
3350
92915f71
GR		 3351 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3352 MAX_TX_QUEUES);
92915f71
GR		 3353 if (!netdev) {
3354 err = -ENOMEM;
3355 goto err_alloc_etherdev;
3356 }
3357
3358 SET_NETDEV_DEV(netdev, &pdev->dev);
3359
3360 pci_set_drvdata(pdev, netdev);
3361 adapter = netdev_priv(netdev);
3362
3363 adapter->netdev = netdev;
3364 adapter->pdev = pdev;
3365 hw = &adapter->hw;
3366 hw->back = adapter;
b3f4d599 3367 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
92915f71
GR		 3368
3369 /*
3370 * call save state here in standalone driver because it relies on
3371 * adapter struct to exist, and needs to call netdev_priv
3372 */
3373 pci_save_state(pdev);
3374
3375 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3376 pci_resource_len(pdev, 0));
3377 if (!hw->hw_addr) {
3378 err = -EIO;
3379 goto err_ioremap;
3380 }
3381
3382 ixgbevf_assign_netdev_ops(netdev);
3383
3384 adapter->bd_number = cards_found;
3385
3386 /* Setup hw api */
3387 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3388 hw->mac.type = ii->mac;
3389
3390 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
f416dfc0 3391 sizeof(struct ixgbe_mbx_operations));
92915f71 3392
92915f71
GR		 3393 /* setup the private structure */
3394 err = ixgbevf_sw_init(adapter);
1a0d6ae5
DK		 3395 if (err)
3396 goto err_sw_init;
3397
3398 /* The HW MAC address was set and/or determined in sw_init */
1a0d6ae5
DK		 3399 if (!is_valid_ether_addr(netdev->dev_addr)) {
3400 pr_err("invalid MAC address\n");
3401 err = -EIO;
3402 goto err_sw_init;
3403 }
92915f71 3404
471a76de 3405 netdev->hw_features = NETIF_F_SG |
92915f71 3406 NETIF_F_IP_CSUM |
471a76de
MM		 3407 NETIF_F_IPV6_CSUM |
3408 NETIF_F_TSO |
3409 NETIF_F_TSO6 |
3410 NETIF_F_RXCSUM;
3411
3412 netdev->features = netdev->hw_features |
f646968f
PM		 3413 NETIF_F_HW_VLAN_CTAG_TX |
3414 NETIF_F_HW_VLAN_CTAG_RX |
3415 NETIF_F_HW_VLAN_CTAG_FILTER;
92915f71 3416
92915f71
GR		 3417 netdev->vlan_features |= NETIF_F_TSO;
3418 netdev->vlan_features |= NETIF_F_TSO6;
3419 netdev->vlan_features |= NETIF_F_IP_CSUM;
3bfacf96 3420 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
92915f71
GR		 3421 netdev->vlan_features |= NETIF_F_SG;
3422
3423 if (pci_using_dac)
3424 netdev->features |= NETIF_F_HIGHDMA;
3425
01789349
JP		 3426 netdev->priv_flags |= IFF_UNICAST_FLT;
3427
92915f71 3428 init_timer(&adapter->watchdog_timer);
c061b18d 3429 adapter->watchdog_timer.function = ixgbevf_watchdog;
92915f71
GR		 3430 adapter->watchdog_timer.data = (unsigned long)adapter;
3431
3432 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3433 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3434
3435 err = ixgbevf_init_interrupt_scheme(adapter);
3436 if (err)
3437 goto err_sw_init;
3438
92915f71
GR		 3439 strcpy(netdev->name, "eth%d");
3440
3441 err = register_netdev(netdev);
3442 if (err)
3443 goto err_register;
3444
5d426ad1
GR		 3445 netif_carrier_off(netdev);
3446
33bd9f60
GR		 3447 ixgbevf_init_last_counter_stats(adapter);
3448
92915f71 3449 /* print the MAC address */
f794e7ef 3450 hw_dbg(hw, "%pM\n", netdev->dev_addr);
92915f71
GR		 3451
3452 hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3453
92915f71
GR		 3454 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3455 cards_found++;
3456 return 0;
3457
3458err_register:
0ac1e8ce 3459 ixgbevf_clear_interrupt_scheme(adapter);
92915f71
GR		 3460err_sw_init:
3461 ixgbevf_reset_interrupt_capability(adapter);
3462 iounmap(hw->hw_addr);
3463err_ioremap:
3464 free_netdev(netdev);
3465err_alloc_etherdev:
3466 pci_release_regions(pdev);
3467err_pci_reg:
3468err_dma:
3469 pci_disable_device(pdev);
3470 return err;
3471}
3472
3473/**
3474 * ixgbevf_remove - Device Removal Routine
3475 * @pdev: PCI device information struct
3476 *
3477 * ixgbevf_remove is called by the PCI subsystem to alert the driver
3478 * that it should release a PCI device. The could be caused by a
3479 * Hot-Plug event, or because the driver is going to be removed from
3480 * memory.
3481 **/
9f9a12f8 3482static void ixgbevf_remove(struct pci_dev *pdev)
92915f71
GR		 3483{
3484 struct net_device *netdev = pci_get_drvdata(pdev);
3485 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3486
3487 set_bit(__IXGBEVF_DOWN, &adapter->state);
3488
3489 del_timer_sync(&adapter->watchdog_timer);
3490
23f333a2 3491 cancel_work_sync(&adapter->reset_task);
92915f71
GR		 3492 cancel_work_sync(&adapter->watchdog_task);
3493
fd13a9ab 3494 if (netdev->reg_state == NETREG_REGISTERED)
92915f71 3495 unregister_netdev(netdev);
92915f71 3496
0ac1e8ce 3497 ixgbevf_clear_interrupt_scheme(adapter);
92915f71
GR		 3498 ixgbevf_reset_interrupt_capability(adapter);
3499
3500 iounmap(adapter->hw.hw_addr);
3501 pci_release_regions(pdev);
3502
3503 hw_dbg(&adapter->hw, "Remove complete\n");
3504
3505 kfree(adapter->tx_ring);
3506 kfree(adapter->rx_ring);
3507
3508 free_netdev(netdev);
3509
3510 pci_disable_device(pdev);
3511}
3512
9f19f31d
AD		 3513/**
3514 * ixgbevf_io_error_detected - called when PCI error is detected
3515 * @pdev: Pointer to PCI device
3516 * @state: The current pci connection state
3517 *
3518 * This function is called after a PCI bus error affecting
3519 * this device has been detected.
3520 */
3521static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
3522 pci_channel_state_t state)
3523{
3524 struct net_device *netdev = pci_get_drvdata(pdev);
3525 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3526
3527 netif_device_detach(netdev);
3528
3529 if (state == pci_channel_io_perm_failure)
3530 return PCI_ERS_RESULT_DISCONNECT;
3531
3532 if (netif_running(netdev))
3533 ixgbevf_down(adapter);
3534
3535 pci_disable_device(pdev);
3536
3537 /* Request a slot slot reset. */
3538 return PCI_ERS_RESULT_NEED_RESET;
3539}
3540
3541/**
3542 * ixgbevf_io_slot_reset - called after the pci bus has been reset.
3543 * @pdev: Pointer to PCI device
3544 *
3545 * Restart the card from scratch, as if from a cold-boot. Implementation
3546 * resembles the first-half of the ixgbevf_resume routine.
3547 */
3548static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
3549{
3550 struct net_device *netdev = pci_get_drvdata(pdev);
3551 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3552
3553 if (pci_enable_device_mem(pdev)) {
3554 dev_err(&pdev->dev,
3555 "Cannot re-enable PCI device after reset.\n");
3556 return PCI_ERS_RESULT_DISCONNECT;
3557 }
3558
3559 pci_set_master(pdev);
3560
3561 ixgbevf_reset(adapter);
3562
3563 return PCI_ERS_RESULT_RECOVERED;
3564}
3565
3566/**
3567 * ixgbevf_io_resume - called when traffic can start flowing again.
3568 * @pdev: Pointer to PCI device
3569 *
3570 * This callback is called when the error recovery driver tells us that
3571 * its OK to resume normal operation. Implementation resembles the
3572 * second-half of the ixgbevf_resume routine.
3573 */
3574static void ixgbevf_io_resume(struct pci_dev *pdev)
3575{
3576 struct net_device *netdev = pci_get_drvdata(pdev);
3577 struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3578
3579 if (netif_running(netdev))
3580 ixgbevf_up(adapter);
3581
3582 netif_device_attach(netdev);
3583}
3584
3585/* PCI Error Recovery (ERS) */
3646f0e5 3586static const struct pci_error_handlers ixgbevf_err_handler = {
9f19f31d
AD		 3587 .error_detected = ixgbevf_io_error_detected,
3588 .slot_reset = ixgbevf_io_slot_reset,
3589 .resume = ixgbevf_io_resume,
3590};
3591
92915f71
GR		 3592static struct pci_driver ixgbevf_driver = {
3593 .name = ixgbevf_driver_name,
3594 .id_table = ixgbevf_pci_tbl,
3595 .probe = ixgbevf_probe,
9f9a12f8 3596 .remove = ixgbevf_remove,
0ac1e8ce
AD		 3597#ifdef CONFIG_PM
3598 /* Power Management Hooks */
3599 .suspend = ixgbevf_suspend,
3600 .resume = ixgbevf_resume,
3601#endif
92915f71 3602 .shutdown = ixgbevf_shutdown,
9f19f31d 3603 .err_handler = &ixgbevf_err_handler
92915f71
GR		 3604};
3605
3606/**
65d676c8 3607 * ixgbevf_init_module - Driver Registration Routine
92915f71 3608 *
65d676c8 3609 * ixgbevf_init_module is the first routine called when the driver is
92915f71
GR		 3610 * loaded. All it does is register with the PCI subsystem.
3611 **/
3612static int __init ixgbevf_init_module(void)
3613{
3614 int ret;
dbd9636e
JK		 3615 pr_info("%s - version %s\n", ixgbevf_driver_string,
3616 ixgbevf_driver_version);
92915f71 3617
dbd9636e 3618 pr_info("%s\n", ixgbevf_copyright);
92915f71
GR		 3619
3620 ret = pci_register_driver(&ixgbevf_driver);
3621 return ret;
3622}
3623
3624module_init(ixgbevf_init_module);
3625
3626/**
65d676c8 3627 * ixgbevf_exit_module - Driver Exit Cleanup Routine
92915f71 3628 *
65d676c8 3629 * ixgbevf_exit_module is called just before the driver is removed
92915f71
GR		 3630 * from memory.
3631 **/
3632static void __exit ixgbevf_exit_module(void)
3633{
3634 pci_unregister_driver(&ixgbevf_driver);
3635}
3636
3637#ifdef DEBUG
3638/**
65d676c8 3639 * ixgbevf_get_hw_dev_name - return device name string
92915f71
GR		 3640 * used by hardware layer to print debugging information
3641 **/
3642char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3643{
3644 struct ixgbevf_adapter *adapter = hw->back;
3645 return adapter->netdev->name;
3646}
3647
3648#endif
3649module_exit(ixgbevf_exit_module);
3650
3651/* ixgbevf_main.c */
Linux kernel - Deliverables
This page took 0.598703 seconds and 5 git commands to generate.