1 /*******************************************************************************
3 Intel 10 Gigabit PCI Express Linux driver
4 Copyright(c) 1999 - 2007 Intel Corporation.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
23 Linux NICS <linux.nics@intel.com>
24 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
25 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
27 *******************************************************************************/
29 #include <linux/types.h>
30 #include <linux/module.h>
31 #include <linux/pci.h>
32 #include <linux/netdevice.h>
33 #include <linux/vmalloc.h>
34 #include <linux/string.h>
37 #include <linux/tcp.h>
38 #include <linux/ipv6.h>
39 #include <net/checksum.h>
40 #include <net/ip6_checksum.h>
41 #include <linux/ethtool.h>
42 #include <linux/if_vlan.h>
45 #include "ixgbe_common.h"
47 char ixgbe_driver_name
[] = "ixgbe";
48 static const char ixgbe_driver_string
[] =
49 "Intel(R) 10 Gigabit PCI Express Network Driver";
51 #define DRV_VERSION "1.1.18"
52 const char ixgbe_driver_version
[] = DRV_VERSION
;
53 static const char ixgbe_copyright
[] =
54 "Copyright (c) 1999-2007 Intel Corporation.";
56 static const struct ixgbe_info
*ixgbe_info_tbl
[] = {
57 [board_82598AF
] = &ixgbe_82598AF_info
,
58 [board_82598EB
] = &ixgbe_82598EB_info
,
59 [board_82598AT
] = &ixgbe_82598AT_info
,
62 /* ixgbe_pci_tbl - PCI Device ID Table
64 * Wildcard entries (PCI_ANY_ID) should come last
65 * Last entry must be all 0s
67 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
68 * Class, Class Mask, private data (not used) }
70 static struct pci_device_id ixgbe_pci_tbl
[] = {
71 {PCI_VDEVICE(INTEL
, IXGBE_DEV_ID_82598AF_DUAL_PORT
),
73 {PCI_VDEVICE(INTEL
, IXGBE_DEV_ID_82598AF_SINGLE_PORT
),
75 {PCI_VDEVICE(INTEL
, IXGBE_DEV_ID_82598AT_DUAL_PORT
),
77 {PCI_VDEVICE(INTEL
, IXGBE_DEV_ID_82598EB_CX4
),
80 /* required last entry */
83 MODULE_DEVICE_TABLE(pci
, ixgbe_pci_tbl
);
85 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
86 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
87 MODULE_LICENSE("GPL");
88 MODULE_VERSION(DRV_VERSION
);
90 #define DEFAULT_DEBUG_LEVEL_SHIFT 3
95 * ixgbe_get_hw_dev_name - return device name string
96 * used by hardware layer to print debugging information
98 char *ixgbe_get_hw_dev_name(struct ixgbe_hw
*hw
)
100 struct ixgbe_adapter
*adapter
= hw
->back
;
101 struct net_device
*netdev
= adapter
->netdev
;
106 static void ixgbe_set_ivar(struct ixgbe_adapter
*adapter
, u16 int_alloc_entry
,
111 msix_vector
|= IXGBE_IVAR_ALLOC_VAL
;
112 index
= (int_alloc_entry
>> 2) & 0x1F;
113 ivar
= IXGBE_READ_REG(&adapter
->hw
, IXGBE_IVAR(index
));
114 ivar
&= ~(0xFF << (8 * (int_alloc_entry
& 0x3)));
115 ivar
|= (msix_vector
<< (8 * (int_alloc_entry
& 0x3)));
116 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_IVAR(index
), ivar
);
119 static void ixgbe_unmap_and_free_tx_resource(struct ixgbe_adapter
*adapter
,
120 struct ixgbe_tx_buffer
123 if (tx_buffer_info
->dma
) {
124 pci_unmap_page(adapter
->pdev
,
126 tx_buffer_info
->length
, PCI_DMA_TODEVICE
);
127 tx_buffer_info
->dma
= 0;
129 if (tx_buffer_info
->skb
) {
130 dev_kfree_skb_any(tx_buffer_info
->skb
);
131 tx_buffer_info
->skb
= NULL
;
133 /* tx_buffer_info must be completely set up in the transmit path */
136 static inline bool ixgbe_check_tx_hang(struct ixgbe_adapter
*adapter
,
137 struct ixgbe_ring
*tx_ring
,
139 union ixgbe_adv_tx_desc
*eop_desc
)
141 /* Detect a transmit hang in hardware, this serializes the
142 * check with the clearing of time_stamp and movement of i */
143 adapter
->detect_tx_hung
= false;
144 if (tx_ring
->tx_buffer_info
[eop
].dma
&&
145 time_after(jiffies
, tx_ring
->tx_buffer_info
[eop
].time_stamp
+ HZ
) &&
146 !(IXGBE_READ_REG(&adapter
->hw
, IXGBE_TFCS
) & IXGBE_TFCS_TXOFF
)) {
147 /* detected Tx unit hang */
148 DPRINTK(DRV
, ERR
, "Detected Tx Unit Hang\n"
151 " next_to_use <%x>\n"
152 " next_to_clean <%x>\n"
153 "tx_buffer_info[next_to_clean]\n"
154 " time_stamp <%lx>\n"
155 " next_to_watch <%x>\n"
157 " next_to_watch.status <%x>\n",
158 readl(adapter
->hw
.hw_addr
+ tx_ring
->head
),
159 readl(adapter
->hw
.hw_addr
+ tx_ring
->tail
),
160 tx_ring
->next_to_use
,
161 tx_ring
->next_to_clean
,
162 tx_ring
->tx_buffer_info
[eop
].time_stamp
,
163 eop
, jiffies
, eop_desc
->wb
.status
);
171 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
172 * @adapter: board private structure
174 static bool ixgbe_clean_tx_irq(struct ixgbe_adapter
*adapter
,
175 struct ixgbe_ring
*tx_ring
)
177 struct net_device
*netdev
= adapter
->netdev
;
178 union ixgbe_adv_tx_desc
*tx_desc
, *eop_desc
;
179 struct ixgbe_tx_buffer
*tx_buffer_info
;
181 bool cleaned
= false;
184 i
= tx_ring
->next_to_clean
;
185 eop
= tx_ring
->tx_buffer_info
[i
].next_to_watch
;
186 eop_desc
= IXGBE_TX_DESC_ADV(*tx_ring
, eop
);
187 while (eop_desc
->wb
.status
& cpu_to_le32(IXGBE_TXD_STAT_DD
)) {
188 for (cleaned
= false; !cleaned
;) {
189 tx_desc
= IXGBE_TX_DESC_ADV(*tx_ring
, i
);
190 tx_buffer_info
= &tx_ring
->tx_buffer_info
[i
];
191 cleaned
= (i
== eop
);
193 tx_ring
->stats
.bytes
+= tx_buffer_info
->length
;
194 ixgbe_unmap_and_free_tx_resource(adapter
,
196 tx_desc
->wb
.status
= 0;
199 if (i
== tx_ring
->count
)
203 tx_ring
->stats
.packets
++;
205 eop
= tx_ring
->tx_buffer_info
[i
].next_to_watch
;
206 eop_desc
= IXGBE_TX_DESC_ADV(*tx_ring
, eop
);
208 /* weight of a sort for tx, avoid endless transmit cleanup */
209 if (count
++ >= tx_ring
->work_limit
)
213 tx_ring
->next_to_clean
= i
;
215 #define TX_WAKE_THRESHOLD 32
216 spin_lock(&tx_ring
->tx_lock
);
218 if (cleaned
&& netif_carrier_ok(netdev
) &&
219 (IXGBE_DESC_UNUSED(tx_ring
) >= TX_WAKE_THRESHOLD
) &&
220 !test_bit(__IXGBE_DOWN
, &adapter
->state
))
221 netif_wake_queue(netdev
);
223 spin_unlock(&tx_ring
->tx_lock
);
225 if (adapter
->detect_tx_hung
)
226 if (ixgbe_check_tx_hang(adapter
, tx_ring
, eop
, eop_desc
))
227 netif_stop_queue(netdev
);
229 if (count
>= tx_ring
->work_limit
)
230 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_EICS
, tx_ring
->eims_value
);
236 * ixgbe_receive_skb - Send a completed packet up the stack
237 * @adapter: board private structure
238 * @skb: packet to send up
239 * @is_vlan: packet has a VLAN tag
240 * @tag: VLAN tag from descriptor
242 static void ixgbe_receive_skb(struct ixgbe_adapter
*adapter
,
243 struct sk_buff
*skb
, bool is_vlan
,
246 if (!(adapter
->flags
& IXGBE_FLAG_IN_NETPOLL
)) {
247 if (adapter
->vlgrp
&& is_vlan
)
248 vlan_hwaccel_receive_skb(skb
, adapter
->vlgrp
, tag
);
250 netif_receive_skb(skb
);
253 if (adapter
->vlgrp
&& is_vlan
)
254 vlan_hwaccel_rx(skb
, adapter
->vlgrp
, tag
);
260 static inline void ixgbe_rx_checksum(struct ixgbe_adapter
*adapter
,
264 skb
->ip_summed
= CHECKSUM_NONE
;
266 /* Ignore Checksum bit is set */
267 if ((status_err
& IXGBE_RXD_STAT_IXSM
) ||
268 !(adapter
->flags
& IXGBE_FLAG_RX_CSUM_ENABLED
))
270 /* TCP/UDP checksum error bit is set */
271 if (status_err
& (IXGBE_RXDADV_ERR_TCPE
| IXGBE_RXDADV_ERR_IPE
)) {
272 /* let the stack verify checksum errors */
273 adapter
->hw_csum_rx_error
++;
276 /* It must be a TCP or UDP packet with a valid checksum */
277 if (status_err
& (IXGBE_RXD_STAT_L4CS
| IXGBE_RXD_STAT_UDPCS
))
278 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
279 adapter
->hw_csum_rx_good
++;
283 * ixgbe_alloc_rx_buffers - Replace used receive buffers; packet split
284 * @adapter: address of board private structure
286 static void ixgbe_alloc_rx_buffers(struct ixgbe_adapter
*adapter
,
287 struct ixgbe_ring
*rx_ring
,
290 struct net_device
*netdev
= adapter
->netdev
;
291 struct pci_dev
*pdev
= adapter
->pdev
;
292 union ixgbe_adv_rx_desc
*rx_desc
;
293 struct ixgbe_rx_buffer
*rx_buffer_info
;
296 unsigned int bufsz
= adapter
->rx_buf_len
+ NET_IP_ALIGN
;
298 i
= rx_ring
->next_to_use
;
299 rx_buffer_info
= &rx_ring
->rx_buffer_info
[i
];
301 while (cleaned_count
--) {
302 rx_desc
= IXGBE_RX_DESC_ADV(*rx_ring
, i
);
304 if (!rx_buffer_info
->page
&&
305 (adapter
->flags
& IXGBE_FLAG_RX_PS_ENABLED
)) {
306 rx_buffer_info
->page
= alloc_page(GFP_ATOMIC
);
307 if (!rx_buffer_info
->page
) {
308 adapter
->alloc_rx_page_failed
++;
311 rx_buffer_info
->page_dma
=
312 pci_map_page(pdev
, rx_buffer_info
->page
,
313 0, PAGE_SIZE
, PCI_DMA_FROMDEVICE
);
316 if (!rx_buffer_info
->skb
) {
317 skb
= netdev_alloc_skb(netdev
, bufsz
);
320 adapter
->alloc_rx_buff_failed
++;
325 * Make buffer alignment 2 beyond a 16 byte boundary
326 * this will result in a 16 byte aligned IP header after
327 * the 14 byte MAC header is removed
329 skb_reserve(skb
, NET_IP_ALIGN
);
331 rx_buffer_info
->skb
= skb
;
332 rx_buffer_info
->dma
= pci_map_single(pdev
, skb
->data
,
336 /* Refresh the desc even if buffer_addrs didn't change because
337 * each write-back erases this info. */
338 if (adapter
->flags
& IXGBE_FLAG_RX_PS_ENABLED
) {
339 rx_desc
->read
.pkt_addr
=
340 cpu_to_le64(rx_buffer_info
->page_dma
);
341 rx_desc
->read
.hdr_addr
=
342 cpu_to_le64(rx_buffer_info
->dma
);
344 rx_desc
->read
.pkt_addr
=
345 cpu_to_le64(rx_buffer_info
->dma
);
349 if (i
== rx_ring
->count
)
351 rx_buffer_info
= &rx_ring
->rx_buffer_info
[i
];
354 if (rx_ring
->next_to_use
!= i
) {
355 rx_ring
->next_to_use
= i
;
357 i
= (rx_ring
->count
- 1);
360 * Force memory writes to complete before letting h/w
361 * know there are new descriptors to fetch. (Only
362 * applicable for weak-ordered memory model archs,
366 writel(i
, adapter
->hw
.hw_addr
+ rx_ring
->tail
);
370 static bool ixgbe_clean_rx_irq(struct ixgbe_adapter
*adapter
,
371 struct ixgbe_ring
*rx_ring
,
372 int *work_done
, int work_to_do
)
374 struct net_device
*netdev
= adapter
->netdev
;
375 struct pci_dev
*pdev
= adapter
->pdev
;
376 union ixgbe_adv_rx_desc
*rx_desc
, *next_rxd
;
377 struct ixgbe_rx_buffer
*rx_buffer_info
, *next_buffer
;
380 u32 upper_len
, len
, staterr
;
381 u16 hdr_info
, vlan_tag
;
382 bool is_vlan
, cleaned
= false;
383 int cleaned_count
= 0;
385 i
= rx_ring
->next_to_clean
;
387 rx_desc
= IXGBE_RX_DESC_ADV(*rx_ring
, i
);
388 staterr
= le32_to_cpu(rx_desc
->wb
.upper
.status_error
);
389 rx_buffer_info
= &rx_ring
->rx_buffer_info
[i
];
390 is_vlan
= (staterr
& IXGBE_RXD_STAT_VP
);
391 vlan_tag
= le16_to_cpu(rx_desc
->wb
.upper
.vlan
);
393 while (staterr
& IXGBE_RXD_STAT_DD
) {
394 if (*work_done
>= work_to_do
)
398 if (adapter
->flags
& IXGBE_FLAG_RX_PS_ENABLED
) {
400 le16_to_cpu(rx_desc
->wb
.lower
.lo_dword
.hdr_info
);
402 ((hdr_info
& IXGBE_RXDADV_HDRBUFLEN_MASK
) >>
403 IXGBE_RXDADV_HDRBUFLEN_SHIFT
);
404 if (hdr_info
& IXGBE_RXDADV_SPH
)
405 adapter
->rx_hdr_split
++;
406 if (len
> IXGBE_RX_HDR_SIZE
)
407 len
= IXGBE_RX_HDR_SIZE
;
408 upper_len
= le16_to_cpu(rx_desc
->wb
.upper
.length
);
410 len
= le16_to_cpu(rx_desc
->wb
.upper
.length
);
413 skb
= rx_buffer_info
->skb
;
414 prefetch(skb
->data
- NET_IP_ALIGN
);
415 rx_buffer_info
->skb
= NULL
;
417 if (len
&& !skb_shinfo(skb
)->nr_frags
) {
418 pci_unmap_single(pdev
, rx_buffer_info
->dma
,
419 adapter
->rx_buf_len
+ NET_IP_ALIGN
,
425 pci_unmap_page(pdev
, rx_buffer_info
->page_dma
,
426 PAGE_SIZE
, PCI_DMA_FROMDEVICE
);
427 rx_buffer_info
->page_dma
= 0;
428 skb_fill_page_desc(skb
, skb_shinfo(skb
)->nr_frags
,
429 rx_buffer_info
->page
, 0, upper_len
);
430 rx_buffer_info
->page
= NULL
;
432 skb
->len
+= upper_len
;
433 skb
->data_len
+= upper_len
;
434 skb
->truesize
+= upper_len
;
438 if (i
== rx_ring
->count
)
440 next_buffer
= &rx_ring
->rx_buffer_info
[i
];
442 next_rxd
= IXGBE_RX_DESC_ADV(*rx_ring
, i
);
446 if (staterr
& IXGBE_RXD_STAT_EOP
) {
447 rx_ring
->stats
.packets
++;
448 rx_ring
->stats
.bytes
+= skb
->len
;
450 rx_buffer_info
->skb
= next_buffer
->skb
;
451 rx_buffer_info
->dma
= next_buffer
->dma
;
452 next_buffer
->skb
= skb
;
453 adapter
->non_eop_descs
++;
457 if (staterr
& IXGBE_RXDADV_ERR_FRAME_ERR_MASK
) {
458 dev_kfree_skb_irq(skb
);
462 ixgbe_rx_checksum(adapter
, staterr
, skb
);
463 skb
->protocol
= eth_type_trans(skb
, netdev
);
464 ixgbe_receive_skb(adapter
, skb
, is_vlan
, vlan_tag
);
465 netdev
->last_rx
= jiffies
;
468 rx_desc
->wb
.upper
.status_error
= 0;
470 /* return some buffers to hardware, one at a time is too slow */
471 if (cleaned_count
>= IXGBE_RX_BUFFER_WRITE
) {
472 ixgbe_alloc_rx_buffers(adapter
, rx_ring
, cleaned_count
);
476 /* use prefetched values */
478 rx_buffer_info
= next_buffer
;
480 staterr
= le32_to_cpu(rx_desc
->wb
.upper
.status_error
);
481 is_vlan
= (staterr
& IXGBE_RXD_STAT_VP
);
482 vlan_tag
= le16_to_cpu(rx_desc
->wb
.upper
.vlan
);
485 rx_ring
->next_to_clean
= i
;
486 cleaned_count
= IXGBE_DESC_UNUSED(rx_ring
);
489 ixgbe_alloc_rx_buffers(adapter
, rx_ring
, cleaned_count
);
494 #define IXGBE_MAX_INTR 10
496 * ixgbe_configure_msix - Configure MSI-X hardware
497 * @adapter: board private structure
499 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
502 static void ixgbe_configure_msix(struct ixgbe_adapter
*adapter
)
506 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
507 ixgbe_set_ivar(adapter
, IXGBE_IVAR_TX_QUEUE(i
),
508 IXGBE_MSIX_VECTOR(vector
));
509 writel(EITR_INTS_PER_SEC_TO_REG(adapter
->tx_eitr
),
510 adapter
->hw
.hw_addr
+ adapter
->tx_ring
[i
].itr_register
);
514 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
515 ixgbe_set_ivar(adapter
, IXGBE_IVAR_RX_QUEUE(i
),
516 IXGBE_MSIX_VECTOR(vector
));
517 writel(EITR_INTS_PER_SEC_TO_REG(adapter
->rx_eitr
),
518 adapter
->hw
.hw_addr
+ adapter
->rx_ring
[i
].itr_register
);
522 vector
= adapter
->num_tx_queues
+ adapter
->num_rx_queues
;
523 ixgbe_set_ivar(adapter
, IXGBE_IVAR_OTHER_CAUSES_INDEX
,
524 IXGBE_MSIX_VECTOR(vector
));
525 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_EITR(vector
), 1950);
528 static irqreturn_t
ixgbe_msix_lsc(int irq
, void *data
)
530 struct net_device
*netdev
= data
;
531 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
532 struct ixgbe_hw
*hw
= &adapter
->hw
;
533 u32 eicr
= IXGBE_READ_REG(hw
, IXGBE_EICR
);
535 if (eicr
& IXGBE_EICR_LSC
) {
537 if (!test_bit(__IXGBE_DOWN
, &adapter
->state
))
538 mod_timer(&adapter
->watchdog_timer
, jiffies
);
540 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_EIMS
, IXGBE_EIMS_OTHER
);
545 static irqreturn_t
ixgbe_msix_clean_tx(int irq
, void *data
)
547 struct ixgbe_ring
*txr
= data
;
548 struct ixgbe_adapter
*adapter
= txr
->adapter
;
550 ixgbe_clean_tx_irq(adapter
, txr
);
555 static irqreturn_t
ixgbe_msix_clean_rx(int irq
, void *data
)
557 struct ixgbe_ring
*rxr
= data
;
558 struct ixgbe_adapter
*adapter
= rxr
->adapter
;
560 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_EIMC
, rxr
->eims_value
);
561 netif_rx_schedule(adapter
->netdev
, &adapter
->napi
);
565 static int ixgbe_clean_rxonly(struct napi_struct
*napi
, int budget
)
567 struct ixgbe_adapter
*adapter
= container_of(napi
,
568 struct ixgbe_adapter
, napi
);
569 struct net_device
*netdev
= adapter
->netdev
;
571 struct ixgbe_ring
*rxr
= adapter
->rx_ring
;
573 /* Keep link state information with original netdev */
574 if (!netif_carrier_ok(netdev
))
577 ixgbe_clean_rx_irq(adapter
, rxr
, &work_done
, budget
);
579 /* If no Tx and not enough Rx work done, exit the polling mode */
580 if ((work_done
< budget
) || !netif_running(netdev
)) {
582 netif_rx_complete(netdev
, napi
);
583 if (!test_bit(__IXGBE_DOWN
, &adapter
->state
))
584 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_EIMS
,
592 * ixgbe_setup_msix - Initialize MSI-X interrupts
594 * ixgbe_setup_msix allocates MSI-X vectors and requests
595 * interrutps from the kernel.
597 static int ixgbe_setup_msix(struct ixgbe_adapter
*adapter
)
599 struct net_device
*netdev
= adapter
->netdev
;
600 int i
, int_vector
= 0, err
= 0;
603 /* +1 for the LSC interrupt */
604 max_msix_count
= adapter
->num_rx_queues
+ adapter
->num_tx_queues
+ 1;
605 adapter
->msix_entries
= kcalloc(max_msix_count
,
606 sizeof(struct msix_entry
), GFP_KERNEL
);
607 if (!adapter
->msix_entries
)
610 for (i
= 0; i
< max_msix_count
; i
++)
611 adapter
->msix_entries
[i
].entry
= i
;
613 err
= pci_enable_msix(adapter
->pdev
, adapter
->msix_entries
,
618 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
619 sprintf(adapter
->tx_ring
[i
].name
, "%s-tx%d", netdev
->name
, i
);
620 err
= request_irq(adapter
->msix_entries
[int_vector
].vector
,
621 &ixgbe_msix_clean_tx
,
623 adapter
->tx_ring
[i
].name
,
624 &(adapter
->tx_ring
[i
]));
627 "request_irq failed for MSIX interrupt "
631 adapter
->tx_ring
[i
].eims_value
=
632 (1 << IXGBE_MSIX_VECTOR(int_vector
));
633 adapter
->tx_ring
[i
].itr_register
= IXGBE_EITR(int_vector
);
637 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
638 if (strlen(netdev
->name
) < (IFNAMSIZ
- 5))
639 sprintf(adapter
->rx_ring
[i
].name
,
640 "%s-rx%d", netdev
->name
, i
);
642 memcpy(adapter
->rx_ring
[i
].name
,
643 netdev
->name
, IFNAMSIZ
);
644 err
= request_irq(adapter
->msix_entries
[int_vector
].vector
,
645 &ixgbe_msix_clean_rx
, 0,
646 adapter
->rx_ring
[i
].name
,
647 &(adapter
->rx_ring
[i
]));
650 "request_irq failed for MSIX interrupt "
655 adapter
->rx_ring
[i
].eims_value
=
656 (1 << IXGBE_MSIX_VECTOR(int_vector
));
657 adapter
->rx_ring
[i
].itr_register
= IXGBE_EITR(int_vector
);
661 sprintf(adapter
->lsc_name
, "%s-lsc", netdev
->name
);
662 err
= request_irq(adapter
->msix_entries
[int_vector
].vector
,
663 &ixgbe_msix_lsc
, 0, adapter
->lsc_name
, netdev
);
666 "request_irq for msix_lsc failed: %d\n", err
);
670 /* FIXME: implement netif_napi_remove() instead */
671 adapter
->napi
.poll
= ixgbe_clean_rxonly
;
672 adapter
->flags
|= IXGBE_FLAG_MSIX_ENABLED
;
677 for (; int_vector
>= adapter
->num_tx_queues
; int_vector
--)
678 free_irq(adapter
->msix_entries
[int_vector
].vector
,
679 &(adapter
->rx_ring
[int_vector
-
680 adapter
->num_tx_queues
]));
682 for (; int_vector
>= 0; int_vector
--)
683 free_irq(adapter
->msix_entries
[int_vector
].vector
,
684 &(adapter
->tx_ring
[int_vector
]));
686 kfree(adapter
->msix_entries
);
687 adapter
->msix_entries
= NULL
;
688 adapter
->flags
&= ~IXGBE_FLAG_MSIX_ENABLED
;
693 * ixgbe_intr - Interrupt Handler
694 * @irq: interrupt number
695 * @data: pointer to a network interface device structure
696 * @pt_regs: CPU registers structure
698 static irqreturn_t
ixgbe_intr(int irq
, void *data
)
700 struct net_device
*netdev
= data
;
701 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
702 struct ixgbe_hw
*hw
= &adapter
->hw
;
705 eicr
= IXGBE_READ_REG(hw
, IXGBE_EICR
);
708 return IRQ_NONE
; /* Not our interrupt */
710 if (eicr
& IXGBE_EICR_LSC
) {
712 if (!test_bit(__IXGBE_DOWN
, &adapter
->state
))
713 mod_timer(&adapter
->watchdog_timer
, jiffies
);
715 if (netif_rx_schedule_prep(netdev
, &adapter
->napi
)) {
716 /* Disable interrupts and register for poll. The flush of the
717 * posted write is intentionally left out. */
718 atomic_inc(&adapter
->irq_sem
);
719 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_EIMC
, ~0);
720 __netif_rx_schedule(netdev
, &adapter
->napi
);
727 * ixgbe_request_irq - initialize interrupts
728 * @adapter: board private structure
730 * Attempts to configure interrupts using the best available
731 * capabilities of the hardware and kernel.
733 static int ixgbe_request_irq(struct ixgbe_adapter
*adapter
, u32
*num_rx_queues
)
735 struct net_device
*netdev
= adapter
->netdev
;
737 irqreturn_t(*handler
) (int, void *) = &ixgbe_intr
;
741 err
= ixgbe_setup_msix(adapter
);
746 * if we can't do MSI-X, fall through and try MSI
747 * No need to reallocate memory since we're decreasing the number of
748 * queues. We just won't use the other ones, also it is freed correctly
754 err
= pci_enable_msi(adapter
->pdev
);
756 adapter
->flags
|= IXGBE_FLAG_MSI_ENABLED
;
757 flags
&= ~IRQF_SHARED
;
758 handler
= &ixgbe_intr
;
761 err
= request_irq(adapter
->pdev
->irq
, handler
, flags
,
762 netdev
->name
, netdev
);
764 DPRINTK(PROBE
, ERR
, "request_irq failed, Error %d\n", err
);
770 static void ixgbe_free_irq(struct ixgbe_adapter
*adapter
)
772 struct net_device
*netdev
= adapter
->netdev
;
774 if (adapter
->flags
& IXGBE_FLAG_MSIX_ENABLED
) {
777 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
778 free_irq(adapter
->msix_entries
[i
].vector
,
779 &(adapter
->tx_ring
[i
]));
780 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
781 free_irq(adapter
->msix_entries
[i
+
782 adapter
->num_tx_queues
].vector
,
783 &(adapter
->rx_ring
[i
]));
784 i
= adapter
->num_rx_queues
+ adapter
->num_tx_queues
;
785 free_irq(adapter
->msix_entries
[i
].vector
, netdev
);
786 pci_disable_msix(adapter
->pdev
);
787 kfree(adapter
->msix_entries
);
788 adapter
->msix_entries
= NULL
;
789 adapter
->flags
&= ~IXGBE_FLAG_MSIX_ENABLED
;
793 free_irq(adapter
->pdev
->irq
, netdev
);
794 if (adapter
->flags
& IXGBE_FLAG_MSI_ENABLED
) {
795 pci_disable_msi(adapter
->pdev
);
796 adapter
->flags
&= ~IXGBE_FLAG_MSI_ENABLED
;
801 * ixgbe_irq_disable - Mask off interrupt generation on the NIC
802 * @adapter: board private structure
804 static inline void ixgbe_irq_disable(struct ixgbe_adapter
*adapter
)
806 atomic_inc(&adapter
->irq_sem
);
807 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_EIMC
, ~0);
808 IXGBE_WRITE_FLUSH(&adapter
->hw
);
809 synchronize_irq(adapter
->pdev
->irq
);
813 * ixgbe_irq_enable - Enable default interrupt generation settings
814 * @adapter: board private structure
816 static inline void ixgbe_irq_enable(struct ixgbe_adapter
*adapter
)
818 if (atomic_dec_and_test(&adapter
->irq_sem
)) {
819 if (adapter
->flags
& IXGBE_FLAG_MSIX_ENABLED
)
820 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_EIAC
,
821 (IXGBE_EIMS_ENABLE_MASK
&
822 ~(IXGBE_EIMS_OTHER
| IXGBE_EIMS_LSC
)));
823 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_EIMS
,
824 IXGBE_EIMS_ENABLE_MASK
);
825 IXGBE_WRITE_FLUSH(&adapter
->hw
);
830 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
833 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter
*adapter
)
836 struct ixgbe_hw
*hw
= &adapter
->hw
;
838 if (adapter
->rx_eitr
)
839 IXGBE_WRITE_REG(hw
, IXGBE_EITR(0),
840 EITR_INTS_PER_SEC_TO_REG(adapter
->rx_eitr
));
842 /* for re-triggering the interrupt in non-NAPI mode */
843 adapter
->rx_ring
[0].eims_value
= (1 << IXGBE_MSIX_VECTOR(0));
844 adapter
->tx_ring
[0].eims_value
= (1 << IXGBE_MSIX_VECTOR(0));
846 ixgbe_set_ivar(adapter
, IXGBE_IVAR_RX_QUEUE(0), 0);
847 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
848 ixgbe_set_ivar(adapter
, IXGBE_IVAR_TX_QUEUE(i
), i
);
852 * ixgbe_configure_tx - Configure 8254x Transmit Unit after Reset
853 * @adapter: board private structure
855 * Configure the Tx unit of the MAC after a reset.
857 static void ixgbe_configure_tx(struct ixgbe_adapter
*adapter
)
860 struct ixgbe_hw
*hw
= &adapter
->hw
;
863 /* Setup the HW Tx Head and Tail descriptor pointers */
864 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
865 tdba
= adapter
->tx_ring
[i
].dma
;
866 tdlen
= adapter
->tx_ring
[i
].count
*
867 sizeof(union ixgbe_adv_tx_desc
);
868 IXGBE_WRITE_REG(hw
, IXGBE_TDBAL(i
), (tdba
& DMA_32BIT_MASK
));
869 IXGBE_WRITE_REG(hw
, IXGBE_TDBAH(i
), (tdba
>> 32));
870 IXGBE_WRITE_REG(hw
, IXGBE_TDLEN(i
), tdlen
);
871 IXGBE_WRITE_REG(hw
, IXGBE_TDH(i
), 0);
872 IXGBE_WRITE_REG(hw
, IXGBE_TDT(i
), 0);
873 adapter
->tx_ring
[i
].head
= IXGBE_TDH(i
);
874 adapter
->tx_ring
[i
].tail
= IXGBE_TDT(i
);
877 IXGBE_WRITE_REG(hw
, IXGBE_TIPG
, IXGBE_TIPG_FIBER_DEFAULT
);
880 #define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \
881 (((S) & (PAGE_SIZE - 1)) ? 1 : 0))
883 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
885 * ixgbe_configure_rx - Configure 8254x Receive Unit after Reset
886 * @adapter: board private structure
888 * Configure the Rx unit of the MAC after a reset.
890 static void ixgbe_configure_rx(struct ixgbe_adapter
*adapter
)
893 struct ixgbe_hw
*hw
= &adapter
->hw
;
894 struct net_device
*netdev
= adapter
->netdev
;
895 int max_frame
= netdev
->mtu
+ ETH_HLEN
+ ETH_FCS_LEN
;
896 u32 rdlen
, rxctrl
, rxcsum
;
904 /* Decide whether to use packet split mode or not */
905 if (netdev
->mtu
> ETH_DATA_LEN
)
906 adapter
->flags
|= IXGBE_FLAG_RX_PS_ENABLED
;
908 adapter
->flags
&= ~IXGBE_FLAG_RX_PS_ENABLED
;
910 /* Set the RX buffer length according to the mode */
911 if (adapter
->flags
& IXGBE_FLAG_RX_PS_ENABLED
) {
912 adapter
->rx_buf_len
= IXGBE_RX_HDR_SIZE
;
914 if (netdev
->mtu
<= ETH_DATA_LEN
)
915 adapter
->rx_buf_len
= MAXIMUM_ETHERNET_VLAN_SIZE
;
917 adapter
->rx_buf_len
= ALIGN(max_frame
, 1024);
920 fctrl
= IXGBE_READ_REG(&adapter
->hw
, IXGBE_FCTRL
);
921 fctrl
|= IXGBE_FCTRL_BAM
;
922 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_FCTRL
, fctrl
);
924 hlreg0
= IXGBE_READ_REG(hw
, IXGBE_HLREG0
);
925 if (adapter
->netdev
->mtu
<= ETH_DATA_LEN
)
926 hlreg0
&= ~IXGBE_HLREG0_JUMBOEN
;
928 hlreg0
|= IXGBE_HLREG0_JUMBOEN
;
929 IXGBE_WRITE_REG(hw
, IXGBE_HLREG0
, hlreg0
);
931 pages
= PAGE_USE_COUNT(adapter
->netdev
->mtu
);
933 srrctl
= IXGBE_READ_REG(&adapter
->hw
, IXGBE_SRRCTL(0));
934 srrctl
&= ~IXGBE_SRRCTL_BSIZEHDR_MASK
;
935 srrctl
&= ~IXGBE_SRRCTL_BSIZEPKT_MASK
;
937 if (adapter
->flags
& IXGBE_FLAG_RX_PS_ENABLED
) {
938 srrctl
|= PAGE_SIZE
>> IXGBE_SRRCTL_BSIZEPKT_SHIFT
;
939 srrctl
|= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS
;
940 srrctl
|= ((IXGBE_RX_HDR_SIZE
<<
941 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT
) &
942 IXGBE_SRRCTL_BSIZEHDR_MASK
);
944 srrctl
|= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF
;
946 if (adapter
->rx_buf_len
== MAXIMUM_ETHERNET_VLAN_SIZE
)
948 IXGBE_RXBUFFER_2048
>> IXGBE_SRRCTL_BSIZEPKT_SHIFT
;
951 adapter
->rx_buf_len
>> IXGBE_SRRCTL_BSIZEPKT_SHIFT
;
953 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_SRRCTL(0), srrctl
);
955 rdlen
= adapter
->rx_ring
[0].count
* sizeof(union ixgbe_adv_rx_desc
);
956 /* disable receives while setting up the descriptors */
957 rxctrl
= IXGBE_READ_REG(hw
, IXGBE_RXCTRL
);
958 IXGBE_WRITE_REG(hw
, IXGBE_RXCTRL
, rxctrl
& ~IXGBE_RXCTRL_RXEN
);
960 /* Setup the HW Rx Head and Tail Descriptor Pointers and
961 * the Base and Length of the Rx Descriptor Ring */
962 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
963 rdba
= adapter
->rx_ring
[i
].dma
;
964 IXGBE_WRITE_REG(hw
, IXGBE_RDBAL(i
), (rdba
& DMA_32BIT_MASK
));
965 IXGBE_WRITE_REG(hw
, IXGBE_RDBAH(i
), (rdba
>> 32));
966 IXGBE_WRITE_REG(hw
, IXGBE_RDLEN(i
), rdlen
);
967 IXGBE_WRITE_REG(hw
, IXGBE_RDH(i
), 0);
968 IXGBE_WRITE_REG(hw
, IXGBE_RDT(i
), 0);
969 adapter
->rx_ring
[i
].head
= IXGBE_RDH(i
);
970 adapter
->rx_ring
[i
].tail
= IXGBE_RDT(i
);
973 if (adapter
->num_rx_queues
> 1) {
974 /* Random 40bytes used as random key in RSS hash function */
975 get_random_bytes(&random
[0], 40);
977 switch (adapter
->num_rx_queues
) {
980 /* Bits [3:0] in each byte refers the Rx queue no */
991 /* Fill out redirection table */
992 for (i
= 0; i
< 32; i
++) {
993 IXGBE_WRITE_REG_ARRAY(hw
, IXGBE_RETA(0), i
, reta
);
994 if (adapter
->num_rx_queues
> 4) {
996 IXGBE_WRITE_REG_ARRAY(hw
, IXGBE_RETA(0), i
,
1001 /* Fill out hash function seeds */
1002 for (i
= 0; i
< 10; i
++)
1003 IXGBE_WRITE_REG_ARRAY(hw
, IXGBE_RSSRK(0), i
, random
[i
]);
1005 mrqc
= IXGBE_MRQC_RSSEN
1006 /* Perform hash on these packet types */
1007 | IXGBE_MRQC_RSS_FIELD_IPV4
1008 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
1009 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
1010 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
1011 | IXGBE_MRQC_RSS_FIELD_IPV6_EX
1012 | IXGBE_MRQC_RSS_FIELD_IPV6
1013 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
1014 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
1015 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP
;
1016 IXGBE_WRITE_REG(hw
, IXGBE_MRQC
, mrqc
);
1018 /* Multiqueue and packet checksumming are mutually exclusive. */
1019 rxcsum
= IXGBE_READ_REG(hw
, IXGBE_RXCSUM
);
1020 rxcsum
|= IXGBE_RXCSUM_PCSD
;
1021 IXGBE_WRITE_REG(hw
, IXGBE_RXCSUM
, rxcsum
);
1023 /* Enable Receive Checksum Offload for TCP and UDP */
1024 rxcsum
= IXGBE_READ_REG(hw
, IXGBE_RXCSUM
);
1025 if (adapter
->flags
& IXGBE_FLAG_RX_CSUM_ENABLED
) {
1026 /* Enable IPv4 payload checksum for UDP fragments
1027 * Must be used in conjunction with packet-split. */
1028 rxcsum
|= IXGBE_RXCSUM_IPPCSE
;
1030 /* don't need to clear IPPCSE as it defaults to 0 */
1032 IXGBE_WRITE_REG(hw
, IXGBE_RXCSUM
, rxcsum
);
1034 /* Enable Receives */
1035 IXGBE_WRITE_REG(hw
, IXGBE_RXCTRL
, rxctrl
);
1036 rxctrl
= IXGBE_READ_REG(hw
, IXGBE_RXCTRL
);
1039 static void ixgbe_vlan_rx_register(struct net_device
*netdev
,
1040 struct vlan_group
*grp
)
1042 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
1045 ixgbe_irq_disable(adapter
);
1046 adapter
->vlgrp
= grp
;
1049 /* enable VLAN tag insert/strip */
1050 ctrl
= IXGBE_READ_REG(&adapter
->hw
, IXGBE_VLNCTRL
);
1051 ctrl
|= IXGBE_VLNCTRL_VME
| IXGBE_VLNCTRL_VFE
;
1052 ctrl
&= ~IXGBE_VLNCTRL_CFIEN
;
1053 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_VLNCTRL
, ctrl
);
1056 ixgbe_irq_enable(adapter
);
1059 static void ixgbe_vlan_rx_add_vid(struct net_device
*netdev
, u16 vid
)
1061 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
1063 /* add VID to filter table */
1064 ixgbe_set_vfta(&adapter
->hw
, vid
, 0, true);
1067 static void ixgbe_vlan_rx_kill_vid(struct net_device
*netdev
, u16 vid
)
1069 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
1071 ixgbe_irq_disable(adapter
);
1072 vlan_group_set_device(adapter
->vlgrp
, vid
, NULL
);
1073 ixgbe_irq_enable(adapter
);
1075 /* remove VID from filter table */
1076 ixgbe_set_vfta(&adapter
->hw
, vid
, 0, false);
1079 static void ixgbe_restore_vlan(struct ixgbe_adapter
*adapter
)
1081 ixgbe_vlan_rx_register(adapter
->netdev
, adapter
->vlgrp
);
1083 if (adapter
->vlgrp
) {
1085 for (vid
= 0; vid
< VLAN_GROUP_ARRAY_LEN
; vid
++) {
1086 if (!vlan_group_get_device(adapter
->vlgrp
, vid
))
1088 ixgbe_vlan_rx_add_vid(adapter
->netdev
, vid
);
1094 * ixgbe_set_multi - Multicast and Promiscuous mode set
1095 * @netdev: network interface device structure
1097 * The set_multi entry point is called whenever the multicast address
1098 * list or the network interface flags are updated. This routine is
1099 * responsible for configuring the hardware for proper multicast,
1100 * promiscuous mode, and all-multi behavior.
1102 static void ixgbe_set_multi(struct net_device
*netdev
)
1104 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
1105 struct ixgbe_hw
*hw
= &adapter
->hw
;
1106 struct dev_mc_list
*mc_ptr
;
1111 /* Check for Promiscuous and All Multicast modes */
1113 fctrl
= IXGBE_READ_REG(hw
, IXGBE_FCTRL
);
1115 if (netdev
->flags
& IFF_PROMISC
) {
1116 fctrl
|= (IXGBE_FCTRL_UPE
| IXGBE_FCTRL_MPE
);
1117 } else if (netdev
->flags
& IFF_ALLMULTI
) {
1118 fctrl
|= IXGBE_FCTRL_MPE
;
1119 fctrl
&= ~IXGBE_FCTRL_UPE
;
1121 fctrl
&= ~(IXGBE_FCTRL_UPE
| IXGBE_FCTRL_MPE
);
1124 IXGBE_WRITE_REG(hw
, IXGBE_FCTRL
, fctrl
);
1126 if (netdev
->mc_count
) {
1127 mta_list
= kcalloc(netdev
->mc_count
, ETH_ALEN
, GFP_ATOMIC
);
1131 /* Shared function expects packed array of only addresses. */
1132 mc_ptr
= netdev
->mc_list
;
1134 for (i
= 0; i
< netdev
->mc_count
; i
++) {
1137 memcpy(mta_list
+ (i
* ETH_ALEN
), mc_ptr
->dmi_addr
,
1139 mc_ptr
= mc_ptr
->next
;
1142 ixgbe_update_mc_addr_list(hw
, mta_list
, i
, 0);
1145 ixgbe_update_mc_addr_list(hw
, NULL
, 0, 0);
1150 static void ixgbe_configure(struct ixgbe_adapter
*adapter
)
1152 struct net_device
*netdev
= adapter
->netdev
;
1155 ixgbe_set_multi(netdev
);
1157 ixgbe_restore_vlan(adapter
);
1159 ixgbe_configure_tx(adapter
);
1160 ixgbe_configure_rx(adapter
);
1161 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
1162 ixgbe_alloc_rx_buffers(adapter
, &adapter
->rx_ring
[i
],
1163 (adapter
->rx_ring
[i
].count
- 1));
1166 static int ixgbe_up_complete(struct ixgbe_adapter
*adapter
)
1168 struct net_device
*netdev
= adapter
->netdev
;
1171 struct ixgbe_hw
*hw
= &adapter
->hw
;
1172 u32 txdctl
, rxdctl
, mhadd
;
1173 int max_frame
= netdev
->mtu
+ ETH_HLEN
+ ETH_FCS_LEN
;
1175 if (adapter
->flags
& (IXGBE_FLAG_MSIX_ENABLED
|
1176 IXGBE_FLAG_MSI_ENABLED
)) {
1177 if (adapter
->flags
& IXGBE_FLAG_MSIX_ENABLED
) {
1178 gpie
= (IXGBE_GPIE_MSIX_MODE
| IXGBE_GPIE_EIAME
|
1179 IXGBE_GPIE_PBA_SUPPORT
| IXGBE_GPIE_OCD
);
1182 gpie
= (IXGBE_GPIE_EIAME
|
1183 IXGBE_GPIE_PBA_SUPPORT
);
1185 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_GPIE
, gpie
);
1186 gpie
= IXGBE_READ_REG(&adapter
->hw
, IXGBE_GPIE
);
1189 mhadd
= IXGBE_READ_REG(hw
, IXGBE_MHADD
);
1191 if (max_frame
!= (mhadd
>> IXGBE_MHADD_MFS_SHIFT
)) {
1192 mhadd
&= ~IXGBE_MHADD_MFS_MASK
;
1193 mhadd
|= max_frame
<< IXGBE_MHADD_MFS_SHIFT
;
1195 IXGBE_WRITE_REG(hw
, IXGBE_MHADD
, mhadd
);
1198 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
1199 txdctl
= IXGBE_READ_REG(&adapter
->hw
, IXGBE_TXDCTL(i
));
1200 txdctl
|= IXGBE_TXDCTL_ENABLE
;
1201 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_TXDCTL(i
), txdctl
);
1204 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
1205 rxdctl
= IXGBE_READ_REG(&adapter
->hw
, IXGBE_RXDCTL(i
));
1206 rxdctl
|= IXGBE_RXDCTL_ENABLE
;
1207 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_RXDCTL(i
), rxdctl
);
1209 /* enable all receives */
1210 rxdctl
= IXGBE_READ_REG(hw
, IXGBE_RXCTRL
);
1211 rxdctl
|= (IXGBE_RXCTRL_DMBYPS
| IXGBE_RXCTRL_RXEN
);
1212 IXGBE_WRITE_REG(hw
, IXGBE_RXCTRL
, rxdctl
);
1214 if (adapter
->flags
& IXGBE_FLAG_MSIX_ENABLED
)
1215 ixgbe_configure_msix(adapter
);
1217 ixgbe_configure_msi_and_legacy(adapter
);
1219 clear_bit(__IXGBE_DOWN
, &adapter
->state
);
1220 napi_enable(&adapter
->napi
);
1221 ixgbe_irq_enable(adapter
);
1223 /* bring the link up in the watchdog, this could race with our first
1224 * link up interrupt but shouldn't be a problem */
1225 mod_timer(&adapter
->watchdog_timer
, jiffies
);
1229 int ixgbe_up(struct ixgbe_adapter
*adapter
)
1231 /* hardware has been reset, we need to reload some things */
1232 ixgbe_configure(adapter
);
1234 return ixgbe_up_complete(adapter
);
1237 void ixgbe_reset(struct ixgbe_adapter
*adapter
)
1239 if (ixgbe_init_hw(&adapter
->hw
))
1240 DPRINTK(PROBE
, ERR
, "Hardware Error\n");
1242 /* reprogram the RAR[0] in case user changed it. */
1243 ixgbe_set_rar(&adapter
->hw
, 0, adapter
->hw
.mac
.addr
, 0, IXGBE_RAH_AV
);
1248 static int ixgbe_resume(struct pci_dev
*pdev
)
1250 struct net_device
*netdev
= pci_get_drvdata(pdev
);
1251 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
1252 u32 err
, num_rx_queues
= adapter
->num_rx_queues
;
1254 pci_set_power_state(pdev
, PCI_D0
);
1255 pci_restore_state(pdev
);
1256 err
= pci_enable_device(pdev
);
1258 printk(KERN_ERR
"ixgbe: Cannot enable PCI device from " \
1262 pci_set_master(pdev
);
1264 pci_enable_wake(pdev
, PCI_D3hot
, 0);
1265 pci_enable_wake(pdev
, PCI_D3cold
, 0);
1267 if (netif_running(netdev
)) {
1268 err
= ixgbe_request_irq(adapter
, &num_rx_queues
);
1273 ixgbe_reset(adapter
);
1275 if (netif_running(netdev
))
1278 netif_device_attach(netdev
);
1285 * ixgbe_clean_rx_ring - Free Rx Buffers per Queue
1286 * @adapter: board private structure
1287 * @rx_ring: ring to free buffers from
1289 static void ixgbe_clean_rx_ring(struct ixgbe_adapter
*adapter
,
1290 struct ixgbe_ring
*rx_ring
)
1292 struct pci_dev
*pdev
= adapter
->pdev
;
1296 /* Free all the Rx ring sk_buffs */
1298 for (i
= 0; i
< rx_ring
->count
; i
++) {
1299 struct ixgbe_rx_buffer
*rx_buffer_info
;
1301 rx_buffer_info
= &rx_ring
->rx_buffer_info
[i
];
1302 if (rx_buffer_info
->dma
) {
1303 pci_unmap_single(pdev
, rx_buffer_info
->dma
,
1304 adapter
->rx_buf_len
,
1305 PCI_DMA_FROMDEVICE
);
1306 rx_buffer_info
->dma
= 0;
1308 if (rx_buffer_info
->skb
) {
1309 dev_kfree_skb(rx_buffer_info
->skb
);
1310 rx_buffer_info
->skb
= NULL
;
1312 if (!rx_buffer_info
->page
)
1314 pci_unmap_page(pdev
, rx_buffer_info
->page_dma
, PAGE_SIZE
,
1315 PCI_DMA_FROMDEVICE
);
1316 rx_buffer_info
->page_dma
= 0;
1318 put_page(rx_buffer_info
->page
);
1319 rx_buffer_info
->page
= NULL
;
1322 size
= sizeof(struct ixgbe_rx_buffer
) * rx_ring
->count
;
1323 memset(rx_ring
->rx_buffer_info
, 0, size
);
1325 /* Zero out the descriptor ring */
1326 memset(rx_ring
->desc
, 0, rx_ring
->size
);
1328 rx_ring
->next_to_clean
= 0;
1329 rx_ring
->next_to_use
= 0;
1331 writel(0, adapter
->hw
.hw_addr
+ rx_ring
->head
);
1332 writel(0, adapter
->hw
.hw_addr
+ rx_ring
->tail
);
1336 * ixgbe_clean_tx_ring - Free Tx Buffers
1337 * @adapter: board private structure
1338 * @tx_ring: ring to be cleaned
1340 static void ixgbe_clean_tx_ring(struct ixgbe_adapter
*adapter
,
1341 struct ixgbe_ring
*tx_ring
)
1343 struct ixgbe_tx_buffer
*tx_buffer_info
;
1347 /* Free all the Tx ring sk_buffs */
1349 for (i
= 0; i
< tx_ring
->count
; i
++) {
1350 tx_buffer_info
= &tx_ring
->tx_buffer_info
[i
];
1351 ixgbe_unmap_and_free_tx_resource(adapter
, tx_buffer_info
);
1354 size
= sizeof(struct ixgbe_tx_buffer
) * tx_ring
->count
;
1355 memset(tx_ring
->tx_buffer_info
, 0, size
);
1357 /* Zero out the descriptor ring */
1358 memset(tx_ring
->desc
, 0, tx_ring
->size
);
1360 tx_ring
->next_to_use
= 0;
1361 tx_ring
->next_to_clean
= 0;
1363 writel(0, adapter
->hw
.hw_addr
+ tx_ring
->head
);
1364 writel(0, adapter
->hw
.hw_addr
+ tx_ring
->tail
);
1368 * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues
1369 * @adapter: board private structure
1371 static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter
*adapter
)
1375 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
1376 ixgbe_clean_tx_ring(adapter
, &adapter
->tx_ring
[i
]);
1380 * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues
1381 * @adapter: board private structure
1383 static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter
*adapter
)
1387 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
1388 ixgbe_clean_rx_ring(adapter
, &adapter
->rx_ring
[i
]);
1391 void ixgbe_down(struct ixgbe_adapter
*adapter
)
1393 struct net_device
*netdev
= adapter
->netdev
;
1396 /* signal that we are down to the interrupt handler */
1397 set_bit(__IXGBE_DOWN
, &adapter
->state
);
1399 /* disable receives */
1400 rxctrl
= IXGBE_READ_REG(&adapter
->hw
, IXGBE_RXCTRL
);
1401 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_RXCTRL
,
1402 rxctrl
& ~IXGBE_RXCTRL_RXEN
);
1404 netif_tx_disable(netdev
);
1406 /* disable transmits in the hardware */
1408 /* flush both disables */
1409 IXGBE_WRITE_FLUSH(&adapter
->hw
);
1412 ixgbe_irq_disable(adapter
);
1414 napi_disable(&adapter
->napi
);
1415 del_timer_sync(&adapter
->watchdog_timer
);
1417 netif_carrier_off(netdev
);
1418 netif_stop_queue(netdev
);
1420 ixgbe_reset(adapter
);
1421 ixgbe_clean_all_tx_rings(adapter
);
1422 ixgbe_clean_all_rx_rings(adapter
);
1426 static int ixgbe_suspend(struct pci_dev
*pdev
, pm_message_t state
)
1428 struct net_device
*netdev
= pci_get_drvdata(pdev
);
1429 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
1434 netif_device_detach(netdev
);
1436 if (netif_running(netdev
)) {
1437 ixgbe_down(adapter
);
1438 ixgbe_free_irq(adapter
);
1442 retval
= pci_save_state(pdev
);
1447 pci_enable_wake(pdev
, PCI_D3hot
, 0);
1448 pci_enable_wake(pdev
, PCI_D3cold
, 0);
1450 pci_disable_device(pdev
);
1452 pci_set_power_state(pdev
, pci_choose_state(pdev
, state
));
1457 static void ixgbe_shutdown(struct pci_dev
*pdev
)
1459 ixgbe_suspend(pdev
, PMSG_SUSPEND
);
1463 * ixgbe_clean - NAPI Rx polling callback
1464 * @adapter: board private structure
1466 static int ixgbe_clean(struct napi_struct
*napi
, int budget
)
1468 struct ixgbe_adapter
*adapter
= container_of(napi
,
1469 struct ixgbe_adapter
, napi
);
1470 struct net_device
*netdev
= adapter
->netdev
;
1471 int tx_cleaned
= 0, work_done
= 0;
1473 /* In non-MSIX case, there is no multi-Tx/Rx queue */
1474 tx_cleaned
= ixgbe_clean_tx_irq(adapter
, adapter
->tx_ring
);
1475 ixgbe_clean_rx_irq(adapter
, &adapter
->rx_ring
[0], &work_done
,
1481 /* If budget not fully consumed, exit the polling mode */
1482 if (work_done
< budget
) {
1483 netif_rx_complete(netdev
, napi
);
1484 ixgbe_irq_enable(adapter
);
1491 * ixgbe_tx_timeout - Respond to a Tx Hang
1492 * @netdev: network interface device structure
1494 static void ixgbe_tx_timeout(struct net_device
*netdev
)
1496 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
1498 /* Do the reset outside of interrupt context */
1499 schedule_work(&adapter
->reset_task
);
1502 static void ixgbe_reset_task(struct work_struct
*work
)
1504 struct ixgbe_adapter
*adapter
;
1505 adapter
= container_of(work
, struct ixgbe_adapter
, reset_task
);
1507 adapter
->tx_timeout_count
++;
1509 ixgbe_down(adapter
);
1514 * ixgbe_alloc_queues - Allocate memory for all rings
1515 * @adapter: board private structure to initialize
1517 * We allocate one ring per queue at run-time since we don't know the
1518 * number of queues at compile-time. The polling_netdev array is
1519 * intended for Multiqueue, but should work fine with a single queue.
1521 static int __devinit
ixgbe_alloc_queues(struct ixgbe_adapter
*adapter
)
1525 adapter
->tx_ring
= kcalloc(adapter
->num_tx_queues
,
1526 sizeof(struct ixgbe_ring
), GFP_KERNEL
);
1527 if (!adapter
->tx_ring
)
1530 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
1531 adapter
->tx_ring
[i
].count
= IXGBE_DEFAULT_TXD
;
1533 adapter
->rx_ring
= kcalloc(adapter
->num_rx_queues
,
1534 sizeof(struct ixgbe_ring
), GFP_KERNEL
);
1535 if (!adapter
->rx_ring
) {
1536 kfree(adapter
->tx_ring
);
1540 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
1541 adapter
->rx_ring
[i
].adapter
= adapter
;
1542 adapter
->rx_ring
[i
].itr_register
= IXGBE_EITR(i
);
1543 adapter
->rx_ring
[i
].count
= IXGBE_DEFAULT_RXD
;
1550 * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)
1551 * @adapter: board private structure to initialize
1553 * ixgbe_sw_init initializes the Adapter private data structure.
1554 * Fields are initialized based on PCI device information and
1555 * OS network device settings (MTU size).
1557 static int __devinit
ixgbe_sw_init(struct ixgbe_adapter
*adapter
)
1559 struct ixgbe_hw
*hw
= &adapter
->hw
;
1560 struct pci_dev
*pdev
= adapter
->pdev
;
1562 /* default flow control settings */
1563 hw
->fc
.original_type
= ixgbe_fc_full
;
1564 hw
->fc
.type
= ixgbe_fc_full
;
1566 hw
->mac
.link_mode_select
= IXGBE_AUTOC_LMS_10G_LINK_NO_AN
;
1567 if (hw
->mac
.ops
.reset(hw
)) {
1568 dev_err(&pdev
->dev
, "HW Init failed\n");
1571 if (hw
->phy
.ops
.setup_speed(hw
, IXGBE_LINK_SPEED_10GB_FULL
, true,
1573 dev_err(&pdev
->dev
, "Link Speed setup failed\n");
1577 /* initialize eeprom parameters */
1578 if (ixgbe_init_eeprom(hw
)) {
1579 dev_err(&pdev
->dev
, "EEPROM initialization failed\n");
1583 /* Set the default values */
1584 adapter
->num_rx_queues
= IXGBE_DEFAULT_RXQ
;
1585 adapter
->num_tx_queues
= 1;
1586 adapter
->flags
|= IXGBE_FLAG_RX_CSUM_ENABLED
;
1588 if (ixgbe_alloc_queues(adapter
)) {
1589 dev_err(&pdev
->dev
, "Unable to allocate memory for queues\n");
1593 atomic_set(&adapter
->irq_sem
, 1);
1594 set_bit(__IXGBE_DOWN
, &adapter
->state
);
1600 * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors)
1601 * @adapter: board private structure
1602 * @txdr: tx descriptor ring (for a specific queue) to setup
1604 * Return 0 on success, negative on failure
1606 int ixgbe_setup_tx_resources(struct ixgbe_adapter
*adapter
,
1607 struct ixgbe_ring
*txdr
)
1609 struct pci_dev
*pdev
= adapter
->pdev
;
1612 size
= sizeof(struct ixgbe_tx_buffer
) * txdr
->count
;
1613 txdr
->tx_buffer_info
= vmalloc(size
);
1614 if (!txdr
->tx_buffer_info
) {
1616 "Unable to allocate memory for the transmit descriptor ring\n");
1619 memset(txdr
->tx_buffer_info
, 0, size
);
1621 /* round up to nearest 4K */
1622 txdr
->size
= txdr
->count
* sizeof(union ixgbe_adv_tx_desc
);
1623 txdr
->size
= ALIGN(txdr
->size
, 4096);
1625 txdr
->desc
= pci_alloc_consistent(pdev
, txdr
->size
, &txdr
->dma
);
1627 vfree(txdr
->tx_buffer_info
);
1629 "Memory allocation failed for the tx desc ring\n");
1633 txdr
->adapter
= adapter
;
1634 txdr
->next_to_use
= 0;
1635 txdr
->next_to_clean
= 0;
1636 txdr
->work_limit
= txdr
->count
;
1637 spin_lock_init(&txdr
->tx_lock
);
1643 * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)
1644 * @adapter: board private structure
1645 * @rxdr: rx descriptor ring (for a specific queue) to setup
1647 * Returns 0 on success, negative on failure
1649 int ixgbe_setup_rx_resources(struct ixgbe_adapter
*adapter
,
1650 struct ixgbe_ring
*rxdr
)
1652 struct pci_dev
*pdev
= adapter
->pdev
;
1655 size
= sizeof(struct ixgbe_rx_buffer
) * rxdr
->count
;
1656 rxdr
->rx_buffer_info
= vmalloc(size
);
1657 if (!rxdr
->rx_buffer_info
) {
1659 "vmalloc allocation failed for the rx desc ring\n");
1662 memset(rxdr
->rx_buffer_info
, 0, size
);
1664 desc_len
= sizeof(union ixgbe_adv_rx_desc
);
1666 /* Round up to nearest 4K */
1667 rxdr
->size
= rxdr
->count
* desc_len
;
1668 rxdr
->size
= ALIGN(rxdr
->size
, 4096);
1670 rxdr
->desc
= pci_alloc_consistent(pdev
, rxdr
->size
, &rxdr
->dma
);
1674 "Memory allocation failed for the rx desc ring\n");
1675 vfree(rxdr
->rx_buffer_info
);
1679 rxdr
->next_to_clean
= 0;
1680 rxdr
->next_to_use
= 0;
1681 rxdr
->adapter
= adapter
;
1687 * ixgbe_free_tx_resources - Free Tx Resources per Queue
1688 * @adapter: board private structure
1689 * @tx_ring: Tx descriptor ring for a specific queue
1691 * Free all transmit software resources
1693 static void ixgbe_free_tx_resources(struct ixgbe_adapter
*adapter
,
1694 struct ixgbe_ring
*tx_ring
)
1696 struct pci_dev
*pdev
= adapter
->pdev
;
1698 ixgbe_clean_tx_ring(adapter
, tx_ring
);
1700 vfree(tx_ring
->tx_buffer_info
);
1701 tx_ring
->tx_buffer_info
= NULL
;
1703 pci_free_consistent(pdev
, tx_ring
->size
, tx_ring
->desc
, tx_ring
->dma
);
1705 tx_ring
->desc
= NULL
;
1709 * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues
1710 * @adapter: board private structure
1712 * Free all transmit software resources
1714 static void ixgbe_free_all_tx_resources(struct ixgbe_adapter
*adapter
)
1718 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
1719 ixgbe_free_tx_resources(adapter
, &adapter
->tx_ring
[i
]);
1723 * ixgbe_free_rx_resources - Free Rx Resources
1724 * @adapter: board private structure
1725 * @rx_ring: ring to clean the resources from
1727 * Free all receive software resources
1729 static void ixgbe_free_rx_resources(struct ixgbe_adapter
*adapter
,
1730 struct ixgbe_ring
*rx_ring
)
1732 struct pci_dev
*pdev
= adapter
->pdev
;
1734 ixgbe_clean_rx_ring(adapter
, rx_ring
);
1736 vfree(rx_ring
->rx_buffer_info
);
1737 rx_ring
->rx_buffer_info
= NULL
;
1739 pci_free_consistent(pdev
, rx_ring
->size
, rx_ring
->desc
, rx_ring
->dma
);
1741 rx_ring
->desc
= NULL
;
1745 * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues
1746 * @adapter: board private structure
1748 * Free all receive software resources
1750 static void ixgbe_free_all_rx_resources(struct ixgbe_adapter
*adapter
)
1754 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
1755 ixgbe_free_rx_resources(adapter
, &adapter
->rx_ring
[i
]);
1759 * ixgbe_setup_all_tx_resources - wrapper to allocate Tx resources
1760 * (Descriptors) for all queues
1761 * @adapter: board private structure
1763 * If this function returns with an error, then it's possible one or
1764 * more of the rings is populated (while the rest are not). It is the
1765 * callers duty to clean those orphaned rings.
1767 * Return 0 on success, negative on failure
1769 static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter
*adapter
)
1773 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
1774 err
= ixgbe_setup_tx_resources(adapter
, &adapter
->tx_ring
[i
]);
1777 "Allocation for Tx Queue %u failed\n", i
);
1786 * ixgbe_setup_all_rx_resources - wrapper to allocate Rx resources
1787 * (Descriptors) for all queues
1788 * @adapter: board private structure
1790 * If this function returns with an error, then it's possible one or
1791 * more of the rings is populated (while the rest are not). It is the
1792 * callers duty to clean those orphaned rings.
1794 * Return 0 on success, negative on failure
1797 static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter
*adapter
)
1801 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
1802 err
= ixgbe_setup_rx_resources(adapter
, &adapter
->rx_ring
[i
]);
1805 "Allocation for Rx Queue %u failed\n", i
);
1814 * ixgbe_change_mtu - Change the Maximum Transfer Unit
1815 * @netdev: network interface device structure
1816 * @new_mtu: new value for maximum frame size
1818 * Returns 0 on success, negative on failure
1820 static int ixgbe_change_mtu(struct net_device
*netdev
, int new_mtu
)
1822 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
1823 int max_frame
= new_mtu
+ ETH_HLEN
+ ETH_FCS_LEN
;
1825 if ((max_frame
< (ETH_ZLEN
+ ETH_FCS_LEN
)) ||
1826 (max_frame
> IXGBE_MAX_JUMBO_FRAME_SIZE
))
1829 netdev
->mtu
= new_mtu
;
1831 if (netif_running(netdev
)) {
1832 ixgbe_down(adapter
);
1840 * ixgbe_open - Called when a network interface is made active
1841 * @netdev: network interface device structure
1843 * Returns 0 on success, negative value on failure
1845 * The open entry point is called when a network interface is made
1846 * active by the system (IFF_UP). At this point all resources needed
1847 * for transmit and receive operations are allocated, the interrupt
1848 * handler is registered with the OS, the watchdog timer is started,
1849 * and the stack is notified that the interface is ready.
1851 static int ixgbe_open(struct net_device
*netdev
)
1853 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
1856 u32 num_rx_queues
= adapter
->num_rx_queues
;
1858 /* Let firmware know the driver has taken over */
1859 ctrl_ext
= IXGBE_READ_REG(&adapter
->hw
, IXGBE_CTRL_EXT
);
1860 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_CTRL_EXT
,
1861 ctrl_ext
| IXGBE_CTRL_EXT_DRV_LOAD
);
1864 /* allocate transmit descriptors */
1865 err
= ixgbe_setup_all_tx_resources(adapter
);
1869 if (!(adapter
->flags
& IXGBE_FLAG_MSIX_ENABLED
)) {
1871 adapter
->num_rx_queues
= num_rx_queues
;
1874 /* allocate receive descriptors */
1875 err
= ixgbe_setup_all_rx_resources(adapter
);
1879 ixgbe_configure(adapter
);
1881 err
= ixgbe_request_irq(adapter
, &num_rx_queues
);
1885 /* ixgbe_request might have reduced num_rx_queues */
1886 if (num_rx_queues
< adapter
->num_rx_queues
) {
1887 /* We didn't get MSI-X, so we need to release everything,
1888 * set our Rx queue count to num_rx_queues, and redo the
1889 * whole init process.
1891 ixgbe_free_irq(adapter
);
1892 if (adapter
->flags
& IXGBE_FLAG_MSI_ENABLED
) {
1893 pci_disable_msi(adapter
->pdev
);
1894 adapter
->flags
&= ~IXGBE_FLAG_MSI_ENABLED
;
1896 ixgbe_free_all_rx_resources(adapter
);
1897 ixgbe_free_all_tx_resources(adapter
);
1898 adapter
->num_rx_queues
= num_rx_queues
;
1900 /* Reset the hardware, and start over. */
1901 ixgbe_reset(adapter
);
1903 goto try_intr_reinit
;
1906 err
= ixgbe_up_complete(adapter
);
1913 ixgbe_free_irq(adapter
);
1915 ixgbe_free_all_rx_resources(adapter
);
1917 ixgbe_free_all_tx_resources(adapter
);
1919 ixgbe_reset(adapter
);
1925 * ixgbe_close - Disables a network interface
1926 * @netdev: network interface device structure
1928 * Returns 0, this is not allowed to fail
1930 * The close entry point is called when an interface is de-activated
1931 * by the OS. The hardware is still under the drivers control, but
1932 * needs to be disabled. A global MAC reset is issued to stop the
1933 * hardware, and all transmit and receive resources are freed.
1935 static int ixgbe_close(struct net_device
*netdev
)
1937 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
1940 ixgbe_down(adapter
);
1941 ixgbe_free_irq(adapter
);
1943 ixgbe_free_all_tx_resources(adapter
);
1944 ixgbe_free_all_rx_resources(adapter
);
1946 ctrl_ext
= IXGBE_READ_REG(&adapter
->hw
, IXGBE_CTRL_EXT
);
1947 IXGBE_WRITE_REG(&adapter
->hw
, IXGBE_CTRL_EXT
,
1948 ctrl_ext
& ~IXGBE_CTRL_EXT_DRV_LOAD
);
1954 * ixgbe_update_stats - Update the board statistics counters.
1955 * @adapter: board private structure
1957 void ixgbe_update_stats(struct ixgbe_adapter
*adapter
)
1959 struct ixgbe_hw
*hw
= &adapter
->hw
;
1960 u64 good_rx
, missed_rx
, bprc
;
1962 adapter
->stats
.crcerrs
+= IXGBE_READ_REG(hw
, IXGBE_CRCERRS
);
1963 good_rx
= IXGBE_READ_REG(hw
, IXGBE_GPRC
);
1964 missed_rx
= IXGBE_READ_REG(hw
, IXGBE_MPC(0));
1965 missed_rx
+= IXGBE_READ_REG(hw
, IXGBE_MPC(1));
1966 missed_rx
+= IXGBE_READ_REG(hw
, IXGBE_MPC(2));
1967 missed_rx
+= IXGBE_READ_REG(hw
, IXGBE_MPC(3));
1968 missed_rx
+= IXGBE_READ_REG(hw
, IXGBE_MPC(4));
1969 missed_rx
+= IXGBE_READ_REG(hw
, IXGBE_MPC(5));
1970 missed_rx
+= IXGBE_READ_REG(hw
, IXGBE_MPC(6));
1971 missed_rx
+= IXGBE_READ_REG(hw
, IXGBE_MPC(7));
1972 adapter
->stats
.gprc
+= (good_rx
- missed_rx
);
1974 adapter
->stats
.mpc
[0] += missed_rx
;
1975 adapter
->stats
.gorc
+= IXGBE_READ_REG(hw
, IXGBE_GORCH
);
1976 bprc
= IXGBE_READ_REG(hw
, IXGBE_BPRC
);
1977 adapter
->stats
.bprc
+= bprc
;
1978 adapter
->stats
.mprc
+= IXGBE_READ_REG(hw
, IXGBE_MPRC
);
1979 adapter
->stats
.mprc
-= bprc
;
1980 adapter
->stats
.roc
+= IXGBE_READ_REG(hw
, IXGBE_ROC
);
1981 adapter
->stats
.prc64
+= IXGBE_READ_REG(hw
, IXGBE_PRC64
);
1982 adapter
->stats
.prc127
+= IXGBE_READ_REG(hw
, IXGBE_PRC127
);
1983 adapter
->stats
.prc255
+= IXGBE_READ_REG(hw
, IXGBE_PRC255
);
1984 adapter
->stats
.prc511
+= IXGBE_READ_REG(hw
, IXGBE_PRC511
);
1985 adapter
->stats
.prc1023
+= IXGBE_READ_REG(hw
, IXGBE_PRC1023
);
1986 adapter
->stats
.prc1522
+= IXGBE_READ_REG(hw
, IXGBE_PRC1522
);
1988 adapter
->stats
.rlec
+= IXGBE_READ_REG(hw
, IXGBE_RLEC
);
1989 adapter
->stats
.lxonrxc
+= IXGBE_READ_REG(hw
, IXGBE_LXONRXC
);
1990 adapter
->stats
.lxontxc
+= IXGBE_READ_REG(hw
, IXGBE_LXONTXC
);
1991 adapter
->stats
.lxoffrxc
+= IXGBE_READ_REG(hw
, IXGBE_LXOFFRXC
);
1992 adapter
->stats
.lxofftxc
+= IXGBE_READ_REG(hw
, IXGBE_LXOFFTXC
);
1993 adapter
->stats
.ruc
+= IXGBE_READ_REG(hw
, IXGBE_RUC
);
1994 adapter
->stats
.gptc
+= IXGBE_READ_REG(hw
, IXGBE_GPTC
);
1995 adapter
->stats
.gotc
+= IXGBE_READ_REG(hw
, IXGBE_GOTCH
);
1996 adapter
->stats
.rnbc
[0] += IXGBE_READ_REG(hw
, IXGBE_RNBC(0));
1997 adapter
->stats
.ruc
+= IXGBE_READ_REG(hw
, IXGBE_RUC
);
1998 adapter
->stats
.rfc
+= IXGBE_READ_REG(hw
, IXGBE_RFC
);
1999 adapter
->stats
.rjc
+= IXGBE_READ_REG(hw
, IXGBE_RJC
);
2000 adapter
->stats
.tor
+= IXGBE_READ_REG(hw
, IXGBE_TORH
);
2001 adapter
->stats
.tpr
+= IXGBE_READ_REG(hw
, IXGBE_TPR
);
2002 adapter
->stats
.ptc64
+= IXGBE_READ_REG(hw
, IXGBE_PTC64
);
2003 adapter
->stats
.ptc127
+= IXGBE_READ_REG(hw
, IXGBE_PTC127
);
2004 adapter
->stats
.ptc255
+= IXGBE_READ_REG(hw
, IXGBE_PTC255
);
2005 adapter
->stats
.ptc511
+= IXGBE_READ_REG(hw
, IXGBE_PTC511
);
2006 adapter
->stats
.ptc1023
+= IXGBE_READ_REG(hw
, IXGBE_PTC1023
);
2007 adapter
->stats
.ptc1522
+= IXGBE_READ_REG(hw
, IXGBE_PTC1522
);
2008 adapter
->stats
.mptc
+= IXGBE_READ_REG(hw
, IXGBE_MPTC
);
2009 adapter
->stats
.bptc
+= IXGBE_READ_REG(hw
, IXGBE_BPTC
);
2011 /* Fill out the OS statistics structure */
2012 adapter
->net_stats
.rx_packets
= adapter
->stats
.gprc
;
2013 adapter
->net_stats
.tx_packets
= adapter
->stats
.gptc
;
2014 adapter
->net_stats
.rx_bytes
= adapter
->stats
.gorc
;
2015 adapter
->net_stats
.tx_bytes
= adapter
->stats
.gotc
;
2016 adapter
->net_stats
.multicast
= adapter
->stats
.mprc
;
2019 adapter
->net_stats
.rx_errors
= adapter
->stats
.crcerrs
+
2020 adapter
->stats
.rlec
;
2021 adapter
->net_stats
.rx_dropped
= 0;
2022 adapter
->net_stats
.rx_length_errors
= adapter
->stats
.rlec
;
2023 adapter
->net_stats
.rx_crc_errors
= adapter
->stats
.crcerrs
;
2024 adapter
->net_stats
.rx_missed_errors
= adapter
->stats
.mpc
[0];
2029 * ixgbe_watchdog - Timer Call-back
2030 * @data: pointer to adapter cast into an unsigned long
2032 static void ixgbe_watchdog(unsigned long data
)
2034 struct ixgbe_adapter
*adapter
= (struct ixgbe_adapter
*)data
;
2035 struct net_device
*netdev
= adapter
->netdev
;
2039 adapter
->hw
.phy
.ops
.check(&adapter
->hw
, &(link_speed
), &link_up
);
2042 if (!netif_carrier_ok(netdev
)) {
2043 u32 frctl
= IXGBE_READ_REG(&adapter
->hw
, IXGBE_FCTRL
);
2044 u32 rmcs
= IXGBE_READ_REG(&adapter
->hw
, IXGBE_RMCS
);
2045 #define FLOW_RX (frctl & IXGBE_FCTRL_RFCE)
2046 #define FLOW_TX (rmcs & IXGBE_RMCS_TFCE_802_3X)
2047 DPRINTK(LINK
, INFO
, "NIC Link is Up %s, "
2048 "Flow Control: %s\n",
2049 (link_speed
== IXGBE_LINK_SPEED_10GB_FULL
?
2051 (link_speed
== IXGBE_LINK_SPEED_1GB_FULL
?
2052 "1 Gpbs" : "unknown speed")),
2053 ((FLOW_RX
&& FLOW_TX
) ? "RX/TX" :
2055 (FLOW_TX
? "TX" : "None"))));
2057 netif_carrier_on(netdev
);
2058 netif_wake_queue(netdev
);
2060 /* Force detection of hung controller */
2061 adapter
->detect_tx_hung
= true;
2064 if (netif_carrier_ok(netdev
)) {
2065 DPRINTK(LINK
, INFO
, "NIC Link is Down\n");
2066 netif_carrier_off(netdev
);
2067 netif_stop_queue(netdev
);
2071 ixgbe_update_stats(adapter
);
2073 /* Reset the timer */
2074 if (!test_bit(__IXGBE_DOWN
, &adapter
->state
))
2075 mod_timer(&adapter
->watchdog_timer
,
2076 round_jiffies(jiffies
+ 2 * HZ
));
2079 #define IXGBE_MAX_TXD_PWR 14
2080 #define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR)
2082 /* Tx Descriptors needed, worst case */
2083 #define TXD_USE_COUNT(S) (((S) >> IXGBE_MAX_TXD_PWR) + \
2084 (((S) & (IXGBE_MAX_DATA_PER_TXD - 1)) ? 1 : 0))
2085 #define DESC_NEEDED (TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) /* skb->data */ + \
2086 MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1) /* for context */
2088 static int ixgbe_tso(struct ixgbe_adapter
*adapter
,
2089 struct ixgbe_ring
*tx_ring
, struct sk_buff
*skb
,
2090 u32 tx_flags
, u8
*hdr_len
)
2092 struct ixgbe_adv_tx_context_desc
*context_desc
;
2095 struct ixgbe_tx_buffer
*tx_buffer_info
;
2096 u32 vlan_macip_lens
= 0, type_tucmd_mlhl
= 0;
2097 u32 mss_l4len_idx
= 0, l4len
;
2100 if (skb_is_gso(skb
)) {
2101 if (skb_header_cloned(skb
)) {
2102 err
= pskb_expand_head(skb
, 0, 0, GFP_ATOMIC
);
2106 l4len
= tcp_hdrlen(skb
);
2109 if (skb
->protocol
== ntohs(ETH_P_IP
)) {
2110 struct iphdr
*iph
= ip_hdr(skb
);
2113 tcp_hdr(skb
)->check
= ~csum_tcpudp_magic(iph
->saddr
,
2117 adapter
->hw_tso_ctxt
++;
2118 } else if (skb_shinfo(skb
)->gso_type
== SKB_GSO_TCPV6
) {
2119 ipv6_hdr(skb
)->payload_len
= 0;
2120 tcp_hdr(skb
)->check
=
2121 ~csum_ipv6_magic(&ipv6_hdr(skb
)->saddr
,
2122 &ipv6_hdr(skb
)->daddr
,
2124 adapter
->hw_tso6_ctxt
++;
2127 i
= tx_ring
->next_to_use
;
2129 tx_buffer_info
= &tx_ring
->tx_buffer_info
[i
];
2130 context_desc
= IXGBE_TX_CTXTDESC_ADV(*tx_ring
, i
);
2132 /* VLAN MACLEN IPLEN */
2133 if (tx_flags
& IXGBE_TX_FLAGS_VLAN
)
2135 (tx_flags
& IXGBE_TX_FLAGS_VLAN_MASK
);
2136 vlan_macip_lens
|= ((skb_network_offset(skb
)) <<
2137 IXGBE_ADVTXD_MACLEN_SHIFT
);
2138 *hdr_len
+= skb_network_offset(skb
);
2140 (skb_transport_header(skb
) - skb_network_header(skb
));
2142 (skb_transport_header(skb
) - skb_network_header(skb
));
2143 context_desc
->vlan_macip_lens
= cpu_to_le32(vlan_macip_lens
);
2144 context_desc
->seqnum_seed
= 0;
2146 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2147 type_tucmd_mlhl
|= (IXGBE_TXD_CMD_DEXT
|
2148 IXGBE_ADVTXD_DTYP_CTXT
);
2150 if (skb
->protocol
== ntohs(ETH_P_IP
))
2151 type_tucmd_mlhl
|= IXGBE_ADVTXD_TUCMD_IPV4
;
2152 type_tucmd_mlhl
|= IXGBE_ADVTXD_TUCMD_L4T_TCP
;
2153 context_desc
->type_tucmd_mlhl
= cpu_to_le32(type_tucmd_mlhl
);
2157 (skb_shinfo(skb
)->gso_size
<< IXGBE_ADVTXD_MSS_SHIFT
);
2158 mss_l4len_idx
|= (l4len
<< IXGBE_ADVTXD_L4LEN_SHIFT
);
2159 context_desc
->mss_l4len_idx
= cpu_to_le32(mss_l4len_idx
);
2161 tx_buffer_info
->time_stamp
= jiffies
;
2162 tx_buffer_info
->next_to_watch
= i
;
2165 if (i
== tx_ring
->count
)
2167 tx_ring
->next_to_use
= i
;
2174 static bool ixgbe_tx_csum(struct ixgbe_adapter
*adapter
,
2175 struct ixgbe_ring
*tx_ring
,
2176 struct sk_buff
*skb
, u32 tx_flags
)
2178 struct ixgbe_adv_tx_context_desc
*context_desc
;
2180 struct ixgbe_tx_buffer
*tx_buffer_info
;
2181 u32 vlan_macip_lens
= 0, type_tucmd_mlhl
= 0;
2183 if (skb
->ip_summed
== CHECKSUM_PARTIAL
||
2184 (tx_flags
& IXGBE_TX_FLAGS_VLAN
)) {
2185 i
= tx_ring
->next_to_use
;
2186 tx_buffer_info
= &tx_ring
->tx_buffer_info
[i
];
2187 context_desc
= IXGBE_TX_CTXTDESC_ADV(*tx_ring
, i
);
2189 if (tx_flags
& IXGBE_TX_FLAGS_VLAN
)
2191 (tx_flags
& IXGBE_TX_FLAGS_VLAN_MASK
);
2192 vlan_macip_lens
|= (skb_network_offset(skb
) <<
2193 IXGBE_ADVTXD_MACLEN_SHIFT
);
2194 if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
2195 vlan_macip_lens
|= (skb_transport_header(skb
) -
2196 skb_network_header(skb
));
2198 context_desc
->vlan_macip_lens
= cpu_to_le32(vlan_macip_lens
);
2199 context_desc
->seqnum_seed
= 0;
2201 type_tucmd_mlhl
|= (IXGBE_TXD_CMD_DEXT
|
2202 IXGBE_ADVTXD_DTYP_CTXT
);
2204 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
2205 if (skb
->protocol
== ntohs(ETH_P_IP
))
2206 type_tucmd_mlhl
|= IXGBE_ADVTXD_TUCMD_IPV4
;
2208 if (skb
->sk
->sk_protocol
== IPPROTO_TCP
)
2209 type_tucmd_mlhl
|= IXGBE_ADVTXD_TUCMD_L4T_TCP
;
2212 context_desc
->type_tucmd_mlhl
= cpu_to_le32(type_tucmd_mlhl
);
2213 context_desc
->mss_l4len_idx
= 0;
2215 tx_buffer_info
->time_stamp
= jiffies
;
2216 tx_buffer_info
->next_to_watch
= i
;
2217 adapter
->hw_csum_tx_good
++;
2219 if (i
== tx_ring
->count
)
2221 tx_ring
->next_to_use
= i
;
2228 static int ixgbe_tx_map(struct ixgbe_adapter
*adapter
,
2229 struct ixgbe_ring
*tx_ring
,
2230 struct sk_buff
*skb
, unsigned int first
)
2232 struct ixgbe_tx_buffer
*tx_buffer_info
;
2233 unsigned int len
= skb
->len
;
2234 unsigned int offset
= 0, size
, count
= 0, i
;
2235 unsigned int nr_frags
= skb_shinfo(skb
)->nr_frags
;
2238 len
-= skb
->data_len
;
2240 i
= tx_ring
->next_to_use
;
2243 tx_buffer_info
= &tx_ring
->tx_buffer_info
[i
];
2244 size
= min(len
, (uint
)IXGBE_MAX_DATA_PER_TXD
);
2246 tx_buffer_info
->length
= size
;
2247 tx_buffer_info
->dma
= pci_map_single(adapter
->pdev
,
2249 size
, PCI_DMA_TODEVICE
);
2250 tx_buffer_info
->time_stamp
= jiffies
;
2251 tx_buffer_info
->next_to_watch
= i
;
2257 if (i
== tx_ring
->count
)
2261 for (f
= 0; f
< nr_frags
; f
++) {
2262 struct skb_frag_struct
*frag
;
2264 frag
= &skb_shinfo(skb
)->frags
[f
];
2266 offset
= frag
->page_offset
;
2269 tx_buffer_info
= &tx_ring
->tx_buffer_info
[i
];
2270 size
= min(len
, (uint
)IXGBE_MAX_DATA_PER_TXD
);
2272 tx_buffer_info
->length
= size
;
2273 tx_buffer_info
->dma
= pci_map_page(adapter
->pdev
,
2276 size
, PCI_DMA_TODEVICE
);
2277 tx_buffer_info
->time_stamp
= jiffies
;
2278 tx_buffer_info
->next_to_watch
= i
;
2284 if (i
== tx_ring
->count
)
2289 i
= tx_ring
->count
- 1;
2292 tx_ring
->tx_buffer_info
[i
].skb
= skb
;
2293 tx_ring
->tx_buffer_info
[first
].next_to_watch
= i
;
2298 static void ixgbe_tx_queue(struct ixgbe_adapter
*adapter
,
2299 struct ixgbe_ring
*tx_ring
,
2300 int tx_flags
, int count
, u32 paylen
, u8 hdr_len
)
2302 union ixgbe_adv_tx_desc
*tx_desc
= NULL
;
2303 struct ixgbe_tx_buffer
*tx_buffer_info
;
2304 u32 olinfo_status
= 0, cmd_type_len
= 0;
2306 u32 txd_cmd
= IXGBE_TXD_CMD_EOP
| IXGBE_TXD_CMD_RS
| IXGBE_TXD_CMD_IFCS
;
2308 cmd_type_len
|= IXGBE_ADVTXD_DTYP_DATA
;
2310 cmd_type_len
|= IXGBE_ADVTXD_DCMD_IFCS
| IXGBE_ADVTXD_DCMD_DEXT
;
2312 if (tx_flags
& IXGBE_TX_FLAGS_VLAN
)
2313 cmd_type_len
|= IXGBE_ADVTXD_DCMD_VLE
;
2315 if (tx_flags
& IXGBE_TX_FLAGS_TSO
) {
2316 cmd_type_len
|= IXGBE_ADVTXD_DCMD_TSE
;
2318 olinfo_status
|= IXGBE_TXD_POPTS_TXSM
<<
2319 IXGBE_ADVTXD_POPTS_SHIFT
;
2321 if (tx_flags
& IXGBE_TX_FLAGS_IPV4
)
2322 olinfo_status
|= IXGBE_TXD_POPTS_IXSM
<<
2323 IXGBE_ADVTXD_POPTS_SHIFT
;
2325 } else if (tx_flags
& IXGBE_TX_FLAGS_CSUM
)
2326 olinfo_status
|= IXGBE_TXD_POPTS_TXSM
<<
2327 IXGBE_ADVTXD_POPTS_SHIFT
;
2329 olinfo_status
|= ((paylen
- hdr_len
) << IXGBE_ADVTXD_PAYLEN_SHIFT
);
2331 i
= tx_ring
->next_to_use
;
2333 tx_buffer_info
= &tx_ring
->tx_buffer_info
[i
];
2334 tx_desc
= IXGBE_TX_DESC_ADV(*tx_ring
, i
);
2335 tx_desc
->read
.buffer_addr
= cpu_to_le64(tx_buffer_info
->dma
);
2336 tx_desc
->read
.cmd_type_len
=
2337 cpu_to_le32(cmd_type_len
| tx_buffer_info
->length
);
2338 tx_desc
->read
.olinfo_status
= cpu_to_le32(olinfo_status
);
2341 if (i
== tx_ring
->count
)
2345 tx_desc
->read
.cmd_type_len
|= cpu_to_le32(txd_cmd
);
2348 * Force memory writes to complete before letting h/w
2349 * know there are new descriptors to fetch. (Only
2350 * applicable for weak-ordered memory model archs,
2355 tx_ring
->next_to_use
= i
;
2356 writel(i
, adapter
->hw
.hw_addr
+ tx_ring
->tail
);
2359 static int ixgbe_xmit_frame(struct sk_buff
*skb
, struct net_device
*netdev
)
2361 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
2362 struct ixgbe_ring
*tx_ring
;
2363 unsigned int len
= skb
->len
;
2365 unsigned int tx_flags
= 0;
2366 unsigned long flags
= 0;
2369 unsigned int mss
= 0;
2372 unsigned int nr_frags
= skb_shinfo(skb
)->nr_frags
;
2373 len
-= skb
->data_len
;
2375 tx_ring
= adapter
->tx_ring
;
2377 if (skb
->len
<= 0) {
2379 return NETDEV_TX_OK
;
2381 mss
= skb_shinfo(skb
)->gso_size
;
2385 else if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
2388 count
+= TXD_USE_COUNT(len
);
2389 for (f
= 0; f
< nr_frags
; f
++)
2390 count
+= TXD_USE_COUNT(skb_shinfo(skb
)->frags
[f
].size
);
2392 spin_lock_irqsave(&tx_ring
->tx_lock
, flags
);
2393 if (IXGBE_DESC_UNUSED(tx_ring
) < (count
+ 2)) {
2395 netif_stop_queue(netdev
);
2396 spin_unlock_irqrestore(&tx_ring
->tx_lock
, flags
);
2397 return NETDEV_TX_BUSY
;
2399 spin_unlock_irqrestore(&tx_ring
->tx_lock
, flags
);
2400 if (adapter
->vlgrp
&& vlan_tx_tag_present(skb
)) {
2401 tx_flags
|= IXGBE_TX_FLAGS_VLAN
;
2402 tx_flags
|= (vlan_tx_tag_get(skb
) << IXGBE_TX_FLAGS_VLAN_SHIFT
);
2405 if (skb
->protocol
== ntohs(ETH_P_IP
))
2406 tx_flags
|= IXGBE_TX_FLAGS_IPV4
;
2407 first
= tx_ring
->next_to_use
;
2408 tso
= ixgbe_tso(adapter
, tx_ring
, skb
, tx_flags
, &hdr_len
);
2410 dev_kfree_skb_any(skb
);
2411 return NETDEV_TX_OK
;
2415 tx_flags
|= IXGBE_TX_FLAGS_TSO
;
2416 else if (ixgbe_tx_csum(adapter
, tx_ring
, skb
, tx_flags
) &&
2417 (skb
->ip_summed
== CHECKSUM_PARTIAL
))
2418 tx_flags
|= IXGBE_TX_FLAGS_CSUM
;
2420 ixgbe_tx_queue(adapter
, tx_ring
, tx_flags
,
2421 ixgbe_tx_map(adapter
, tx_ring
, skb
, first
),
2424 netdev
->trans_start
= jiffies
;
2426 spin_lock_irqsave(&tx_ring
->tx_lock
, flags
);
2427 /* Make sure there is space in the ring for the next send. */
2428 if (IXGBE_DESC_UNUSED(tx_ring
) < DESC_NEEDED
)
2429 netif_stop_queue(netdev
);
2430 spin_unlock_irqrestore(&tx_ring
->tx_lock
, flags
);
2432 return NETDEV_TX_OK
;
2436 * ixgbe_get_stats - Get System Network Statistics
2437 * @netdev: network interface device structure
2439 * Returns the address of the device statistics structure.
2440 * The statistics are actually updated from the timer callback.
2442 static struct net_device_stats
*ixgbe_get_stats(struct net_device
*netdev
)
2444 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
2446 /* only return the current stats */
2447 return &adapter
->net_stats
;
2451 * ixgbe_set_mac - Change the Ethernet Address of the NIC
2452 * @netdev: network interface device structure
2453 * @p: pointer to an address structure
2455 * Returns 0 on success, negative on failure
2457 static int ixgbe_set_mac(struct net_device
*netdev
, void *p
)
2459 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
2460 struct sockaddr
*addr
= p
;
2462 if (!is_valid_ether_addr(addr
->sa_data
))
2463 return -EADDRNOTAVAIL
;
2465 memcpy(netdev
->dev_addr
, addr
->sa_data
, netdev
->addr_len
);
2466 memcpy(adapter
->hw
.mac
.addr
, addr
->sa_data
, netdev
->addr_len
);
2468 ixgbe_set_rar(&adapter
->hw
, 0, adapter
->hw
.mac
.addr
, 0, IXGBE_RAH_AV
);
2473 #ifdef CONFIG_NET_POLL_CONTROLLER
2475 * Polling 'interrupt' - used by things like netconsole to send skbs
2476 * without having to re-enable interrupts. It's not called while
2477 * the interrupt routine is executing.
2479 static void ixgbe_netpoll(struct net_device
*netdev
)
2481 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
2483 disable_irq(adapter
->pdev
->irq
);
2484 adapter
->flags
|= IXGBE_FLAG_IN_NETPOLL
;
2485 ixgbe_intr(adapter
->pdev
->irq
, netdev
);
2486 adapter
->flags
&= ~IXGBE_FLAG_IN_NETPOLL
;
2487 enable_irq(adapter
->pdev
->irq
);
2492 * ixgbe_probe - Device Initialization Routine
2493 * @pdev: PCI device information struct
2494 * @ent: entry in ixgbe_pci_tbl
2496 * Returns 0 on success, negative on failure
2498 * ixgbe_probe initializes an adapter identified by a pci_dev structure.
2499 * The OS initialization, configuring of the adapter private structure,
2500 * and a hardware reset occur.
2502 static int __devinit
ixgbe_probe(struct pci_dev
*pdev
,
2503 const struct pci_device_id
*ent
)
2505 struct net_device
*netdev
;
2506 struct ixgbe_adapter
*adapter
= NULL
;
2507 struct ixgbe_hw
*hw
;
2508 const struct ixgbe_info
*ii
= ixgbe_info_tbl
[ent
->driver_data
];
2509 unsigned long mmio_start
, mmio_len
;
2510 static int cards_found
;
2511 int i
, err
, pci_using_dac
;
2512 u16 link_status
, link_speed
, link_width
;
2515 err
= pci_enable_device(pdev
);
2519 if (!pci_set_dma_mask(pdev
, DMA_64BIT_MASK
) &&
2520 !pci_set_consistent_dma_mask(pdev
, DMA_64BIT_MASK
)) {
2523 err
= pci_set_dma_mask(pdev
, DMA_32BIT_MASK
);
2525 err
= pci_set_consistent_dma_mask(pdev
, DMA_32BIT_MASK
);
2527 dev_err(&pdev
->dev
, "No usable DMA "
2528 "configuration, aborting\n");
2535 err
= pci_request_regions(pdev
, ixgbe_driver_name
);
2537 dev_err(&pdev
->dev
, "pci_request_regions failed 0x%x\n", err
);
2541 pci_set_master(pdev
);
2543 netdev
= alloc_etherdev(sizeof(struct ixgbe_adapter
));
2546 goto err_alloc_etherdev
;
2549 SET_NETDEV_DEV(netdev
, &pdev
->dev
);
2551 pci_set_drvdata(pdev
, netdev
);
2552 adapter
= netdev_priv(netdev
);
2554 adapter
->netdev
= netdev
;
2555 adapter
->pdev
= pdev
;
2558 adapter
->msg_enable
= (1 << DEFAULT_DEBUG_LEVEL_SHIFT
) - 1;
2560 mmio_start
= pci_resource_start(pdev
, 0);
2561 mmio_len
= pci_resource_len(pdev
, 0);
2563 hw
->hw_addr
= ioremap(mmio_start
, mmio_len
);
2569 for (i
= 1; i
<= 5; i
++) {
2570 if (pci_resource_len(pdev
, i
) == 0)
2574 netdev
->open
= &ixgbe_open
;
2575 netdev
->stop
= &ixgbe_close
;
2576 netdev
->hard_start_xmit
= &ixgbe_xmit_frame
;
2577 netdev
->get_stats
= &ixgbe_get_stats
;
2578 netdev
->set_multicast_list
= &ixgbe_set_multi
;
2579 netdev
->set_mac_address
= &ixgbe_set_mac
;
2580 netdev
->change_mtu
= &ixgbe_change_mtu
;
2581 ixgbe_set_ethtool_ops(netdev
);
2582 netdev
->tx_timeout
= &ixgbe_tx_timeout
;
2583 netdev
->watchdog_timeo
= 5 * HZ
;
2584 netif_napi_add(netdev
, &adapter
->napi
, ixgbe_clean
, 64);
2585 netdev
->vlan_rx_register
= ixgbe_vlan_rx_register
;
2586 netdev
->vlan_rx_add_vid
= ixgbe_vlan_rx_add_vid
;
2587 netdev
->vlan_rx_kill_vid
= ixgbe_vlan_rx_kill_vid
;
2588 #ifdef CONFIG_NET_POLL_CONTROLLER
2589 netdev
->poll_controller
= ixgbe_netpoll
;
2591 strcpy(netdev
->name
, pci_name(pdev
));
2593 netdev
->mem_start
= mmio_start
;
2594 netdev
->mem_end
= mmio_start
+ mmio_len
;
2596 adapter
->bd_number
= cards_found
;
2598 /* PCI config space info */
2599 hw
->vendor_id
= pdev
->vendor
;
2600 hw
->device_id
= pdev
->device
;
2601 hw
->revision_id
= pdev
->revision
;
2602 hw
->subsystem_vendor_id
= pdev
->subsystem_vendor
;
2603 hw
->subsystem_device_id
= pdev
->subsystem_device
;
2606 memcpy(&hw
->mac
.ops
, ii
->mac_ops
, sizeof(hw
->mac
.ops
));
2607 memcpy(&hw
->phy
.ops
, ii
->phy_ops
, sizeof(hw
->phy
.ops
));
2609 err
= ii
->get_invariants(hw
);
2613 /* setup the private structure */
2614 err
= ixgbe_sw_init(adapter
);
2618 netdev
->features
= NETIF_F_SG
|
2620 NETIF_F_HW_VLAN_TX
|
2621 NETIF_F_HW_VLAN_RX
|
2622 NETIF_F_HW_VLAN_FILTER
;
2624 netdev
->features
|= NETIF_F_TSO
;
2626 netdev
->features
|= NETIF_F_TSO6
;
2628 netdev
->features
|= NETIF_F_HIGHDMA
;
2631 /* make sure the EEPROM is good */
2632 if (ixgbe_validate_eeprom_checksum(hw
, NULL
) < 0) {
2633 dev_err(&pdev
->dev
, "The EEPROM Checksum Is Not Valid\n");
2638 memcpy(netdev
->dev_addr
, hw
->mac
.perm_addr
, netdev
->addr_len
);
2639 memcpy(netdev
->perm_addr
, hw
->mac
.perm_addr
, netdev
->addr_len
);
2641 if (ixgbe_validate_mac_addr(netdev
->dev_addr
)) {
2646 init_timer(&adapter
->watchdog_timer
);
2647 adapter
->watchdog_timer
.function
= &ixgbe_watchdog
;
2648 adapter
->watchdog_timer
.data
= (unsigned long)adapter
;
2650 INIT_WORK(&adapter
->reset_task
, ixgbe_reset_task
);
2652 /* initialize default flow control settings */
2653 hw
->fc
.original_type
= ixgbe_fc_full
;
2654 hw
->fc
.type
= ixgbe_fc_full
;
2655 hw
->fc
.high_water
= IXGBE_DEFAULT_FCRTH
;
2656 hw
->fc
.low_water
= IXGBE_DEFAULT_FCRTL
;
2657 hw
->fc
.pause_time
= IXGBE_DEFAULT_FCPAUSE
;
2659 /* Interrupt Throttle Rate */
2660 adapter
->rx_eitr
= (1000000 / IXGBE_DEFAULT_ITR_RX_USECS
);
2661 adapter
->tx_eitr
= (1000000 / IXGBE_DEFAULT_ITR_TX_USECS
);
2663 /* print bus type/speed/width info */
2664 pci_read_config_word(pdev
, IXGBE_PCI_LINK_STATUS
, &link_status
);
2665 link_speed
= link_status
& IXGBE_PCI_LINK_SPEED
;
2666 link_width
= link_status
& IXGBE_PCI_LINK_WIDTH
;
2667 dev_info(&pdev
->dev
, "(PCI Express:%s:%s) "
2668 "%02x:%02x:%02x:%02x:%02x:%02x\n",
2669 ((link_speed
== IXGBE_PCI_LINK_SPEED_5000
) ? "5.0Gb/s" :
2670 (link_speed
== IXGBE_PCI_LINK_SPEED_2500
) ? "2.5Gb/s" :
2672 ((link_width
== IXGBE_PCI_LINK_WIDTH_8
) ? "Width x8" :
2673 (link_width
== IXGBE_PCI_LINK_WIDTH_4
) ? "Width x4" :
2674 (link_width
== IXGBE_PCI_LINK_WIDTH_2
) ? "Width x2" :
2675 (link_width
== IXGBE_PCI_LINK_WIDTH_1
) ? "Width x1" :
2677 netdev
->dev_addr
[0], netdev
->dev_addr
[1], netdev
->dev_addr
[2],
2678 netdev
->dev_addr
[3], netdev
->dev_addr
[4], netdev
->dev_addr
[5]);
2679 ixgbe_read_part_num(hw
, &part_num
);
2680 dev_info(&pdev
->dev
, "MAC: %d, PHY: %d, PBA No: %06x-%03x\n",
2681 hw
->mac
.type
, hw
->phy
.type
,
2682 (part_num
>> 8), (part_num
& 0xff));
2684 /* reset the hardware with the new settings */
2687 netif_carrier_off(netdev
);
2688 netif_stop_queue(netdev
);
2690 strcpy(netdev
->name
, "eth%d");
2691 err
= register_netdev(netdev
);
2696 dev_info(&pdev
->dev
, "Intel(R) 10 Gigabit Network Connection\n");
2704 iounmap(hw
->hw_addr
);
2706 free_netdev(netdev
);
2708 pci_release_regions(pdev
);
2711 pci_disable_device(pdev
);
2716 * ixgbe_remove - Device Removal Routine
2717 * @pdev: PCI device information struct
2719 * ixgbe_remove is called by the PCI subsystem to alert the driver
2720 * that it should release a PCI device. The could be caused by a
2721 * Hot-Plug event, or because the driver is going to be removed from
2724 static void __devexit
ixgbe_remove(struct pci_dev
*pdev
)
2726 struct net_device
*netdev
= pci_get_drvdata(pdev
);
2727 struct ixgbe_adapter
*adapter
= netdev_priv(netdev
);
2729 set_bit(__IXGBE_DOWN
, &adapter
->state
);
2730 del_timer_sync(&adapter
->watchdog_timer
);
2732 flush_scheduled_work();
2734 unregister_netdev(netdev
);
2736 kfree(adapter
->tx_ring
);
2737 kfree(adapter
->rx_ring
);
2739 iounmap(adapter
->hw
.hw_addr
);
2740 pci_release_regions(pdev
);
2742 free_netdev(netdev
);
2744 pci_disable_device(pdev
);
2748 * ixgbe_io_error_detected - called when PCI error is detected
2749 * @pdev: Pointer to PCI device
2750 * @state: The current pci connection state
2752 * This function is called after a PCI bus error affecting
2753 * this device has been detected.
2755 static pci_ers_result_t
ixgbe_io_error_detected(struct pci_dev
*pdev
,
2756 pci_channel_state_t state
)
2758 struct net_device
*netdev
= pci_get_drvdata(pdev
);
2759 struct ixgbe_adapter
*adapter
= netdev
->priv
;
2761 netif_device_detach(netdev
);
2763 if (netif_running(netdev
))
2764 ixgbe_down(adapter
);
2765 pci_disable_device(pdev
);
2767 /* Request a slot slot reset. */
2768 return PCI_ERS_RESULT_NEED_RESET
;
2772 * ixgbe_io_slot_reset - called after the pci bus has been reset.
2773 * @pdev: Pointer to PCI device
2775 * Restart the card from scratch, as if from a cold-boot.
2777 static pci_ers_result_t
ixgbe_io_slot_reset(struct pci_dev
*pdev
)
2779 struct net_device
*netdev
= pci_get_drvdata(pdev
);
2780 struct ixgbe_adapter
*adapter
= netdev
->priv
;
2782 if (pci_enable_device(pdev
)) {
2784 "Cannot re-enable PCI device after reset.\n");
2785 return PCI_ERS_RESULT_DISCONNECT
;
2787 pci_set_master(pdev
);
2789 pci_enable_wake(pdev
, PCI_D3hot
, 0);
2790 pci_enable_wake(pdev
, PCI_D3cold
, 0);
2792 ixgbe_reset(adapter
);
2794 return PCI_ERS_RESULT_RECOVERED
;
2798 * ixgbe_io_resume - called when traffic can start flowing again.
2799 * @pdev: Pointer to PCI device
2801 * This callback is called when the error recovery driver tells us that
2802 * its OK to resume normal operation.
2804 static void ixgbe_io_resume(struct pci_dev
*pdev
)
2806 struct net_device
*netdev
= pci_get_drvdata(pdev
);
2807 struct ixgbe_adapter
*adapter
= netdev
->priv
;
2809 if (netif_running(netdev
)) {
2810 if (ixgbe_up(adapter
)) {
2811 DPRINTK(PROBE
, INFO
, "ixgbe_up failed after reset\n");
2816 netif_device_attach(netdev
);
2820 static struct pci_error_handlers ixgbe_err_handler
= {
2821 .error_detected
= ixgbe_io_error_detected
,
2822 .slot_reset
= ixgbe_io_slot_reset
,
2823 .resume
= ixgbe_io_resume
,
2826 static struct pci_driver ixgbe_driver
= {
2827 .name
= ixgbe_driver_name
,
2828 .id_table
= ixgbe_pci_tbl
,
2829 .probe
= ixgbe_probe
,
2830 .remove
= __devexit_p(ixgbe_remove
),
2832 .suspend
= ixgbe_suspend
,
2833 .resume
= ixgbe_resume
,
2835 .shutdown
= ixgbe_shutdown
,
2836 .err_handler
= &ixgbe_err_handler
2840 * ixgbe_init_module - Driver Registration Routine
2842 * ixgbe_init_module is the first routine called when the driver is
2843 * loaded. All it does is register with the PCI subsystem.
2845 static int __init
ixgbe_init_module(void)
2848 printk(KERN_INFO
"%s: %s - version %s\n", ixgbe_driver_name
,
2849 ixgbe_driver_string
, ixgbe_driver_version
);
2851 printk(KERN_INFO
"%s: %s\n", ixgbe_driver_name
, ixgbe_copyright
);
2853 ret
= pci_register_driver(&ixgbe_driver
);
2856 module_init(ixgbe_init_module
);
2859 * ixgbe_exit_module - Driver Exit Cleanup Routine
2861 * ixgbe_exit_module is called just before the driver is removed
2864 static void __exit
ixgbe_exit_module(void)
2866 pci_unregister_driver(&ixgbe_driver
);
2868 module_exit(ixgbe_exit_module
);