1 /****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
4 * Copyright 2005-2009 Solarflare Communications Inc.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
11 #include <linux/module.h>
12 #include <linux/pci.h>
13 #include <linux/netdevice.h>
14 #include <linux/etherdevice.h>
15 #include <linux/delay.h>
16 #include <linux/notifier.h>
18 #include <linux/tcp.h>
20 #include <linux/crc32.h>
21 #include <linux/ethtool.h>
22 #include <linux/topology.h>
23 #include <linux/gfp.h>
24 #include "net_driver.h"
30 #include "workarounds.h"
32 /**************************************************************************
36 **************************************************************************
39 /* Loopback mode names (see LOOPBACK_MODE()) */
40 const unsigned int efx_loopback_mode_max
= LOOPBACK_MAX
;
41 const char *efx_loopback_mode_names
[] = {
42 [LOOPBACK_NONE
] = "NONE",
43 [LOOPBACK_DATA
] = "DATAPATH",
44 [LOOPBACK_GMAC
] = "GMAC",
45 [LOOPBACK_XGMII
] = "XGMII",
46 [LOOPBACK_XGXS
] = "XGXS",
47 [LOOPBACK_XAUI
] = "XAUI",
48 [LOOPBACK_GMII
] = "GMII",
49 [LOOPBACK_SGMII
] = "SGMII",
50 [LOOPBACK_XGBR
] = "XGBR",
51 [LOOPBACK_XFI
] = "XFI",
52 [LOOPBACK_XAUI_FAR
] = "XAUI_FAR",
53 [LOOPBACK_GMII_FAR
] = "GMII_FAR",
54 [LOOPBACK_SGMII_FAR
] = "SGMII_FAR",
55 [LOOPBACK_XFI_FAR
] = "XFI_FAR",
56 [LOOPBACK_GPHY
] = "GPHY",
57 [LOOPBACK_PHYXS
] = "PHYXS",
58 [LOOPBACK_PCS
] = "PCS",
59 [LOOPBACK_PMAPMD
] = "PMA/PMD",
60 [LOOPBACK_XPORT
] = "XPORT",
61 [LOOPBACK_XGMII_WS
] = "XGMII_WS",
62 [LOOPBACK_XAUI_WS
] = "XAUI_WS",
63 [LOOPBACK_XAUI_WS_FAR
] = "XAUI_WS_FAR",
64 [LOOPBACK_XAUI_WS_NEAR
] = "XAUI_WS_NEAR",
65 [LOOPBACK_GMII_WS
] = "GMII_WS",
66 [LOOPBACK_XFI_WS
] = "XFI_WS",
67 [LOOPBACK_XFI_WS_FAR
] = "XFI_WS_FAR",
68 [LOOPBACK_PHYXS_WS
] = "PHYXS_WS",
71 /* Interrupt mode names (see INT_MODE())) */
72 const unsigned int efx_interrupt_mode_max
= EFX_INT_MODE_MAX
;
73 const char *efx_interrupt_mode_names
[] = {
74 [EFX_INT_MODE_MSIX
] = "MSI-X",
75 [EFX_INT_MODE_MSI
] = "MSI",
76 [EFX_INT_MODE_LEGACY
] = "legacy",
79 const unsigned int efx_reset_type_max
= RESET_TYPE_MAX
;
80 const char *efx_reset_type_names
[] = {
81 [RESET_TYPE_INVISIBLE
] = "INVISIBLE",
82 [RESET_TYPE_ALL
] = "ALL",
83 [RESET_TYPE_WORLD
] = "WORLD",
84 [RESET_TYPE_DISABLE
] = "DISABLE",
85 [RESET_TYPE_TX_WATCHDOG
] = "TX_WATCHDOG",
86 [RESET_TYPE_INT_ERROR
] = "INT_ERROR",
87 [RESET_TYPE_RX_RECOVERY
] = "RX_RECOVERY",
88 [RESET_TYPE_RX_DESC_FETCH
] = "RX_DESC_FETCH",
89 [RESET_TYPE_TX_DESC_FETCH
] = "TX_DESC_FETCH",
90 [RESET_TYPE_TX_SKIP
] = "TX_SKIP",
91 [RESET_TYPE_MC_FAILURE
] = "MC_FAILURE",
94 #define EFX_MAX_MTU (9 * 1024)
96 /* Reset workqueue. If any NIC has a hardware failure then a reset will be
97 * queued onto this work queue. This is not a per-nic work queue, because
98 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.
100 static struct workqueue_struct
*reset_workqueue
;
102 /**************************************************************************
104 * Configurable values
106 *************************************************************************/
109 * Use separate channels for TX and RX events
111 * Set this to 1 to use separate channels for TX and RX. It allows us
112 * to control interrupt affinity separately for TX and RX.
114 * This is only used in MSI-X interrupt mode
116 static unsigned int separate_tx_channels
;
117 module_param(separate_tx_channels
, uint
, 0444);
118 MODULE_PARM_DESC(separate_tx_channels
,
119 "Use separate channels for TX and RX");
121 /* This is the weight assigned to each of the (per-channel) virtual
124 static int napi_weight
= 64;
126 /* This is the time (in jiffies) between invocations of the hardware
127 * monitor, which checks for known hardware bugs and resets the
128 * hardware and driver as necessary.
130 unsigned int efx_monitor_interval
= 1 * HZ
;
132 /* This controls whether or not the driver will initialise devices
133 * with invalid MAC addresses stored in the EEPROM or flash. If true,
134 * such devices will be initialised with a random locally-generated
135 * MAC address. This allows for loading the sfc_mtd driver to
136 * reprogram the flash, even if the flash contents (including the MAC
137 * address) have previously been erased.
139 static unsigned int allow_bad_hwaddr
;
141 /* Initial interrupt moderation settings. They can be modified after
142 * module load with ethtool.
144 * The default for RX should strike a balance between increasing the
145 * round-trip latency and reducing overhead.
147 static unsigned int rx_irq_mod_usec
= 60;
149 /* Initial interrupt moderation settings. They can be modified after
150 * module load with ethtool.
152 * This default is chosen to ensure that a 10G link does not go idle
153 * while a TX queue is stopped after it has become full. A queue is
154 * restarted when it drops below half full. The time this takes (assuming
155 * worst case 3 descriptors per packet and 1024 descriptors) is
156 * 512 / 3 * 1.2 = 205 usec.
158 static unsigned int tx_irq_mod_usec
= 150;
160 /* This is the first interrupt mode to try out of:
165 static unsigned int interrupt_mode
;
167 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
168 * i.e. the number of CPUs among which we may distribute simultaneous
169 * interrupt handling.
171 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
172 * The default (0) means to assign an interrupt to each package (level II cache)
174 static unsigned int rss_cpus
;
175 module_param(rss_cpus
, uint
, 0444);
176 MODULE_PARM_DESC(rss_cpus
, "Number of CPUs to use for Receive-Side Scaling");
178 static int phy_flash_cfg
;
179 module_param(phy_flash_cfg
, int, 0644);
180 MODULE_PARM_DESC(phy_flash_cfg
, "Set PHYs into reflash mode initially");
182 static unsigned irq_adapt_low_thresh
= 10000;
183 module_param(irq_adapt_low_thresh
, uint
, 0644);
184 MODULE_PARM_DESC(irq_adapt_low_thresh
,
185 "Threshold score for reducing IRQ moderation");
187 static unsigned irq_adapt_high_thresh
= 20000;
188 module_param(irq_adapt_high_thresh
, uint
, 0644);
189 MODULE_PARM_DESC(irq_adapt_high_thresh
,
190 "Threshold score for increasing IRQ moderation");
192 static unsigned debug
= (NETIF_MSG_DRV
| NETIF_MSG_PROBE
|
193 NETIF_MSG_LINK
| NETIF_MSG_IFDOWN
|
194 NETIF_MSG_IFUP
| NETIF_MSG_RX_ERR
|
195 NETIF_MSG_TX_ERR
| NETIF_MSG_HW
);
196 module_param(debug
, uint
, 0);
197 MODULE_PARM_DESC(debug
, "Bitmapped debugging message enable value");
199 /**************************************************************************
201 * Utility functions and prototypes
203 *************************************************************************/
204 static void efx_remove_channel(struct efx_channel
*channel
);
205 static void efx_remove_port(struct efx_nic
*efx
);
206 static void efx_fini_napi(struct efx_nic
*efx
);
207 static void efx_fini_channels(struct efx_nic
*efx
);
209 #define EFX_ASSERT_RESET_SERIALISED(efx) \
211 if ((efx->state == STATE_RUNNING) || \
212 (efx->state == STATE_DISABLED)) \
216 /**************************************************************************
218 * Event queue processing
220 *************************************************************************/
222 /* Process channel's event queue
224 * This function is responsible for processing the event queue of a
225 * single channel. The caller must guarantee that this function will
226 * never be concurrently called more than once on the same channel,
227 * though different channels may be being processed concurrently.
229 static int efx_process_channel(struct efx_channel
*channel
, int budget
)
231 struct efx_nic
*efx
= channel
->efx
;
234 if (unlikely(efx
->reset_pending
!= RESET_TYPE_NONE
||
238 spent
= efx_nic_process_eventq(channel
, budget
);
242 /* Deliver last RX packet. */
243 if (channel
->rx_pkt
) {
244 __efx_rx_packet(channel
, channel
->rx_pkt
,
245 channel
->rx_pkt_csummed
);
246 channel
->rx_pkt
= NULL
;
249 efx_rx_strategy(channel
);
251 efx_fast_push_rx_descriptors(efx_channel_get_rx_queue(channel
));
256 /* Mark channel as finished processing
258 * Note that since we will not receive further interrupts for this
259 * channel before we finish processing and call the eventq_read_ack()
260 * method, there is no need to use the interrupt hold-off timers.
262 static inline void efx_channel_processed(struct efx_channel
*channel
)
264 /* The interrupt handler for this channel may set work_pending
265 * as soon as we acknowledge the events we've seen. Make sure
266 * it's cleared before then. */
267 channel
->work_pending
= false;
270 efx_nic_eventq_read_ack(channel
);
275 * NAPI guarantees serialisation of polls of the same device, which
276 * provides the guarantee required by efx_process_channel().
278 static int efx_poll(struct napi_struct
*napi
, int budget
)
280 struct efx_channel
*channel
=
281 container_of(napi
, struct efx_channel
, napi_str
);
282 struct efx_nic
*efx
= channel
->efx
;
285 netif_vdbg(efx
, intr
, efx
->net_dev
,
286 "channel %d NAPI poll executing on CPU %d\n",
287 channel
->channel
, raw_smp_processor_id());
289 spent
= efx_process_channel(channel
, budget
);
291 if (spent
< budget
) {
292 if (channel
->channel
< efx
->n_rx_channels
&&
293 efx
->irq_rx_adaptive
&&
294 unlikely(++channel
->irq_count
== 1000)) {
295 if (unlikely(channel
->irq_mod_score
<
296 irq_adapt_low_thresh
)) {
297 if (channel
->irq_moderation
> 1) {
298 channel
->irq_moderation
-= 1;
299 efx
->type
->push_irq_moderation(channel
);
301 } else if (unlikely(channel
->irq_mod_score
>
302 irq_adapt_high_thresh
)) {
303 if (channel
->irq_moderation
<
304 efx
->irq_rx_moderation
) {
305 channel
->irq_moderation
+= 1;
306 efx
->type
->push_irq_moderation(channel
);
309 channel
->irq_count
= 0;
310 channel
->irq_mod_score
= 0;
313 /* There is no race here; although napi_disable() will
314 * only wait for napi_complete(), this isn't a problem
315 * since efx_channel_processed() will have no effect if
316 * interrupts have already been disabled.
319 efx_channel_processed(channel
);
325 /* Process the eventq of the specified channel immediately on this CPU
327 * Disable hardware generated interrupts, wait for any existing
328 * processing to finish, then directly poll (and ack ) the eventq.
329 * Finally reenable NAPI and interrupts.
331 * Since we are touching interrupts the caller should hold the suspend lock
333 void efx_process_channel_now(struct efx_channel
*channel
)
335 struct efx_nic
*efx
= channel
->efx
;
337 BUG_ON(channel
->channel
>= efx
->n_channels
);
338 BUG_ON(!channel
->enabled
);
340 /* Disable interrupts and wait for ISRs to complete */
341 efx_nic_disable_interrupts(efx
);
343 synchronize_irq(efx
->legacy_irq
);
345 synchronize_irq(channel
->irq
);
347 /* Wait for any NAPI processing to complete */
348 napi_disable(&channel
->napi_str
);
350 /* Poll the channel */
351 efx_process_channel(channel
, channel
->eventq_mask
+ 1);
353 /* Ack the eventq. This may cause an interrupt to be generated
354 * when they are reenabled */
355 efx_channel_processed(channel
);
357 napi_enable(&channel
->napi_str
);
358 efx_nic_enable_interrupts(efx
);
361 /* Create event queue
362 * Event queue memory allocations are done only once. If the channel
363 * is reset, the memory buffer will be reused; this guards against
364 * errors during channel reset and also simplifies interrupt handling.
366 static int efx_probe_eventq(struct efx_channel
*channel
)
368 struct efx_nic
*efx
= channel
->efx
;
369 unsigned long entries
;
371 netif_dbg(channel
->efx
, probe
, channel
->efx
->net_dev
,
372 "chan %d create event queue\n", channel
->channel
);
374 /* Build an event queue with room for one event per tx and rx buffer,
375 * plus some extra for link state events and MCDI completions. */
376 entries
= roundup_pow_of_two(efx
->rxq_entries
+ efx
->txq_entries
+ 128);
377 EFX_BUG_ON_PARANOID(entries
> EFX_MAX_EVQ_SIZE
);
378 channel
->eventq_mask
= max(entries
, EFX_MIN_EVQ_SIZE
) - 1;
380 return efx_nic_probe_eventq(channel
);
383 /* Prepare channel's event queue */
384 static void efx_init_eventq(struct efx_channel
*channel
)
386 netif_dbg(channel
->efx
, drv
, channel
->efx
->net_dev
,
387 "chan %d init event queue\n", channel
->channel
);
389 channel
->eventq_read_ptr
= 0;
391 efx_nic_init_eventq(channel
);
394 static void efx_fini_eventq(struct efx_channel
*channel
)
396 netif_dbg(channel
->efx
, drv
, channel
->efx
->net_dev
,
397 "chan %d fini event queue\n", channel
->channel
);
399 efx_nic_fini_eventq(channel
);
402 static void efx_remove_eventq(struct efx_channel
*channel
)
404 netif_dbg(channel
->efx
, drv
, channel
->efx
->net_dev
,
405 "chan %d remove event queue\n", channel
->channel
);
407 efx_nic_remove_eventq(channel
);
410 /**************************************************************************
414 *************************************************************************/
416 static int efx_probe_channel(struct efx_channel
*channel
)
418 struct efx_tx_queue
*tx_queue
;
419 struct efx_rx_queue
*rx_queue
;
422 netif_dbg(channel
->efx
, probe
, channel
->efx
->net_dev
,
423 "creating channel %d\n", channel
->channel
);
425 rc
= efx_probe_eventq(channel
);
429 efx_for_each_channel_tx_queue(tx_queue
, channel
) {
430 rc
= efx_probe_tx_queue(tx_queue
);
435 efx_for_each_channel_rx_queue(rx_queue
, channel
) {
436 rc
= efx_probe_rx_queue(rx_queue
);
441 channel
->n_rx_frm_trunc
= 0;
446 efx_for_each_channel_rx_queue(rx_queue
, channel
)
447 efx_remove_rx_queue(rx_queue
);
449 efx_for_each_channel_tx_queue(tx_queue
, channel
)
450 efx_remove_tx_queue(tx_queue
);
456 static void efx_set_channel_names(struct efx_nic
*efx
)
458 struct efx_channel
*channel
;
459 const char *type
= "";
462 efx_for_each_channel(channel
, efx
) {
463 number
= channel
->channel
;
464 if (efx
->n_channels
> efx
->n_rx_channels
) {
465 if (channel
->channel
< efx
->n_rx_channels
) {
469 number
-= efx
->n_rx_channels
;
472 snprintf(channel
->name
, sizeof(channel
->name
),
473 "%s%s-%d", efx
->name
, type
, number
);
477 /* Channels are shutdown and reinitialised whilst the NIC is running
478 * to propagate configuration changes (mtu, checksum offload), or
479 * to clear hardware error conditions
481 static void efx_init_channels(struct efx_nic
*efx
)
483 struct efx_tx_queue
*tx_queue
;
484 struct efx_rx_queue
*rx_queue
;
485 struct efx_channel
*channel
;
487 /* Calculate the rx buffer allocation parameters required to
488 * support the current MTU, including padding for header
489 * alignment and overruns.
491 efx
->rx_buffer_len
= (max(EFX_PAGE_IP_ALIGN
, NET_IP_ALIGN
) +
492 EFX_MAX_FRAME_LEN(efx
->net_dev
->mtu
) +
493 efx
->type
->rx_buffer_hash_size
+
494 efx
->type
->rx_buffer_padding
);
495 efx
->rx_buffer_order
= get_order(efx
->rx_buffer_len
+
496 sizeof(struct efx_rx_page_state
));
498 /* Initialise the channels */
499 efx_for_each_channel(channel
, efx
) {
500 netif_dbg(channel
->efx
, drv
, channel
->efx
->net_dev
,
501 "init chan %d\n", channel
->channel
);
503 efx_init_eventq(channel
);
505 efx_for_each_channel_tx_queue(tx_queue
, channel
)
506 efx_init_tx_queue(tx_queue
);
508 /* The rx buffer allocation strategy is MTU dependent */
509 efx_rx_strategy(channel
);
511 efx_for_each_channel_rx_queue(rx_queue
, channel
)
512 efx_init_rx_queue(rx_queue
);
514 WARN_ON(channel
->rx_pkt
!= NULL
);
515 efx_rx_strategy(channel
);
519 /* This enables event queue processing and packet transmission.
521 * Note that this function is not allowed to fail, since that would
522 * introduce too much complexity into the suspend/resume path.
524 static void efx_start_channel(struct efx_channel
*channel
)
526 struct efx_rx_queue
*rx_queue
;
528 netif_dbg(channel
->efx
, ifup
, channel
->efx
->net_dev
,
529 "starting chan %d\n", channel
->channel
);
531 /* The interrupt handler for this channel may set work_pending
532 * as soon as we enable it. Make sure it's cleared before
533 * then. Similarly, make sure it sees the enabled flag set. */
534 channel
->work_pending
= false;
535 channel
->enabled
= true;
538 /* Fill the queues before enabling NAPI */
539 efx_for_each_channel_rx_queue(rx_queue
, channel
)
540 efx_fast_push_rx_descriptors(rx_queue
);
542 napi_enable(&channel
->napi_str
);
545 /* This disables event queue processing and packet transmission.
546 * This function does not guarantee that all queue processing
547 * (e.g. RX refill) is complete.
549 static void efx_stop_channel(struct efx_channel
*channel
)
551 if (!channel
->enabled
)
554 netif_dbg(channel
->efx
, ifdown
, channel
->efx
->net_dev
,
555 "stop chan %d\n", channel
->channel
);
557 channel
->enabled
= false;
558 napi_disable(&channel
->napi_str
);
561 static void efx_fini_channels(struct efx_nic
*efx
)
563 struct efx_channel
*channel
;
564 struct efx_tx_queue
*tx_queue
;
565 struct efx_rx_queue
*rx_queue
;
568 EFX_ASSERT_RESET_SERIALISED(efx
);
569 BUG_ON(efx
->port_enabled
);
571 rc
= efx_nic_flush_queues(efx
);
572 if (rc
&& EFX_WORKAROUND_7803(efx
)) {
573 /* Schedule a reset to recover from the flush failure. The
574 * descriptor caches reference memory we're about to free,
575 * but falcon_reconfigure_mac_wrapper() won't reconnect
576 * the MACs because of the pending reset. */
577 netif_err(efx
, drv
, efx
->net_dev
,
578 "Resetting to recover from flush failure\n");
579 efx_schedule_reset(efx
, RESET_TYPE_ALL
);
581 netif_err(efx
, drv
, efx
->net_dev
, "failed to flush queues\n");
583 netif_dbg(efx
, drv
, efx
->net_dev
,
584 "successfully flushed all queues\n");
587 efx_for_each_channel(channel
, efx
) {
588 netif_dbg(channel
->efx
, drv
, channel
->efx
->net_dev
,
589 "shut down chan %d\n", channel
->channel
);
591 efx_for_each_channel_rx_queue(rx_queue
, channel
)
592 efx_fini_rx_queue(rx_queue
);
593 efx_for_each_channel_tx_queue(tx_queue
, channel
)
594 efx_fini_tx_queue(tx_queue
);
595 efx_fini_eventq(channel
);
599 static void efx_remove_channel(struct efx_channel
*channel
)
601 struct efx_tx_queue
*tx_queue
;
602 struct efx_rx_queue
*rx_queue
;
604 netif_dbg(channel
->efx
, drv
, channel
->efx
->net_dev
,
605 "destroy chan %d\n", channel
->channel
);
607 efx_for_each_channel_rx_queue(rx_queue
, channel
)
608 efx_remove_rx_queue(rx_queue
);
609 efx_for_each_channel_tx_queue(tx_queue
, channel
)
610 efx_remove_tx_queue(tx_queue
);
611 efx_remove_eventq(channel
);
614 void efx_schedule_slow_fill(struct efx_rx_queue
*rx_queue
)
616 mod_timer(&rx_queue
->slow_fill
, jiffies
+ msecs_to_jiffies(100));
619 /**************************************************************************
623 **************************************************************************/
625 /* This ensures that the kernel is kept informed (via
626 * netif_carrier_on/off) of the link status, and also maintains the
627 * link status's stop on the port's TX queue.
629 void efx_link_status_changed(struct efx_nic
*efx
)
631 struct efx_link_state
*link_state
= &efx
->link_state
;
633 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure
634 * that no events are triggered between unregister_netdev() and the
635 * driver unloading. A more general condition is that NETDEV_CHANGE
636 * can only be generated between NETDEV_UP and NETDEV_DOWN */
637 if (!netif_running(efx
->net_dev
))
640 if (efx
->port_inhibited
) {
641 netif_carrier_off(efx
->net_dev
);
645 if (link_state
->up
!= netif_carrier_ok(efx
->net_dev
)) {
646 efx
->n_link_state_changes
++;
649 netif_carrier_on(efx
->net_dev
);
651 netif_carrier_off(efx
->net_dev
);
654 /* Status message for kernel log */
655 if (link_state
->up
) {
656 netif_info(efx
, link
, efx
->net_dev
,
657 "link up at %uMbps %s-duplex (MTU %d)%s\n",
658 link_state
->speed
, link_state
->fd
? "full" : "half",
660 (efx
->promiscuous
? " [PROMISC]" : ""));
662 netif_info(efx
, link
, efx
->net_dev
, "link down\n");
667 void efx_link_set_advertising(struct efx_nic
*efx
, u32 advertising
)
669 efx
->link_advertising
= advertising
;
671 if (advertising
& ADVERTISED_Pause
)
672 efx
->wanted_fc
|= (EFX_FC_TX
| EFX_FC_RX
);
674 efx
->wanted_fc
&= ~(EFX_FC_TX
| EFX_FC_RX
);
675 if (advertising
& ADVERTISED_Asym_Pause
)
676 efx
->wanted_fc
^= EFX_FC_TX
;
680 void efx_link_set_wanted_fc(struct efx_nic
*efx
, enum efx_fc_type wanted_fc
)
682 efx
->wanted_fc
= wanted_fc
;
683 if (efx
->link_advertising
) {
684 if (wanted_fc
& EFX_FC_RX
)
685 efx
->link_advertising
|= (ADVERTISED_Pause
|
686 ADVERTISED_Asym_Pause
);
688 efx
->link_advertising
&= ~(ADVERTISED_Pause
|
689 ADVERTISED_Asym_Pause
);
690 if (wanted_fc
& EFX_FC_TX
)
691 efx
->link_advertising
^= ADVERTISED_Asym_Pause
;
695 static void efx_fini_port(struct efx_nic
*efx
);
697 /* Push loopback/power/transmit disable settings to the PHY, and reconfigure
698 * the MAC appropriately. All other PHY configuration changes are pushed
699 * through phy_op->set_settings(), and pushed asynchronously to the MAC
700 * through efx_monitor().
702 * Callers must hold the mac_lock
704 int __efx_reconfigure_port(struct efx_nic
*efx
)
706 enum efx_phy_mode phy_mode
;
709 WARN_ON(!mutex_is_locked(&efx
->mac_lock
));
711 /* Serialise the promiscuous flag with efx_set_multicast_list. */
712 if (efx_dev_registered(efx
)) {
713 netif_addr_lock_bh(efx
->net_dev
);
714 netif_addr_unlock_bh(efx
->net_dev
);
717 /* Disable PHY transmit in mac level loopbacks */
718 phy_mode
= efx
->phy_mode
;
719 if (LOOPBACK_INTERNAL(efx
))
720 efx
->phy_mode
|= PHY_MODE_TX_DISABLED
;
722 efx
->phy_mode
&= ~PHY_MODE_TX_DISABLED
;
724 rc
= efx
->type
->reconfigure_port(efx
);
727 efx
->phy_mode
= phy_mode
;
732 /* Reinitialise the MAC to pick up new PHY settings, even if the port is
734 int efx_reconfigure_port(struct efx_nic
*efx
)
738 EFX_ASSERT_RESET_SERIALISED(efx
);
740 mutex_lock(&efx
->mac_lock
);
741 rc
= __efx_reconfigure_port(efx
);
742 mutex_unlock(&efx
->mac_lock
);
747 /* Asynchronous work item for changing MAC promiscuity and multicast
748 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current
750 static void efx_mac_work(struct work_struct
*data
)
752 struct efx_nic
*efx
= container_of(data
, struct efx_nic
, mac_work
);
754 mutex_lock(&efx
->mac_lock
);
755 if (efx
->port_enabled
) {
756 efx
->type
->push_multicast_hash(efx
);
757 efx
->mac_op
->reconfigure(efx
);
759 mutex_unlock(&efx
->mac_lock
);
762 static int efx_probe_port(struct efx_nic
*efx
)
766 netif_dbg(efx
, probe
, efx
->net_dev
, "create port\n");
769 efx
->phy_mode
= PHY_MODE_SPECIAL
;
771 /* Connect up MAC/PHY operations table */
772 rc
= efx
->type
->probe_port(efx
);
776 /* Sanity check MAC address */
777 if (is_valid_ether_addr(efx
->mac_address
)) {
778 memcpy(efx
->net_dev
->dev_addr
, efx
->mac_address
, ETH_ALEN
);
780 netif_err(efx
, probe
, efx
->net_dev
, "invalid MAC address %pM\n",
782 if (!allow_bad_hwaddr
) {
786 random_ether_addr(efx
->net_dev
->dev_addr
);
787 netif_info(efx
, probe
, efx
->net_dev
,
788 "using locally-generated MAC %pM\n",
789 efx
->net_dev
->dev_addr
);
795 efx
->type
->remove_port(efx
);
799 static int efx_init_port(struct efx_nic
*efx
)
803 netif_dbg(efx
, drv
, efx
->net_dev
, "init port\n");
805 mutex_lock(&efx
->mac_lock
);
807 rc
= efx
->phy_op
->init(efx
);
811 efx
->port_initialized
= true;
813 /* Reconfigure the MAC before creating dma queues (required for
814 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */
815 efx
->mac_op
->reconfigure(efx
);
817 /* Ensure the PHY advertises the correct flow control settings */
818 rc
= efx
->phy_op
->reconfigure(efx
);
822 mutex_unlock(&efx
->mac_lock
);
826 efx
->phy_op
->fini(efx
);
828 mutex_unlock(&efx
->mac_lock
);
832 static void efx_start_port(struct efx_nic
*efx
)
834 netif_dbg(efx
, ifup
, efx
->net_dev
, "start port\n");
835 BUG_ON(efx
->port_enabled
);
837 mutex_lock(&efx
->mac_lock
);
838 efx
->port_enabled
= true;
840 /* efx_mac_work() might have been scheduled after efx_stop_port(),
841 * and then cancelled by efx_flush_all() */
842 efx
->type
->push_multicast_hash(efx
);
843 efx
->mac_op
->reconfigure(efx
);
845 mutex_unlock(&efx
->mac_lock
);
848 /* Prevent efx_mac_work() and efx_monitor() from working */
849 static void efx_stop_port(struct efx_nic
*efx
)
851 netif_dbg(efx
, ifdown
, efx
->net_dev
, "stop port\n");
853 mutex_lock(&efx
->mac_lock
);
854 efx
->port_enabled
= false;
855 mutex_unlock(&efx
->mac_lock
);
857 /* Serialise against efx_set_multicast_list() */
858 if (efx_dev_registered(efx
)) {
859 netif_addr_lock_bh(efx
->net_dev
);
860 netif_addr_unlock_bh(efx
->net_dev
);
864 static void efx_fini_port(struct efx_nic
*efx
)
866 netif_dbg(efx
, drv
, efx
->net_dev
, "shut down port\n");
868 if (!efx
->port_initialized
)
871 efx
->phy_op
->fini(efx
);
872 efx
->port_initialized
= false;
874 efx
->link_state
.up
= false;
875 efx_link_status_changed(efx
);
878 static void efx_remove_port(struct efx_nic
*efx
)
880 netif_dbg(efx
, drv
, efx
->net_dev
, "destroying port\n");
882 efx
->type
->remove_port(efx
);
885 /**************************************************************************
889 **************************************************************************/
891 /* This configures the PCI device to enable I/O and DMA. */
892 static int efx_init_io(struct efx_nic
*efx
)
894 struct pci_dev
*pci_dev
= efx
->pci_dev
;
895 dma_addr_t dma_mask
= efx
->type
->max_dma_mask
;
898 netif_dbg(efx
, probe
, efx
->net_dev
, "initialising I/O\n");
900 rc
= pci_enable_device(pci_dev
);
902 netif_err(efx
, probe
, efx
->net_dev
,
903 "failed to enable PCI device\n");
907 pci_set_master(pci_dev
);
909 /* Set the PCI DMA mask. Try all possibilities from our
910 * genuine mask down to 32 bits, because some architectures
911 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit
912 * masks event though they reject 46 bit masks.
914 while (dma_mask
> 0x7fffffffUL
) {
915 if (pci_dma_supported(pci_dev
, dma_mask
) &&
916 ((rc
= pci_set_dma_mask(pci_dev
, dma_mask
)) == 0))
921 netif_err(efx
, probe
, efx
->net_dev
,
922 "could not find a suitable DMA mask\n");
925 netif_dbg(efx
, probe
, efx
->net_dev
,
926 "using DMA mask %llx\n", (unsigned long long) dma_mask
);
927 rc
= pci_set_consistent_dma_mask(pci_dev
, dma_mask
);
929 /* pci_set_consistent_dma_mask() is not *allowed* to
930 * fail with a mask that pci_set_dma_mask() accepted,
931 * but just in case...
933 netif_err(efx
, probe
, efx
->net_dev
,
934 "failed to set consistent DMA mask\n");
938 efx
->membase_phys
= pci_resource_start(efx
->pci_dev
, EFX_MEM_BAR
);
939 rc
= pci_request_region(pci_dev
, EFX_MEM_BAR
, "sfc");
941 netif_err(efx
, probe
, efx
->net_dev
,
942 "request for memory BAR failed\n");
946 efx
->membase
= ioremap_nocache(efx
->membase_phys
,
947 efx
->type
->mem_map_size
);
949 netif_err(efx
, probe
, efx
->net_dev
,
950 "could not map memory BAR at %llx+%x\n",
951 (unsigned long long)efx
->membase_phys
,
952 efx
->type
->mem_map_size
);
956 netif_dbg(efx
, probe
, efx
->net_dev
,
957 "memory BAR at %llx+%x (virtual %p)\n",
958 (unsigned long long)efx
->membase_phys
,
959 efx
->type
->mem_map_size
, efx
->membase
);
964 pci_release_region(efx
->pci_dev
, EFX_MEM_BAR
);
966 efx
->membase_phys
= 0;
968 pci_disable_device(efx
->pci_dev
);
973 static void efx_fini_io(struct efx_nic
*efx
)
975 netif_dbg(efx
, drv
, efx
->net_dev
, "shutting down I/O\n");
978 iounmap(efx
->membase
);
982 if (efx
->membase_phys
) {
983 pci_release_region(efx
->pci_dev
, EFX_MEM_BAR
);
984 efx
->membase_phys
= 0;
987 pci_disable_device(efx
->pci_dev
);
990 /* Get number of channels wanted. Each channel will have its own IRQ,
991 * 1 RX queue and/or 2 TX queues. */
992 static int efx_wanted_channels(void)
994 cpumask_var_t core_mask
;
998 if (unlikely(!zalloc_cpumask_var(&core_mask
, GFP_KERNEL
))) {
1000 "sfc: RSS disabled due to allocation failure\n");
1005 for_each_online_cpu(cpu
) {
1006 if (!cpumask_test_cpu(cpu
, core_mask
)) {
1008 cpumask_or(core_mask
, core_mask
,
1009 topology_core_cpumask(cpu
));
1013 free_cpumask_var(core_mask
);
1017 /* Probe the number and type of interrupts we are able to obtain, and
1018 * the resulting numbers of channels and RX queues.
1020 static void efx_probe_interrupts(struct efx_nic
*efx
)
1023 min_t(int, efx
->type
->phys_addr_channels
, EFX_MAX_CHANNELS
);
1026 if (efx
->interrupt_mode
== EFX_INT_MODE_MSIX
) {
1027 struct msix_entry xentries
[EFX_MAX_CHANNELS
];
1030 n_channels
= efx_wanted_channels();
1031 if (separate_tx_channels
)
1033 n_channels
= min(n_channels
, max_channels
);
1035 for (i
= 0; i
< n_channels
; i
++)
1036 xentries
[i
].entry
= i
;
1037 rc
= pci_enable_msix(efx
->pci_dev
, xentries
, n_channels
);
1039 netif_err(efx
, drv
, efx
->net_dev
,
1040 "WARNING: Insufficient MSI-X vectors"
1041 " available (%d < %d).\n", rc
, n_channels
);
1042 netif_err(efx
, drv
, efx
->net_dev
,
1043 "WARNING: Performance may be reduced.\n");
1044 EFX_BUG_ON_PARANOID(rc
>= n_channels
);
1046 rc
= pci_enable_msix(efx
->pci_dev
, xentries
,
1051 efx
->n_channels
= n_channels
;
1052 if (separate_tx_channels
) {
1053 efx
->n_tx_channels
=
1054 max(efx
->n_channels
/ 2, 1U);
1055 efx
->n_rx_channels
=
1056 max(efx
->n_channels
-
1057 efx
->n_tx_channels
, 1U);
1059 efx
->n_tx_channels
= efx
->n_channels
;
1060 efx
->n_rx_channels
= efx
->n_channels
;
1062 for (i
= 0; i
< n_channels
; i
++)
1063 efx_get_channel(efx
, i
)->irq
=
1066 /* Fall back to single channel MSI */
1067 efx
->interrupt_mode
= EFX_INT_MODE_MSI
;
1068 netif_err(efx
, drv
, efx
->net_dev
,
1069 "could not enable MSI-X\n");
1073 /* Try single interrupt MSI */
1074 if (efx
->interrupt_mode
== EFX_INT_MODE_MSI
) {
1075 efx
->n_channels
= 1;
1076 efx
->n_rx_channels
= 1;
1077 efx
->n_tx_channels
= 1;
1078 rc
= pci_enable_msi(efx
->pci_dev
);
1080 efx_get_channel(efx
, 0)->irq
= efx
->pci_dev
->irq
;
1082 netif_err(efx
, drv
, efx
->net_dev
,
1083 "could not enable MSI\n");
1084 efx
->interrupt_mode
= EFX_INT_MODE_LEGACY
;
1088 /* Assume legacy interrupts */
1089 if (efx
->interrupt_mode
== EFX_INT_MODE_LEGACY
) {
1090 efx
->n_channels
= 1 + (separate_tx_channels
? 1 : 0);
1091 efx
->n_rx_channels
= 1;
1092 efx
->n_tx_channels
= 1;
1093 efx
->legacy_irq
= efx
->pci_dev
->irq
;
1097 static void efx_remove_interrupts(struct efx_nic
*efx
)
1099 struct efx_channel
*channel
;
1101 /* Remove MSI/MSI-X interrupts */
1102 efx_for_each_channel(channel
, efx
)
1104 pci_disable_msi(efx
->pci_dev
);
1105 pci_disable_msix(efx
->pci_dev
);
1107 /* Remove legacy interrupt */
1108 efx
->legacy_irq
= 0;
1111 struct efx_tx_queue
*
1112 efx_get_tx_queue(struct efx_nic
*efx
, unsigned index
, unsigned type
)
1114 unsigned tx_channel_offset
=
1115 separate_tx_channels
? efx
->n_channels
- efx
->n_tx_channels
: 0;
1116 EFX_BUG_ON_PARANOID(index
>= efx
->n_tx_channels
||
1117 type
>= EFX_TXQ_TYPES
);
1118 return &efx
->channel
[tx_channel_offset
+ index
]->tx_queue
[type
];
1121 static void efx_set_channels(struct efx_nic
*efx
)
1123 struct efx_channel
*channel
;
1124 struct efx_tx_queue
*tx_queue
;
1125 unsigned tx_channel_offset
=
1126 separate_tx_channels
? efx
->n_channels
- efx
->n_tx_channels
: 0;
1128 /* Channel pointers were set in efx_init_struct() but we now
1129 * need to clear them for TX queues in any RX-only channels. */
1130 efx_for_each_channel(channel
, efx
) {
1131 if (channel
->channel
- tx_channel_offset
>=
1132 efx
->n_tx_channels
) {
1133 efx_for_each_channel_tx_queue(tx_queue
, channel
)
1134 tx_queue
->channel
= NULL
;
1139 static int efx_probe_nic(struct efx_nic
*efx
)
1144 netif_dbg(efx
, probe
, efx
->net_dev
, "creating NIC\n");
1146 /* Carry out hardware-type specific initialisation */
1147 rc
= efx
->type
->probe(efx
);
1151 /* Determine the number of channels and queues by trying to hook
1152 * in MSI-X interrupts. */
1153 efx_probe_interrupts(efx
);
1155 if (efx
->n_channels
> 1)
1156 get_random_bytes(&efx
->rx_hash_key
, sizeof(efx
->rx_hash_key
));
1157 for (i
= 0; i
< ARRAY_SIZE(efx
->rx_indir_table
); i
++)
1158 efx
->rx_indir_table
[i
] = i
% efx
->n_rx_channels
;
1160 efx_set_channels(efx
);
1161 efx
->net_dev
->real_num_tx_queues
= efx
->n_tx_channels
;
1163 /* Initialise the interrupt moderation settings */
1164 efx_init_irq_moderation(efx
, tx_irq_mod_usec
, rx_irq_mod_usec
, true);
1169 static void efx_remove_nic(struct efx_nic
*efx
)
1171 netif_dbg(efx
, drv
, efx
->net_dev
, "destroying NIC\n");
1173 efx_remove_interrupts(efx
);
1174 efx
->type
->remove(efx
);
1177 /**************************************************************************
1179 * NIC startup/shutdown
1181 *************************************************************************/
1183 static int efx_probe_all(struct efx_nic
*efx
)
1185 struct efx_channel
*channel
;
1189 rc
= efx_probe_nic(efx
);
1191 netif_err(efx
, probe
, efx
->net_dev
, "failed to create NIC\n");
1196 rc
= efx_probe_port(efx
);
1198 netif_err(efx
, probe
, efx
->net_dev
, "failed to create port\n");
1202 /* Create channels */
1203 efx
->rxq_entries
= efx
->txq_entries
= EFX_DEFAULT_DMAQ_SIZE
;
1204 efx_for_each_channel(channel
, efx
) {
1205 rc
= efx_probe_channel(channel
);
1207 netif_err(efx
, probe
, efx
->net_dev
,
1208 "failed to create channel %d\n",
1213 efx_set_channel_names(efx
);
1218 efx_for_each_channel(channel
, efx
)
1219 efx_remove_channel(channel
);
1220 efx_remove_port(efx
);
1222 efx_remove_nic(efx
);
1227 /* Called after previous invocation(s) of efx_stop_all, restarts the
1228 * port, kernel transmit queue, NAPI processing and hardware interrupts,
1229 * and ensures that the port is scheduled to be reconfigured.
1230 * This function is safe to call multiple times when the NIC is in any
1232 static void efx_start_all(struct efx_nic
*efx
)
1234 struct efx_channel
*channel
;
1236 EFX_ASSERT_RESET_SERIALISED(efx
);
1238 /* Check that it is appropriate to restart the interface. All
1239 * of these flags are safe to read under just the rtnl lock */
1240 if (efx
->port_enabled
)
1242 if ((efx
->state
!= STATE_RUNNING
) && (efx
->state
!= STATE_INIT
))
1244 if (efx_dev_registered(efx
) && !netif_running(efx
->net_dev
))
1247 /* Mark the port as enabled so port reconfigurations can start, then
1248 * restart the transmit interface early so the watchdog timer stops */
1249 efx_start_port(efx
);
1251 efx_for_each_channel(channel
, efx
) {
1252 if (efx_dev_registered(efx
))
1253 efx_wake_queue(channel
);
1254 efx_start_channel(channel
);
1257 efx_nic_enable_interrupts(efx
);
1259 /* Switch to event based MCDI completions after enabling interrupts.
1260 * If a reset has been scheduled, then we need to stay in polled mode.
1261 * Rather than serialising efx_mcdi_mode_event() [which sleeps] and
1262 * reset_pending [modified from an atomic context], we instead guarantee
1263 * that efx_mcdi_mode_poll() isn't reverted erroneously */
1264 efx_mcdi_mode_event(efx
);
1265 if (efx
->reset_pending
!= RESET_TYPE_NONE
)
1266 efx_mcdi_mode_poll(efx
);
1268 /* Start the hardware monitor if there is one. Otherwise (we're link
1269 * event driven), we have to poll the PHY because after an event queue
1270 * flush, we could have a missed a link state change */
1271 if (efx
->type
->monitor
!= NULL
) {
1272 queue_delayed_work(efx
->workqueue
, &efx
->monitor_work
,
1273 efx_monitor_interval
);
1275 mutex_lock(&efx
->mac_lock
);
1276 if (efx
->phy_op
->poll(efx
))
1277 efx_link_status_changed(efx
);
1278 mutex_unlock(&efx
->mac_lock
);
1281 efx
->type
->start_stats(efx
);
1284 /* Flush all delayed work. Should only be called when no more delayed work
1285 * will be scheduled. This doesn't flush pending online resets (efx_reset),
1286 * since we're holding the rtnl_lock at this point. */
1287 static void efx_flush_all(struct efx_nic
*efx
)
1289 /* Make sure the hardware monitor is stopped */
1290 cancel_delayed_work_sync(&efx
->monitor_work
);
1291 /* Stop scheduled port reconfigurations */
1292 cancel_work_sync(&efx
->mac_work
);
1295 /* Quiesce hardware and software without bringing the link down.
1296 * Safe to call multiple times, when the nic and interface is in any
1297 * state. The caller is guaranteed to subsequently be in a position
1298 * to modify any hardware and software state they see fit without
1300 static void efx_stop_all(struct efx_nic
*efx
)
1302 struct efx_channel
*channel
;
1304 EFX_ASSERT_RESET_SERIALISED(efx
);
1306 /* port_enabled can be read safely under the rtnl lock */
1307 if (!efx
->port_enabled
)
1310 efx
->type
->stop_stats(efx
);
1312 /* Switch to MCDI polling on Siena before disabling interrupts */
1313 efx_mcdi_mode_poll(efx
);
1315 /* Disable interrupts and wait for ISR to complete */
1316 efx_nic_disable_interrupts(efx
);
1317 if (efx
->legacy_irq
)
1318 synchronize_irq(efx
->legacy_irq
);
1319 efx_for_each_channel(channel
, efx
) {
1321 synchronize_irq(channel
->irq
);
1324 /* Stop all NAPI processing and synchronous rx refills */
1325 efx_for_each_channel(channel
, efx
)
1326 efx_stop_channel(channel
);
1328 /* Stop all asynchronous port reconfigurations. Since all
1329 * event processing has already been stopped, there is no
1330 * window to loose phy events */
1333 /* Flush efx_mac_work(), refill_workqueue, monitor_work */
1336 /* Stop the kernel transmit interface late, so the watchdog
1337 * timer isn't ticking over the flush */
1338 if (efx_dev_registered(efx
)) {
1339 struct efx_channel
*channel
;
1340 efx_for_each_channel(channel
, efx
)
1341 efx_stop_queue(channel
);
1342 netif_tx_lock_bh(efx
->net_dev
);
1343 netif_tx_unlock_bh(efx
->net_dev
);
1347 static void efx_remove_all(struct efx_nic
*efx
)
1349 struct efx_channel
*channel
;
1351 efx_for_each_channel(channel
, efx
)
1352 efx_remove_channel(channel
);
1353 efx_remove_port(efx
);
1354 efx_remove_nic(efx
);
1357 /**************************************************************************
1359 * Interrupt moderation
1361 **************************************************************************/
1363 static unsigned irq_mod_ticks(int usecs
, int resolution
)
1366 return 0; /* cannot receive interrupts ahead of time :-) */
1367 if (usecs
< resolution
)
1368 return 1; /* never round down to 0 */
1369 return usecs
/ resolution
;
1372 /* Set interrupt moderation parameters */
1373 void efx_init_irq_moderation(struct efx_nic
*efx
, int tx_usecs
, int rx_usecs
,
1376 struct efx_channel
*channel
;
1377 unsigned tx_ticks
= irq_mod_ticks(tx_usecs
, EFX_IRQ_MOD_RESOLUTION
);
1378 unsigned rx_ticks
= irq_mod_ticks(rx_usecs
, EFX_IRQ_MOD_RESOLUTION
);
1380 EFX_ASSERT_RESET_SERIALISED(efx
);
1382 efx
->irq_rx_adaptive
= rx_adaptive
;
1383 efx
->irq_rx_moderation
= rx_ticks
;
1384 efx_for_each_channel(channel
, efx
) {
1385 if (efx_channel_get_rx_queue(channel
))
1386 channel
->irq_moderation
= rx_ticks
;
1387 else if (efx_channel_get_tx_queue(channel
, 0))
1388 channel
->irq_moderation
= tx_ticks
;
1392 /**************************************************************************
1396 **************************************************************************/
1398 /* Run periodically off the general workqueue. Serialised against
1399 * efx_reconfigure_port via the mac_lock */
1400 static void efx_monitor(struct work_struct
*data
)
1402 struct efx_nic
*efx
= container_of(data
, struct efx_nic
,
1405 netif_vdbg(efx
, timer
, efx
->net_dev
,
1406 "hardware monitor executing on CPU %d\n",
1407 raw_smp_processor_id());
1408 BUG_ON(efx
->type
->monitor
== NULL
);
1410 /* If the mac_lock is already held then it is likely a port
1411 * reconfiguration is already in place, which will likely do
1412 * most of the work of check_hw() anyway. */
1413 if (!mutex_trylock(&efx
->mac_lock
))
1415 if (!efx
->port_enabled
)
1417 efx
->type
->monitor(efx
);
1420 mutex_unlock(&efx
->mac_lock
);
1422 queue_delayed_work(efx
->workqueue
, &efx
->monitor_work
,
1423 efx_monitor_interval
);
1426 /**************************************************************************
1430 *************************************************************************/
1433 * Context: process, rtnl_lock() held.
1435 static int efx_ioctl(struct net_device
*net_dev
, struct ifreq
*ifr
, int cmd
)
1437 struct efx_nic
*efx
= netdev_priv(net_dev
);
1438 struct mii_ioctl_data
*data
= if_mii(ifr
);
1440 EFX_ASSERT_RESET_SERIALISED(efx
);
1442 /* Convert phy_id from older PRTAD/DEVAD format */
1443 if ((cmd
== SIOCGMIIREG
|| cmd
== SIOCSMIIREG
) &&
1444 (data
->phy_id
& 0xfc00) == 0x0400)
1445 data
->phy_id
^= MDIO_PHY_ID_C45
| 0x0400;
1447 return mdio_mii_ioctl(&efx
->mdio
, data
, cmd
);
1450 /**************************************************************************
1454 **************************************************************************/
1456 static int efx_init_napi(struct efx_nic
*efx
)
1458 struct efx_channel
*channel
;
1460 efx_for_each_channel(channel
, efx
) {
1461 channel
->napi_dev
= efx
->net_dev
;
1462 netif_napi_add(channel
->napi_dev
, &channel
->napi_str
,
1463 efx_poll
, napi_weight
);
1468 static void efx_fini_napi(struct efx_nic
*efx
)
1470 struct efx_channel
*channel
;
1472 efx_for_each_channel(channel
, efx
) {
1473 if (channel
->napi_dev
)
1474 netif_napi_del(&channel
->napi_str
);
1475 channel
->napi_dev
= NULL
;
1479 /**************************************************************************
1481 * Kernel netpoll interface
1483 *************************************************************************/
1485 #ifdef CONFIG_NET_POLL_CONTROLLER
1487 /* Although in the common case interrupts will be disabled, this is not
1488 * guaranteed. However, all our work happens inside the NAPI callback,
1489 * so no locking is required.
1491 static void efx_netpoll(struct net_device
*net_dev
)
1493 struct efx_nic
*efx
= netdev_priv(net_dev
);
1494 struct efx_channel
*channel
;
1496 efx_for_each_channel(channel
, efx
)
1497 efx_schedule_channel(channel
);
1502 /**************************************************************************
1504 * Kernel net device interface
1506 *************************************************************************/
1508 /* Context: process, rtnl_lock() held. */
1509 static int efx_net_open(struct net_device
*net_dev
)
1511 struct efx_nic
*efx
= netdev_priv(net_dev
);
1512 EFX_ASSERT_RESET_SERIALISED(efx
);
1514 netif_dbg(efx
, ifup
, efx
->net_dev
, "opening device on CPU %d\n",
1515 raw_smp_processor_id());
1517 if (efx
->state
== STATE_DISABLED
)
1519 if (efx
->phy_mode
& PHY_MODE_SPECIAL
)
1521 if (efx_mcdi_poll_reboot(efx
) && efx_reset(efx
, RESET_TYPE_ALL
))
1524 /* Notify the kernel of the link state polled during driver load,
1525 * before the monitor starts running */
1526 efx_link_status_changed(efx
);
1532 /* Context: process, rtnl_lock() held.
1533 * Note that the kernel will ignore our return code; this method
1534 * should really be a void.
1536 static int efx_net_stop(struct net_device
*net_dev
)
1538 struct efx_nic
*efx
= netdev_priv(net_dev
);
1540 netif_dbg(efx
, ifdown
, efx
->net_dev
, "closing on CPU %d\n",
1541 raw_smp_processor_id());
1543 if (efx
->state
!= STATE_DISABLED
) {
1544 /* Stop the device and flush all the channels */
1546 efx_fini_channels(efx
);
1547 efx_init_channels(efx
);
1553 /* Context: process, dev_base_lock or RTNL held, non-blocking. */
1554 static struct rtnl_link_stats64
*efx_net_stats(struct net_device
*net_dev
, struct rtnl_link_stats64
*stats
)
1556 struct efx_nic
*efx
= netdev_priv(net_dev
);
1557 struct efx_mac_stats
*mac_stats
= &efx
->mac_stats
;
1559 spin_lock_bh(&efx
->stats_lock
);
1560 efx
->type
->update_stats(efx
);
1561 spin_unlock_bh(&efx
->stats_lock
);
1563 stats
->rx_packets
= mac_stats
->rx_packets
;
1564 stats
->tx_packets
= mac_stats
->tx_packets
;
1565 stats
->rx_bytes
= mac_stats
->rx_bytes
;
1566 stats
->tx_bytes
= mac_stats
->tx_bytes
;
1567 stats
->rx_dropped
= efx
->n_rx_nodesc_drop_cnt
;
1568 stats
->multicast
= mac_stats
->rx_multicast
;
1569 stats
->collisions
= mac_stats
->tx_collision
;
1570 stats
->rx_length_errors
= (mac_stats
->rx_gtjumbo
+
1571 mac_stats
->rx_length_error
);
1572 stats
->rx_crc_errors
= mac_stats
->rx_bad
;
1573 stats
->rx_frame_errors
= mac_stats
->rx_align_error
;
1574 stats
->rx_fifo_errors
= mac_stats
->rx_overflow
;
1575 stats
->rx_missed_errors
= mac_stats
->rx_missed
;
1576 stats
->tx_window_errors
= mac_stats
->tx_late_collision
;
1578 stats
->rx_errors
= (stats
->rx_length_errors
+
1579 stats
->rx_crc_errors
+
1580 stats
->rx_frame_errors
+
1581 mac_stats
->rx_symbol_error
);
1582 stats
->tx_errors
= (stats
->tx_window_errors
+
1588 /* Context: netif_tx_lock held, BHs disabled. */
1589 static void efx_watchdog(struct net_device
*net_dev
)
1591 struct efx_nic
*efx
= netdev_priv(net_dev
);
1593 netif_err(efx
, tx_err
, efx
->net_dev
,
1594 "TX stuck with port_enabled=%d: resetting channels\n",
1597 efx_schedule_reset(efx
, RESET_TYPE_TX_WATCHDOG
);
1601 /* Context: process, rtnl_lock() held. */
1602 static int efx_change_mtu(struct net_device
*net_dev
, int new_mtu
)
1604 struct efx_nic
*efx
= netdev_priv(net_dev
);
1607 EFX_ASSERT_RESET_SERIALISED(efx
);
1609 if (new_mtu
> EFX_MAX_MTU
)
1614 netif_dbg(efx
, drv
, efx
->net_dev
, "changing MTU to %d\n", new_mtu
);
1616 efx_fini_channels(efx
);
1618 mutex_lock(&efx
->mac_lock
);
1619 /* Reconfigure the MAC before enabling the dma queues so that
1620 * the RX buffers don't overflow */
1621 net_dev
->mtu
= new_mtu
;
1622 efx
->mac_op
->reconfigure(efx
);
1623 mutex_unlock(&efx
->mac_lock
);
1625 efx_init_channels(efx
);
1631 static int efx_set_mac_address(struct net_device
*net_dev
, void *data
)
1633 struct efx_nic
*efx
= netdev_priv(net_dev
);
1634 struct sockaddr
*addr
= data
;
1635 char *new_addr
= addr
->sa_data
;
1637 EFX_ASSERT_RESET_SERIALISED(efx
);
1639 if (!is_valid_ether_addr(new_addr
)) {
1640 netif_err(efx
, drv
, efx
->net_dev
,
1641 "invalid ethernet MAC address requested: %pM\n",
1646 memcpy(net_dev
->dev_addr
, new_addr
, net_dev
->addr_len
);
1648 /* Reconfigure the MAC */
1649 mutex_lock(&efx
->mac_lock
);
1650 efx
->mac_op
->reconfigure(efx
);
1651 mutex_unlock(&efx
->mac_lock
);
1656 /* Context: netif_addr_lock held, BHs disabled. */
1657 static void efx_set_multicast_list(struct net_device
*net_dev
)
1659 struct efx_nic
*efx
= netdev_priv(net_dev
);
1660 struct netdev_hw_addr
*ha
;
1661 union efx_multicast_hash
*mc_hash
= &efx
->multicast_hash
;
1665 efx
->promiscuous
= !!(net_dev
->flags
& IFF_PROMISC
);
1667 /* Build multicast hash table */
1668 if (efx
->promiscuous
|| (net_dev
->flags
& IFF_ALLMULTI
)) {
1669 memset(mc_hash
, 0xff, sizeof(*mc_hash
));
1671 memset(mc_hash
, 0x00, sizeof(*mc_hash
));
1672 netdev_for_each_mc_addr(ha
, net_dev
) {
1673 crc
= ether_crc_le(ETH_ALEN
, ha
->addr
);
1674 bit
= crc
& (EFX_MCAST_HASH_ENTRIES
- 1);
1675 set_bit_le(bit
, mc_hash
->byte
);
1678 /* Broadcast packets go through the multicast hash filter.
1679 * ether_crc_le() of the broadcast address is 0xbe2612ff
1680 * so we always add bit 0xff to the mask.
1682 set_bit_le(0xff, mc_hash
->byte
);
1685 if (efx
->port_enabled
)
1686 queue_work(efx
->workqueue
, &efx
->mac_work
);
1687 /* Otherwise efx_start_port() will do this */
1690 static const struct net_device_ops efx_netdev_ops
= {
1691 .ndo_open
= efx_net_open
,
1692 .ndo_stop
= efx_net_stop
,
1693 .ndo_get_stats64
= efx_net_stats
,
1694 .ndo_tx_timeout
= efx_watchdog
,
1695 .ndo_start_xmit
= efx_hard_start_xmit
,
1696 .ndo_validate_addr
= eth_validate_addr
,
1697 .ndo_do_ioctl
= efx_ioctl
,
1698 .ndo_change_mtu
= efx_change_mtu
,
1699 .ndo_set_mac_address
= efx_set_mac_address
,
1700 .ndo_set_multicast_list
= efx_set_multicast_list
,
1701 #ifdef CONFIG_NET_POLL_CONTROLLER
1702 .ndo_poll_controller
= efx_netpoll
,
1706 static void efx_update_name(struct efx_nic
*efx
)
1708 strcpy(efx
->name
, efx
->net_dev
->name
);
1709 efx_mtd_rename(efx
);
1710 efx_set_channel_names(efx
);
1713 static int efx_netdev_event(struct notifier_block
*this,
1714 unsigned long event
, void *ptr
)
1716 struct net_device
*net_dev
= ptr
;
1718 if (net_dev
->netdev_ops
== &efx_netdev_ops
&&
1719 event
== NETDEV_CHANGENAME
)
1720 efx_update_name(netdev_priv(net_dev
));
1725 static struct notifier_block efx_netdev_notifier
= {
1726 .notifier_call
= efx_netdev_event
,
1730 show_phy_type(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
1732 struct efx_nic
*efx
= pci_get_drvdata(to_pci_dev(dev
));
1733 return sprintf(buf
, "%d\n", efx
->phy_type
);
1735 static DEVICE_ATTR(phy_type
, 0644, show_phy_type
, NULL
);
1737 static int efx_register_netdev(struct efx_nic
*efx
)
1739 struct net_device
*net_dev
= efx
->net_dev
;
1742 net_dev
->watchdog_timeo
= 5 * HZ
;
1743 net_dev
->irq
= efx
->pci_dev
->irq
;
1744 net_dev
->netdev_ops
= &efx_netdev_ops
;
1745 SET_ETHTOOL_OPS(net_dev
, &efx_ethtool_ops
);
1747 /* Clear MAC statistics */
1748 efx
->mac_op
->update_stats(efx
);
1749 memset(&efx
->mac_stats
, 0, sizeof(efx
->mac_stats
));
1753 rc
= dev_alloc_name(net_dev
, net_dev
->name
);
1756 efx_update_name(efx
);
1758 rc
= register_netdevice(net_dev
);
1762 /* Always start with carrier off; PHY events will detect the link */
1763 netif_carrier_off(efx
->net_dev
);
1767 rc
= device_create_file(&efx
->pci_dev
->dev
, &dev_attr_phy_type
);
1769 netif_err(efx
, drv
, efx
->net_dev
,
1770 "failed to init net dev attributes\n");
1771 goto fail_registered
;
1778 netif_err(efx
, drv
, efx
->net_dev
, "could not register net dev\n");
1782 unregister_netdev(net_dev
);
1786 static void efx_unregister_netdev(struct efx_nic
*efx
)
1788 struct efx_channel
*channel
;
1789 struct efx_tx_queue
*tx_queue
;
1794 BUG_ON(netdev_priv(efx
->net_dev
) != efx
);
1796 /* Free up any skbs still remaining. This has to happen before
1797 * we try to unregister the netdev as running their destructors
1798 * may be needed to get the device ref. count to 0. */
1799 efx_for_each_channel(channel
, efx
) {
1800 efx_for_each_channel_tx_queue(tx_queue
, channel
)
1801 efx_release_tx_buffers(tx_queue
);
1804 if (efx_dev_registered(efx
)) {
1805 strlcpy(efx
->name
, pci_name(efx
->pci_dev
), sizeof(efx
->name
));
1806 device_remove_file(&efx
->pci_dev
->dev
, &dev_attr_phy_type
);
1807 unregister_netdev(efx
->net_dev
);
1811 /**************************************************************************
1813 * Device reset and suspend
1815 **************************************************************************/
1817 /* Tears down the entire software state and most of the hardware state
1819 void efx_reset_down(struct efx_nic
*efx
, enum reset_type method
)
1821 EFX_ASSERT_RESET_SERIALISED(efx
);
1824 mutex_lock(&efx
->mac_lock
);
1825 mutex_lock(&efx
->spi_lock
);
1827 efx_fini_channels(efx
);
1828 if (efx
->port_initialized
&& method
!= RESET_TYPE_INVISIBLE
)
1829 efx
->phy_op
->fini(efx
);
1830 efx
->type
->fini(efx
);
1833 /* This function will always ensure that the locks acquired in
1834 * efx_reset_down() are released. A failure return code indicates
1835 * that we were unable to reinitialise the hardware, and the
1836 * driver should be disabled. If ok is false, then the rx and tx
1837 * engines are not restarted, pending a RESET_DISABLE. */
1838 int efx_reset_up(struct efx_nic
*efx
, enum reset_type method
, bool ok
)
1842 EFX_ASSERT_RESET_SERIALISED(efx
);
1844 rc
= efx
->type
->init(efx
);
1846 netif_err(efx
, drv
, efx
->net_dev
, "failed to initialise NIC\n");
1853 if (efx
->port_initialized
&& method
!= RESET_TYPE_INVISIBLE
) {
1854 rc
= efx
->phy_op
->init(efx
);
1857 if (efx
->phy_op
->reconfigure(efx
))
1858 netif_err(efx
, drv
, efx
->net_dev
,
1859 "could not restore PHY settings\n");
1862 efx
->mac_op
->reconfigure(efx
);
1864 efx_init_channels(efx
);
1866 mutex_unlock(&efx
->spi_lock
);
1867 mutex_unlock(&efx
->mac_lock
);
1874 efx
->port_initialized
= false;
1876 mutex_unlock(&efx
->spi_lock
);
1877 mutex_unlock(&efx
->mac_lock
);
1882 /* Reset the NIC using the specified method. Note that the reset may
1883 * fail, in which case the card will be left in an unusable state.
1885 * Caller must hold the rtnl_lock.
1887 int efx_reset(struct efx_nic
*efx
, enum reset_type method
)
1892 netif_info(efx
, drv
, efx
->net_dev
, "resetting (%s)\n",
1893 RESET_TYPE(method
));
1895 efx_reset_down(efx
, method
);
1897 rc
= efx
->type
->reset(efx
, method
);
1899 netif_err(efx
, drv
, efx
->net_dev
, "failed to reset hardware\n");
1903 /* Allow resets to be rescheduled. */
1904 efx
->reset_pending
= RESET_TYPE_NONE
;
1906 /* Reinitialise bus-mastering, which may have been turned off before
1907 * the reset was scheduled. This is still appropriate, even in the
1908 * RESET_TYPE_DISABLE since this driver generally assumes the hardware
1909 * can respond to requests. */
1910 pci_set_master(efx
->pci_dev
);
1913 /* Leave device stopped if necessary */
1914 disabled
= rc
|| method
== RESET_TYPE_DISABLE
;
1915 rc2
= efx_reset_up(efx
, method
, !disabled
);
1923 dev_close(efx
->net_dev
);
1924 netif_err(efx
, drv
, efx
->net_dev
, "has been disabled\n");
1925 efx
->state
= STATE_DISABLED
;
1927 netif_dbg(efx
, drv
, efx
->net_dev
, "reset complete\n");
1932 /* The worker thread exists so that code that cannot sleep can
1933 * schedule a reset for later.
1935 static void efx_reset_work(struct work_struct
*data
)
1937 struct efx_nic
*efx
= container_of(data
, struct efx_nic
, reset_work
);
1939 if (efx
->reset_pending
== RESET_TYPE_NONE
)
1942 /* If we're not RUNNING then don't reset. Leave the reset_pending
1943 * flag set so that efx_pci_probe_main will be retried */
1944 if (efx
->state
!= STATE_RUNNING
) {
1945 netif_info(efx
, drv
, efx
->net_dev
,
1946 "scheduled reset quenched. NIC not RUNNING\n");
1951 (void)efx_reset(efx
, efx
->reset_pending
);
1955 void efx_schedule_reset(struct efx_nic
*efx
, enum reset_type type
)
1957 enum reset_type method
;
1959 if (efx
->reset_pending
!= RESET_TYPE_NONE
) {
1960 netif_info(efx
, drv
, efx
->net_dev
,
1961 "quenching already scheduled reset\n");
1966 case RESET_TYPE_INVISIBLE
:
1967 case RESET_TYPE_ALL
:
1968 case RESET_TYPE_WORLD
:
1969 case RESET_TYPE_DISABLE
:
1972 case RESET_TYPE_RX_RECOVERY
:
1973 case RESET_TYPE_RX_DESC_FETCH
:
1974 case RESET_TYPE_TX_DESC_FETCH
:
1975 case RESET_TYPE_TX_SKIP
:
1976 method
= RESET_TYPE_INVISIBLE
;
1978 case RESET_TYPE_MC_FAILURE
:
1980 method
= RESET_TYPE_ALL
;
1985 netif_dbg(efx
, drv
, efx
->net_dev
,
1986 "scheduling %s reset for %s\n",
1987 RESET_TYPE(method
), RESET_TYPE(type
));
1989 netif_dbg(efx
, drv
, efx
->net_dev
, "scheduling %s reset\n",
1990 RESET_TYPE(method
));
1992 efx
->reset_pending
= method
;
1994 /* efx_process_channel() will no longer read events once a
1995 * reset is scheduled. So switch back to poll'd MCDI completions. */
1996 efx_mcdi_mode_poll(efx
);
1998 queue_work(reset_workqueue
, &efx
->reset_work
);
2001 /**************************************************************************
2003 * List of NICs we support
2005 **************************************************************************/
2007 /* PCI device ID table */
2008 static DEFINE_PCI_DEVICE_TABLE(efx_pci_table
) = {
2009 {PCI_DEVICE(EFX_VENDID_SFC
, FALCON_A_P_DEVID
),
2010 .driver_data
= (unsigned long) &falcon_a1_nic_type
},
2011 {PCI_DEVICE(EFX_VENDID_SFC
, FALCON_B_P_DEVID
),
2012 .driver_data
= (unsigned long) &falcon_b0_nic_type
},
2013 {PCI_DEVICE(EFX_VENDID_SFC
, BETHPAGE_A_P_DEVID
),
2014 .driver_data
= (unsigned long) &siena_a0_nic_type
},
2015 {PCI_DEVICE(EFX_VENDID_SFC
, SIENA_A_P_DEVID
),
2016 .driver_data
= (unsigned long) &siena_a0_nic_type
},
2017 {0} /* end of list */
2020 /**************************************************************************
2022 * Dummy PHY/MAC operations
2024 * Can be used for some unimplemented operations
2025 * Needed so all function pointers are valid and do not have to be tested
2028 **************************************************************************/
2029 int efx_port_dummy_op_int(struct efx_nic
*efx
)
2033 void efx_port_dummy_op_void(struct efx_nic
*efx
) {}
2034 void efx_port_dummy_op_set_id_led(struct efx_nic
*efx
, enum efx_led_mode mode
)
2037 bool efx_port_dummy_op_poll(struct efx_nic
*efx
)
2042 static struct efx_phy_operations efx_dummy_phy_operations
= {
2043 .init
= efx_port_dummy_op_int
,
2044 .reconfigure
= efx_port_dummy_op_int
,
2045 .poll
= efx_port_dummy_op_poll
,
2046 .fini
= efx_port_dummy_op_void
,
2049 /**************************************************************************
2053 **************************************************************************/
2055 /* This zeroes out and then fills in the invariants in a struct
2056 * efx_nic (including all sub-structures).
2058 static int efx_init_struct(struct efx_nic
*efx
, struct efx_nic_type
*type
,
2059 struct pci_dev
*pci_dev
, struct net_device
*net_dev
)
2061 struct efx_channel
*channel
;
2062 struct efx_tx_queue
*tx_queue
;
2063 struct efx_rx_queue
*rx_queue
;
2066 /* Initialise common structures */
2067 memset(efx
, 0, sizeof(*efx
));
2068 spin_lock_init(&efx
->biu_lock
);
2069 mutex_init(&efx
->mdio_lock
);
2070 mutex_init(&efx
->spi_lock
);
2071 #ifdef CONFIG_SFC_MTD
2072 INIT_LIST_HEAD(&efx
->mtd_list
);
2074 INIT_WORK(&efx
->reset_work
, efx_reset_work
);
2075 INIT_DELAYED_WORK(&efx
->monitor_work
, efx_monitor
);
2076 efx
->pci_dev
= pci_dev
;
2077 efx
->msg_enable
= debug
;
2078 efx
->state
= STATE_INIT
;
2079 efx
->reset_pending
= RESET_TYPE_NONE
;
2080 strlcpy(efx
->name
, pci_name(pci_dev
), sizeof(efx
->name
));
2082 efx
->net_dev
= net_dev
;
2083 efx
->rx_checksum_enabled
= true;
2084 spin_lock_init(&efx
->stats_lock
);
2085 mutex_init(&efx
->mac_lock
);
2086 efx
->mac_op
= type
->default_mac_ops
;
2087 efx
->phy_op
= &efx_dummy_phy_operations
;
2088 efx
->mdio
.dev
= net_dev
;
2089 INIT_WORK(&efx
->mac_work
, efx_mac_work
);
2091 for (i
= 0; i
< EFX_MAX_CHANNELS
; i
++) {
2092 efx
->channel
[i
] = kzalloc(sizeof(*channel
), GFP_KERNEL
);
2093 channel
= efx
->channel
[i
];
2095 channel
->channel
= i
;
2096 spin_lock_init(&channel
->tx_stop_lock
);
2097 atomic_set(&channel
->tx_stop_count
, 1);
2099 for (j
= 0; j
< EFX_TXQ_TYPES
; j
++) {
2100 tx_queue
= &channel
->tx_queue
[j
];
2101 tx_queue
->efx
= efx
;
2102 tx_queue
->queue
= i
* EFX_TXQ_TYPES
+ j
;
2103 tx_queue
->channel
= channel
;
2106 rx_queue
= &channel
->rx_queue
;
2107 rx_queue
->efx
= efx
;
2108 setup_timer(&rx_queue
->slow_fill
, efx_rx_slow_fill
,
2109 (unsigned long)rx_queue
);
2114 EFX_BUG_ON_PARANOID(efx
->type
->phys_addr_channels
> EFX_MAX_CHANNELS
);
2116 /* Higher numbered interrupt modes are less capable! */
2117 efx
->interrupt_mode
= max(efx
->type
->max_interrupt_mode
,
2120 /* Would be good to use the net_dev name, but we're too early */
2121 snprintf(efx
->workqueue_name
, sizeof(efx
->workqueue_name
), "sfc%s",
2123 efx
->workqueue
= create_singlethread_workqueue(efx
->workqueue_name
);
2124 if (!efx
->workqueue
)
2130 static void efx_fini_struct(struct efx_nic
*efx
)
2134 for (i
= 0; i
< EFX_MAX_CHANNELS
; i
++)
2135 kfree(efx
->channel
[i
]);
2137 if (efx
->workqueue
) {
2138 destroy_workqueue(efx
->workqueue
);
2139 efx
->workqueue
= NULL
;
2143 /**************************************************************************
2147 **************************************************************************/
2149 /* Main body of final NIC shutdown code
2150 * This is called only at module unload (or hotplug removal).
2152 static void efx_pci_remove_main(struct efx_nic
*efx
)
2154 efx_nic_fini_interrupt(efx
);
2155 efx_fini_channels(efx
);
2157 efx
->type
->fini(efx
);
2159 efx_remove_all(efx
);
2162 /* Final NIC shutdown
2163 * This is called only at module unload (or hotplug removal).
2165 static void efx_pci_remove(struct pci_dev
*pci_dev
)
2167 struct efx_nic
*efx
;
2169 efx
= pci_get_drvdata(pci_dev
);
2173 /* Mark the NIC as fini, then stop the interface */
2175 efx
->state
= STATE_FINI
;
2176 dev_close(efx
->net_dev
);
2178 /* Allow any queued efx_resets() to complete */
2181 efx_unregister_netdev(efx
);
2183 efx_mtd_remove(efx
);
2185 /* Wait for any scheduled resets to complete. No more will be
2186 * scheduled from this point because efx_stop_all() has been
2187 * called, we are no longer registered with driverlink, and
2188 * the net_device's have been removed. */
2189 cancel_work_sync(&efx
->reset_work
);
2191 efx_pci_remove_main(efx
);
2194 netif_dbg(efx
, drv
, efx
->net_dev
, "shutdown successful\n");
2196 pci_set_drvdata(pci_dev
, NULL
);
2197 efx_fini_struct(efx
);
2198 free_netdev(efx
->net_dev
);
2201 /* Main body of NIC initialisation
2202 * This is called at module load (or hotplug insertion, theoretically).
2204 static int efx_pci_probe_main(struct efx_nic
*efx
)
2208 /* Do start-of-day initialisation */
2209 rc
= efx_probe_all(efx
);
2213 rc
= efx_init_napi(efx
);
2217 rc
= efx
->type
->init(efx
);
2219 netif_err(efx
, probe
, efx
->net_dev
,
2220 "failed to initialise NIC\n");
2224 rc
= efx_init_port(efx
);
2226 netif_err(efx
, probe
, efx
->net_dev
,
2227 "failed to initialise port\n");
2231 efx_init_channels(efx
);
2233 rc
= efx_nic_init_interrupt(efx
);
2240 efx_fini_channels(efx
);
2243 efx
->type
->fini(efx
);
2247 efx_remove_all(efx
);
2252 /* NIC initialisation
2254 * This is called at module load (or hotplug insertion,
2255 * theoretically). It sets up PCI mappings, tests and resets the NIC,
2256 * sets up and registers the network devices with the kernel and hooks
2257 * the interrupt service routine. It does not prepare the device for
2258 * transmission; this is left to the first time one of the network
2259 * interfaces is brought up (i.e. efx_net_open).
2261 static int __devinit
efx_pci_probe(struct pci_dev
*pci_dev
,
2262 const struct pci_device_id
*entry
)
2264 struct efx_nic_type
*type
= (struct efx_nic_type
*) entry
->driver_data
;
2265 struct net_device
*net_dev
;
2266 struct efx_nic
*efx
;
2269 /* Allocate and initialise a struct net_device and struct efx_nic */
2270 net_dev
= alloc_etherdev_mq(sizeof(*efx
), EFX_MAX_CORE_TX_QUEUES
);
2273 net_dev
->features
|= (type
->offload_features
| NETIF_F_SG
|
2274 NETIF_F_HIGHDMA
| NETIF_F_TSO
|
2276 if (type
->offload_features
& NETIF_F_V6_CSUM
)
2277 net_dev
->features
|= NETIF_F_TSO6
;
2278 /* Mask for features that also apply to VLAN devices */
2279 net_dev
->vlan_features
|= (NETIF_F_ALL_CSUM
| NETIF_F_SG
|
2280 NETIF_F_HIGHDMA
| NETIF_F_TSO
);
2281 efx
= netdev_priv(net_dev
);
2282 pci_set_drvdata(pci_dev
, efx
);
2283 SET_NETDEV_DEV(net_dev
, &pci_dev
->dev
);
2284 rc
= efx_init_struct(efx
, type
, pci_dev
, net_dev
);
2288 netif_info(efx
, probe
, efx
->net_dev
,
2289 "Solarflare Communications NIC detected\n");
2291 /* Set up basic I/O (BAR mappings etc) */
2292 rc
= efx_init_io(efx
);
2296 /* No serialisation is required with the reset path because
2297 * we're in STATE_INIT. */
2298 for (i
= 0; i
< 5; i
++) {
2299 rc
= efx_pci_probe_main(efx
);
2301 /* Serialise against efx_reset(). No more resets will be
2302 * scheduled since efx_stop_all() has been called, and we
2303 * have not and never have been registered with either
2304 * the rtnetlink or driverlink layers. */
2305 cancel_work_sync(&efx
->reset_work
);
2308 if (efx
->reset_pending
!= RESET_TYPE_NONE
) {
2309 /* If there was a scheduled reset during
2310 * probe, the NIC is probably hosed anyway */
2311 efx_pci_remove_main(efx
);
2318 /* Retry if a recoverably reset event has been scheduled */
2319 if ((efx
->reset_pending
!= RESET_TYPE_INVISIBLE
) &&
2320 (efx
->reset_pending
!= RESET_TYPE_ALL
))
2323 efx
->reset_pending
= RESET_TYPE_NONE
;
2327 netif_err(efx
, probe
, efx
->net_dev
, "Could not reset NIC\n");
2331 /* Switch to the running state before we expose the device to the OS,
2332 * so that dev_open()|efx_start_all() will actually start the device */
2333 efx
->state
= STATE_RUNNING
;
2335 rc
= efx_register_netdev(efx
);
2339 netif_dbg(efx
, probe
, efx
->net_dev
, "initialisation successful\n");
2342 efx_mtd_probe(efx
); /* allowed to fail */
2347 efx_pci_remove_main(efx
);
2352 efx_fini_struct(efx
);
2355 netif_dbg(efx
, drv
, efx
->net_dev
, "initialisation failed. rc=%d\n", rc
);
2356 free_netdev(net_dev
);
2360 static int efx_pm_freeze(struct device
*dev
)
2362 struct efx_nic
*efx
= pci_get_drvdata(to_pci_dev(dev
));
2364 efx
->state
= STATE_FINI
;
2366 netif_device_detach(efx
->net_dev
);
2369 efx_fini_channels(efx
);
2374 static int efx_pm_thaw(struct device
*dev
)
2376 struct efx_nic
*efx
= pci_get_drvdata(to_pci_dev(dev
));
2378 efx
->state
= STATE_INIT
;
2380 efx_init_channels(efx
);
2382 mutex_lock(&efx
->mac_lock
);
2383 efx
->phy_op
->reconfigure(efx
);
2384 mutex_unlock(&efx
->mac_lock
);
2388 netif_device_attach(efx
->net_dev
);
2390 efx
->state
= STATE_RUNNING
;
2392 efx
->type
->resume_wol(efx
);
2394 /* Reschedule any quenched resets scheduled during efx_pm_freeze() */
2395 queue_work(reset_workqueue
, &efx
->reset_work
);
2400 static int efx_pm_poweroff(struct device
*dev
)
2402 struct pci_dev
*pci_dev
= to_pci_dev(dev
);
2403 struct efx_nic
*efx
= pci_get_drvdata(pci_dev
);
2405 efx
->type
->fini(efx
);
2407 efx
->reset_pending
= RESET_TYPE_NONE
;
2409 pci_save_state(pci_dev
);
2410 return pci_set_power_state(pci_dev
, PCI_D3hot
);
2413 /* Used for both resume and restore */
2414 static int efx_pm_resume(struct device
*dev
)
2416 struct pci_dev
*pci_dev
= to_pci_dev(dev
);
2417 struct efx_nic
*efx
= pci_get_drvdata(pci_dev
);
2420 rc
= pci_set_power_state(pci_dev
, PCI_D0
);
2423 pci_restore_state(pci_dev
);
2424 rc
= pci_enable_device(pci_dev
);
2427 pci_set_master(efx
->pci_dev
);
2428 rc
= efx
->type
->reset(efx
, RESET_TYPE_ALL
);
2431 rc
= efx
->type
->init(efx
);
2438 static int efx_pm_suspend(struct device
*dev
)
2443 rc
= efx_pm_poweroff(dev
);
2449 static struct dev_pm_ops efx_pm_ops
= {
2450 .suspend
= efx_pm_suspend
,
2451 .resume
= efx_pm_resume
,
2452 .freeze
= efx_pm_freeze
,
2453 .thaw
= efx_pm_thaw
,
2454 .poweroff
= efx_pm_poweroff
,
2455 .restore
= efx_pm_resume
,
2458 static struct pci_driver efx_pci_driver
= {
2459 .name
= KBUILD_MODNAME
,
2460 .id_table
= efx_pci_table
,
2461 .probe
= efx_pci_probe
,
2462 .remove
= efx_pci_remove
,
2463 .driver
.pm
= &efx_pm_ops
,
2466 /**************************************************************************
2468 * Kernel module interface
2470 *************************************************************************/
2472 module_param(interrupt_mode
, uint
, 0444);
2473 MODULE_PARM_DESC(interrupt_mode
,
2474 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");
2476 static int __init
efx_init_module(void)
2480 printk(KERN_INFO
"Solarflare NET driver v" EFX_DRIVER_VERSION
"\n");
2482 rc
= register_netdevice_notifier(&efx_netdev_notifier
);
2486 reset_workqueue
= create_singlethread_workqueue("sfc_reset");
2487 if (!reset_workqueue
) {
2492 rc
= pci_register_driver(&efx_pci_driver
);
2499 destroy_workqueue(reset_workqueue
);
2501 unregister_netdevice_notifier(&efx_netdev_notifier
);
2506 static void __exit
efx_exit_module(void)
2508 printk(KERN_INFO
"Solarflare NET driver unloading\n");
2510 pci_unregister_driver(&efx_pci_driver
);
2511 destroy_workqueue(reset_workqueue
);
2512 unregister_netdevice_notifier(&efx_netdev_notifier
);
2516 module_init(efx_init_module
);
2517 module_exit(efx_exit_module
);
2519 MODULE_AUTHOR("Solarflare Communications and "
2520 "Michael Brown <mbrown@fensystems.co.uk>");
2521 MODULE_DESCRIPTION("Solarflare Communications network driver");
2522 MODULE_LICENSE("GPL");
2523 MODULE_DEVICE_TABLE(pci
, efx_pci_table
);