projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
sfc: Add use of shared RSS contexts.
[deliverable/linux.git] / drivers / net / ethernet / sfc / efx.c
CommitLineData
8ceee660 1/****************************************************************************
f7a6d2c4 2 * Driver for Solarflare network controllers and boards
8ceee660 3 * Copyright 2005-2006 Fen Systems Ltd.
f7a6d2c4 4 * Copyright 2005-2013 Solarflare Communications Inc.
8ceee660
BH		 5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
9 */
10
11#include <linux/module.h>
12#include <linux/pci.h>
13#include <linux/netdevice.h>
14#include <linux/etherdevice.h>
15#include <linux/delay.h>
16#include <linux/notifier.h>
17#include <linux/ip.h>
18#include <linux/tcp.h>
19#include <linux/in.h>
8ceee660 20#include <linux/ethtool.h>
aa6ef27e 21#include <linux/topology.h>
5a0e3ad6 22#include <linux/gfp.h>
626950db 23#include <linux/aer.h>
b28405b0 24#include <linux/interrupt.h>
8ceee660 25#include "net_driver.h"
8ceee660 26#include "efx.h"
744093c9 27#include "nic.h"
dd40781e 28#include "selftest.h"
7fa8d547 29#include "sriov.h"
8ceee660 30
8880f4ec 31#include "mcdi.h"
fd371e32 32#include "workarounds.h"
8880f4ec 33
c459302d
BH		 34/**************************************************************************
35 *
36 * Type name strings
37 *
38 **************************************************************************
39 */
40
41/* Loopback mode names (see LOOPBACK_MODE()) */
42const unsigned int efx_loopback_mode_max = LOOPBACK_MAX;
18e83e4c 43const char *const efx_loopback_mode_names[] = {
c459302d 44 [LOOPBACK_NONE] = "NONE",
e58f69f4 45 [LOOPBACK_DATA] = "DATAPATH",
c459302d
BH		 46 [LOOPBACK_GMAC] = "GMAC",
47 [LOOPBACK_XGMII] = "XGMII",
48 [LOOPBACK_XGXS] = "XGXS",
9c636baf
BH		 49 [LOOPBACK_XAUI] = "XAUI",
50 [LOOPBACK_GMII] = "GMII",
51 [LOOPBACK_SGMII] = "SGMII",
e58f69f4
BH		 52 [LOOPBACK_XGBR] = "XGBR",
53 [LOOPBACK_XFI] = "XFI",
54 [LOOPBACK_XAUI_FAR] = "XAUI_FAR",
55 [LOOPBACK_GMII_FAR] = "GMII_FAR",
56 [LOOPBACK_SGMII_FAR] = "SGMII_FAR",
57 [LOOPBACK_XFI_FAR] = "XFI_FAR",
c459302d
BH		 58 [LOOPBACK_GPHY] = "GPHY",
59 [LOOPBACK_PHYXS] = "PHYXS",
9c636baf
BH		 60 [LOOPBACK_PCS] = "PCS",
61 [LOOPBACK_PMAPMD] = "PMA/PMD",
e58f69f4
BH		 62 [LOOPBACK_XPORT] = "XPORT",
63 [LOOPBACK_XGMII_WS] = "XGMII_WS",
9c636baf 64 [LOOPBACK_XAUI_WS] = "XAUI_WS",
e58f69f4
BH		 65 [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR",
66 [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR",
9c636baf 67 [LOOPBACK_GMII_WS] = "GMII_WS",
e58f69f4
BH		 68 [LOOPBACK_XFI_WS] = "XFI_WS",
69 [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR",
9c636baf 70 [LOOPBACK_PHYXS_WS] = "PHYXS_WS",
c459302d
BH		 71};
72
c459302d 73const unsigned int efx_reset_type_max = RESET_TYPE_MAX;
18e83e4c 74const char *const efx_reset_type_names[] = {
626950db
AR		 75 [RESET_TYPE_INVISIBLE] = "INVISIBLE",
76 [RESET_TYPE_ALL] = "ALL",
77 [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL",
78 [RESET_TYPE_WORLD] = "WORLD",
79 [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE",
e283546c 80 [RESET_TYPE_MC_BIST] = "MC_BIST",
626950db
AR		 81 [RESET_TYPE_DISABLE] = "DISABLE",
82 [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG",
83 [RESET_TYPE_INT_ERROR] = "INT_ERROR",
84 [RESET_TYPE_RX_RECOVERY] = "RX_RECOVERY",
3de82b91 85 [RESET_TYPE_DMA_ERROR] = "DMA_ERROR",
626950db
AR		 86 [RESET_TYPE_TX_SKIP] = "TX_SKIP",
87 [RESET_TYPE_MC_FAILURE] = "MC_FAILURE",
e283546c 88 [RESET_TYPE_MCDI_TIMEOUT] = "MCDI_TIMEOUT (FLR)",
c459302d
BH		 89};
90
1ab00629
SH		 91/* Reset workqueue. If any NIC has a hardware failure then a reset will be
92 * queued onto this work queue. This is not a per-nic work queue, because
93 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.
94 */
95static struct workqueue_struct *reset_workqueue;
96
74cd60a4
JC		 97/* How often and how many times to poll for a reset while waiting for a
98 * BIST that another function started to complete.
99 */
100#define BIST_WAIT_DELAY_MS 100
101#define BIST_WAIT_DELAY_COUNT 100
102
8ceee660
BH		 103/**************************************************************************
104 *
105 * Configurable values
106 *
107 *************************************************************************/
108
8ceee660
BH		 109/*
110 * Use separate channels for TX and RX events
111 *
28b581ab
NT		 112 * Set this to 1 to use separate channels for TX and RX. It allows us
113 * to control interrupt affinity separately for TX and RX.
8ceee660 114 *
28b581ab 115 * This is only used in MSI-X interrupt mode
8ceee660 116 */
b9cc977d
BH		 117static bool separate_tx_channels;
118module_param(separate_tx_channels, bool, 0444);
28b581ab
NT		 119MODULE_PARM_DESC(separate_tx_channels,
120 "Use separate channels for TX and RX");
8ceee660
BH		 121
122/* This is the weight assigned to each of the (per-channel) virtual
123 * NAPI devices.
124 */
125static int napi_weight = 64;
126
127/* This is the time (in jiffies) between invocations of the hardware
626950db
AR		 128 * monitor.
129 * On Falcon-based NICs, this will:
e254c274
BH		 130 * - Check the on-board hardware monitor;
131 * - Poll the link state and reconfigure the hardware as necessary.
626950db
AR		 132 * On Siena-based NICs for power systems with EEH support, this will give EEH a
133 * chance to start.
8ceee660 134 */
d215697f 135static unsigned int efx_monitor_interval = 1 * HZ;
8ceee660 136
8ceee660
BH		 137/* Initial interrupt moderation settings. They can be modified after
138 * module load with ethtool.
139 *
140 * The default for RX should strike a balance between increasing the
141 * round-trip latency and reducing overhead.
142 */
143static unsigned int rx_irq_mod_usec = 60;
144
145/* Initial interrupt moderation settings. They can be modified after
146 * module load with ethtool.
147 *
148 * This default is chosen to ensure that a 10G link does not go idle
149 * while a TX queue is stopped after it has become full. A queue is
150 * restarted when it drops below half full. The time this takes (assuming
151 * worst case 3 descriptors per packet and 1024 descriptors) is
152 * 512 / 3 * 1.2 = 205 usec.
153 */
154static unsigned int tx_irq_mod_usec = 150;
155
156/* This is the first interrupt mode to try out of:
157 * 0 => MSI-X
158 * 1 => MSI
159 * 2 => legacy
160 */
161static unsigned int interrupt_mode;
162
163/* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
164 * i.e. the number of CPUs among which we may distribute simultaneous
165 * interrupt handling.
166 *
167 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
cdb08f8f 168 * The default (0) means to assign an interrupt to each core.
8ceee660
BH		 169 */
170static unsigned int rss_cpus;
171module_param(rss_cpus, uint, 0444);
172MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");
173
b9cc977d
BH		 174static bool phy_flash_cfg;
175module_param(phy_flash_cfg, bool, 0644);
84ae48fe
BH		 176MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");
177
e7bed9c8 178static unsigned irq_adapt_low_thresh = 8000;
6fb70fd1
BH		 179module_param(irq_adapt_low_thresh, uint, 0644);
180MODULE_PARM_DESC(irq_adapt_low_thresh,
181 "Threshold score for reducing IRQ moderation");
182
e7bed9c8 183static unsigned irq_adapt_high_thresh = 16000;
6fb70fd1
BH		 184module_param(irq_adapt_high_thresh, uint, 0644);
185MODULE_PARM_DESC(irq_adapt_high_thresh,
186 "Threshold score for increasing IRQ moderation");
187
62776d03
BH		 188static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
189 NETIF_MSG_LINK | NETIF_MSG_IFDOWN |
190 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR |
191 NETIF_MSG_TX_ERR | NETIF_MSG_HW);
192module_param(debug, uint, 0);
193MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value");
194
8ceee660
BH		 195/**************************************************************************
196 *
197 * Utility functions and prototypes
198 *
199 *************************************************************************/
4642610c 200
261e4d96 201static int efx_soft_enable_interrupts(struct efx_nic *efx);
d8291187 202static void efx_soft_disable_interrupts(struct efx_nic *efx);
7f967c01 203static void efx_remove_channel(struct efx_channel *channel);
4642610c 204static void efx_remove_channels(struct efx_nic *efx);
7f967c01 205static const struct efx_channel_type efx_default_channel_type;
8ceee660 206static void efx_remove_port(struct efx_nic *efx);
7f967c01 207static void efx_init_napi_channel(struct efx_channel *channel);
8ceee660 208static void efx_fini_napi(struct efx_nic *efx);
e8f14992 209static void efx_fini_napi_channel(struct efx_channel *channel);
4642610c
BH		 210static void efx_fini_struct(struct efx_nic *efx);
211static void efx_start_all(struct efx_nic *efx);
212static void efx_stop_all(struct efx_nic *efx);
8ceee660
BH		 213
214#define EFX_ASSERT_RESET_SERIALISED(efx) \
215 do { \
f16aeea0 216 if ((efx->state == STATE_READY) || \
626950db 217 (efx->state == STATE_RECOVERY) || \
332c1ce9 218 (efx->state == STATE_DISABLED)) \
8ceee660
BH		 219 ASSERT_RTNL(); \
220 } while (0)
221
8b7325b4
BH		 222static int efx_check_disabled(struct efx_nic *efx)
223{
626950db 224 if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) {
8b7325b4
BH		 225 netif_err(efx, drv, efx->net_dev,
226 "device is disabled due to earlier errors\n");
227 return -EIO;
228 }
229 return 0;
230}
231
8ceee660
BH		 232/**************************************************************************
233 *
234 * Event queue processing
235 *
236 *************************************************************************/
237
238/* Process channel's event queue
239 *
240 * This function is responsible for processing the event queue of a
241 * single channel. The caller must guarantee that this function will
242 * never be concurrently called more than once on the same channel,
243 * though different channels may be being processed concurrently.
244 */
fa236e18 245static int efx_process_channel(struct efx_channel *channel, int budget)
8ceee660 246{
fa236e18 247 int spent;
8ceee660 248
9f2cb71c 249 if (unlikely(!channel->enabled))
42cbe2d7 250 return 0;
8ceee660 251
fa236e18 252 spent = efx_nic_process_eventq(channel, budget);
d9ab7007
BH		 253 if (spent && efx_channel_has_rx_queue(channel)) {
254 struct efx_rx_queue *rx_queue =
255 efx_channel_get_rx_queue(channel);
256
ff734ef4 257 efx_rx_flush_packet(channel);
cce28794 258 efx_fast_push_rx_descriptors(rx_queue, true);
8ceee660
BH		 259 }
260
fa236e18 261 return spent;
8ceee660
BH		 262}
263
8ceee660
BH		 264/* NAPI poll handler
265 *
266 * NAPI guarantees serialisation of polls of the same device, which
267 * provides the guarantee required by efx_process_channel().
268 */
269static int efx_poll(struct napi_struct *napi, int budget)
270{
271 struct efx_channel *channel =
272 container_of(napi, struct efx_channel, napi_str);
62776d03 273 struct efx_nic *efx = channel->efx;
fa236e18 274 int spent;
8ceee660 275
36763266
AR		 276 if (!efx_channel_lock_napi(channel))
277 return budget;
278
62776d03
BH		 279 netif_vdbg(efx, intr, efx->net_dev,
280 "channel %d NAPI poll executing on CPU %d\n",
281 channel->channel, raw_smp_processor_id());
8ceee660 282
fa236e18 283 spent = efx_process_channel(channel, budget);
8ceee660 284
fa236e18 285 if (spent < budget) {
9d9a6973 286 if (efx_channel_has_rx_queue(channel) &&
6fb70fd1
BH		 287 efx->irq_rx_adaptive &&
288 unlikely(++channel->irq_count == 1000)) {
6fb70fd1
BH		 289 if (unlikely(channel->irq_mod_score <
290 irq_adapt_low_thresh)) {
0d86ebd8
BH		 291 if (channel->irq_moderation > 1) {
292 channel->irq_moderation -= 1;
ef2b90ee 293 efx->type->push_irq_moderation(channel);
0d86ebd8 294 }
6fb70fd1
BH		 295 } else if (unlikely(channel->irq_mod_score >
296 irq_adapt_high_thresh)) {
0d86ebd8
BH		 297 if (channel->irq_moderation <
298 efx->irq_rx_moderation) {
299 channel->irq_moderation += 1;
ef2b90ee 300 efx->type->push_irq_moderation(channel);
0d86ebd8 301 }
6fb70fd1 302 }
6fb70fd1
BH		 303 channel->irq_count = 0;
304 channel->irq_mod_score = 0;
305 }
306
64d8ad6d
BH		 307 efx_filter_rfs_expire(channel);
308
8ceee660 309 /* There is no race here; although napi_disable() will
288379f0 310 * only wait for napi_complete(), this isn't a problem
514bedbc 311 * since efx_nic_eventq_read_ack() will have no effect if
8ceee660
BH		 312 * interrupts have already been disabled.
313 */
288379f0 314 napi_complete(napi);
514bedbc 315 efx_nic_eventq_read_ack(channel);
8ceee660
BH		 316 }
317
36763266 318 efx_channel_unlock_napi(channel);
fa236e18 319 return spent;
8ceee660
BH		 320}
321
8ceee660
BH		 322/* Create event queue
323 * Event queue memory allocations are done only once. If the channel
324 * is reset, the memory buffer will be reused; this guards against
325 * errors during channel reset and also simplifies interrupt handling.
326 */
327static int efx_probe_eventq(struct efx_channel *channel)
328{
ecc910f5
SH		 329 struct efx_nic *efx = channel->efx;
330 unsigned long entries;
331
86ee5302 332 netif_dbg(efx, probe, efx->net_dev,
62776d03 333 "chan %d create event queue\n", channel->channel);
8ceee660 334
ecc910f5
SH		 335 /* Build an event queue with room for one event per tx and rx buffer,
336 * plus some extra for link state events and MCDI completions. */
337 entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
338 EFX_BUG_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
339 channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
340
152b6a62 341 return efx_nic_probe_eventq(channel);
8ceee660
BH		 342}
343
344/* Prepare channel's event queue */
261e4d96 345static int efx_init_eventq(struct efx_channel *channel)
8ceee660 346{
15acb1ce 347 struct efx_nic *efx = channel->efx;
261e4d96
JC		 348 int rc;
349
350 EFX_WARN_ON_PARANOID(channel->eventq_init);
351
15acb1ce 352 netif_dbg(efx, drv, efx->net_dev,
62776d03 353 "chan %d init event queue\n", channel->channel);
8ceee660 354
261e4d96
JC		 355 rc = efx_nic_init_eventq(channel);
356 if (rc == 0) {
15acb1ce 357 efx->type->push_irq_moderation(channel);
261e4d96
JC		 358 channel->eventq_read_ptr = 0;
359 channel->eventq_init = true;
360 }
361 return rc;
8ceee660
BH		 362}
363
9f2cb71c 364/* Enable event queue processing and NAPI */
36763266 365void efx_start_eventq(struct efx_channel *channel)
9f2cb71c
BH		 366{
367 netif_dbg(channel->efx, ifup, channel->efx->net_dev,
368 "chan %d start event queue\n", channel->channel);
369
514bedbc 370 /* Make sure the NAPI handler sees the enabled flag set */
9f2cb71c
BH		 371 channel->enabled = true;
372 smp_wmb();
373
36763266 374 efx_channel_enable(channel);
9f2cb71c
BH		 375 napi_enable(&channel->napi_str);
376 efx_nic_eventq_read_ack(channel);
377}
378
379/* Disable event queue processing and NAPI */
36763266 380void efx_stop_eventq(struct efx_channel *channel)
9f2cb71c
BH		 381{
382 if (!channel->enabled)
383 return;
384
385 napi_disable(&channel->napi_str);
36763266
AR		 386 while (!efx_channel_disable(channel))
387 usleep_range(1000, 20000);
9f2cb71c
BH		 388 channel->enabled = false;
389}
390
8ceee660
BH		 391static void efx_fini_eventq(struct efx_channel *channel)
392{
be3fc09c
BH		 393 if (!channel->eventq_init)
394 return;
395
62776d03
BH		 396 netif_dbg(channel->efx, drv, channel->efx->net_dev,
397 "chan %d fini event queue\n", channel->channel);
8ceee660 398
152b6a62 399 efx_nic_fini_eventq(channel);
be3fc09c 400 channel->eventq_init = false;
8ceee660
BH		 401}
402
403static void efx_remove_eventq(struct efx_channel *channel)
404{
62776d03
BH		 405 netif_dbg(channel->efx, drv, channel->efx->net_dev,
406 "chan %d remove event queue\n", channel->channel);
8ceee660 407
152b6a62 408 efx_nic_remove_eventq(channel);
8ceee660
BH		 409}
410
411/**************************************************************************
412 *
413 * Channel handling
414 *
415 *************************************************************************/
416
7f967c01 417/* Allocate and initialise a channel structure. */
4642610c
BH		 418static struct efx_channel *
419efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel)
420{
421 struct efx_channel *channel;
422 struct efx_rx_queue *rx_queue;
423 struct efx_tx_queue *tx_queue;
424 int j;
425
7f967c01
BH		 426 channel = kzalloc(sizeof(*channel), GFP_KERNEL);
427 if (!channel)
428 return NULL;
4642610c 429
7f967c01
BH		 430 channel->efx = efx;
431 channel->channel = i;
432 channel->type = &efx_default_channel_type;
4642610c 433
7f967c01
BH		 434 for (j = 0; j < EFX_TXQ_TYPES; j++) {
435 tx_queue = &channel->tx_queue[j];
436 tx_queue->efx = efx;
437 tx_queue->queue = i * EFX_TXQ_TYPES + j;
438 tx_queue->channel = channel;
439 }
4642610c 440
7f967c01
BH		 441 rx_queue = &channel->rx_queue;
442 rx_queue->efx = efx;
443 setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill,
444 (unsigned long)rx_queue);
4642610c 445
7f967c01
BH		 446 return channel;
447}
448
449/* Allocate and initialise a channel structure, copying parameters
450 * (but not resources) from an old channel structure.
451 */
452static struct efx_channel *
453efx_copy_channel(const struct efx_channel *old_channel)
454{
455 struct efx_channel *channel;
456 struct efx_rx_queue *rx_queue;
457 struct efx_tx_queue *tx_queue;
458 int j;
4642610c 459
7f967c01
BH		 460 channel = kmalloc(sizeof(*channel), GFP_KERNEL);
461 if (!channel)
462 return NULL;
463
464 *channel = *old_channel;
465
466 channel->napi_dev = NULL;
467 memset(&channel->eventq, 0, sizeof(channel->eventq));
4642610c 468
7f967c01
BH		 469 for (j = 0; j < EFX_TXQ_TYPES; j++) {
470 tx_queue = &channel->tx_queue[j];
471 if (tx_queue->channel)
4642610c 472 tx_queue->channel = channel;
7f967c01
BH		 473 tx_queue->buffer = NULL;
474 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
4642610c
BH		 475 }
476
4642610c 477 rx_queue = &channel->rx_queue;
7f967c01
BH		 478 rx_queue->buffer = NULL;
479 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
4642610c
BH		 480 setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill,
481 (unsigned long)rx_queue);
482
483 return channel;
484}
485
8ceee660
BH		 486static int efx_probe_channel(struct efx_channel *channel)
487{
488 struct efx_tx_queue *tx_queue;
489 struct efx_rx_queue *rx_queue;
490 int rc;
491
62776d03
BH		 492 netif_dbg(channel->efx, probe, channel->efx->net_dev,
493 "creating channel %d\n", channel->channel);
8ceee660 494
7f967c01
BH		 495 rc = channel->type->pre_probe(channel);
496 if (rc)
497 goto fail;
498
8ceee660
BH		 499 rc = efx_probe_eventq(channel);
500 if (rc)
7f967c01 501 goto fail;
8ceee660
BH		 502
503 efx_for_each_channel_tx_queue(tx_queue, channel) {
504 rc = efx_probe_tx_queue(tx_queue);
505 if (rc)
7f967c01 506 goto fail;
8ceee660
BH		 507 }
508
509 efx_for_each_channel_rx_queue(rx_queue, channel) {
510 rc = efx_probe_rx_queue(rx_queue);
511 if (rc)
7f967c01 512 goto fail;
8ceee660
BH		 513 }
514
8ceee660
BH		 515 return 0;
516
7f967c01
BH		 517fail:
518 efx_remove_channel(channel);
8ceee660
BH		 519 return rc;
520}
521
7f967c01
BH		 522static void
523efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len)
524{
525 struct efx_nic *efx = channel->efx;
526 const char *type;
527 int number;
528
529 number = channel->channel;
530 if (efx->tx_channel_offset == 0) {
531 type = "";
532 } else if (channel->channel < efx->tx_channel_offset) {
533 type = "-rx";
534 } else {
535 type = "-tx";
536 number -= efx->tx_channel_offset;
537 }
538 snprintf(buf, len, "%s%s-%d", efx->name, type, number);
539}
8ceee660 540
56536e9c
BH		 541static void efx_set_channel_names(struct efx_nic *efx)
542{
543 struct efx_channel *channel;
56536e9c 544
7f967c01
BH		 545 efx_for_each_channel(channel, efx)
546 channel->type->get_name(channel,
d8291187
BH		 547 efx->msi_context[channel->channel].name,
548 sizeof(efx->msi_context[0].name));
56536e9c
BH		 549}
550
4642610c
BH		 551static int efx_probe_channels(struct efx_nic *efx)
552{
553 struct efx_channel *channel;
554 int rc;
555
556 /* Restart special buffer allocation */
557 efx->next_buffer_table = 0;
558
c92aaff1
BH		 559 /* Probe channels in reverse, so that any 'extra' channels
560 * use the start of the buffer table. This allows the traffic
561 * channels to be resized without moving them or wasting the
562 * entries before them.
563 */
564 efx_for_each_channel_rev(channel, efx) {
4642610c
BH		 565 rc = efx_probe_channel(channel);
566 if (rc) {
567 netif_err(efx, probe, efx->net_dev,
568 "failed to create channel %d\n",
569 channel->channel);
570 goto fail;
571 }
572 }
573 efx_set_channel_names(efx);
574
575 return 0;
576
577fail:
578 efx_remove_channels(efx);
579 return rc;
580}
581
8ceee660
BH		 582/* Channels are shutdown and reinitialised whilst the NIC is running
583 * to propagate configuration changes (mtu, checksum offload), or
584 * to clear hardware error conditions
585 */
9f2cb71c 586static void efx_start_datapath(struct efx_nic *efx)
8ceee660 587{
85740cdf 588 bool old_rx_scatter = efx->rx_scatter;
8ceee660
BH		 589 struct efx_tx_queue *tx_queue;
590 struct efx_rx_queue *rx_queue;
591 struct efx_channel *channel;
85740cdf 592 size_t rx_buf_len;
8ceee660 593
f7f13b0b
BH		 594 /* Calculate the rx buffer allocation parameters required to
595 * support the current MTU, including padding for header
596 * alignment and overruns.
597 */
43a3739d 598 efx->rx_dma_len = (efx->rx_prefix_size +
272baeeb
BH		 599 EFX_MAX_FRAME_LEN(efx->net_dev->mtu) +
600 efx->type->rx_buffer_padding);
85740cdf 601 rx_buf_len = (sizeof(struct efx_rx_page_state) +
2ec03014 602 efx->rx_ip_align + efx->rx_dma_len);
85740cdf 603 if (rx_buf_len <= PAGE_SIZE) {
e8c68c0a 604 efx->rx_scatter = efx->type->always_rx_scatter;
85740cdf 605 efx->rx_buffer_order = 0;
85740cdf 606 } else if (efx->type->can_rx_scatter) {
950c54df 607 BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES);
85740cdf 608 BUILD_BUG_ON(sizeof(struct efx_rx_page_state) +
950c54df
BH		 609 2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE,
610 EFX_RX_BUF_ALIGNMENT) >
611 PAGE_SIZE);
85740cdf
BH		 612 efx->rx_scatter = true;
613 efx->rx_dma_len = EFX_RX_USR_BUF_SIZE;
614 efx->rx_buffer_order = 0;
85740cdf
BH		 615 } else {
616 efx->rx_scatter = false;
617 efx->rx_buffer_order = get_order(rx_buf_len);
85740cdf
BH		 618 }
619
1648a23f
DP		 620 efx_rx_config_page_split(efx);
621 if (efx->rx_buffer_order)
622 netif_dbg(efx, drv, efx->net_dev,
623 "RX buf len=%u; page order=%u batch=%u\n",
624 efx->rx_dma_len, efx->rx_buffer_order,
625 efx->rx_pages_per_batch);
626 else
627 netif_dbg(efx, drv, efx->net_dev,
628 "RX buf len=%u step=%u bpp=%u; page batch=%u\n",
629 efx->rx_dma_len, efx->rx_page_buf_step,
630 efx->rx_bufs_per_page, efx->rx_pages_per_batch);
2768935a 631
e8c68c0a 632 /* RX filters may also have scatter-enabled flags */
85740cdf 633 if (efx->rx_scatter != old_rx_scatter)
add72477 634 efx->type->filter_update_rx_scatter(efx);
8ceee660 635
14bf718f
BH		 636 /* We must keep at least one descriptor in a TX ring empty.
637 * We could avoid this when the queue size does not exactly
638 * match the hardware ring size, but it's not that important.
639 * Therefore we stop the queue when one more skb might fill
640 * the ring completely. We wake it when half way back to
641 * empty.
642 */
643 efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx);
644 efx->txq_wake_thresh = efx->txq_stop_thresh / 2;
645
8ceee660
BH		 646 /* Initialise the channels */
647 efx_for_each_channel(channel, efx) {
3881d8ab 648 efx_for_each_channel_tx_queue(tx_queue, channel) {
bc3c90a2 649 efx_init_tx_queue(tx_queue);
3881d8ab
AR		 650 atomic_inc(&efx->active_queues);
651 }
8ceee660 652
9f2cb71c 653 efx_for_each_channel_rx_queue(rx_queue, channel) {
bc3c90a2 654 efx_init_rx_queue(rx_queue);
3881d8ab 655 atomic_inc(&efx->active_queues);
cce28794
JC		 656 efx_stop_eventq(channel);
657 efx_fast_push_rx_descriptors(rx_queue, false);
658 efx_start_eventq(channel);
9f2cb71c 659 }
8ceee660 660
85740cdf 661 WARN_ON(channel->rx_pkt_n_frags);
8ceee660 662 }
8ceee660 663
2ea4dc28
AR		 664 efx_ptp_start_datapath(efx);
665
9f2cb71c
BH		 666 if (netif_device_present(efx->net_dev))
667 netif_tx_wake_all_queues(efx->net_dev);
8ceee660
BH		 668}
669
9f2cb71c 670static void efx_stop_datapath(struct efx_nic *efx)
8ceee660
BH		 671{
672 struct efx_channel *channel;
673 struct efx_tx_queue *tx_queue;
674 struct efx_rx_queue *rx_queue;
6bc5d3a9 675 int rc;
8ceee660
BH		 676
677 EFX_ASSERT_RESET_SERIALISED(efx);
678 BUG_ON(efx->port_enabled);
679
2ea4dc28
AR		 680 efx_ptp_stop_datapath(efx);
681
d8aec745
BH		 682 /* Stop RX refill */
683 efx_for_each_channel(channel, efx) {
684 efx_for_each_channel_rx_queue(rx_queue, channel)
685 rx_queue->refill_enabled = false;
686 }
687
8ceee660 688 efx_for_each_channel(channel, efx) {
9f2cb71c
BH		 689 /* RX packet processing is pipelined, so wait for the
690 * NAPI handler to complete. At least event queue 0
691 * might be kept active by non-data events, so don't
692 * use napi_synchronize() but actually disable NAPI
693 * temporarily.
694 */
695 if (efx_channel_has_rx_queue(channel)) {
696 efx_stop_eventq(channel);
697 efx_start_eventq(channel);
698 }
e42c3d85 699 }
8ceee660 700
e42c3d85
BH		 701 rc = efx->type->fini_dmaq(efx);
702 if (rc && EFX_WORKAROUND_7803(efx)) {
703 /* Schedule a reset to recover from the flush failure. The
704 * descriptor caches reference memory we're about to free,
705 * but falcon_reconfigure_mac_wrapper() won't reconnect
706 * the MACs because of the pending reset.
707 */
708 netif_err(efx, drv, efx->net_dev,
709 "Resetting to recover from flush failure\n");
710 efx_schedule_reset(efx, RESET_TYPE_ALL);
711 } else if (rc) {
712 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
713 } else {
714 netif_dbg(efx, drv, efx->net_dev,
715 "successfully flushed all queues\n");
716 }
717
718 efx_for_each_channel(channel, efx) {
8ceee660
BH		 719 efx_for_each_channel_rx_queue(rx_queue, channel)
720 efx_fini_rx_queue(rx_queue);
94b274bf 721 efx_for_each_possible_channel_tx_queue(tx_queue, channel)
8ceee660 722 efx_fini_tx_queue(tx_queue);
8ceee660
BH		 723 }
724}
725
726static void efx_remove_channel(struct efx_channel *channel)
727{
728 struct efx_tx_queue *tx_queue;
729 struct efx_rx_queue *rx_queue;
730
62776d03
BH		 731 netif_dbg(channel->efx, drv, channel->efx->net_dev,
732 "destroy chan %d\n", channel->channel);
8ceee660
BH		 733
734 efx_for_each_channel_rx_queue(rx_queue, channel)
735 efx_remove_rx_queue(rx_queue);
94b274bf 736 efx_for_each_possible_channel_tx_queue(tx_queue, channel)
8ceee660
BH		 737 efx_remove_tx_queue(tx_queue);
738 efx_remove_eventq(channel);
c31e5f9f 739 channel->type->post_remove(channel);
8ceee660
BH		 740}
741
4642610c
BH		 742static void efx_remove_channels(struct efx_nic *efx)
743{
744 struct efx_channel *channel;
745
746 efx_for_each_channel(channel, efx)
747 efx_remove_channel(channel);
748}
749
750int
751efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
752{
753 struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
754 u32 old_rxq_entries, old_txq_entries;
7f967c01 755 unsigned i, next_buffer_table = 0;
261e4d96 756 int rc, rc2;
8b7325b4
BH		 757
758 rc = efx_check_disabled(efx);
759 if (rc)
760 return rc;
7f967c01
BH		 761
762 /* Not all channels should be reallocated. We must avoid
763 * reallocating their buffer table entries.
764 */
765 efx_for_each_channel(channel, efx) {
766 struct efx_rx_queue *rx_queue;
767 struct efx_tx_queue *tx_queue;
768
769 if (channel->type->copy)
770 continue;
771 next_buffer_table = max(next_buffer_table,
772 channel->eventq.index +
773 channel->eventq.entries);
774 efx_for_each_channel_rx_queue(rx_queue, channel)
775 next_buffer_table = max(next_buffer_table,
776 rx_queue->rxd.index +
777 rx_queue->rxd.entries);
778 efx_for_each_channel_tx_queue(tx_queue, channel)
779 next_buffer_table = max(next_buffer_table,
780 tx_queue->txd.index +
781 tx_queue->txd.entries);
782 }
4642610c 783
29c69a48 784 efx_device_detach_sync(efx);
4642610c 785 efx_stop_all(efx);
d8291187 786 efx_soft_disable_interrupts(efx);
4642610c 787
7f967c01 788 /* Clone channels (where possible) */
4642610c
BH		 789 memset(other_channel, 0, sizeof(other_channel));
790 for (i = 0; i < efx->n_channels; i++) {
7f967c01
BH		 791 channel = efx->channel[i];
792 if (channel->type->copy)
793 channel = channel->type->copy(channel);
4642610c
BH		 794 if (!channel) {
795 rc = -ENOMEM;
796 goto out;
797 }
798 other_channel[i] = channel;
799 }
800
801 /* Swap entry counts and channel pointers */
802 old_rxq_entries = efx->rxq_entries;
803 old_txq_entries = efx->txq_entries;
804 efx->rxq_entries = rxq_entries;
805 efx->txq_entries = txq_entries;
806 for (i = 0; i < efx->n_channels; i++) {
807 channel = efx->channel[i];
808 efx->channel[i] = other_channel[i];
809 other_channel[i] = channel;
810 }
811
7f967c01
BH		 812 /* Restart buffer table allocation */
813 efx->next_buffer_table = next_buffer_table;
e8f14992 814
e8f14992 815 for (i = 0; i < efx->n_channels; i++) {
7f967c01
BH		 816 channel = efx->channel[i];
817 if (!channel->type->copy)
818 continue;
819 rc = efx_probe_channel(channel);
820 if (rc)
821 goto rollback;
822 efx_init_napi_channel(efx->channel[i]);
e8f14992 823 }
7f967c01 824
4642610c 825out:
7f967c01
BH		 826 /* Destroy unused channel structures */
827 for (i = 0; i < efx->n_channels; i++) {
828 channel = other_channel[i];
829 if (channel && channel->type->copy) {
830 efx_fini_napi_channel(channel);
831 efx_remove_channel(channel);
832 kfree(channel);
833 }
834 }
4642610c 835
261e4d96
JC		 836 rc2 = efx_soft_enable_interrupts(efx);
837 if (rc2) {
838 rc = rc ? rc : rc2;
839 netif_err(efx, drv, efx->net_dev,
840 "unable to restart interrupts on channel reallocation\n");
841 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
842 } else {
843 efx_start_all(efx);
844 netif_device_attach(efx->net_dev);
845 }
4642610c
BH		 846 return rc;
847
848rollback:
849 /* Swap back */
850 efx->rxq_entries = old_rxq_entries;
851 efx->txq_entries = old_txq_entries;
852 for (i = 0; i < efx->n_channels; i++) {
853 channel = efx->channel[i];
854 efx->channel[i] = other_channel[i];
855 other_channel[i] = channel;
856 }
857 goto out;
858}
859
90d683af 860void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
8ceee660 861{
90d683af 862 mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100));
8ceee660
BH		 863}
864
7f967c01
BH		 865static const struct efx_channel_type efx_default_channel_type = {
866 .pre_probe = efx_channel_dummy_op_int,
c31e5f9f 867 .post_remove = efx_channel_dummy_op_void,
7f967c01
BH		 868 .get_name = efx_get_channel_name,
869 .copy = efx_copy_channel,
870 .keep_eventq = false,
871};
872
873int efx_channel_dummy_op_int(struct efx_channel *channel)
874{
875 return 0;
876}
877
c31e5f9f
SH		 878void efx_channel_dummy_op_void(struct efx_channel *channel)
879{
880}
881
8ceee660
BH		 882/**************************************************************************
883 *
884 * Port handling
885 *
886 **************************************************************************/
887
888/* This ensures that the kernel is kept informed (via
889 * netif_carrier_on/off) of the link status, and also maintains the
890 * link status's stop on the port's TX queue.
891 */
fdaa9aed 892void efx_link_status_changed(struct efx_nic *efx)
8ceee660 893{
eb50c0d6
BH		 894 struct efx_link_state *link_state = &efx->link_state;
895
8ceee660
BH		 896 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure
897 * that no events are triggered between unregister_netdev() and the
898 * driver unloading. A more general condition is that NETDEV_CHANGE
899 * can only be generated between NETDEV_UP and NETDEV_DOWN */
900 if (!netif_running(efx->net_dev))
901 return;
902
eb50c0d6 903 if (link_state->up != netif_carrier_ok(efx->net_dev)) {
8ceee660
BH		 904 efx->n_link_state_changes++;
905
eb50c0d6 906 if (link_state->up)
8ceee660
BH		 907 netif_carrier_on(efx->net_dev);
908 else
909 netif_carrier_off(efx->net_dev);
910 }
911
912 /* Status message for kernel log */
2aa9ef11 913 if (link_state->up)
62776d03 914 netif_info(efx, link, efx->net_dev,
964e6135 915 "link up at %uMbps %s-duplex (MTU %d)\n",
62776d03 916 link_state->speed, link_state->fd ? "full" : "half",
964e6135 917 efx->net_dev->mtu);
2aa9ef11 918 else
62776d03 919 netif_info(efx, link, efx->net_dev, "link down\n");
8ceee660
BH		 920}
921
d3245b28
BH		 922void efx_link_set_advertising(struct efx_nic *efx, u32 advertising)
923{
924 efx->link_advertising = advertising;
925 if (advertising) {
926 if (advertising & ADVERTISED_Pause)
927 efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX);
928 else
929 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
930 if (advertising & ADVERTISED_Asym_Pause)
931 efx->wanted_fc ^= EFX_FC_TX;
932 }
933}
934
b5626946 935void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc)
d3245b28
BH		 936{
937 efx->wanted_fc = wanted_fc;
938 if (efx->link_advertising) {
939 if (wanted_fc & EFX_FC_RX)
940 efx->link_advertising |= (ADVERTISED_Pause |
941 ADVERTISED_Asym_Pause);
942 else
943 efx->link_advertising &= ~(ADVERTISED_Pause |
944 ADVERTISED_Asym_Pause);
945 if (wanted_fc & EFX_FC_TX)
946 efx->link_advertising ^= ADVERTISED_Asym_Pause;
947 }
948}
949
115122af
BH		 950static void efx_fini_port(struct efx_nic *efx);
951
d3245b28
BH		 952/* Push loopback/power/transmit disable settings to the PHY, and reconfigure
953 * the MAC appropriately. All other PHY configuration changes are pushed
954 * through phy_op->set_settings(), and pushed asynchronously to the MAC
955 * through efx_monitor().
956 *
957 * Callers must hold the mac_lock
958 */
959int __efx_reconfigure_port(struct efx_nic *efx)
8ceee660 960{
d3245b28
BH		 961 enum efx_phy_mode phy_mode;
962 int rc;
8ceee660 963
d3245b28 964 WARN_ON(!mutex_is_locked(&efx->mac_lock));
8ceee660 965
d3245b28
BH		 966 /* Disable PHY transmit in mac level loopbacks */
967 phy_mode = efx->phy_mode;
177dfcd8
BH		 968 if (LOOPBACK_INTERNAL(efx))
969 efx->phy_mode |= PHY_MODE_TX_DISABLED;
970 else
971 efx->phy_mode &= ~PHY_MODE_TX_DISABLED;
177dfcd8 972
d3245b28 973 rc = efx->type->reconfigure_port(efx);
8ceee660 974
d3245b28
BH		 975 if (rc)
976 efx->phy_mode = phy_mode;
177dfcd8 977
d3245b28 978 return rc;
8ceee660
BH		 979}
980
981/* Reinitialise the MAC to pick up new PHY settings, even if the port is
982 * disabled. */
d3245b28 983int efx_reconfigure_port(struct efx_nic *efx)
8ceee660 984{
d3245b28
BH		 985 int rc;
986
8ceee660
BH		 987 EFX_ASSERT_RESET_SERIALISED(efx);
988
989 mutex_lock(&efx->mac_lock);
d3245b28 990 rc = __efx_reconfigure_port(efx);
8ceee660 991 mutex_unlock(&efx->mac_lock);
d3245b28
BH		 992
993 return rc;
8ceee660
BH		 994}
995
8be4f3e6
BH		 996/* Asynchronous work item for changing MAC promiscuity and multicast
997 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current
998 * MAC directly. */
766ca0fa
BH		 999static void efx_mac_work(struct work_struct *data)
1000{
1001 struct efx_nic *efx = container_of(data, struct efx_nic, mac_work);
1002
1003 mutex_lock(&efx->mac_lock);
30b81cda 1004 if (efx->port_enabled)
710b208d 1005 efx->type->reconfigure_mac(efx);
766ca0fa
BH		 1006 mutex_unlock(&efx->mac_lock);
1007}
1008
8ceee660
BH		 1009static int efx_probe_port(struct efx_nic *efx)
1010{
1011 int rc;
1012
62776d03 1013 netif_dbg(efx, probe, efx->net_dev, "create port\n");
8ceee660 1014
ff3b00a0
SH		 1015 if (phy_flash_cfg)
1016 efx->phy_mode = PHY_MODE_SPECIAL;
1017
ef2b90ee
BH		 1018 /* Connect up MAC/PHY operations table */
1019 rc = efx->type->probe_port(efx);
8ceee660 1020 if (rc)
e42de262 1021 return rc;
8ceee660 1022
e332bcb3 1023 /* Initialise MAC address to permanent address */
cd84ff4d 1024 ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr);
8ceee660
BH		 1025
1026 return 0;
8ceee660
BH		 1027}
1028
1029static int efx_init_port(struct efx_nic *efx)
1030{
1031 int rc;
1032
62776d03 1033 netif_dbg(efx, drv, efx->net_dev, "init port\n");
8ceee660 1034
1dfc5cea
BH		 1035 mutex_lock(&efx->mac_lock);
1036
177dfcd8 1037 rc = efx->phy_op->init(efx);
8ceee660 1038 if (rc)
1dfc5cea 1039 goto fail1;
8ceee660 1040
dc8cfa55 1041 efx->port_initialized = true;
1dfc5cea 1042
d3245b28
BH		 1043 /* Reconfigure the MAC before creating dma queues (required for
1044 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */
710b208d 1045 efx->type->reconfigure_mac(efx);
d3245b28
BH		 1046
1047 /* Ensure the PHY advertises the correct flow control settings */
1048 rc = efx->phy_op->reconfigure(efx);
267d9d73 1049 if (rc && rc != -EPERM)
d3245b28
BH		 1050 goto fail2;
1051
1dfc5cea 1052 mutex_unlock(&efx->mac_lock);
8ceee660 1053 return 0;
177dfcd8 1054
1dfc5cea 1055fail2:
177dfcd8 1056 efx->phy_op->fini(efx);
1dfc5cea
BH		 1057fail1:
1058 mutex_unlock(&efx->mac_lock);
177dfcd8 1059 return rc;
8ceee660
BH		 1060}
1061
8ceee660
BH		 1062static void efx_start_port(struct efx_nic *efx)
1063{
62776d03 1064 netif_dbg(efx, ifup, efx->net_dev, "start port\n");
8ceee660
BH		 1065 BUG_ON(efx->port_enabled);
1066
1067 mutex_lock(&efx->mac_lock);
dc8cfa55 1068 efx->port_enabled = true;
8be4f3e6 1069
d615c039 1070 /* Ensure MAC ingress/egress is enabled */
710b208d 1071 efx->type->reconfigure_mac(efx);
8be4f3e6 1072
8ceee660
BH		 1073 mutex_unlock(&efx->mac_lock);
1074}
1075
d615c039
BH		 1076/* Cancel work for MAC reconfiguration, periodic hardware monitoring
1077 * and the async self-test, wait for them to finish and prevent them
1078 * being scheduled again. This doesn't cover online resets, which
1079 * should only be cancelled when removing the device.
1080 */
8ceee660
BH		 1081static void efx_stop_port(struct efx_nic *efx)
1082{
62776d03 1083 netif_dbg(efx, ifdown, efx->net_dev, "stop port\n");
8ceee660 1084
d615c039
BH		 1085 EFX_ASSERT_RESET_SERIALISED(efx);
1086
8ceee660 1087 mutex_lock(&efx->mac_lock);
dc8cfa55 1088 efx->port_enabled = false;
8ceee660
BH		 1089 mutex_unlock(&efx->mac_lock);
1090
1091 /* Serialise against efx_set_multicast_list() */
73ba7b68
BH		 1092 netif_addr_lock_bh(efx->net_dev);
1093 netif_addr_unlock_bh(efx->net_dev);
d615c039
BH		 1094
1095 cancel_delayed_work_sync(&efx->monitor_work);
1096 efx_selftest_async_cancel(efx);
1097 cancel_work_sync(&efx->mac_work);
8ceee660
BH		 1098}
1099
1100static void efx_fini_port(struct efx_nic *efx)
1101{
62776d03 1102 netif_dbg(efx, drv, efx->net_dev, "shut down port\n");
8ceee660
BH		 1103
1104 if (!efx->port_initialized)
1105 return;
1106
177dfcd8 1107 efx->phy_op->fini(efx);
dc8cfa55 1108 efx->port_initialized = false;
8ceee660 1109
eb50c0d6 1110 efx->link_state.up = false;
8ceee660
BH		 1111 efx_link_status_changed(efx);
1112}
1113
1114static void efx_remove_port(struct efx_nic *efx)
1115{
62776d03 1116 netif_dbg(efx, drv, efx->net_dev, "destroying port\n");
8ceee660 1117
ef2b90ee 1118 efx->type->remove_port(efx);
8ceee660
BH		 1119}
1120
1121/**************************************************************************
1122 *
1123 * NIC handling
1124 *
1125 **************************************************************************/
1126
0bcf4a64
BH		 1127static LIST_HEAD(efx_primary_list);
1128static LIST_HEAD(efx_unassociated_list);
1129
1130static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right)
1131{
1132 return left->type == right->type &&
1133 left->vpd_sn && right->vpd_sn &&
1134 !strcmp(left->vpd_sn, right->vpd_sn);
1135}
1136
1137static void efx_associate(struct efx_nic *efx)
1138{
1139 struct efx_nic *other, *next;
1140
1141 if (efx->primary == efx) {
1142 /* Adding primary function; look for secondaries */
1143
1144 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n");
1145 list_add_tail(&efx->node, &efx_primary_list);
1146
1147 list_for_each_entry_safe(other, next, &efx_unassociated_list,
1148 node) {
1149 if (efx_same_controller(efx, other)) {
1150 list_del(&other->node);
1151 netif_dbg(other, probe, other->net_dev,
1152 "moving to secondary list of %s %s\n",
1153 pci_name(efx->pci_dev),
1154 efx->net_dev->name);
1155 list_add_tail(&other->node,
1156 &efx->secondary_list);
1157 other->primary = efx;
1158 }
1159 }
1160 } else {
1161 /* Adding secondary function; look for primary */
1162
1163 list_for_each_entry(other, &efx_primary_list, node) {
1164 if (efx_same_controller(efx, other)) {
1165 netif_dbg(efx, probe, efx->net_dev,
1166 "adding to secondary list of %s %s\n",
1167 pci_name(other->pci_dev),
1168 other->net_dev->name);
1169 list_add_tail(&efx->node,
1170 &other->secondary_list);
1171 efx->primary = other;
1172 return;
1173 }
1174 }
1175
1176 netif_dbg(efx, probe, efx->net_dev,
1177 "adding to unassociated list\n");
1178 list_add_tail(&efx->node, &efx_unassociated_list);
1179 }
1180}
1181
1182static void efx_dissociate(struct efx_nic *efx)
1183{
1184 struct efx_nic *other, *next;
1185
1186 list_del(&efx->node);
1187 efx->primary = NULL;
1188
1189 list_for_each_entry_safe(other, next, &efx->secondary_list, node) {
1190 list_del(&other->node);
1191 netif_dbg(other, probe, other->net_dev,
1192 "moving to unassociated list\n");
1193 list_add_tail(&other->node, &efx_unassociated_list);
1194 other->primary = NULL;
1195 }
1196}
1197
8ceee660
BH		 1198/* This configures the PCI device to enable I/O and DMA. */
1199static int efx_init_io(struct efx_nic *efx)
1200{
1201 struct pci_dev *pci_dev = efx->pci_dev;
1202 dma_addr_t dma_mask = efx->type->max_dma_mask;
b105798f 1203 unsigned int mem_map_size = efx->type->mem_map_size(efx);
02246a7f 1204 int rc, bar;
8ceee660 1205
62776d03 1206 netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n");
8ceee660 1207
02246a7f
SS		 1208 bar = efx->type->mem_bar;
1209
8ceee660
BH		 1210 rc = pci_enable_device(pci_dev);
1211 if (rc) {
62776d03
BH		 1212 netif_err(efx, probe, efx->net_dev,
1213 "failed to enable PCI device\n");
8ceee660
BH		 1214 goto fail1;
1215 }
1216
1217 pci_set_master(pci_dev);
1218
1219 /* Set the PCI DMA mask. Try all possibilities from our
1220 * genuine mask down to 32 bits, because some architectures
1221 * (e.g. x86_64 with iommu_sac_force set) will allow 40 bit
1222 * masks event though they reject 46 bit masks.
1223 */
1224 while (dma_mask > 0x7fffffffUL) {
0e33d870 1225 if (dma_supported(&pci_dev->dev, dma_mask)) {
9663deda 1226 rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask);
e9e01846
BH		 1227 if (rc == 0)
1228 break;
1229 }
8ceee660
BH		 1230 dma_mask >>= 1;
1231 }
1232 if (rc) {
62776d03
BH		 1233 netif_err(efx, probe, efx->net_dev,
1234 "could not find a suitable DMA mask\n");
8ceee660
BH		 1235 goto fail2;
1236 }
62776d03
BH		 1237 netif_dbg(efx, probe, efx->net_dev,
1238 "using DMA mask %llx\n", (unsigned long long) dma_mask);
8ceee660 1239
02246a7f
SS		 1240 efx->membase_phys = pci_resource_start(efx->pci_dev, bar);
1241 rc = pci_request_region(pci_dev, bar, "sfc");
8ceee660 1242 if (rc) {
62776d03
BH		 1243 netif_err(efx, probe, efx->net_dev,
1244 "request for memory BAR failed\n");
8ceee660
BH		 1245 rc = -EIO;
1246 goto fail3;
1247 }
b105798f 1248 efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size);
8ceee660 1249 if (!efx->membase) {
62776d03
BH		 1250 netif_err(efx, probe, efx->net_dev,
1251 "could not map memory BAR at %llx+%x\n",
b105798f 1252 (unsigned long long)efx->membase_phys, mem_map_size);
8ceee660
BH		 1253 rc = -ENOMEM;
1254 goto fail4;
1255 }
62776d03
BH		 1256 netif_dbg(efx, probe, efx->net_dev,
1257 "memory BAR at %llx+%x (virtual %p)\n",
b105798f
BH		 1258 (unsigned long long)efx->membase_phys, mem_map_size,
1259 efx->membase);
8ceee660
BH		 1260
1261 return 0;
1262
1263 fail4:
02246a7f 1264 pci_release_region(efx->pci_dev, bar);
8ceee660 1265 fail3:
2c118e0f 1266 efx->membase_phys = 0;
8ceee660
BH		 1267 fail2:
1268 pci_disable_device(efx->pci_dev);
1269 fail1:
1270 return rc;
1271}
1272
1273static void efx_fini_io(struct efx_nic *efx)
1274{
02246a7f
SS		 1275 int bar;
1276
62776d03 1277 netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n");
8ceee660
BH		 1278
1279 if (efx->membase) {
1280 iounmap(efx->membase);
1281 efx->membase = NULL;
1282 }
1283
1284 if (efx->membase_phys) {
02246a7f
SS		 1285 bar = efx->type->mem_bar;
1286 pci_release_region(efx->pci_dev, bar);
2c118e0f 1287 efx->membase_phys = 0;
8ceee660
BH		 1288 }
1289
1290 pci_disable_device(efx->pci_dev);
1291}
1292
267c0157
JC		 1293void efx_set_default_rx_indir_table(struct efx_nic *efx)
1294{
1295 size_t i;
1296
1297 for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++)
1298 efx->rx_indir_table[i] =
1299 ethtool_rxfh_indir_default(i, efx->rss_spread);
1300}
1301
a9a52506 1302static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
46123d04 1303{
cdb08f8f 1304 cpumask_var_t thread_mask;
a16e5b24 1305 unsigned int count;
46123d04 1306 int cpu;
5b874e25 1307
cd2d5b52
BH		 1308 if (rss_cpus) {
1309 count = rss_cpus;
1310 } else {
1311 if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) {
1312 netif_warn(efx, probe, efx->net_dev,
1313 "RSS disabled due to allocation failure\n");
1314 return 1;
1315 }
46123d04 1316
cd2d5b52
BH		 1317 count = 0;
1318 for_each_online_cpu(cpu) {
1319 if (!cpumask_test_cpu(cpu, thread_mask)) {
1320 ++count;
1321 cpumask_or(thread_mask, thread_mask,
1322 topology_thread_cpumask(cpu));
1323 }
1324 }
1325
1326 free_cpumask_var(thread_mask);
2f8975fb
RR		 1327 }
1328
cd2d5b52
BH		 1329 /* If RSS is requested for the PF *and* VFs then we can't write RSS
1330 * table entries that are inaccessible to VFs
1331 */
7fa8d547
SS		 1332#ifdef CONFIG_SFC_SRIOV
1333 if (efx->type->sriov_wanted) {
1334 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
1335 count > efx_vf_size(efx)) {
1336 netif_warn(efx, probe, efx->net_dev,
1337 "Reducing number of RSS channels from %u to %u for "
1338 "VF support. Increase vf-msix-limit to use more "
1339 "channels on the PF.\n",
1340 count, efx_vf_size(efx));
1341 count = efx_vf_size(efx);
1342 }
46123d04 1343 }
7fa8d547 1344#endif
46123d04
BH		 1345
1346 return count;
1347}
1348
1349/* Probe the number and type of interrupts we are able to obtain, and
1350 * the resulting numbers of channels and RX queues.
1351 */
64d8ad6d 1352static int efx_probe_interrupts(struct efx_nic *efx)
8ceee660 1353{
7f967c01
BH		 1354 unsigned int extra_channels = 0;
1355 unsigned int i, j;
a16e5b24 1356 int rc;
8ceee660 1357
7f967c01
BH		 1358 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
1359 if (efx->extra_channel_type[i])
1360 ++extra_channels;
1361
8ceee660 1362 if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
46123d04 1363 struct msix_entry xentries[EFX_MAX_CHANNELS];
a16e5b24 1364 unsigned int n_channels;
aa6ef27e 1365
a9a52506 1366 n_channels = efx_wanted_parallelism(efx);
a4900ac9
BH		 1367 if (separate_tx_channels)
1368 n_channels *= 2;
7f967c01 1369 n_channels += extra_channels;
b105798f 1370 n_channels = min(n_channels, efx->max_channels);
8ceee660 1371
a4900ac9 1372 for (i = 0; i < n_channels; i++)
8ceee660 1373 xentries[i].entry = i;
184603d8
AG		 1374 rc = pci_enable_msix_range(efx->pci_dev,
1375 xentries, 1, n_channels);
1376 if (rc < 0) {
1377 /* Fall back to single channel MSI */
1378 efx->interrupt_mode = EFX_INT_MODE_MSI;
1379 netif_err(efx, drv, efx->net_dev,
1380 "could not enable MSI-X\n");
1381 } else if (rc < n_channels) {
62776d03
BH		 1382 netif_err(efx, drv, efx->net_dev,
1383 "WARNING: Insufficient MSI-X vectors"
a16e5b24 1384 " available (%d < %u).\n", rc, n_channels);
62776d03
BH		 1385 netif_err(efx, drv, efx->net_dev,
1386 "WARNING: Performance may be reduced.\n");
a4900ac9 1387 n_channels = rc;
8ceee660
BH		 1388 }
1389
184603d8 1390 if (rc > 0) {
a4900ac9 1391 efx->n_channels = n_channels;
7f967c01
BH		 1392 if (n_channels > extra_channels)
1393 n_channels -= extra_channels;
a4900ac9 1394 if (separate_tx_channels) {
7f967c01
BH		 1395 efx->n_tx_channels = max(n_channels / 2, 1U);
1396 efx->n_rx_channels = max(n_channels -
1397 efx->n_tx_channels,
1398 1U);
a4900ac9 1399 } else {
7f967c01
BH		 1400 efx->n_tx_channels = n_channels;
1401 efx->n_rx_channels = n_channels;
a4900ac9 1402 }
7f967c01 1403 for (i = 0; i < efx->n_channels; i++)
f7d12cdc
BH		 1404 efx_get_channel(efx, i)->irq =
1405 xentries[i].vector;
8ceee660
BH		 1406 }
1407 }
1408
1409 /* Try single interrupt MSI */
1410 if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
28b581ab 1411 efx->n_channels = 1;
a4900ac9
BH		 1412 efx->n_rx_channels = 1;
1413 efx->n_tx_channels = 1;
8ceee660
BH		 1414 rc = pci_enable_msi(efx->pci_dev);
1415 if (rc == 0) {
f7d12cdc 1416 efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
8ceee660 1417 } else {
62776d03
BH		 1418 netif_err(efx, drv, efx->net_dev,
1419 "could not enable MSI\n");
8ceee660
BH		 1420 efx->interrupt_mode = EFX_INT_MODE_LEGACY;
1421 }
1422 }
1423
1424 /* Assume legacy interrupts */
1425 if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
28b581ab 1426 efx->n_channels = 1 + (separate_tx_channels ? 1 : 0);
a4900ac9
BH		 1427 efx->n_rx_channels = 1;
1428 efx->n_tx_channels = 1;
8ceee660
BH		 1429 efx->legacy_irq = efx->pci_dev->irq;
1430 }
64d8ad6d 1431
7f967c01
BH		 1432 /* Assign extra channels if possible */
1433 j = efx->n_channels;
1434 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
1435 if (!efx->extra_channel_type[i])
1436 continue;
1437 if (efx->interrupt_mode != EFX_INT_MODE_MSIX ||
1438 efx->n_channels <= extra_channels) {
1439 efx->extra_channel_type[i]->handle_no_channel(efx);
1440 } else {
1441 --j;
1442 efx_get_channel(efx, j)->type =
1443 efx->extra_channel_type[i];
1444 }
1445 }
1446
cd2d5b52 1447 /* RSS might be usable on VFs even if it is disabled on the PF */
7fa8d547
SS		 1448#ifdef CONFIG_SFC_SRIOV
1449 if (efx->type->sriov_wanted) {
1450 efx->rss_spread = ((efx->n_rx_channels > 1 ||
1451 !efx->type->sriov_wanted(efx)) ?
1452 efx->n_rx_channels : efx_vf_size(efx));
1453 return 0;
1454 }
1455#endif
1456 efx->rss_spread = efx->n_rx_channels;
64d8ad6d 1457 return 0;
8ceee660
BH		 1458}
1459
261e4d96 1460static int efx_soft_enable_interrupts(struct efx_nic *efx)
9f2cb71c 1461{
261e4d96
JC		 1462 struct efx_channel *channel, *end_channel;
1463 int rc;
9f2cb71c 1464
8b7325b4
BH		 1465 BUG_ON(efx->state == STATE_DISABLED);
1466
d8291187
BH		 1467 efx->irq_soft_enabled = true;
1468 smp_wmb();
9f2cb71c
BH		 1469
1470 efx_for_each_channel(channel, efx) {
261e4d96
JC		 1471 if (!channel->type->keep_eventq) {
1472 rc = efx_init_eventq(channel);
1473 if (rc)
1474 goto fail;
1475 }
9f2cb71c
BH		 1476 efx_start_eventq(channel);
1477 }
1478
1479 efx_mcdi_mode_event(efx);
261e4d96
JC		 1480
1481 return 0;
1482fail:
1483 end_channel = channel;
1484 efx_for_each_channel(channel, efx) {
1485 if (channel == end_channel)
1486 break;
1487 efx_stop_eventq(channel);
1488 if (!channel->type->keep_eventq)
1489 efx_fini_eventq(channel);
1490 }
1491
1492 return rc;
9f2cb71c
BH		 1493}
1494
d8291187 1495static void efx_soft_disable_interrupts(struct efx_nic *efx)
9f2cb71c
BH		 1496{
1497 struct efx_channel *channel;
1498
8b7325b4
BH		 1499 if (efx->state == STATE_DISABLED)
1500 return;
1501
9f2cb71c
BH		 1502 efx_mcdi_mode_poll(efx);
1503
d8291187
BH		 1504 efx->irq_soft_enabled = false;
1505 smp_wmb();
1506
1507 if (efx->legacy_irq)
9f2cb71c 1508 synchronize_irq(efx->legacy_irq);
9f2cb71c
BH		 1509
1510 efx_for_each_channel(channel, efx) {
1511 if (channel->irq)
1512 synchronize_irq(channel->irq);
1513
1514 efx_stop_eventq(channel);
d8291187 1515 if (!channel->type->keep_eventq)
7f967c01 1516 efx_fini_eventq(channel);
9f2cb71c 1517 }
cade715f
BH		 1518
1519 /* Flush the asynchronous MCDI request queue */
1520 efx_mcdi_flush_async(efx);
9f2cb71c
BH		 1521}
1522
261e4d96 1523static int efx_enable_interrupts(struct efx_nic *efx)
d8291187 1524{
261e4d96
JC		 1525 struct efx_channel *channel, *end_channel;
1526 int rc;
d8291187
BH		 1527
1528 BUG_ON(efx->state == STATE_DISABLED);
1529
1530 if (efx->eeh_disabled_legacy_irq) {
1531 enable_irq(efx->legacy_irq);
1532 efx->eeh_disabled_legacy_irq = false;
1533 }
1534
86094f7f 1535 efx->type->irq_enable_master(efx);
d8291187
BH		 1536
1537 efx_for_each_channel(channel, efx) {
261e4d96
JC		 1538 if (channel->type->keep_eventq) {
1539 rc = efx_init_eventq(channel);
1540 if (rc)
1541 goto fail;
1542 }
1543 }
1544
1545 rc = efx_soft_enable_interrupts(efx);
1546 if (rc)
1547 goto fail;
1548
1549 return 0;
1550
1551fail:
1552 end_channel = channel;
1553 efx_for_each_channel(channel, efx) {
1554 if (channel == end_channel)
1555 break;
d8291187 1556 if (channel->type->keep_eventq)
261e4d96 1557 efx_fini_eventq(channel);
d8291187
BH		 1558 }
1559
261e4d96
JC		 1560 efx->type->irq_disable_non_ev(efx);
1561
1562 return rc;
d8291187
BH		 1563}
1564
1565static void efx_disable_interrupts(struct efx_nic *efx)
1566{
1567 struct efx_channel *channel;
1568
1569 efx_soft_disable_interrupts(efx);
1570
1571 efx_for_each_channel(channel, efx) {
1572 if (channel->type->keep_eventq)
1573 efx_fini_eventq(channel);
1574 }
1575
86094f7f 1576 efx->type->irq_disable_non_ev(efx);
d8291187
BH		 1577}
1578
8ceee660
BH		 1579static void efx_remove_interrupts(struct efx_nic *efx)
1580{
1581 struct efx_channel *channel;
1582
1583 /* Remove MSI/MSI-X interrupts */
64ee3120 1584 efx_for_each_channel(channel, efx)
8ceee660
BH		 1585 channel->irq = 0;
1586 pci_disable_msi(efx->pci_dev);
1587 pci_disable_msix(efx->pci_dev);
1588
1589 /* Remove legacy interrupt */
1590 efx->legacy_irq = 0;
1591}
1592
8831da7b 1593static void efx_set_channels(struct efx_nic *efx)
8ceee660 1594{
602a5322
BH		 1595 struct efx_channel *channel;
1596 struct efx_tx_queue *tx_queue;
1597
97653431 1598 efx->tx_channel_offset =
a4900ac9 1599 separate_tx_channels ? efx->n_channels - efx->n_tx_channels : 0;
602a5322 1600
79d68b37
SH		 1601 /* We need to mark which channels really have RX and TX
1602 * queues, and adjust the TX queue numbers if we have separate
602a5322
BH		 1603 * RX-only and TX-only channels.
1604 */
1605 efx_for_each_channel(channel, efx) {
79d68b37
SH		 1606 if (channel->channel < efx->n_rx_channels)
1607 channel->rx_queue.core_index = channel->channel;
1608 else
1609 channel->rx_queue.core_index = -1;
1610
602a5322
BH		 1611 efx_for_each_channel_tx_queue(tx_queue, channel)
1612 tx_queue->queue -= (efx->tx_channel_offset *
1613 EFX_TXQ_TYPES);
1614 }
8ceee660
BH		 1615}
1616
1617static int efx_probe_nic(struct efx_nic *efx)
1618{
1619 int rc;
1620
62776d03 1621 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n");
8ceee660
BH		 1622
1623 /* Carry out hardware-type specific initialisation */
ef2b90ee 1624 rc = efx->type->probe(efx);
8ceee660
BH		 1625 if (rc)
1626 return rc;
1627
a4900ac9 1628 /* Determine the number of channels and queues by trying to hook
8ceee660 1629 * in MSI-X interrupts. */
64d8ad6d
BH		 1630 rc = efx_probe_interrupts(efx);
1631 if (rc)
c15eed22 1632 goto fail1;
8ceee660 1633
52ad762b
DP		 1634 efx_set_channels(efx);
1635
c15eed22
BH		 1636 rc = efx->type->dimension_resources(efx);
1637 if (rc)
1638 goto fail2;
28e47c49 1639
5d3a6fca 1640 if (efx->n_channels > 1)
267c0157
JC		 1641 netdev_rss_key_fill(&efx->rx_hash_key,
1642 sizeof(efx->rx_hash_key));
1643 efx_set_default_rx_indir_table(efx);
5d3a6fca 1644
c4f4adc7
BH		 1645 netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
1646 netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
8ceee660
BH		 1647
1648 /* Initialise the interrupt moderation settings */
9e393b30
BH		 1649 efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true,
1650 true);
8ceee660
BH		 1651
1652 return 0;
64d8ad6d 1653
c15eed22
BH		 1654fail2:
1655 efx_remove_interrupts(efx);
1656fail1:
64d8ad6d
BH		 1657 efx->type->remove(efx);
1658 return rc;
8ceee660
BH		 1659}
1660
1661static void efx_remove_nic(struct efx_nic *efx)
1662{
62776d03 1663 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n");
8ceee660
BH		 1664
1665 efx_remove_interrupts(efx);
ef2b90ee 1666 efx->type->remove(efx);
8ceee660
BH		 1667}
1668
add72477
BH		 1669static int efx_probe_filters(struct efx_nic *efx)
1670{
1671 int rc;
1672
1673 spin_lock_init(&efx->filter_lock);
1674
1675 rc = efx->type->filter_table_probe(efx);
1676 if (rc)
1677 return rc;
1678
1679#ifdef CONFIG_RFS_ACCEL
1680 if (efx->type->offload_features & NETIF_F_NTUPLE) {
1681 efx->rps_flow_id = kcalloc(efx->type->max_rx_ip_filters,
1682 sizeof(*efx->rps_flow_id),
1683 GFP_KERNEL);
1684 if (!efx->rps_flow_id) {
1685 efx->type->filter_table_remove(efx);
1686 return -ENOMEM;
1687 }
1688 }
1689#endif
1690
1691 return 0;
1692}
1693
1694static void efx_remove_filters(struct efx_nic *efx)
1695{
1696#ifdef CONFIG_RFS_ACCEL
1697 kfree(efx->rps_flow_id);
1698#endif
1699 efx->type->filter_table_remove(efx);
1700}
1701
1702static void efx_restore_filters(struct efx_nic *efx)
1703{
1704 efx->type->filter_table_restore(efx);
1705}
1706
8ceee660
BH		 1707/**************************************************************************
1708 *
1709 * NIC startup/shutdown
1710 *
1711 *************************************************************************/
1712
1713static int efx_probe_all(struct efx_nic *efx)
1714{
8ceee660
BH		 1715 int rc;
1716
8ceee660
BH		 1717 rc = efx_probe_nic(efx);
1718 if (rc) {
62776d03 1719 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");
8ceee660
BH		 1720 goto fail1;
1721 }
1722
8ceee660
BH		 1723 rc = efx_probe_port(efx);
1724 if (rc) {
62776d03 1725 netif_err(efx, probe, efx->net_dev, "failed to create port\n");
8ceee660
BH		 1726 goto fail2;
1727 }
1728
7e6d06f0
BH		 1729 BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT);
1730 if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) {
1731 rc = -EINVAL;
1732 goto fail3;
1733 }
ecc910f5 1734 efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE;
8ceee660 1735
6d8aaaf6
DP		 1736#ifdef CONFIG_SFC_SRIOV
1737 rc = efx->type->vswitching_probe(efx);
1738 if (rc) /* not fatal; the PF will still work fine */
1739 netif_warn(efx, probe, efx->net_dev,
1740 "failed to setup vswitching rc=%d;"
1741 " VFs may not function\n", rc);
1742#endif
1743
64eebcfd
BH		 1744 rc = efx_probe_filters(efx);
1745 if (rc) {
1746 netif_err(efx, probe, efx->net_dev,
1747 "failed to create filter tables\n");
6d8aaaf6 1748 goto fail4;
64eebcfd
BH		 1749 }
1750
7f967c01
BH		 1751 rc = efx_probe_channels(efx);
1752 if (rc)
6d8aaaf6 1753 goto fail5;
7f967c01 1754
8ceee660
BH		 1755 return 0;
1756
6d8aaaf6 1757 fail5:
7f967c01 1758 efx_remove_filters(efx);
6d8aaaf6
DP		 1759 fail4:
1760#ifdef CONFIG_SFC_SRIOV
1761 efx->type->vswitching_remove(efx);
1762#endif
8ceee660 1763 fail3:
8ceee660
BH		 1764 efx_remove_port(efx);
1765 fail2:
1766 efx_remove_nic(efx);
1767 fail1:
1768 return rc;
1769}
1770
8b7325b4
BH		 1771/* If the interface is supposed to be running but is not, start
1772 * the hardware and software data path, regular activity for the port
1773 * (MAC statistics, link polling, etc.) and schedule the port to be
1774 * reconfigured. Interrupts must already be enabled. This function
1775 * is safe to call multiple times, so long as the NIC is not disabled.
1776 * Requires the RTNL lock.
9f2cb71c 1777 */
8ceee660
BH		 1778static void efx_start_all(struct efx_nic *efx)
1779{
8ceee660 1780 EFX_ASSERT_RESET_SERIALISED(efx);
8b7325b4 1781 BUG_ON(efx->state == STATE_DISABLED);
8ceee660
BH		 1782
1783 /* Check that it is appropriate to restart the interface. All
1784 * of these flags are safe to read under just the rtnl lock */
e283546c
EC		 1785 if (efx->port_enabled || !netif_running(efx->net_dev) ||
1786 efx->reset_pending)
8ceee660
BH		 1787 return;
1788
8ceee660 1789 efx_start_port(efx);
9f2cb71c 1790 efx_start_datapath(efx);
8880f4ec 1791
626950db
AR		 1792 /* Start the hardware monitor if there is one */
1793 if (efx->type->monitor != NULL)
8ceee660
BH		 1794 queue_delayed_work(efx->workqueue, &efx->monitor_work,
1795 efx_monitor_interval);
626950db
AR		 1796
1797 /* If link state detection is normally event-driven, we have
1798 * to poll now because we could have missed a change
1799 */
1800 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
78c1f0a0
SH		 1801 mutex_lock(&efx->mac_lock);
1802 if (efx->phy_op->poll(efx))
1803 efx_link_status_changed(efx);
1804 mutex_unlock(&efx->mac_lock);
1805 }
55edc6e6 1806
ef2b90ee 1807 efx->type->start_stats(efx);
f8f3b5ae
JC		 1808 efx->type->pull_stats(efx);
1809 spin_lock_bh(&efx->stats_lock);
1810 efx->type->update_stats(efx, NULL, NULL);
1811 spin_unlock_bh(&efx->stats_lock);
8ceee660
BH		 1812}
1813
8b7325b4
BH		 1814/* Quiesce the hardware and software data path, and regular activity
1815 * for the port without bringing the link down. Safe to call multiple
1816 * times with the NIC in almost any state, but interrupts should be
1817 * enabled. Requires the RTNL lock.
1818 */
8ceee660
BH		 1819static void efx_stop_all(struct efx_nic *efx)
1820{
8ceee660
BH		 1821 EFX_ASSERT_RESET_SERIALISED(efx);
1822
1823 /* port_enabled can be read safely under the rtnl lock */
1824 if (!efx->port_enabled)
1825 return;
1826
f8f3b5ae
JC		 1827 /* update stats before we go down so we can accurately count
1828 * rx_nodesc_drops
1829 */
1830 efx->type->pull_stats(efx);
1831 spin_lock_bh(&efx->stats_lock);
1832 efx->type->update_stats(efx, NULL, NULL);
1833 spin_unlock_bh(&efx->stats_lock);
ef2b90ee 1834 efx->type->stop_stats(efx);
8ceee660
BH		 1835 efx_stop_port(efx);
1836
29c69a48
BH		 1837 /* Stop the kernel transmit interface. This is only valid if
1838 * the device is stopped or detached; otherwise the watchdog
1839 * may fire immediately.
1840 */
1841 WARN_ON(netif_running(efx->net_dev) &&
1842 netif_device_present(efx->net_dev));
9f2cb71c
BH		 1843 netif_tx_disable(efx->net_dev);
1844
1845 efx_stop_datapath(efx);
8ceee660
BH		 1846}
1847
1848static void efx_remove_all(struct efx_nic *efx)
1849{
4642610c 1850 efx_remove_channels(efx);
7f967c01 1851 efx_remove_filters(efx);
6d8aaaf6
DP		 1852#ifdef CONFIG_SFC_SRIOV
1853 efx->type->vswitching_remove(efx);
1854#endif
8ceee660
BH		 1855 efx_remove_port(efx);
1856 efx_remove_nic(efx);
1857}
1858
8ceee660
BH		 1859/**************************************************************************
1860 *
1861 * Interrupt moderation
1862 *
1863 **************************************************************************/
1864
cc180b69 1865static unsigned int irq_mod_ticks(unsigned int usecs, unsigned int quantum_ns)
0d86ebd8 1866{
b548f976
BH		 1867 if (usecs == 0)
1868 return 0;
cc180b69 1869 if (usecs * 1000 < quantum_ns)
0d86ebd8 1870 return 1; /* never round down to 0 */
cc180b69 1871 return usecs * 1000 / quantum_ns;
0d86ebd8
BH		 1872}
1873
8ceee660 1874/* Set interrupt moderation parameters */
9e393b30
BH		 1875int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
1876 unsigned int rx_usecs, bool rx_adaptive,
1877 bool rx_may_override_tx)
8ceee660 1878{
f7d12cdc 1879 struct efx_channel *channel;
cc180b69
BH		 1880 unsigned int irq_mod_max = DIV_ROUND_UP(efx->type->timer_period_max *
1881 efx->timer_quantum_ns,
1882 1000);
1883 unsigned int tx_ticks;
1884 unsigned int rx_ticks;
8ceee660
BH		 1885
1886 EFX_ASSERT_RESET_SERIALISED(efx);
1887
cc180b69 1888 if (tx_usecs > irq_mod_max || rx_usecs > irq_mod_max)
9e393b30
BH		 1889 return -EINVAL;
1890
cc180b69
BH		 1891 tx_ticks = irq_mod_ticks(tx_usecs, efx->timer_quantum_ns);
1892 rx_ticks = irq_mod_ticks(rx_usecs, efx->timer_quantum_ns);
1893
9e393b30
BH		 1894 if (tx_ticks != rx_ticks && efx->tx_channel_offset == 0 &&
1895 !rx_may_override_tx) {
1896 netif_err(efx, drv, efx->net_dev, "Channels are shared. "
1897 "RX and TX IRQ moderation must be equal\n");
1898 return -EINVAL;
1899 }
1900
6fb70fd1 1901 efx->irq_rx_adaptive = rx_adaptive;
0d86ebd8 1902 efx->irq_rx_moderation = rx_ticks;
f7d12cdc 1903 efx_for_each_channel(channel, efx) {
525da907 1904 if (efx_channel_has_rx_queue(channel))
f7d12cdc 1905 channel->irq_moderation = rx_ticks;
525da907 1906 else if (efx_channel_has_tx_queues(channel))
f7d12cdc
BH		 1907 channel->irq_moderation = tx_ticks;
1908 }
9e393b30
BH		 1909
1910 return 0;
8ceee660
BH		 1911}
1912
a0c4faf5
BH		 1913void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs,
1914 unsigned int *rx_usecs, bool *rx_adaptive)
1915{
cc180b69
BH		 1916 /* We must round up when converting ticks to microseconds
1917 * because we round down when converting the other way.
1918 */
1919
a0c4faf5 1920 *rx_adaptive = efx->irq_rx_adaptive;
cc180b69
BH		 1921 *rx_usecs = DIV_ROUND_UP(efx->irq_rx_moderation *
1922 efx->timer_quantum_ns,
1923 1000);
a0c4faf5
BH		 1924
1925 /* If channels are shared between RX and TX, so is IRQ
1926 * moderation. Otherwise, IRQ moderation is the same for all
1927 * TX channels and is not adaptive.
1928 */
1929 if (efx->tx_channel_offset == 0)
1930 *tx_usecs = *rx_usecs;
1931 else
cc180b69 1932 *tx_usecs = DIV_ROUND_UP(
a0c4faf5 1933 efx->channel[efx->tx_channel_offset]->irq_moderation *
cc180b69
BH		 1934 efx->timer_quantum_ns,
1935 1000);
a0c4faf5
BH		 1936}
1937
8ceee660
BH		 1938/**************************************************************************
1939 *
1940 * Hardware monitor
1941 *
1942 **************************************************************************/
1943
e254c274 1944/* Run periodically off the general workqueue */
8ceee660
BH		 1945static void efx_monitor(struct work_struct *data)
1946{
1947 struct efx_nic *efx = container_of(data, struct efx_nic,
1948 monitor_work.work);
8ceee660 1949
62776d03
BH		 1950 netif_vdbg(efx, timer, efx->net_dev,
1951 "hardware monitor executing on CPU %d\n",
1952 raw_smp_processor_id());
ef2b90ee 1953 BUG_ON(efx->type->monitor == NULL);
8ceee660 1954
8ceee660
BH		 1955 /* If the mac_lock is already held then it is likely a port
1956 * reconfiguration is already in place, which will likely do
e254c274
BH		 1957 * most of the work of monitor() anyway. */
1958 if (mutex_trylock(&efx->mac_lock)) {
1959 if (efx->port_enabled)
1960 efx->type->monitor(efx);
1961 mutex_unlock(&efx->mac_lock);
1962 }
8ceee660 1963
8ceee660
BH		 1964 queue_delayed_work(efx->workqueue, &efx->monitor_work,
1965 efx_monitor_interval);
1966}
1967
1968/**************************************************************************
1969 *
1970 * ioctls
1971 *
1972 *************************************************************************/
1973
1974/* Net device ioctl
1975 * Context: process, rtnl_lock() held.
1976 */
1977static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
1978{
767e468c 1979 struct efx_nic *efx = netdev_priv(net_dev);
68e7f45e 1980 struct mii_ioctl_data *data = if_mii(ifr);
8ceee660 1981
7c236c43 1982 if (cmd == SIOCSHWTSTAMP)
433dc9b3
BH		 1983 return efx_ptp_set_ts_config(efx, ifr);
1984 if (cmd == SIOCGHWTSTAMP)
1985 return efx_ptp_get_ts_config(efx, ifr);
7c236c43 1986
68e7f45e
BH		 1987 /* Convert phy_id from older PRTAD/DEVAD format */
1988 if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) &&
1989 (data->phy_id & 0xfc00) == 0x0400)
1990 data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400;
1991
1992 return mdio_mii_ioctl(&efx->mdio, data, cmd);
8ceee660
BH		 1993}
1994
1995/**************************************************************************
1996 *
1997 * NAPI interface
1998 *
1999 **************************************************************************/
2000
7f967c01
BH		 2001static void efx_init_napi_channel(struct efx_channel *channel)
2002{
2003 struct efx_nic *efx = channel->efx;
2004
2005 channel->napi_dev = efx->net_dev;
2006 netif_napi_add(channel->napi_dev, &channel->napi_str,
2007 efx_poll, napi_weight);
36763266
AR		 2008 napi_hash_add(&channel->napi_str);
2009 efx_channel_init_lock(channel);
7f967c01
BH		 2010}
2011
e8f14992 2012static void efx_init_napi(struct efx_nic *efx)
8ceee660
BH		 2013{
2014 struct efx_channel *channel;
8ceee660 2015
7f967c01
BH		 2016 efx_for_each_channel(channel, efx)
2017 efx_init_napi_channel(channel);
e8f14992
BH		 2018}
2019
2020static void efx_fini_napi_channel(struct efx_channel *channel)
2021{
36763266 2022 if (channel->napi_dev) {
e8f14992 2023 netif_napi_del(&channel->napi_str);
36763266
AR		 2024 napi_hash_del(&channel->napi_str);
2025 }
e8f14992 2026 channel->napi_dev = NULL;
8ceee660
BH		 2027}
2028
2029static void efx_fini_napi(struct efx_nic *efx)
2030{
2031 struct efx_channel *channel;
2032
e8f14992
BH		 2033 efx_for_each_channel(channel, efx)
2034 efx_fini_napi_channel(channel);
8ceee660
BH		 2035}
2036
2037/**************************************************************************
2038 *
2039 * Kernel netpoll interface
2040 *
2041 *************************************************************************/
2042
2043#ifdef CONFIG_NET_POLL_CONTROLLER
2044
2045/* Although in the common case interrupts will be disabled, this is not
2046 * guaranteed. However, all our work happens inside the NAPI callback,
2047 * so no locking is required.
2048 */
2049static void efx_netpoll(struct net_device *net_dev)
2050{
767e468c 2051 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660
BH		 2052 struct efx_channel *channel;
2053
64ee3120 2054 efx_for_each_channel(channel, efx)
8ceee660
BH		 2055 efx_schedule_channel(channel);
2056}
2057
2058#endif
2059
36763266
AR		 2060#ifdef CONFIG_NET_RX_BUSY_POLL
2061static int efx_busy_poll(struct napi_struct *napi)
2062{
2063 struct efx_channel *channel =
2064 container_of(napi, struct efx_channel, napi_str);
2065 struct efx_nic *efx = channel->efx;
2066 int budget = 4;
2067 int old_rx_packets, rx_packets;
2068
2069 if (!netif_running(efx->net_dev))
2070 return LL_FLUSH_FAILED;
2071
2072 if (!efx_channel_lock_poll(channel))
2073 return LL_FLUSH_BUSY;
2074
2075 old_rx_packets = channel->rx_queue.rx_packets;
2076 efx_process_channel(channel, budget);
2077
2078 rx_packets = channel->rx_queue.rx_packets - old_rx_packets;
2079
2080 /* There is no race condition with NAPI here.
2081 * NAPI will automatically be rescheduled if it yielded during busy
2082 * polling, because it was not able to take the lock and thus returned
2083 * the full budget.
2084 */
2085 efx_channel_unlock_poll(channel);
2086
2087 return rx_packets;
2088}
2089#endif
2090
8ceee660
BH		 2091/**************************************************************************
2092 *
2093 * Kernel net device interface
2094 *
2095 *************************************************************************/
2096
2097/* Context: process, rtnl_lock() held. */
2098static int efx_net_open(struct net_device *net_dev)
2099{
767e468c 2100 struct efx_nic *efx = netdev_priv(net_dev);
8b7325b4
BH		 2101 int rc;
2102
62776d03
BH		 2103 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n",
2104 raw_smp_processor_id());
8ceee660 2105
8b7325b4
BH		 2106 rc = efx_check_disabled(efx);
2107 if (rc)
2108 return rc;
f8b87c17
BH		 2109 if (efx->phy_mode & PHY_MODE_SPECIAL)
2110 return -EBUSY;
8880f4ec
BH		 2111 if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL))
2112 return -EIO;
f8b87c17 2113
78c1f0a0
SH		 2114 /* Notify the kernel of the link state polled during driver load,
2115 * before the monitor starts running */
2116 efx_link_status_changed(efx);
2117
8ceee660 2118 efx_start_all(efx);
dd40781e 2119 efx_selftest_async_start(efx);
8ceee660
BH		 2120 return 0;
2121}
2122
2123/* Context: process, rtnl_lock() held.
2124 * Note that the kernel will ignore our return code; this method
2125 * should really be a void.
2126 */
2127static int efx_net_stop(struct net_device *net_dev)
2128{
767e468c 2129 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660 2130
62776d03
BH		 2131 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n",
2132 raw_smp_processor_id());
8ceee660 2133
8b7325b4
BH		 2134 /* Stop the device and flush all the channels */
2135 efx_stop_all(efx);
8ceee660
BH		 2136
2137 return 0;
2138}
2139
5b9e207c 2140/* Context: process, dev_base_lock or RTNL held, non-blocking. */
2aa9ef11
BH		 2141static struct rtnl_link_stats64 *efx_net_stats(struct net_device *net_dev,
2142 struct rtnl_link_stats64 *stats)
8ceee660 2143{
767e468c 2144 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660 2145
55edc6e6 2146 spin_lock_bh(&efx->stats_lock);
cd0ecc9a 2147 efx->type->update_stats(efx, NULL, stats);
1cb34522
BH		 2148 spin_unlock_bh(&efx->stats_lock);
2149
8ceee660
BH		 2150 return stats;
2151}
2152
2153/* Context: netif_tx_lock held, BHs disabled. */
2154static void efx_watchdog(struct net_device *net_dev)
2155{
767e468c 2156 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660 2157
62776d03
BH		 2158 netif_err(efx, tx_err, efx->net_dev,
2159 "TX stuck with port_enabled=%d: resetting channels\n",
2160 efx->port_enabled);
8ceee660 2161
739bb23d 2162 efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG);
8ceee660
BH		 2163}
2164
2165
2166/* Context: process, rtnl_lock() held. */
2167static int efx_change_mtu(struct net_device *net_dev, int new_mtu)
2168{
767e468c 2169 struct efx_nic *efx = netdev_priv(net_dev);
8b7325b4 2170 int rc;
8ceee660 2171
8b7325b4
BH		 2172 rc = efx_check_disabled(efx);
2173 if (rc)
2174 return rc;
8ceee660
BH		 2175 if (new_mtu > EFX_MAX_MTU)
2176 return -EINVAL;
2177
62776d03 2178 netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu);
8ceee660 2179
29c69a48
BH		 2180 efx_device_detach_sync(efx);
2181 efx_stop_all(efx);
2182
d3245b28 2183 mutex_lock(&efx->mac_lock);
8ceee660 2184 net_dev->mtu = new_mtu;
710b208d 2185 efx->type->reconfigure_mac(efx);
d3245b28
BH		 2186 mutex_unlock(&efx->mac_lock);
2187
8ceee660 2188 efx_start_all(efx);
29c69a48 2189 netif_device_attach(efx->net_dev);
6c8eef4a 2190 return 0;
8ceee660
BH		 2191}
2192
2193static int efx_set_mac_address(struct net_device *net_dev, void *data)
2194{
767e468c 2195 struct efx_nic *efx = netdev_priv(net_dev);
8ceee660 2196 struct sockaddr *addr = data;
e0b3ae30 2197 u8 *new_addr = addr->sa_data;
8ceee660 2198
8ceee660 2199 if (!is_valid_ether_addr(new_addr)) {
62776d03
BH		 2200 netif_err(efx, drv, efx->net_dev,
2201 "invalid ethernet MAC address requested: %pM\n",
2202 new_addr);
504f9b5a 2203 return -EADDRNOTAVAIL;
8ceee660
BH		 2204 }
2205
cd84ff4d 2206 ether_addr_copy(net_dev->dev_addr, new_addr);
7fa8d547
SS		 2207 if (efx->type->sriov_mac_address_changed)
2208 efx->type->sriov_mac_address_changed(efx);
8ceee660
BH		 2209
2210 /* Reconfigure the MAC */
d3245b28 2211 mutex_lock(&efx->mac_lock);
710b208d 2212 efx->type->reconfigure_mac(efx);
d3245b28 2213 mutex_unlock(&efx->mac_lock);
8ceee660
BH		 2214
2215 return 0;
2216}
2217
a816f75a 2218/* Context: netif_addr_lock held, BHs disabled. */
0fca8c97 2219static void efx_set_rx_mode(struct net_device *net_dev)
8ceee660 2220{
767e468c 2221 struct efx_nic *efx = netdev_priv(net_dev);
a816f75a 2222
8be4f3e6
BH		 2223 if (efx->port_enabled)
2224 queue_work(efx->workqueue, &efx->mac_work);
2225 /* Otherwise efx_start_port() will do this */
8ceee660
BH		 2226}
2227
c8f44aff 2228static int efx_set_features(struct net_device *net_dev, netdev_features_t data)
abfe9039
BH		 2229{
2230 struct efx_nic *efx = netdev_priv(net_dev);
2231
2232 /* If disabling RX n-tuple filtering, clear existing filters */
2233 if (net_dev->features & ~data & NETIF_F_NTUPLE)
fbd79120 2234 return efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL);
abfe9039
BH		 2235
2236 return 0;
2237}
2238
7fa8d547 2239static const struct net_device_ops efx_netdev_ops = {
c3ecb9f3
SH		 2240 .ndo_open = efx_net_open,
2241 .ndo_stop = efx_net_stop,
4472702e 2242 .ndo_get_stats64 = efx_net_stats,
c3ecb9f3
SH		 2243 .ndo_tx_timeout = efx_watchdog,
2244 .ndo_start_xmit = efx_hard_start_xmit,
2245 .ndo_validate_addr = eth_validate_addr,
2246 .ndo_do_ioctl = efx_ioctl,
2247 .ndo_change_mtu = efx_change_mtu,
2248 .ndo_set_mac_address = efx_set_mac_address,
0fca8c97 2249 .ndo_set_rx_mode = efx_set_rx_mode,
abfe9039 2250 .ndo_set_features = efx_set_features,
cd2d5b52 2251#ifdef CONFIG_SFC_SRIOV
7fa8d547
SS		 2252 .ndo_set_vf_mac = efx_sriov_set_vf_mac,
2253 .ndo_set_vf_vlan = efx_sriov_set_vf_vlan,
2254 .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk,
2255 .ndo_get_vf_config = efx_sriov_get_vf_config,
cd2d5b52 2256#endif
c3ecb9f3
SH		 2257#ifdef CONFIG_NET_POLL_CONTROLLER
2258 .ndo_poll_controller = efx_netpoll,
2259#endif
94b274bf 2260 .ndo_setup_tc = efx_setup_tc,
36763266
AR		 2261#ifdef CONFIG_NET_RX_BUSY_POLL
2262 .ndo_busy_poll = efx_busy_poll,
2263#endif
64d8ad6d
BH		 2264#ifdef CONFIG_RFS_ACCEL
2265 .ndo_rx_flow_steer = efx_filter_rfs,
2266#endif
c3ecb9f3
SH		 2267};
2268
7dde596e
BH		 2269static void efx_update_name(struct efx_nic *efx)
2270{
2271 strcpy(efx->name, efx->net_dev->name);
2272 efx_mtd_rename(efx);
2273 efx_set_channel_names(efx);
2274}
2275
8ceee660
BH		 2276static int efx_netdev_event(struct notifier_block *this,
2277 unsigned long event, void *ptr)
2278{
351638e7 2279 struct net_device *net_dev = netdev_notifier_info_to_dev(ptr);
8ceee660 2280
7fa8d547 2281 if ((net_dev->netdev_ops == &efx_netdev_ops) &&
7dde596e
BH		 2282 event == NETDEV_CHANGENAME)
2283 efx_update_name(netdev_priv(net_dev));
8ceee660
BH		 2284
2285 return NOTIFY_DONE;
2286}
2287
2288static struct notifier_block efx_netdev_notifier = {
2289 .notifier_call = efx_netdev_event,
2290};
2291
06d5e193
BH		 2292static ssize_t
2293show_phy_type(struct device *dev, struct device_attribute *attr, char *buf)
2294{
2295 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
2296 return sprintf(buf, "%d\n", efx->phy_type);
2297}
776fbcc9 2298static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL);
06d5e193 2299
8ceee660
BH		 2300static int efx_register_netdev(struct efx_nic *efx)
2301{
2302 struct net_device *net_dev = efx->net_dev;
c04bfc6b 2303 struct efx_channel *channel;
8ceee660
BH		 2304 int rc;
2305
2306 net_dev->watchdog_timeo = 5 * HZ;
2307 net_dev->irq = efx->pci_dev->irq;
7fa8d547
SS		 2308 net_dev->netdev_ops = &efx_netdev_ops;
2309 if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0)
8127d661 2310 net_dev->priv_flags |= IFF_UNICAST_FLT;
7ad24ea4 2311 net_dev->ethtool_ops = &efx_ethtool_ops;
7e6d06f0 2312 net_dev->gso_max_segs = EFX_TSO_MAX_SEGS;
8ceee660 2313
7dde596e 2314 rtnl_lock();
aed0628d 2315
7153f623
BH		 2316 /* Enable resets to be scheduled and check whether any were
2317 * already requested. If so, the NIC is probably hosed so we
2318 * abort.
2319 */
2320 efx->state = STATE_READY;
2321 smp_mb(); /* ensure we change state before checking reset_pending */
2322 if (efx->reset_pending) {
2323 netif_err(efx, probe, efx->net_dev,
2324 "aborting probe due to scheduled reset\n");
2325 rc = -EIO;
2326 goto fail_locked;
2327 }
2328
aed0628d
BH		 2329 rc = dev_alloc_name(net_dev, net_dev->name);
2330 if (rc < 0)
2331 goto fail_locked;
7dde596e 2332 efx_update_name(efx);
aed0628d 2333
8f8b3d51
BH		 2334 /* Always start with carrier off; PHY events will detect the link */
2335 netif_carrier_off(net_dev);
2336
aed0628d
BH		 2337 rc = register_netdevice(net_dev);
2338 if (rc)
2339 goto fail_locked;
2340
c04bfc6b
BH		 2341 efx_for_each_channel(channel, efx) {
2342 struct efx_tx_queue *tx_queue;
60031fcc
BH		 2343 efx_for_each_channel_tx_queue(tx_queue, channel)
2344 efx_init_tx_queue_core_txq(tx_queue);
c04bfc6b
BH		 2345 }
2346
0bcf4a64
BH		 2347 efx_associate(efx);
2348
7dde596e 2349 rtnl_unlock();
8ceee660 2350
06d5e193
BH		 2351 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2352 if (rc) {
62776d03
BH		 2353 netif_err(efx, drv, efx->net_dev,
2354 "failed to init net dev attributes\n");
06d5e193
BH		 2355 goto fail_registered;
2356 }
2357
8ceee660 2358 return 0;
06d5e193 2359
7153f623
BH		 2360fail_registered:
2361 rtnl_lock();
0bcf4a64 2362 efx_dissociate(efx);
7153f623 2363 unregister_netdevice(net_dev);
aed0628d 2364fail_locked:
7153f623 2365 efx->state = STATE_UNINIT;
aed0628d 2366 rtnl_unlock();
62776d03 2367 netif_err(efx, drv, efx->net_dev, "could not register net dev\n");
aed0628d 2368 return rc;
8ceee660
BH		 2369}
2370
2371static void efx_unregister_netdev(struct efx_nic *efx)
2372{
8ceee660
BH		 2373 if (!efx->net_dev)
2374 return;
2375
767e468c 2376 BUG_ON(netdev_priv(efx->net_dev) != efx);
8ceee660 2377
73ba7b68
BH		 2378 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
2379 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
7153f623
BH		 2380
2381 rtnl_lock();
2382 unregister_netdevice(efx->net_dev);
2383 efx->state = STATE_UNINIT;
2384 rtnl_unlock();
8ceee660
BH		 2385}
2386
2387/**************************************************************************
2388 *
2389 * Device reset and suspend
2390 *
2391 **************************************************************************/
2392
2467ca46
BH		 2393/* Tears down the entire software state and most of the hardware state
2394 * before reset. */
d3245b28 2395void efx_reset_down(struct efx_nic *efx, enum reset_type method)
8ceee660 2396{
8ceee660
BH		 2397 EFX_ASSERT_RESET_SERIALISED(efx);
2398
e283546c
EC		 2399 if (method == RESET_TYPE_MCDI_TIMEOUT)
2400 efx->type->prepare_flr(efx);
2401
2467ca46 2402 efx_stop_all(efx);
d8291187 2403 efx_disable_interrupts(efx);
5642ceef
BH		 2404
2405 mutex_lock(&efx->mac_lock);
4b988280
SH		 2406 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE)
2407 efx->phy_op->fini(efx);
ef2b90ee 2408 efx->type->fini(efx);
8ceee660
BH		 2409}
2410
2467ca46
BH		 2411/* This function will always ensure that the locks acquired in
2412 * efx_reset_down() are released. A failure return code indicates
2413 * that we were unable to reinitialise the hardware, and the
2414 * driver should be disabled. If ok is false, then the rx and tx
2415 * engines are not restarted, pending a RESET_DISABLE. */
d3245b28 2416int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)
8ceee660
BH		 2417{
2418 int rc;
2419
2467ca46 2420 EFX_ASSERT_RESET_SERIALISED(efx);
8ceee660 2421
e283546c
EC		 2422 if (method == RESET_TYPE_MCDI_TIMEOUT)
2423 efx->type->finish_flr(efx);
2424
2425 /* Ensure that SRAM is initialised even if we're disabling the device */
ef2b90ee 2426 rc = efx->type->init(efx);
8ceee660 2427 if (rc) {
62776d03 2428 netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n");
eb9f6744 2429 goto fail;
8ceee660
BH		 2430 }
2431
eb9f6744
BH		 2432 if (!ok)
2433 goto fail;
2434
4b988280 2435 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE) {
eb9f6744
BH		 2436 rc = efx->phy_op->init(efx);
2437 if (rc)
2438 goto fail;
267d9d73
EC		 2439 rc = efx->phy_op->reconfigure(efx);
2440 if (rc && rc != -EPERM)
62776d03
BH		 2441 netif_err(efx, drv, efx->net_dev,
2442 "could not restore PHY settings\n");
4b988280
SH		 2443 }
2444
261e4d96
JC		 2445 rc = efx_enable_interrupts(efx);
2446 if (rc)
2447 goto fail;
6d8aaaf6
DP		 2448
2449#ifdef CONFIG_SFC_SRIOV
2450 rc = efx->type->vswitching_restore(efx);
2451 if (rc) /* not fatal; the PF will still work fine */
2452 netif_warn(efx, probe, efx->net_dev,
2453 "failed to restore vswitching rc=%d;"
2454 " VFs may not function\n", rc);
2455#endif
2456
64eebcfd 2457 efx_restore_filters(efx);
7fa8d547
SS		 2458 if (efx->type->sriov_reset)
2459 efx->type->sriov_reset(efx);
eb9f6744 2460
eb9f6744
BH		 2461 mutex_unlock(&efx->mac_lock);
2462
2463 efx_start_all(efx);
2464
2465 return 0;
2466
2467fail:
2468 efx->port_initialized = false;
2467ca46
BH		 2469
2470 mutex_unlock(&efx->mac_lock);
2471
8ceee660
BH		 2472 return rc;
2473}
2474
eb9f6744
BH		 2475/* Reset the NIC using the specified method. Note that the reset may
2476 * fail, in which case the card will be left in an unusable state.
8ceee660 2477 *
eb9f6744 2478 * Caller must hold the rtnl_lock.
8ceee660 2479 */
eb9f6744 2480int efx_reset(struct efx_nic *efx, enum reset_type method)
8ceee660 2481{
eb9f6744
BH		 2482 int rc, rc2;
2483 bool disabled;
8ceee660 2484
62776d03
BH		 2485 netif_info(efx, drv, efx->net_dev, "resetting (%s)\n",
2486 RESET_TYPE(method));
8ceee660 2487
c2f3b8e3 2488 efx_device_detach_sync(efx);
d3245b28 2489 efx_reset_down(efx, method);
8ceee660 2490
ef2b90ee 2491 rc = efx->type->reset(efx, method);
8ceee660 2492 if (rc) {
62776d03 2493 netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n");
eb9f6744 2494 goto out;
8ceee660
BH		 2495 }
2496
a7d529ae
BH		 2497 /* Clear flags for the scopes we covered. We assume the NIC and
2498 * driver are now quiescent so that there is no race here.
2499 */
e283546c
EC		 2500 if (method < RESET_TYPE_MAX_METHOD)
2501 efx->reset_pending &= -(1 << (method + 1));
2502 else /* it doesn't fit into the well-ordered scope hierarchy */
2503 __clear_bit(method, &efx->reset_pending);
8ceee660
BH		 2504
2505 /* Reinitialise bus-mastering, which may have been turned off before
2506 * the reset was scheduled. This is still appropriate, even in the
2507 * RESET_TYPE_DISABLE since this driver generally assumes the hardware
2508 * can respond to requests. */
2509 pci_set_master(efx->pci_dev);
2510
eb9f6744 2511out:
8ceee660 2512 /* Leave device stopped if necessary */
626950db
AR		 2513 disabled = rc ||
2514 method == RESET_TYPE_DISABLE ||
2515 method == RESET_TYPE_RECOVER_OR_DISABLE;
eb9f6744
BH		 2516 rc2 = efx_reset_up(efx, method, !disabled);
2517 if (rc2) {
2518 disabled = true;
2519 if (!rc)
2520 rc = rc2;
8ceee660
BH		 2521 }
2522
eb9f6744 2523 if (disabled) {
f49a4589 2524 dev_close(efx->net_dev);
62776d03 2525 netif_err(efx, drv, efx->net_dev, "has been disabled\n");
f4bd954e 2526 efx->state = STATE_DISABLED;
f4bd954e 2527 } else {
62776d03 2528 netif_dbg(efx, drv, efx->net_dev, "reset complete\n");
e4abce85 2529 netif_device_attach(efx->net_dev);
f4bd954e 2530 }
8ceee660
BH		 2531 return rc;
2532}
2533
626950db
AR		 2534/* Try recovery mechanisms.
2535 * For now only EEH is supported.
2536 * Returns 0 if the recovery mechanisms are unsuccessful.
2537 * Returns a non-zero value otherwise.
2538 */
b28405b0 2539int efx_try_recovery(struct efx_nic *efx)
626950db
AR		 2540{
2541#ifdef CONFIG_EEH
2542 /* A PCI error can occur and not be seen by EEH because nothing
2543 * happens on the PCI bus. In this case the driver may fail and
2544 * schedule a 'recover or reset', leading to this recovery handler.
2545 * Manually call the eeh failure check function.
2546 */
12a89dba 2547 struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev);
626950db
AR		 2548 if (eeh_dev_check_failure(eehdev)) {
2549 /* The EEH mechanisms will handle the error and reset the
2550 * device if necessary.
2551 */
2552 return 1;
2553 }
2554#endif
2555 return 0;
2556}
2557
74cd60a4
JC		 2558static void efx_wait_for_bist_end(struct efx_nic *efx)
2559{
2560 int i;
2561
2562 for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) {
2563 if (efx_mcdi_poll_reboot(efx))
2564 goto out;
2565 msleep(BIST_WAIT_DELAY_MS);
2566 }
2567
2568 netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n");
2569out:
2570 /* Either way unset the BIST flag. If we found no reboot we probably
2571 * won't recover, but we should try.
2572 */
2573 efx->mc_bist_for_other_fn = false;
2574}
2575
8ceee660
BH		 2576/* The worker thread exists so that code that cannot sleep can
2577 * schedule a reset for later.
2578 */
2579static void efx_reset_work(struct work_struct *data)
2580{
eb9f6744 2581 struct efx_nic *efx = container_of(data, struct efx_nic, reset_work);
626950db
AR		 2582 unsigned long pending;
2583 enum reset_type method;
2584
2585 pending = ACCESS_ONCE(efx->reset_pending);
2586 method = fls(pending) - 1;
2587
74cd60a4
JC		 2588 if (method == RESET_TYPE_MC_BIST)
2589 efx_wait_for_bist_end(efx);
2590
626950db
AR		 2591 if ((method == RESET_TYPE_RECOVER_OR_DISABLE ||
2592 method == RESET_TYPE_RECOVER_OR_ALL) &&
2593 efx_try_recovery(efx))
2594 return;
8ceee660 2595
a7d529ae 2596 if (!pending)
319ba649
SH		 2597 return;
2598
eb9f6744 2599 rtnl_lock();
7153f623
BH		 2600
2601 /* We checked the state in efx_schedule_reset() but it may
2602 * have changed by now. Now that we have the RTNL lock,
2603 * it cannot change again.
2604 */
2605 if (efx->state == STATE_READY)
626950db 2606 (void)efx_reset(efx, method);
7153f623 2607
eb9f6744 2608 rtnl_unlock();
8ceee660
BH		 2609}
2610
2611void efx_schedule_reset(struct efx_nic *efx, enum reset_type type)
2612{
2613 enum reset_type method;
2614
626950db
AR		 2615 if (efx->state == STATE_RECOVERY) {
2616 netif_dbg(efx, drv, efx->net_dev,
2617 "recovering: skip scheduling %s reset\n",
2618 RESET_TYPE(type));
2619 return;
2620 }
2621
8ceee660
BH		 2622 switch (type) {
2623 case RESET_TYPE_INVISIBLE:
2624 case RESET_TYPE_ALL:
626950db 2625 case RESET_TYPE_RECOVER_OR_ALL:
8ceee660
BH		 2626 case RESET_TYPE_WORLD:
2627 case RESET_TYPE_DISABLE:
626950db 2628 case RESET_TYPE_RECOVER_OR_DISABLE:
74cd60a4 2629 case RESET_TYPE_MC_BIST:
e283546c 2630 case RESET_TYPE_MCDI_TIMEOUT:
8ceee660 2631 method = type;
0e2a9c7c
BH		 2632 netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n",
2633 RESET_TYPE(method));
8ceee660 2634 break;
8ceee660 2635 default:
0e2a9c7c 2636 method = efx->type->map_reset_reason(type);
62776d03
BH		 2637 netif_dbg(efx, drv, efx->net_dev,
2638 "scheduling %s reset for %s\n",
2639 RESET_TYPE(method), RESET_TYPE(type));
0e2a9c7c
BH		 2640 break;
2641 }
8ceee660 2642
a7d529ae 2643 set_bit(method, &efx->reset_pending);
7153f623
BH		 2644 smp_mb(); /* ensure we change reset_pending before checking state */
2645
2646 /* If we're not READY then just leave the flags set as the cue
2647 * to abort probing or reschedule the reset later.
2648 */
2649 if (ACCESS_ONCE(efx->state) != STATE_READY)
2650 return;
8ceee660 2651
8880f4ec
BH		 2652 /* efx_process_channel() will no longer read events once a
2653 * reset is scheduled. So switch back to poll'd MCDI completions. */
2654 efx_mcdi_mode_poll(efx);
2655
1ab00629 2656 queue_work(reset_workqueue, &efx->reset_work);
8ceee660
BH		 2657}
2658
2659/**************************************************************************
2660 *
2661 * List of NICs we support
2662 *
2663 **************************************************************************/
2664
2665/* PCI device ID table */
9baa3c34 2666static const struct pci_device_id efx_pci_table[] = {
937383a5
BH		 2667 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE,
2668 PCI_DEVICE_ID_SOLARFLARE_SFC4000A_0),
daeda630 2669 .driver_data = (unsigned long) &falcon_a1_nic_type},
937383a5
BH		 2670 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE,
2671 PCI_DEVICE_ID_SOLARFLARE_SFC4000B),
daeda630 2672 .driver_data = (unsigned long) &falcon_b0_nic_type},
547c474f 2673 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */
8880f4ec 2674 .driver_data = (unsigned long) &siena_a0_nic_type},
547c474f 2675 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */
8880f4ec 2676 .driver_data = (unsigned long) &siena_a0_nic_type},
8127d661
BH		 2677 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */
2678 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
3b06a00e
MW		 2679 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */
2680 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
8ceee660
BH		 2681 {0} /* end of list */
2682};
2683
2684/**************************************************************************
2685 *
3759433d 2686 * Dummy PHY/MAC operations
8ceee660 2687 *
01aad7b6 2688 * Can be used for some unimplemented operations
8ceee660
BH		 2689 * Needed so all function pointers are valid and do not have to be tested
2690 * before use
2691 *
2692 **************************************************************************/
2693int efx_port_dummy_op_int(struct efx_nic *efx)
2694{
2695 return 0;
2696}
2697void efx_port_dummy_op_void(struct efx_nic *efx) {}
d215697f 2698
2699static bool efx_port_dummy_op_poll(struct efx_nic *efx)
fdaa9aed
SH		 2700{
2701 return false;
2702}
8ceee660 2703
6c8c2513 2704static const struct efx_phy_operations efx_dummy_phy_operations = {
8ceee660 2705 .init = efx_port_dummy_op_int,
d3245b28 2706 .reconfigure = efx_port_dummy_op_int,
fdaa9aed 2707 .poll = efx_port_dummy_op_poll,
8ceee660 2708 .fini = efx_port_dummy_op_void,
8ceee660
BH		 2709};
2710
8ceee660
BH		 2711/**************************************************************************
2712 *
2713 * Data housekeeping
2714 *
2715 **************************************************************************/
2716
2717/* This zeroes out and then fills in the invariants in a struct
2718 * efx_nic (including all sub-structures).
2719 */
adeb15aa 2720static int efx_init_struct(struct efx_nic *efx,
8ceee660
BH		 2721 struct pci_dev *pci_dev, struct net_device *net_dev)
2722{
4642610c 2723 int i;
8ceee660
BH		 2724
2725 /* Initialise common structures */
0bcf4a64
BH		 2726 INIT_LIST_HEAD(&efx->node);
2727 INIT_LIST_HEAD(&efx->secondary_list);
8ceee660 2728 spin_lock_init(&efx->biu_lock);
76884835
BH		 2729#ifdef CONFIG_SFC_MTD
2730 INIT_LIST_HEAD(&efx->mtd_list);
2731#endif
8ceee660
BH		 2732 INIT_WORK(&efx->reset_work, efx_reset_work);
2733 INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor);
dd40781e 2734 INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work);
8ceee660 2735 efx->pci_dev = pci_dev;
62776d03 2736 efx->msg_enable = debug;
f16aeea0 2737 efx->state = STATE_UNINIT;
8ceee660 2738 strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
8ceee660
BH		 2739
2740 efx->net_dev = net_dev;
43a3739d 2741 efx->rx_prefix_size = efx->type->rx_prefix_size;
2ec03014
AR		 2742 efx->rx_ip_align =
2743 NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0;
43a3739d
JC		 2744 efx->rx_packet_hash_offset =
2745 efx->type->rx_hash_offset - efx->type->rx_prefix_size;
bd9a265d
JC		 2746 efx->rx_packet_ts_offset =
2747 efx->type->rx_ts_offset - efx->type->rx_prefix_size;
8ceee660
BH		 2748 spin_lock_init(&efx->stats_lock);
2749 mutex_init(&efx->mac_lock);
2750 efx->phy_op = &efx_dummy_phy_operations;
68e7f45e 2751 efx->mdio.dev = net_dev;
766ca0fa 2752 INIT_WORK(&efx->mac_work, efx_mac_work);
9f2cb71c 2753 init_waitqueue_head(&efx->flush_wq);
8ceee660
BH		 2754
2755 for (i = 0; i < EFX_MAX_CHANNELS; i++) {
4642610c
BH		 2756 efx->channel[i] = efx_alloc_channel(efx, i, NULL);
2757 if (!efx->channel[i])
2758 goto fail;
d8291187
BH		 2759 efx->msi_context[i].efx = efx;
2760 efx->msi_context[i].index = i;
8ceee660
BH		 2761 }
2762
8ceee660
BH		 2763 /* Higher numbered interrupt modes are less capable! */
2764 efx->interrupt_mode = max(efx->type->max_interrupt_mode,
2765 interrupt_mode);
2766
6977dc63
BH		 2767 /* Would be good to use the net_dev name, but we're too early */
2768 snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s",
2769 pci_name(pci_dev));
2770 efx->workqueue = create_singlethread_workqueue(efx->workqueue_name);
1ab00629 2771 if (!efx->workqueue)
4642610c 2772 goto fail;
8d9853d9 2773
8ceee660 2774 return 0;
4642610c
BH		 2775
2776fail:
2777 efx_fini_struct(efx);
2778 return -ENOMEM;
8ceee660
BH		 2779}
2780
2781static void efx_fini_struct(struct efx_nic *efx)
2782{
8313aca3
BH		 2783 int i;
2784
2785 for (i = 0; i < EFX_MAX_CHANNELS; i++)
2786 kfree(efx->channel[i]);
2787
ef215e64
BH		 2788 kfree(efx->vpd_sn);
2789
8ceee660
BH		 2790 if (efx->workqueue) {
2791 destroy_workqueue(efx->workqueue);
2792 efx->workqueue = NULL;
2793 }
2794}
2795
e4d112e4
EC		 2796void efx_update_sw_stats(struct efx_nic *efx, u64 *stats)
2797{
2798 u64 n_rx_nodesc_trunc = 0;
2799 struct efx_channel *channel;
2800
2801 efx_for_each_channel(channel, efx)
2802 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc;
2803 stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc;
2804 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops);
2805}
2806
8ceee660
BH		 2807/**************************************************************************
2808 *
2809 * PCI interface
2810 *
2811 **************************************************************************/
2812
2813/* Main body of final NIC shutdown code
2814 * This is called only at module unload (or hotplug removal).
2815 */
2816static void efx_pci_remove_main(struct efx_nic *efx)
2817{
7153f623
BH		 2818 /* Flush reset_work. It can no longer be scheduled since we
2819 * are not READY.
2820 */
2821 BUG_ON(efx->state == STATE_READY);
2822 cancel_work_sync(&efx->reset_work);
2823
d8291187 2824 efx_disable_interrupts(efx);
152b6a62 2825 efx_nic_fini_interrupt(efx);
8ceee660 2826 efx_fini_port(efx);
ef2b90ee 2827 efx->type->fini(efx);
8ceee660
BH		 2828 efx_fini_napi(efx);
2829 efx_remove_all(efx);
2830}
2831
2832/* Final NIC shutdown
2833 * This is called only at module unload (or hotplug removal).
2834 */
2835static void efx_pci_remove(struct pci_dev *pci_dev)
2836{
2837 struct efx_nic *efx;
2838
2839 efx = pci_get_drvdata(pci_dev);
2840 if (!efx)
2841 return;
2842
2843 /* Mark the NIC as fini, then stop the interface */
2844 rtnl_lock();
0bcf4a64 2845 efx_dissociate(efx);
8ceee660 2846 dev_close(efx->net_dev);
d8291187 2847 efx_disable_interrupts(efx);
8ceee660
BH		 2848 rtnl_unlock();
2849
7fa8d547
SS		 2850 if (efx->type->sriov_fini)
2851 efx->type->sriov_fini(efx);
2852
8ceee660
BH		 2853 efx_unregister_netdev(efx);
2854
7dde596e
BH		 2855 efx_mtd_remove(efx);
2856
8ceee660
BH		 2857 efx_pci_remove_main(efx);
2858
8ceee660 2859 efx_fini_io(efx);
62776d03 2860 netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n");
8ceee660 2861
8ceee660
BH		 2862 efx_fini_struct(efx);
2863 free_netdev(efx->net_dev);
626950db
AR		 2864
2865 pci_disable_pcie_error_reporting(pci_dev);
8ceee660
BH		 2866};
2867
460eeaa0
BH		 2868/* NIC VPD information
2869 * Called during probe to display the part number of the
2870 * installed NIC. VPD is potentially very large but this should
2871 * always appear within the first 512 bytes.
2872 */
2873#define SFC_VPD_LEN 512
ef215e64 2874static void efx_probe_vpd_strings(struct efx_nic *efx)
460eeaa0
BH		 2875{
2876 struct pci_dev *dev = efx->pci_dev;
2877 char vpd_data[SFC_VPD_LEN];
2878 ssize_t vpd_size;
ef215e64 2879 int ro_start, ro_size, i, j;
460eeaa0
BH		 2880
2881 /* Get the vpd data from the device */
2882 vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data);
2883 if (vpd_size <= 0) {
2884 netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n");
2885 return;
2886 }
2887
2888 /* Get the Read only section */
ef215e64
BH		 2889 ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
2890 if (ro_start < 0) {
460eeaa0
BH		 2891 netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n");
2892 return;
2893 }
2894
ef215e64
BH		 2895 ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]);
2896 j = ro_size;
2897 i = ro_start + PCI_VPD_LRDT_TAG_SIZE;
460eeaa0
BH		 2898 if (i + j > vpd_size)
2899 j = vpd_size - i;
2900
2901 /* Get the Part number */
2902 i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN");
2903 if (i < 0) {
2904 netif_err(efx, drv, efx->net_dev, "Part number not found\n");
2905 return;
2906 }
2907
2908 j = pci_vpd_info_field_size(&vpd_data[i]);
2909 i += PCI_VPD_INFO_FLD_HDR_SIZE;
2910 if (i + j > vpd_size) {
2911 netif_err(efx, drv, efx->net_dev, "Incomplete part number\n");
2912 return;
2913 }
2914
2915 netif_info(efx, drv, efx->net_dev,
2916 "Part Number : %.*s\n", j, &vpd_data[i]);
ef215e64
BH		 2917
2918 i = ro_start + PCI_VPD_LRDT_TAG_SIZE;
2919 j = ro_size;
2920 i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN");
2921 if (i < 0) {
2922 netif_err(efx, drv, efx->net_dev, "Serial number not found\n");
2923 return;
2924 }
2925
2926 j = pci_vpd_info_field_size(&vpd_data[i]);
2927 i += PCI_VPD_INFO_FLD_HDR_SIZE;
2928 if (i + j > vpd_size) {
2929 netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n");
2930 return;
2931 }
2932
2933 efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL);
2934 if (!efx->vpd_sn)
2935 return;
2936
2937 snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]);
460eeaa0
BH		 2938}
2939
2940
8ceee660
BH		 2941/* Main body of NIC initialisation
2942 * This is called at module load (or hotplug insertion, theoretically).
2943 */
2944static int efx_pci_probe_main(struct efx_nic *efx)
2945{
2946 int rc;
2947
2948 /* Do start-of-day initialisation */
2949 rc = efx_probe_all(efx);
2950 if (rc)
2951 goto fail1;
2952
e8f14992 2953 efx_init_napi(efx);
8ceee660 2954
ef2b90ee 2955 rc = efx->type->init(efx);
8ceee660 2956 if (rc) {
62776d03
BH		 2957 netif_err(efx, probe, efx->net_dev,
2958 "failed to initialise NIC\n");
278c0621 2959 goto fail3;
8ceee660
BH		 2960 }
2961
2962 rc = efx_init_port(efx);
2963 if (rc) {
62776d03
BH		 2964 netif_err(efx, probe, efx->net_dev,
2965 "failed to initialise port\n");
278c0621 2966 goto fail4;
8ceee660
BH		 2967 }
2968
152b6a62 2969 rc = efx_nic_init_interrupt(efx);
8ceee660 2970 if (rc)
278c0621 2971 goto fail5;
261e4d96
JC		 2972 rc = efx_enable_interrupts(efx);
2973 if (rc)
2974 goto fail6;
8ceee660
BH		 2975
2976 return 0;
2977
261e4d96
JC		 2978 fail6:
2979 efx_nic_fini_interrupt(efx);
278c0621 2980 fail5:
8ceee660 2981 efx_fini_port(efx);
8ceee660 2982 fail4:
ef2b90ee 2983 efx->type->fini(efx);
8ceee660
BH		 2984 fail3:
2985 efx_fini_napi(efx);
8ceee660
BH		 2986 efx_remove_all(efx);
2987 fail1:
2988 return rc;
2989}
2990
2991/* NIC initialisation
2992 *
2993 * This is called at module load (or hotplug insertion,
73ba7b68 2994 * theoretically). It sets up PCI mappings, resets the NIC,
8ceee660
BH		 2995 * sets up and registers the network devices with the kernel and hooks
2996 * the interrupt service routine. It does not prepare the device for
2997 * transmission; this is left to the first time one of the network
2998 * interfaces is brought up (i.e. efx_net_open).
2999 */
87d1fc11 3000static int efx_pci_probe(struct pci_dev *pci_dev,
1dd06ae8 3001 const struct pci_device_id *entry)
8ceee660 3002{
8ceee660
BH		 3003 struct net_device *net_dev;
3004 struct efx_nic *efx;
fadac6aa 3005 int rc;
8ceee660
BH		 3006
3007 /* Allocate and initialise a struct net_device and struct efx_nic */
94b274bf
BH		 3008 net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES,
3009 EFX_MAX_RX_QUEUES);
8ceee660
BH		 3010 if (!net_dev)
3011 return -ENOMEM;
adeb15aa
BH		 3012 efx = netdev_priv(net_dev);
3013 efx->type = (const struct efx_nic_type *) entry->driver_data;
3014 net_dev->features |= (efx->type->offload_features | NETIF_F_SG |
97bc5415 3015 NETIF_F_HIGHDMA | NETIF_F_TSO |
abfe9039 3016 NETIF_F_RXCSUM);
adeb15aa 3017 if (efx->type->offload_features & NETIF_F_V6_CSUM)
738a8f4b 3018 net_dev->features |= NETIF_F_TSO6;
28506563
BH		 3019 /* Mask for features that also apply to VLAN devices */
3020 net_dev->vlan_features |= (NETIF_F_ALL_CSUM | NETIF_F_SG |
abfe9039
BH		 3021 NETIF_F_HIGHDMA | NETIF_F_ALL_TSO |
3022 NETIF_F_RXCSUM);
3023 /* All offloads can be toggled */
3024 net_dev->hw_features = net_dev->features & ~NETIF_F_HIGHDMA;
8ceee660 3025 pci_set_drvdata(pci_dev, efx);
62776d03 3026 SET_NETDEV_DEV(net_dev, &pci_dev->dev);
adeb15aa 3027 rc = efx_init_struct(efx, pci_dev, net_dev);
8ceee660
BH		 3028 if (rc)
3029 goto fail1;
3030
62776d03 3031 netif_info(efx, probe, efx->net_dev,
ff79c8ac 3032 "Solarflare NIC detected\n");
8ceee660 3033
ef215e64 3034 efx_probe_vpd_strings(efx);
460eeaa0 3035
8ceee660
BH		 3036 /* Set up basic I/O (BAR mappings etc) */
3037 rc = efx_init_io(efx);
3038 if (rc)
3039 goto fail2;
3040
fadac6aa 3041 rc = efx_pci_probe_main(efx);
fadac6aa
BH		 3042 if (rc)
3043 goto fail3;
8ceee660 3044
8ceee660
BH		 3045 rc = efx_register_netdev(efx);
3046 if (rc)
fadac6aa 3047 goto fail4;
8ceee660 3048
7fa8d547
SS		 3049 if (efx->type->sriov_init) {
3050 rc = efx->type->sriov_init(efx);
3051 if (rc)
3052 netif_err(efx, probe, efx->net_dev,
3053 "SR-IOV can't be enabled rc %d\n", rc);
3054 }
cd2d5b52 3055
62776d03 3056 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n");
a5211bb5 3057
7c43161c 3058 /* Try to create MTDs, but allow this to fail */
a5211bb5 3059 rtnl_lock();
7c43161c 3060 rc = efx_mtd_probe(efx);
a5211bb5 3061 rtnl_unlock();
7c43161c
BH		 3062 if (rc)
3063 netif_warn(efx, probe, efx->net_dev,
3064 "failed to create MTDs (%d)\n", rc);
3065
626950db
AR		 3066 rc = pci_enable_pcie_error_reporting(pci_dev);
3067 if (rc && rc != -EINVAL)
3068 netif_warn(efx, probe, efx->net_dev,
3069 "pci_enable_pcie_error_reporting failed (%d)\n", rc);
3070
8ceee660
BH		 3071 return 0;
3072
8ceee660 3073 fail4:
fadac6aa 3074 efx_pci_remove_main(efx);
8ceee660
BH		 3075 fail3:
3076 efx_fini_io(efx);
3077 fail2:
3078 efx_fini_struct(efx);
3079 fail1:
5e2a911c 3080 WARN_ON(rc > 0);
62776d03 3081 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc);
8ceee660
BH		 3082 free_netdev(net_dev);
3083 return rc;
3084}
3085
834e23dd
SS		 3086/* efx_pci_sriov_configure returns the actual number of Virtual Functions
3087 * enabled on success
3088 */
3089#ifdef CONFIG_SFC_SRIOV
3090static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
3091{
3092 int rc;
3093 struct efx_nic *efx = pci_get_drvdata(dev);
3094
3095 if (efx->type->sriov_configure) {
3096 rc = efx->type->sriov_configure(efx, num_vfs);
3097 if (rc)
3098 return rc;
3099 else
3100 return num_vfs;
3101 } else
3102 return -EOPNOTSUPP;
3103}
3104#endif
3105
89c758fa
BH		 3106static int efx_pm_freeze(struct device *dev)
3107{
3108 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
3109
61da026d
BH		 3110 rtnl_lock();
3111
6032fb56
BH		 3112 if (efx->state != STATE_DISABLED) {
3113 efx->state = STATE_UNINIT;
89c758fa 3114
c2f3b8e3 3115 efx_device_detach_sync(efx);
89c758fa 3116
6032fb56 3117 efx_stop_all(efx);
d8291187 3118 efx_disable_interrupts(efx);
6032fb56 3119 }
89c758fa 3120
61da026d
BH		 3121 rtnl_unlock();
3122
89c758fa
BH		 3123 return 0;
3124}
3125
3126static int efx_pm_thaw(struct device *dev)
3127{
261e4d96 3128 int rc;
89c758fa
BH		 3129 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
3130
61da026d
BH		 3131 rtnl_lock();
3132
6032fb56 3133 if (efx->state != STATE_DISABLED) {
261e4d96
JC		 3134 rc = efx_enable_interrupts(efx);
3135 if (rc)
3136 goto fail;
89c758fa 3137
6032fb56
BH		 3138 mutex_lock(&efx->mac_lock);
3139 efx->phy_op->reconfigure(efx);
3140 mutex_unlock(&efx->mac_lock);
89c758fa 3141
6032fb56 3142 efx_start_all(efx);
89c758fa 3143
6032fb56 3144 netif_device_attach(efx->net_dev);
89c758fa 3145
6032fb56 3146 efx->state = STATE_READY;
89c758fa 3147
6032fb56
BH		 3148 efx->type->resume_wol(efx);
3149 }
89c758fa 3150
61da026d
BH		 3151 rtnl_unlock();
3152
319ba649
SH		 3153 /* Reschedule any quenched resets scheduled during efx_pm_freeze() */
3154 queue_work(reset_workqueue, &efx->reset_work);
3155
89c758fa 3156 return 0;
261e4d96
JC		 3157
3158fail:
3159 rtnl_unlock();
3160
3161 return rc;
89c758fa
BH		 3162}
3163
3164static int efx_pm_poweroff(struct device *dev)
3165{
3166 struct pci_dev *pci_dev = to_pci_dev(dev);
3167 struct efx_nic *efx = pci_get_drvdata(pci_dev);
3168
3169 efx->type->fini(efx);
3170
a7d529ae 3171 efx->reset_pending = 0;
89c758fa
BH		 3172
3173 pci_save_state(pci_dev);
3174 return pci_set_power_state(pci_dev, PCI_D3hot);
3175}
3176
3177/* Used for both resume and restore */
3178static int efx_pm_resume(struct device *dev)
3179{
3180 struct pci_dev *pci_dev = to_pci_dev(dev);
3181 struct efx_nic *efx = pci_get_drvdata(pci_dev);
3182 int rc;
3183
3184 rc = pci_set_power_state(pci_dev, PCI_D0);
3185 if (rc)
3186 return rc;
3187 pci_restore_state(pci_dev);
3188 rc = pci_enable_device(pci_dev);
3189 if (rc)
3190 return rc;
3191 pci_set_master(efx->pci_dev);
3192 rc = efx->type->reset(efx, RESET_TYPE_ALL);
3193 if (rc)
3194 return rc;
3195 rc = efx->type->init(efx);
3196 if (rc)
3197 return rc;
261e4d96
JC		 3198 rc = efx_pm_thaw(dev);
3199 return rc;
89c758fa
BH		 3200}
3201
3202static int efx_pm_suspend(struct device *dev)
3203{
3204 int rc;
3205
3206 efx_pm_freeze(dev);
3207 rc = efx_pm_poweroff(dev);
3208 if (rc)
3209 efx_pm_resume(dev);
3210 return rc;
3211}
3212
18e83e4c 3213static const struct dev_pm_ops efx_pm_ops = {
89c758fa
BH		 3214 .suspend = efx_pm_suspend,
3215 .resume = efx_pm_resume,
3216 .freeze = efx_pm_freeze,
3217 .thaw = efx_pm_thaw,
3218 .poweroff = efx_pm_poweroff,
3219 .restore = efx_pm_resume,
3220};
3221
626950db
AR		 3222/* A PCI error affecting this device was detected.
3223 * At this point MMIO and DMA may be disabled.
3224 * Stop the software path and request a slot reset.
3225 */
debd0034 3226static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev,
3227 enum pci_channel_state state)
626950db
AR		 3228{
3229 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED;
3230 struct efx_nic *efx = pci_get_drvdata(pdev);
3231
3232 if (state == pci_channel_io_perm_failure)
3233 return PCI_ERS_RESULT_DISCONNECT;
3234
3235 rtnl_lock();
3236
3237 if (efx->state != STATE_DISABLED) {
3238 efx->state = STATE_RECOVERY;
3239 efx->reset_pending = 0;
3240
3241 efx_device_detach_sync(efx);
3242
3243 efx_stop_all(efx);
d8291187 3244 efx_disable_interrupts(efx);
626950db
AR		 3245
3246 status = PCI_ERS_RESULT_NEED_RESET;
3247 } else {
3248 /* If the interface is disabled we don't want to do anything
3249 * with it.
3250 */
3251 status = PCI_ERS_RESULT_RECOVERED;
3252 }
3253
3254 rtnl_unlock();
3255
3256 pci_disable_device(pdev);
3257
3258 return status;
3259}
3260
dbedd44e 3261/* Fake a successful reset, which will be performed later in efx_io_resume. */
debd0034 3262static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev)
626950db
AR		 3263{
3264 struct efx_nic *efx = pci_get_drvdata(pdev);
3265 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED;
3266 int rc;
3267
3268 if (pci_enable_device(pdev)) {
3269 netif_err(efx, hw, efx->net_dev,
3270 "Cannot re-enable PCI device after reset.\n");
3271 status = PCI_ERS_RESULT_DISCONNECT;
3272 }
3273
3274 rc = pci_cleanup_aer_uncorrect_error_status(pdev);
3275 if (rc) {
3276 netif_err(efx, hw, efx->net_dev,
3277 "pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc);
3278 /* Non-fatal error. Continue. */
3279 }
3280
3281 return status;
3282}
3283
3284/* Perform the actual reset and resume I/O operations. */
3285static void efx_io_resume(struct pci_dev *pdev)
3286{
3287 struct efx_nic *efx = pci_get_drvdata(pdev);
3288 int rc;
3289
3290 rtnl_lock();
3291
3292 if (efx->state == STATE_DISABLED)
3293 goto out;
3294
3295 rc = efx_reset(efx, RESET_TYPE_ALL);
3296 if (rc) {
3297 netif_err(efx, hw, efx->net_dev,
3298 "efx_reset failed after PCI error (%d)\n", rc);
3299 } else {
3300 efx->state = STATE_READY;
3301 netif_dbg(efx, hw, efx->net_dev,
3302 "Done resetting and resuming IO after PCI error.\n");
3303 }
3304
3305out:
3306 rtnl_unlock();
3307}
3308
3309/* For simplicity and reliability, we always require a slot reset and try to
3310 * reset the hardware when a pci error affecting the device is detected.
3311 * We leave both the link_reset and mmio_enabled callback unimplemented:
3312 * with our request for slot reset the mmio_enabled callback will never be
3313 * called, and the link_reset callback is not used by AER or EEH mechanisms.
3314 */
3315static struct pci_error_handlers efx_err_handlers = {
3316 .error_detected = efx_io_error_detected,
3317 .slot_reset = efx_io_slot_reset,
3318 .resume = efx_io_resume,
3319};
3320
8ceee660 3321static struct pci_driver efx_pci_driver = {
c5d5f5fd 3322 .name = KBUILD_MODNAME,
8ceee660
BH		 3323 .id_table = efx_pci_table,
3324 .probe = efx_pci_probe,
3325 .remove = efx_pci_remove,
89c758fa 3326 .driver.pm = &efx_pm_ops,
626950db 3327 .err_handler = &efx_err_handlers,
834e23dd
SS		 3328#ifdef CONFIG_SFC_SRIOV
3329 .sriov_configure = efx_pci_sriov_configure,
3330#endif
8ceee660
BH		 3331};
3332
3333/**************************************************************************
3334 *
3335 * Kernel module interface
3336 *
3337 *************************************************************************/
3338
3339module_param(interrupt_mode, uint, 0444);
3340MODULE_PARM_DESC(interrupt_mode,
3341 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");
3342
3343static int __init efx_init_module(void)
3344{
3345 int rc;
3346
3347 printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n");
3348
3349 rc = register_netdevice_notifier(&efx_netdev_notifier);
3350 if (rc)
3351 goto err_notifier;
3352
7fa8d547 3353#ifdef CONFIG_SFC_SRIOV
cd2d5b52
BH		 3354 rc = efx_init_sriov();
3355 if (rc)
3356 goto err_sriov;
7fa8d547 3357#endif
cd2d5b52 3358
1ab00629
SH		 3359 reset_workqueue = create_singlethread_workqueue("sfc_reset");
3360 if (!reset_workqueue) {
3361 rc = -ENOMEM;
3362 goto err_reset;
3363 }
8ceee660
BH		 3364
3365 rc = pci_register_driver(&efx_pci_driver);
3366 if (rc < 0)
3367 goto err_pci;
3368
3369 return 0;
3370
3371 err_pci:
1ab00629
SH		 3372 destroy_workqueue(reset_workqueue);
3373 err_reset:
7fa8d547 3374#ifdef CONFIG_SFC_SRIOV
cd2d5b52
BH		 3375 efx_fini_sriov();
3376 err_sriov:
7fa8d547 3377#endif
8ceee660
BH		 3378 unregister_netdevice_notifier(&efx_netdev_notifier);
3379 err_notifier:
3380 return rc;
3381}
3382
3383static void __exit efx_exit_module(void)
3384{
3385 printk(KERN_INFO "Solarflare NET driver unloading\n");
3386
3387 pci_unregister_driver(&efx_pci_driver);
1ab00629 3388 destroy_workqueue(reset_workqueue);
7fa8d547 3389#ifdef CONFIG_SFC_SRIOV
cd2d5b52 3390 efx_fini_sriov();
7fa8d547 3391#endif
8ceee660
BH		 3392 unregister_netdevice_notifier(&efx_netdev_notifier);
3393
3394}
3395
3396module_init(efx_init_module);
3397module_exit(efx_exit_module);
3398
906bb26c
BH		 3399MODULE_AUTHOR("Solarflare Communications and "
3400 "Michael Brown <mbrown@fensystems.co.uk>");
6a350fdb 3401MODULE_DESCRIPTION("Solarflare network driver");
8ceee660
BH		 3402MODULE_LICENSE("GPL");
3403MODULE_DEVICE_TABLE(pci, efx_pci_table);
Linux kernel - Deliverables
This page took 1.04118 seconds and 5 git commands to generate.