projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
sfc: Disable PTP on EF10 until we're ready to handle inline RX timestamps
[deliverable/linux.git] / drivers / net / ethernet / sfc / ef10.c
CommitLineData
8127d661
BH		 1/****************************************************************************
2 * Driver for Solarflare network controllers and boards
3 * Copyright 2012-2013 Solarflare Communications Inc.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation, incorporated herein by reference.
8 */
9
10#include "net_driver.h"
11#include "ef10_regs.h"
12#include "io.h"
13#include "mcdi.h"
14#include "mcdi_pcol.h"
15#include "nic.h"
16#include "workarounds.h"
17#include <linux/in.h>
18#include <linux/jhash.h>
19#include <linux/wait.h>
20#include <linux/workqueue.h>
21
22/* Hardware control for EF10 architecture including 'Huntington'. */
23
24#define EFX_EF10_DRVGEN_EV 7
25enum {
26 EFX_EF10_TEST = 1,
27 EFX_EF10_REFILL,
28};
29
30/* The reserved RSS context value */
31#define EFX_EF10_RSS_CONTEXT_INVALID 0xffffffff
32
33/* The filter table(s) are managed by firmware and we have write-only
34 * access. When removing filters we must identify them to the
35 * firmware by a 64-bit handle, but this is too wide for Linux kernel
36 * interfaces (32-bit for RX NFC, 16-bit for RFS). Also, we need to
37 * be able to tell in advance whether a requested insertion will
38 * replace an existing filter. Therefore we maintain a software hash
39 * table, which should be at least as large as the hardware hash
40 * table.
41 *
42 * Huntington has a single 8K filter table shared between all filter
43 * types and both ports.
44 */
45#define HUNT_FILTER_TBL_ROWS 8192
46
47struct efx_ef10_filter_table {
48/* The RX match field masks supported by this fw & hw, in order of priority */
49 enum efx_filter_match_flags rx_match_flags[
50 MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_MAXNUM];
51 unsigned int rx_match_count;
52
53 struct {
54 unsigned long spec; /* pointer to spec plus flag bits */
55/* BUSY flag indicates that an update is in progress. STACK_OLD is
56 * used to mark and sweep stack-owned MAC filters.
57 */
58#define EFX_EF10_FILTER_FLAG_BUSY 1UL
59#define EFX_EF10_FILTER_FLAG_STACK_OLD 2UL
60#define EFX_EF10_FILTER_FLAGS 3UL
61 u64 handle; /* firmware handle */
62 } *entry;
63 wait_queue_head_t waitq;
64/* Shadow of net_device address lists, guarded by mac_lock */
65#define EFX_EF10_FILTER_STACK_UC_MAX 32
66#define EFX_EF10_FILTER_STACK_MC_MAX 256
67 struct {
68 u8 addr[ETH_ALEN];
69 u16 id;
70 } stack_uc_list[EFX_EF10_FILTER_STACK_UC_MAX],
71 stack_mc_list[EFX_EF10_FILTER_STACK_MC_MAX];
72 int stack_uc_count; /* negative for PROMISC */
73 int stack_mc_count; /* negative for PROMISC/ALLMULTI */
74};
75
76/* An arbitrary search limit for the software hash table */
77#define EFX_EF10_FILTER_SEARCH_LIMIT 200
78
79static void efx_ef10_rx_push_indir_table(struct efx_nic *efx);
80static void efx_ef10_rx_free_indir_table(struct efx_nic *efx);
81static void efx_ef10_filter_table_remove(struct efx_nic *efx);
82
83static int efx_ef10_get_warm_boot_count(struct efx_nic *efx)
84{
85 efx_dword_t reg;
86
87 efx_readd(efx, &reg, ER_DZ_BIU_MC_SFT_STATUS);
88 return EFX_DWORD_FIELD(reg, EFX_WORD_1) == 0xb007 ?
89 EFX_DWORD_FIELD(reg, EFX_WORD_0) : -EIO;
90}
91
92static unsigned int efx_ef10_mem_map_size(struct efx_nic *efx)
93{
94 return resource_size(&efx->pci_dev->resource[EFX_MEM_BAR]);
95}
96
97static int efx_ef10_init_capabilities(struct efx_nic *efx)
98{
99 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CAPABILITIES_OUT_LEN);
100 struct efx_ef10_nic_data *nic_data = efx->nic_data;
101 size_t outlen;
102 int rc;
103
104 BUILD_BUG_ON(MC_CMD_GET_CAPABILITIES_IN_LEN != 0);
105
106 rc = efx_mcdi_rpc(efx, MC_CMD_GET_CAPABILITIES, NULL, 0,
107 outbuf, sizeof(outbuf), &outlen);
108 if (rc)
109 return rc;
110
111 if (outlen >= sizeof(outbuf)) {
112 nic_data->datapath_caps =
113 MCDI_DWORD(outbuf, GET_CAPABILITIES_OUT_FLAGS1);
114 if (!(nic_data->datapath_caps &
115 (1 << MC_CMD_GET_CAPABILITIES_OUT_TX_TSO_LBN))) {
116 netif_err(efx, drv, efx->net_dev,
117 "Capabilities don't indicate TSO support.\n");
118 return -ENODEV;
119 }
120 }
121
122 return 0;
123}
124
125static int efx_ef10_get_sysclk_freq(struct efx_nic *efx)
126{
127 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CLOCK_OUT_LEN);
128 int rc;
129
130 rc = efx_mcdi_rpc(efx, MC_CMD_GET_CLOCK, NULL, 0,
131 outbuf, sizeof(outbuf), NULL);
132 if (rc)
133 return rc;
134 rc = MCDI_DWORD(outbuf, GET_CLOCK_OUT_SYS_FREQ);
135 return rc > 0 ? rc : -ERANGE;
136}
137
138static int efx_ef10_get_mac_address(struct efx_nic *efx, u8 *mac_address)
139{
140 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_MAC_ADDRESSES_OUT_LEN);
141 size_t outlen;
142 int rc;
143
144 BUILD_BUG_ON(MC_CMD_GET_MAC_ADDRESSES_IN_LEN != 0);
145
146 rc = efx_mcdi_rpc(efx, MC_CMD_GET_MAC_ADDRESSES, NULL, 0,
147 outbuf, sizeof(outbuf), &outlen);
148 if (rc)
149 return rc;
150 if (outlen < MC_CMD_GET_MAC_ADDRESSES_OUT_LEN)
151 return -EIO;
152
153 memcpy(mac_address,
154 MCDI_PTR(outbuf, GET_MAC_ADDRESSES_OUT_MAC_ADDR_BASE), ETH_ALEN);
155 return 0;
156}
157
158static int efx_ef10_probe(struct efx_nic *efx)
159{
160 struct efx_ef10_nic_data *nic_data;
161 int i, rc;
162
163 /* We can have one VI for each 8K region. However we need
164 * multiple TX queues per channel.
165 */
166 efx->max_channels =
167 min_t(unsigned int,
168 EFX_MAX_CHANNELS,
169 resource_size(&efx->pci_dev->resource[EFX_MEM_BAR]) /
170 (EFX_VI_PAGE_SIZE * EFX_TXQ_TYPES));
171 BUG_ON(efx->max_channels == 0);
172
173 nic_data = kzalloc(sizeof(*nic_data), GFP_KERNEL);
174 if (!nic_data)
175 return -ENOMEM;
176 efx->nic_data = nic_data;
177
178 rc = efx_nic_alloc_buffer(efx, &nic_data->mcdi_buf,
179 8 + MCDI_CTL_SDU_LEN_MAX_V2, GFP_KERNEL);
180 if (rc)
181 goto fail1;
182
183 /* Get the MC's warm boot count. In case it's rebooting right
184 * now, be prepared to retry.
185 */
186 i = 0;
187 for (;;) {
188 rc = efx_ef10_get_warm_boot_count(efx);
189 if (rc >= 0)
190 break;
191 if (++i == 5)
192 goto fail2;
193 ssleep(1);
194 }
195 nic_data->warm_boot_count = rc;
196
197 nic_data->rx_rss_context = EFX_EF10_RSS_CONTEXT_INVALID;
198
199 /* In case we're recovering from a crash (kexec), we want to
200 * cancel any outstanding request by the previous user of this
201 * function. We send a special message using the least
202 * significant bits of the 'high' (doorbell) register.
203 */
204 _efx_writed(efx, cpu_to_le32(1), ER_DZ_MC_DB_HWRD);
205
206 rc = efx_mcdi_init(efx);
207 if (rc)
208 goto fail2;
209
210 /* Reset (most) configuration for this function */
211 rc = efx_mcdi_reset(efx, RESET_TYPE_ALL);
212 if (rc)
213 goto fail3;
214
215 /* Enable event logging */
216 rc = efx_mcdi_log_ctrl(efx, true, false, 0);
217 if (rc)
218 goto fail3;
219
220 rc = efx_ef10_init_capabilities(efx);
221 if (rc < 0)
222 goto fail3;
223
224 efx->rx_packet_len_offset =
225 ES_DZ_RX_PREFIX_PKTLEN_OFST - ES_DZ_RX_PREFIX_SIZE;
226
227 if (!(nic_data->datapath_caps &
228 (1 << MC_CMD_GET_CAPABILITIES_OUT_RX_PREFIX_LEN_14_LBN))) {
229 netif_err(efx, probe, efx->net_dev,
230 "current firmware does not support an RX prefix\n");
231 rc = -ENODEV;
232 goto fail3;
233 }
234
235 rc = efx_mcdi_port_get_number(efx);
236 if (rc < 0)
237 goto fail3;
238 efx->port_num = rc;
239
240 rc = efx_ef10_get_mac_address(efx, efx->net_dev->perm_addr);
241 if (rc)
242 goto fail3;
243
244 rc = efx_ef10_get_sysclk_freq(efx);
245 if (rc < 0)
246 goto fail3;
247 efx->timer_quantum_ns = 1536000 / rc; /* 1536 cycles */
248
249 /* Check whether firmware supports bug 35388 workaround */
250 rc = efx_mcdi_set_workaround(efx, MC_CMD_WORKAROUND_BUG35388, true);
251 if (rc == 0)
252 nic_data->workaround_35388 = true;
253 else if (rc != -ENOSYS && rc != -ENOENT)
254 goto fail3;
255 netif_dbg(efx, probe, efx->net_dev,
256 "workaround for bug 35388 is %sabled\n",
257 nic_data->workaround_35388 ? "en" : "dis");
258
259 rc = efx_mcdi_mon_probe(efx);
260 if (rc)
261 goto fail3;
262
8127d661
BH		 263 return 0;
264
265fail3:
266 efx_mcdi_fini(efx);
267fail2:
268 efx_nic_free_buffer(efx, &nic_data->mcdi_buf);
269fail1:
270 kfree(nic_data);
271 efx->nic_data = NULL;
272 return rc;
273}
274
275static int efx_ef10_free_vis(struct efx_nic *efx)
276{
277 int rc = efx_mcdi_rpc(efx, MC_CMD_FREE_VIS, NULL, 0, NULL, 0, NULL);
278
279 /* -EALREADY means nothing to free, so ignore */
280 if (rc == -EALREADY)
281 rc = 0;
282 return rc;
283}
284
285static void efx_ef10_remove(struct efx_nic *efx)
286{
287 struct efx_ef10_nic_data *nic_data = efx->nic_data;
288 int rc;
289
290 efx_mcdi_mon_remove(efx);
291
292 /* This needs to be after efx_ptp_remove_channel() with no filters */
293 efx_ef10_rx_free_indir_table(efx);
294
295 rc = efx_ef10_free_vis(efx);
296 WARN_ON(rc != 0);
297
298 efx_mcdi_fini(efx);
299 efx_nic_free_buffer(efx, &nic_data->mcdi_buf);
300 kfree(nic_data);
301}
302
303static int efx_ef10_alloc_vis(struct efx_nic *efx,
304 unsigned int min_vis, unsigned int max_vis)
305{
306 MCDI_DECLARE_BUF(inbuf, MC_CMD_ALLOC_VIS_IN_LEN);
307 MCDI_DECLARE_BUF(outbuf, MC_CMD_ALLOC_VIS_OUT_LEN);
308 struct efx_ef10_nic_data *nic_data = efx->nic_data;
309 size_t outlen;
310 int rc;
311
312 MCDI_SET_DWORD(inbuf, ALLOC_VIS_IN_MIN_VI_COUNT, min_vis);
313 MCDI_SET_DWORD(inbuf, ALLOC_VIS_IN_MAX_VI_COUNT, max_vis);
314 rc = efx_mcdi_rpc(efx, MC_CMD_ALLOC_VIS, inbuf, sizeof(inbuf),
315 outbuf, sizeof(outbuf), &outlen);
316 if (rc != 0)
317 return rc;
318
319 if (outlen < MC_CMD_ALLOC_VIS_OUT_LEN)
320 return -EIO;
321
322 netif_dbg(efx, drv, efx->net_dev, "base VI is A0x%03x\n",
323 MCDI_DWORD(outbuf, ALLOC_VIS_OUT_VI_BASE));
324
325 nic_data->vi_base = MCDI_DWORD(outbuf, ALLOC_VIS_OUT_VI_BASE);
326 nic_data->n_allocated_vis = MCDI_DWORD(outbuf, ALLOC_VIS_OUT_VI_COUNT);
327 return 0;
328}
329
330static int efx_ef10_dimension_resources(struct efx_nic *efx)
331{
332 unsigned int n_vis =
333 max(efx->n_channels, efx->n_tx_channels * EFX_TXQ_TYPES);
334
335 return efx_ef10_alloc_vis(efx, n_vis, n_vis);
336}
337
338static int efx_ef10_init_nic(struct efx_nic *efx)
339{
340 struct efx_ef10_nic_data *nic_data = efx->nic_data;
341 int rc;
342
343 if (nic_data->must_realloc_vis) {
344 /* We cannot let the number of VIs change now */
345 rc = efx_ef10_alloc_vis(efx, nic_data->n_allocated_vis,
346 nic_data->n_allocated_vis);
347 if (rc)
348 return rc;
349 nic_data->must_realloc_vis = false;
350 }
351
352 efx_ef10_rx_push_indir_table(efx);
353 return 0;
354}
355
356static int efx_ef10_map_reset_flags(u32 *flags)
357{
358 enum {
359 EF10_RESET_PORT = ((ETH_RESET_MAC | ETH_RESET_PHY) <<
360 ETH_RESET_SHARED_SHIFT),
361 EF10_RESET_MC = ((ETH_RESET_DMA | ETH_RESET_FILTER |
362 ETH_RESET_OFFLOAD | ETH_RESET_MAC |
363 ETH_RESET_PHY | ETH_RESET_MGMT) <<
364 ETH_RESET_SHARED_SHIFT)
365 };
366
367 /* We assume for now that our PCI function is permitted to
368 * reset everything.
369 */
370
371 if ((*flags & EF10_RESET_MC) == EF10_RESET_MC) {
372 *flags &= ~EF10_RESET_MC;
373 return RESET_TYPE_WORLD;
374 }
375
376 if ((*flags & EF10_RESET_PORT) == EF10_RESET_PORT) {
377 *flags &= ~EF10_RESET_PORT;
378 return RESET_TYPE_ALL;
379 }
380
381 /* no invisible reset implemented */
382
383 return -EINVAL;
384}
385
386#define EF10_DMA_STAT(ext_name, mcdi_name) \
387 [EF10_STAT_ ## ext_name] = \
388 { #ext_name, 64, 8 * MC_CMD_MAC_ ## mcdi_name }
389#define EF10_DMA_INVIS_STAT(int_name, mcdi_name) \
390 [EF10_STAT_ ## int_name] = \
391 { NULL, 64, 8 * MC_CMD_MAC_ ## mcdi_name }
392#define EF10_OTHER_STAT(ext_name) \
393 [EF10_STAT_ ## ext_name] = { #ext_name, 0, 0 }
394
395static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = {
396 EF10_DMA_STAT(tx_bytes, TX_BYTES),
397 EF10_DMA_STAT(tx_packets, TX_PKTS),
398 EF10_DMA_STAT(tx_pause, TX_PAUSE_PKTS),
399 EF10_DMA_STAT(tx_control, TX_CONTROL_PKTS),
400 EF10_DMA_STAT(tx_unicast, TX_UNICAST_PKTS),
401 EF10_DMA_STAT(tx_multicast, TX_MULTICAST_PKTS),
402 EF10_DMA_STAT(tx_broadcast, TX_BROADCAST_PKTS),
403 EF10_DMA_STAT(tx_lt64, TX_LT64_PKTS),
404 EF10_DMA_STAT(tx_64, TX_64_PKTS),
405 EF10_DMA_STAT(tx_65_to_127, TX_65_TO_127_PKTS),
406 EF10_DMA_STAT(tx_128_to_255, TX_128_TO_255_PKTS),
407 EF10_DMA_STAT(tx_256_to_511, TX_256_TO_511_PKTS),
408 EF10_DMA_STAT(tx_512_to_1023, TX_512_TO_1023_PKTS),
409 EF10_DMA_STAT(tx_1024_to_15xx, TX_1024_TO_15XX_PKTS),
410 EF10_DMA_STAT(tx_15xx_to_jumbo, TX_15XX_TO_JUMBO_PKTS),
411 EF10_DMA_STAT(rx_bytes, RX_BYTES),
412 EF10_DMA_INVIS_STAT(rx_bytes_minus_good_bytes, RX_BAD_BYTES),
413 EF10_OTHER_STAT(rx_good_bytes),
414 EF10_OTHER_STAT(rx_bad_bytes),
415 EF10_DMA_STAT(rx_packets, RX_PKTS),
416 EF10_DMA_STAT(rx_good, RX_GOOD_PKTS),
417 EF10_DMA_STAT(rx_bad, RX_BAD_FCS_PKTS),
418 EF10_DMA_STAT(rx_pause, RX_PAUSE_PKTS),
419 EF10_DMA_STAT(rx_control, RX_CONTROL_PKTS),
420 EF10_DMA_STAT(rx_unicast, RX_UNICAST_PKTS),
421 EF10_DMA_STAT(rx_multicast, RX_MULTICAST_PKTS),
422 EF10_DMA_STAT(rx_broadcast, RX_BROADCAST_PKTS),
423 EF10_DMA_STAT(rx_lt64, RX_UNDERSIZE_PKTS),
424 EF10_DMA_STAT(rx_64, RX_64_PKTS),
425 EF10_DMA_STAT(rx_65_to_127, RX_65_TO_127_PKTS),
426 EF10_DMA_STAT(rx_128_to_255, RX_128_TO_255_PKTS),
427 EF10_DMA_STAT(rx_256_to_511, RX_256_TO_511_PKTS),
428 EF10_DMA_STAT(rx_512_to_1023, RX_512_TO_1023_PKTS),
429 EF10_DMA_STAT(rx_1024_to_15xx, RX_1024_TO_15XX_PKTS),
430 EF10_DMA_STAT(rx_15xx_to_jumbo, RX_15XX_TO_JUMBO_PKTS),
431 EF10_DMA_STAT(rx_gtjumbo, RX_GTJUMBO_PKTS),
432 EF10_DMA_STAT(rx_bad_gtjumbo, RX_JABBER_PKTS),
433 EF10_DMA_STAT(rx_overflow, RX_OVERFLOW_PKTS),
434 EF10_DMA_STAT(rx_align_error, RX_ALIGN_ERROR_PKTS),
435 EF10_DMA_STAT(rx_length_error, RX_LENGTH_ERROR_PKTS),
436 EF10_DMA_STAT(rx_nodesc_drops, RX_NODESC_DROPS),
437};
438
439#define HUNT_COMMON_STAT_MASK ((1ULL << EF10_STAT_tx_bytes) | \
440 (1ULL << EF10_STAT_tx_packets) | \
441 (1ULL << EF10_STAT_tx_pause) | \
442 (1ULL << EF10_STAT_tx_unicast) | \
443 (1ULL << EF10_STAT_tx_multicast) | \
444 (1ULL << EF10_STAT_tx_broadcast) | \
445 (1ULL << EF10_STAT_rx_bytes) | \
446 (1ULL << EF10_STAT_rx_bytes_minus_good_bytes) | \
447 (1ULL << EF10_STAT_rx_good_bytes) | \
448 (1ULL << EF10_STAT_rx_bad_bytes) | \
449 (1ULL << EF10_STAT_rx_packets) | \
450 (1ULL << EF10_STAT_rx_good) | \
451 (1ULL << EF10_STAT_rx_bad) | \
452 (1ULL << EF10_STAT_rx_pause) | \
453 (1ULL << EF10_STAT_rx_control) | \
454 (1ULL << EF10_STAT_rx_unicast) | \
455 (1ULL << EF10_STAT_rx_multicast) | \
456 (1ULL << EF10_STAT_rx_broadcast) | \
457 (1ULL << EF10_STAT_rx_lt64) | \
458 (1ULL << EF10_STAT_rx_64) | \
459 (1ULL << EF10_STAT_rx_65_to_127) | \
460 (1ULL << EF10_STAT_rx_128_to_255) | \
461 (1ULL << EF10_STAT_rx_256_to_511) | \
462 (1ULL << EF10_STAT_rx_512_to_1023) | \
463 (1ULL << EF10_STAT_rx_1024_to_15xx) | \
464 (1ULL << EF10_STAT_rx_15xx_to_jumbo) | \
465 (1ULL << EF10_STAT_rx_gtjumbo) | \
466 (1ULL << EF10_STAT_rx_bad_gtjumbo) | \
467 (1ULL << EF10_STAT_rx_overflow) | \
468 (1ULL << EF10_STAT_rx_nodesc_drops))
469
470/* These statistics are only provided by the 10G MAC. For a 10G/40G
471 * switchable port we do not expose these because they might not
472 * include all the packets they should.
473 */
474#define HUNT_10G_ONLY_STAT_MASK ((1ULL << EF10_STAT_tx_control) | \
475 (1ULL << EF10_STAT_tx_lt64) | \
476 (1ULL << EF10_STAT_tx_64) | \
477 (1ULL << EF10_STAT_tx_65_to_127) | \
478 (1ULL << EF10_STAT_tx_128_to_255) | \
479 (1ULL << EF10_STAT_tx_256_to_511) | \
480 (1ULL << EF10_STAT_tx_512_to_1023) | \
481 (1ULL << EF10_STAT_tx_1024_to_15xx) | \
482 (1ULL << EF10_STAT_tx_15xx_to_jumbo))
483
484/* These statistics are only provided by the 40G MAC. For a 10G/40G
485 * switchable port we do expose these because the errors will otherwise
486 * be silent.
487 */
488#define HUNT_40G_EXTRA_STAT_MASK ((1ULL << EF10_STAT_rx_align_error) | \
489 (1ULL << EF10_STAT_rx_length_error))
490
491#if BITS_PER_LONG == 64
492#define STAT_MASK_BITMAP(bits) (bits)
493#else
494#define STAT_MASK_BITMAP(bits) (bits) & 0xffffffff, (bits) >> 32
495#endif
496
497static const unsigned long *efx_ef10_stat_mask(struct efx_nic *efx)
498{
499 static const unsigned long hunt_40g_stat_mask[] = {
500 STAT_MASK_BITMAP(HUNT_COMMON_STAT_MASK |
501 HUNT_40G_EXTRA_STAT_MASK)
502 };
503 static const unsigned long hunt_10g_only_stat_mask[] = {
504 STAT_MASK_BITMAP(HUNT_COMMON_STAT_MASK |
505 HUNT_10G_ONLY_STAT_MASK)
506 };
507 u32 port_caps = efx_mcdi_phy_get_caps(efx);
508
509 if (port_caps & (1 << MC_CMD_PHY_CAP_40000FDX_LBN))
510 return hunt_40g_stat_mask;
511 else
512 return hunt_10g_only_stat_mask;
513}
514
515static size_t efx_ef10_describe_stats(struct efx_nic *efx, u8 *names)
516{
517 return efx_nic_describe_stats(efx_ef10_stat_desc, EF10_STAT_COUNT,
518 efx_ef10_stat_mask(efx), names);
519}
520
521static int efx_ef10_try_update_nic_stats(struct efx_nic *efx)
522{
523 struct efx_ef10_nic_data *nic_data = efx->nic_data;
524 const unsigned long *stats_mask = efx_ef10_stat_mask(efx);
525 __le64 generation_start, generation_end;
526 u64 *stats = nic_data->stats;
527 __le64 *dma_stats;
528
529 dma_stats = efx->stats_buffer.addr;
530 nic_data = efx->nic_data;
531
532 generation_end = dma_stats[MC_CMD_MAC_GENERATION_END];
533 if (generation_end == EFX_MC_STATS_GENERATION_INVALID)
534 return 0;
535 rmb();
536 efx_nic_update_stats(efx_ef10_stat_desc, EF10_STAT_COUNT, stats_mask,
537 stats, efx->stats_buffer.addr, false);
538 generation_start = dma_stats[MC_CMD_MAC_GENERATION_START];
539 if (generation_end != generation_start)
540 return -EAGAIN;
541
542 /* Update derived statistics */
543 stats[EF10_STAT_rx_good_bytes] =
544 stats[EF10_STAT_rx_bytes] -
545 stats[EF10_STAT_rx_bytes_minus_good_bytes];
546 efx_update_diff_stat(&stats[EF10_STAT_rx_bad_bytes],
547 stats[EF10_STAT_rx_bytes_minus_good_bytes]);
548
549 return 0;
550}
551
552
553static size_t efx_ef10_update_stats(struct efx_nic *efx, u64 *full_stats,
554 struct rtnl_link_stats64 *core_stats)
555{
556 const unsigned long *mask = efx_ef10_stat_mask(efx);
557 struct efx_ef10_nic_data *nic_data = efx->nic_data;
558 u64 *stats = nic_data->stats;
559 size_t stats_count = 0, index;
560 int retry;
561
562 /* If we're unlucky enough to read statistics during the DMA, wait
563 * up to 10ms for it to finish (typically takes <500us)
564 */
565 for (retry = 0; retry < 100; ++retry) {
566 if (efx_ef10_try_update_nic_stats(efx) == 0)
567 break;
568 udelay(100);
569 }
570
571 if (full_stats) {
572 for_each_set_bit(index, mask, EF10_STAT_COUNT) {
573 if (efx_ef10_stat_desc[index].name) {
574 *full_stats++ = stats[index];
575 ++stats_count;
576 }
577 }
578 }
579
580 if (core_stats) {
581 core_stats->rx_packets = stats[EF10_STAT_rx_packets];
582 core_stats->tx_packets = stats[EF10_STAT_tx_packets];
583 core_stats->rx_bytes = stats[EF10_STAT_rx_bytes];
584 core_stats->tx_bytes = stats[EF10_STAT_tx_bytes];
585 core_stats->rx_dropped = stats[EF10_STAT_rx_nodesc_drops];
586 core_stats->multicast = stats[EF10_STAT_rx_multicast];
587 core_stats->rx_length_errors =
588 stats[EF10_STAT_rx_gtjumbo] +
589 stats[EF10_STAT_rx_length_error];
590 core_stats->rx_crc_errors = stats[EF10_STAT_rx_bad];
591 core_stats->rx_frame_errors = stats[EF10_STAT_rx_align_error];
592 core_stats->rx_fifo_errors = stats[EF10_STAT_rx_overflow];
593 core_stats->rx_errors = (core_stats->rx_length_errors +
594 core_stats->rx_crc_errors +
595 core_stats->rx_frame_errors);
596 }
597
598 return stats_count;
599}
600
601static void efx_ef10_push_irq_moderation(struct efx_channel *channel)
602{
603 struct efx_nic *efx = channel->efx;
604 unsigned int mode, value;
605 efx_dword_t timer_cmd;
606
607 if (channel->irq_moderation) {
608 mode = 3;
609 value = channel->irq_moderation - 1;
610 } else {
611 mode = 0;
612 value = 0;
613 }
614
615 if (EFX_EF10_WORKAROUND_35388(efx)) {
616 EFX_POPULATE_DWORD_3(timer_cmd, ERF_DD_EVQ_IND_TIMER_FLAGS,
617 EFE_DD_EVQ_IND_TIMER_FLAGS,
618 ERF_DD_EVQ_IND_TIMER_MODE, mode,
619 ERF_DD_EVQ_IND_TIMER_VAL, value);
620 efx_writed_page(efx, &timer_cmd, ER_DD_EVQ_INDIRECT,
621 channel->channel);
622 } else {
623 EFX_POPULATE_DWORD_2(timer_cmd, ERF_DZ_TC_TIMER_MODE, mode,
624 ERF_DZ_TC_TIMER_VAL, value);
625 efx_writed_page(efx, &timer_cmd, ER_DZ_EVQ_TMR,
626 channel->channel);
627 }
628}
629
630static void efx_ef10_get_wol(struct efx_nic *efx, struct ethtool_wolinfo *wol)
631{
632 wol->supported = 0;
633 wol->wolopts = 0;
634 memset(&wol->sopass, 0, sizeof(wol->sopass));
635}
636
637static int efx_ef10_set_wol(struct efx_nic *efx, u32 type)
638{
639 if (type != 0)
640 return -EINVAL;
641 return 0;
642}
643
644static void efx_ef10_mcdi_request(struct efx_nic *efx,
645 const efx_dword_t *hdr, size_t hdr_len,
646 const efx_dword_t *sdu, size_t sdu_len)
647{
648 struct efx_ef10_nic_data *nic_data = efx->nic_data;
649 u8 *pdu = nic_data->mcdi_buf.addr;
650
651 memcpy(pdu, hdr, hdr_len);
652 memcpy(pdu + hdr_len, sdu, sdu_len);
653 wmb();
654
655 /* The hardware provides 'low' and 'high' (doorbell) registers
656 * for passing the 64-bit address of an MCDI request to
657 * firmware. However the dwords are swapped by firmware. The
658 * least significant bits of the doorbell are then 0 for all
659 * MCDI requests due to alignment.
660 */
661 _efx_writed(efx, cpu_to_le32((u64)nic_data->mcdi_buf.dma_addr >> 32),
662 ER_DZ_MC_DB_LWRD);
663 _efx_writed(efx, cpu_to_le32((u32)nic_data->mcdi_buf.dma_addr),
664 ER_DZ_MC_DB_HWRD);
665}
666
667static bool efx_ef10_mcdi_poll_response(struct efx_nic *efx)
668{
669 struct efx_ef10_nic_data *nic_data = efx->nic_data;
670 const efx_dword_t hdr = *(const efx_dword_t *)nic_data->mcdi_buf.addr;
671
672 rmb();
673 return EFX_DWORD_FIELD(hdr, MCDI_HEADER_RESPONSE);
674}
675
676static void
677efx_ef10_mcdi_read_response(struct efx_nic *efx, efx_dword_t *outbuf,
678 size_t offset, size_t outlen)
679{
680 struct efx_ef10_nic_data *nic_data = efx->nic_data;
681 const u8 *pdu = nic_data->mcdi_buf.addr;
682
683 memcpy(outbuf, pdu + offset, outlen);
684}
685
686static int efx_ef10_mcdi_poll_reboot(struct efx_nic *efx)
687{
688 struct efx_ef10_nic_data *nic_data = efx->nic_data;
689 int rc;
690
691 rc = efx_ef10_get_warm_boot_count(efx);
692 if (rc < 0) {
693 /* The firmware is presumably in the process of
694 * rebooting. However, we are supposed to report each
695 * reboot just once, so we must only do that once we
696 * can read and store the updated warm boot count.
697 */
698 return 0;
699 }
700
701 if (rc == nic_data->warm_boot_count)
702 return 0;
703
704 nic_data->warm_boot_count = rc;
705
706 /* All our allocations have been reset */
707 nic_data->must_realloc_vis = true;
708 nic_data->must_restore_filters = true;
709 nic_data->rx_rss_context = EFX_EF10_RSS_CONTEXT_INVALID;
710
711 return -EIO;
712}
713
714/* Handle an MSI interrupt
715 *
716 * Handle an MSI hardware interrupt. This routine schedules event
717 * queue processing. No interrupt acknowledgement cycle is necessary.
718 * Also, we never need to check that the interrupt is for us, since
719 * MSI interrupts cannot be shared.
720 */
721static irqreturn_t efx_ef10_msi_interrupt(int irq, void *dev_id)
722{
723 struct efx_msi_context *context = dev_id;
724 struct efx_nic *efx = context->efx;
725
726 netif_vdbg(efx, intr, efx->net_dev,
727 "IRQ %d on CPU %d\n", irq, raw_smp_processor_id());
728
729 if (likely(ACCESS_ONCE(efx->irq_soft_enabled))) {
730 /* Note test interrupts */
731 if (context->index == efx->irq_level)
732 efx->last_irq_cpu = raw_smp_processor_id();
733
734 /* Schedule processing of the channel */
735 efx_schedule_channel_irq(efx->channel[context->index]);
736 }
737
738 return IRQ_HANDLED;
739}
740
741static irqreturn_t efx_ef10_legacy_interrupt(int irq, void *dev_id)
742{
743 struct efx_nic *efx = dev_id;
744 bool soft_enabled = ACCESS_ONCE(efx->irq_soft_enabled);
745 struct efx_channel *channel;
746 efx_dword_t reg;
747 u32 queues;
748
749 /* Read the ISR which also ACKs the interrupts */
750 efx_readd(efx, &reg, ER_DZ_BIU_INT_ISR);
751 queues = EFX_DWORD_FIELD(reg, ERF_DZ_ISR_REG);
752
753 if (queues == 0)
754 return IRQ_NONE;
755
756 if (likely(soft_enabled)) {
757 /* Note test interrupts */
758 if (queues & (1U << efx->irq_level))
759 efx->last_irq_cpu = raw_smp_processor_id();
760
761 efx_for_each_channel(channel, efx) {
762 if (queues & 1)
763 efx_schedule_channel_irq(channel);
764 queues >>= 1;
765 }
766 }
767
768 netif_vdbg(efx, intr, efx->net_dev,
769 "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n",
770 irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg));
771
772 return IRQ_HANDLED;
773}
774
775static void efx_ef10_irq_test_generate(struct efx_nic *efx)
776{
777 MCDI_DECLARE_BUF(inbuf, MC_CMD_TRIGGER_INTERRUPT_IN_LEN);
778
779 BUILD_BUG_ON(MC_CMD_TRIGGER_INTERRUPT_OUT_LEN != 0);
780
781 MCDI_SET_DWORD(inbuf, TRIGGER_INTERRUPT_IN_INTR_LEVEL, efx->irq_level);
782 (void) efx_mcdi_rpc(efx, MC_CMD_TRIGGER_INTERRUPT,
783 inbuf, sizeof(inbuf), NULL, 0, NULL);
784}
785
786static int efx_ef10_tx_probe(struct efx_tx_queue *tx_queue)
787{
788 return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd.buf,
789 (tx_queue->ptr_mask + 1) *
790 sizeof(efx_qword_t),
791 GFP_KERNEL);
792}
793
794/* This writes to the TX_DESC_WPTR and also pushes data */
795static inline void efx_ef10_push_tx_desc(struct efx_tx_queue *tx_queue,
796 const efx_qword_t *txd)
797{
798 unsigned int write_ptr;
799 efx_oword_t reg;
800
801 write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
802 EFX_POPULATE_OWORD_1(reg, ERF_DZ_TX_DESC_WPTR, write_ptr);
803 reg.qword[0] = *txd;
804 efx_writeo_page(tx_queue->efx, &reg,
805 ER_DZ_TX_DESC_UPD, tx_queue->queue);
806}
807
808static void efx_ef10_tx_init(struct efx_tx_queue *tx_queue)
809{
810 MCDI_DECLARE_BUF(inbuf, MC_CMD_INIT_TXQ_IN_LEN(EFX_MAX_DMAQ_SIZE * 8 /
811 EFX_BUF_SIZE));
812 MCDI_DECLARE_BUF(outbuf, MC_CMD_INIT_TXQ_OUT_LEN);
813 bool csum_offload = tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD;
814 size_t entries = tx_queue->txd.buf.len / EFX_BUF_SIZE;
815 struct efx_channel *channel = tx_queue->channel;
816 struct efx_nic *efx = tx_queue->efx;
817 size_t inlen, outlen;
818 dma_addr_t dma_addr;
819 efx_qword_t *txd;
820 int rc;
821 int i;
822
823 MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_SIZE, tx_queue->ptr_mask + 1);
824 MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_TARGET_EVQ, channel->channel);
825 MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_LABEL, tx_queue->queue);
826 MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_INSTANCE, tx_queue->queue);
827 MCDI_POPULATE_DWORD_2(inbuf, INIT_TXQ_IN_FLAGS,
828 INIT_TXQ_IN_FLAG_IP_CSUM_DIS, !csum_offload,
829 INIT_TXQ_IN_FLAG_TCP_CSUM_DIS, !csum_offload);
830 MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_OWNER_ID, 0);
831 MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
832
833 dma_addr = tx_queue->txd.buf.dma_addr;
834
835 netif_dbg(efx, hw, efx->net_dev, "pushing TXQ %d. %zu entries (%llx)\n",
836 tx_queue->queue, entries, (u64)dma_addr);
837
838 for (i = 0; i < entries; ++i) {
839 MCDI_SET_ARRAY_QWORD(inbuf, INIT_TXQ_IN_DMA_ADDR, i, dma_addr);
840 dma_addr += EFX_BUF_SIZE;
841 }
842
843 inlen = MC_CMD_INIT_TXQ_IN_LEN(entries);
844
845 rc = efx_mcdi_rpc(efx, MC_CMD_INIT_TXQ, inbuf, inlen,
846 outbuf, sizeof(outbuf), &outlen);
847 if (rc)
848 goto fail;
849
850 /* A previous user of this TX queue might have set us up the
851 * bomb by writing a descriptor to the TX push collector but
852 * not the doorbell. (Each collector belongs to a port, not a
853 * queue or function, so cannot easily be reset.) We must
854 * attempt to push a no-op descriptor in its place.
855 */
856 tx_queue->buffer[0].flags = EFX_TX_BUF_OPTION;
857 tx_queue->insert_count = 1;
858 txd = efx_tx_desc(tx_queue, 0);
859 EFX_POPULATE_QWORD_4(*txd,
860 ESF_DZ_TX_DESC_IS_OPT, true,
861 ESF_DZ_TX_OPTION_TYPE,
862 ESE_DZ_TX_OPTION_DESC_CRC_CSUM,
863 ESF_DZ_TX_OPTION_UDP_TCP_CSUM, csum_offload,
864 ESF_DZ_TX_OPTION_IP_CSUM, csum_offload);
865 tx_queue->write_count = 1;
866 wmb();
867 efx_ef10_push_tx_desc(tx_queue, txd);
868
869 return;
870
871fail:
872 WARN_ON(true);
873 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
874}
875
876static void efx_ef10_tx_fini(struct efx_tx_queue *tx_queue)
877{
878 MCDI_DECLARE_BUF(inbuf, MC_CMD_FINI_TXQ_IN_LEN);
879 MCDI_DECLARE_BUF(outbuf, MC_CMD_FINI_TXQ_OUT_LEN);
880 struct efx_nic *efx = tx_queue->efx;
881 size_t outlen;
882 int rc;
883
884 MCDI_SET_DWORD(inbuf, FINI_TXQ_IN_INSTANCE,
885 tx_queue->queue);
886
887 rc = efx_mcdi_rpc(efx, MC_CMD_FINI_TXQ, inbuf, sizeof(inbuf),
888 outbuf, sizeof(outbuf), &outlen);
889
890 if (rc && rc != -EALREADY)
891 goto fail;
892
893 return;
894
895fail:
896 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
897}
898
899static void efx_ef10_tx_remove(struct efx_tx_queue *tx_queue)
900{
901 efx_nic_free_buffer(tx_queue->efx, &tx_queue->txd.buf);
902}
903
904/* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */
905static inline void efx_ef10_notify_tx_desc(struct efx_tx_queue *tx_queue)
906{
907 unsigned int write_ptr;
908 efx_dword_t reg;
909
910 write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
911 EFX_POPULATE_DWORD_1(reg, ERF_DZ_TX_DESC_WPTR_DWORD, write_ptr);
912 efx_writed_page(tx_queue->efx, &reg,
913 ER_DZ_TX_DESC_UPD_DWORD, tx_queue->queue);
914}
915
916static void efx_ef10_tx_write(struct efx_tx_queue *tx_queue)
917{
918 unsigned int old_write_count = tx_queue->write_count;
919 struct efx_tx_buffer *buffer;
920 unsigned int write_ptr;
921 efx_qword_t *txd;
922
923 BUG_ON(tx_queue->write_count == tx_queue->insert_count);
924
925 do {
926 write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
927 buffer = &tx_queue->buffer[write_ptr];
928 txd = efx_tx_desc(tx_queue, write_ptr);
929 ++tx_queue->write_count;
930
931 /* Create TX descriptor ring entry */
932 if (buffer->flags & EFX_TX_BUF_OPTION) {
933 *txd = buffer->option;
934 } else {
935 BUILD_BUG_ON(EFX_TX_BUF_CONT != 1);
936 EFX_POPULATE_QWORD_3(
937 *txd,
938 ESF_DZ_TX_KER_CONT,
939 buffer->flags & EFX_TX_BUF_CONT,
940 ESF_DZ_TX_KER_BYTE_CNT, buffer->len,
941 ESF_DZ_TX_KER_BUF_ADDR, buffer->dma_addr);
942 }
943 } while (tx_queue->write_count != tx_queue->insert_count);
944
945 wmb(); /* Ensure descriptors are written before they are fetched */
946
947 if (efx_nic_may_push_tx_desc(tx_queue, old_write_count)) {
948 txd = efx_tx_desc(tx_queue,
949 old_write_count & tx_queue->ptr_mask);
950 efx_ef10_push_tx_desc(tx_queue, txd);
951 ++tx_queue->pushes;
952 } else {
953 efx_ef10_notify_tx_desc(tx_queue);
954 }
955}
956
957static int efx_ef10_alloc_rss_context(struct efx_nic *efx, u32 *context)
958{
959 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN);
960 MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
961 size_t outlen;
962 int rc;
963
964 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
965 EVB_PORT_ID_ASSIGNED);
966 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_TYPE,
967 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE);
968 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES,
969 EFX_MAX_CHANNELS);
970
971 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_ALLOC, inbuf, sizeof(inbuf),
972 outbuf, sizeof(outbuf), &outlen);
973 if (rc != 0)
974 return rc;
975
976 if (outlen < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)
977 return -EIO;
978
979 *context = MCDI_DWORD(outbuf, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
980
981 return 0;
982}
983
984static void efx_ef10_free_rss_context(struct efx_nic *efx, u32 context)
985{
986 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_FREE_IN_LEN);
987 int rc;
988
989 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID,
990 context);
991
992 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_FREE, inbuf, sizeof(inbuf),
993 NULL, 0, NULL);
994 WARN_ON(rc != 0);
995}
996
997static int efx_ef10_populate_rss_table(struct efx_nic *efx, u32 context)
998{
999 MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN);
1000 MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN);
1001 int i, rc;
1002
1003 MCDI_SET_DWORD(tablebuf, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
1004 context);
1005 BUILD_BUG_ON(ARRAY_SIZE(efx->rx_indir_table) !=
1006 MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN);
1007
1008 for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); ++i)
1009 MCDI_PTR(tablebuf,
1010 RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE)[i] =
1011 (u8) efx->rx_indir_table[i];
1012
1013 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_TABLE, tablebuf,
1014 sizeof(tablebuf), NULL, 0, NULL);
1015 if (rc != 0)
1016 return rc;
1017
1018 MCDI_SET_DWORD(keybuf, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
1019 context);
1020 BUILD_BUG_ON(ARRAY_SIZE(efx->rx_hash_key) !=
1021 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
1022 for (i = 0; i < ARRAY_SIZE(efx->rx_hash_key); ++i)
1023 MCDI_PTR(keybuf, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY)[i] =
1024 efx->rx_hash_key[i];
1025
1026 return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_KEY, keybuf,
1027 sizeof(keybuf), NULL, 0, NULL);
1028}
1029
1030static void efx_ef10_rx_free_indir_table(struct efx_nic *efx)
1031{
1032 struct efx_ef10_nic_data *nic_data = efx->nic_data;
1033
1034 if (nic_data->rx_rss_context != EFX_EF10_RSS_CONTEXT_INVALID)
1035 efx_ef10_free_rss_context(efx, nic_data->rx_rss_context);
1036 nic_data->rx_rss_context = EFX_EF10_RSS_CONTEXT_INVALID;
1037}
1038
1039static void efx_ef10_rx_push_indir_table(struct efx_nic *efx)
1040{
1041 struct efx_ef10_nic_data *nic_data = efx->nic_data;
1042 int rc;
1043
1044 netif_dbg(efx, drv, efx->net_dev, "pushing RX indirection table\n");
1045
1046 if (nic_data->rx_rss_context == EFX_EF10_RSS_CONTEXT_INVALID) {
1047 rc = efx_ef10_alloc_rss_context(efx, &nic_data->rx_rss_context);
1048 if (rc != 0)
1049 goto fail;
1050 }
1051
1052 rc = efx_ef10_populate_rss_table(efx, nic_data->rx_rss_context);
1053 if (rc != 0)
1054 goto fail;
1055
1056 return;
1057
1058fail:
1059 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
1060}
1061
1062static int efx_ef10_rx_probe(struct efx_rx_queue *rx_queue)
1063{
1064 return efx_nic_alloc_buffer(rx_queue->efx, &rx_queue->rxd.buf,
1065 (rx_queue->ptr_mask + 1) *
1066 sizeof(efx_qword_t),
1067 GFP_KERNEL);
1068}
1069
1070static void efx_ef10_rx_init(struct efx_rx_queue *rx_queue)
1071{
1072 MCDI_DECLARE_BUF(inbuf,
1073 MC_CMD_INIT_RXQ_IN_LEN(EFX_MAX_DMAQ_SIZE * 8 /
1074 EFX_BUF_SIZE));
1075 MCDI_DECLARE_BUF(outbuf, MC_CMD_INIT_RXQ_OUT_LEN);
1076 struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
1077 size_t entries = rx_queue->rxd.buf.len / EFX_BUF_SIZE;
1078 struct efx_nic *efx = rx_queue->efx;
1079 size_t inlen, outlen;
1080 dma_addr_t dma_addr;
1081 int rc;
1082 int i;
1083
1084 rx_queue->scatter_n = 0;
1085 rx_queue->scatter_len = 0;
1086
1087 MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_SIZE, rx_queue->ptr_mask + 1);
1088 MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_TARGET_EVQ, channel->channel);
1089 MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_LABEL, efx_rx_queue_index(rx_queue));
1090 MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_INSTANCE,
1091 efx_rx_queue_index(rx_queue));
1092 MCDI_POPULATE_DWORD_1(inbuf, INIT_RXQ_IN_FLAGS,
1093 INIT_RXQ_IN_FLAG_PREFIX, 1);
1094 MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_OWNER_ID, 0);
1095 MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
1096
1097 dma_addr = rx_queue->rxd.buf.dma_addr;
1098
1099 netif_dbg(efx, hw, efx->net_dev, "pushing RXQ %d. %zu entries (%llx)\n",
1100 efx_rx_queue_index(rx_queue), entries, (u64)dma_addr);
1101
1102 for (i = 0; i < entries; ++i) {
1103 MCDI_SET_ARRAY_QWORD(inbuf, INIT_RXQ_IN_DMA_ADDR, i, dma_addr);
1104 dma_addr += EFX_BUF_SIZE;
1105 }
1106
1107 inlen = MC_CMD_INIT_RXQ_IN_LEN(entries);
1108
1109 rc = efx_mcdi_rpc(efx, MC_CMD_INIT_RXQ, inbuf, inlen,
1110 outbuf, sizeof(outbuf), &outlen);
1111 if (rc)
1112 goto fail;
1113
1114 return;
1115
1116fail:
1117 WARN_ON(true);
1118 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
1119}
1120
1121static void efx_ef10_rx_fini(struct efx_rx_queue *rx_queue)
1122{
1123 MCDI_DECLARE_BUF(inbuf, MC_CMD_FINI_RXQ_IN_LEN);
1124 MCDI_DECLARE_BUF(outbuf, MC_CMD_FINI_RXQ_OUT_LEN);
1125 struct efx_nic *efx = rx_queue->efx;
1126 size_t outlen;
1127 int rc;
1128
1129 MCDI_SET_DWORD(inbuf, FINI_RXQ_IN_INSTANCE,
1130 efx_rx_queue_index(rx_queue));
1131
1132 rc = efx_mcdi_rpc(efx, MC_CMD_FINI_RXQ, inbuf, sizeof(inbuf),
1133 outbuf, sizeof(outbuf), &outlen);
1134
1135 if (rc && rc != -EALREADY)
1136 goto fail;
1137
1138 return;
1139
1140fail:
1141 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
1142}
1143
1144static void efx_ef10_rx_remove(struct efx_rx_queue *rx_queue)
1145{
1146 efx_nic_free_buffer(rx_queue->efx, &rx_queue->rxd.buf);
1147}
1148
1149/* This creates an entry in the RX descriptor queue */
1150static inline void
1151efx_ef10_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index)
1152{
1153 struct efx_rx_buffer *rx_buf;
1154 efx_qword_t *rxd;
1155
1156 rxd = efx_rx_desc(rx_queue, index);
1157 rx_buf = efx_rx_buffer(rx_queue, index);
1158 EFX_POPULATE_QWORD_2(*rxd,
1159 ESF_DZ_RX_KER_BYTE_CNT, rx_buf->len,
1160 ESF_DZ_RX_KER_BUF_ADDR, rx_buf->dma_addr);
1161}
1162
1163static void efx_ef10_rx_write(struct efx_rx_queue *rx_queue)
1164{
1165 struct efx_nic *efx = rx_queue->efx;
1166 unsigned int write_count;
1167 efx_dword_t reg;
1168
1169 /* Firmware requires that RX_DESC_WPTR be a multiple of 8 */
1170 write_count = rx_queue->added_count & ~7;
1171 if (rx_queue->notified_count == write_count)
1172 return;
1173
1174 do
1175 efx_ef10_build_rx_desc(
1176 rx_queue,
1177 rx_queue->notified_count & rx_queue->ptr_mask);
1178 while (++rx_queue->notified_count != write_count);
1179
1180 wmb();
1181 EFX_POPULATE_DWORD_1(reg, ERF_DZ_RX_DESC_WPTR,
1182 write_count & rx_queue->ptr_mask);
1183 efx_writed_page(efx, &reg, ER_DZ_RX_DESC_UPD,
1184 efx_rx_queue_index(rx_queue));
1185}
1186
1187static efx_mcdi_async_completer efx_ef10_rx_defer_refill_complete;
1188
1189static void efx_ef10_rx_defer_refill(struct efx_rx_queue *rx_queue)
1190{
1191 struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
1192 MCDI_DECLARE_BUF(inbuf, MC_CMD_DRIVER_EVENT_IN_LEN);
1193 efx_qword_t event;
1194
1195 EFX_POPULATE_QWORD_2(event,
1196 ESF_DZ_EV_CODE, EFX_EF10_DRVGEN_EV,
1197 ESF_DZ_EV_DATA, EFX_EF10_REFILL);
1198
1199 MCDI_SET_DWORD(inbuf, DRIVER_EVENT_IN_EVQ, channel->channel);
1200
1201 /* MCDI_SET_QWORD is not appropriate here since EFX_POPULATE_* has
1202 * already swapped the data to little-endian order.
1203 */
1204 memcpy(MCDI_PTR(inbuf, DRIVER_EVENT_IN_DATA), &event.u64[0],
1205 sizeof(efx_qword_t));
1206
1207 efx_mcdi_rpc_async(channel->efx, MC_CMD_DRIVER_EVENT,
1208 inbuf, sizeof(inbuf), 0,
1209 efx_ef10_rx_defer_refill_complete, 0);
1210}
1211
1212static void
1213efx_ef10_rx_defer_refill_complete(struct efx_nic *efx, unsigned long cookie,
1214 int rc, efx_dword_t *outbuf,
1215 size_t outlen_actual)
1216{
1217 /* nothing to do */
1218}
1219
1220static int efx_ef10_ev_probe(struct efx_channel *channel)
1221{
1222 return efx_nic_alloc_buffer(channel->efx, &channel->eventq.buf,
1223 (channel->eventq_mask + 1) *
1224 sizeof(efx_qword_t),
1225 GFP_KERNEL);
1226}
1227
1228static int efx_ef10_ev_init(struct efx_channel *channel)
1229{
1230 MCDI_DECLARE_BUF(inbuf,
1231 MC_CMD_INIT_EVQ_IN_LEN(EFX_MAX_EVQ_SIZE * 8 /
1232 EFX_BUF_SIZE));
1233 MCDI_DECLARE_BUF(outbuf, MC_CMD_INIT_EVQ_OUT_LEN);
1234 size_t entries = channel->eventq.buf.len / EFX_BUF_SIZE;
1235 struct efx_nic *efx = channel->efx;
1236 struct efx_ef10_nic_data *nic_data;
1237 bool supports_rx_merge;
1238 size_t inlen, outlen;
1239 dma_addr_t dma_addr;
1240 int rc;
1241 int i;
1242
1243 nic_data = efx->nic_data;
1244 supports_rx_merge =
1245 !!(nic_data->datapath_caps &
1246 1 << MC_CMD_GET_CAPABILITIES_OUT_RX_BATCHING_LBN);
1247
1248 /* Fill event queue with all ones (i.e. empty events) */
1249 memset(channel->eventq.buf.addr, 0xff, channel->eventq.buf.len);
1250
1251 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_SIZE, channel->eventq_mask + 1);
1252 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_INSTANCE, channel->channel);
1253 /* INIT_EVQ expects index in vector table, not absolute */
1254 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_IRQ_NUM, channel->channel);
1255 MCDI_POPULATE_DWORD_4(inbuf, INIT_EVQ_IN_FLAGS,
1256 INIT_EVQ_IN_FLAG_INTERRUPTING, 1,
1257 INIT_EVQ_IN_FLAG_RX_MERGE, 1,
1258 INIT_EVQ_IN_FLAG_TX_MERGE, 1,
1259 INIT_EVQ_IN_FLAG_CUT_THRU, !supports_rx_merge);
1260 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_TMR_MODE,
1261 MC_CMD_INIT_EVQ_IN_TMR_MODE_DIS);
1262 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_TMR_LOAD, 0);
1263 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_TMR_RELOAD, 0);
1264 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_COUNT_MODE,
1265 MC_CMD_INIT_EVQ_IN_COUNT_MODE_DIS);
1266 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_COUNT_THRSHLD, 0);
1267
1268 dma_addr = channel->eventq.buf.dma_addr;
1269 for (i = 0; i < entries; ++i) {
1270 MCDI_SET_ARRAY_QWORD(inbuf, INIT_EVQ_IN_DMA_ADDR, i, dma_addr);
1271 dma_addr += EFX_BUF_SIZE;
1272 }
1273
1274 inlen = MC_CMD_INIT_EVQ_IN_LEN(entries);
1275
1276 rc = efx_mcdi_rpc(efx, MC_CMD_INIT_EVQ, inbuf, inlen,
1277 outbuf, sizeof(outbuf), &outlen);
1278 if (rc)
1279 goto fail;
1280
1281 /* IRQ return is ignored */
1282
1283 return 0;
1284
1285fail:
1286 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
1287 return rc;
1288}
1289
1290static void efx_ef10_ev_fini(struct efx_channel *channel)
1291{
1292 MCDI_DECLARE_BUF(inbuf, MC_CMD_FINI_EVQ_IN_LEN);
1293 MCDI_DECLARE_BUF(outbuf, MC_CMD_FINI_EVQ_OUT_LEN);
1294 struct efx_nic *efx = channel->efx;
1295 size_t outlen;
1296 int rc;
1297
1298 MCDI_SET_DWORD(inbuf, FINI_EVQ_IN_INSTANCE, channel->channel);
1299
1300 rc = efx_mcdi_rpc(efx, MC_CMD_FINI_EVQ, inbuf, sizeof(inbuf),
1301 outbuf, sizeof(outbuf), &outlen);
1302
1303 if (rc && rc != -EALREADY)
1304 goto fail;
1305
1306 return;
1307
1308fail:
1309 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
1310}
1311
1312static void efx_ef10_ev_remove(struct efx_channel *channel)
1313{
1314 efx_nic_free_buffer(channel->efx, &channel->eventq.buf);
1315}
1316
1317static void efx_ef10_handle_rx_wrong_queue(struct efx_rx_queue *rx_queue,
1318 unsigned int rx_queue_label)
1319{
1320 struct efx_nic *efx = rx_queue->efx;
1321
1322 netif_info(efx, hw, efx->net_dev,
1323 "rx event arrived on queue %d labeled as queue %u\n",
1324 efx_rx_queue_index(rx_queue), rx_queue_label);
1325
1326 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
1327}
1328
1329static void
1330efx_ef10_handle_rx_bad_lbits(struct efx_rx_queue *rx_queue,
1331 unsigned int actual, unsigned int expected)
1332{
1333 unsigned int dropped = (actual - expected) & rx_queue->ptr_mask;
1334 struct efx_nic *efx = rx_queue->efx;
1335
1336 netif_info(efx, hw, efx->net_dev,
1337 "dropped %d events (index=%d expected=%d)\n",
1338 dropped, actual, expected);
1339
1340 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
1341}
1342
1343/* partially received RX was aborted. clean up. */
1344static void efx_ef10_handle_rx_abort(struct efx_rx_queue *rx_queue)
1345{
1346 unsigned int rx_desc_ptr;
1347
1348 WARN_ON(rx_queue->scatter_n == 0);
1349
1350 netif_dbg(rx_queue->efx, hw, rx_queue->efx->net_dev,
1351 "scattered RX aborted (dropping %u buffers)\n",
1352 rx_queue->scatter_n);
1353
1354 rx_desc_ptr = rx_queue->removed_count & rx_queue->ptr_mask;
1355
1356 efx_rx_packet(rx_queue, rx_desc_ptr, rx_queue->scatter_n,
1357 0, EFX_RX_PKT_DISCARD);
1358
1359 rx_queue->removed_count += rx_queue->scatter_n;
1360 rx_queue->scatter_n = 0;
1361 rx_queue->scatter_len = 0;
1362 ++efx_rx_queue_channel(rx_queue)->n_rx_nodesc_trunc;
1363}
1364
1365static int efx_ef10_handle_rx_event(struct efx_channel *channel,
1366 const efx_qword_t *event)
1367{
1368 unsigned int rx_bytes, next_ptr_lbits, rx_queue_label, rx_l4_class;
1369 unsigned int n_descs, n_packets, i;
1370 struct efx_nic *efx = channel->efx;
1371 struct efx_rx_queue *rx_queue;
1372 bool rx_cont;
1373 u16 flags = 0;
1374
1375 if (unlikely(ACCESS_ONCE(efx->reset_pending)))
1376 return 0;
1377
1378 /* Basic packet information */
1379 rx_bytes = EFX_QWORD_FIELD(*event, ESF_DZ_RX_BYTES);
1380 next_ptr_lbits = EFX_QWORD_FIELD(*event, ESF_DZ_RX_DSC_PTR_LBITS);
1381 rx_queue_label = EFX_QWORD_FIELD(*event, ESF_DZ_RX_QLABEL);
1382 rx_l4_class = EFX_QWORD_FIELD(*event, ESF_DZ_RX_L4_CLASS);
1383 rx_cont = EFX_QWORD_FIELD(*event, ESF_DZ_RX_CONT);
1384
1385 WARN_ON(EFX_QWORD_FIELD(*event, ESF_DZ_RX_DROP_EVENT));
1386
1387 rx_queue = efx_channel_get_rx_queue(channel);
1388
1389 if (unlikely(rx_queue_label != efx_rx_queue_index(rx_queue)))
1390 efx_ef10_handle_rx_wrong_queue(rx_queue, rx_queue_label);
1391
1392 n_descs = ((next_ptr_lbits - rx_queue->removed_count) &
1393 ((1 << ESF_DZ_RX_DSC_PTR_LBITS_WIDTH) - 1));
1394
1395 if (n_descs != rx_queue->scatter_n + 1) {
1396 /* detect rx abort */
1397 if (unlikely(n_descs == rx_queue->scatter_n)) {
1398 WARN_ON(rx_bytes != 0);
1399 efx_ef10_handle_rx_abort(rx_queue);
1400 return 0;
1401 }
1402
1403 if (unlikely(rx_queue->scatter_n != 0)) {
1404 /* Scattered packet completions cannot be
1405 * merged, so something has gone wrong.
1406 */
1407 efx_ef10_handle_rx_bad_lbits(
1408 rx_queue, next_ptr_lbits,
1409 (rx_queue->removed_count +
1410 rx_queue->scatter_n + 1) &
1411 ((1 << ESF_DZ_RX_DSC_PTR_LBITS_WIDTH) - 1));
1412 return 0;
1413 }
1414
1415 /* Merged completion for multiple non-scattered packets */
1416 rx_queue->scatter_n = 1;
1417 rx_queue->scatter_len = 0;
1418 n_packets = n_descs;
1419 ++channel->n_rx_merge_events;
1420 channel->n_rx_merge_packets += n_packets;
1421 flags |= EFX_RX_PKT_PREFIX_LEN;
1422 } else {
1423 ++rx_queue->scatter_n;
1424 rx_queue->scatter_len += rx_bytes;
1425 if (rx_cont)
1426 return 0;
1427 n_packets = 1;
1428 }
1429
1430 if (unlikely(EFX_QWORD_FIELD(*event, ESF_DZ_RX_ECRC_ERR)))
1431 flags |= EFX_RX_PKT_DISCARD;
1432
1433 if (unlikely(EFX_QWORD_FIELD(*event, ESF_DZ_RX_IPCKSUM_ERR))) {
1434 channel->n_rx_ip_hdr_chksum_err += n_packets;
1435 } else if (unlikely(EFX_QWORD_FIELD(*event,
1436 ESF_DZ_RX_TCPUDP_CKSUM_ERR))) {
1437 channel->n_rx_tcp_udp_chksum_err += n_packets;
1438 } else if (rx_l4_class == ESE_DZ_L4_CLASS_TCP ||
1439 rx_l4_class == ESE_DZ_L4_CLASS_UDP) {
1440 flags |= EFX_RX_PKT_CSUMMED;
1441 }
1442
1443 if (rx_l4_class == ESE_DZ_L4_CLASS_TCP)
1444 flags |= EFX_RX_PKT_TCP;
1445
1446 channel->irq_mod_score += 2 * n_packets;
1447
1448 /* Handle received packet(s) */
1449 for (i = 0; i < n_packets; i++) {
1450 efx_rx_packet(rx_queue,
1451 rx_queue->removed_count & rx_queue->ptr_mask,
1452 rx_queue->scatter_n, rx_queue->scatter_len,
1453 flags);
1454 rx_queue->removed_count += rx_queue->scatter_n;
1455 }
1456
1457 rx_queue->scatter_n = 0;
1458 rx_queue->scatter_len = 0;
1459
1460 return n_packets;
1461}
1462
1463static int
1464efx_ef10_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
1465{
1466 struct efx_nic *efx = channel->efx;
1467 struct efx_tx_queue *tx_queue;
1468 unsigned int tx_ev_desc_ptr;
1469 unsigned int tx_ev_q_label;
1470 int tx_descs = 0;
1471
1472 if (unlikely(ACCESS_ONCE(efx->reset_pending)))
1473 return 0;
1474
1475 if (unlikely(EFX_QWORD_FIELD(*event, ESF_DZ_TX_DROP_EVENT)))
1476 return 0;
1477
1478 /* Transmit completion */
1479 tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, ESF_DZ_TX_DESCR_INDX);
1480 tx_ev_q_label = EFX_QWORD_FIELD(*event, ESF_DZ_TX_QLABEL);
1481 tx_queue = efx_channel_get_tx_queue(channel,
1482 tx_ev_q_label % EFX_TXQ_TYPES);
1483 tx_descs = ((tx_ev_desc_ptr + 1 - tx_queue->read_count) &
1484 tx_queue->ptr_mask);
1485 efx_xmit_done(tx_queue, tx_ev_desc_ptr & tx_queue->ptr_mask);
1486
1487 return tx_descs;
1488}
1489
1490static void
1491efx_ef10_handle_driver_event(struct efx_channel *channel, efx_qword_t *event)
1492{
1493 struct efx_nic *efx = channel->efx;
1494 int subcode;
1495
1496 subcode = EFX_QWORD_FIELD(*event, ESF_DZ_DRV_SUB_CODE);
1497
1498 switch (subcode) {
1499 case ESE_DZ_DRV_TIMER_EV:
1500 case ESE_DZ_DRV_WAKE_UP_EV:
1501 break;
1502 case ESE_DZ_DRV_START_UP_EV:
1503 /* event queue init complete. ok. */
1504 break;
1505 default:
1506 netif_err(efx, hw, efx->net_dev,
1507 "channel %d unknown driver event type %d"
1508 " (data " EFX_QWORD_FMT ")\n",
1509 channel->channel, subcode,
1510 EFX_QWORD_VAL(*event));
1511
1512 }
1513}
1514
1515static void efx_ef10_handle_driver_generated_event(struct efx_channel *channel,
1516 efx_qword_t *event)
1517{
1518 struct efx_nic *efx = channel->efx;
1519 u32 subcode;
1520
1521 subcode = EFX_QWORD_FIELD(*event, EFX_DWORD_0);
1522
1523 switch (subcode) {
1524 case EFX_EF10_TEST:
1525 channel->event_test_cpu = raw_smp_processor_id();
1526 break;
1527 case EFX_EF10_REFILL:
1528 /* The queue must be empty, so we won't receive any rx
1529 * events, so efx_process_channel() won't refill the
1530 * queue. Refill it here
1531 */
1532 efx_fast_push_rx_descriptors(&channel->rx_queue);
1533 break;
1534 default:
1535 netif_err(efx, hw, efx->net_dev,
1536 "channel %d unknown driver event type %u"
1537 " (data " EFX_QWORD_FMT ")\n",
1538 channel->channel, (unsigned) subcode,
1539 EFX_QWORD_VAL(*event));
1540 }
1541}
1542
1543static int efx_ef10_ev_process(struct efx_channel *channel, int quota)
1544{
1545 struct efx_nic *efx = channel->efx;
1546 efx_qword_t event, *p_event;
1547 unsigned int read_ptr;
1548 int ev_code;
1549 int tx_descs = 0;
1550 int spent = 0;
1551
1552 read_ptr = channel->eventq_read_ptr;
1553
1554 for (;;) {
1555 p_event = efx_event(channel, read_ptr);
1556 event = *p_event;
1557
1558 if (!efx_event_present(&event))
1559 break;
1560
1561 EFX_SET_QWORD(*p_event);
1562
1563 ++read_ptr;
1564
1565 ev_code = EFX_QWORD_FIELD(event, ESF_DZ_EV_CODE);
1566
1567 netif_vdbg(efx, drv, efx->net_dev,
1568 "processing event on %d " EFX_QWORD_FMT "\n",
1569 channel->channel, EFX_QWORD_VAL(event));
1570
1571 switch (ev_code) {
1572 case ESE_DZ_EV_CODE_MCDI_EV:
1573 efx_mcdi_process_event(channel, &event);
1574 break;
1575 case ESE_DZ_EV_CODE_RX_EV:
1576 spent += efx_ef10_handle_rx_event(channel, &event);
1577 if (spent >= quota) {
1578 /* XXX can we split a merged event to
1579 * avoid going over-quota?
1580 */
1581 spent = quota;
1582 goto out;
1583 }
1584 break;
1585 case ESE_DZ_EV_CODE_TX_EV:
1586 tx_descs += efx_ef10_handle_tx_event(channel, &event);
1587 if (tx_descs > efx->txq_entries) {
1588 spent = quota;
1589 goto out;
1590 } else if (++spent == quota) {
1591 goto out;
1592 }
1593 break;
1594 case ESE_DZ_EV_CODE_DRIVER_EV:
1595 efx_ef10_handle_driver_event(channel, &event);
1596 if (++spent == quota)
1597 goto out;
1598 break;
1599 case EFX_EF10_DRVGEN_EV:
1600 efx_ef10_handle_driver_generated_event(channel, &event);
1601 break;
1602 default:
1603 netif_err(efx, hw, efx->net_dev,
1604 "channel %d unknown event type %d"
1605 " (data " EFX_QWORD_FMT ")\n",
1606 channel->channel, ev_code,
1607 EFX_QWORD_VAL(event));
1608 }
1609 }
1610
1611out:
1612 channel->eventq_read_ptr = read_ptr;
1613 return spent;
1614}
1615
1616static void efx_ef10_ev_read_ack(struct efx_channel *channel)
1617{
1618 struct efx_nic *efx = channel->efx;
1619 efx_dword_t rptr;
1620
1621 if (EFX_EF10_WORKAROUND_35388(efx)) {
1622 BUILD_BUG_ON(EFX_MIN_EVQ_SIZE <
1623 (1 << ERF_DD_EVQ_IND_RPTR_WIDTH));
1624 BUILD_BUG_ON(EFX_MAX_EVQ_SIZE >
1625 (1 << 2 * ERF_DD_EVQ_IND_RPTR_WIDTH));
1626
1627 EFX_POPULATE_DWORD_2(rptr, ERF_DD_EVQ_IND_RPTR_FLAGS,
1628 EFE_DD_EVQ_IND_RPTR_FLAGS_HIGH,
1629 ERF_DD_EVQ_IND_RPTR,
1630 (channel->eventq_read_ptr &
1631 channel->eventq_mask) >>
1632 ERF_DD_EVQ_IND_RPTR_WIDTH);
1633 efx_writed_page(efx, &rptr, ER_DD_EVQ_INDIRECT,
1634 channel->channel);
1635 EFX_POPULATE_DWORD_2(rptr, ERF_DD_EVQ_IND_RPTR_FLAGS,
1636 EFE_DD_EVQ_IND_RPTR_FLAGS_LOW,
1637 ERF_DD_EVQ_IND_RPTR,
1638 channel->eventq_read_ptr &
1639 ((1 << ERF_DD_EVQ_IND_RPTR_WIDTH) - 1));
1640 efx_writed_page(efx, &rptr, ER_DD_EVQ_INDIRECT,
1641 channel->channel);
1642 } else {
1643 EFX_POPULATE_DWORD_1(rptr, ERF_DZ_EVQ_RPTR,
1644 channel->eventq_read_ptr &
1645 channel->eventq_mask);
1646 efx_writed_page(efx, &rptr, ER_DZ_EVQ_RPTR, channel->channel);
1647 }
1648}
1649
1650static void efx_ef10_ev_test_generate(struct efx_channel *channel)
1651{
1652 MCDI_DECLARE_BUF(inbuf, MC_CMD_DRIVER_EVENT_IN_LEN);
1653 struct efx_nic *efx = channel->efx;
1654 efx_qword_t event;
1655 int rc;
1656
1657 EFX_POPULATE_QWORD_2(event,
1658 ESF_DZ_EV_CODE, EFX_EF10_DRVGEN_EV,
1659 ESF_DZ_EV_DATA, EFX_EF10_TEST);
1660
1661 MCDI_SET_DWORD(inbuf, DRIVER_EVENT_IN_EVQ, channel->channel);
1662
1663 /* MCDI_SET_QWORD is not appropriate here since EFX_POPULATE_* has
1664 * already swapped the data to little-endian order.
1665 */
1666 memcpy(MCDI_PTR(inbuf, DRIVER_EVENT_IN_DATA), &event.u64[0],
1667 sizeof(efx_qword_t));
1668
1669 rc = efx_mcdi_rpc(efx, MC_CMD_DRIVER_EVENT, inbuf, sizeof(inbuf),
1670 NULL, 0, NULL);
1671 if (rc != 0)
1672 goto fail;
1673
1674 return;
1675
1676fail:
1677 WARN_ON(true);
1678 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
1679}
1680
1681void efx_ef10_handle_drain_event(struct efx_nic *efx)
1682{
1683 if (atomic_dec_and_test(&efx->active_queues))
1684 wake_up(&efx->flush_wq);
1685
1686 WARN_ON(atomic_read(&efx->active_queues) < 0);
1687}
1688
1689static int efx_ef10_fini_dmaq(struct efx_nic *efx)
1690{
1691 struct efx_ef10_nic_data *nic_data = efx->nic_data;
1692 struct efx_channel *channel;
1693 struct efx_tx_queue *tx_queue;
1694 struct efx_rx_queue *rx_queue;
1695 int pending;
1696
1697 /* If the MC has just rebooted, the TX/RX queues will have already been
1698 * torn down, but efx->active_queues needs to be set to zero.
1699 */
1700 if (nic_data->must_realloc_vis) {
1701 atomic_set(&efx->active_queues, 0);
1702 return 0;
1703 }
1704
1705 /* Do not attempt to write to the NIC during EEH recovery */
1706 if (efx->state != STATE_RECOVERY) {
1707 efx_for_each_channel(channel, efx) {
1708 efx_for_each_channel_rx_queue(rx_queue, channel)
1709 efx_ef10_rx_fini(rx_queue);
1710 efx_for_each_channel_tx_queue(tx_queue, channel)
1711 efx_ef10_tx_fini(tx_queue);
1712 }
1713
1714 wait_event_timeout(efx->flush_wq,
1715 atomic_read(&efx->active_queues) == 0,
1716 msecs_to_jiffies(EFX_MAX_FLUSH_TIME));
1717 pending = atomic_read(&efx->active_queues);
1718 if (pending) {
1719 netif_err(efx, hw, efx->net_dev, "failed to flush %d queues\n",
1720 pending);
1721 return -ETIMEDOUT;
1722 }
1723 }
1724
1725 return 0;
1726}
1727
1728static bool efx_ef10_filter_equal(const struct efx_filter_spec *left,
1729 const struct efx_filter_spec *right)
1730{
1731 if ((left->match_flags ^ right->match_flags) |
1732 ((left->flags ^ right->flags) &
1733 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)))
1734 return false;
1735
1736 return memcmp(&left->outer_vid, &right->outer_vid,
1737 sizeof(struct efx_filter_spec) -
1738 offsetof(struct efx_filter_spec, outer_vid)) == 0;
1739}
1740
1741static unsigned int efx_ef10_filter_hash(const struct efx_filter_spec *spec)
1742{
1743 BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3);
1744 return jhash2((const u32 *)&spec->outer_vid,
1745 (sizeof(struct efx_filter_spec) -
1746 offsetof(struct efx_filter_spec, outer_vid)) / 4,
1747 0);
1748 /* XXX should we randomise the initval? */
1749}
1750
1751/* Decide whether a filter should be exclusive or else should allow
1752 * delivery to additional recipients. Currently we decide that
1753 * filters for specific local unicast MAC and IP addresses are
1754 * exclusive.
1755 */
1756static bool efx_ef10_filter_is_exclusive(const struct efx_filter_spec *spec)
1757{
1758 if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC &&
1759 !is_multicast_ether_addr(spec->loc_mac))
1760 return true;
1761
1762 if ((spec->match_flags &
1763 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==
1764 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {
1765 if (spec->ether_type == htons(ETH_P_IP) &&
1766 !ipv4_is_multicast(spec->loc_host[0]))
1767 return true;
1768 if (spec->ether_type == htons(ETH_P_IPV6) &&
1769 ((const u8 *)spec->loc_host)[0] != 0xff)
1770 return true;
1771 }
1772
1773 return false;
1774}
1775
1776static struct efx_filter_spec *
1777efx_ef10_filter_entry_spec(const struct efx_ef10_filter_table *table,
1778 unsigned int filter_idx)
1779{
1780 return (struct efx_filter_spec *)(table->entry[filter_idx].spec &
1781 ~EFX_EF10_FILTER_FLAGS);
1782}
1783
1784static unsigned int
1785efx_ef10_filter_entry_flags(const struct efx_ef10_filter_table *table,
1786 unsigned int filter_idx)
1787{
1788 return table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAGS;
1789}
1790
1791static void
1792efx_ef10_filter_set_entry(struct efx_ef10_filter_table *table,
1793 unsigned int filter_idx,
1794 const struct efx_filter_spec *spec,
1795 unsigned int flags)
1796{
1797 table->entry[filter_idx].spec = (unsigned long)spec | flags;
1798}
1799
1800static void efx_ef10_filter_push_prep(struct efx_nic *efx,
1801 const struct efx_filter_spec *spec,
1802 efx_dword_t *inbuf, u64 handle,
1803 bool replacing)
1804{
1805 struct efx_ef10_nic_data *nic_data = efx->nic_data;
1806
1807 memset(inbuf, 0, MC_CMD_FILTER_OP_IN_LEN);
1808
1809 if (replacing) {
1810 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
1811 MC_CMD_FILTER_OP_IN_OP_REPLACE);
1812 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, handle);
1813 } else {
1814 u32 match_fields = 0;
1815
1816 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
1817 efx_ef10_filter_is_exclusive(spec) ?
1818 MC_CMD_FILTER_OP_IN_OP_INSERT :
1819 MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE);
1820
1821 /* Convert match flags and values. Unlike almost
1822 * everything else in MCDI, these fields are in
1823 * network byte order.
1824 */
1825 if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC_IG)
1826 match_fields |=
1827 is_multicast_ether_addr(spec->loc_mac) ?
1828 1 << MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_MCAST_DST_LBN :
1829 1 << MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_UCAST_DST_LBN;
1830#define COPY_FIELD(gen_flag, gen_field, mcdi_field) \
1831 if (spec->match_flags & EFX_FILTER_MATCH_ ## gen_flag) { \
1832 match_fields |= \
1833 1 << MC_CMD_FILTER_OP_IN_MATCH_ ## \
1834 mcdi_field ## _LBN; \
1835 BUILD_BUG_ON( \
1836 MC_CMD_FILTER_OP_IN_ ## mcdi_field ## _LEN < \
1837 sizeof(spec->gen_field)); \
1838 memcpy(MCDI_PTR(inbuf, FILTER_OP_IN_ ## mcdi_field), \
1839 &spec->gen_field, sizeof(spec->gen_field)); \
1840 }
1841 COPY_FIELD(REM_HOST, rem_host, SRC_IP);
1842 COPY_FIELD(LOC_HOST, loc_host, DST_IP);
1843 COPY_FIELD(REM_MAC, rem_mac, SRC_MAC);
1844 COPY_FIELD(REM_PORT, rem_port, SRC_PORT);
1845 COPY_FIELD(LOC_MAC, loc_mac, DST_MAC);
1846 COPY_FIELD(LOC_PORT, loc_port, DST_PORT);
1847 COPY_FIELD(ETHER_TYPE, ether_type, ETHER_TYPE);
1848 COPY_FIELD(INNER_VID, inner_vid, INNER_VLAN);
1849 COPY_FIELD(OUTER_VID, outer_vid, OUTER_VLAN);
1850 COPY_FIELD(IP_PROTO, ip_proto, IP_PROTO);
1851#undef COPY_FIELD
1852 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_MATCH_FIELDS,
1853 match_fields);
1854 }
1855
1856 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
1857 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_DEST,
1858 spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ?
1859 MC_CMD_FILTER_OP_IN_RX_DEST_DROP :
1860 MC_CMD_FILTER_OP_IN_RX_DEST_HOST);
1861 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DEST,
1862 MC_CMD_FILTER_OP_IN_TX_DEST_DEFAULT);
1863 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_QUEUE, spec->dmaq_id);
1864 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_MODE,
1865 (spec->flags & EFX_FILTER_FLAG_RX_RSS) ?
1866 MC_CMD_FILTER_OP_IN_RX_MODE_RSS :
1867 MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE);
1868 if (spec->flags & EFX_FILTER_FLAG_RX_RSS)
1869 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_CONTEXT,
1870 spec->rss_context !=
1871 EFX_FILTER_RSS_CONTEXT_DEFAULT ?
1872 spec->rss_context : nic_data->rx_rss_context);
1873}
1874
1875static int efx_ef10_filter_push(struct efx_nic *efx,
1876 const struct efx_filter_spec *spec,
1877 u64 *handle, bool replacing)
1878{
1879 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_IN_LEN);
1880 MCDI_DECLARE_BUF(outbuf, MC_CMD_FILTER_OP_OUT_LEN);
1881 int rc;
1882
1883 efx_ef10_filter_push_prep(efx, spec, inbuf, *handle, replacing);
1884 rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf),
1885 outbuf, sizeof(outbuf), NULL);
1886 if (rc == 0)
1887 *handle = MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE);
1888 return rc;
1889}
1890
1891static int efx_ef10_filter_rx_match_pri(struct efx_ef10_filter_table *table,
1892 enum efx_filter_match_flags match_flags)
1893{
1894 unsigned int match_pri;
1895
1896 for (match_pri = 0;
1897 match_pri < table->rx_match_count;
1898 match_pri++)
1899 if (table->rx_match_flags[match_pri] == match_flags)
1900 return match_pri;
1901
1902 return -EPROTONOSUPPORT;
1903}
1904
1905static s32 efx_ef10_filter_insert(struct efx_nic *efx,
1906 struct efx_filter_spec *spec,
1907 bool replace_equal)
1908{
1909 struct efx_ef10_filter_table *table = efx->filter_state;
1910 DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
1911 struct efx_filter_spec *saved_spec;
1912 unsigned int match_pri, hash;
1913 unsigned int priv_flags;
1914 bool replacing = false;
1915 int ins_index = -1;
1916 DEFINE_WAIT(wait);
1917 bool is_mc_recip;
1918 s32 rc;
1919
1920 /* For now, only support RX filters */
1921 if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) !=
1922 EFX_FILTER_FLAG_RX)
1923 return -EINVAL;
1924
1925 rc = efx_ef10_filter_rx_match_pri(table, spec->match_flags);
1926 if (rc < 0)
1927 return rc;
1928 match_pri = rc;
1929
1930 hash = efx_ef10_filter_hash(spec);
1931 is_mc_recip = efx_filter_is_mc_recipient(spec);
1932 if (is_mc_recip)
1933 bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
1934
1935 /* Find any existing filters with the same match tuple or
1936 * else a free slot to insert at. If any of them are busy,
1937 * we have to wait and retry.
1938 */
1939 for (;;) {
1940 unsigned int depth = 1;
1941 unsigned int i;
1942
1943 spin_lock_bh(&efx->filter_lock);
1944
1945 for (;;) {
1946 i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1);
1947 saved_spec = efx_ef10_filter_entry_spec(table, i);
1948
1949 if (!saved_spec) {
1950 if (ins_index < 0)
1951 ins_index = i;
1952 } else if (efx_ef10_filter_equal(spec, saved_spec)) {
1953 if (table->entry[i].spec &
1954 EFX_EF10_FILTER_FLAG_BUSY)
1955 break;
1956 if (spec->priority < saved_spec->priority &&
1957 !(saved_spec->priority ==
1958 EFX_FILTER_PRI_REQUIRED &&
1959 saved_spec->flags &
1960 EFX_FILTER_FLAG_RX_STACK)) {
1961 rc = -EPERM;
1962 goto out_unlock;
1963 }
1964 if (!is_mc_recip) {
1965 /* This is the only one */
1966 if (spec->priority ==
1967 saved_spec->priority &&
1968 !replace_equal) {
1969 rc = -EEXIST;
1970 goto out_unlock;
1971 }
1972 ins_index = i;
1973 goto found;
1974 } else if (spec->priority >
1975 saved_spec->priority ||
1976 (spec->priority ==
1977 saved_spec->priority &&
1978 replace_equal)) {
1979 if (ins_index < 0)
1980 ins_index = i;
1981 else
1982 __set_bit(depth, mc_rem_map);
1983 }
1984 }
1985
1986 /* Once we reach the maximum search depth, use
1987 * the first suitable slot or return -EBUSY if
1988 * there was none
1989 */
1990 if (depth == EFX_EF10_FILTER_SEARCH_LIMIT) {
1991 if (ins_index < 0) {
1992 rc = -EBUSY;
1993 goto out_unlock;
1994 }
1995 goto found;
1996 }
1997
1998 ++depth;
1999 }
2000
2001 prepare_to_wait(&table->waitq, &wait, TASK_UNINTERRUPTIBLE);
2002 spin_unlock_bh(&efx->filter_lock);
2003 schedule();
2004 }
2005
2006found:
2007 /* Create a software table entry if necessary, and mark it
2008 * busy. We might yet fail to insert, but any attempt to
2009 * insert a conflicting filter while we're waiting for the
2010 * firmware must find the busy entry.
2011 */
2012 saved_spec = efx_ef10_filter_entry_spec(table, ins_index);
2013 if (saved_spec) {
2014 if (spec->flags & EFX_FILTER_FLAG_RX_STACK) {
2015 /* Just make sure it won't be removed */
2016 saved_spec->flags |= EFX_FILTER_FLAG_RX_STACK;
2017 table->entry[ins_index].spec &=
2018 ~EFX_EF10_FILTER_FLAG_STACK_OLD;
2019 rc = ins_index;
2020 goto out_unlock;
2021 }
2022 replacing = true;
2023 priv_flags = efx_ef10_filter_entry_flags(table, ins_index);
2024 } else {
2025 saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC);
2026 if (!saved_spec) {
2027 rc = -ENOMEM;
2028 goto out_unlock;
2029 }
2030 *saved_spec = *spec;
2031 priv_flags = 0;
2032 }
2033 efx_ef10_filter_set_entry(table, ins_index, saved_spec,
2034 priv_flags | EFX_EF10_FILTER_FLAG_BUSY);
2035
2036 /* Mark lower-priority multicast recipients busy prior to removal */
2037 if (is_mc_recip) {
2038 unsigned int depth, i;
2039
2040 for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
2041 i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1);
2042 if (test_bit(depth, mc_rem_map))
2043 table->entry[i].spec |=
2044 EFX_EF10_FILTER_FLAG_BUSY;
2045 }
2046 }
2047
2048 spin_unlock_bh(&efx->filter_lock);
2049
2050 rc = efx_ef10_filter_push(efx, spec, &table->entry[ins_index].handle,
2051 replacing);
2052
2053 /* Finalise the software table entry */
2054 spin_lock_bh(&efx->filter_lock);
2055 if (rc == 0) {
2056 if (replacing) {
2057 /* Update the fields that may differ */
2058 saved_spec->priority = spec->priority;
2059 saved_spec->flags &= EFX_FILTER_FLAG_RX_STACK;
2060 saved_spec->flags |= spec->flags;
2061 saved_spec->rss_context = spec->rss_context;
2062 saved_spec->dmaq_id = spec->dmaq_id;
2063 }
2064 } else if (!replacing) {
2065 kfree(saved_spec);
2066 saved_spec = NULL;
2067 }
2068 efx_ef10_filter_set_entry(table, ins_index, saved_spec, priv_flags);
2069
2070 /* Remove and finalise entries for lower-priority multicast
2071 * recipients
2072 */
2073 if (is_mc_recip) {
2074 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_IN_LEN);
2075 unsigned int depth, i;
2076
2077 memset(inbuf, 0, sizeof(inbuf));
2078
2079 for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
2080 if (!test_bit(depth, mc_rem_map))
2081 continue;
2082
2083 i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1);
2084 saved_spec = efx_ef10_filter_entry_spec(table, i);
2085 priv_flags = efx_ef10_filter_entry_flags(table, i);
2086
2087 if (rc == 0) {
2088 spin_unlock_bh(&efx->filter_lock);
2089 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
2090 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
2091 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
2092 table->entry[i].handle);
2093 rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP,
2094 inbuf, sizeof(inbuf),
2095 NULL, 0, NULL);
2096 spin_lock_bh(&efx->filter_lock);
2097 }
2098
2099 if (rc == 0) {
2100 kfree(saved_spec);
2101 saved_spec = NULL;
2102 priv_flags = 0;
2103 } else {
2104 priv_flags &= ~EFX_EF10_FILTER_FLAG_BUSY;
2105 }
2106 efx_ef10_filter_set_entry(table, i, saved_spec,
2107 priv_flags);
2108 }
2109 }
2110
2111 /* If successful, return the inserted filter ID */
2112 if (rc == 0)
2113 rc = match_pri * HUNT_FILTER_TBL_ROWS + ins_index;
2114
2115 wake_up_all(&table->waitq);
2116out_unlock:
2117 spin_unlock_bh(&efx->filter_lock);
2118 finish_wait(&table->waitq, &wait);
2119 return rc;
2120}
2121
2122void efx_ef10_filter_update_rx_scatter(struct efx_nic *efx)
2123{
2124 /* no need to do anything here on EF10 */
2125}
2126
2127/* Remove a filter.
2128 * If !stack_requested, remove by ID
2129 * If stack_requested, remove by index
2130 * Filter ID may come from userland and must be range-checked.
2131 */
2132static int efx_ef10_filter_remove_internal(struct efx_nic *efx,
2133 enum efx_filter_priority priority,
2134 u32 filter_id, bool stack_requested)
2135{
2136 unsigned int filter_idx = filter_id % HUNT_FILTER_TBL_ROWS;
2137 struct efx_ef10_filter_table *table = efx->filter_state;
2138 MCDI_DECLARE_BUF(inbuf,
2139 MC_CMD_FILTER_OP_IN_HANDLE_OFST +
2140 MC_CMD_FILTER_OP_IN_HANDLE_LEN);
2141 struct efx_filter_spec *spec;
2142 DEFINE_WAIT(wait);
2143 int rc;
2144
2145 /* Find the software table entry and mark it busy. Don't
2146 * remove it yet; any attempt to update while we're waiting
2147 * for the firmware must find the busy entry.
2148 */
2149 for (;;) {
2150 spin_lock_bh(&efx->filter_lock);
2151 if (!(table->entry[filter_idx].spec &
2152 EFX_EF10_FILTER_FLAG_BUSY))
2153 break;
2154 prepare_to_wait(&table->waitq, &wait, TASK_UNINTERRUPTIBLE);
2155 spin_unlock_bh(&efx->filter_lock);
2156 schedule();
2157 }
2158 spec = efx_ef10_filter_entry_spec(table, filter_idx);
2159 if (!spec || spec->priority > priority ||
2160 (!stack_requested &&
2161 efx_ef10_filter_rx_match_pri(table, spec->match_flags) !=
2162 filter_id / HUNT_FILTER_TBL_ROWS)) {
2163 rc = -ENOENT;
2164 goto out_unlock;
2165 }
2166 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY;
2167 spin_unlock_bh(&efx->filter_lock);
2168
2169 if (spec->flags & EFX_FILTER_FLAG_RX_STACK && !stack_requested) {
2170 /* Reset steering of a stack-owned filter */
2171
2172 struct efx_filter_spec new_spec = *spec;
2173
2174 new_spec.priority = EFX_FILTER_PRI_REQUIRED;
2175 new_spec.flags = (EFX_FILTER_FLAG_RX |
2176 EFX_FILTER_FLAG_RX_RSS |
2177 EFX_FILTER_FLAG_RX_STACK);
2178 new_spec.dmaq_id = 0;
2179 new_spec.rss_context = EFX_FILTER_RSS_CONTEXT_DEFAULT;
2180 rc = efx_ef10_filter_push(efx, &new_spec,
2181 &table->entry[filter_idx].handle,
2182 true);
2183
2184 spin_lock_bh(&efx->filter_lock);
2185 if (rc == 0)
2186 *spec = new_spec;
2187 } else {
2188 /* Really remove the filter */
2189
2190 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
2191 efx_ef10_filter_is_exclusive(spec) ?
2192 MC_CMD_FILTER_OP_IN_OP_REMOVE :
2193 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
2194 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
2195 table->entry[filter_idx].handle);
2196 rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP,
2197 inbuf, sizeof(inbuf), NULL, 0, NULL);
2198
2199 spin_lock_bh(&efx->filter_lock);
2200 if (rc == 0) {
2201 kfree(spec);
2202 efx_ef10_filter_set_entry(table, filter_idx, NULL, 0);
2203 }
2204 }
2205 table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_BUSY;
2206 wake_up_all(&table->waitq);
2207out_unlock:
2208 spin_unlock_bh(&efx->filter_lock);
2209 finish_wait(&table->waitq, &wait);
2210 return rc;
2211}
2212
2213static int efx_ef10_filter_remove_safe(struct efx_nic *efx,
2214 enum efx_filter_priority priority,
2215 u32 filter_id)
2216{
2217 return efx_ef10_filter_remove_internal(efx, priority, filter_id, false);
2218}
2219
2220static int efx_ef10_filter_get_safe(struct efx_nic *efx,
2221 enum efx_filter_priority priority,
2222 u32 filter_id, struct efx_filter_spec *spec)
2223{
2224 unsigned int filter_idx = filter_id % HUNT_FILTER_TBL_ROWS;
2225 struct efx_ef10_filter_table *table = efx->filter_state;
2226 const struct efx_filter_spec *saved_spec;
2227 int rc;
2228
2229 spin_lock_bh(&efx->filter_lock);
2230 saved_spec = efx_ef10_filter_entry_spec(table, filter_idx);
2231 if (saved_spec && saved_spec->priority == priority &&
2232 efx_ef10_filter_rx_match_pri(table, saved_spec->match_flags) ==
2233 filter_id / HUNT_FILTER_TBL_ROWS) {
2234 *spec = *saved_spec;
2235 rc = 0;
2236 } else {
2237 rc = -ENOENT;
2238 }
2239 spin_unlock_bh(&efx->filter_lock);
2240 return rc;
2241}
2242
2243static void efx_ef10_filter_clear_rx(struct efx_nic *efx,
2244 enum efx_filter_priority priority)
2245{
2246 /* TODO */
2247}
2248
2249static u32 efx_ef10_filter_count_rx_used(struct efx_nic *efx,
2250 enum efx_filter_priority priority)
2251{
2252 struct efx_ef10_filter_table *table = efx->filter_state;
2253 unsigned int filter_idx;
2254 s32 count = 0;
2255
2256 spin_lock_bh(&efx->filter_lock);
2257 for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) {
2258 if (table->entry[filter_idx].spec &&
2259 efx_ef10_filter_entry_spec(table, filter_idx)->priority ==
2260 priority)
2261 ++count;
2262 }
2263 spin_unlock_bh(&efx->filter_lock);
2264 return count;
2265}
2266
2267static u32 efx_ef10_filter_get_rx_id_limit(struct efx_nic *efx)
2268{
2269 struct efx_ef10_filter_table *table = efx->filter_state;
2270
2271 return table->rx_match_count * HUNT_FILTER_TBL_ROWS;
2272}
2273
2274static s32 efx_ef10_filter_get_rx_ids(struct efx_nic *efx,
2275 enum efx_filter_priority priority,
2276 u32 *buf, u32 size)
2277{
2278 struct efx_ef10_filter_table *table = efx->filter_state;
2279 struct efx_filter_spec *spec;
2280 unsigned int filter_idx;
2281 s32 count = 0;
2282
2283 spin_lock_bh(&efx->filter_lock);
2284 for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) {
2285 spec = efx_ef10_filter_entry_spec(table, filter_idx);
2286 if (spec && spec->priority == priority) {
2287 if (count == size) {
2288 count = -EMSGSIZE;
2289 break;
2290 }
2291 buf[count++] = (efx_ef10_filter_rx_match_pri(
2292 table, spec->match_flags) *
2293 HUNT_FILTER_TBL_ROWS +
2294 filter_idx);
2295 }
2296 }
2297 spin_unlock_bh(&efx->filter_lock);
2298 return count;
2299}
2300
2301#ifdef CONFIG_RFS_ACCEL
2302
2303static efx_mcdi_async_completer efx_ef10_filter_rfs_insert_complete;
2304
2305static s32 efx_ef10_filter_rfs_insert(struct efx_nic *efx,
2306 struct efx_filter_spec *spec)
2307{
2308 struct efx_ef10_filter_table *table = efx->filter_state;
2309 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_IN_LEN);
2310 struct efx_filter_spec *saved_spec;
2311 unsigned int hash, i, depth = 1;
2312 bool replacing = false;
2313 int ins_index = -1;
2314 u64 cookie;
2315 s32 rc;
2316
2317 /* Must be an RX filter without RSS and not for a multicast
2318 * destination address (RFS only works for connected sockets).
2319 * These restrictions allow us to pass only a tiny amount of
2320 * data through to the completion function.
2321 */
2322 EFX_WARN_ON_PARANOID(spec->flags !=
2323 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_RX_SCATTER));
2324 EFX_WARN_ON_PARANOID(spec->priority != EFX_FILTER_PRI_HINT);
2325 EFX_WARN_ON_PARANOID(efx_filter_is_mc_recipient(spec));
2326
2327 hash = efx_ef10_filter_hash(spec);
2328
2329 spin_lock_bh(&efx->filter_lock);
2330
2331 /* Find any existing filter with the same match tuple or else
2332 * a free slot to insert at. If an existing filter is busy,
2333 * we have to give up.
2334 */
2335 for (;;) {
2336 i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1);
2337 saved_spec = efx_ef10_filter_entry_spec(table, i);
2338
2339 if (!saved_spec) {
2340 if (ins_index < 0)
2341 ins_index = i;
2342 } else if (efx_ef10_filter_equal(spec, saved_spec)) {
2343 if (table->entry[i].spec & EFX_EF10_FILTER_FLAG_BUSY) {
2344 rc = -EBUSY;
2345 goto fail_unlock;
2346 }
2347 EFX_WARN_ON_PARANOID(saved_spec->flags &
2348 EFX_FILTER_FLAG_RX_STACK);
2349 if (spec->priority < saved_spec->priority) {
2350 rc = -EPERM;
2351 goto fail_unlock;
2352 }
2353 ins_index = i;
2354 break;
2355 }
2356
2357 /* Once we reach the maximum search depth, use the
2358 * first suitable slot or return -EBUSY if there was
2359 * none
2360 */
2361 if (depth == EFX_EF10_FILTER_SEARCH_LIMIT) {
2362 if (ins_index < 0) {
2363 rc = -EBUSY;
2364 goto fail_unlock;
2365 }
2366 break;
2367 }
2368
2369 ++depth;
2370 }
2371
2372 /* Create a software table entry if necessary, and mark it
2373 * busy. We might yet fail to insert, but any attempt to
2374 * insert a conflicting filter while we're waiting for the
2375 * firmware must find the busy entry.
2376 */
2377 saved_spec = efx_ef10_filter_entry_spec(table, ins_index);
2378 if (saved_spec) {
2379 replacing = true;
2380 } else {
2381 saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC);
2382 if (!saved_spec) {
2383 rc = -ENOMEM;
2384 goto fail_unlock;
2385 }
2386 *saved_spec = *spec;
2387 }
2388 efx_ef10_filter_set_entry(table, ins_index, saved_spec,
2389 EFX_EF10_FILTER_FLAG_BUSY);
2390
2391 spin_unlock_bh(&efx->filter_lock);
2392
2393 /* Pack up the variables needed on completion */
2394 cookie = replacing << 31 | ins_index << 16 | spec->dmaq_id;
2395
2396 efx_ef10_filter_push_prep(efx, spec, inbuf,
2397 table->entry[ins_index].handle, replacing);
2398 efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf),
2399 MC_CMD_FILTER_OP_OUT_LEN,
2400 efx_ef10_filter_rfs_insert_complete, cookie);
2401
2402 return ins_index;
2403
2404fail_unlock:
2405 spin_unlock_bh(&efx->filter_lock);
2406 return rc;
2407}
2408
2409static void
2410efx_ef10_filter_rfs_insert_complete(struct efx_nic *efx, unsigned long cookie,
2411 int rc, efx_dword_t *outbuf,
2412 size_t outlen_actual)
2413{
2414 struct efx_ef10_filter_table *table = efx->filter_state;
2415 unsigned int ins_index, dmaq_id;
2416 struct efx_filter_spec *spec;
2417 bool replacing;
2418
2419 /* Unpack the cookie */
2420 replacing = cookie >> 31;
2421 ins_index = (cookie >> 16) & (HUNT_FILTER_TBL_ROWS - 1);
2422 dmaq_id = cookie & 0xffff;
2423
2424 spin_lock_bh(&efx->filter_lock);
2425 spec = efx_ef10_filter_entry_spec(table, ins_index);
2426 if (rc == 0) {
2427 table->entry[ins_index].handle =
2428 MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE);
2429 if (replacing)
2430 spec->dmaq_id = dmaq_id;
2431 } else if (!replacing) {
2432 kfree(spec);
2433 spec = NULL;
2434 }
2435 efx_ef10_filter_set_entry(table, ins_index, spec, 0);
2436 spin_unlock_bh(&efx->filter_lock);
2437
2438 wake_up_all(&table->waitq);
2439}
2440
2441static void
2442efx_ef10_filter_rfs_expire_complete(struct efx_nic *efx,
2443 unsigned long filter_idx,
2444 int rc, efx_dword_t *outbuf,
2445 size_t outlen_actual);
2446
2447static bool efx_ef10_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
2448 unsigned int filter_idx)
2449{
2450 struct efx_ef10_filter_table *table = efx->filter_state;
2451 struct efx_filter_spec *spec =
2452 efx_ef10_filter_entry_spec(table, filter_idx);
2453 MCDI_DECLARE_BUF(inbuf,
2454 MC_CMD_FILTER_OP_IN_HANDLE_OFST +
2455 MC_CMD_FILTER_OP_IN_HANDLE_LEN);
2456
2457 if (!spec ||
2458 (table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAG_BUSY) ||
2459 spec->priority != EFX_FILTER_PRI_HINT ||
2460 !rps_may_expire_flow(efx->net_dev, spec->dmaq_id,
2461 flow_id, filter_idx))
2462 return false;
2463
2464 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
2465 MC_CMD_FILTER_OP_IN_OP_REMOVE);
2466 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
2467 table->entry[filter_idx].handle);
2468 if (efx_mcdi_rpc_async(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), 0,
2469 efx_ef10_filter_rfs_expire_complete, filter_idx))
2470 return false;
2471
2472 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY;
2473 return true;
2474}
2475
2476static void
2477efx_ef10_filter_rfs_expire_complete(struct efx_nic *efx,
2478 unsigned long filter_idx,
2479 int rc, efx_dword_t *outbuf,
2480 size_t outlen_actual)
2481{
2482 struct efx_ef10_filter_table *table = efx->filter_state;
2483 struct efx_filter_spec *spec =
2484 efx_ef10_filter_entry_spec(table, filter_idx);
2485
2486 spin_lock_bh(&efx->filter_lock);
2487 if (rc == 0) {
2488 kfree(spec);
2489 efx_ef10_filter_set_entry(table, filter_idx, NULL, 0);
2490 }
2491 table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_BUSY;
2492 wake_up_all(&table->waitq);
2493 spin_unlock_bh(&efx->filter_lock);
2494}
2495
2496#endif /* CONFIG_RFS_ACCEL */
2497
2498static int efx_ef10_filter_match_flags_from_mcdi(u32 mcdi_flags)
2499{
2500 int match_flags = 0;
2501
2502#define MAP_FLAG(gen_flag, mcdi_field) { \
2503 u32 old_mcdi_flags = mcdi_flags; \
2504 mcdi_flags &= ~(1 << MC_CMD_FILTER_OP_IN_MATCH_ ## \
2505 mcdi_field ## _LBN); \
2506 if (mcdi_flags != old_mcdi_flags) \
2507 match_flags |= EFX_FILTER_MATCH_ ## gen_flag; \
2508 }
2509 MAP_FLAG(LOC_MAC_IG, UNKNOWN_UCAST_DST);
2510 MAP_FLAG(LOC_MAC_IG, UNKNOWN_MCAST_DST);
2511 MAP_FLAG(REM_HOST, SRC_IP);
2512 MAP_FLAG(LOC_HOST, DST_IP);
2513 MAP_FLAG(REM_MAC, SRC_MAC);
2514 MAP_FLAG(REM_PORT, SRC_PORT);
2515 MAP_FLAG(LOC_MAC, DST_MAC);
2516 MAP_FLAG(LOC_PORT, DST_PORT);
2517 MAP_FLAG(ETHER_TYPE, ETHER_TYPE);
2518 MAP_FLAG(INNER_VID, INNER_VLAN);
2519 MAP_FLAG(OUTER_VID, OUTER_VLAN);
2520 MAP_FLAG(IP_PROTO, IP_PROTO);
2521#undef MAP_FLAG
2522
2523 /* Did we map them all? */
2524 if (mcdi_flags)
2525 return -EINVAL;
2526
2527 return match_flags;
2528}
2529
2530static int efx_ef10_filter_table_probe(struct efx_nic *efx)
2531{
2532 MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN);
2533 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX);
2534 unsigned int pd_match_pri, pd_match_count;
2535 struct efx_ef10_filter_table *table;
2536 size_t outlen;
2537 int rc;
2538
2539 table = kzalloc(sizeof(*table), GFP_KERNEL);
2540 if (!table)
2541 return -ENOMEM;
2542
2543 /* Find out which RX filter types are supported, and their priorities */
2544 MCDI_SET_DWORD(inbuf, GET_PARSER_DISP_INFO_IN_OP,
2545 MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES);
2546 rc = efx_mcdi_rpc(efx, MC_CMD_GET_PARSER_DISP_INFO,
2547 inbuf, sizeof(inbuf), outbuf, sizeof(outbuf),
2548 &outlen);
2549 if (rc)
2550 goto fail;
2551 pd_match_count = MCDI_VAR_ARRAY_LEN(
2552 outlen, GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES);
2553 table->rx_match_count = 0;
2554
2555 for (pd_match_pri = 0; pd_match_pri < pd_match_count; pd_match_pri++) {
2556 u32 mcdi_flags =
2557 MCDI_ARRAY_DWORD(
2558 outbuf,
2559 GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES,
2560 pd_match_pri);
2561 rc = efx_ef10_filter_match_flags_from_mcdi(mcdi_flags);
2562 if (rc < 0) {
2563 netif_dbg(efx, probe, efx->net_dev,
2564 "%s: fw flags %#x pri %u not supported in driver\n",
2565 __func__, mcdi_flags, pd_match_pri);
2566 } else {
2567 netif_dbg(efx, probe, efx->net_dev,
2568 "%s: fw flags %#x pri %u supported as driver flags %#x pri %u\n",
2569 __func__, mcdi_flags, pd_match_pri,
2570 rc, table->rx_match_count);
2571 table->rx_match_flags[table->rx_match_count++] = rc;
2572 }
2573 }
2574
2575 table->entry = vzalloc(HUNT_FILTER_TBL_ROWS * sizeof(*table->entry));
2576 if (!table->entry) {
2577 rc = -ENOMEM;
2578 goto fail;
2579 }
2580
2581 efx->filter_state = table;
2582 init_waitqueue_head(&table->waitq);
2583 return 0;
2584
2585fail:
2586 kfree(table);
2587 return rc;
2588}
2589
2590static void efx_ef10_filter_table_restore(struct efx_nic *efx)
2591{
2592 struct efx_ef10_filter_table *table = efx->filter_state;
2593 struct efx_ef10_nic_data *nic_data = efx->nic_data;
2594 struct efx_filter_spec *spec;
2595 unsigned int filter_idx;
2596 bool failed = false;
2597 int rc;
2598
2599 if (!nic_data->must_restore_filters)
2600 return;
2601
2602 spin_lock_bh(&efx->filter_lock);
2603
2604 for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) {
2605 spec = efx_ef10_filter_entry_spec(table, filter_idx);
2606 if (!spec)
2607 continue;
2608
2609 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_BUSY;
2610 spin_unlock_bh(&efx->filter_lock);
2611
2612 rc = efx_ef10_filter_push(efx, spec,
2613 &table->entry[filter_idx].handle,
2614 false);
2615 if (rc)
2616 failed = true;
2617
2618 spin_lock_bh(&efx->filter_lock);
2619 if (rc) {
2620 kfree(spec);
2621 efx_ef10_filter_set_entry(table, filter_idx, NULL, 0);
2622 } else {
2623 table->entry[filter_idx].spec &=
2624 ~EFX_EF10_FILTER_FLAG_BUSY;
2625 }
2626 }
2627
2628 spin_unlock_bh(&efx->filter_lock);
2629
2630 if (failed)
2631 netif_err(efx, hw, efx->net_dev,
2632 "unable to restore all filters\n");
2633 else
2634 nic_data->must_restore_filters = false;
2635}
2636
2637static void efx_ef10_filter_table_remove(struct efx_nic *efx)
2638{
2639 struct efx_ef10_filter_table *table = efx->filter_state;
2640 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_IN_LEN);
2641 struct efx_filter_spec *spec;
2642 unsigned int filter_idx;
2643 int rc;
2644
2645 for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) {
2646 spec = efx_ef10_filter_entry_spec(table, filter_idx);
2647 if (!spec)
2648 continue;
2649
2650 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
2651 efx_ef10_filter_is_exclusive(spec) ?
2652 MC_CMD_FILTER_OP_IN_OP_REMOVE :
2653 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
2654 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
2655 table->entry[filter_idx].handle);
2656 rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf),
2657 NULL, 0, NULL);
2658
2659 WARN_ON(rc != 0);
2660 kfree(spec);
2661 }
2662
2663 vfree(table->entry);
2664 kfree(table);
2665}
2666
2667static void efx_ef10_filter_sync_rx_mode(struct efx_nic *efx)
2668{
2669 struct efx_ef10_filter_table *table = efx->filter_state;
2670 struct net_device *net_dev = efx->net_dev;
2671 struct efx_filter_spec spec;
2672 bool remove_failed = false;
2673 struct netdev_hw_addr *uc;
2674 struct netdev_hw_addr *mc;
2675 unsigned int filter_idx;
2676 int i, n, rc;
2677
2678 if (!efx_dev_registered(efx))
2679 return;
2680
2681 /* Mark old filters that may need to be removed */
2682 spin_lock_bh(&efx->filter_lock);
2683 n = table->stack_uc_count < 0 ? 1 : table->stack_uc_count;
2684 for (i = 0; i < n; i++) {
2685 filter_idx = table->stack_uc_list[i].id % HUNT_FILTER_TBL_ROWS;
2686 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_STACK_OLD;
2687 }
2688 n = table->stack_mc_count < 0 ? 1 : table->stack_mc_count;
2689 for (i = 0; i < n; i++) {
2690 filter_idx = table->stack_mc_list[i].id % HUNT_FILTER_TBL_ROWS;
2691 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_STACK_OLD;
2692 }
2693 spin_unlock_bh(&efx->filter_lock);
2694
2695 /* Copy/convert the address lists; add the primary station
2696 * address and broadcast address
2697 */
2698 netif_addr_lock_bh(net_dev);
2699 if (net_dev->flags & IFF_PROMISC ||
2700 netdev_uc_count(net_dev) >= EFX_EF10_FILTER_STACK_UC_MAX) {
2701 table->stack_uc_count = -1;
2702 } else {
2703 table->stack_uc_count = 1 + netdev_uc_count(net_dev);
2704 memcpy(table->stack_uc_list[0].addr, net_dev->dev_addr,
2705 ETH_ALEN);
2706 i = 1;
2707 netdev_for_each_uc_addr(uc, net_dev) {
2708 memcpy(table->stack_uc_list[i].addr,
2709 uc->addr, ETH_ALEN);
2710 i++;
2711 }
2712 }
2713 if (net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI) ||
2714 netdev_mc_count(net_dev) >= EFX_EF10_FILTER_STACK_MC_MAX) {
2715 table->stack_mc_count = -1;
2716 } else {
2717 table->stack_mc_count = 1 + netdev_mc_count(net_dev);
2718 eth_broadcast_addr(table->stack_mc_list[0].addr);
2719 i = 1;
2720 netdev_for_each_mc_addr(mc, net_dev) {
2721 memcpy(table->stack_mc_list[i].addr,
2722 mc->addr, ETH_ALEN);
2723 i++;
2724 }
2725 }
2726 netif_addr_unlock_bh(net_dev);
2727
2728 /* Insert/renew unicast filters */
2729 if (table->stack_uc_count >= 0) {
2730 for (i = 0; i < table->stack_uc_count; i++) {
2731 efx_filter_init_rx(&spec, EFX_FILTER_PRI_REQUIRED,
2732 EFX_FILTER_FLAG_RX_RSS |
2733 EFX_FILTER_FLAG_RX_STACK,
2734 0);
2735 efx_filter_set_eth_local(&spec, EFX_FILTER_VID_UNSPEC,
2736 table->stack_uc_list[i].addr);
2737 rc = efx_ef10_filter_insert(efx, &spec, true);
2738 if (rc < 0) {
2739 /* Fall back to unicast-promisc */
2740 while (i--)
2741 efx_ef10_filter_remove_safe(
2742 efx, EFX_FILTER_PRI_REQUIRED,
2743 table->stack_uc_list[i].id);
2744 table->stack_uc_count = -1;
2745 break;
2746 }
2747 table->stack_uc_list[i].id = rc;
2748 }
2749 }
2750 if (table->stack_uc_count < 0) {
2751 efx_filter_init_rx(&spec, EFX_FILTER_PRI_REQUIRED,
2752 EFX_FILTER_FLAG_RX_RSS |
2753 EFX_FILTER_FLAG_RX_STACK,
2754 0);
2755 efx_filter_set_uc_def(&spec);
2756 rc = efx_ef10_filter_insert(efx, &spec, true);
2757 if (rc < 0) {
2758 WARN_ON(1);
2759 table->stack_uc_count = 0;
2760 } else {
2761 table->stack_uc_list[0].id = rc;
2762 }
2763 }
2764
2765 /* Insert/renew multicast filters */
2766 if (table->stack_mc_count >= 0) {
2767 for (i = 0; i < table->stack_mc_count; i++) {
2768 efx_filter_init_rx(&spec, EFX_FILTER_PRI_REQUIRED,
2769 EFX_FILTER_FLAG_RX_RSS |
2770 EFX_FILTER_FLAG_RX_STACK,
2771 0);
2772 efx_filter_set_eth_local(&spec, EFX_FILTER_VID_UNSPEC,
2773 table->stack_mc_list[i].addr);
2774 rc = efx_ef10_filter_insert(efx, &spec, true);
2775 if (rc < 0) {
2776 /* Fall back to multicast-promisc */
2777 while (i--)
2778 efx_ef10_filter_remove_safe(
2779 efx, EFX_FILTER_PRI_REQUIRED,
2780 table->stack_mc_list[i].id);
2781 table->stack_mc_count = -1;
2782 break;
2783 }
2784 table->stack_mc_list[i].id = rc;
2785 }
2786 }
2787 if (table->stack_mc_count < 0) {
2788 efx_filter_init_rx(&spec, EFX_FILTER_PRI_REQUIRED,
2789 EFX_FILTER_FLAG_RX_RSS |
2790 EFX_FILTER_FLAG_RX_STACK,
2791 0);
2792 efx_filter_set_mc_def(&spec);
2793 rc = efx_ef10_filter_insert(efx, &spec, true);
2794 if (rc < 0) {
2795 WARN_ON(1);
2796 table->stack_mc_count = 0;
2797 } else {
2798 table->stack_mc_list[0].id = rc;
2799 }
2800 }
2801
2802 /* Remove filters that weren't renewed. Since nothing else
2803 * changes the STACK_OLD flag or removes these filters, we
2804 * don't need to hold the filter_lock while scanning for
2805 * these filters.
2806 */
2807 for (i = 0; i < HUNT_FILTER_TBL_ROWS; i++) {
2808 if (ACCESS_ONCE(table->entry[i].spec) &
2809 EFX_EF10_FILTER_FLAG_STACK_OLD) {
2810 if (efx_ef10_filter_remove_internal(efx,
2811 EFX_FILTER_PRI_REQUIRED,
2812 i, true) < 0)
2813 remove_failed = true;
2814 }
2815 }
2816 WARN_ON(remove_failed);
2817}
2818
2819static int efx_ef10_mac_reconfigure(struct efx_nic *efx)
2820{
2821 efx_ef10_filter_sync_rx_mode(efx);
2822
2823 return efx_mcdi_set_mac(efx);
2824}
2825
2826#ifdef CONFIG_SFC_MTD
2827
2828struct efx_ef10_nvram_type_info {
2829 u16 type, type_mask;
2830 u8 port;
2831 const char *name;
2832};
2833
2834static const struct efx_ef10_nvram_type_info efx_ef10_nvram_types[] = {
2835 { NVRAM_PARTITION_TYPE_MC_FIRMWARE, 0, 0, "sfc_mcfw" },
2836 { NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP, 0, 0, "sfc_mcfw_backup" },
2837 { NVRAM_PARTITION_TYPE_EXPANSION_ROM, 0, 0, "sfc_exp_rom" },
2838 { NVRAM_PARTITION_TYPE_STATIC_CONFIG, 0, 0, "sfc_static_cfg" },
2839 { NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, 0, 0, "sfc_dynamic_cfg" },
2840 { NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT0, 0, 0, "sfc_exp_rom_cfg" },
2841 { NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT1, 0, 1, "sfc_exp_rom_cfg" },
2842 { NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT2, 0, 2, "sfc_exp_rom_cfg" },
2843 { NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT3, 0, 3, "sfc_exp_rom_cfg" },
2844 { NVRAM_PARTITION_TYPE_PHY_MIN, 0xff, 0, "sfc_phy_fw" },
2845};
2846
2847static int efx_ef10_mtd_probe_partition(struct efx_nic *efx,
2848 struct efx_mcdi_mtd_partition *part,
2849 unsigned int type)
2850{
2851 MCDI_DECLARE_BUF(inbuf, MC_CMD_NVRAM_METADATA_IN_LEN);
2852 MCDI_DECLARE_BUF(outbuf, MC_CMD_NVRAM_METADATA_OUT_LENMAX);
2853 const struct efx_ef10_nvram_type_info *info;
2854 size_t size, erase_size, outlen;
2855 bool protected;
2856 int rc;
2857
2858 for (info = efx_ef10_nvram_types; ; info++) {
2859 if (info ==
2860 efx_ef10_nvram_types + ARRAY_SIZE(efx_ef10_nvram_types))
2861 return -ENODEV;
2862 if ((type & ~info->type_mask) == info->type)
2863 break;
2864 }
2865 if (info->port != efx_port_num(efx))
2866 return -ENODEV;
2867
2868 rc = efx_mcdi_nvram_info(efx, type, &size, &erase_size, &protected);
2869 if (rc)
2870 return rc;
2871 if (protected)
2872 return -ENODEV; /* hide it */
2873
2874 part->nvram_type = type;
2875
2876 MCDI_SET_DWORD(inbuf, NVRAM_METADATA_IN_TYPE, type);
2877 rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_METADATA, inbuf, sizeof(inbuf),
2878 outbuf, sizeof(outbuf), &outlen);
2879 if (rc)
2880 return rc;
2881 if (outlen < MC_CMD_NVRAM_METADATA_OUT_LENMIN)
2882 return -EIO;
2883 if (MCDI_DWORD(outbuf, NVRAM_METADATA_OUT_FLAGS) &
2884 (1 << MC_CMD_NVRAM_METADATA_OUT_SUBTYPE_VALID_LBN))
2885 part->fw_subtype = MCDI_DWORD(outbuf,
2886 NVRAM_METADATA_OUT_SUBTYPE);
2887
2888 part->common.dev_type_name = "EF10 NVRAM manager";
2889 part->common.type_name = info->name;
2890
2891 part->common.mtd.type = MTD_NORFLASH;
2892 part->common.mtd.flags = MTD_CAP_NORFLASH;
2893 part->common.mtd.size = size;
2894 part->common.mtd.erasesize = erase_size;
2895
2896 return 0;
2897}
2898
2899static int efx_ef10_mtd_probe(struct efx_nic *efx)
2900{
2901 MCDI_DECLARE_BUF(outbuf, MC_CMD_NVRAM_PARTITIONS_OUT_LENMAX);
2902 struct efx_mcdi_mtd_partition *parts;
2903 size_t outlen, n_parts_total, i, n_parts;
2904 unsigned int type;
2905 int rc;
2906
2907 ASSERT_RTNL();
2908
2909 BUILD_BUG_ON(MC_CMD_NVRAM_PARTITIONS_IN_LEN != 0);
2910 rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_PARTITIONS, NULL, 0,
2911 outbuf, sizeof(outbuf), &outlen);
2912 if (rc)
2913 return rc;
2914 if (outlen < MC_CMD_NVRAM_PARTITIONS_OUT_LENMIN)
2915 return -EIO;
2916
2917 n_parts_total = MCDI_DWORD(outbuf, NVRAM_PARTITIONS_OUT_NUM_PARTITIONS);
2918 if (n_parts_total >
2919 MCDI_VAR_ARRAY_LEN(outlen, NVRAM_PARTITIONS_OUT_TYPE_ID))
2920 return -EIO;
2921
2922 parts = kcalloc(n_parts_total, sizeof(*parts), GFP_KERNEL);
2923 if (!parts)
2924 return -ENOMEM;
2925
2926 n_parts = 0;
2927 for (i = 0; i < n_parts_total; i++) {
2928 type = MCDI_ARRAY_DWORD(outbuf, NVRAM_PARTITIONS_OUT_TYPE_ID,
2929 i);
2930 rc = efx_ef10_mtd_probe_partition(efx, &parts[n_parts], type);
2931 if (rc == 0)
2932 n_parts++;
2933 else if (rc != -ENODEV)
2934 goto fail;
2935 }
2936
2937 rc = efx_mtd_add(efx, &parts[0].common, n_parts, sizeof(*parts));
2938fail:
2939 if (rc)
2940 kfree(parts);
2941 return rc;
2942}
2943
2944#endif /* CONFIG_SFC_MTD */
2945
2946static void efx_ef10_ptp_write_host_time(struct efx_nic *efx, u32 host_time)
2947{
2948 _efx_writed(efx, cpu_to_le32(host_time), ER_DZ_MC_DB_LWRD);
2949}
2950
2951const struct efx_nic_type efx_hunt_a0_nic_type = {
2952 .mem_map_size = efx_ef10_mem_map_size,
2953 .probe = efx_ef10_probe,
2954 .remove = efx_ef10_remove,
2955 .dimension_resources = efx_ef10_dimension_resources,
2956 .init = efx_ef10_init_nic,
2957 .fini = efx_port_dummy_op_void,
2958 .map_reset_reason = efx_mcdi_map_reset_reason,
2959 .map_reset_flags = efx_ef10_map_reset_flags,
2960 .reset = efx_mcdi_reset,
2961 .probe_port = efx_mcdi_port_probe,
2962 .remove_port = efx_mcdi_port_remove,
2963 .fini_dmaq = efx_ef10_fini_dmaq,
2964 .describe_stats = efx_ef10_describe_stats,
2965 .update_stats = efx_ef10_update_stats,
2966 .start_stats = efx_mcdi_mac_start_stats,
2967 .stop_stats = efx_mcdi_mac_stop_stats,
2968 .set_id_led = efx_mcdi_set_id_led,
2969 .push_irq_moderation = efx_ef10_push_irq_moderation,
2970 .reconfigure_mac = efx_ef10_mac_reconfigure,
2971 .check_mac_fault = efx_mcdi_mac_check_fault,
2972 .reconfigure_port = efx_mcdi_port_reconfigure,
2973 .get_wol = efx_ef10_get_wol,
2974 .set_wol = efx_ef10_set_wol,
2975 .resume_wol = efx_port_dummy_op_void,
2976 /* TODO: test_chip */
2977 .test_nvram = efx_mcdi_nvram_test_all,
2978 .mcdi_request = efx_ef10_mcdi_request,
2979 .mcdi_poll_response = efx_ef10_mcdi_poll_response,
2980 .mcdi_read_response = efx_ef10_mcdi_read_response,
2981 .mcdi_poll_reboot = efx_ef10_mcdi_poll_reboot,
2982 .irq_enable_master = efx_port_dummy_op_void,
2983 .irq_test_generate = efx_ef10_irq_test_generate,
2984 .irq_disable_non_ev = efx_port_dummy_op_void,
2985 .irq_handle_msi = efx_ef10_msi_interrupt,
2986 .irq_handle_legacy = efx_ef10_legacy_interrupt,
2987 .tx_probe = efx_ef10_tx_probe,
2988 .tx_init = efx_ef10_tx_init,
2989 .tx_remove = efx_ef10_tx_remove,
2990 .tx_write = efx_ef10_tx_write,
2991 .rx_push_indir_table = efx_ef10_rx_push_indir_table,
2992 .rx_probe = efx_ef10_rx_probe,
2993 .rx_init = efx_ef10_rx_init,
2994 .rx_remove = efx_ef10_rx_remove,
2995 .rx_write = efx_ef10_rx_write,
2996 .rx_defer_refill = efx_ef10_rx_defer_refill,
2997 .ev_probe = efx_ef10_ev_probe,
2998 .ev_init = efx_ef10_ev_init,
2999 .ev_fini = efx_ef10_ev_fini,
3000 .ev_remove = efx_ef10_ev_remove,
3001 .ev_process = efx_ef10_ev_process,
3002 .ev_read_ack = efx_ef10_ev_read_ack,
3003 .ev_test_generate = efx_ef10_ev_test_generate,
3004 .filter_table_probe = efx_ef10_filter_table_probe,
3005 .filter_table_restore = efx_ef10_filter_table_restore,
3006 .filter_table_remove = efx_ef10_filter_table_remove,
3007 .filter_update_rx_scatter = efx_ef10_filter_update_rx_scatter,
3008 .filter_insert = efx_ef10_filter_insert,
3009 .filter_remove_safe = efx_ef10_filter_remove_safe,
3010 .filter_get_safe = efx_ef10_filter_get_safe,
3011 .filter_clear_rx = efx_ef10_filter_clear_rx,
3012 .filter_count_rx_used = efx_ef10_filter_count_rx_used,
3013 .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit,
3014 .filter_get_rx_ids = efx_ef10_filter_get_rx_ids,
3015#ifdef CONFIG_RFS_ACCEL
3016 .filter_rfs_insert = efx_ef10_filter_rfs_insert,
3017 .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one,
3018#endif
3019#ifdef CONFIG_SFC_MTD
3020 .mtd_probe = efx_ef10_mtd_probe,
3021 .mtd_rename = efx_mcdi_mtd_rename,
3022 .mtd_read = efx_mcdi_mtd_read,
3023 .mtd_erase = efx_mcdi_mtd_erase,
3024 .mtd_write = efx_mcdi_mtd_write,
3025 .mtd_sync = efx_mcdi_mtd_sync,
3026#endif
3027 .ptp_write_host_time = efx_ef10_ptp_write_host_time,
3028
3029 .revision = EFX_REV_HUNT_A0,
3030 .max_dma_mask = DMA_BIT_MASK(ESF_DZ_TX_KER_BUF_ADDR_WIDTH),
3031 .rx_prefix_size = ES_DZ_RX_PREFIX_SIZE,
3032 .rx_hash_offset = ES_DZ_RX_PREFIX_HASH_OFST,
3033 .can_rx_scatter = true,
3034 .always_rx_scatter = true,
3035 .max_interrupt_mode = EFX_INT_MODE_MSIX,
3036 .timer_period_max = 1 << ERF_DD_EVQ_IND_TIMER_VAL_WIDTH,
3037 .offload_features = (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3038 NETIF_F_RXHASH | NETIF_F_NTUPLE),
3039 .mcdi_max_ver = 2,
3040 .max_rx_ip_filters = HUNT_FILTER_TBL_ROWS,
3041};
Linux kernel - Deliverables
This page took 0.133879 seconds and 5 git commands to generate.