1 /****************************************************************************
2 * Driver for Solarflare network controllers and boards
3 * Copyright 2010-2012 Solarflare Communications Inc.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation, incorporated herein by reference.
10 #include <linux/module.h>
11 #include "net_driver.h"
17 #include "mcdi_pcol.h"
18 #include "farch_regs.h"
21 /* Number of longs required to track all the VIs in a VF */
22 #define VI_MASK_LENGTH BITS_TO_LONGS(1 << EFX_VI_SCALE_MAX)
24 /* Maximum number of RX queues supported */
25 #define VF_MAX_RX_QUEUES 63
28 * enum efx_vf_tx_filter_mode - TX MAC filtering behaviour
29 * @VF_TX_FILTER_OFF: Disabled
30 * @VF_TX_FILTER_AUTO: Enabled if MAC address assigned to VF and only
31 * 2 TX queues allowed per VF.
32 * @VF_TX_FILTER_ON: Enabled
34 enum efx_vf_tx_filter_mode
{
41 * struct efx_vf - Back-end resource and protocol state for a PCI VF
42 * @efx: The Efx NIC owning this VF
43 * @pci_rid: The PCI requester ID for this VF
44 * @pci_name: The PCI name (formatted address) of this VF
45 * @index: Index of VF within its port and PF.
46 * @req: VFDI incoming request work item. Incoming USR_EV events are received
47 * by the NAPI handler, but must be handled by executing MCDI requests
49 * @req_addr: VFDI incoming request DMA address (in VF's PCI address space).
50 * @req_type: Expected next incoming (from VF) %VFDI_EV_TYPE member.
51 * @req_seqno: Expected next incoming (from VF) %VFDI_EV_SEQ member.
52 * @msg_seqno: Next %VFDI_EV_SEQ member to reply to VF. Protected by
54 * @busy: VFDI request queued to be processed or being processed. Receiving
55 * a VFDI request when @busy is set is an error condition.
56 * @buf: Incoming VFDI requests are DMA from the VF into this buffer.
57 * @buftbl_base: Buffer table entries for this VF start at this index.
58 * @rx_filtering: Receive filtering has been requested by the VF driver.
59 * @rx_filter_flags: The flags sent in the %VFDI_OP_INSERT_FILTER request.
60 * @rx_filter_qid: VF relative qid for RX filter requested by VF.
61 * @rx_filter_id: Receive MAC filter ID. Only one filter per VF is supported.
62 * @tx_filter_mode: Transmit MAC filtering mode.
63 * @tx_filter_id: Transmit MAC filter ID.
64 * @addr: The MAC address and outer vlan tag of the VF.
65 * @status_addr: VF DMA address of page for &struct vfdi_status updates.
66 * @status_lock: Mutex protecting @msg_seqno, @status_addr, @addr,
67 * @peer_page_addrs and @peer_page_count from simultaneous
68 * updates by the VM and consumption by
69 * efx_siena_sriov_update_vf_addr()
70 * @peer_page_addrs: Pointer to an array of guest pages for local addresses.
71 * @peer_page_count: Number of entries in @peer_page_count.
72 * @evq0_addrs: Array of guest pages backing evq0.
73 * @evq0_count: Number of entries in @evq0_addrs.
74 * @flush_waitq: wait queue used by %VFDI_OP_FINI_ALL_QUEUES handler
75 * to wait for flush completions.
76 * @txq_lock: Mutex for TX queue allocation.
77 * @txq_mask: Mask of initialized transmit queues.
78 * @txq_count: Number of initialized transmit queues.
79 * @rxq_mask: Mask of initialized receive queues.
80 * @rxq_count: Number of initialized receive queues.
81 * @rxq_retry_mask: Mask or receive queues that need to be flushed again
82 * due to flush failure.
83 * @rxq_retry_count: Number of receive queues in @rxq_retry_mask.
84 * @reset_work: Work item to schedule a VF reset.
89 char pci_name
[13]; /* dddd:bb:dd.f */
91 struct work_struct req
;
97 struct efx_buffer buf
;
100 enum efx_filter_flags rx_filter_flags
;
101 unsigned rx_filter_qid
;
103 enum efx_vf_tx_filter_mode tx_filter_mode
;
105 struct vfdi_endpoint addr
;
107 struct mutex status_lock
;
108 u64
*peer_page_addrs
;
109 unsigned peer_page_count
;
110 u64 evq0_addrs
[EFX_MAX_VF_EVQ_SIZE
* sizeof(efx_qword_t
) /
113 wait_queue_head_t flush_waitq
;
114 struct mutex txq_lock
;
115 unsigned long txq_mask
[VI_MASK_LENGTH
];
117 unsigned long rxq_mask
[VI_MASK_LENGTH
];
119 unsigned long rxq_retry_mask
[VI_MASK_LENGTH
];
120 atomic_t rxq_retry_count
;
121 struct work_struct reset_work
;
124 struct efx_memcpy_req
{
125 unsigned int from_rid
;
134 * struct efx_local_addr - A MAC address on the vswitch without a VF.
136 * Siena does not have a switch, so VFs can't transmit data to each
137 * other. Instead the VFs must be made aware of the local addresses
138 * on the vswitch, so that they can arrange for an alternative
139 * software datapath to be used.
141 * @link: List head for insertion into efx->local_addr_list.
142 * @addr: Ethernet address
144 struct efx_local_addr
{
145 struct list_head link
;
150 * struct efx_endpoint_page - Page of vfdi_endpoint structures
152 * @link: List head for insertion into efx->local_page_list.
153 * @ptr: Pointer to page.
154 * @addr: DMA address of page.
156 struct efx_endpoint_page
{
157 struct list_head link
;
162 /* Buffer table entries are reserved txq0,rxq0,evq0,txq1,rxq1,evq1 */
163 #define EFX_BUFTBL_TXQ_BASE(_vf, _qid) \
164 ((_vf)->buftbl_base + EFX_VF_BUFTBL_PER_VI * (_qid))
165 #define EFX_BUFTBL_RXQ_BASE(_vf, _qid) \
166 (EFX_BUFTBL_TXQ_BASE(_vf, _qid) + \
167 (EFX_MAX_DMAQ_SIZE * sizeof(efx_qword_t) / EFX_BUF_SIZE))
168 #define EFX_BUFTBL_EVQ_BASE(_vf, _qid) \
169 (EFX_BUFTBL_TXQ_BASE(_vf, _qid) + \
170 (2 * EFX_MAX_DMAQ_SIZE * sizeof(efx_qword_t) / EFX_BUF_SIZE))
172 #define EFX_FIELD_MASK(_field) \
173 ((1 << _field ## _WIDTH) - 1)
175 /* VFs can only use this many transmit channels */
176 static unsigned int vf_max_tx_channels
= 2;
177 module_param(vf_max_tx_channels
, uint
, 0444);
178 MODULE_PARM_DESC(vf_max_tx_channels
,
179 "Limit the number of TX channels VFs can use");
181 static int max_vfs
= -1;
182 module_param(max_vfs
, int, 0444);
183 MODULE_PARM_DESC(max_vfs
,
184 "Reduce the number of VFs initialized by the driver");
186 /* Workqueue used by VFDI communication. We can't use the global
187 * workqueue because it may be running the VF driver's probe()
188 * routine, which will be blocked there waiting for a VFDI response.
190 static struct workqueue_struct
*vfdi_workqueue
;
192 static unsigned abs_index(struct efx_vf
*vf
, unsigned index
)
194 return EFX_VI_BASE
+ vf
->index
* efx_vf_size(vf
->efx
) + index
;
197 static int efx_siena_sriov_cmd(struct efx_nic
*efx
, bool enable
,
198 unsigned *vi_scale_out
, unsigned *vf_total_out
)
200 MCDI_DECLARE_BUF(inbuf
, MC_CMD_SRIOV_IN_LEN
);
201 MCDI_DECLARE_BUF(outbuf
, MC_CMD_SRIOV_OUT_LEN
);
202 unsigned vi_scale
, vf_total
;
206 MCDI_SET_DWORD(inbuf
, SRIOV_IN_ENABLE
, enable
? 1 : 0);
207 MCDI_SET_DWORD(inbuf
, SRIOV_IN_VI_BASE
, EFX_VI_BASE
);
208 MCDI_SET_DWORD(inbuf
, SRIOV_IN_VF_COUNT
, efx
->vf_count
);
210 rc
= efx_mcdi_rpc(efx
, MC_CMD_SRIOV
, inbuf
, MC_CMD_SRIOV_IN_LEN
,
211 outbuf
, MC_CMD_SRIOV_OUT_LEN
, &outlen
);
214 if (outlen
< MC_CMD_SRIOV_OUT_LEN
)
217 vf_total
= MCDI_DWORD(outbuf
, SRIOV_OUT_VF_TOTAL
);
218 vi_scale
= MCDI_DWORD(outbuf
, SRIOV_OUT_VI_SCALE
);
219 if (vi_scale
> EFX_VI_SCALE_MAX
)
223 *vi_scale_out
= vi_scale
;
225 *vf_total_out
= vf_total
;
230 static void efx_siena_sriov_usrev(struct efx_nic
*efx
, bool enabled
)
232 struct siena_nic_data
*nic_data
= efx
->nic_data
;
235 EFX_POPULATE_OWORD_2(reg
,
236 FRF_CZ_USREV_DIS
, enabled
? 0 : 1,
237 FRF_CZ_DFLT_EVQ
, nic_data
->vfdi_channel
->channel
);
238 efx_writeo(efx
, ®
, FR_CZ_USR_EV_CFG
);
241 static int efx_siena_sriov_memcpy(struct efx_nic
*efx
,
242 struct efx_memcpy_req
*req
,
245 MCDI_DECLARE_BUF(inbuf
, MCDI_CTL_SDU_LEN_MAX_V1
);
246 MCDI_DECLARE_STRUCT_PTR(record
);
247 unsigned int index
, used
;
252 mb(); /* Finish writing source/reading dest before DMA starts */
254 if (WARN_ON(count
> MC_CMD_MEMCPY_IN_RECORD_MAXNUM
))
256 used
= MC_CMD_MEMCPY_IN_LEN(count
);
258 for (index
= 0; index
< count
; index
++) {
259 record
= MCDI_ARRAY_STRUCT_PTR(inbuf
, MEMCPY_IN_RECORD
, index
);
260 MCDI_SET_DWORD(record
, MEMCPY_RECORD_TYPEDEF_NUM_RECORDS
,
262 MCDI_SET_DWORD(record
, MEMCPY_RECORD_TYPEDEF_TO_RID
,
264 MCDI_SET_QWORD(record
, MEMCPY_RECORD_TYPEDEF_TO_ADDR
,
266 if (req
->from_buf
== NULL
) {
267 from_rid
= req
->from_rid
;
268 from_addr
= req
->from_addr
;
270 if (WARN_ON(used
+ req
->length
>
271 MCDI_CTL_SDU_LEN_MAX_V1
)) {
276 from_rid
= MC_CMD_MEMCPY_RECORD_TYPEDEF_RID_INLINE
;
278 memcpy(_MCDI_PTR(inbuf
, used
), req
->from_buf
,
283 MCDI_SET_DWORD(record
, MEMCPY_RECORD_TYPEDEF_FROM_RID
, from_rid
);
284 MCDI_SET_QWORD(record
, MEMCPY_RECORD_TYPEDEF_FROM_ADDR
,
286 MCDI_SET_DWORD(record
, MEMCPY_RECORD_TYPEDEF_LENGTH
,
292 rc
= efx_mcdi_rpc(efx
, MC_CMD_MEMCPY
, inbuf
, used
, NULL
, 0, NULL
);
294 mb(); /* Don't write source/read dest before DMA is complete */
299 /* The TX filter is entirely controlled by this driver, and is modified
300 * underneath the feet of the VF
302 static void efx_siena_sriov_reset_tx_filter(struct efx_vf
*vf
)
304 struct efx_nic
*efx
= vf
->efx
;
305 struct efx_filter_spec filter
;
309 if (vf
->tx_filter_id
!= -1) {
310 efx_filter_remove_id_safe(efx
, EFX_FILTER_PRI_REQUIRED
,
312 netif_dbg(efx
, hw
, efx
->net_dev
, "Removed vf %s tx filter %d\n",
313 vf
->pci_name
, vf
->tx_filter_id
);
314 vf
->tx_filter_id
= -1;
317 if (is_zero_ether_addr(vf
->addr
.mac_addr
))
320 /* Turn on TX filtering automatically if not explicitly
321 * enabled or disabled.
323 if (vf
->tx_filter_mode
== VF_TX_FILTER_AUTO
&& vf_max_tx_channels
<= 2)
324 vf
->tx_filter_mode
= VF_TX_FILTER_ON
;
326 vlan
= ntohs(vf
->addr
.tci
) & VLAN_VID_MASK
;
327 efx_filter_init_tx(&filter
, abs_index(vf
, 0));
328 rc
= efx_filter_set_eth_local(&filter
,
329 vlan
? vlan
: EFX_FILTER_VID_UNSPEC
,
333 rc
= efx_filter_insert_filter(efx
, &filter
, true);
335 netif_warn(efx
, hw
, efx
->net_dev
,
336 "Unable to migrate tx filter for vf %s\n",
339 netif_dbg(efx
, hw
, efx
->net_dev
, "Inserted vf %s tx filter %d\n",
341 vf
->tx_filter_id
= rc
;
345 /* The RX filter is managed here on behalf of the VF driver */
346 static void efx_siena_sriov_reset_rx_filter(struct efx_vf
*vf
)
348 struct efx_nic
*efx
= vf
->efx
;
349 struct efx_filter_spec filter
;
353 if (vf
->rx_filter_id
!= -1) {
354 efx_filter_remove_id_safe(efx
, EFX_FILTER_PRI_REQUIRED
,
356 netif_dbg(efx
, hw
, efx
->net_dev
, "Removed vf %s rx filter %d\n",
357 vf
->pci_name
, vf
->rx_filter_id
);
358 vf
->rx_filter_id
= -1;
361 if (!vf
->rx_filtering
|| is_zero_ether_addr(vf
->addr
.mac_addr
))
364 vlan
= ntohs(vf
->addr
.tci
) & VLAN_VID_MASK
;
365 efx_filter_init_rx(&filter
, EFX_FILTER_PRI_REQUIRED
,
367 abs_index(vf
, vf
->rx_filter_qid
));
368 rc
= efx_filter_set_eth_local(&filter
,
369 vlan
? vlan
: EFX_FILTER_VID_UNSPEC
,
373 rc
= efx_filter_insert_filter(efx
, &filter
, true);
375 netif_warn(efx
, hw
, efx
->net_dev
,
376 "Unable to insert rx filter for vf %s\n",
379 netif_dbg(efx
, hw
, efx
->net_dev
, "Inserted vf %s rx filter %d\n",
381 vf
->rx_filter_id
= rc
;
385 static void __efx_siena_sriov_update_vf_addr(struct efx_vf
*vf
)
387 struct efx_nic
*efx
= vf
->efx
;
388 struct siena_nic_data
*nic_data
= efx
->nic_data
;
390 efx_siena_sriov_reset_tx_filter(vf
);
391 efx_siena_sriov_reset_rx_filter(vf
);
392 queue_work(vfdi_workqueue
, &nic_data
->peer_work
);
395 /* Push the peer list to this VF. The caller must hold status_lock to interlock
396 * with VFDI requests, and they must be serialised against manipulation of
397 * local_page_list, either by acquiring local_lock or by running from
398 * efx_siena_sriov_peer_work()
400 static void __efx_siena_sriov_push_vf_status(struct efx_vf
*vf
)
402 struct efx_nic
*efx
= vf
->efx
;
403 struct siena_nic_data
*nic_data
= efx
->nic_data
;
404 struct vfdi_status
*status
= nic_data
->vfdi_status
.addr
;
405 struct efx_memcpy_req copy
[4];
406 struct efx_endpoint_page
*epp
;
407 unsigned int pos
, count
;
408 unsigned data_offset
;
411 WARN_ON(!mutex_is_locked(&vf
->status_lock
));
412 WARN_ON(!vf
->status_addr
);
414 status
->local
= vf
->addr
;
415 status
->generation_end
= ++status
->generation_start
;
417 memset(copy
, '\0', sizeof(copy
));
418 /* Write generation_start */
419 copy
[0].from_buf
= &status
->generation_start
;
420 copy
[0].to_rid
= vf
->pci_rid
;
421 copy
[0].to_addr
= vf
->status_addr
+ offsetof(struct vfdi_status
,
423 copy
[0].length
= sizeof(status
->generation_start
);
424 /* DMA the rest of the structure (excluding the generations). This
425 * assumes that the non-generation portion of vfdi_status is in
426 * one chunk starting at the version member.
428 data_offset
= offsetof(struct vfdi_status
, version
);
429 copy
[1].from_rid
= efx
->pci_dev
->devfn
;
430 copy
[1].from_addr
= nic_data
->vfdi_status
.dma_addr
+ data_offset
;
431 copy
[1].to_rid
= vf
->pci_rid
;
432 copy
[1].to_addr
= vf
->status_addr
+ data_offset
;
433 copy
[1].length
= status
->length
- data_offset
;
435 /* Copy the peer pages */
438 list_for_each_entry(epp
, &nic_data
->local_page_list
, link
) {
439 if (count
== vf
->peer_page_count
) {
440 /* The VF driver will know they need to provide more
441 * pages because peer_addr_count is too large.
445 copy
[pos
].from_buf
= NULL
;
446 copy
[pos
].from_rid
= efx
->pci_dev
->devfn
;
447 copy
[pos
].from_addr
= epp
->addr
;
448 copy
[pos
].to_rid
= vf
->pci_rid
;
449 copy
[pos
].to_addr
= vf
->peer_page_addrs
[count
];
450 copy
[pos
].length
= EFX_PAGE_SIZE
;
452 if (++pos
== ARRAY_SIZE(copy
)) {
453 efx_siena_sriov_memcpy(efx
, copy
, ARRAY_SIZE(copy
));
459 /* Write generation_end */
460 copy
[pos
].from_buf
= &status
->generation_end
;
461 copy
[pos
].to_rid
= vf
->pci_rid
;
462 copy
[pos
].to_addr
= vf
->status_addr
+ offsetof(struct vfdi_status
,
464 copy
[pos
].length
= sizeof(status
->generation_end
);
465 efx_siena_sriov_memcpy(efx
, copy
, pos
+ 1);
467 /* Notify the guest */
468 EFX_POPULATE_QWORD_3(event
,
469 FSF_AZ_EV_CODE
, FSE_CZ_EV_CODE_USER_EV
,
470 VFDI_EV_SEQ
, (vf
->msg_seqno
& 0xff),
471 VFDI_EV_TYPE
, VFDI_EV_TYPE_STATUS
);
473 efx_farch_generate_event(efx
,
474 EFX_VI_BASE
+ vf
->index
* efx_vf_size(efx
),
478 static void efx_siena_sriov_bufs(struct efx_nic
*efx
, unsigned offset
,
479 u64
*addr
, unsigned count
)
484 for (pos
= 0; pos
< count
; ++pos
) {
485 EFX_POPULATE_QWORD_3(buf
,
486 FRF_AZ_BUF_ADR_REGION
, 0,
488 addr
? addr
[pos
] >> 12 : 0,
489 FRF_AZ_BUF_OWNER_ID_FBUF
, 0);
490 efx_sram_writeq(efx
, efx
->membase
+ FR_BZ_BUF_FULL_TBL
,
495 static bool bad_vf_index(struct efx_nic
*efx
, unsigned index
)
497 return index
>= efx_vf_size(efx
);
500 static bool bad_buf_count(unsigned buf_count
, unsigned max_entry_count
)
502 unsigned max_buf_count
= max_entry_count
*
503 sizeof(efx_qword_t
) / EFX_BUF_SIZE
;
505 return ((buf_count
& (buf_count
- 1)) || buf_count
> max_buf_count
);
508 /* Check that VI specified by per-port index belongs to a VF.
509 * Optionally set VF index and VI index within the VF.
511 static bool map_vi_index(struct efx_nic
*efx
, unsigned abs_index
,
512 struct efx_vf
**vf_out
, unsigned *rel_index_out
)
516 if (abs_index
< EFX_VI_BASE
)
518 vf_i
= (abs_index
- EFX_VI_BASE
) / efx_vf_size(efx
);
519 if (vf_i
>= efx
->vf_init_count
)
523 *vf_out
= efx
->vf
+ vf_i
;
525 *rel_index_out
= abs_index
% efx_vf_size(efx
);
529 static int efx_vfdi_init_evq(struct efx_vf
*vf
)
531 struct efx_nic
*efx
= vf
->efx
;
532 struct vfdi_req
*req
= vf
->buf
.addr
;
533 unsigned vf_evq
= req
->u
.init_evq
.index
;
534 unsigned buf_count
= req
->u
.init_evq
.buf_count
;
535 unsigned abs_evq
= abs_index(vf
, vf_evq
);
536 unsigned buftbl
= EFX_BUFTBL_EVQ_BASE(vf
, vf_evq
);
539 if (bad_vf_index(efx
, vf_evq
) ||
540 bad_buf_count(buf_count
, EFX_MAX_VF_EVQ_SIZE
)) {
542 netif_err(efx
, hw
, efx
->net_dev
,
543 "ERROR: Invalid INIT_EVQ from %s: evq %d bufs %d\n",
544 vf
->pci_name
, vf_evq
, buf_count
);
545 return VFDI_RC_EINVAL
;
548 efx_siena_sriov_bufs(efx
, buftbl
, req
->u
.init_evq
.addr
, buf_count
);
550 EFX_POPULATE_OWORD_3(reg
,
551 FRF_CZ_TIMER_Q_EN
, 1,
552 FRF_CZ_HOST_NOTIFY_MODE
, 0,
553 FRF_CZ_TIMER_MODE
, FFE_CZ_TIMER_MODE_DIS
);
554 efx_writeo_table(efx
, ®
, FR_BZ_TIMER_TBL
, abs_evq
);
555 EFX_POPULATE_OWORD_3(reg
,
557 FRF_AZ_EVQ_SIZE
, __ffs(buf_count
),
558 FRF_AZ_EVQ_BUF_BASE_ID
, buftbl
);
559 efx_writeo_table(efx
, ®
, FR_BZ_EVQ_PTR_TBL
, abs_evq
);
562 memcpy(vf
->evq0_addrs
, req
->u
.init_evq
.addr
,
563 buf_count
* sizeof(u64
));
564 vf
->evq0_count
= buf_count
;
567 return VFDI_RC_SUCCESS
;
570 static int efx_vfdi_init_rxq(struct efx_vf
*vf
)
572 struct efx_nic
*efx
= vf
->efx
;
573 struct vfdi_req
*req
= vf
->buf
.addr
;
574 unsigned vf_rxq
= req
->u
.init_rxq
.index
;
575 unsigned vf_evq
= req
->u
.init_rxq
.evq
;
576 unsigned buf_count
= req
->u
.init_rxq
.buf_count
;
577 unsigned buftbl
= EFX_BUFTBL_RXQ_BASE(vf
, vf_rxq
);
581 if (bad_vf_index(efx
, vf_evq
) || bad_vf_index(efx
, vf_rxq
) ||
582 vf_rxq
>= VF_MAX_RX_QUEUES
||
583 bad_buf_count(buf_count
, EFX_MAX_DMAQ_SIZE
)) {
585 netif_err(efx
, hw
, efx
->net_dev
,
586 "ERROR: Invalid INIT_RXQ from %s: rxq %d evq %d "
587 "buf_count %d\n", vf
->pci_name
, vf_rxq
,
589 return VFDI_RC_EINVAL
;
591 if (__test_and_set_bit(req
->u
.init_rxq
.index
, vf
->rxq_mask
))
593 efx_siena_sriov_bufs(efx
, buftbl
, req
->u
.init_rxq
.addr
, buf_count
);
595 label
= req
->u
.init_rxq
.label
& EFX_FIELD_MASK(FRF_AZ_RX_DESCQ_LABEL
);
596 EFX_POPULATE_OWORD_6(reg
,
597 FRF_AZ_RX_DESCQ_BUF_BASE_ID
, buftbl
,
598 FRF_AZ_RX_DESCQ_EVQ_ID
, abs_index(vf
, vf_evq
),
599 FRF_AZ_RX_DESCQ_LABEL
, label
,
600 FRF_AZ_RX_DESCQ_SIZE
, __ffs(buf_count
),
601 FRF_AZ_RX_DESCQ_JUMBO
,
602 !!(req
->u
.init_rxq
.flags
&
603 VFDI_RXQ_FLAG_SCATTER_EN
),
604 FRF_AZ_RX_DESCQ_EN
, 1);
605 efx_writeo_table(efx
, ®
, FR_BZ_RX_DESC_PTR_TBL
,
606 abs_index(vf
, vf_rxq
));
608 return VFDI_RC_SUCCESS
;
611 static int efx_vfdi_init_txq(struct efx_vf
*vf
)
613 struct efx_nic
*efx
= vf
->efx
;
614 struct vfdi_req
*req
= vf
->buf
.addr
;
615 unsigned vf_txq
= req
->u
.init_txq
.index
;
616 unsigned vf_evq
= req
->u
.init_txq
.evq
;
617 unsigned buf_count
= req
->u
.init_txq
.buf_count
;
618 unsigned buftbl
= EFX_BUFTBL_TXQ_BASE(vf
, vf_txq
);
619 unsigned label
, eth_filt_en
;
622 if (bad_vf_index(efx
, vf_evq
) || bad_vf_index(efx
, vf_txq
) ||
623 vf_txq
>= vf_max_tx_channels
||
624 bad_buf_count(buf_count
, EFX_MAX_DMAQ_SIZE
)) {
626 netif_err(efx
, hw
, efx
->net_dev
,
627 "ERROR: Invalid INIT_TXQ from %s: txq %d evq %d "
628 "buf_count %d\n", vf
->pci_name
, vf_txq
,
630 return VFDI_RC_EINVAL
;
633 mutex_lock(&vf
->txq_lock
);
634 if (__test_and_set_bit(req
->u
.init_txq
.index
, vf
->txq_mask
))
636 mutex_unlock(&vf
->txq_lock
);
637 efx_siena_sriov_bufs(efx
, buftbl
, req
->u
.init_txq
.addr
, buf_count
);
639 eth_filt_en
= vf
->tx_filter_mode
== VF_TX_FILTER_ON
;
641 label
= req
->u
.init_txq
.label
& EFX_FIELD_MASK(FRF_AZ_TX_DESCQ_LABEL
);
642 EFX_POPULATE_OWORD_8(reg
,
643 FRF_CZ_TX_DPT_Q_MASK_WIDTH
, min(efx
->vi_scale
, 1U),
644 FRF_CZ_TX_DPT_ETH_FILT_EN
, eth_filt_en
,
645 FRF_AZ_TX_DESCQ_EN
, 1,
646 FRF_AZ_TX_DESCQ_BUF_BASE_ID
, buftbl
,
647 FRF_AZ_TX_DESCQ_EVQ_ID
, abs_index(vf
, vf_evq
),
648 FRF_AZ_TX_DESCQ_LABEL
, label
,
649 FRF_AZ_TX_DESCQ_SIZE
, __ffs(buf_count
),
650 FRF_BZ_TX_NON_IP_DROP_DIS
, 1);
651 efx_writeo_table(efx
, ®
, FR_BZ_TX_DESC_PTR_TBL
,
652 abs_index(vf
, vf_txq
));
654 return VFDI_RC_SUCCESS
;
657 /* Returns true when efx_vfdi_fini_all_queues should wake */
658 static bool efx_vfdi_flush_wake(struct efx_vf
*vf
)
660 /* Ensure that all updates are visible to efx_vfdi_fini_all_queues() */
663 return (!vf
->txq_count
&& !vf
->rxq_count
) ||
664 atomic_read(&vf
->rxq_retry_count
);
667 static void efx_vfdi_flush_clear(struct efx_vf
*vf
)
669 memset(vf
->txq_mask
, 0, sizeof(vf
->txq_mask
));
671 memset(vf
->rxq_mask
, 0, sizeof(vf
->rxq_mask
));
673 memset(vf
->rxq_retry_mask
, 0, sizeof(vf
->rxq_retry_mask
));
674 atomic_set(&vf
->rxq_retry_count
, 0);
677 static int efx_vfdi_fini_all_queues(struct efx_vf
*vf
)
679 struct efx_nic
*efx
= vf
->efx
;
681 unsigned count
= efx_vf_size(efx
);
682 unsigned vf_offset
= EFX_VI_BASE
+ vf
->index
* efx_vf_size(efx
);
683 unsigned timeout
= HZ
;
684 unsigned index
, rxqs_count
;
685 MCDI_DECLARE_BUF(inbuf
, MC_CMD_FLUSH_RX_QUEUES_IN_LENMAX
);
688 BUILD_BUG_ON(VF_MAX_RX_QUEUES
>
689 MC_CMD_FLUSH_RX_QUEUES_IN_QID_OFST_MAXNUM
);
692 siena_prepare_flush(efx
);
695 /* Flush all the initialized queues */
697 for (index
= 0; index
< count
; ++index
) {
698 if (test_bit(index
, vf
->txq_mask
)) {
699 EFX_POPULATE_OWORD_2(reg
,
700 FRF_AZ_TX_FLUSH_DESCQ_CMD
, 1,
701 FRF_AZ_TX_FLUSH_DESCQ
,
703 efx_writeo(efx
, ®
, FR_AZ_TX_FLUSH_DESCQ
);
705 if (test_bit(index
, vf
->rxq_mask
)) {
706 MCDI_SET_ARRAY_DWORD(
707 inbuf
, FLUSH_RX_QUEUES_IN_QID_OFST
,
708 rxqs_count
, vf_offset
+ index
);
713 atomic_set(&vf
->rxq_retry_count
, 0);
714 while (timeout
&& (vf
->rxq_count
|| vf
->txq_count
)) {
715 rc
= efx_mcdi_rpc(efx
, MC_CMD_FLUSH_RX_QUEUES
, inbuf
,
716 MC_CMD_FLUSH_RX_QUEUES_IN_LEN(rxqs_count
),
720 timeout
= wait_event_timeout(vf
->flush_waitq
,
721 efx_vfdi_flush_wake(vf
),
724 for (index
= 0; index
< count
; ++index
) {
725 if (test_and_clear_bit(index
, vf
->rxq_retry_mask
)) {
726 atomic_dec(&vf
->rxq_retry_count
);
727 MCDI_SET_ARRAY_DWORD(
728 inbuf
, FLUSH_RX_QUEUES_IN_QID_OFST
,
729 rxqs_count
, vf_offset
+ index
);
736 siena_finish_flush(efx
);
739 /* Irrespective of success/failure, fini the queues */
741 for (index
= 0; index
< count
; ++index
) {
742 efx_writeo_table(efx
, ®
, FR_BZ_RX_DESC_PTR_TBL
,
744 efx_writeo_table(efx
, ®
, FR_BZ_TX_DESC_PTR_TBL
,
746 efx_writeo_table(efx
, ®
, FR_BZ_EVQ_PTR_TBL
,
748 efx_writeo_table(efx
, ®
, FR_BZ_TIMER_TBL
,
751 efx_siena_sriov_bufs(efx
, vf
->buftbl_base
, NULL
,
752 EFX_VF_BUFTBL_PER_VI
* efx_vf_size(efx
));
753 efx_vfdi_flush_clear(vf
);
757 return timeout
? 0 : VFDI_RC_ETIMEDOUT
;
760 static int efx_vfdi_insert_filter(struct efx_vf
*vf
)
762 struct efx_nic
*efx
= vf
->efx
;
763 struct siena_nic_data
*nic_data
= efx
->nic_data
;
764 struct vfdi_req
*req
= vf
->buf
.addr
;
765 unsigned vf_rxq
= req
->u
.mac_filter
.rxq
;
768 if (bad_vf_index(efx
, vf_rxq
) || vf
->rx_filtering
) {
770 netif_err(efx
, hw
, efx
->net_dev
,
771 "ERROR: Invalid INSERT_FILTER from %s: rxq %d "
772 "flags 0x%x\n", vf
->pci_name
, vf_rxq
,
773 req
->u
.mac_filter
.flags
);
774 return VFDI_RC_EINVAL
;
778 if (req
->u
.mac_filter
.flags
& VFDI_MAC_FILTER_FLAG_RSS
)
779 flags
|= EFX_FILTER_FLAG_RX_RSS
;
780 if (req
->u
.mac_filter
.flags
& VFDI_MAC_FILTER_FLAG_SCATTER
)
781 flags
|= EFX_FILTER_FLAG_RX_SCATTER
;
782 vf
->rx_filter_flags
= flags
;
783 vf
->rx_filter_qid
= vf_rxq
;
784 vf
->rx_filtering
= true;
786 efx_siena_sriov_reset_rx_filter(vf
);
787 queue_work(vfdi_workqueue
, &nic_data
->peer_work
);
789 return VFDI_RC_SUCCESS
;
792 static int efx_vfdi_remove_all_filters(struct efx_vf
*vf
)
794 struct efx_nic
*efx
= vf
->efx
;
795 struct siena_nic_data
*nic_data
= efx
->nic_data
;
797 vf
->rx_filtering
= false;
798 efx_siena_sriov_reset_rx_filter(vf
);
799 queue_work(vfdi_workqueue
, &nic_data
->peer_work
);
801 return VFDI_RC_SUCCESS
;
804 static int efx_vfdi_set_status_page(struct efx_vf
*vf
)
806 struct efx_nic
*efx
= vf
->efx
;
807 struct siena_nic_data
*nic_data
= efx
->nic_data
;
808 struct vfdi_req
*req
= vf
->buf
.addr
;
809 u64 page_count
= req
->u
.set_status_page
.peer_page_count
;
812 offsetof(struct vfdi_req
, u
.set_status_page
.peer_page_addr
[0]))
813 / sizeof(req
->u
.set_status_page
.peer_page_addr
[0]);
815 if (!req
->u
.set_status_page
.dma_addr
|| page_count
> max_page_count
) {
817 netif_err(efx
, hw
, efx
->net_dev
,
818 "ERROR: Invalid SET_STATUS_PAGE from %s\n",
820 return VFDI_RC_EINVAL
;
823 mutex_lock(&nic_data
->local_lock
);
824 mutex_lock(&vf
->status_lock
);
825 vf
->status_addr
= req
->u
.set_status_page
.dma_addr
;
827 kfree(vf
->peer_page_addrs
);
828 vf
->peer_page_addrs
= NULL
;
829 vf
->peer_page_count
= 0;
832 vf
->peer_page_addrs
= kcalloc(page_count
, sizeof(u64
),
834 if (vf
->peer_page_addrs
) {
835 memcpy(vf
->peer_page_addrs
,
836 req
->u
.set_status_page
.peer_page_addr
,
837 page_count
* sizeof(u64
));
838 vf
->peer_page_count
= page_count
;
842 __efx_siena_sriov_push_vf_status(vf
);
843 mutex_unlock(&vf
->status_lock
);
844 mutex_unlock(&nic_data
->local_lock
);
846 return VFDI_RC_SUCCESS
;
849 static int efx_vfdi_clear_status_page(struct efx_vf
*vf
)
851 mutex_lock(&vf
->status_lock
);
853 mutex_unlock(&vf
->status_lock
);
855 return VFDI_RC_SUCCESS
;
858 typedef int (*efx_vfdi_op_t
)(struct efx_vf
*vf
);
860 static const efx_vfdi_op_t vfdi_ops
[VFDI_OP_LIMIT
] = {
861 [VFDI_OP_INIT_EVQ
] = efx_vfdi_init_evq
,
862 [VFDI_OP_INIT_TXQ
] = efx_vfdi_init_txq
,
863 [VFDI_OP_INIT_RXQ
] = efx_vfdi_init_rxq
,
864 [VFDI_OP_FINI_ALL_QUEUES
] = efx_vfdi_fini_all_queues
,
865 [VFDI_OP_INSERT_FILTER
] = efx_vfdi_insert_filter
,
866 [VFDI_OP_REMOVE_ALL_FILTERS
] = efx_vfdi_remove_all_filters
,
867 [VFDI_OP_SET_STATUS_PAGE
] = efx_vfdi_set_status_page
,
868 [VFDI_OP_CLEAR_STATUS_PAGE
] = efx_vfdi_clear_status_page
,
871 static void efx_siena_sriov_vfdi(struct work_struct
*work
)
873 struct efx_vf
*vf
= container_of(work
, struct efx_vf
, req
);
874 struct efx_nic
*efx
= vf
->efx
;
875 struct vfdi_req
*req
= vf
->buf
.addr
;
876 struct efx_memcpy_req copy
[2];
879 /* Copy this page into the local address space */
880 memset(copy
, '\0', sizeof(copy
));
881 copy
[0].from_rid
= vf
->pci_rid
;
882 copy
[0].from_addr
= vf
->req_addr
;
883 copy
[0].to_rid
= efx
->pci_dev
->devfn
;
884 copy
[0].to_addr
= vf
->buf
.dma_addr
;
885 copy
[0].length
= EFX_PAGE_SIZE
;
886 rc
= efx_siena_sriov_memcpy(efx
, copy
, 1);
888 /* If we can't get the request, we can't reply to the caller */
890 netif_err(efx
, hw
, efx
->net_dev
,
891 "ERROR: Unable to fetch VFDI request from %s rc %d\n",
897 if (req
->op
< VFDI_OP_LIMIT
&& vfdi_ops
[req
->op
] != NULL
) {
898 rc
= vfdi_ops
[req
->op
](vf
);
900 netif_dbg(efx
, hw
, efx
->net_dev
,
901 "vfdi request %d from %s ok\n",
902 req
->op
, vf
->pci_name
);
905 netif_dbg(efx
, hw
, efx
->net_dev
,
906 "ERROR: Unrecognised request %d from VF %s addr "
907 "%llx\n", req
->op
, vf
->pci_name
,
908 (unsigned long long)vf
->req_addr
);
909 rc
= VFDI_RC_EOPNOTSUPP
;
912 /* Allow subsequent VF requests */
916 /* Respond to the request */
918 req
->op
= VFDI_OP_RESPONSE
;
920 memset(copy
, '\0', sizeof(copy
));
921 copy
[0].from_buf
= &req
->rc
;
922 copy
[0].to_rid
= vf
->pci_rid
;
923 copy
[0].to_addr
= vf
->req_addr
+ offsetof(struct vfdi_req
, rc
);
924 copy
[0].length
= sizeof(req
->rc
);
925 copy
[1].from_buf
= &req
->op
;
926 copy
[1].to_rid
= vf
->pci_rid
;
927 copy
[1].to_addr
= vf
->req_addr
+ offsetof(struct vfdi_req
, op
);
928 copy
[1].length
= sizeof(req
->op
);
930 (void)efx_siena_sriov_memcpy(efx
, copy
, ARRAY_SIZE(copy
));
935 /* After a reset the event queues inside the guests no longer exist. Fill the
936 * event ring in guest memory with VFDI reset events, then (re-initialise) the
937 * event queue to raise an interrupt. The guest driver will then recover.
939 static void efx_siena_sriov_reset_vf(struct efx_vf
*vf
,
940 struct efx_buffer
*buffer
)
942 struct efx_nic
*efx
= vf
->efx
;
943 struct efx_memcpy_req copy_req
[4];
945 unsigned int pos
, count
, k
, buftbl
, abs_evq
;
950 BUG_ON(buffer
->len
!= EFX_PAGE_SIZE
);
954 BUG_ON(vf
->evq0_count
& (vf
->evq0_count
- 1));
956 mutex_lock(&vf
->status_lock
);
957 EFX_POPULATE_QWORD_3(event
,
958 FSF_AZ_EV_CODE
, FSE_CZ_EV_CODE_USER_EV
,
959 VFDI_EV_SEQ
, vf
->msg_seqno
,
960 VFDI_EV_TYPE
, VFDI_EV_TYPE_RESET
);
962 for (pos
= 0; pos
< EFX_PAGE_SIZE
; pos
+= sizeof(event
))
963 memcpy(buffer
->addr
+ pos
, &event
, sizeof(event
));
965 for (pos
= 0; pos
< vf
->evq0_count
; pos
+= count
) {
966 count
= min_t(unsigned, vf
->evq0_count
- pos
,
967 ARRAY_SIZE(copy_req
));
968 for (k
= 0; k
< count
; k
++) {
969 copy_req
[k
].from_buf
= NULL
;
970 copy_req
[k
].from_rid
= efx
->pci_dev
->devfn
;
971 copy_req
[k
].from_addr
= buffer
->dma_addr
;
972 copy_req
[k
].to_rid
= vf
->pci_rid
;
973 copy_req
[k
].to_addr
= vf
->evq0_addrs
[pos
+ k
];
974 copy_req
[k
].length
= EFX_PAGE_SIZE
;
976 rc
= efx_siena_sriov_memcpy(efx
, copy_req
, count
);
979 netif_err(efx
, hw
, efx
->net_dev
,
980 "ERROR: Unable to notify %s of reset"
981 ": %d\n", vf
->pci_name
, -rc
);
986 /* Reinitialise, arm and trigger evq0 */
987 abs_evq
= abs_index(vf
, 0);
988 buftbl
= EFX_BUFTBL_EVQ_BASE(vf
, 0);
989 efx_siena_sriov_bufs(efx
, buftbl
, vf
->evq0_addrs
, vf
->evq0_count
);
991 EFX_POPULATE_OWORD_3(reg
,
992 FRF_CZ_TIMER_Q_EN
, 1,
993 FRF_CZ_HOST_NOTIFY_MODE
, 0,
994 FRF_CZ_TIMER_MODE
, FFE_CZ_TIMER_MODE_DIS
);
995 efx_writeo_table(efx
, ®
, FR_BZ_TIMER_TBL
, abs_evq
);
996 EFX_POPULATE_OWORD_3(reg
,
998 FRF_AZ_EVQ_SIZE
, __ffs(vf
->evq0_count
),
999 FRF_AZ_EVQ_BUF_BASE_ID
, buftbl
);
1000 efx_writeo_table(efx
, ®
, FR_BZ_EVQ_PTR_TBL
, abs_evq
);
1001 EFX_POPULATE_DWORD_1(ptr
, FRF_AZ_EVQ_RPTR
, 0);
1002 efx_writed(efx
, &ptr
, FR_BZ_EVQ_RPTR
+ FR_BZ_EVQ_RPTR_STEP
* abs_evq
);
1004 mutex_unlock(&vf
->status_lock
);
1007 static void efx_siena_sriov_reset_vf_work(struct work_struct
*work
)
1009 struct efx_vf
*vf
= container_of(work
, struct efx_vf
, req
);
1010 struct efx_nic
*efx
= vf
->efx
;
1011 struct efx_buffer buf
;
1013 if (!efx_nic_alloc_buffer(efx
, &buf
, EFX_PAGE_SIZE
, GFP_NOIO
)) {
1014 efx_siena_sriov_reset_vf(vf
, &buf
);
1015 efx_nic_free_buffer(efx
, &buf
);
1019 static void efx_siena_sriov_handle_no_channel(struct efx_nic
*efx
)
1021 netif_err(efx
, drv
, efx
->net_dev
,
1022 "ERROR: IOV requires MSI-X and 1 additional interrupt"
1023 "vector. IOV disabled\n");
1027 static int efx_siena_sriov_probe_channel(struct efx_channel
*channel
)
1029 struct siena_nic_data
*nic_data
= channel
->efx
->nic_data
;
1030 nic_data
->vfdi_channel
= channel
;
1036 efx_siena_sriov_get_channel_name(struct efx_channel
*channel
,
1037 char *buf
, size_t len
)
1039 snprintf(buf
, len
, "%s-iov", channel
->efx
->name
);
1042 static const struct efx_channel_type efx_siena_sriov_channel_type
= {
1043 .handle_no_channel
= efx_siena_sriov_handle_no_channel
,
1044 .pre_probe
= efx_siena_sriov_probe_channel
,
1045 .post_remove
= efx_channel_dummy_op_void
,
1046 .get_name
= efx_siena_sriov_get_channel_name
,
1047 /* no copy operation; channel must not be reallocated */
1048 .keep_eventq
= true,
1051 void efx_siena_sriov_probe(struct efx_nic
*efx
)
1058 if (efx_siena_sriov_cmd(efx
, false, &efx
->vi_scale
, &count
))
1060 if (count
> 0 && count
> max_vfs
)
1063 /* efx_nic_dimension_resources() will reduce vf_count as appopriate */
1064 efx
->vf_count
= count
;
1066 efx
->extra_channel_type
[EFX_EXTRA_CHANNEL_IOV
] = &efx_siena_sriov_channel_type
;
1069 /* Copy the list of individual addresses into the vfdi_status.peers
1070 * array and auxillary pages, protected by %local_lock. Drop that lock
1071 * and then broadcast the address list to every VF.
1073 static void efx_siena_sriov_peer_work(struct work_struct
*data
)
1075 struct siena_nic_data
*nic_data
= container_of(data
,
1076 struct siena_nic_data
,
1078 struct efx_nic
*efx
= nic_data
->efx
;
1079 struct vfdi_status
*vfdi_status
= nic_data
->vfdi_status
.addr
;
1081 struct efx_local_addr
*local_addr
;
1082 struct vfdi_endpoint
*peer
;
1083 struct efx_endpoint_page
*epp
;
1084 struct list_head pages
;
1085 unsigned int peer_space
;
1086 unsigned int peer_count
;
1089 mutex_lock(&nic_data
->local_lock
);
1091 /* Move the existing peer pages off %local_page_list */
1092 INIT_LIST_HEAD(&pages
);
1093 list_splice_tail_init(&nic_data
->local_page_list
, &pages
);
1095 /* Populate the VF addresses starting from entry 1 (entry 0 is
1098 peer
= vfdi_status
->peers
+ 1;
1099 peer_space
= ARRAY_SIZE(vfdi_status
->peers
) - 1;
1101 for (pos
= 0; pos
< efx
->vf_count
; ++pos
) {
1104 mutex_lock(&vf
->status_lock
);
1105 if (vf
->rx_filtering
&& !is_zero_ether_addr(vf
->addr
.mac_addr
)) {
1109 BUG_ON(peer_space
== 0);
1111 mutex_unlock(&vf
->status_lock
);
1114 /* Fill the remaining addresses */
1115 list_for_each_entry(local_addr
, &nic_data
->local_addr_list
, link
) {
1116 ether_addr_copy(peer
->mac_addr
, local_addr
->addr
);
1120 if (--peer_space
== 0) {
1121 if (list_empty(&pages
)) {
1122 epp
= kmalloc(sizeof(*epp
), GFP_KERNEL
);
1125 epp
->ptr
= dma_alloc_coherent(
1126 &efx
->pci_dev
->dev
, EFX_PAGE_SIZE
,
1127 &epp
->addr
, GFP_KERNEL
);
1133 epp
= list_first_entry(
1134 &pages
, struct efx_endpoint_page
, link
);
1135 list_del(&epp
->link
);
1138 list_add_tail(&epp
->link
, &nic_data
->local_page_list
);
1139 peer
= (struct vfdi_endpoint
*)epp
->ptr
;
1140 peer_space
= EFX_PAGE_SIZE
/ sizeof(struct vfdi_endpoint
);
1143 vfdi_status
->peer_count
= peer_count
;
1144 mutex_unlock(&nic_data
->local_lock
);
1146 /* Free any now unused endpoint pages */
1147 while (!list_empty(&pages
)) {
1148 epp
= list_first_entry(
1149 &pages
, struct efx_endpoint_page
, link
);
1150 list_del(&epp
->link
);
1151 dma_free_coherent(&efx
->pci_dev
->dev
, EFX_PAGE_SIZE
,
1152 epp
->ptr
, epp
->addr
);
1156 /* Finally, push the pages */
1157 for (pos
= 0; pos
< efx
->vf_count
; ++pos
) {
1160 mutex_lock(&vf
->status_lock
);
1161 if (vf
->status_addr
)
1162 __efx_siena_sriov_push_vf_status(vf
);
1163 mutex_unlock(&vf
->status_lock
);
1167 static void efx_siena_sriov_free_local(struct efx_nic
*efx
)
1169 struct siena_nic_data
*nic_data
= efx
->nic_data
;
1170 struct efx_local_addr
*local_addr
;
1171 struct efx_endpoint_page
*epp
;
1173 while (!list_empty(&nic_data
->local_addr_list
)) {
1174 local_addr
= list_first_entry(&nic_data
->local_addr_list
,
1175 struct efx_local_addr
, link
);
1176 list_del(&local_addr
->link
);
1180 while (!list_empty(&nic_data
->local_page_list
)) {
1181 epp
= list_first_entry(&nic_data
->local_page_list
,
1182 struct efx_endpoint_page
, link
);
1183 list_del(&epp
->link
);
1184 dma_free_coherent(&efx
->pci_dev
->dev
, EFX_PAGE_SIZE
,
1185 epp
->ptr
, epp
->addr
);
1190 static int efx_siena_sriov_vf_alloc(struct efx_nic
*efx
)
1195 efx
->vf
= kzalloc(sizeof(struct efx_vf
) * efx
->vf_count
, GFP_KERNEL
);
1199 for (index
= 0; index
< efx
->vf_count
; ++index
) {
1200 vf
= efx
->vf
+ index
;
1204 vf
->rx_filter_id
= -1;
1205 vf
->tx_filter_mode
= VF_TX_FILTER_AUTO
;
1206 vf
->tx_filter_id
= -1;
1207 INIT_WORK(&vf
->req
, efx_siena_sriov_vfdi
);
1208 INIT_WORK(&vf
->reset_work
, efx_siena_sriov_reset_vf_work
);
1209 init_waitqueue_head(&vf
->flush_waitq
);
1210 mutex_init(&vf
->status_lock
);
1211 mutex_init(&vf
->txq_lock
);
1217 static void efx_siena_sriov_vfs_fini(struct efx_nic
*efx
)
1222 for (pos
= 0; pos
< efx
->vf_count
; ++pos
) {
1225 efx_nic_free_buffer(efx
, &vf
->buf
);
1226 kfree(vf
->peer_page_addrs
);
1227 vf
->peer_page_addrs
= NULL
;
1228 vf
->peer_page_count
= 0;
1234 static int efx_siena_sriov_vfs_init(struct efx_nic
*efx
)
1236 struct pci_dev
*pci_dev
= efx
->pci_dev
;
1237 struct siena_nic_data
*nic_data
= efx
->nic_data
;
1238 unsigned index
, devfn
, sriov
, buftbl_base
;
1243 sriov
= pci_find_ext_capability(pci_dev
, PCI_EXT_CAP_ID_SRIOV
);
1247 pci_read_config_word(pci_dev
, sriov
+ PCI_SRIOV_VF_OFFSET
, &offset
);
1248 pci_read_config_word(pci_dev
, sriov
+ PCI_SRIOV_VF_STRIDE
, &stride
);
1250 buftbl_base
= nic_data
->vf_buftbl_base
;
1251 devfn
= pci_dev
->devfn
+ offset
;
1252 for (index
= 0; index
< efx
->vf_count
; ++index
) {
1253 vf
= efx
->vf
+ index
;
1255 /* Reserve buffer entries */
1256 vf
->buftbl_base
= buftbl_base
;
1257 buftbl_base
+= EFX_VF_BUFTBL_PER_VI
* efx_vf_size(efx
);
1259 vf
->pci_rid
= devfn
;
1260 snprintf(vf
->pci_name
, sizeof(vf
->pci_name
),
1261 "%04x:%02x:%02x.%d",
1262 pci_domain_nr(pci_dev
->bus
), pci_dev
->bus
->number
,
1263 PCI_SLOT(devfn
), PCI_FUNC(devfn
));
1265 rc
= efx_nic_alloc_buffer(efx
, &vf
->buf
, EFX_PAGE_SIZE
,
1276 efx_siena_sriov_vfs_fini(efx
);
1280 int efx_siena_sriov_init(struct efx_nic
*efx
)
1282 struct net_device
*net_dev
= efx
->net_dev
;
1283 struct siena_nic_data
*nic_data
= efx
->nic_data
;
1284 struct vfdi_status
*vfdi_status
;
1287 /* Ensure there's room for vf_channel */
1288 BUILD_BUG_ON(EFX_MAX_CHANNELS
+ 1 >= EFX_VI_BASE
);
1289 /* Ensure that VI_BASE is aligned on VI_SCALE */
1290 BUILD_BUG_ON(EFX_VI_BASE
& ((1 << EFX_VI_SCALE_MAX
) - 1));
1292 if (efx
->vf_count
== 0)
1295 rc
= efx_siena_sriov_cmd(efx
, true, NULL
, NULL
);
1299 rc
= efx_nic_alloc_buffer(efx
, &nic_data
->vfdi_status
,
1300 sizeof(*vfdi_status
), GFP_KERNEL
);
1303 vfdi_status
= nic_data
->vfdi_status
.addr
;
1304 memset(vfdi_status
, 0, sizeof(*vfdi_status
));
1305 vfdi_status
->version
= 1;
1306 vfdi_status
->length
= sizeof(*vfdi_status
);
1307 vfdi_status
->max_tx_channels
= vf_max_tx_channels
;
1308 vfdi_status
->vi_scale
= efx
->vi_scale
;
1309 vfdi_status
->rss_rxq_count
= efx
->rss_spread
;
1310 vfdi_status
->peer_count
= 1 + efx
->vf_count
;
1311 vfdi_status
->timer_quantum_ns
= efx
->timer_quantum_ns
;
1313 rc
= efx_siena_sriov_vf_alloc(efx
);
1317 mutex_init(&nic_data
->local_lock
);
1318 INIT_WORK(&nic_data
->peer_work
, efx_siena_sriov_peer_work
);
1319 INIT_LIST_HEAD(&nic_data
->local_addr_list
);
1320 INIT_LIST_HEAD(&nic_data
->local_page_list
);
1322 rc
= efx_siena_sriov_vfs_init(efx
);
1327 ether_addr_copy(vfdi_status
->peers
[0].mac_addr
, net_dev
->dev_addr
);
1328 efx
->vf_init_count
= efx
->vf_count
;
1331 efx_siena_sriov_usrev(efx
, true);
1333 /* At this point we must be ready to accept VFDI requests */
1335 rc
= pci_enable_sriov(efx
->pci_dev
, efx
->vf_count
);
1339 netif_info(efx
, probe
, net_dev
,
1340 "enabled SR-IOV for %d VFs, %d VI per VF\n",
1341 efx
->vf_count
, efx_vf_size(efx
));
1345 efx_siena_sriov_usrev(efx
, false);
1347 efx
->vf_init_count
= 0;
1349 efx_siena_sriov_vfs_fini(efx
);
1351 cancel_work_sync(&nic_data
->peer_work
);
1352 efx_siena_sriov_free_local(efx
);
1355 efx_nic_free_buffer(efx
, &nic_data
->vfdi_status
);
1357 efx_siena_sriov_cmd(efx
, false, NULL
, NULL
);
1362 void efx_siena_sriov_fini(struct efx_nic
*efx
)
1366 struct siena_nic_data
*nic_data
= efx
->nic_data
;
1368 if (efx
->vf_init_count
== 0)
1371 /* Disable all interfaces to reconfiguration */
1372 BUG_ON(nic_data
->vfdi_channel
->enabled
);
1373 efx_siena_sriov_usrev(efx
, false);
1375 efx
->vf_init_count
= 0;
1378 /* Flush all reconfiguration work */
1379 for (pos
= 0; pos
< efx
->vf_count
; ++pos
) {
1381 cancel_work_sync(&vf
->req
);
1382 cancel_work_sync(&vf
->reset_work
);
1384 cancel_work_sync(&nic_data
->peer_work
);
1386 pci_disable_sriov(efx
->pci_dev
);
1388 /* Tear down back-end state */
1389 efx_siena_sriov_vfs_fini(efx
);
1390 efx_siena_sriov_free_local(efx
);
1392 efx_nic_free_buffer(efx
, &nic_data
->vfdi_status
);
1393 efx_siena_sriov_cmd(efx
, false, NULL
, NULL
);
1396 void efx_siena_sriov_event(struct efx_channel
*channel
, efx_qword_t
*event
)
1398 struct efx_nic
*efx
= channel
->efx
;
1400 unsigned qid
, seq
, type
, data
;
1402 qid
= EFX_QWORD_FIELD(*event
, FSF_CZ_USER_QID
);
1404 /* USR_EV_REG_VALUE is dword0, so access the VFDI_EV fields directly */
1405 BUILD_BUG_ON(FSF_CZ_USER_EV_REG_VALUE_LBN
!= 0);
1406 seq
= EFX_QWORD_FIELD(*event
, VFDI_EV_SEQ
);
1407 type
= EFX_QWORD_FIELD(*event
, VFDI_EV_TYPE
);
1408 data
= EFX_QWORD_FIELD(*event
, VFDI_EV_DATA
);
1410 netif_vdbg(efx
, hw
, efx
->net_dev
,
1411 "USR_EV event from qid %d seq 0x%x type %d data 0x%x\n",
1412 qid
, seq
, type
, data
);
1414 if (map_vi_index(efx
, qid
, &vf
, NULL
))
1419 if (type
== VFDI_EV_TYPE_REQ_WORD0
) {
1421 vf
->req_type
= VFDI_EV_TYPE_REQ_WORD0
;
1422 vf
->req_seqno
= seq
+ 1;
1424 } else if (seq
!= (vf
->req_seqno
++ & 0xff) || type
!= vf
->req_type
)
1427 switch (vf
->req_type
) {
1428 case VFDI_EV_TYPE_REQ_WORD0
:
1429 case VFDI_EV_TYPE_REQ_WORD1
:
1430 case VFDI_EV_TYPE_REQ_WORD2
:
1431 vf
->req_addr
|= (u64
)data
<< (vf
->req_type
<< 4);
1435 case VFDI_EV_TYPE_REQ_WORD3
:
1436 vf
->req_addr
|= (u64
)data
<< 48;
1437 vf
->req_type
= VFDI_EV_TYPE_REQ_WORD0
;
1439 queue_work(vfdi_workqueue
, &vf
->req
);
1444 if (net_ratelimit())
1445 netif_err(efx
, hw
, efx
->net_dev
,
1446 "ERROR: Screaming VFDI request from %s\n",
1448 /* Reset the request and sequence number */
1449 vf
->req_type
= VFDI_EV_TYPE_REQ_WORD0
;
1450 vf
->req_seqno
= seq
+ 1;
1453 void efx_siena_sriov_flr(struct efx_nic
*efx
, unsigned vf_i
)
1457 if (vf_i
> efx
->vf_init_count
)
1459 vf
= efx
->vf
+ vf_i
;
1460 netif_info(efx
, hw
, efx
->net_dev
,
1461 "FLR on VF %s\n", vf
->pci_name
);
1463 vf
->status_addr
= 0;
1464 efx_vfdi_remove_all_filters(vf
);
1465 efx_vfdi_flush_clear(vf
);
1470 void efx_siena_sriov_mac_address_changed(struct efx_nic
*efx
)
1472 struct siena_nic_data
*nic_data
= efx
->nic_data
;
1473 struct vfdi_status
*vfdi_status
= nic_data
->vfdi_status
.addr
;
1475 if (!efx
->vf_init_count
)
1477 ether_addr_copy(vfdi_status
->peers
[0].mac_addr
,
1478 efx
->net_dev
->dev_addr
);
1479 queue_work(vfdi_workqueue
, &nic_data
->peer_work
);
1482 void efx_siena_sriov_tx_flush_done(struct efx_nic
*efx
, efx_qword_t
*event
)
1485 unsigned queue
, qid
;
1487 queue
= EFX_QWORD_FIELD(*event
, FSF_AZ_DRIVER_EV_SUBDATA
);
1488 if (map_vi_index(efx
, queue
, &vf
, &qid
))
1490 /* Ignore flush completions triggered by an FLR */
1491 if (!test_bit(qid
, vf
->txq_mask
))
1494 __clear_bit(qid
, vf
->txq_mask
);
1497 if (efx_vfdi_flush_wake(vf
))
1498 wake_up(&vf
->flush_waitq
);
1501 void efx_siena_sriov_rx_flush_done(struct efx_nic
*efx
, efx_qword_t
*event
)
1504 unsigned ev_failed
, queue
, qid
;
1506 queue
= EFX_QWORD_FIELD(*event
, FSF_AZ_DRIVER_EV_RX_DESCQ_ID
);
1507 ev_failed
= EFX_QWORD_FIELD(*event
,
1508 FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL
);
1509 if (map_vi_index(efx
, queue
, &vf
, &qid
))
1511 if (!test_bit(qid
, vf
->rxq_mask
))
1515 set_bit(qid
, vf
->rxq_retry_mask
);
1516 atomic_inc(&vf
->rxq_retry_count
);
1518 __clear_bit(qid
, vf
->rxq_mask
);
1521 if (efx_vfdi_flush_wake(vf
))
1522 wake_up(&vf
->flush_waitq
);
1525 /* Called from napi. Schedule the reset work item */
1526 void efx_siena_sriov_desc_fetch_err(struct efx_nic
*efx
, unsigned dmaq
)
1531 if (map_vi_index(efx
, dmaq
, &vf
, &rel
))
1534 if (net_ratelimit())
1535 netif_err(efx
, hw
, efx
->net_dev
,
1536 "VF %d DMA Q %d reports descriptor fetch error.\n",
1538 queue_work(vfdi_workqueue
, &vf
->reset_work
);
1542 void efx_siena_sriov_reset(struct efx_nic
*efx
)
1545 struct efx_buffer buf
;
1550 if (efx
->vf_init_count
== 0)
1553 efx_siena_sriov_usrev(efx
, true);
1554 (void)efx_siena_sriov_cmd(efx
, true, NULL
, NULL
);
1556 if (efx_nic_alloc_buffer(efx
, &buf
, EFX_PAGE_SIZE
, GFP_NOIO
))
1559 for (vf_i
= 0; vf_i
< efx
->vf_init_count
; ++vf_i
) {
1560 vf
= efx
->vf
+ vf_i
;
1561 efx_siena_sriov_reset_vf(vf
, &buf
);
1564 efx_nic_free_buffer(efx
, &buf
);
1567 int efx_init_sriov(void)
1569 /* A single threaded workqueue is sufficient. efx_siena_sriov_vfdi() and
1570 * efx_siena_sriov_peer_work() spend almost all their time sleeping for
1571 * MCDI to complete anyway
1573 vfdi_workqueue
= create_singlethread_workqueue("sfc_vfdi");
1574 if (!vfdi_workqueue
)
1580 void efx_fini_sriov(void)
1582 destroy_workqueue(vfdi_workqueue
);
1585 int efx_siena_sriov_set_vf_mac(struct net_device
*net_dev
, int vf_i
, u8
*mac
)
1587 struct efx_nic
*efx
= netdev_priv(net_dev
);
1590 if (vf_i
>= efx
->vf_init_count
)
1592 vf
= efx
->vf
+ vf_i
;
1594 mutex_lock(&vf
->status_lock
);
1595 ether_addr_copy(vf
->addr
.mac_addr
, mac
);
1596 __efx_siena_sriov_update_vf_addr(vf
);
1597 mutex_unlock(&vf
->status_lock
);
1602 int efx_siena_sriov_set_vf_vlan(struct net_device
*net_dev
, int vf_i
,
1605 struct efx_nic
*efx
= netdev_priv(net_dev
);
1609 if (vf_i
>= efx
->vf_init_count
)
1611 vf
= efx
->vf
+ vf_i
;
1613 mutex_lock(&vf
->status_lock
);
1614 tci
= (vlan
& VLAN_VID_MASK
) | ((qos
& 0x7) << VLAN_PRIO_SHIFT
);
1615 vf
->addr
.tci
= htons(tci
);
1616 __efx_siena_sriov_update_vf_addr(vf
);
1617 mutex_unlock(&vf
->status_lock
);
1622 int efx_siena_sriov_set_vf_spoofchk(struct net_device
*net_dev
, int vf_i
,
1625 struct efx_nic
*efx
= netdev_priv(net_dev
);
1629 if (vf_i
>= efx
->vf_init_count
)
1631 vf
= efx
->vf
+ vf_i
;
1633 mutex_lock(&vf
->txq_lock
);
1634 if (vf
->txq_count
== 0) {
1635 vf
->tx_filter_mode
=
1636 spoofchk
? VF_TX_FILTER_ON
: VF_TX_FILTER_OFF
;
1639 /* This cannot be changed while TX queues are running */
1642 mutex_unlock(&vf
->txq_lock
);
1646 int efx_siena_sriov_get_vf_config(struct net_device
*net_dev
, int vf_i
,
1647 struct ifla_vf_info
*ivi
)
1649 struct efx_nic
*efx
= netdev_priv(net_dev
);
1653 if (vf_i
>= efx
->vf_init_count
)
1655 vf
= efx
->vf
+ vf_i
;
1658 ether_addr_copy(ivi
->mac
, vf
->addr
.mac_addr
);
1659 ivi
->max_tx_rate
= 0;
1660 ivi
->min_tx_rate
= 0;
1661 tci
= ntohs(vf
->addr
.tci
);
1662 ivi
->vlan
= tci
& VLAN_VID_MASK
;
1663 ivi
->qos
= (tci
>> VLAN_PRIO_SHIFT
) & 0x7;
1664 ivi
->spoofchk
= vf
->tx_filter_mode
== VF_TX_FILTER_ON
;