1 /* Intel(R) Ethernet Switch Host Interface Driver
2 * Copyright(c) 2013 - 2016 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * The full GNU General Public License is included in this distribution in
14 * the file called "COPYING".
16 * Contact Information:
17 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
18 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
22 #include <linux/vmalloc.h>
23 #include <net/udp_tunnel.h>
26 * fm10k_setup_tx_resources - allocate Tx resources (Descriptors)
27 * @tx_ring: tx descriptor ring (for a specific queue) to setup
29 * Return 0 on success, negative on failure
31 int fm10k_setup_tx_resources(struct fm10k_ring
*tx_ring
)
33 struct device
*dev
= tx_ring
->dev
;
36 size
= sizeof(struct fm10k_tx_buffer
) * tx_ring
->count
;
38 tx_ring
->tx_buffer
= vzalloc(size
);
39 if (!tx_ring
->tx_buffer
)
42 u64_stats_init(&tx_ring
->syncp
);
44 /* round up to nearest 4K */
45 tx_ring
->size
= tx_ring
->count
* sizeof(struct fm10k_tx_desc
);
46 tx_ring
->size
= ALIGN(tx_ring
->size
, 4096);
48 tx_ring
->desc
= dma_alloc_coherent(dev
, tx_ring
->size
,
49 &tx_ring
->dma
, GFP_KERNEL
);
56 vfree(tx_ring
->tx_buffer
);
57 tx_ring
->tx_buffer
= NULL
;
62 * fm10k_setup_all_tx_resources - allocate all queues Tx resources
63 * @interface: board private structure
65 * If this function returns with an error, then it's possible one or
66 * more of the rings is populated (while the rest are not). It is the
67 * callers duty to clean those orphaned rings.
69 * Return 0 on success, negative on failure
71 static int fm10k_setup_all_tx_resources(struct fm10k_intfc
*interface
)
75 for (i
= 0; i
< interface
->num_tx_queues
; i
++) {
76 err
= fm10k_setup_tx_resources(interface
->tx_ring
[i
]);
80 netif_err(interface
, probe
, interface
->netdev
,
81 "Allocation for Tx Queue %u failed\n", i
);
87 /* rewind the index freeing the rings as we go */
89 fm10k_free_tx_resources(interface
->tx_ring
[i
]);
94 * fm10k_setup_rx_resources - allocate Rx resources (Descriptors)
95 * @rx_ring: rx descriptor ring (for a specific queue) to setup
97 * Returns 0 on success, negative on failure
99 int fm10k_setup_rx_resources(struct fm10k_ring
*rx_ring
)
101 struct device
*dev
= rx_ring
->dev
;
104 size
= sizeof(struct fm10k_rx_buffer
) * rx_ring
->count
;
106 rx_ring
->rx_buffer
= vzalloc(size
);
107 if (!rx_ring
->rx_buffer
)
110 u64_stats_init(&rx_ring
->syncp
);
112 /* Round up to nearest 4K */
113 rx_ring
->size
= rx_ring
->count
* sizeof(union fm10k_rx_desc
);
114 rx_ring
->size
= ALIGN(rx_ring
->size
, 4096);
116 rx_ring
->desc
= dma_alloc_coherent(dev
, rx_ring
->size
,
117 &rx_ring
->dma
, GFP_KERNEL
);
123 vfree(rx_ring
->rx_buffer
);
124 rx_ring
->rx_buffer
= NULL
;
129 * fm10k_setup_all_rx_resources - allocate all queues Rx resources
130 * @interface: board private structure
132 * If this function returns with an error, then it's possible one or
133 * more of the rings is populated (while the rest are not). It is the
134 * callers duty to clean those orphaned rings.
136 * Return 0 on success, negative on failure
138 static int fm10k_setup_all_rx_resources(struct fm10k_intfc
*interface
)
142 for (i
= 0; i
< interface
->num_rx_queues
; i
++) {
143 err
= fm10k_setup_rx_resources(interface
->rx_ring
[i
]);
147 netif_err(interface
, probe
, interface
->netdev
,
148 "Allocation for Rx Queue %u failed\n", i
);
154 /* rewind the index freeing the rings as we go */
156 fm10k_free_rx_resources(interface
->rx_ring
[i
]);
160 void fm10k_unmap_and_free_tx_resource(struct fm10k_ring
*ring
,
161 struct fm10k_tx_buffer
*tx_buffer
)
163 if (tx_buffer
->skb
) {
164 dev_kfree_skb_any(tx_buffer
->skb
);
165 if (dma_unmap_len(tx_buffer
, len
))
166 dma_unmap_single(ring
->dev
,
167 dma_unmap_addr(tx_buffer
, dma
),
168 dma_unmap_len(tx_buffer
, len
),
170 } else if (dma_unmap_len(tx_buffer
, len
)) {
171 dma_unmap_page(ring
->dev
,
172 dma_unmap_addr(tx_buffer
, dma
),
173 dma_unmap_len(tx_buffer
, len
),
176 tx_buffer
->next_to_watch
= NULL
;
177 tx_buffer
->skb
= NULL
;
178 dma_unmap_len_set(tx_buffer
, len
, 0);
179 /* tx_buffer must be completely set up in the transmit path */
183 * fm10k_clean_tx_ring - Free Tx Buffers
184 * @tx_ring: ring to be cleaned
186 static void fm10k_clean_tx_ring(struct fm10k_ring
*tx_ring
)
188 struct fm10k_tx_buffer
*tx_buffer
;
192 /* ring already cleared, nothing to do */
193 if (!tx_ring
->tx_buffer
)
196 /* Free all the Tx ring sk_buffs */
197 for (i
= 0; i
< tx_ring
->count
; i
++) {
198 tx_buffer
= &tx_ring
->tx_buffer
[i
];
199 fm10k_unmap_and_free_tx_resource(tx_ring
, tx_buffer
);
202 /* reset BQL values */
203 netdev_tx_reset_queue(txring_txq(tx_ring
));
205 size
= sizeof(struct fm10k_tx_buffer
) * tx_ring
->count
;
206 memset(tx_ring
->tx_buffer
, 0, size
);
208 /* Zero out the descriptor ring */
209 memset(tx_ring
->desc
, 0, tx_ring
->size
);
213 * fm10k_free_tx_resources - Free Tx Resources per Queue
214 * @tx_ring: Tx descriptor ring for a specific queue
216 * Free all transmit software resources
218 void fm10k_free_tx_resources(struct fm10k_ring
*tx_ring
)
220 fm10k_clean_tx_ring(tx_ring
);
222 vfree(tx_ring
->tx_buffer
);
223 tx_ring
->tx_buffer
= NULL
;
225 /* if not set, then don't free */
229 dma_free_coherent(tx_ring
->dev
, tx_ring
->size
,
230 tx_ring
->desc
, tx_ring
->dma
);
231 tx_ring
->desc
= NULL
;
235 * fm10k_clean_all_tx_rings - Free Tx Buffers for all queues
236 * @interface: board private structure
238 void fm10k_clean_all_tx_rings(struct fm10k_intfc
*interface
)
242 for (i
= 0; i
< interface
->num_tx_queues
; i
++)
243 fm10k_clean_tx_ring(interface
->tx_ring
[i
]);
247 * fm10k_free_all_tx_resources - Free Tx Resources for All Queues
248 * @interface: board private structure
250 * Free all transmit software resources
252 static void fm10k_free_all_tx_resources(struct fm10k_intfc
*interface
)
254 int i
= interface
->num_tx_queues
;
257 fm10k_free_tx_resources(interface
->tx_ring
[i
]);
261 * fm10k_clean_rx_ring - Free Rx Buffers per Queue
262 * @rx_ring: ring to free buffers from
264 static void fm10k_clean_rx_ring(struct fm10k_ring
*rx_ring
)
269 if (!rx_ring
->rx_buffer
)
273 dev_kfree_skb(rx_ring
->skb
);
276 /* Free all the Rx ring sk_buffs */
277 for (i
= 0; i
< rx_ring
->count
; i
++) {
278 struct fm10k_rx_buffer
*buffer
= &rx_ring
->rx_buffer
[i
];
279 /* clean-up will only set page pointer to NULL */
283 dma_unmap_page(rx_ring
->dev
, buffer
->dma
,
284 PAGE_SIZE
, DMA_FROM_DEVICE
);
285 __free_page(buffer
->page
);
290 size
= sizeof(struct fm10k_rx_buffer
) * rx_ring
->count
;
291 memset(rx_ring
->rx_buffer
, 0, size
);
293 /* Zero out the descriptor ring */
294 memset(rx_ring
->desc
, 0, rx_ring
->size
);
296 rx_ring
->next_to_alloc
= 0;
297 rx_ring
->next_to_clean
= 0;
298 rx_ring
->next_to_use
= 0;
302 * fm10k_free_rx_resources - Free Rx Resources
303 * @rx_ring: ring to clean the resources from
305 * Free all receive software resources
307 void fm10k_free_rx_resources(struct fm10k_ring
*rx_ring
)
309 fm10k_clean_rx_ring(rx_ring
);
311 vfree(rx_ring
->rx_buffer
);
312 rx_ring
->rx_buffer
= NULL
;
314 /* if not set, then don't free */
318 dma_free_coherent(rx_ring
->dev
, rx_ring
->size
,
319 rx_ring
->desc
, rx_ring
->dma
);
321 rx_ring
->desc
= NULL
;
325 * fm10k_clean_all_rx_rings - Free Rx Buffers for all queues
326 * @interface: board private structure
328 void fm10k_clean_all_rx_rings(struct fm10k_intfc
*interface
)
332 for (i
= 0; i
< interface
->num_rx_queues
; i
++)
333 fm10k_clean_rx_ring(interface
->rx_ring
[i
]);
337 * fm10k_free_all_rx_resources - Free Rx Resources for All Queues
338 * @interface: board private structure
340 * Free all receive software resources
342 static void fm10k_free_all_rx_resources(struct fm10k_intfc
*interface
)
344 int i
= interface
->num_rx_queues
;
347 fm10k_free_rx_resources(interface
->rx_ring
[i
]);
351 * fm10k_request_glort_range - Request GLORTs for use in configuring rules
352 * @interface: board private structure
354 * This function allocates a range of glorts for this interface to use.
356 static void fm10k_request_glort_range(struct fm10k_intfc
*interface
)
358 struct fm10k_hw
*hw
= &interface
->hw
;
359 u16 mask
= (~hw
->mac
.dglort_map
) >> FM10K_DGLORTMAP_MASK_SHIFT
;
361 /* establish GLORT base */
362 interface
->glort
= hw
->mac
.dglort_map
& FM10K_DGLORTMAP_NONE
;
363 interface
->glort_count
= 0;
365 /* nothing we can do until mask is allocated */
366 if (hw
->mac
.dglort_map
== FM10K_DGLORTMAP_NONE
)
369 /* we support 3 possible GLORT configurations.
370 * 1: VFs consume all but the last 1
371 * 2: VFs and PF split glorts with possible gap between
372 * 3: VFs allocated first 64, all others belong to PF
374 if (mask
<= hw
->iov
.total_vfs
) {
375 interface
->glort_count
= 1;
376 interface
->glort
+= mask
;
377 } else if (mask
< 64) {
378 interface
->glort_count
= (mask
+ 1) / 2;
379 interface
->glort
+= interface
->glort_count
;
381 interface
->glort_count
= mask
- 63;
382 interface
->glort
+= 64;
387 * fm10k_del_vxlan_port_all
388 * @interface: board private structure
390 * This function frees the entire vxlan_port list
392 static void fm10k_del_vxlan_port_all(struct fm10k_intfc
*interface
)
394 struct fm10k_vxlan_port
*vxlan_port
;
396 /* flush all entries from list */
397 vxlan_port
= list_first_entry_or_null(&interface
->vxlan_port
,
398 struct fm10k_vxlan_port
, list
);
400 list_del(&vxlan_port
->list
);
402 vxlan_port
= list_first_entry_or_null(&interface
->vxlan_port
,
403 struct fm10k_vxlan_port
,
409 * fm10k_restore_vxlan_port
410 * @interface: board private structure
412 * This function restores the value in the tunnel_cfg register after reset
414 static void fm10k_restore_vxlan_port(struct fm10k_intfc
*interface
)
416 struct fm10k_hw
*hw
= &interface
->hw
;
417 struct fm10k_vxlan_port
*vxlan_port
;
419 /* only the PF supports configuring tunnels */
420 if (hw
->mac
.type
!= fm10k_mac_pf
)
423 vxlan_port
= list_first_entry_or_null(&interface
->vxlan_port
,
424 struct fm10k_vxlan_port
, list
);
426 /* restore tunnel configuration register */
427 fm10k_write_reg(hw
, FM10K_TUNNEL_CFG
,
428 (vxlan_port
? ntohs(vxlan_port
->port
) : 0) |
429 (ETH_P_TEB
<< FM10K_TUNNEL_CFG_NVGRE_SHIFT
));
433 * fm10k_add_vxlan_port
434 * @netdev: network interface device structure
435 * @ti: Tunnel endpoint information
437 * This function is called when a new VXLAN interface has added a new port
438 * number to the range that is currently in use for VXLAN. The new port
439 * number is always added to the tail so that the port number list should
440 * match the order in which the ports were allocated. The head of the list
441 * is always used as the VXLAN port number for offloads.
443 static void fm10k_add_vxlan_port(struct net_device
*dev
,
444 struct udp_tunnel_info
*ti
)
446 struct fm10k_intfc
*interface
= netdev_priv(dev
);
447 struct fm10k_vxlan_port
*vxlan_port
;
449 if (ti
->type
!= UDP_TUNNEL_TYPE_VXLAN
)
451 /* only the PF supports configuring tunnels */
452 if (interface
->hw
.mac
.type
!= fm10k_mac_pf
)
455 /* existing ports are pulled out so our new entry is always last */
456 fm10k_vxlan_port_for_each(vxlan_port
, interface
) {
457 if ((vxlan_port
->port
== ti
->port
) &&
458 (vxlan_port
->sa_family
== ti
->sa_family
)) {
459 list_del(&vxlan_port
->list
);
464 /* allocate memory to track ports */
465 vxlan_port
= kmalloc(sizeof(*vxlan_port
), GFP_ATOMIC
);
468 vxlan_port
->port
= ti
->port
;
469 vxlan_port
->sa_family
= ti
->sa_family
;
472 /* add new port value to list */
473 list_add_tail(&vxlan_port
->list
, &interface
->vxlan_port
);
475 fm10k_restore_vxlan_port(interface
);
479 * fm10k_del_vxlan_port
480 * @netdev: network interface device structure
481 * @ti: Tunnel endpoint information
483 * This function is called when a new VXLAN interface has freed a port
484 * number from the range that is currently in use for VXLAN. The freed
485 * port is removed from the list and the new head is used to determine
486 * the port number for offloads.
488 static void fm10k_del_vxlan_port(struct net_device
*dev
,
489 struct udp_tunnel_info
*ti
)
491 struct fm10k_intfc
*interface
= netdev_priv(dev
);
492 struct fm10k_vxlan_port
*vxlan_port
;
494 if (ti
->type
!= UDP_TUNNEL_TYPE_VXLAN
)
496 if (interface
->hw
.mac
.type
!= fm10k_mac_pf
)
499 /* find the port in the list and free it */
500 fm10k_vxlan_port_for_each(vxlan_port
, interface
) {
501 if ((vxlan_port
->port
== ti
->port
) &&
502 (vxlan_port
->sa_family
== ti
->sa_family
)) {
503 list_del(&vxlan_port
->list
);
509 fm10k_restore_vxlan_port(interface
);
513 * fm10k_open - Called when a network interface is made active
514 * @netdev: network interface device structure
516 * Returns 0 on success, negative value on failure
518 * The open entry point is called when a network interface is made
519 * active by the system (IFF_UP). At this point all resources needed
520 * for transmit and receive operations are allocated, the interrupt
521 * handler is registered with the OS, the watchdog timer is started,
522 * and the stack is notified that the interface is ready.
524 int fm10k_open(struct net_device
*netdev
)
526 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
529 /* allocate transmit descriptors */
530 err
= fm10k_setup_all_tx_resources(interface
);
534 /* allocate receive descriptors */
535 err
= fm10k_setup_all_rx_resources(interface
);
539 /* allocate interrupt resources */
540 err
= fm10k_qv_request_irq(interface
);
544 /* setup GLORT assignment for this port */
545 fm10k_request_glort_range(interface
);
547 /* Notify the stack of the actual queue counts */
548 err
= netif_set_real_num_tx_queues(netdev
,
549 interface
->num_tx_queues
);
553 err
= netif_set_real_num_rx_queues(netdev
,
554 interface
->num_rx_queues
);
558 /* update VXLAN port configuration */
559 udp_tunnel_get_rx_info(netdev
);
566 fm10k_qv_free_irq(interface
);
568 fm10k_free_all_rx_resources(interface
);
570 fm10k_free_all_tx_resources(interface
);
576 * fm10k_close - Disables a network interface
577 * @netdev: network interface device structure
579 * Returns 0, this is not allowed to fail
581 * The close entry point is called when an interface is de-activated
582 * by the OS. The hardware is still under the drivers control, but
583 * needs to be disabled. A global MAC reset is issued to stop the
584 * hardware, and all transmit and receive resources are freed.
586 int fm10k_close(struct net_device
*netdev
)
588 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
590 fm10k_down(interface
);
592 fm10k_qv_free_irq(interface
);
594 fm10k_del_vxlan_port_all(interface
);
596 fm10k_free_all_tx_resources(interface
);
597 fm10k_free_all_rx_resources(interface
);
602 static netdev_tx_t
fm10k_xmit_frame(struct sk_buff
*skb
, struct net_device
*dev
)
604 struct fm10k_intfc
*interface
= netdev_priv(dev
);
605 unsigned int r_idx
= skb
->queue_mapping
;
608 if ((skb
->protocol
== htons(ETH_P_8021Q
)) &&
609 !skb_vlan_tag_present(skb
)) {
610 /* FM10K only supports hardware tagging, any tags in frame
611 * are considered 2nd level or "outer" tags
613 struct vlan_hdr
*vhdr
;
616 /* make sure skb is not shared */
617 skb
= skb_share_check(skb
, GFP_ATOMIC
);
621 /* make sure there is enough room to move the ethernet header */
622 if (unlikely(!pskb_may_pull(skb
, VLAN_ETH_HLEN
)))
625 /* verify the skb head is not shared */
626 err
= skb_cow_head(skb
, 0);
632 /* locate VLAN header */
633 vhdr
= (struct vlan_hdr
*)(skb
->data
+ ETH_HLEN
);
635 /* pull the 2 key pieces of data out of it */
636 __vlan_hwaccel_put_tag(skb
,
638 ntohs(vhdr
->h_vlan_TCI
));
639 proto
= vhdr
->h_vlan_encapsulated_proto
;
640 skb
->protocol
= (ntohs(proto
) >= 1536) ? proto
:
643 /* squash it by moving the ethernet addresses up 4 bytes */
644 memmove(skb
->data
+ VLAN_HLEN
, skb
->data
, 12);
645 __skb_pull(skb
, VLAN_HLEN
);
646 skb_reset_mac_header(skb
);
649 /* The minimum packet size for a single buffer is 17B so pad the skb
650 * in order to meet this minimum size requirement.
652 if (unlikely(skb
->len
< 17)) {
653 int pad_len
= 17 - skb
->len
;
655 if (skb_pad(skb
, pad_len
))
657 __skb_put(skb
, pad_len
);
660 if (r_idx
>= interface
->num_tx_queues
)
661 r_idx
%= interface
->num_tx_queues
;
663 err
= fm10k_xmit_frame_ring(skb
, interface
->tx_ring
[r_idx
]);
668 static int fm10k_change_mtu(struct net_device
*dev
, int new_mtu
)
670 if (new_mtu
< 68 || new_mtu
> FM10K_MAX_JUMBO_FRAME_SIZE
)
679 * fm10k_tx_timeout - Respond to a Tx Hang
680 * @netdev: network interface device structure
682 static void fm10k_tx_timeout(struct net_device
*netdev
)
684 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
685 bool real_tx_hang
= false;
688 #define TX_TIMEO_LIMIT 16000
689 for (i
= 0; i
< interface
->num_tx_queues
; i
++) {
690 struct fm10k_ring
*tx_ring
= interface
->tx_ring
[i
];
692 if (check_for_tx_hang(tx_ring
) && fm10k_check_tx_hang(tx_ring
))
697 fm10k_tx_timeout_reset(interface
);
699 netif_info(interface
, drv
, netdev
,
700 "Fake Tx hang detected with timeout of %d seconds\n",
701 netdev
->watchdog_timeo
/ HZ
);
703 /* fake Tx hang - increase the kernel timeout */
704 if (netdev
->watchdog_timeo
< TX_TIMEO_LIMIT
)
705 netdev
->watchdog_timeo
*= 2;
709 static int fm10k_uc_vlan_unsync(struct net_device
*netdev
,
710 const unsigned char *uc_addr
)
712 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
713 struct fm10k_hw
*hw
= &interface
->hw
;
714 u16 glort
= interface
->glort
;
715 u16 vid
= interface
->vid
;
716 bool set
= !!(vid
/ VLAN_N_VID
);
719 /* drop any leading bits on the VLAN ID */
720 vid
&= VLAN_N_VID
- 1;
722 err
= hw
->mac
.ops
.update_uc_addr(hw
, glort
, uc_addr
, vid
, set
, 0);
726 /* return non-zero value as we are only doing a partial sync/unsync */
730 static int fm10k_mc_vlan_unsync(struct net_device
*netdev
,
731 const unsigned char *mc_addr
)
733 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
734 struct fm10k_hw
*hw
= &interface
->hw
;
735 u16 glort
= interface
->glort
;
736 u16 vid
= interface
->vid
;
737 bool set
= !!(vid
/ VLAN_N_VID
);
740 /* drop any leading bits on the VLAN ID */
741 vid
&= VLAN_N_VID
- 1;
743 err
= hw
->mac
.ops
.update_mc_addr(hw
, glort
, mc_addr
, vid
, set
);
747 /* return non-zero value as we are only doing a partial sync/unsync */
751 static int fm10k_update_vid(struct net_device
*netdev
, u16 vid
, bool set
)
753 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
754 struct fm10k_hw
*hw
= &interface
->hw
;
758 /* updates do not apply to VLAN 0 */
762 if (vid
>= VLAN_N_VID
)
765 /* Verify we have permission to add VLANs */
766 if (hw
->mac
.vlan_override
)
769 /* update active_vlans bitmask */
770 set_bit(vid
, interface
->active_vlans
);
772 clear_bit(vid
, interface
->active_vlans
);
774 /* disable the default VLAN ID on ring if we have an active VLAN */
775 for (i
= 0; i
< interface
->num_rx_queues
; i
++) {
776 struct fm10k_ring
*rx_ring
= interface
->rx_ring
[i
];
777 u16 rx_vid
= rx_ring
->vid
& (VLAN_N_VID
- 1);
779 if (test_bit(rx_vid
, interface
->active_vlans
))
780 rx_ring
->vid
|= FM10K_VLAN_CLEAR
;
782 rx_ring
->vid
&= ~FM10K_VLAN_CLEAR
;
785 /* Do not remove default VLAN ID related entries from VLAN and MAC
788 if (!set
&& vid
== hw
->mac
.default_vid
)
791 /* Do not throw an error if the interface is down. We will sync once
794 if (test_bit(__FM10K_DOWN
, &interface
->state
))
797 fm10k_mbx_lock(interface
);
799 /* only need to update the VLAN if not in promiscuous mode */
800 if (!(netdev
->flags
& IFF_PROMISC
)) {
801 err
= hw
->mac
.ops
.update_vlan(hw
, vid
, 0, set
);
806 /* update our base MAC address */
807 err
= hw
->mac
.ops
.update_uc_addr(hw
, interface
->glort
, hw
->mac
.addr
,
812 /* set VLAN ID prior to syncing/unsyncing the VLAN */
813 interface
->vid
= vid
+ (set
? VLAN_N_VID
: 0);
815 /* Update the unicast and multicast address list to add/drop VLAN */
816 __dev_uc_unsync(netdev
, fm10k_uc_vlan_unsync
);
817 __dev_mc_unsync(netdev
, fm10k_mc_vlan_unsync
);
820 fm10k_mbx_unlock(interface
);
825 static int fm10k_vlan_rx_add_vid(struct net_device
*netdev
,
826 __always_unused __be16 proto
, u16 vid
)
828 /* update VLAN and address table based on changes */
829 return fm10k_update_vid(netdev
, vid
, true);
832 static int fm10k_vlan_rx_kill_vid(struct net_device
*netdev
,
833 __always_unused __be16 proto
, u16 vid
)
835 /* update VLAN and address table based on changes */
836 return fm10k_update_vid(netdev
, vid
, false);
839 static u16
fm10k_find_next_vlan(struct fm10k_intfc
*interface
, u16 vid
)
841 struct fm10k_hw
*hw
= &interface
->hw
;
842 u16 default_vid
= hw
->mac
.default_vid
;
843 u16 vid_limit
= vid
< default_vid
? default_vid
: VLAN_N_VID
;
845 vid
= find_next_bit(interface
->active_vlans
, vid_limit
, ++vid
);
850 static void fm10k_clear_unused_vlans(struct fm10k_intfc
*interface
)
852 struct fm10k_hw
*hw
= &interface
->hw
;
855 /* loop through and find any gaps in the table */
856 for (vid
= 0, prev_vid
= 0;
857 prev_vid
< VLAN_N_VID
;
858 prev_vid
= vid
+ 1, vid
= fm10k_find_next_vlan(interface
, vid
)) {
862 /* send request to clear multiple bits at a time */
863 prev_vid
+= (vid
- prev_vid
- 1) << FM10K_VLAN_LENGTH_SHIFT
;
864 hw
->mac
.ops
.update_vlan(hw
, prev_vid
, 0, false);
868 static int __fm10k_uc_sync(struct net_device
*dev
,
869 const unsigned char *addr
, bool sync
)
871 struct fm10k_intfc
*interface
= netdev_priv(dev
);
872 struct fm10k_hw
*hw
= &interface
->hw
;
873 u16 vid
, glort
= interface
->glort
;
876 if (!is_valid_ether_addr(addr
))
877 return -EADDRNOTAVAIL
;
879 /* update table with current entries */
880 for (vid
= hw
->mac
.default_vid
? fm10k_find_next_vlan(interface
, 0) : 1;
882 vid
= fm10k_find_next_vlan(interface
, vid
)) {
883 err
= hw
->mac
.ops
.update_uc_addr(hw
, glort
, addr
,
892 static int fm10k_uc_sync(struct net_device
*dev
,
893 const unsigned char *addr
)
895 return __fm10k_uc_sync(dev
, addr
, true);
898 static int fm10k_uc_unsync(struct net_device
*dev
,
899 const unsigned char *addr
)
901 return __fm10k_uc_sync(dev
, addr
, false);
904 static int fm10k_set_mac(struct net_device
*dev
, void *p
)
906 struct fm10k_intfc
*interface
= netdev_priv(dev
);
907 struct fm10k_hw
*hw
= &interface
->hw
;
908 struct sockaddr
*addr
= p
;
911 if (!is_valid_ether_addr(addr
->sa_data
))
912 return -EADDRNOTAVAIL
;
914 if (dev
->flags
& IFF_UP
) {
915 /* setting MAC address requires mailbox */
916 fm10k_mbx_lock(interface
);
918 err
= fm10k_uc_sync(dev
, addr
->sa_data
);
920 fm10k_uc_unsync(dev
, hw
->mac
.addr
);
922 fm10k_mbx_unlock(interface
);
926 ether_addr_copy(dev
->dev_addr
, addr
->sa_data
);
927 ether_addr_copy(hw
->mac
.addr
, addr
->sa_data
);
928 dev
->addr_assign_type
&= ~NET_ADDR_RANDOM
;
931 /* if we had a mailbox error suggest trying again */
932 return err
? -EAGAIN
: 0;
935 static int __fm10k_mc_sync(struct net_device
*dev
,
936 const unsigned char *addr
, bool sync
)
938 struct fm10k_intfc
*interface
= netdev_priv(dev
);
939 struct fm10k_hw
*hw
= &interface
->hw
;
940 u16 vid
, glort
= interface
->glort
;
942 /* update table with current entries */
943 for (vid
= hw
->mac
.default_vid
? fm10k_find_next_vlan(interface
, 0) : 1;
945 vid
= fm10k_find_next_vlan(interface
, vid
)) {
946 hw
->mac
.ops
.update_mc_addr(hw
, glort
, addr
, vid
, sync
);
952 static int fm10k_mc_sync(struct net_device
*dev
,
953 const unsigned char *addr
)
955 return __fm10k_mc_sync(dev
, addr
, true);
958 static int fm10k_mc_unsync(struct net_device
*dev
,
959 const unsigned char *addr
)
961 return __fm10k_mc_sync(dev
, addr
, false);
964 static void fm10k_set_rx_mode(struct net_device
*dev
)
966 struct fm10k_intfc
*interface
= netdev_priv(dev
);
967 struct fm10k_hw
*hw
= &interface
->hw
;
970 /* no need to update the harwdare if we are not running */
971 if (!(dev
->flags
& IFF_UP
))
974 /* determine new mode based on flags */
975 xcast_mode
= (dev
->flags
& IFF_PROMISC
) ? FM10K_XCAST_MODE_PROMISC
:
976 (dev
->flags
& IFF_ALLMULTI
) ? FM10K_XCAST_MODE_ALLMULTI
:
977 (dev
->flags
& (IFF_BROADCAST
| IFF_MULTICAST
)) ?
978 FM10K_XCAST_MODE_MULTI
: FM10K_XCAST_MODE_NONE
;
980 fm10k_mbx_lock(interface
);
982 /* update xcast mode first, but only if it changed */
983 if (interface
->xcast_mode
!= xcast_mode
) {
984 /* update VLAN table */
985 if (xcast_mode
== FM10K_XCAST_MODE_PROMISC
)
986 hw
->mac
.ops
.update_vlan(hw
, FM10K_VLAN_ALL
, 0, true);
987 if (interface
->xcast_mode
== FM10K_XCAST_MODE_PROMISC
)
988 fm10k_clear_unused_vlans(interface
);
990 /* update xcast mode */
991 hw
->mac
.ops
.update_xcast_mode(hw
, interface
->glort
, xcast_mode
);
993 /* record updated xcast mode state */
994 interface
->xcast_mode
= xcast_mode
;
997 /* synchronize all of the addresses */
998 __dev_uc_sync(dev
, fm10k_uc_sync
, fm10k_uc_unsync
);
999 __dev_mc_sync(dev
, fm10k_mc_sync
, fm10k_mc_unsync
);
1001 fm10k_mbx_unlock(interface
);
1004 void fm10k_restore_rx_state(struct fm10k_intfc
*interface
)
1006 struct net_device
*netdev
= interface
->netdev
;
1007 struct fm10k_hw
*hw
= &interface
->hw
;
1011 /* record glort for this interface */
1012 glort
= interface
->glort
;
1014 /* convert interface flags to xcast mode */
1015 if (netdev
->flags
& IFF_PROMISC
)
1016 xcast_mode
= FM10K_XCAST_MODE_PROMISC
;
1017 else if (netdev
->flags
& IFF_ALLMULTI
)
1018 xcast_mode
= FM10K_XCAST_MODE_ALLMULTI
;
1019 else if (netdev
->flags
& (IFF_BROADCAST
| IFF_MULTICAST
))
1020 xcast_mode
= FM10K_XCAST_MODE_MULTI
;
1022 xcast_mode
= FM10K_XCAST_MODE_NONE
;
1024 fm10k_mbx_lock(interface
);
1026 /* Enable logical port */
1027 hw
->mac
.ops
.update_lport_state(hw
, glort
, interface
->glort_count
, true);
1029 /* update VLAN table */
1030 hw
->mac
.ops
.update_vlan(hw
, FM10K_VLAN_ALL
, 0,
1031 xcast_mode
== FM10K_XCAST_MODE_PROMISC
);
1033 /* Add filter for VLAN 0 */
1034 hw
->mac
.ops
.update_vlan(hw
, 0, 0, true);
1036 /* update table with current entries */
1037 for (vid
= hw
->mac
.default_vid
? fm10k_find_next_vlan(interface
, 0) : 1;
1039 vid
= fm10k_find_next_vlan(interface
, vid
)) {
1040 hw
->mac
.ops
.update_vlan(hw
, vid
, 0, true);
1041 hw
->mac
.ops
.update_uc_addr(hw
, glort
, hw
->mac
.addr
,
1045 /* update xcast mode before synchronizing addresses */
1046 hw
->mac
.ops
.update_xcast_mode(hw
, glort
, xcast_mode
);
1048 /* synchronize all of the addresses */
1049 __dev_uc_sync(netdev
, fm10k_uc_sync
, fm10k_uc_unsync
);
1050 __dev_mc_sync(netdev
, fm10k_mc_sync
, fm10k_mc_unsync
);
1052 fm10k_mbx_unlock(interface
);
1054 /* record updated xcast mode state */
1055 interface
->xcast_mode
= xcast_mode
;
1057 /* Restore tunnel configuration */
1058 fm10k_restore_vxlan_port(interface
);
1061 void fm10k_reset_rx_state(struct fm10k_intfc
*interface
)
1063 struct net_device
*netdev
= interface
->netdev
;
1064 struct fm10k_hw
*hw
= &interface
->hw
;
1066 fm10k_mbx_lock(interface
);
1068 /* clear the logical port state on lower device */
1069 hw
->mac
.ops
.update_lport_state(hw
, interface
->glort
,
1070 interface
->glort_count
, false);
1072 fm10k_mbx_unlock(interface
);
1074 /* reset flags to default state */
1075 interface
->xcast_mode
= FM10K_XCAST_MODE_NONE
;
1077 /* clear the sync flag since the lport has been dropped */
1078 __dev_uc_unsync(netdev
, NULL
);
1079 __dev_mc_unsync(netdev
, NULL
);
1083 * fm10k_get_stats64 - Get System Network Statistics
1084 * @netdev: network interface device structure
1085 * @stats: storage space for 64bit statistics
1087 * Returns 64bit statistics, for use in the ndo_get_stats64 callback. This
1088 * function replaces fm10k_get_stats for kernels which support it.
1090 static struct rtnl_link_stats64
*fm10k_get_stats64(struct net_device
*netdev
,
1091 struct rtnl_link_stats64
*stats
)
1093 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
1094 struct fm10k_ring
*ring
;
1095 unsigned int start
, i
;
1100 for (i
= 0; i
< interface
->num_rx_queues
; i
++) {
1101 ring
= ACCESS_ONCE(interface
->rx_ring
[i
]);
1107 start
= u64_stats_fetch_begin_irq(&ring
->syncp
);
1108 packets
= ring
->stats
.packets
;
1109 bytes
= ring
->stats
.bytes
;
1110 } while (u64_stats_fetch_retry_irq(&ring
->syncp
, start
));
1112 stats
->rx_packets
+= packets
;
1113 stats
->rx_bytes
+= bytes
;
1116 for (i
= 0; i
< interface
->num_tx_queues
; i
++) {
1117 ring
= ACCESS_ONCE(interface
->tx_ring
[i
]);
1123 start
= u64_stats_fetch_begin_irq(&ring
->syncp
);
1124 packets
= ring
->stats
.packets
;
1125 bytes
= ring
->stats
.bytes
;
1126 } while (u64_stats_fetch_retry_irq(&ring
->syncp
, start
));
1128 stats
->tx_packets
+= packets
;
1129 stats
->tx_bytes
+= bytes
;
1134 /* following stats updated by fm10k_service_task() */
1135 stats
->rx_missed_errors
= netdev
->stats
.rx_missed_errors
;
1140 int fm10k_setup_tc(struct net_device
*dev
, u8 tc
)
1142 struct fm10k_intfc
*interface
= netdev_priv(dev
);
1145 /* Currently only the PF supports priority classes */
1146 if (tc
&& (interface
->hw
.mac
.type
!= fm10k_mac_pf
))
1149 /* Hardware supports up to 8 traffic classes */
1153 /* Hardware has to reinitialize queues to match packet
1154 * buffer alignment. Unfortunately, the hardware is not
1155 * flexible enough to do this dynamically.
1157 if (netif_running(dev
))
1160 fm10k_mbx_free_irq(interface
);
1162 fm10k_clear_queueing_scheme(interface
);
1164 /* we expect the prio_tc map to be repopulated later */
1165 netdev_reset_tc(dev
);
1166 netdev_set_num_tc(dev
, tc
);
1168 err
= fm10k_init_queueing_scheme(interface
);
1170 goto err_queueing_scheme
;
1172 err
= fm10k_mbx_request_irq(interface
);
1176 err
= netif_running(dev
) ? fm10k_open(dev
) : 0;
1180 /* flag to indicate SWPRI has yet to be updated */
1181 interface
->flags
|= FM10K_FLAG_SWPRI_CONFIG
;
1185 fm10k_mbx_free_irq(interface
);
1187 fm10k_clear_queueing_scheme(interface
);
1188 err_queueing_scheme
:
1189 netif_device_detach(dev
);
1194 static int __fm10k_setup_tc(struct net_device
*dev
, u32 handle
, __be16 proto
,
1195 struct tc_to_netdev
*tc
)
1197 if (tc
->type
!= TC_SETUP_MQPRIO
)
1200 return fm10k_setup_tc(dev
, tc
->tc
);
1203 static void fm10k_assign_l2_accel(struct fm10k_intfc
*interface
,
1204 struct fm10k_l2_accel
*l2_accel
)
1206 struct fm10k_ring
*ring
;
1209 for (i
= 0; i
< interface
->num_rx_queues
; i
++) {
1210 ring
= interface
->rx_ring
[i
];
1211 rcu_assign_pointer(ring
->l2_accel
, l2_accel
);
1214 interface
->l2_accel
= l2_accel
;
1217 static void *fm10k_dfwd_add_station(struct net_device
*dev
,
1218 struct net_device
*sdev
)
1220 struct fm10k_intfc
*interface
= netdev_priv(dev
);
1221 struct fm10k_l2_accel
*l2_accel
= interface
->l2_accel
;
1222 struct fm10k_l2_accel
*old_l2_accel
= NULL
;
1223 struct fm10k_dglort_cfg dglort
= { 0 };
1224 struct fm10k_hw
*hw
= &interface
->hw
;
1228 /* allocate l2 accel structure if it is not available */
1230 /* verify there is enough free GLORTs to support l2_accel */
1231 if (interface
->glort_count
< 7)
1232 return ERR_PTR(-EBUSY
);
1234 size
= offsetof(struct fm10k_l2_accel
, macvlan
[7]);
1235 l2_accel
= kzalloc(size
, GFP_KERNEL
);
1237 return ERR_PTR(-ENOMEM
);
1240 l2_accel
->dglort
= interface
->glort
;
1242 /* update pointers */
1243 fm10k_assign_l2_accel(interface
, l2_accel
);
1244 /* do not expand if we are at our limit */
1245 } else if ((l2_accel
->count
== FM10K_MAX_STATIONS
) ||
1246 (l2_accel
->count
== (interface
->glort_count
- 1))) {
1247 return ERR_PTR(-EBUSY
);
1248 /* expand if we have hit the size limit */
1249 } else if (l2_accel
->count
== l2_accel
->size
) {
1250 old_l2_accel
= l2_accel
;
1251 size
= offsetof(struct fm10k_l2_accel
,
1252 macvlan
[(l2_accel
->size
* 2) + 1]);
1253 l2_accel
= kzalloc(size
, GFP_KERNEL
);
1255 return ERR_PTR(-ENOMEM
);
1257 memcpy(l2_accel
, old_l2_accel
,
1258 offsetof(struct fm10k_l2_accel
,
1259 macvlan
[old_l2_accel
->size
]));
1261 l2_accel
->size
= (old_l2_accel
->size
* 2) + 1;
1263 /* update pointers */
1264 fm10k_assign_l2_accel(interface
, l2_accel
);
1265 kfree_rcu(old_l2_accel
, rcu
);
1268 /* add macvlan to accel table, and record GLORT for position */
1269 for (i
= 0; i
< l2_accel
->size
; i
++) {
1270 if (!l2_accel
->macvlan
[i
])
1274 /* record station */
1275 l2_accel
->macvlan
[i
] = sdev
;
1278 /* configure default DGLORT mapping for RSS/DCB */
1279 dglort
.idx
= fm10k_dglort_pf_rss
;
1280 dglort
.inner_rss
= 1;
1281 dglort
.rss_l
= fls(interface
->ring_feature
[RING_F_RSS
].mask
);
1282 dglort
.pc_l
= fls(interface
->ring_feature
[RING_F_QOS
].mask
);
1283 dglort
.glort
= interface
->glort
;
1284 dglort
.shared_l
= fls(l2_accel
->size
);
1285 hw
->mac
.ops
.configure_dglort_map(hw
, &dglort
);
1287 /* Add rules for this specific dglort to the switch */
1288 fm10k_mbx_lock(interface
);
1290 glort
= l2_accel
->dglort
+ 1 + i
;
1291 hw
->mac
.ops
.update_xcast_mode(hw
, glort
, FM10K_XCAST_MODE_MULTI
);
1292 hw
->mac
.ops
.update_uc_addr(hw
, glort
, sdev
->dev_addr
, 0, true, 0);
1294 fm10k_mbx_unlock(interface
);
1299 static void fm10k_dfwd_del_station(struct net_device
*dev
, void *priv
)
1301 struct fm10k_intfc
*interface
= netdev_priv(dev
);
1302 struct fm10k_l2_accel
*l2_accel
= ACCESS_ONCE(interface
->l2_accel
);
1303 struct fm10k_dglort_cfg dglort
= { 0 };
1304 struct fm10k_hw
*hw
= &interface
->hw
;
1305 struct net_device
*sdev
= priv
;
1312 /* search table for matching interface */
1313 for (i
= 0; i
< l2_accel
->size
; i
++) {
1314 if (l2_accel
->macvlan
[i
] == sdev
)
1318 /* exit if macvlan not found */
1319 if (i
== l2_accel
->size
)
1322 /* Remove any rules specific to this dglort */
1323 fm10k_mbx_lock(interface
);
1325 glort
= l2_accel
->dglort
+ 1 + i
;
1326 hw
->mac
.ops
.update_xcast_mode(hw
, glort
, FM10K_XCAST_MODE_NONE
);
1327 hw
->mac
.ops
.update_uc_addr(hw
, glort
, sdev
->dev_addr
, 0, false, 0);
1329 fm10k_mbx_unlock(interface
);
1331 /* record removal */
1332 l2_accel
->macvlan
[i
] = NULL
;
1335 /* configure default DGLORT mapping for RSS/DCB */
1336 dglort
.idx
= fm10k_dglort_pf_rss
;
1337 dglort
.inner_rss
= 1;
1338 dglort
.rss_l
= fls(interface
->ring_feature
[RING_F_RSS
].mask
);
1339 dglort
.pc_l
= fls(interface
->ring_feature
[RING_F_QOS
].mask
);
1340 dglort
.glort
= interface
->glort
;
1341 dglort
.shared_l
= fls(l2_accel
->size
);
1342 hw
->mac
.ops
.configure_dglort_map(hw
, &dglort
);
1344 /* If table is empty remove it */
1345 if (l2_accel
->count
== 0) {
1346 fm10k_assign_l2_accel(interface
, NULL
);
1347 kfree_rcu(l2_accel
, rcu
);
1351 static netdev_features_t
fm10k_features_check(struct sk_buff
*skb
,
1352 struct net_device
*dev
,
1353 netdev_features_t features
)
1355 if (!skb
->encapsulation
|| fm10k_tx_encap_offload(skb
))
1358 return features
& ~(NETIF_F_CSUM_MASK
| NETIF_F_GSO_MASK
);
1361 static const struct net_device_ops fm10k_netdev_ops
= {
1362 .ndo_open
= fm10k_open
,
1363 .ndo_stop
= fm10k_close
,
1364 .ndo_validate_addr
= eth_validate_addr
,
1365 .ndo_start_xmit
= fm10k_xmit_frame
,
1366 .ndo_set_mac_address
= fm10k_set_mac
,
1367 .ndo_change_mtu
= fm10k_change_mtu
,
1368 .ndo_tx_timeout
= fm10k_tx_timeout
,
1369 .ndo_vlan_rx_add_vid
= fm10k_vlan_rx_add_vid
,
1370 .ndo_vlan_rx_kill_vid
= fm10k_vlan_rx_kill_vid
,
1371 .ndo_set_rx_mode
= fm10k_set_rx_mode
,
1372 .ndo_get_stats64
= fm10k_get_stats64
,
1373 .ndo_setup_tc
= __fm10k_setup_tc
,
1374 .ndo_set_vf_mac
= fm10k_ndo_set_vf_mac
,
1375 .ndo_set_vf_vlan
= fm10k_ndo_set_vf_vlan
,
1376 .ndo_set_vf_rate
= fm10k_ndo_set_vf_bw
,
1377 .ndo_get_vf_config
= fm10k_ndo_get_vf_config
,
1378 .ndo_udp_tunnel_add
= fm10k_add_vxlan_port
,
1379 .ndo_udp_tunnel_del
= fm10k_del_vxlan_port
,
1380 .ndo_dfwd_add_station
= fm10k_dfwd_add_station
,
1381 .ndo_dfwd_del_station
= fm10k_dfwd_del_station
,
1382 #ifdef CONFIG_NET_POLL_CONTROLLER
1383 .ndo_poll_controller
= fm10k_netpoll
,
1385 .ndo_features_check
= fm10k_features_check
,
1388 #define DEFAULT_DEBUG_LEVEL_SHIFT 3
1390 struct net_device
*fm10k_alloc_netdev(const struct fm10k_info
*info
)
1392 netdev_features_t hw_features
;
1393 struct fm10k_intfc
*interface
;
1394 struct net_device
*dev
;
1396 dev
= alloc_etherdev_mq(sizeof(struct fm10k_intfc
), MAX_QUEUES
);
1400 /* set net device and ethtool ops */
1401 dev
->netdev_ops
= &fm10k_netdev_ops
;
1402 fm10k_set_ethtool_ops(dev
);
1404 /* configure default debug level */
1405 interface
= netdev_priv(dev
);
1406 interface
->msg_enable
= BIT(DEFAULT_DEBUG_LEVEL_SHIFT
) - 1;
1408 /* configure default features */
1409 dev
->features
|= NETIF_F_IP_CSUM
|
1418 /* Only the PF can support VXLAN and NVGRE tunnel offloads */
1419 if (info
->mac
== fm10k_mac_pf
) {
1420 dev
->hw_enc_features
= NETIF_F_IP_CSUM
|
1424 NETIF_F_GSO_UDP_TUNNEL
|
1428 dev
->features
|= NETIF_F_GSO_UDP_TUNNEL
;
1431 /* all features defined to this point should be changeable */
1432 hw_features
= dev
->features
;
1434 /* allow user to enable L2 forwarding acceleration */
1435 hw_features
|= NETIF_F_HW_L2FW_DOFFLOAD
;
1437 /* configure VLAN features */
1438 dev
->vlan_features
|= dev
->features
;
1440 /* we want to leave these both on as we cannot disable VLAN tag
1441 * insertion or stripping on the hardware since it is contained
1442 * in the FTAG and not in the frame itself.
1444 dev
->features
|= NETIF_F_HW_VLAN_CTAG_TX
|
1445 NETIF_F_HW_VLAN_CTAG_RX
|
1446 NETIF_F_HW_VLAN_CTAG_FILTER
;
1448 dev
->priv_flags
|= IFF_UNICAST_FLT
;
1450 dev
->hw_features
|= hw_features
;