1 /* Intel(R) Ethernet Switch Host Interface Driver
2 * Copyright(c) 2013 - 2016 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * The full GNU General Public License is included in this distribution in
14 * the file called "COPYING".
16 * Contact Information:
17 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
18 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
22 #include <linux/vmalloc.h>
23 #include <net/udp_tunnel.h>
26 * fm10k_setup_tx_resources - allocate Tx resources (Descriptors)
27 * @tx_ring: tx descriptor ring (for a specific queue) to setup
29 * Return 0 on success, negative on failure
31 int fm10k_setup_tx_resources(struct fm10k_ring
*tx_ring
)
33 struct device
*dev
= tx_ring
->dev
;
36 size
= sizeof(struct fm10k_tx_buffer
) * tx_ring
->count
;
38 tx_ring
->tx_buffer
= vzalloc(size
);
39 if (!tx_ring
->tx_buffer
)
42 u64_stats_init(&tx_ring
->syncp
);
44 /* round up to nearest 4K */
45 tx_ring
->size
= tx_ring
->count
* sizeof(struct fm10k_tx_desc
);
46 tx_ring
->size
= ALIGN(tx_ring
->size
, 4096);
48 tx_ring
->desc
= dma_alloc_coherent(dev
, tx_ring
->size
,
49 &tx_ring
->dma
, GFP_KERNEL
);
56 vfree(tx_ring
->tx_buffer
);
57 tx_ring
->tx_buffer
= NULL
;
62 * fm10k_setup_all_tx_resources - allocate all queues Tx resources
63 * @interface: board private structure
65 * If this function returns with an error, then it's possible one or
66 * more of the rings is populated (while the rest are not). It is the
67 * callers duty to clean those orphaned rings.
69 * Return 0 on success, negative on failure
71 static int fm10k_setup_all_tx_resources(struct fm10k_intfc
*interface
)
75 for (i
= 0; i
< interface
->num_tx_queues
; i
++) {
76 err
= fm10k_setup_tx_resources(interface
->tx_ring
[i
]);
80 netif_err(interface
, probe
, interface
->netdev
,
81 "Allocation for Tx Queue %u failed\n", i
);
87 /* rewind the index freeing the rings as we go */
89 fm10k_free_tx_resources(interface
->tx_ring
[i
]);
94 * fm10k_setup_rx_resources - allocate Rx resources (Descriptors)
95 * @rx_ring: rx descriptor ring (for a specific queue) to setup
97 * Returns 0 on success, negative on failure
99 int fm10k_setup_rx_resources(struct fm10k_ring
*rx_ring
)
101 struct device
*dev
= rx_ring
->dev
;
104 size
= sizeof(struct fm10k_rx_buffer
) * rx_ring
->count
;
106 rx_ring
->rx_buffer
= vzalloc(size
);
107 if (!rx_ring
->rx_buffer
)
110 u64_stats_init(&rx_ring
->syncp
);
112 /* Round up to nearest 4K */
113 rx_ring
->size
= rx_ring
->count
* sizeof(union fm10k_rx_desc
);
114 rx_ring
->size
= ALIGN(rx_ring
->size
, 4096);
116 rx_ring
->desc
= dma_alloc_coherent(dev
, rx_ring
->size
,
117 &rx_ring
->dma
, GFP_KERNEL
);
123 vfree(rx_ring
->rx_buffer
);
124 rx_ring
->rx_buffer
= NULL
;
129 * fm10k_setup_all_rx_resources - allocate all queues Rx resources
130 * @interface: board private structure
132 * If this function returns with an error, then it's possible one or
133 * more of the rings is populated (while the rest are not). It is the
134 * callers duty to clean those orphaned rings.
136 * Return 0 on success, negative on failure
138 static int fm10k_setup_all_rx_resources(struct fm10k_intfc
*interface
)
142 for (i
= 0; i
< interface
->num_rx_queues
; i
++) {
143 err
= fm10k_setup_rx_resources(interface
->rx_ring
[i
]);
147 netif_err(interface
, probe
, interface
->netdev
,
148 "Allocation for Rx Queue %u failed\n", i
);
154 /* rewind the index freeing the rings as we go */
156 fm10k_free_rx_resources(interface
->rx_ring
[i
]);
160 void fm10k_unmap_and_free_tx_resource(struct fm10k_ring
*ring
,
161 struct fm10k_tx_buffer
*tx_buffer
)
163 if (tx_buffer
->skb
) {
164 dev_kfree_skb_any(tx_buffer
->skb
);
165 if (dma_unmap_len(tx_buffer
, len
))
166 dma_unmap_single(ring
->dev
,
167 dma_unmap_addr(tx_buffer
, dma
),
168 dma_unmap_len(tx_buffer
, len
),
170 } else if (dma_unmap_len(tx_buffer
, len
)) {
171 dma_unmap_page(ring
->dev
,
172 dma_unmap_addr(tx_buffer
, dma
),
173 dma_unmap_len(tx_buffer
, len
),
176 tx_buffer
->next_to_watch
= NULL
;
177 tx_buffer
->skb
= NULL
;
178 dma_unmap_len_set(tx_buffer
, len
, 0);
179 /* tx_buffer must be completely set up in the transmit path */
183 * fm10k_clean_tx_ring - Free Tx Buffers
184 * @tx_ring: ring to be cleaned
186 static void fm10k_clean_tx_ring(struct fm10k_ring
*tx_ring
)
188 struct fm10k_tx_buffer
*tx_buffer
;
192 /* ring already cleared, nothing to do */
193 if (!tx_ring
->tx_buffer
)
196 /* Free all the Tx ring sk_buffs */
197 for (i
= 0; i
< tx_ring
->count
; i
++) {
198 tx_buffer
= &tx_ring
->tx_buffer
[i
];
199 fm10k_unmap_and_free_tx_resource(tx_ring
, tx_buffer
);
202 /* reset BQL values */
203 netdev_tx_reset_queue(txring_txq(tx_ring
));
205 size
= sizeof(struct fm10k_tx_buffer
) * tx_ring
->count
;
206 memset(tx_ring
->tx_buffer
, 0, size
);
208 /* Zero out the descriptor ring */
209 memset(tx_ring
->desc
, 0, tx_ring
->size
);
213 * fm10k_free_tx_resources - Free Tx Resources per Queue
214 * @tx_ring: Tx descriptor ring for a specific queue
216 * Free all transmit software resources
218 void fm10k_free_tx_resources(struct fm10k_ring
*tx_ring
)
220 fm10k_clean_tx_ring(tx_ring
);
222 vfree(tx_ring
->tx_buffer
);
223 tx_ring
->tx_buffer
= NULL
;
225 /* if not set, then don't free */
229 dma_free_coherent(tx_ring
->dev
, tx_ring
->size
,
230 tx_ring
->desc
, tx_ring
->dma
);
231 tx_ring
->desc
= NULL
;
235 * fm10k_clean_all_tx_rings - Free Tx Buffers for all queues
236 * @interface: board private structure
238 void fm10k_clean_all_tx_rings(struct fm10k_intfc
*interface
)
242 for (i
= 0; i
< interface
->num_tx_queues
; i
++)
243 fm10k_clean_tx_ring(interface
->tx_ring
[i
]);
247 * fm10k_free_all_tx_resources - Free Tx Resources for All Queues
248 * @interface: board private structure
250 * Free all transmit software resources
252 static void fm10k_free_all_tx_resources(struct fm10k_intfc
*interface
)
254 int i
= interface
->num_tx_queues
;
257 fm10k_free_tx_resources(interface
->tx_ring
[i
]);
261 * fm10k_clean_rx_ring - Free Rx Buffers per Queue
262 * @rx_ring: ring to free buffers from
264 static void fm10k_clean_rx_ring(struct fm10k_ring
*rx_ring
)
269 if (!rx_ring
->rx_buffer
)
273 dev_kfree_skb(rx_ring
->skb
);
276 /* Free all the Rx ring sk_buffs */
277 for (i
= 0; i
< rx_ring
->count
; i
++) {
278 struct fm10k_rx_buffer
*buffer
= &rx_ring
->rx_buffer
[i
];
279 /* clean-up will only set page pointer to NULL */
283 dma_unmap_page(rx_ring
->dev
, buffer
->dma
,
284 PAGE_SIZE
, DMA_FROM_DEVICE
);
285 __free_page(buffer
->page
);
290 size
= sizeof(struct fm10k_rx_buffer
) * rx_ring
->count
;
291 memset(rx_ring
->rx_buffer
, 0, size
);
293 /* Zero out the descriptor ring */
294 memset(rx_ring
->desc
, 0, rx_ring
->size
);
296 rx_ring
->next_to_alloc
= 0;
297 rx_ring
->next_to_clean
= 0;
298 rx_ring
->next_to_use
= 0;
302 * fm10k_free_rx_resources - Free Rx Resources
303 * @rx_ring: ring to clean the resources from
305 * Free all receive software resources
307 void fm10k_free_rx_resources(struct fm10k_ring
*rx_ring
)
309 fm10k_clean_rx_ring(rx_ring
);
311 vfree(rx_ring
->rx_buffer
);
312 rx_ring
->rx_buffer
= NULL
;
314 /* if not set, then don't free */
318 dma_free_coherent(rx_ring
->dev
, rx_ring
->size
,
319 rx_ring
->desc
, rx_ring
->dma
);
321 rx_ring
->desc
= NULL
;
325 * fm10k_clean_all_rx_rings - Free Rx Buffers for all queues
326 * @interface: board private structure
328 void fm10k_clean_all_rx_rings(struct fm10k_intfc
*interface
)
332 for (i
= 0; i
< interface
->num_rx_queues
; i
++)
333 fm10k_clean_rx_ring(interface
->rx_ring
[i
]);
337 * fm10k_free_all_rx_resources - Free Rx Resources for All Queues
338 * @interface: board private structure
340 * Free all receive software resources
342 static void fm10k_free_all_rx_resources(struct fm10k_intfc
*interface
)
344 int i
= interface
->num_rx_queues
;
347 fm10k_free_rx_resources(interface
->rx_ring
[i
]);
351 * fm10k_request_glort_range - Request GLORTs for use in configuring rules
352 * @interface: board private structure
354 * This function allocates a range of glorts for this interface to use.
356 static void fm10k_request_glort_range(struct fm10k_intfc
*interface
)
358 struct fm10k_hw
*hw
= &interface
->hw
;
359 u16 mask
= (~hw
->mac
.dglort_map
) >> FM10K_DGLORTMAP_MASK_SHIFT
;
361 /* establish GLORT base */
362 interface
->glort
= hw
->mac
.dglort_map
& FM10K_DGLORTMAP_NONE
;
363 interface
->glort_count
= 0;
365 /* nothing we can do until mask is allocated */
366 if (hw
->mac
.dglort_map
== FM10K_DGLORTMAP_NONE
)
369 /* we support 3 possible GLORT configurations.
370 * 1: VFs consume all but the last 1
371 * 2: VFs and PF split glorts with possible gap between
372 * 3: VFs allocated first 64, all others belong to PF
374 if (mask
<= hw
->iov
.total_vfs
) {
375 interface
->glort_count
= 1;
376 interface
->glort
+= mask
;
377 } else if (mask
< 64) {
378 interface
->glort_count
= (mask
+ 1) / 2;
379 interface
->glort
+= interface
->glort_count
;
381 interface
->glort_count
= mask
- 63;
382 interface
->glort
+= 64;
387 * fm10k_del_vxlan_port_all
388 * @interface: board private structure
390 * This function frees the entire vxlan_port list
392 static void fm10k_del_vxlan_port_all(struct fm10k_intfc
*interface
)
394 struct fm10k_vxlan_port
*vxlan_port
;
396 /* flush all entries from list */
397 vxlan_port
= list_first_entry_or_null(&interface
->vxlan_port
,
398 struct fm10k_vxlan_port
, list
);
400 list_del(&vxlan_port
->list
);
402 vxlan_port
= list_first_entry_or_null(&interface
->vxlan_port
,
403 struct fm10k_vxlan_port
,
409 * fm10k_restore_vxlan_port
410 * @interface: board private structure
412 * This function restores the value in the tunnel_cfg register after reset
414 static void fm10k_restore_vxlan_port(struct fm10k_intfc
*interface
)
416 struct fm10k_hw
*hw
= &interface
->hw
;
417 struct fm10k_vxlan_port
*vxlan_port
;
419 /* only the PF supports configuring tunnels */
420 if (hw
->mac
.type
!= fm10k_mac_pf
)
423 vxlan_port
= list_first_entry_or_null(&interface
->vxlan_port
,
424 struct fm10k_vxlan_port
, list
);
426 /* restore tunnel configuration register */
427 fm10k_write_reg(hw
, FM10K_TUNNEL_CFG
,
428 (vxlan_port
? ntohs(vxlan_port
->port
) : 0) |
429 (ETH_P_TEB
<< FM10K_TUNNEL_CFG_NVGRE_SHIFT
));
433 * fm10k_add_vxlan_port
434 * @netdev: network interface device structure
435 * @sa_family: Address family of new port
436 * @port: port number used for VXLAN
437 * @type: Enumerated value specifying udp encapsulation type
439 * This function is called when a new VXLAN interface has added a new port
440 * number to the range that is currently in use for VXLAN. The new port
441 * number is always added to the tail so that the port number list should
442 * match the order in which the ports were allocated. The head of the list
443 * is always used as the VXLAN port number for offloads.
445 static void fm10k_add_vxlan_port(struct net_device
*dev
,
446 struct udp_tunnel_info
*ti
)
448 struct fm10k_intfc
*interface
= netdev_priv(dev
);
449 struct fm10k_vxlan_port
*vxlan_port
;
451 if (ti
->type
!= UDP_TUNNEL_TYPE_VXLAN
)
453 /* only the PF supports configuring tunnels */
454 if (interface
->hw
.mac
.type
!= fm10k_mac_pf
)
457 /* existing ports are pulled out so our new entry is always last */
458 fm10k_vxlan_port_for_each(vxlan_port
, interface
) {
459 if ((vxlan_port
->port
== ti
->port
) &&
460 (vxlan_port
->sa_family
== ti
->sa_family
)) {
461 list_del(&vxlan_port
->list
);
466 /* allocate memory to track ports */
467 vxlan_port
= kmalloc(sizeof(*vxlan_port
), GFP_ATOMIC
);
470 vxlan_port
->port
= ti
->port
;
471 vxlan_port
->sa_family
= ti
->sa_family
;
474 /* add new port value to list */
475 list_add_tail(&vxlan_port
->list
, &interface
->vxlan_port
);
477 fm10k_restore_vxlan_port(interface
);
481 * fm10k_del_vxlan_port
482 * @netdev: network interface device structure
483 * @sa_family: Address family of freed port
484 * @port: port number used for VXLAN
485 * @type: Enumerated value specifying udp encapsulation type
487 * This function is called when a new VXLAN interface has freed a port
488 * number from the range that is currently in use for VXLAN. The freed
489 * port is removed from the list and the new head is used to determine
490 * the port number for offloads.
492 static void fm10k_del_vxlan_port(struct net_device
*dev
,
493 struct udp_tunnel_info
*ti
)
495 struct fm10k_intfc
*interface
= netdev_priv(dev
);
496 struct fm10k_vxlan_port
*vxlan_port
;
498 if (ti
->type
!= UDP_TUNNEL_TYPE_VXLAN
)
500 if (interface
->hw
.mac
.type
!= fm10k_mac_pf
)
503 /* find the port in the list and free it */
504 fm10k_vxlan_port_for_each(vxlan_port
, interface
) {
505 if ((vxlan_port
->port
== ti
->port
) &&
506 (vxlan_port
->sa_family
== ti
->sa_family
)) {
507 list_del(&vxlan_port
->list
);
513 fm10k_restore_vxlan_port(interface
);
517 * fm10k_open - Called when a network interface is made active
518 * @netdev: network interface device structure
520 * Returns 0 on success, negative value on failure
522 * The open entry point is called when a network interface is made
523 * active by the system (IFF_UP). At this point all resources needed
524 * for transmit and receive operations are allocated, the interrupt
525 * handler is registered with the OS, the watchdog timer is started,
526 * and the stack is notified that the interface is ready.
528 int fm10k_open(struct net_device
*netdev
)
530 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
533 /* allocate transmit descriptors */
534 err
= fm10k_setup_all_tx_resources(interface
);
538 /* allocate receive descriptors */
539 err
= fm10k_setup_all_rx_resources(interface
);
543 /* allocate interrupt resources */
544 err
= fm10k_qv_request_irq(interface
);
548 /* setup GLORT assignment for this port */
549 fm10k_request_glort_range(interface
);
551 /* Notify the stack of the actual queue counts */
552 err
= netif_set_real_num_tx_queues(netdev
,
553 interface
->num_tx_queues
);
557 err
= netif_set_real_num_rx_queues(netdev
,
558 interface
->num_rx_queues
);
562 /* update VXLAN port configuration */
563 udp_tunnel_get_rx_info(netdev
);
570 fm10k_qv_free_irq(interface
);
572 fm10k_free_all_rx_resources(interface
);
574 fm10k_free_all_tx_resources(interface
);
580 * fm10k_close - Disables a network interface
581 * @netdev: network interface device structure
583 * Returns 0, this is not allowed to fail
585 * The close entry point is called when an interface is de-activated
586 * by the OS. The hardware is still under the drivers control, but
587 * needs to be disabled. A global MAC reset is issued to stop the
588 * hardware, and all transmit and receive resources are freed.
590 int fm10k_close(struct net_device
*netdev
)
592 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
594 fm10k_down(interface
);
596 fm10k_qv_free_irq(interface
);
598 fm10k_del_vxlan_port_all(interface
);
600 fm10k_free_all_tx_resources(interface
);
601 fm10k_free_all_rx_resources(interface
);
606 static netdev_tx_t
fm10k_xmit_frame(struct sk_buff
*skb
, struct net_device
*dev
)
608 struct fm10k_intfc
*interface
= netdev_priv(dev
);
609 unsigned int r_idx
= skb
->queue_mapping
;
612 if ((skb
->protocol
== htons(ETH_P_8021Q
)) &&
613 !skb_vlan_tag_present(skb
)) {
614 /* FM10K only supports hardware tagging, any tags in frame
615 * are considered 2nd level or "outer" tags
617 struct vlan_hdr
*vhdr
;
620 /* make sure skb is not shared */
621 skb
= skb_share_check(skb
, GFP_ATOMIC
);
625 /* make sure there is enough room to move the ethernet header */
626 if (unlikely(!pskb_may_pull(skb
, VLAN_ETH_HLEN
)))
629 /* verify the skb head is not shared */
630 err
= skb_cow_head(skb
, 0);
636 /* locate VLAN header */
637 vhdr
= (struct vlan_hdr
*)(skb
->data
+ ETH_HLEN
);
639 /* pull the 2 key pieces of data out of it */
640 __vlan_hwaccel_put_tag(skb
,
642 ntohs(vhdr
->h_vlan_TCI
));
643 proto
= vhdr
->h_vlan_encapsulated_proto
;
644 skb
->protocol
= (ntohs(proto
) >= 1536) ? proto
:
647 /* squash it by moving the ethernet addresses up 4 bytes */
648 memmove(skb
->data
+ VLAN_HLEN
, skb
->data
, 12);
649 __skb_pull(skb
, VLAN_HLEN
);
650 skb_reset_mac_header(skb
);
653 /* The minimum packet size for a single buffer is 17B so pad the skb
654 * in order to meet this minimum size requirement.
656 if (unlikely(skb
->len
< 17)) {
657 int pad_len
= 17 - skb
->len
;
659 if (skb_pad(skb
, pad_len
))
661 __skb_put(skb
, pad_len
);
664 if (r_idx
>= interface
->num_tx_queues
)
665 r_idx
%= interface
->num_tx_queues
;
667 err
= fm10k_xmit_frame_ring(skb
, interface
->tx_ring
[r_idx
]);
672 static int fm10k_change_mtu(struct net_device
*dev
, int new_mtu
)
674 if (new_mtu
< 68 || new_mtu
> FM10K_MAX_JUMBO_FRAME_SIZE
)
683 * fm10k_tx_timeout - Respond to a Tx Hang
684 * @netdev: network interface device structure
686 static void fm10k_tx_timeout(struct net_device
*netdev
)
688 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
689 bool real_tx_hang
= false;
692 #define TX_TIMEO_LIMIT 16000
693 for (i
= 0; i
< interface
->num_tx_queues
; i
++) {
694 struct fm10k_ring
*tx_ring
= interface
->tx_ring
[i
];
696 if (check_for_tx_hang(tx_ring
) && fm10k_check_tx_hang(tx_ring
))
701 fm10k_tx_timeout_reset(interface
);
703 netif_info(interface
, drv
, netdev
,
704 "Fake Tx hang detected with timeout of %d seconds\n",
705 netdev
->watchdog_timeo
/ HZ
);
707 /* fake Tx hang - increase the kernel timeout */
708 if (netdev
->watchdog_timeo
< TX_TIMEO_LIMIT
)
709 netdev
->watchdog_timeo
*= 2;
713 static int fm10k_uc_vlan_unsync(struct net_device
*netdev
,
714 const unsigned char *uc_addr
)
716 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
717 struct fm10k_hw
*hw
= &interface
->hw
;
718 u16 glort
= interface
->glort
;
719 u16 vid
= interface
->vid
;
720 bool set
= !!(vid
/ VLAN_N_VID
);
723 /* drop any leading bits on the VLAN ID */
724 vid
&= VLAN_N_VID
- 1;
726 err
= hw
->mac
.ops
.update_uc_addr(hw
, glort
, uc_addr
, vid
, set
, 0);
730 /* return non-zero value as we are only doing a partial sync/unsync */
734 static int fm10k_mc_vlan_unsync(struct net_device
*netdev
,
735 const unsigned char *mc_addr
)
737 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
738 struct fm10k_hw
*hw
= &interface
->hw
;
739 u16 glort
= interface
->glort
;
740 u16 vid
= interface
->vid
;
741 bool set
= !!(vid
/ VLAN_N_VID
);
744 /* drop any leading bits on the VLAN ID */
745 vid
&= VLAN_N_VID
- 1;
747 err
= hw
->mac
.ops
.update_mc_addr(hw
, glort
, mc_addr
, vid
, set
);
751 /* return non-zero value as we are only doing a partial sync/unsync */
755 static int fm10k_update_vid(struct net_device
*netdev
, u16 vid
, bool set
)
757 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
758 struct fm10k_hw
*hw
= &interface
->hw
;
762 /* updates do not apply to VLAN 0 */
766 if (vid
>= VLAN_N_VID
)
769 /* Verify we have permission to add VLANs */
770 if (hw
->mac
.vlan_override
)
773 /* update active_vlans bitmask */
774 set_bit(vid
, interface
->active_vlans
);
776 clear_bit(vid
, interface
->active_vlans
);
778 /* disable the default VLAN ID on ring if we have an active VLAN */
779 for (i
= 0; i
< interface
->num_rx_queues
; i
++) {
780 struct fm10k_ring
*rx_ring
= interface
->rx_ring
[i
];
781 u16 rx_vid
= rx_ring
->vid
& (VLAN_N_VID
- 1);
783 if (test_bit(rx_vid
, interface
->active_vlans
))
784 rx_ring
->vid
|= FM10K_VLAN_CLEAR
;
786 rx_ring
->vid
&= ~FM10K_VLAN_CLEAR
;
789 /* Do not remove default VLAN ID related entries from VLAN and MAC
792 if (!set
&& vid
== hw
->mac
.default_vid
)
795 /* Do not throw an error if the interface is down. We will sync once
798 if (test_bit(__FM10K_DOWN
, &interface
->state
))
801 fm10k_mbx_lock(interface
);
803 /* only need to update the VLAN if not in promiscuous mode */
804 if (!(netdev
->flags
& IFF_PROMISC
)) {
805 err
= hw
->mac
.ops
.update_vlan(hw
, vid
, 0, set
);
810 /* update our base MAC address */
811 err
= hw
->mac
.ops
.update_uc_addr(hw
, interface
->glort
, hw
->mac
.addr
,
816 /* set VLAN ID prior to syncing/unsyncing the VLAN */
817 interface
->vid
= vid
+ (set
? VLAN_N_VID
: 0);
819 /* Update the unicast and multicast address list to add/drop VLAN */
820 __dev_uc_unsync(netdev
, fm10k_uc_vlan_unsync
);
821 __dev_mc_unsync(netdev
, fm10k_mc_vlan_unsync
);
824 fm10k_mbx_unlock(interface
);
829 static int fm10k_vlan_rx_add_vid(struct net_device
*netdev
,
830 __always_unused __be16 proto
, u16 vid
)
832 /* update VLAN and address table based on changes */
833 return fm10k_update_vid(netdev
, vid
, true);
836 static int fm10k_vlan_rx_kill_vid(struct net_device
*netdev
,
837 __always_unused __be16 proto
, u16 vid
)
839 /* update VLAN and address table based on changes */
840 return fm10k_update_vid(netdev
, vid
, false);
843 static u16
fm10k_find_next_vlan(struct fm10k_intfc
*interface
, u16 vid
)
845 struct fm10k_hw
*hw
= &interface
->hw
;
846 u16 default_vid
= hw
->mac
.default_vid
;
847 u16 vid_limit
= vid
< default_vid
? default_vid
: VLAN_N_VID
;
849 vid
= find_next_bit(interface
->active_vlans
, vid_limit
, ++vid
);
854 static void fm10k_clear_unused_vlans(struct fm10k_intfc
*interface
)
856 struct fm10k_hw
*hw
= &interface
->hw
;
859 /* loop through and find any gaps in the table */
860 for (vid
= 0, prev_vid
= 0;
861 prev_vid
< VLAN_N_VID
;
862 prev_vid
= vid
+ 1, vid
= fm10k_find_next_vlan(interface
, vid
)) {
866 /* send request to clear multiple bits at a time */
867 prev_vid
+= (vid
- prev_vid
- 1) << FM10K_VLAN_LENGTH_SHIFT
;
868 hw
->mac
.ops
.update_vlan(hw
, prev_vid
, 0, false);
872 static int __fm10k_uc_sync(struct net_device
*dev
,
873 const unsigned char *addr
, bool sync
)
875 struct fm10k_intfc
*interface
= netdev_priv(dev
);
876 struct fm10k_hw
*hw
= &interface
->hw
;
877 u16 vid
, glort
= interface
->glort
;
880 if (!is_valid_ether_addr(addr
))
881 return -EADDRNOTAVAIL
;
883 /* update table with current entries */
884 for (vid
= hw
->mac
.default_vid
? fm10k_find_next_vlan(interface
, 0) : 1;
886 vid
= fm10k_find_next_vlan(interface
, vid
)) {
887 err
= hw
->mac
.ops
.update_uc_addr(hw
, glort
, addr
,
896 static int fm10k_uc_sync(struct net_device
*dev
,
897 const unsigned char *addr
)
899 return __fm10k_uc_sync(dev
, addr
, true);
902 static int fm10k_uc_unsync(struct net_device
*dev
,
903 const unsigned char *addr
)
905 return __fm10k_uc_sync(dev
, addr
, false);
908 static int fm10k_set_mac(struct net_device
*dev
, void *p
)
910 struct fm10k_intfc
*interface
= netdev_priv(dev
);
911 struct fm10k_hw
*hw
= &interface
->hw
;
912 struct sockaddr
*addr
= p
;
915 if (!is_valid_ether_addr(addr
->sa_data
))
916 return -EADDRNOTAVAIL
;
918 if (dev
->flags
& IFF_UP
) {
919 /* setting MAC address requires mailbox */
920 fm10k_mbx_lock(interface
);
922 err
= fm10k_uc_sync(dev
, addr
->sa_data
);
924 fm10k_uc_unsync(dev
, hw
->mac
.addr
);
926 fm10k_mbx_unlock(interface
);
930 ether_addr_copy(dev
->dev_addr
, addr
->sa_data
);
931 ether_addr_copy(hw
->mac
.addr
, addr
->sa_data
);
932 dev
->addr_assign_type
&= ~NET_ADDR_RANDOM
;
935 /* if we had a mailbox error suggest trying again */
936 return err
? -EAGAIN
: 0;
939 static int __fm10k_mc_sync(struct net_device
*dev
,
940 const unsigned char *addr
, bool sync
)
942 struct fm10k_intfc
*interface
= netdev_priv(dev
);
943 struct fm10k_hw
*hw
= &interface
->hw
;
944 u16 vid
, glort
= interface
->glort
;
946 /* update table with current entries */
947 for (vid
= hw
->mac
.default_vid
? fm10k_find_next_vlan(interface
, 0) : 1;
949 vid
= fm10k_find_next_vlan(interface
, vid
)) {
950 hw
->mac
.ops
.update_mc_addr(hw
, glort
, addr
, vid
, sync
);
956 static int fm10k_mc_sync(struct net_device
*dev
,
957 const unsigned char *addr
)
959 return __fm10k_mc_sync(dev
, addr
, true);
962 static int fm10k_mc_unsync(struct net_device
*dev
,
963 const unsigned char *addr
)
965 return __fm10k_mc_sync(dev
, addr
, false);
968 static void fm10k_set_rx_mode(struct net_device
*dev
)
970 struct fm10k_intfc
*interface
= netdev_priv(dev
);
971 struct fm10k_hw
*hw
= &interface
->hw
;
974 /* no need to update the harwdare if we are not running */
975 if (!(dev
->flags
& IFF_UP
))
978 /* determine new mode based on flags */
979 xcast_mode
= (dev
->flags
& IFF_PROMISC
) ? FM10K_XCAST_MODE_PROMISC
:
980 (dev
->flags
& IFF_ALLMULTI
) ? FM10K_XCAST_MODE_ALLMULTI
:
981 (dev
->flags
& (IFF_BROADCAST
| IFF_MULTICAST
)) ?
982 FM10K_XCAST_MODE_MULTI
: FM10K_XCAST_MODE_NONE
;
984 fm10k_mbx_lock(interface
);
986 /* update xcast mode first, but only if it changed */
987 if (interface
->xcast_mode
!= xcast_mode
) {
988 /* update VLAN table */
989 if (xcast_mode
== FM10K_XCAST_MODE_PROMISC
)
990 hw
->mac
.ops
.update_vlan(hw
, FM10K_VLAN_ALL
, 0, true);
991 if (interface
->xcast_mode
== FM10K_XCAST_MODE_PROMISC
)
992 fm10k_clear_unused_vlans(interface
);
994 /* update xcast mode */
995 hw
->mac
.ops
.update_xcast_mode(hw
, interface
->glort
, xcast_mode
);
997 /* record updated xcast mode state */
998 interface
->xcast_mode
= xcast_mode
;
1001 /* synchronize all of the addresses */
1002 __dev_uc_sync(dev
, fm10k_uc_sync
, fm10k_uc_unsync
);
1003 __dev_mc_sync(dev
, fm10k_mc_sync
, fm10k_mc_unsync
);
1005 fm10k_mbx_unlock(interface
);
1008 void fm10k_restore_rx_state(struct fm10k_intfc
*interface
)
1010 struct net_device
*netdev
= interface
->netdev
;
1011 struct fm10k_hw
*hw
= &interface
->hw
;
1015 /* record glort for this interface */
1016 glort
= interface
->glort
;
1018 /* convert interface flags to xcast mode */
1019 if (netdev
->flags
& IFF_PROMISC
)
1020 xcast_mode
= FM10K_XCAST_MODE_PROMISC
;
1021 else if (netdev
->flags
& IFF_ALLMULTI
)
1022 xcast_mode
= FM10K_XCAST_MODE_ALLMULTI
;
1023 else if (netdev
->flags
& (IFF_BROADCAST
| IFF_MULTICAST
))
1024 xcast_mode
= FM10K_XCAST_MODE_MULTI
;
1026 xcast_mode
= FM10K_XCAST_MODE_NONE
;
1028 fm10k_mbx_lock(interface
);
1030 /* Enable logical port */
1031 hw
->mac
.ops
.update_lport_state(hw
, glort
, interface
->glort_count
, true);
1033 /* update VLAN table */
1034 hw
->mac
.ops
.update_vlan(hw
, FM10K_VLAN_ALL
, 0,
1035 xcast_mode
== FM10K_XCAST_MODE_PROMISC
);
1037 /* Add filter for VLAN 0 */
1038 hw
->mac
.ops
.update_vlan(hw
, 0, 0, true);
1040 /* update table with current entries */
1041 for (vid
= hw
->mac
.default_vid
? fm10k_find_next_vlan(interface
, 0) : 1;
1043 vid
= fm10k_find_next_vlan(interface
, vid
)) {
1044 hw
->mac
.ops
.update_vlan(hw
, vid
, 0, true);
1045 hw
->mac
.ops
.update_uc_addr(hw
, glort
, hw
->mac
.addr
,
1049 /* update xcast mode before synchronizing addresses */
1050 hw
->mac
.ops
.update_xcast_mode(hw
, glort
, xcast_mode
);
1052 /* synchronize all of the addresses */
1053 __dev_uc_sync(netdev
, fm10k_uc_sync
, fm10k_uc_unsync
);
1054 __dev_mc_sync(netdev
, fm10k_mc_sync
, fm10k_mc_unsync
);
1056 fm10k_mbx_unlock(interface
);
1058 /* record updated xcast mode state */
1059 interface
->xcast_mode
= xcast_mode
;
1061 /* Restore tunnel configuration */
1062 fm10k_restore_vxlan_port(interface
);
1065 void fm10k_reset_rx_state(struct fm10k_intfc
*interface
)
1067 struct net_device
*netdev
= interface
->netdev
;
1068 struct fm10k_hw
*hw
= &interface
->hw
;
1070 fm10k_mbx_lock(interface
);
1072 /* clear the logical port state on lower device */
1073 hw
->mac
.ops
.update_lport_state(hw
, interface
->glort
,
1074 interface
->glort_count
, false);
1076 fm10k_mbx_unlock(interface
);
1078 /* reset flags to default state */
1079 interface
->xcast_mode
= FM10K_XCAST_MODE_NONE
;
1081 /* clear the sync flag since the lport has been dropped */
1082 __dev_uc_unsync(netdev
, NULL
);
1083 __dev_mc_unsync(netdev
, NULL
);
1087 * fm10k_get_stats64 - Get System Network Statistics
1088 * @netdev: network interface device structure
1089 * @stats: storage space for 64bit statistics
1091 * Returns 64bit statistics, for use in the ndo_get_stats64 callback. This
1092 * function replaces fm10k_get_stats for kernels which support it.
1094 static struct rtnl_link_stats64
*fm10k_get_stats64(struct net_device
*netdev
,
1095 struct rtnl_link_stats64
*stats
)
1097 struct fm10k_intfc
*interface
= netdev_priv(netdev
);
1098 struct fm10k_ring
*ring
;
1099 unsigned int start
, i
;
1104 for (i
= 0; i
< interface
->num_rx_queues
; i
++) {
1105 ring
= ACCESS_ONCE(interface
->rx_ring
[i
]);
1111 start
= u64_stats_fetch_begin_irq(&ring
->syncp
);
1112 packets
= ring
->stats
.packets
;
1113 bytes
= ring
->stats
.bytes
;
1114 } while (u64_stats_fetch_retry_irq(&ring
->syncp
, start
));
1116 stats
->rx_packets
+= packets
;
1117 stats
->rx_bytes
+= bytes
;
1120 for (i
= 0; i
< interface
->num_tx_queues
; i
++) {
1121 ring
= ACCESS_ONCE(interface
->tx_ring
[i
]);
1127 start
= u64_stats_fetch_begin_irq(&ring
->syncp
);
1128 packets
= ring
->stats
.packets
;
1129 bytes
= ring
->stats
.bytes
;
1130 } while (u64_stats_fetch_retry_irq(&ring
->syncp
, start
));
1132 stats
->tx_packets
+= packets
;
1133 stats
->tx_bytes
+= bytes
;
1138 /* following stats updated by fm10k_service_task() */
1139 stats
->rx_missed_errors
= netdev
->stats
.rx_missed_errors
;
1144 int fm10k_setup_tc(struct net_device
*dev
, u8 tc
)
1146 struct fm10k_intfc
*interface
= netdev_priv(dev
);
1149 /* Currently only the PF supports priority classes */
1150 if (tc
&& (interface
->hw
.mac
.type
!= fm10k_mac_pf
))
1153 /* Hardware supports up to 8 traffic classes */
1157 /* Hardware has to reinitialize queues to match packet
1158 * buffer alignment. Unfortunately, the hardware is not
1159 * flexible enough to do this dynamically.
1161 if (netif_running(dev
))
1164 fm10k_mbx_free_irq(interface
);
1166 fm10k_clear_queueing_scheme(interface
);
1168 /* we expect the prio_tc map to be repopulated later */
1169 netdev_reset_tc(dev
);
1170 netdev_set_num_tc(dev
, tc
);
1172 err
= fm10k_init_queueing_scheme(interface
);
1174 goto err_queueing_scheme
;
1176 err
= fm10k_mbx_request_irq(interface
);
1180 err
= netif_running(dev
) ? fm10k_open(dev
) : 0;
1184 /* flag to indicate SWPRI has yet to be updated */
1185 interface
->flags
|= FM10K_FLAG_SWPRI_CONFIG
;
1189 fm10k_mbx_free_irq(interface
);
1191 fm10k_clear_queueing_scheme(interface
);
1192 err_queueing_scheme
:
1193 netif_device_detach(dev
);
1198 static int __fm10k_setup_tc(struct net_device
*dev
, u32 handle
, __be16 proto
,
1199 struct tc_to_netdev
*tc
)
1201 if (tc
->type
!= TC_SETUP_MQPRIO
)
1204 return fm10k_setup_tc(dev
, tc
->tc
);
1207 static void fm10k_assign_l2_accel(struct fm10k_intfc
*interface
,
1208 struct fm10k_l2_accel
*l2_accel
)
1210 struct fm10k_ring
*ring
;
1213 for (i
= 0; i
< interface
->num_rx_queues
; i
++) {
1214 ring
= interface
->rx_ring
[i
];
1215 rcu_assign_pointer(ring
->l2_accel
, l2_accel
);
1218 interface
->l2_accel
= l2_accel
;
1221 static void *fm10k_dfwd_add_station(struct net_device
*dev
,
1222 struct net_device
*sdev
)
1224 struct fm10k_intfc
*interface
= netdev_priv(dev
);
1225 struct fm10k_l2_accel
*l2_accel
= interface
->l2_accel
;
1226 struct fm10k_l2_accel
*old_l2_accel
= NULL
;
1227 struct fm10k_dglort_cfg dglort
= { 0 };
1228 struct fm10k_hw
*hw
= &interface
->hw
;
1232 /* allocate l2 accel structure if it is not available */
1234 /* verify there is enough free GLORTs to support l2_accel */
1235 if (interface
->glort_count
< 7)
1236 return ERR_PTR(-EBUSY
);
1238 size
= offsetof(struct fm10k_l2_accel
, macvlan
[7]);
1239 l2_accel
= kzalloc(size
, GFP_KERNEL
);
1241 return ERR_PTR(-ENOMEM
);
1244 l2_accel
->dglort
= interface
->glort
;
1246 /* update pointers */
1247 fm10k_assign_l2_accel(interface
, l2_accel
);
1248 /* do not expand if we are at our limit */
1249 } else if ((l2_accel
->count
== FM10K_MAX_STATIONS
) ||
1250 (l2_accel
->count
== (interface
->glort_count
- 1))) {
1251 return ERR_PTR(-EBUSY
);
1252 /* expand if we have hit the size limit */
1253 } else if (l2_accel
->count
== l2_accel
->size
) {
1254 old_l2_accel
= l2_accel
;
1255 size
= offsetof(struct fm10k_l2_accel
,
1256 macvlan
[(l2_accel
->size
* 2) + 1]);
1257 l2_accel
= kzalloc(size
, GFP_KERNEL
);
1259 return ERR_PTR(-ENOMEM
);
1261 memcpy(l2_accel
, old_l2_accel
,
1262 offsetof(struct fm10k_l2_accel
,
1263 macvlan
[old_l2_accel
->size
]));
1265 l2_accel
->size
= (old_l2_accel
->size
* 2) + 1;
1267 /* update pointers */
1268 fm10k_assign_l2_accel(interface
, l2_accel
);
1269 kfree_rcu(old_l2_accel
, rcu
);
1272 /* add macvlan to accel table, and record GLORT for position */
1273 for (i
= 0; i
< l2_accel
->size
; i
++) {
1274 if (!l2_accel
->macvlan
[i
])
1278 /* record station */
1279 l2_accel
->macvlan
[i
] = sdev
;
1282 /* configure default DGLORT mapping for RSS/DCB */
1283 dglort
.idx
= fm10k_dglort_pf_rss
;
1284 dglort
.inner_rss
= 1;
1285 dglort
.rss_l
= fls(interface
->ring_feature
[RING_F_RSS
].mask
);
1286 dglort
.pc_l
= fls(interface
->ring_feature
[RING_F_QOS
].mask
);
1287 dglort
.glort
= interface
->glort
;
1288 dglort
.shared_l
= fls(l2_accel
->size
);
1289 hw
->mac
.ops
.configure_dglort_map(hw
, &dglort
);
1291 /* Add rules for this specific dglort to the switch */
1292 fm10k_mbx_lock(interface
);
1294 glort
= l2_accel
->dglort
+ 1 + i
;
1295 hw
->mac
.ops
.update_xcast_mode(hw
, glort
, FM10K_XCAST_MODE_MULTI
);
1296 hw
->mac
.ops
.update_uc_addr(hw
, glort
, sdev
->dev_addr
, 0, true, 0);
1298 fm10k_mbx_unlock(interface
);
1303 static void fm10k_dfwd_del_station(struct net_device
*dev
, void *priv
)
1305 struct fm10k_intfc
*interface
= netdev_priv(dev
);
1306 struct fm10k_l2_accel
*l2_accel
= ACCESS_ONCE(interface
->l2_accel
);
1307 struct fm10k_dglort_cfg dglort
= { 0 };
1308 struct fm10k_hw
*hw
= &interface
->hw
;
1309 struct net_device
*sdev
= priv
;
1316 /* search table for matching interface */
1317 for (i
= 0; i
< l2_accel
->size
; i
++) {
1318 if (l2_accel
->macvlan
[i
] == sdev
)
1322 /* exit if macvlan not found */
1323 if (i
== l2_accel
->size
)
1326 /* Remove any rules specific to this dglort */
1327 fm10k_mbx_lock(interface
);
1329 glort
= l2_accel
->dglort
+ 1 + i
;
1330 hw
->mac
.ops
.update_xcast_mode(hw
, glort
, FM10K_XCAST_MODE_NONE
);
1331 hw
->mac
.ops
.update_uc_addr(hw
, glort
, sdev
->dev_addr
, 0, false, 0);
1333 fm10k_mbx_unlock(interface
);
1335 /* record removal */
1336 l2_accel
->macvlan
[i
] = NULL
;
1339 /* configure default DGLORT mapping for RSS/DCB */
1340 dglort
.idx
= fm10k_dglort_pf_rss
;
1341 dglort
.inner_rss
= 1;
1342 dglort
.rss_l
= fls(interface
->ring_feature
[RING_F_RSS
].mask
);
1343 dglort
.pc_l
= fls(interface
->ring_feature
[RING_F_QOS
].mask
);
1344 dglort
.glort
= interface
->glort
;
1345 dglort
.shared_l
= fls(l2_accel
->size
);
1346 hw
->mac
.ops
.configure_dglort_map(hw
, &dglort
);
1348 /* If table is empty remove it */
1349 if (l2_accel
->count
== 0) {
1350 fm10k_assign_l2_accel(interface
, NULL
);
1351 kfree_rcu(l2_accel
, rcu
);
1355 static netdev_features_t
fm10k_features_check(struct sk_buff
*skb
,
1356 struct net_device
*dev
,
1357 netdev_features_t features
)
1359 if (!skb
->encapsulation
|| fm10k_tx_encap_offload(skb
))
1362 return features
& ~(NETIF_F_CSUM_MASK
| NETIF_F_GSO_MASK
);
1365 static const struct net_device_ops fm10k_netdev_ops
= {
1366 .ndo_open
= fm10k_open
,
1367 .ndo_stop
= fm10k_close
,
1368 .ndo_validate_addr
= eth_validate_addr
,
1369 .ndo_start_xmit
= fm10k_xmit_frame
,
1370 .ndo_set_mac_address
= fm10k_set_mac
,
1371 .ndo_change_mtu
= fm10k_change_mtu
,
1372 .ndo_tx_timeout
= fm10k_tx_timeout
,
1373 .ndo_vlan_rx_add_vid
= fm10k_vlan_rx_add_vid
,
1374 .ndo_vlan_rx_kill_vid
= fm10k_vlan_rx_kill_vid
,
1375 .ndo_set_rx_mode
= fm10k_set_rx_mode
,
1376 .ndo_get_stats64
= fm10k_get_stats64
,
1377 .ndo_setup_tc
= __fm10k_setup_tc
,
1378 .ndo_set_vf_mac
= fm10k_ndo_set_vf_mac
,
1379 .ndo_set_vf_vlan
= fm10k_ndo_set_vf_vlan
,
1380 .ndo_set_vf_rate
= fm10k_ndo_set_vf_bw
,
1381 .ndo_get_vf_config
= fm10k_ndo_get_vf_config
,
1382 .ndo_udp_tunnel_add
= fm10k_add_vxlan_port
,
1383 .ndo_udp_tunnel_del
= fm10k_del_vxlan_port
,
1384 .ndo_dfwd_add_station
= fm10k_dfwd_add_station
,
1385 .ndo_dfwd_del_station
= fm10k_dfwd_del_station
,
1386 #ifdef CONFIG_NET_POLL_CONTROLLER
1387 .ndo_poll_controller
= fm10k_netpoll
,
1389 .ndo_features_check
= fm10k_features_check
,
1392 #define DEFAULT_DEBUG_LEVEL_SHIFT 3
1394 struct net_device
*fm10k_alloc_netdev(const struct fm10k_info
*info
)
1396 netdev_features_t hw_features
;
1397 struct fm10k_intfc
*interface
;
1398 struct net_device
*dev
;
1400 dev
= alloc_etherdev_mq(sizeof(struct fm10k_intfc
), MAX_QUEUES
);
1404 /* set net device and ethtool ops */
1405 dev
->netdev_ops
= &fm10k_netdev_ops
;
1406 fm10k_set_ethtool_ops(dev
);
1408 /* configure default debug level */
1409 interface
= netdev_priv(dev
);
1410 interface
->msg_enable
= BIT(DEFAULT_DEBUG_LEVEL_SHIFT
) - 1;
1412 /* configure default features */
1413 dev
->features
|= NETIF_F_IP_CSUM
|
1422 /* Only the PF can support VXLAN and NVGRE tunnel offloads */
1423 if (info
->mac
== fm10k_mac_pf
) {
1424 dev
->hw_enc_features
= NETIF_F_IP_CSUM
|
1428 NETIF_F_GSO_UDP_TUNNEL
|
1432 dev
->features
|= NETIF_F_GSO_UDP_TUNNEL
;
1435 /* all features defined to this point should be changeable */
1436 hw_features
= dev
->features
;
1438 /* allow user to enable L2 forwarding acceleration */
1439 hw_features
|= NETIF_F_HW_L2FW_DOFFLOAD
;
1441 /* configure VLAN features */
1442 dev
->vlan_features
|= dev
->features
;
1444 /* we want to leave these both on as we cannot disable VLAN tag
1445 * insertion or stripping on the hardware since it is contained
1446 * in the FTAG and not in the frame itself.
1448 dev
->features
|= NETIF_F_HW_VLAN_CTAG_TX
|
1449 NETIF_F_HW_VLAN_CTAG_RX
|
1450 NETIF_F_HW_VLAN_CTAG_FILTER
;
1452 dev
->priv_flags
|= IFF_UNICAST_FLT
;
1454 dev
->hw_features
|= hw_features
;