net/mlx5e: TX latency optimization to save DMA reads

[deliverable/linux.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_tx.c

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
index 03f28f438e55ab690cc865b6bc3509a53d280e25..64380bc0cd6a5df34b99531c0565718a4d2b207c 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
@@ -57,7 +57,7 @@ void mlx5e_send_nop(struct mlx5e_sq *sq, bool notify_hw)
 
        if (notify_hw) {
                cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
-               mlx5e_tx_notify_hw(sq, wqe);
+               mlx5e_tx_notify_hw(sq, wqe, 0);
        }
 }
 
@@ -110,9 +110,17 @@ u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
 }
 
 static inline u16 mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq,
-                                           struct sk_buff *skb)
+                                           struct sk_buff *skb, bool bf)
 {
-#define MLX5E_MIN_INLINE 16 /* eth header with vlan (w/o next ethertype) */
+       /* Some NIC TX decisions, e.g loopback, are based on the packet
+        * headers and occur before the data gather.
+        * Therefore these headers must be copied into the WQE
+        */
+#define MLX5E_MIN_INLINE (ETH_HLEN + 2/*vlan tag*/)
+
+       if (bf && (skb_headlen(skb) <= sq->max_inline))
+               return skb_headlen(skb);
+
        return MLX5E_MIN_INLINE;
 }
 
@@ -129,6 +137,7 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
 
        u8  opcode = MLX5_OPCODE_SEND;
        dma_addr_t dma_addr = 0;
+       bool bf = false;
        u16 headlen;
        u16 ds_cnt;
        u16 ihs;
@@ -141,6 +150,11 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
        else
                sq->stats.csum_offload_none++;
 
+       if (sq->cc != sq->prev_cc) {
+               sq->prev_cc = sq->cc;
+               sq->bf_budget = (sq->cc == sq->pc) ? MLX5E_SQ_BF_BUDGET : 0;
+       }
+
        if (skb_is_gso(skb)) {
                u32 payload_len;
 
@@ -153,7 +167,10 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
                sq->stats.tso_packets++;
                sq->stats.tso_bytes += payload_len;
        } else {
-               ihs = mlx5e_get_inline_hdr_size(sq, skb);
+               bf = sq->bf_budget &&
+                    !skb->xmit_more &&
+                    !skb_shinfo(skb)->nr_frags;
+               ihs = mlx5e_get_inline_hdr_size(sq, skb, bf);
                MLX5E_TX_SKB_CB(skb)->num_bytes = max_t(unsigned int, skb->len,
                                                        ETH_ZLEN);
        }
@@ -225,14 +242,21 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
        }
 
        if (!skb->xmit_more || netif_xmit_stopped(sq->txq)) {
+               int bf_sz = 0;
+
+               if (bf && sq->uar_bf_map)
+                       bf_sz = MLX5E_TX_SKB_CB(skb)->num_wqebbs << 3;
+
                cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
-               mlx5e_tx_notify_hw(sq, wqe);
+               mlx5e_tx_notify_hw(sq, wqe, bf_sz);
        }
 
        /* fill sq edge with nops to avoid wqe wrap around */
        while ((sq->pc & wq->sz_m1) > sq->edge)
                mlx5e_send_nop(sq, false);
 
+       sq->bf_budget = bf ? sq->bf_budget - 1 : 0;
+
        sq->stats.packets++;
        return NETDEV_TX_OK;