2 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/ipv6.h>
35 #include <linux/tcp.h>
36 #include <net/busy_poll.h>
40 static inline bool mlx5e_rx_hw_stamp(struct mlx5e_tstamp
*tstamp
)
42 return tstamp
->hwtstamp_config
.rx_filter
== HWTSTAMP_FILTER_ALL
;
45 static inline int mlx5e_alloc_rx_wqe(struct mlx5e_rq
*rq
,
46 struct mlx5e_rx_wqe
*wqe
, u16 ix
)
51 skb
= netdev_alloc_skb(rq
->netdev
, rq
->wqe_sz
);
55 dma_addr
= dma_map_single(rq
->pdev
,
56 /* hw start padding */
62 if (unlikely(dma_mapping_error(rq
->pdev
, dma_addr
)))
65 skb_reserve(skb
, MLX5E_NET_IP_ALIGN
);
67 *((dma_addr_t
*)skb
->cb
) = dma_addr
;
68 wqe
->data
.addr
= cpu_to_be64(dma_addr
+ MLX5E_NET_IP_ALIGN
);
80 bool mlx5e_post_rx_wqes(struct mlx5e_rq
*rq
)
82 struct mlx5_wq_ll
*wq
= &rq
->wq
;
84 if (unlikely(!test_bit(MLX5E_RQ_STATE_POST_WQES_ENABLE
, &rq
->state
)))
87 while (!mlx5_wq_ll_is_full(wq
)) {
88 struct mlx5e_rx_wqe
*wqe
= mlx5_wq_ll_get_wqe(wq
, wq
->head
);
90 if (unlikely(mlx5e_alloc_rx_wqe(rq
, wqe
, wq
->head
)))
93 mlx5_wq_ll_push(wq
, be16_to_cpu(wqe
->next
.next_wqe_index
));
96 /* ensure wqes are visible to device before updating doorbell record */
99 mlx5_wq_ll_update_db_record(wq
);
101 return !mlx5_wq_ll_is_full(wq
);
104 static void mlx5e_lro_update_hdr(struct sk_buff
*skb
, struct mlx5_cqe64
*cqe
)
106 struct ethhdr
*eth
= (struct ethhdr
*)(skb
->data
);
107 struct iphdr
*ipv4
= (struct iphdr
*)(skb
->data
+ ETH_HLEN
);
108 struct ipv6hdr
*ipv6
= (struct ipv6hdr
*)(skb
->data
+ ETH_HLEN
);
111 u8 l4_hdr_type
= get_cqe_l4_hdr_type(cqe
);
112 int tcp_ack
= ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA
== l4_hdr_type
) ||
113 (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA
== l4_hdr_type
));
115 u16 tot_len
= be32_to_cpu(cqe
->byte_cnt
) - ETH_HLEN
;
117 if (eth
->h_proto
== htons(ETH_P_IP
)) {
118 tcp
= (struct tcphdr
*)(skb
->data
+ ETH_HLEN
+
119 sizeof(struct iphdr
));
121 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
123 tcp
= (struct tcphdr
*)(skb
->data
+ ETH_HLEN
+
124 sizeof(struct ipv6hdr
));
126 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV6
;
129 if (get_cqe_lro_tcppsh(cqe
))
134 tcp
->ack_seq
= cqe
->lro_ack_seq_num
;
135 tcp
->window
= cqe
->lro_tcp_win
;
139 ipv4
->ttl
= cqe
->lro_min_ttl
;
140 ipv4
->tot_len
= cpu_to_be16(tot_len
);
142 ipv4
->check
= ip_fast_csum((unsigned char *)ipv4
,
145 ipv6
->hop_limit
= cqe
->lro_min_ttl
;
146 ipv6
->payload_len
= cpu_to_be16(tot_len
-
147 sizeof(struct ipv6hdr
));
151 static inline void mlx5e_skb_set_hash(struct mlx5_cqe64
*cqe
,
154 u8 cht
= cqe
->rss_hash_type
;
155 int ht
= (cht
& CQE_RSS_HTYPE_L4
) ? PKT_HASH_TYPE_L4
:
156 (cht
& CQE_RSS_HTYPE_IP
) ? PKT_HASH_TYPE_L3
:
158 skb_set_hash(skb
, be32_to_cpu(cqe
->rss_hash_result
), ht
);
161 static inline bool is_first_ethertype_ip(struct sk_buff
*skb
)
163 __be16 ethertype
= ((struct ethhdr
*)skb
->data
)->h_proto
;
165 return (ethertype
== htons(ETH_P_IP
) || ethertype
== htons(ETH_P_IPV6
));
168 static inline void mlx5e_handle_csum(struct net_device
*netdev
,
169 struct mlx5_cqe64
*cqe
,
174 if (unlikely(!(netdev
->features
& NETIF_F_RXCSUM
)))
178 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
179 } else if (likely(is_first_ethertype_ip(skb
))) {
180 skb
->ip_summed
= CHECKSUM_COMPLETE
;
181 skb
->csum
= csum_unfold((__force __sum16
)cqe
->check_sum
);
190 skb
->ip_summed
= CHECKSUM_NONE
;
191 rq
->stats
.csum_none
++;
194 static inline void mlx5e_build_rx_skb(struct mlx5_cqe64
*cqe
,
198 struct net_device
*netdev
= rq
->netdev
;
199 u32 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
200 struct mlx5e_tstamp
*tstamp
= rq
->tstamp
;
203 skb_put(skb
, cqe_bcnt
);
205 lro_num_seg
= be32_to_cpu(cqe
->srqn
) >> 24;
206 if (lro_num_seg
> 1) {
207 mlx5e_lro_update_hdr(skb
, cqe
);
208 skb_shinfo(skb
)->gso_size
= DIV_ROUND_UP(cqe_bcnt
, lro_num_seg
);
209 rq
->stats
.lro_packets
++;
210 rq
->stats
.lro_bytes
+= cqe_bcnt
;
213 if (unlikely(mlx5e_rx_hw_stamp(tstamp
)))
214 mlx5e_fill_hwstamp(tstamp
, get_cqe_ts(cqe
), skb_hwtstamps(skb
));
216 mlx5e_handle_csum(netdev
, cqe
, rq
, skb
, !!lro_num_seg
);
218 skb
->protocol
= eth_type_trans(skb
, netdev
);
220 skb_record_rx_queue(skb
, rq
->ix
);
222 if (likely(netdev
->features
& NETIF_F_RXHASH
))
223 mlx5e_skb_set_hash(cqe
, skb
);
225 if (cqe_has_vlan(cqe
))
226 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
),
227 be16_to_cpu(cqe
->vlan_info
));
229 skb
->mark
= be32_to_cpu(cqe
->sop_drop_qpn
) & MLX5E_TC_FLOW_ID_MASK
;
232 int mlx5e_poll_rx_cq(struct mlx5e_cq
*cq
, int budget
)
234 struct mlx5e_rq
*rq
= container_of(cq
, struct mlx5e_rq
, cq
);
237 for (work_done
= 0; work_done
< budget
; work_done
++) {
238 struct mlx5e_rx_wqe
*wqe
;
239 struct mlx5_cqe64
*cqe
;
241 __be16 wqe_counter_be
;
244 cqe
= mlx5e_get_cqe(cq
);
248 mlx5_cqwq_pop(&cq
->wq
);
250 wqe_counter_be
= cqe
->wqe_counter
;
251 wqe_counter
= be16_to_cpu(wqe_counter_be
);
252 wqe
= mlx5_wq_ll_get_wqe(&rq
->wq
, wqe_counter
);
253 skb
= rq
->skb
[wqe_counter
];
255 rq
->skb
[wqe_counter
] = NULL
;
257 dma_unmap_single(rq
->pdev
,
258 *((dma_addr_t
*)skb
->cb
),
262 if (unlikely((cqe
->op_own
>> 4) != MLX5_CQE_RESP_SEND
)) {
268 mlx5e_build_rx_skb(cqe
, rq
, skb
);
270 rq
->stats
.bytes
+= be32_to_cpu(cqe
->byte_cnt
);
271 napi_gro_receive(cq
->napi
, skb
);
274 mlx5_wq_ll_pop(&rq
->wq
, wqe_counter_be
,
275 &wqe
->next
.next_wqe_index
);
278 mlx5_cqwq_update_db_record(&cq
->wq
);
280 /* ensure cq space is freed before enabling more cqes */