2 * Copyright (c) 2012-2015 Qualcomm Atheros, Inc.
4 * Permission to use, copy, modify, and/or distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 #include <linux/etherdevice.h>
18 #include <net/ieee80211_radiotap.h>
19 #include <linux/if_arp.h>
20 #include <linux/moduleparam.h>
22 #include <linux/ipv6.h>
24 #include <linux/prefetch.h>
31 static bool rtap_include_phy_info
;
32 module_param(rtap_include_phy_info
, bool, S_IRUGO
);
33 MODULE_PARM_DESC(rtap_include_phy_info
,
34 " Include PHY info in the radiotap header, default - no");
37 module_param(rx_align_2
, bool, S_IRUGO
);
38 MODULE_PARM_DESC(rx_align_2
, " align Rx buffers on 4*n+2, default - no");
40 static inline uint
wil_rx_snaplen(void)
42 return rx_align_2
? 6 : 0;
45 static inline int wil_vring_is_empty(struct vring
*vring
)
47 return vring
->swhead
== vring
->swtail
;
50 static inline u32
wil_vring_next_tail(struct vring
*vring
)
52 return (vring
->swtail
+ 1) % vring
->size
;
55 static inline void wil_vring_advance_head(struct vring
*vring
, int n
)
57 vring
->swhead
= (vring
->swhead
+ n
) % vring
->size
;
60 static inline int wil_vring_is_full(struct vring
*vring
)
62 return wil_vring_next_tail(vring
) == vring
->swhead
;
65 /* Used space in Tx Vring */
66 static inline int wil_vring_used_tx(struct vring
*vring
)
68 u32 swhead
= vring
->swhead
;
69 u32 swtail
= vring
->swtail
;
70 return (vring
->size
+ swhead
- swtail
) % vring
->size
;
73 /* Available space in Tx Vring */
74 static inline int wil_vring_avail_tx(struct vring
*vring
)
76 return vring
->size
- wil_vring_used_tx(vring
) - 1;
79 /* wil_vring_wmark_low - low watermark for available descriptor space */
80 static inline int wil_vring_wmark_low(struct vring
*vring
)
85 /* wil_vring_wmark_high - high watermark for available descriptor space */
86 static inline int wil_vring_wmark_high(struct vring
*vring
)
91 /* wil_val_in_range - check if value in [min,max) */
92 static inline bool wil_val_in_range(int val
, int min
, int max
)
94 return val
>= min
&& val
< max
;
97 static int wil_vring_alloc(struct wil6210_priv
*wil
, struct vring
*vring
)
99 struct device
*dev
= wil_to_dev(wil
);
100 size_t sz
= vring
->size
* sizeof(vring
->va
[0]);
103 wil_dbg_misc(wil
, "%s()\n", __func__
);
105 BUILD_BUG_ON(sizeof(vring
->va
[0]) != 32);
109 vring
->ctx
= kcalloc(vring
->size
, sizeof(vring
->ctx
[0]), GFP_KERNEL
);
114 /* vring->va should be aligned on its size rounded up to power of 2
115 * This is granted by the dma_alloc_coherent
117 vring
->va
= dma_alloc_coherent(dev
, sz
, &vring
->pa
, GFP_KERNEL
);
123 /* initially, all descriptors are SW owned
124 * For Tx and Rx, ownership bit is at the same location, thus
127 for (i
= 0; i
< vring
->size
; i
++) {
128 volatile struct vring_tx_desc
*_d
= &vring
->va
[i
].tx
;
130 _d
->dma
.status
= TX_DMA_STATUS_DU
;
133 wil_dbg_misc(wil
, "vring[%d] 0x%p:%pad 0x%p\n", vring
->size
,
134 vring
->va
, &vring
->pa
, vring
->ctx
);
139 static void wil_txdesc_unmap(struct device
*dev
, struct vring_tx_desc
*d
,
142 dma_addr_t pa
= wil_desc_addr(&d
->dma
.addr
);
143 u16 dmalen
= le16_to_cpu(d
->dma
.length
);
145 switch (ctx
->mapped_as
) {
146 case wil_mapped_as_single
:
147 dma_unmap_single(dev
, pa
, dmalen
, DMA_TO_DEVICE
);
149 case wil_mapped_as_page
:
150 dma_unmap_page(dev
, pa
, dmalen
, DMA_TO_DEVICE
);
157 static void wil_vring_free(struct wil6210_priv
*wil
, struct vring
*vring
,
160 struct device
*dev
= wil_to_dev(wil
);
161 size_t sz
= vring
->size
* sizeof(vring
->va
[0]);
164 int vring_index
= vring
- wil
->vring_tx
;
166 wil_dbg_misc(wil
, "free Tx vring %d [%d] 0x%p:%pad 0x%p\n",
167 vring_index
, vring
->size
, vring
->va
,
168 &vring
->pa
, vring
->ctx
);
170 wil_dbg_misc(wil
, "free Rx vring [%d] 0x%p:%pad 0x%p\n",
171 vring
->size
, vring
->va
,
172 &vring
->pa
, vring
->ctx
);
175 while (!wil_vring_is_empty(vring
)) {
181 struct vring_tx_desc dd
, *d
= &dd
;
182 volatile struct vring_tx_desc
*_d
=
183 &vring
->va
[vring
->swtail
].tx
;
185 ctx
= &vring
->ctx
[vring
->swtail
];
187 wil_txdesc_unmap(dev
, d
, ctx
);
189 dev_kfree_skb_any(ctx
->skb
);
190 vring
->swtail
= wil_vring_next_tail(vring
);
192 struct vring_rx_desc dd
, *d
= &dd
;
193 volatile struct vring_rx_desc
*_d
=
194 &vring
->va
[vring
->swhead
].rx
;
196 ctx
= &vring
->ctx
[vring
->swhead
];
198 pa
= wil_desc_addr(&d
->dma
.addr
);
199 dmalen
= le16_to_cpu(d
->dma
.length
);
200 dma_unmap_single(dev
, pa
, dmalen
, DMA_FROM_DEVICE
);
202 wil_vring_advance_head(vring
, 1);
205 dma_free_coherent(dev
, sz
, (void *)vring
->va
, vring
->pa
);
213 * Allocate one skb for Rx VRING
215 * Safe to call from IRQ
217 static int wil_vring_alloc_skb(struct wil6210_priv
*wil
, struct vring
*vring
,
220 struct device
*dev
= wil_to_dev(wil
);
221 unsigned int sz
= mtu_max
+ ETH_HLEN
+ wil_rx_snaplen();
222 struct vring_rx_desc dd
, *d
= &dd
;
223 volatile struct vring_rx_desc
*_d
= &vring
->va
[i
].rx
;
225 struct sk_buff
*skb
= dev_alloc_skb(sz
+ headroom
);
230 skb_reserve(skb
, headroom
);
233 pa
= dma_map_single(dev
, skb
->data
, skb
->len
, DMA_FROM_DEVICE
);
234 if (unlikely(dma_mapping_error(dev
, pa
))) {
239 d
->dma
.d0
= RX_DMA_D0_CMD_DMA_RT
| RX_DMA_D0_CMD_DMA_IT
;
240 wil_desc_addr_set(&d
->dma
.addr
, pa
);
241 /* ip_length don't care */
243 /* error don't care */
244 d
->dma
.status
= 0; /* BIT(0) should be 0 for HW_OWNED */
245 d
->dma
.length
= cpu_to_le16(sz
);
247 vring
->ctx
[i
].skb
= skb
;
253 * Adds radiotap header
255 * Any error indicated as "Bad FCS"
257 * Vendor data for 04:ce:14-1 (Wilocity-1) consists of:
258 * - Rx descriptor: 32 bytes
261 static void wil_rx_add_radiotap_header(struct wil6210_priv
*wil
,
264 struct wireless_dev
*wdev
= wil
->wdev
;
265 struct wil6210_rtap
{
266 struct ieee80211_radiotap_header rthdr
;
267 /* fields should be in the order of bits in rthdr.it_present */
271 __le16 chnl_freq
__aligned(2);
278 struct wil6210_rtap_vendor
{
279 struct wil6210_rtap rtap
;
281 u8 vendor_oui
[3] __aligned(2);
286 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
287 struct wil6210_rtap_vendor
*rtap_vendor
;
288 int rtap_len
= sizeof(struct wil6210_rtap
);
289 int phy_length
= 0; /* phy info header size, bytes */
290 static char phy_data
[128];
291 struct ieee80211_channel
*ch
= wdev
->preset_chandef
.chan
;
293 if (rtap_include_phy_info
) {
294 rtap_len
= sizeof(*rtap_vendor
) + sizeof(*d
);
295 /* calculate additional length */
296 if (d
->dma
.status
& RX_DMA_STATUS_PHY_INFO
) {
298 * PHY info starts from 8-byte boundary
299 * there are 8-byte lines, last line may be partially
300 * written (HW bug), thus FW configures for last line
301 * to be excessive. Driver skips this last line.
303 int len
= min_t(int, 8 + sizeof(phy_data
),
304 wil_rxdesc_phy_length(d
));
307 void *p
= skb_tail_pointer(skb
);
308 void *pa
= PTR_ALIGN(p
, 8);
310 if (skb_tailroom(skb
) >= len
+ (pa
- p
)) {
311 phy_length
= len
- 8;
312 memcpy(phy_data
, pa
, phy_length
);
316 rtap_len
+= phy_length
;
319 if (skb_headroom(skb
) < rtap_len
&&
320 pskb_expand_head(skb
, rtap_len
, 0, GFP_ATOMIC
)) {
321 wil_err(wil
, "Unable to expand headrom to %d\n", rtap_len
);
325 rtap_vendor
= (void *)skb_push(skb
, rtap_len
);
326 memset(rtap_vendor
, 0, rtap_len
);
328 rtap_vendor
->rtap
.rthdr
.it_version
= PKTHDR_RADIOTAP_VERSION
;
329 rtap_vendor
->rtap
.rthdr
.it_len
= cpu_to_le16(rtap_len
);
330 rtap_vendor
->rtap
.rthdr
.it_present
= cpu_to_le32(
331 (1 << IEEE80211_RADIOTAP_FLAGS
) |
332 (1 << IEEE80211_RADIOTAP_CHANNEL
) |
333 (1 << IEEE80211_RADIOTAP_MCS
));
334 if (d
->dma
.status
& RX_DMA_STATUS_ERROR
)
335 rtap_vendor
->rtap
.flags
|= IEEE80211_RADIOTAP_F_BADFCS
;
337 rtap_vendor
->rtap
.chnl_freq
= cpu_to_le16(ch
? ch
->center_freq
: 58320);
338 rtap_vendor
->rtap
.chnl_flags
= cpu_to_le16(0);
340 rtap_vendor
->rtap
.mcs_present
= IEEE80211_RADIOTAP_MCS_HAVE_MCS
;
341 rtap_vendor
->rtap
.mcs_flags
= 0;
342 rtap_vendor
->rtap
.mcs_index
= wil_rxdesc_mcs(d
);
344 if (rtap_include_phy_info
) {
345 rtap_vendor
->rtap
.rthdr
.it_present
|= cpu_to_le32(1 <<
346 IEEE80211_RADIOTAP_VENDOR_NAMESPACE
);
347 /* OUI for Wilocity 04:ce:14 */
348 rtap_vendor
->vendor_oui
[0] = 0x04;
349 rtap_vendor
->vendor_oui
[1] = 0xce;
350 rtap_vendor
->vendor_oui
[2] = 0x14;
351 rtap_vendor
->vendor_ns
= 1;
352 /* Rx descriptor + PHY data */
353 rtap_vendor
->vendor_skip
= cpu_to_le16(sizeof(*d
) +
355 memcpy(rtap_vendor
->vendor_data
, (void *)d
, sizeof(*d
));
356 memcpy(rtap_vendor
->vendor_data
+ sizeof(*d
), phy_data
,
362 * reap 1 frame from @swhead
364 * Rx descriptor copied to skb->cb
366 * Safe to call from IRQ
368 static struct sk_buff
*wil_vring_reap_rx(struct wil6210_priv
*wil
,
371 struct device
*dev
= wil_to_dev(wil
);
372 struct net_device
*ndev
= wil_to_ndev(wil
);
373 volatile struct vring_rx_desc
*_d
;
374 struct vring_rx_desc
*d
;
377 unsigned int snaplen
= wil_rx_snaplen();
378 unsigned int sz
= mtu_max
+ ETH_HLEN
+ snaplen
;
383 struct wil_net_stats
*stats
;
385 BUILD_BUG_ON(sizeof(struct vring_rx_desc
) > sizeof(skb
->cb
));
388 if (unlikely(wil_vring_is_empty(vring
)))
391 i
= (int)vring
->swhead
;
392 _d
= &vring
->va
[i
].rx
;
393 if (unlikely(!(_d
->dma
.status
& RX_DMA_STATUS_DU
))) {
394 /* it is not error, we just reached end of Rx done area */
398 skb
= vring
->ctx
[i
].skb
;
399 vring
->ctx
[i
].skb
= NULL
;
400 wil_vring_advance_head(vring
, 1);
402 wil_err(wil
, "No Rx skb at [%d]\n", i
);
405 d
= wil_skb_rxdesc(skb
);
407 pa
= wil_desc_addr(&d
->dma
.addr
);
409 dma_unmap_single(dev
, pa
, sz
, DMA_FROM_DEVICE
);
410 dmalen
= le16_to_cpu(d
->dma
.length
);
412 trace_wil6210_rx(i
, d
);
413 wil_dbg_txrx(wil
, "Rx[%3d] : %d bytes\n", i
, dmalen
);
414 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_NONE
, 32, 4,
415 (const void *)d
, sizeof(*d
), false);
417 cid
= wil_rxdesc_cid(d
);
418 stats
= &wil
->sta
[cid
].stats
;
420 if (unlikely(dmalen
> sz
)) {
421 wil_err(wil
, "Rx size too large: %d bytes!\n", dmalen
);
422 stats
->rx_large_frame
++;
426 skb_trim(skb
, dmalen
);
430 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET
, 16, 1,
431 skb
->data
, skb_headlen(skb
), false);
433 stats
->last_mcs_rx
= wil_rxdesc_mcs(d
);
434 if (stats
->last_mcs_rx
< ARRAY_SIZE(stats
->rx_per_mcs
))
435 stats
->rx_per_mcs
[stats
->last_mcs_rx
]++;
437 /* use radiotap header only if required */
438 if (ndev
->type
== ARPHRD_IEEE80211_RADIOTAP
)
439 wil_rx_add_radiotap_header(wil
, skb
);
441 /* no extra checks if in sniffer mode */
442 if (ndev
->type
!= ARPHRD_ETHER
)
445 * Non-data frames may be delivered through Rx DMA channel (ex: BAR)
446 * Driver should recognize it by frame type, that is found
447 * in Rx descriptor. If type is not data, it is 802.11 frame as is
449 ftype
= wil_rxdesc_ftype(d
) << 2;
450 if (unlikely(ftype
!= IEEE80211_FTYPE_DATA
)) {
451 wil_dbg_txrx(wil
, "Non-data frame ftype 0x%08x\n", ftype
);
452 /* TODO: process it */
453 stats
->rx_non_data_frame
++;
458 if (unlikely(skb
->len
< ETH_HLEN
+ snaplen
)) {
459 wil_err(wil
, "Short frame, len = %d\n", skb
->len
);
460 /* TODO: process it (i.e. BAR) */
461 stats
->rx_short_frame
++;
466 /* L4 IDENT is on when HW calculated checksum, check status
467 * and in case of error drop the packet
468 * higher stack layers will handle retransmission (if required)
470 if (likely(d
->dma
.status
& RX_DMA_STATUS_L4I
)) {
471 /* L4 protocol identified, csum calculated */
472 if (likely((d
->dma
.error
& RX_DMA_ERROR_L4_ERR
) == 0))
473 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
474 /* If HW reports bad checksum, let IP stack re-check it
475 * For example, HW don't understand Microsoft IP stack that
476 * mis-calculates TCP checksum - if it should be 0x0,
477 * it writes 0xffff in violation of RFC 1624
483 * +-------+-------+---------+------------+------+
484 * | SA(6) | DA(6) | SNAP(6) | ETHTYPE(2) | DATA |
485 * +-------+-------+---------+------------+------+
486 * Need to remove SNAP, shifting SA and DA forward
488 memmove(skb
->data
+ snaplen
, skb
->data
, 2 * ETH_ALEN
);
489 skb_pull(skb
, snaplen
);
496 * allocate and fill up to @count buffers in rx ring
497 * buffers posted at @swtail
499 static int wil_rx_refill(struct wil6210_priv
*wil
, int count
)
501 struct net_device
*ndev
= wil_to_ndev(wil
);
502 struct vring
*v
= &wil
->vring_rx
;
505 int headroom
= ndev
->type
== ARPHRD_IEEE80211_RADIOTAP
?
506 WIL6210_RTAP_SIZE
: 0;
508 for (; next_tail
= wil_vring_next_tail(v
),
509 (next_tail
!= v
->swhead
) && (count
-- > 0);
510 v
->swtail
= next_tail
) {
511 rc
= wil_vring_alloc_skb(wil
, v
, v
->swtail
, headroom
);
513 wil_err(wil
, "Error %d in wil_rx_refill[%d]\n",
518 wil_w(wil
, v
->hwtail
, v
->swtail
);
524 * Pass Rx packet to the netif. Update statistics.
525 * Called in softirq context (NAPI poll).
527 void wil_netif_rx_any(struct sk_buff
*skb
, struct net_device
*ndev
)
529 gro_result_t rc
= GRO_NORMAL
;
530 struct wil6210_priv
*wil
= ndev_to_wil(ndev
);
531 struct wireless_dev
*wdev
= wil_to_wdev(wil
);
532 unsigned int len
= skb
->len
;
533 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
534 int cid
= wil_rxdesc_cid(d
); /* always 0..7, no need to check */
535 struct ethhdr
*eth
= (void *)skb
->data
;
536 /* here looking for DA, not A1, thus Rxdesc's 'mcast' indication
537 * is not suitable, need to look at data
539 int mcast
= is_multicast_ether_addr(eth
->h_dest
);
540 struct wil_net_stats
*stats
= &wil
->sta
[cid
].stats
;
541 struct sk_buff
*xmit_skb
= NULL
;
542 static const char * const gro_res_str
[] = {
543 [GRO_MERGED
] = "GRO_MERGED",
544 [GRO_MERGED_FREE
] = "GRO_MERGED_FREE",
545 [GRO_HELD
] = "GRO_HELD",
546 [GRO_NORMAL
] = "GRO_NORMAL",
547 [GRO_DROP
] = "GRO_DROP",
550 if (ndev
->features
& NETIF_F_RXHASH
)
551 /* fake L4 to ensure it won't be re-calculated later
552 * set hash to any non-zero value to activate rps
553 * mechanism, core will be chosen according
554 * to user-level rps configuration.
556 skb_set_hash(skb
, 1, PKT_HASH_TYPE_L4
);
560 if (wdev
->iftype
== NL80211_IFTYPE_AP
&& !wil
->ap_isolate
) {
562 /* send multicast frames both to higher layers in
563 * local net stack and back to the wireless medium
565 xmit_skb
= skb_copy(skb
, GFP_ATOMIC
);
567 int xmit_cid
= wil_find_cid(wil
, eth
->h_dest
);
570 /* The destination station is associated to
571 * this AP (in this VLAN), so send the frame
572 * directly to it and do not pass it to local
581 /* Send to wireless media and increase priority by 256 to
582 * keep the received priority instead of reclassifying
583 * the frame (see cfg80211_classify8021d).
585 xmit_skb
->dev
= ndev
;
586 xmit_skb
->priority
+= 256;
587 xmit_skb
->protocol
= htons(ETH_P_802_3
);
588 skb_reset_network_header(xmit_skb
);
589 skb_reset_mac_header(xmit_skb
);
590 wil_dbg_txrx(wil
, "Rx -> Tx %d bytes\n", len
);
591 dev_queue_xmit(xmit_skb
);
594 if (skb
) { /* deliver to local stack */
596 skb
->protocol
= eth_type_trans(skb
, ndev
);
597 rc
= napi_gro_receive(&wil
->napi_rx
, skb
);
598 wil_dbg_txrx(wil
, "Rx complete %d bytes => %s\n",
599 len
, gro_res_str
[rc
]);
601 /* statistics. rc set to GRO_NORMAL for AP bridging */
602 if (unlikely(rc
== GRO_DROP
)) {
603 ndev
->stats
.rx_dropped
++;
605 wil_dbg_txrx(wil
, "Rx drop %d bytes\n", len
);
607 ndev
->stats
.rx_packets
++;
609 ndev
->stats
.rx_bytes
+= len
;
610 stats
->rx_bytes
+= len
;
612 ndev
->stats
.multicast
++;
617 * Proceed all completed skb's from Rx VRING
619 * Safe to call from NAPI poll, i.e. softirq with interrupts enabled
621 void wil_rx_handle(struct wil6210_priv
*wil
, int *quota
)
623 struct net_device
*ndev
= wil_to_ndev(wil
);
624 struct vring
*v
= &wil
->vring_rx
;
627 if (unlikely(!v
->va
)) {
628 wil_err(wil
, "Rx IRQ while Rx not yet initialized\n");
631 wil_dbg_txrx(wil
, "%s()\n", __func__
);
632 while ((*quota
> 0) && (NULL
!= (skb
= wil_vring_reap_rx(wil
, v
)))) {
635 if (wil
->wdev
->iftype
== NL80211_IFTYPE_MONITOR
) {
637 skb_reset_mac_header(skb
);
638 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
639 skb
->pkt_type
= PACKET_OTHERHOST
;
640 skb
->protocol
= htons(ETH_P_802_2
);
641 wil_netif_rx_any(skb
, ndev
);
643 wil_rx_reorder(wil
, skb
);
646 wil_rx_refill(wil
, v
->size
);
649 int wil_rx_init(struct wil6210_priv
*wil
, u16 size
)
651 struct vring
*vring
= &wil
->vring_rx
;
654 wil_dbg_misc(wil
, "%s()\n", __func__
);
657 wil_err(wil
, "Rx ring already allocated\n");
662 rc
= wil_vring_alloc(wil
, vring
);
666 rc
= wmi_rx_chain_add(wil
, vring
);
670 rc
= wil_rx_refill(wil
, vring
->size
);
676 wil_vring_free(wil
, vring
, 0);
681 void wil_rx_fini(struct wil6210_priv
*wil
)
683 struct vring
*vring
= &wil
->vring_rx
;
685 wil_dbg_misc(wil
, "%s()\n", __func__
);
688 wil_vring_free(wil
, vring
, 0);
691 int wil_vring_init_tx(struct wil6210_priv
*wil
, int id
, int size
,
695 struct wmi_vring_cfg_cmd cmd
= {
696 .action
= cpu_to_le32(WMI_VRING_CMD_ADD
),
700 cpu_to_le16(wil_mtu2macbuf(mtu_max
)),
701 .ring_size
= cpu_to_le16(size
),
704 .cidxtid
= mk_cidxtid(cid
, tid
),
705 .encap_trans_type
= WMI_VRING_ENC_TYPE_802_3
,
710 .priority
= cpu_to_le16(0),
711 .timeslot_us
= cpu_to_le16(0xfff),
716 struct wil6210_mbox_hdr_wmi wmi
;
717 struct wmi_vring_cfg_done_event cmd
;
719 struct vring
*vring
= &wil
->vring_tx
[id
];
720 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[id
];
722 wil_dbg_misc(wil
, "%s() max_mpdu_size %d\n", __func__
,
723 cmd
.vring_cfg
.tx_sw_ring
.max_mpdu_size
);
726 wil_err(wil
, "Tx ring [%d] already allocated\n", id
);
731 memset(txdata
, 0, sizeof(*txdata
));
732 spin_lock_init(&txdata
->lock
);
734 rc
= wil_vring_alloc(wil
, vring
);
738 wil
->vring2cid_tid
[id
][0] = cid
;
739 wil
->vring2cid_tid
[id
][1] = tid
;
741 cmd
.vring_cfg
.tx_sw_ring
.ring_mem_base
= cpu_to_le64(vring
->pa
);
744 txdata
->dot1x_open
= true;
745 rc
= wmi_call(wil
, WMI_VRING_CFG_CMDID
, &cmd
, sizeof(cmd
),
746 WMI_VRING_CFG_DONE_EVENTID
, &reply
, sizeof(reply
), 100);
750 if (reply
.cmd
.status
!= WMI_FW_STATUS_SUCCESS
) {
751 wil_err(wil
, "Tx config failed, status 0x%02x\n",
756 vring
->hwtail
= le32_to_cpu(reply
.cmd
.tx_vring_tail_ptr
);
759 if (txdata
->dot1x_open
&& (agg_wsize
>= 0))
760 wil_addba_tx_request(wil
, id
, agg_wsize
);
764 txdata
->dot1x_open
= false;
766 wil_vring_free(wil
, vring
, 1);
772 int wil_vring_init_bcast(struct wil6210_priv
*wil
, int id
, int size
)
775 struct wmi_bcast_vring_cfg_cmd cmd
= {
776 .action
= cpu_to_le32(WMI_VRING_CMD_ADD
),
780 cpu_to_le16(wil_mtu2macbuf(mtu_max
)),
781 .ring_size
= cpu_to_le16(size
),
784 .encap_trans_type
= WMI_VRING_ENC_TYPE_802_3
,
788 struct wil6210_mbox_hdr_wmi wmi
;
789 struct wmi_vring_cfg_done_event cmd
;
791 struct vring
*vring
= &wil
->vring_tx
[id
];
792 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[id
];
794 wil_dbg_misc(wil
, "%s() max_mpdu_size %d\n", __func__
,
795 cmd
.vring_cfg
.tx_sw_ring
.max_mpdu_size
);
798 wil_err(wil
, "Tx ring [%d] already allocated\n", id
);
803 memset(txdata
, 0, sizeof(*txdata
));
804 spin_lock_init(&txdata
->lock
);
806 rc
= wil_vring_alloc(wil
, vring
);
810 wil
->vring2cid_tid
[id
][0] = WIL6210_MAX_CID
; /* CID */
811 wil
->vring2cid_tid
[id
][1] = 0; /* TID */
813 cmd
.vring_cfg
.tx_sw_ring
.ring_mem_base
= cpu_to_le64(vring
->pa
);
816 txdata
->dot1x_open
= true;
817 rc
= wmi_call(wil
, WMI_BCAST_VRING_CFG_CMDID
, &cmd
, sizeof(cmd
),
818 WMI_VRING_CFG_DONE_EVENTID
, &reply
, sizeof(reply
), 100);
822 if (reply
.cmd
.status
!= WMI_FW_STATUS_SUCCESS
) {
823 wil_err(wil
, "Tx config failed, status 0x%02x\n",
828 vring
->hwtail
= le32_to_cpu(reply
.cmd
.tx_vring_tail_ptr
);
835 txdata
->dot1x_open
= false;
836 wil_vring_free(wil
, vring
, 1);
842 void wil_vring_fini_tx(struct wil6210_priv
*wil
, int id
)
844 struct vring
*vring
= &wil
->vring_tx
[id
];
845 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[id
];
847 WARN_ON(!mutex_is_locked(&wil
->mutex
));
852 wil_dbg_misc(wil
, "%s() id=%d\n", __func__
, id
);
854 spin_lock_bh(&txdata
->lock
);
855 txdata
->dot1x_open
= false;
856 txdata
->enabled
= 0; /* no Tx can be in progress or start anew */
857 spin_unlock_bh(&txdata
->lock
);
858 /* make sure NAPI won't touch this vring */
859 if (test_bit(wil_status_napi_en
, wil
->status
))
860 napi_synchronize(&wil
->napi_tx
);
862 wil_vring_free(wil
, vring
, 1);
863 memset(txdata
, 0, sizeof(*txdata
));
866 static struct vring
*wil_find_tx_ucast(struct wil6210_priv
*wil
,
870 struct ethhdr
*eth
= (void *)skb
->data
;
871 int cid
= wil_find_cid(wil
, eth
->h_dest
);
876 /* TODO: fix for multiple TID */
877 for (i
= 0; i
< ARRAY_SIZE(wil
->vring2cid_tid
); i
++) {
878 if (!wil
->vring_tx_data
[i
].dot1x_open
&&
879 (skb
->protocol
!= cpu_to_be16(ETH_P_PAE
)))
881 if (wil
->vring2cid_tid
[i
][0] == cid
) {
882 struct vring
*v
= &wil
->vring_tx
[i
];
884 wil_dbg_txrx(wil
, "%s(%pM) -> [%d]\n",
885 __func__
, eth
->h_dest
, i
);
889 wil_dbg_txrx(wil
, "vring[%d] not valid\n", i
);
898 static int wil_tx_vring(struct wil6210_priv
*wil
, struct vring
*vring
,
899 struct sk_buff
*skb
);
901 static struct vring
*wil_find_tx_vring_sta(struct wil6210_priv
*wil
,
908 /* In the STA mode, it is expected to have only 1 VRING
909 * for the AP we connected to.
910 * find 1-st vring eligible for this skb and use it.
912 for (i
= 0; i
< WIL6210_MAX_TX_RINGS
; i
++) {
913 v
= &wil
->vring_tx
[i
];
917 cid
= wil
->vring2cid_tid
[i
][0];
918 if (cid
>= WIL6210_MAX_CID
) /* skip BCAST */
921 if (!wil
->vring_tx_data
[i
].dot1x_open
&&
922 (skb
->protocol
!= cpu_to_be16(ETH_P_PAE
)))
925 wil_dbg_txrx(wil
, "Tx -> ring %d\n", i
);
930 wil_dbg_txrx(wil
, "Tx while no vrings active?\n");
935 /* Use one of 2 strategies:
937 * 1. New (real broadcast):
938 * use dedicated broadcast vring
939 * 2. Old (pseudo-DMS):
940 * Find 1-st vring and return it;
941 * duplicate skb and send it to other active vrings;
942 * in all cases override dest address to unicast peer's address
943 * Use old strategy when new is not supported yet:
946 static struct vring
*wil_find_tx_bcast_1(struct wil6210_priv
*wil
,
950 int i
= wil
->bcast_vring
;
954 v
= &wil
->vring_tx
[i
];
957 if (!wil
->vring_tx_data
[i
].dot1x_open
&&
958 (skb
->protocol
!= cpu_to_be16(ETH_P_PAE
)))
964 static void wil_set_da_for_vring(struct wil6210_priv
*wil
,
965 struct sk_buff
*skb
, int vring_index
)
967 struct ethhdr
*eth
= (void *)skb
->data
;
968 int cid
= wil
->vring2cid_tid
[vring_index
][0];
970 ether_addr_copy(eth
->h_dest
, wil
->sta
[cid
].addr
);
973 static struct vring
*wil_find_tx_bcast_2(struct wil6210_priv
*wil
,
976 struct vring
*v
, *v2
;
977 struct sk_buff
*skb2
;
980 struct ethhdr
*eth
= (void *)skb
->data
;
981 char *src
= eth
->h_source
;
983 /* find 1-st vring eligible for data */
984 for (i
= 0; i
< WIL6210_MAX_TX_RINGS
; i
++) {
985 v
= &wil
->vring_tx
[i
];
989 cid
= wil
->vring2cid_tid
[i
][0];
990 if (cid
>= WIL6210_MAX_CID
) /* skip BCAST */
992 if (!wil
->vring_tx_data
[i
].dot1x_open
&&
993 (skb
->protocol
!= cpu_to_be16(ETH_P_PAE
)))
996 /* don't Tx back to source when re-routing Rx->Tx at the AP */
997 if (0 == memcmp(wil
->sta
[cid
].addr
, src
, ETH_ALEN
))
1003 wil_dbg_txrx(wil
, "Tx while no vrings active?\n");
1008 wil_dbg_txrx(wil
, "BCAST -> ring %d\n", i
);
1009 wil_set_da_for_vring(wil
, skb
, i
);
1011 /* find other active vrings and duplicate skb for each */
1012 for (i
++; i
< WIL6210_MAX_TX_RINGS
; i
++) {
1013 v2
= &wil
->vring_tx
[i
];
1016 cid
= wil
->vring2cid_tid
[i
][0];
1017 if (cid
>= WIL6210_MAX_CID
) /* skip BCAST */
1019 if (!wil
->vring_tx_data
[i
].dot1x_open
&&
1020 (skb
->protocol
!= cpu_to_be16(ETH_P_PAE
)))
1023 if (0 == memcmp(wil
->sta
[cid
].addr
, src
, ETH_ALEN
))
1026 skb2
= skb_copy(skb
, GFP_ATOMIC
);
1028 wil_dbg_txrx(wil
, "BCAST DUP -> ring %d\n", i
);
1029 wil_set_da_for_vring(wil
, skb2
, i
);
1030 wil_tx_vring(wil
, v2
, skb2
);
1032 wil_err(wil
, "skb_copy failed\n");
1039 static struct vring
*wil_find_tx_bcast(struct wil6210_priv
*wil
,
1040 struct sk_buff
*skb
)
1042 struct wireless_dev
*wdev
= wil
->wdev
;
1044 if (wdev
->iftype
!= NL80211_IFTYPE_AP
)
1045 return wil_find_tx_bcast_2(wil
, skb
);
1047 return wil_find_tx_bcast_1(wil
, skb
);
1050 static int wil_tx_desc_map(struct vring_tx_desc
*d
, dma_addr_t pa
, u32 len
,
1053 wil_desc_addr_set(&d
->dma
.addr
, pa
);
1054 d
->dma
.ip_length
= 0;
1055 /* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/
1056 d
->dma
.b11
= 0/*14 | BIT(7)*/;
1058 d
->dma
.status
= 0; /* BIT(0) should be 0 for HW_OWNED */
1059 d
->dma
.length
= cpu_to_le16((u16
)len
);
1060 d
->dma
.d0
= (vring_index
<< DMA_CFG_DESC_TX_0_QID_POS
);
1064 d
->mac
.ucode_cmd
= 0;
1065 /* translation type: 0 - bypass; 1 - 802.3; 2 - native wifi */
1066 d
->mac
.d
[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS
) |
1067 (1 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS
);
1073 void wil_tx_desc_set_nr_frags(struct vring_tx_desc
*d
, int nr_frags
)
1075 d
->mac
.d
[2] |= (nr_frags
<< MAC_CFG_DESC_TX_2_NUM_OF_DESCRIPTORS_POS
);
1079 * Sets the descriptor @d up for csum and/or TSO offloading. The corresponding
1080 * @skb is used to obtain the protocol and headers length.
1081 * @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data,
1082 * 2 - middle, 3 - last descriptor.
1085 static void wil_tx_desc_offload_setup_tso(struct vring_tx_desc
*d
,
1086 struct sk_buff
*skb
,
1087 int tso_desc_type
, bool is_ipv4
,
1088 int tcp_hdr_len
, int skb_net_hdr_len
)
1090 d
->dma
.b11
= ETH_HLEN
; /* MAC header length */
1091 d
->dma
.b11
|= is_ipv4
<< DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS
;
1093 d
->dma
.d0
|= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS
);
1094 /* L4 header len: TCP header length */
1095 d
->dma
.d0
|= (tcp_hdr_len
& DMA_CFG_DESC_TX_0_L4_LENGTH_MSK
);
1097 /* Setup TSO: bit and desc type */
1098 d
->dma
.d0
|= (BIT(DMA_CFG_DESC_TX_0_TCP_SEG_EN_POS
)) |
1099 (tso_desc_type
<< DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS
);
1100 d
->dma
.d0
|= (is_ipv4
<< DMA_CFG_DESC_TX_0_IPV4_CHECKSUM_EN_POS
);
1102 d
->dma
.ip_length
= skb_net_hdr_len
;
1103 /* Enable TCP/UDP checksum */
1104 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS
);
1105 /* Calculate pseudo-header */
1106 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS
);
1110 * Sets the descriptor @d up for csum. The corresponding
1111 * @skb is used to obtain the protocol and headers length.
1112 * Returns the protocol: 0 - not TCP, 1 - TCPv4, 2 - TCPv6.
1113 * Note, if d==NULL, the function only returns the protocol result.
1115 * It is very similar to previous wil_tx_desc_offload_setup_tso. This
1116 * is "if unrolling" to optimize the critical path.
1119 static int wil_tx_desc_offload_setup(struct vring_tx_desc
*d
,
1120 struct sk_buff
*skb
){
1123 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1126 d
->dma
.b11
= ETH_HLEN
; /* MAC header length */
1128 switch (skb
->protocol
) {
1129 case cpu_to_be16(ETH_P_IP
):
1130 protocol
= ip_hdr(skb
)->protocol
;
1131 d
->dma
.b11
|= BIT(DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS
);
1133 case cpu_to_be16(ETH_P_IPV6
):
1134 protocol
= ipv6_hdr(skb
)->nexthdr
;
1142 d
->dma
.d0
|= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS
);
1143 /* L4 header len: TCP header length */
1145 (tcp_hdrlen(skb
) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK
);
1148 /* L4 header len: UDP header length */
1150 (sizeof(struct udphdr
) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK
);
1156 d
->dma
.ip_length
= skb_network_header_len(skb
);
1157 /* Enable TCP/UDP checksum */
1158 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS
);
1159 /* Calculate pseudo-header */
1160 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS
);
1165 static inline void wil_tx_last_desc(struct vring_tx_desc
*d
)
1167 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS
) |
1168 BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS
) |
1169 BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS
);
1172 static inline void wil_set_tx_desc_last_tso(volatile struct vring_tx_desc
*d
)
1174 d
->dma
.d0
|= wil_tso_type_lst
<<
1175 DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS
;
1178 static int __wil_tx_vring_tso(struct wil6210_priv
*wil
, struct vring
*vring
,
1179 struct sk_buff
*skb
)
1181 struct device
*dev
= wil_to_dev(wil
);
1183 /* point to descriptors in shared memory */
1184 volatile struct vring_tx_desc
*_desc
= NULL
, *_hdr_desc
,
1185 *_first_desc
= NULL
;
1187 /* pointers to shadow descriptors */
1188 struct vring_tx_desc desc_mem
, hdr_desc_mem
, first_desc_mem
,
1189 *d
= &hdr_desc_mem
, *hdr_desc
= &hdr_desc_mem
,
1190 *first_desc
= &first_desc_mem
;
1192 /* pointer to shadow descriptors' context */
1193 struct wil_ctx
*hdr_ctx
, *first_ctx
= NULL
;
1195 int descs_used
= 0; /* total number of used descriptors */
1196 int sg_desc_cnt
= 0; /* number of descriptors for current mss*/
1198 u32 swhead
= vring
->swhead
;
1199 int used
, avail
= wil_vring_avail_tx(vring
);
1200 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
1201 int min_desc_required
= nr_frags
+ 1;
1202 int mss
= skb_shinfo(skb
)->gso_size
; /* payload size w/o headers */
1203 int f
, len
, hdrlen
, headlen
;
1204 int vring_index
= vring
- wil
->vring_tx
;
1205 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[vring_index
];
1208 const skb_frag_t
*frag
= NULL
;
1211 int hdr_compensation_need
= true;
1212 int desc_tso_type
= wil_tso_type_first
;
1215 int skb_net_hdr_len
;
1218 wil_dbg_txrx(wil
, "%s() %d bytes to vring %d\n",
1219 __func__
, skb
->len
, vring_index
);
1221 if (unlikely(!txdata
->enabled
))
1224 /* A typical page 4K is 3-4 payloads, we assume each fragment
1225 * is a full payload, that's how min_desc_required has been
1226 * calculated. In real we might need more or less descriptors,
1227 * this is the initial check only.
1229 if (unlikely(avail
< min_desc_required
)) {
1230 wil_err_ratelimited(wil
,
1231 "TSO: Tx ring[%2d] full. No space for %d fragments\n",
1232 vring_index
, min_desc_required
);
1236 /* Header Length = MAC header len + IP header len + TCP header len*/
1238 (int)skb_network_header_len(skb
) +
1241 gso_type
= skb_shinfo(skb
)->gso_type
& (SKB_GSO_TCPV6
| SKB_GSO_TCPV4
);
1244 /* TCP v4, zero out the IP length and IPv4 checksum fields
1245 * as required by the offloading doc
1247 ip_hdr(skb
)->tot_len
= 0;
1248 ip_hdr(skb
)->check
= 0;
1252 /* TCP v6, zero out the payload length */
1253 ipv6_hdr(skb
)->payload_len
= 0;
1257 /* other than TCPv4 or TCPv6 types are not supported for TSO.
1258 * It is also illegal for both to be set simultaneously
1263 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1266 /* tcp header length and skb network header length are fixed for all
1267 * packet's descriptors - read then once here
1269 tcp_hdr_len
= tcp_hdrlen(skb
);
1270 skb_net_hdr_len
= skb_network_header_len(skb
);
1272 _hdr_desc
= &vring
->va
[i
].tx
;
1274 pa
= dma_map_single(dev
, skb
->data
, hdrlen
, DMA_TO_DEVICE
);
1275 if (unlikely(dma_mapping_error(dev
, pa
))) {
1276 wil_err(wil
, "TSO: Skb head DMA map error\n");
1280 wil_tx_desc_map(hdr_desc
, pa
, hdrlen
, vring_index
);
1281 wil_tx_desc_offload_setup_tso(hdr_desc
, skb
, wil_tso_type_hdr
, is_ipv4
,
1282 tcp_hdr_len
, skb_net_hdr_len
);
1283 wil_tx_last_desc(hdr_desc
);
1285 vring
->ctx
[i
].mapped_as
= wil_mapped_as_single
;
1286 hdr_ctx
= &vring
->ctx
[i
];
1289 headlen
= skb_headlen(skb
) - hdrlen
;
1291 for (f
= headlen
? -1 : 0; f
< nr_frags
; f
++) {
1294 wil_dbg_txrx(wil
, "TSO: process skb head, len %u\n",
1297 frag
= &skb_shinfo(skb
)->frags
[f
];
1299 wil_dbg_txrx(wil
, "TSO: frag[%d]: len %u\n", f
, len
);
1304 "TSO: len %d, rem_data %d, descs_used %d\n",
1305 len
, rem_data
, descs_used
);
1307 if (descs_used
== avail
) {
1308 wil_err(wil
, "TSO: ring overflow\n");
1312 lenmss
= min_t(int, rem_data
, len
);
1313 i
= (swhead
+ descs_used
) % vring
->size
;
1314 wil_dbg_txrx(wil
, "TSO: lenmss %d, i %d\n", lenmss
, i
);
1317 pa
= skb_frag_dma_map(dev
, frag
,
1318 frag
->size
- len
, lenmss
,
1320 vring
->ctx
[i
].mapped_as
= wil_mapped_as_page
;
1322 pa
= dma_map_single(dev
,
1324 skb_headlen(skb
) - headlen
,
1327 vring
->ctx
[i
].mapped_as
= wil_mapped_as_single
;
1331 if (unlikely(dma_mapping_error(dev
, pa
)))
1334 _desc
= &vring
->va
[i
].tx
;
1337 _first_desc
= _desc
;
1338 first_ctx
= &vring
->ctx
[i
];
1344 wil_tx_desc_map(d
, pa
, lenmss
, vring_index
);
1345 wil_tx_desc_offload_setup_tso(d
, skb
, desc_tso_type
,
1346 is_ipv4
, tcp_hdr_len
,
1349 /* use tso_type_first only once */
1350 desc_tso_type
= wil_tso_type_mid
;
1352 descs_used
++; /* desc used so far */
1353 sg_desc_cnt
++; /* desc used for this segment */
1358 "TSO: len %d, rem_data %d, descs_used %d, sg_desc_cnt %d,\n",
1359 len
, rem_data
, descs_used
, sg_desc_cnt
);
1361 /* Close the segment if reached mss size or last frag*/
1362 if (rem_data
== 0 || (f
== nr_frags
- 1 && len
== 0)) {
1363 if (hdr_compensation_need
) {
1364 /* first segment include hdr desc for
1367 hdr_ctx
->nr_frags
= sg_desc_cnt
;
1368 wil_tx_desc_set_nr_frags(first_desc
,
1371 hdr_compensation_need
= false;
1373 wil_tx_desc_set_nr_frags(first_desc
,
1376 first_ctx
->nr_frags
= sg_desc_cnt
- 1;
1378 wil_tx_last_desc(d
);
1380 /* first descriptor may also be the last
1381 * for this mss - make sure not to copy
1384 if (first_desc
!= d
)
1385 *_first_desc
= *first_desc
;
1387 /*last descriptor will be copied at the end
1388 * of this TS processing
1390 if (f
< nr_frags
- 1 || len
> 0)
1396 } else if (first_desc
!= d
) /* update mid descriptor */
1401 /* first descriptor may also be the last.
1402 * in this case d pointer is invalid
1404 if (_first_desc
== _desc
)
1407 /* Last data descriptor */
1408 wil_set_tx_desc_last_tso(d
);
1411 /* Fill the total number of descriptors in first desc (hdr)*/
1412 wil_tx_desc_set_nr_frags(hdr_desc
, descs_used
);
1413 *_hdr_desc
= *hdr_desc
;
1415 /* hold reference to skb
1416 * to prevent skb release before accounting
1417 * in case of immediate "tx done"
1419 vring
->ctx
[i
].skb
= skb_get(skb
);
1421 /* performance monitoring */
1422 used
= wil_vring_used_tx(vring
);
1423 if (wil_val_in_range(vring_idle_trsh
,
1424 used
, used
+ descs_used
)) {
1425 txdata
->idle
+= get_cycles() - txdata
->last_idle
;
1426 wil_dbg_txrx(wil
, "Ring[%2d] not idle %d -> %d\n",
1427 vring_index
, used
, used
+ descs_used
);
1430 /* advance swhead */
1431 wil_dbg_txrx(wil
, "TSO: Tx swhead %d -> %d\n", swhead
, vring
->swhead
);
1432 wil_vring_advance_head(vring
, descs_used
);
1434 /* make sure all writes to descriptors (shared memory) are done before
1435 * committing them to HW
1439 wil_w(wil
, vring
->hwtail
, vring
->swhead
);
1443 wil_err(wil
, "TSO: DMA map page error\n");
1444 while (descs_used
> 0) {
1445 struct wil_ctx
*ctx
;
1447 i
= (swhead
+ descs_used
) % vring
->size
;
1448 d
= (struct vring_tx_desc
*)&vring
->va
[i
].tx
;
1449 _desc
= &vring
->va
[i
].tx
;
1451 _desc
->dma
.status
= TX_DMA_STATUS_DU
;
1452 ctx
= &vring
->ctx
[i
];
1453 wil_txdesc_unmap(dev
, d
, ctx
);
1455 dev_kfree_skb_any(ctx
->skb
);
1456 memset(ctx
, 0, sizeof(*ctx
));
1464 static int __wil_tx_vring(struct wil6210_priv
*wil
, struct vring
*vring
,
1465 struct sk_buff
*skb
)
1467 struct device
*dev
= wil_to_dev(wil
);
1468 struct vring_tx_desc dd
, *d
= &dd
;
1469 volatile struct vring_tx_desc
*_d
;
1470 u32 swhead
= vring
->swhead
;
1471 int avail
= wil_vring_avail_tx(vring
);
1472 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
1474 int vring_index
= vring
- wil
->vring_tx
;
1475 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[vring_index
];
1479 bool mcast
= (vring_index
== wil
->bcast_vring
);
1480 uint len
= skb_headlen(skb
);
1482 wil_dbg_txrx(wil
, "%s() %d bytes to vring %d\n",
1483 __func__
, skb
->len
, vring_index
);
1485 if (unlikely(!txdata
->enabled
))
1488 if (unlikely(avail
< 1 + nr_frags
)) {
1489 wil_err_ratelimited(wil
,
1490 "Tx ring[%2d] full. No space for %d fragments\n",
1491 vring_index
, 1 + nr_frags
);
1494 _d
= &vring
->va
[i
].tx
;
1496 pa
= dma_map_single(dev
, skb
->data
, skb_headlen(skb
), DMA_TO_DEVICE
);
1498 wil_dbg_txrx(wil
, "Tx[%2d] skb %d bytes 0x%p -> %pad\n", vring_index
,
1499 skb_headlen(skb
), skb
->data
, &pa
);
1500 wil_hex_dump_txrx("Tx ", DUMP_PREFIX_OFFSET
, 16, 1,
1501 skb
->data
, skb_headlen(skb
), false);
1503 if (unlikely(dma_mapping_error(dev
, pa
)))
1505 vring
->ctx
[i
].mapped_as
= wil_mapped_as_single
;
1507 wil_tx_desc_map(d
, pa
, len
, vring_index
);
1508 if (unlikely(mcast
)) {
1509 d
->mac
.d
[0] |= BIT(MAC_CFG_DESC_TX_0_MCS_EN_POS
); /* MCS 0 */
1510 if (unlikely(len
> WIL_BCAST_MCS0_LIMIT
)) /* set MCS 1 */
1511 d
->mac
.d
[0] |= (1 << MAC_CFG_DESC_TX_0_MCS_INDEX_POS
);
1513 /* Process TCP/UDP checksum offloading */
1514 if (unlikely(wil_tx_desc_offload_setup(d
, skb
))) {
1515 wil_err(wil
, "Tx[%2d] Failed to set cksum, drop packet\n",
1520 vring
->ctx
[i
].nr_frags
= nr_frags
;
1521 wil_tx_desc_set_nr_frags(d
, nr_frags
+ 1);
1523 /* middle segments */
1524 for (; f
< nr_frags
; f
++) {
1525 const struct skb_frag_struct
*frag
=
1526 &skb_shinfo(skb
)->frags
[f
];
1527 int len
= skb_frag_size(frag
);
1530 wil_dbg_txrx(wil
, "Tx[%2d] desc[%4d]\n", vring_index
, i
);
1531 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE
, 32, 4,
1532 (const void *)d
, sizeof(*d
), false);
1533 i
= (swhead
+ f
+ 1) % vring
->size
;
1534 _d
= &vring
->va
[i
].tx
;
1535 pa
= skb_frag_dma_map(dev
, frag
, 0, skb_frag_size(frag
),
1537 if (unlikely(dma_mapping_error(dev
, pa
)))
1539 vring
->ctx
[i
].mapped_as
= wil_mapped_as_page
;
1540 wil_tx_desc_map(d
, pa
, len
, vring_index
);
1541 /* no need to check return code -
1542 * if it succeeded for 1-st descriptor,
1543 * it will succeed here too
1545 wil_tx_desc_offload_setup(d
, skb
);
1547 /* for the last seg only */
1548 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS
);
1549 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS
);
1550 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS
);
1552 wil_dbg_txrx(wil
, "Tx[%2d] desc[%4d]\n", vring_index
, i
);
1553 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE
, 32, 4,
1554 (const void *)d
, sizeof(*d
), false);
1556 /* hold reference to skb
1557 * to prevent skb release before accounting
1558 * in case of immediate "tx done"
1560 vring
->ctx
[i
].skb
= skb_get(skb
);
1562 /* performance monitoring */
1563 used
= wil_vring_used_tx(vring
);
1564 if (wil_val_in_range(vring_idle_trsh
,
1565 used
, used
+ nr_frags
+ 1)) {
1566 txdata
->idle
+= get_cycles() - txdata
->last_idle
;
1567 wil_dbg_txrx(wil
, "Ring[%2d] not idle %d -> %d\n",
1568 vring_index
, used
, used
+ nr_frags
+ 1);
1571 /* advance swhead */
1572 wil_vring_advance_head(vring
, nr_frags
+ 1);
1573 wil_dbg_txrx(wil
, "Tx[%2d] swhead %d -> %d\n", vring_index
, swhead
,
1575 trace_wil6210_tx(vring_index
, swhead
, skb
->len
, nr_frags
);
1577 /* make sure all writes to descriptors (shared memory) are done before
1578 * committing them to HW
1582 wil_w(wil
, vring
->hwtail
, vring
->swhead
);
1586 /* unmap what we have mapped */
1587 nr_frags
= f
+ 1; /* frags mapped + one for skb head */
1588 for (f
= 0; f
< nr_frags
; f
++) {
1589 struct wil_ctx
*ctx
;
1591 i
= (swhead
+ f
) % vring
->size
;
1592 ctx
= &vring
->ctx
[i
];
1593 _d
= &vring
->va
[i
].tx
;
1595 _d
->dma
.status
= TX_DMA_STATUS_DU
;
1596 wil_txdesc_unmap(dev
, d
, ctx
);
1599 dev_kfree_skb_any(ctx
->skb
);
1601 memset(ctx
, 0, sizeof(*ctx
));
1607 static int wil_tx_vring(struct wil6210_priv
*wil
, struct vring
*vring
,
1608 struct sk_buff
*skb
)
1610 int vring_index
= vring
- wil
->vring_tx
;
1611 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[vring_index
];
1614 spin_lock(&txdata
->lock
);
1616 rc
= (skb_is_gso(skb
) ? __wil_tx_vring_tso
: __wil_tx_vring
)
1619 spin_unlock(&txdata
->lock
);
1624 netdev_tx_t
wil_start_xmit(struct sk_buff
*skb
, struct net_device
*ndev
)
1626 struct wil6210_priv
*wil
= ndev_to_wil(ndev
);
1627 struct ethhdr
*eth
= (void *)skb
->data
;
1628 bool bcast
= is_multicast_ether_addr(eth
->h_dest
);
1629 struct vring
*vring
;
1630 static bool pr_once_fw
;
1633 wil_dbg_txrx(wil
, "%s()\n", __func__
);
1634 if (unlikely(!test_bit(wil_status_fwready
, wil
->status
))) {
1636 wil_err(wil
, "FW not ready\n");
1641 if (unlikely(!test_bit(wil_status_fwconnected
, wil
->status
))) {
1642 wil_err(wil
, "FW not connected\n");
1645 if (unlikely(wil
->wdev
->iftype
== NL80211_IFTYPE_MONITOR
)) {
1646 wil_err(wil
, "Xmit in monitor mode not supported\n");
1652 if (wil
->wdev
->iftype
== NL80211_IFTYPE_STATION
) {
1653 /* in STA mode (ESS), all to same VRING */
1654 vring
= wil_find_tx_vring_sta(wil
, skb
);
1655 } else { /* direct communication, find matching VRING */
1656 vring
= bcast
? wil_find_tx_bcast(wil
, skb
) :
1657 wil_find_tx_ucast(wil
, skb
);
1659 if (unlikely(!vring
)) {
1660 wil_dbg_txrx(wil
, "No Tx VRING found for %pM\n", eth
->h_dest
);
1663 /* set up vring entry */
1664 rc
= wil_tx_vring(wil
, vring
, skb
);
1666 /* do we still have enough room in the vring? */
1667 if (unlikely(wil_vring_avail_tx(vring
) < wil_vring_wmark_low(vring
))) {
1668 netif_tx_stop_all_queues(wil_to_ndev(wil
));
1669 wil_dbg_txrx(wil
, "netif_tx_stop : ring full\n");
1674 /* statistics will be updated on the tx_complete */
1675 dev_kfree_skb_any(skb
);
1676 return NETDEV_TX_OK
;
1678 return NETDEV_TX_BUSY
;
1680 break; /* goto drop; */
1683 ndev
->stats
.tx_dropped
++;
1684 dev_kfree_skb_any(skb
);
1686 return NET_XMIT_DROP
;
1689 static inline bool wil_need_txstat(struct sk_buff
*skb
)
1691 struct ethhdr
*eth
= (void *)skb
->data
;
1693 return is_unicast_ether_addr(eth
->h_dest
) && skb
->sk
&&
1694 (skb_shinfo(skb
)->tx_flags
& SKBTX_WIFI_STATUS
);
1697 static inline void wil_consume_skb(struct sk_buff
*skb
, bool acked
)
1699 if (unlikely(wil_need_txstat(skb
)))
1700 skb_complete_wifi_ack(skb
, acked
);
1702 acked
? dev_consume_skb_any(skb
) : dev_kfree_skb_any(skb
);
1706 * Clean up transmitted skb's from the Tx VRING
1708 * Return number of descriptors cleared
1710 * Safe to call from IRQ
1712 int wil_tx_complete(struct wil6210_priv
*wil
, int ringid
)
1714 struct net_device
*ndev
= wil_to_ndev(wil
);
1715 struct device
*dev
= wil_to_dev(wil
);
1716 struct vring
*vring
= &wil
->vring_tx
[ringid
];
1717 struct vring_tx_data
*txdata
= &wil
->vring_tx_data
[ringid
];
1719 int cid
= wil
->vring2cid_tid
[ringid
][0];
1720 struct wil_net_stats
*stats
= NULL
;
1721 volatile struct vring_tx_desc
*_d
;
1722 int used_before_complete
;
1725 if (unlikely(!vring
->va
)) {
1726 wil_err(wil
, "Tx irq[%d]: vring not initialized\n", ringid
);
1730 if (unlikely(!txdata
->enabled
)) {
1731 wil_info(wil
, "Tx irq[%d]: vring disabled\n", ringid
);
1735 wil_dbg_txrx(wil
, "%s(%d)\n", __func__
, ringid
);
1737 used_before_complete
= wil_vring_used_tx(vring
);
1739 if (cid
< WIL6210_MAX_CID
)
1740 stats
= &wil
->sta
[cid
].stats
;
1742 while (!wil_vring_is_empty(vring
)) {
1744 struct wil_ctx
*ctx
= &vring
->ctx
[vring
->swtail
];
1746 * For the fragmented skb, HW will set DU bit only for the
1747 * last fragment. look for it.
1748 * In TSO the first DU will include hdr desc
1750 int lf
= (vring
->swtail
+ ctx
->nr_frags
) % vring
->size
;
1751 /* TODO: check we are not past head */
1753 _d
= &vring
->va
[lf
].tx
;
1754 if (unlikely(!(_d
->dma
.status
& TX_DMA_STATUS_DU
)))
1757 new_swtail
= (lf
+ 1) % vring
->size
;
1758 while (vring
->swtail
!= new_swtail
) {
1759 struct vring_tx_desc dd
, *d
= &dd
;
1761 struct sk_buff
*skb
;
1763 ctx
= &vring
->ctx
[vring
->swtail
];
1765 _d
= &vring
->va
[vring
->swtail
].tx
;
1769 dmalen
= le16_to_cpu(d
->dma
.length
);
1770 trace_wil6210_tx_done(ringid
, vring
->swtail
, dmalen
,
1773 "TxC[%2d][%3d] : %d bytes, status 0x%02x err 0x%02x\n",
1774 ringid
, vring
->swtail
, dmalen
,
1775 d
->dma
.status
, d
->dma
.error
);
1776 wil_hex_dump_txrx("TxCD ", DUMP_PREFIX_NONE
, 32, 4,
1777 (const void *)d
, sizeof(*d
), false);
1779 wil_txdesc_unmap(dev
, d
, ctx
);
1782 if (likely(d
->dma
.error
== 0)) {
1783 ndev
->stats
.tx_packets
++;
1784 ndev
->stats
.tx_bytes
+= skb
->len
;
1786 stats
->tx_packets
++;
1787 stats
->tx_bytes
+= skb
->len
;
1790 ndev
->stats
.tx_errors
++;
1794 wil_consume_skb(skb
, d
->dma
.error
== 0);
1796 memset(ctx
, 0, sizeof(*ctx
));
1797 /* There is no need to touch HW descriptor:
1798 * - ststus bit TX_DMA_STATUS_DU is set by design,
1799 * so hardware will not try to process this desc.,
1800 * - rest of descriptor will be initialized on Tx.
1802 vring
->swtail
= wil_vring_next_tail(vring
);
1807 /* performance monitoring */
1808 used_new
= wil_vring_used_tx(vring
);
1809 if (wil_val_in_range(vring_idle_trsh
,
1810 used_new
, used_before_complete
)) {
1811 wil_dbg_txrx(wil
, "Ring[%2d] idle %d -> %d\n",
1812 ringid
, used_before_complete
, used_new
);
1813 txdata
->last_idle
= get_cycles();
1816 if (wil_vring_avail_tx(vring
) > wil_vring_wmark_high(vring
)) {
1817 wil_dbg_txrx(wil
, "netif_tx_wake : ring not full\n");
1818 netif_tx_wake_all_queues(wil_to_ndev(wil
));