2 * Copyright (c) 2005-2011 Atheros Communications Inc.
3 * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
5 * Permission to use, copy, modify, and/or distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
24 * Support for Copy Engine hardware, which is mainly used for
25 * communication between Host and Target over a PCIe interconnect.
29 * A single CopyEngine (CE) comprises two "rings":
33 * Each ring consists of a number of descriptors which specify
34 * an address, length, and meta-data.
36 * Typically, one side of the PCIe interconnect (Host or Target)
37 * controls one ring and the other side controls the other ring.
38 * The source side chooses when to initiate a transfer and it
39 * chooses what to send (buffer address, length). The destination
40 * side keeps a supply of "anonymous receive buffers" available and
41 * it handles incoming data as it arrives (when the destination
42 * recieves an interrupt).
44 * The sender may send a simple buffer (address/length) or it may
45 * send a small list of buffers. When a small list is sent, hardware
46 * "gathers" these and they end up in a single destination buffer
47 * with a single interrupt.
49 * There are several "contexts" managed by this layer -- more, it
50 * may seem -- than should be needed. These are provided mainly for
51 * maximum flexibility and especially to facilitate a simpler HIF
52 * implementation. There are per-CopyEngine recv, send, and watermark
53 * contexts. These are supplied by the caller when a recv, send,
54 * or watermark handler is established and they are echoed back to
55 * the caller when the respective callbacks are invoked. There is
56 * also a per-transfer context supplied by the caller when a buffer
57 * (or sendlist) is sent and when a buffer is enqueued for recv.
58 * These per-transfer contexts are echoed back to the caller when
59 * the buffer is sent/received.
62 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k
*ar
,
66 ath10k_pci_write32(ar
, ce_ctrl_addr
+ DST_WR_INDEX_ADDRESS
, n
);
69 static inline u32
ath10k_ce_dest_ring_write_index_get(struct ath10k
*ar
,
72 return ath10k_pci_read32(ar
, ce_ctrl_addr
+ DST_WR_INDEX_ADDRESS
);
75 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k
*ar
,
79 ath10k_pci_write32(ar
, ce_ctrl_addr
+ SR_WR_INDEX_ADDRESS
, n
);
82 static inline u32
ath10k_ce_src_ring_write_index_get(struct ath10k
*ar
,
85 return ath10k_pci_read32(ar
, ce_ctrl_addr
+ SR_WR_INDEX_ADDRESS
);
88 static inline u32
ath10k_ce_src_ring_read_index_get(struct ath10k
*ar
,
91 return ath10k_pci_read32(ar
, ce_ctrl_addr
+ CURRENT_SRRI_ADDRESS
);
94 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k
*ar
,
98 ath10k_pci_write32(ar
, ce_ctrl_addr
+ SR_BA_ADDRESS
, addr
);
101 static inline void ath10k_ce_src_ring_size_set(struct ath10k
*ar
,
105 ath10k_pci_write32(ar
, ce_ctrl_addr
+ SR_SIZE_ADDRESS
, n
);
108 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k
*ar
,
112 u32 ctrl1_addr
= ath10k_pci_read32((ar
),
113 (ce_ctrl_addr
) + CE_CTRL1_ADDRESS
);
115 ath10k_pci_write32(ar
, ce_ctrl_addr
+ CE_CTRL1_ADDRESS
,
116 (ctrl1_addr
& ~CE_CTRL1_DMAX_LENGTH_MASK
) |
117 CE_CTRL1_DMAX_LENGTH_SET(n
));
120 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k
*ar
,
124 u32 ctrl1_addr
= ath10k_pci_read32(ar
, ce_ctrl_addr
+ CE_CTRL1_ADDRESS
);
126 ath10k_pci_write32(ar
, ce_ctrl_addr
+ CE_CTRL1_ADDRESS
,
127 (ctrl1_addr
& ~CE_CTRL1_SRC_RING_BYTE_SWAP_EN_MASK
) |
128 CE_CTRL1_SRC_RING_BYTE_SWAP_EN_SET(n
));
131 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k
*ar
,
135 u32 ctrl1_addr
= ath10k_pci_read32(ar
, ce_ctrl_addr
+ CE_CTRL1_ADDRESS
);
137 ath10k_pci_write32(ar
, ce_ctrl_addr
+ CE_CTRL1_ADDRESS
,
138 (ctrl1_addr
& ~CE_CTRL1_DST_RING_BYTE_SWAP_EN_MASK
) |
139 CE_CTRL1_DST_RING_BYTE_SWAP_EN_SET(n
));
142 static inline u32
ath10k_ce_dest_ring_read_index_get(struct ath10k
*ar
,
145 return ath10k_pci_read32(ar
, ce_ctrl_addr
+ CURRENT_DRRI_ADDRESS
);
148 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k
*ar
,
152 ath10k_pci_write32(ar
, ce_ctrl_addr
+ DR_BA_ADDRESS
, addr
);
155 static inline void ath10k_ce_dest_ring_size_set(struct ath10k
*ar
,
159 ath10k_pci_write32(ar
, ce_ctrl_addr
+ DR_SIZE_ADDRESS
, n
);
162 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k
*ar
,
166 u32 addr
= ath10k_pci_read32(ar
, ce_ctrl_addr
+ SRC_WATERMARK_ADDRESS
);
168 ath10k_pci_write32(ar
, ce_ctrl_addr
+ SRC_WATERMARK_ADDRESS
,
169 (addr
& ~SRC_WATERMARK_HIGH_MASK
) |
170 SRC_WATERMARK_HIGH_SET(n
));
173 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k
*ar
,
177 u32 addr
= ath10k_pci_read32(ar
, ce_ctrl_addr
+ SRC_WATERMARK_ADDRESS
);
179 ath10k_pci_write32(ar
, ce_ctrl_addr
+ SRC_WATERMARK_ADDRESS
,
180 (addr
& ~SRC_WATERMARK_LOW_MASK
) |
181 SRC_WATERMARK_LOW_SET(n
));
184 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k
*ar
,
188 u32 addr
= ath10k_pci_read32(ar
, ce_ctrl_addr
+ DST_WATERMARK_ADDRESS
);
190 ath10k_pci_write32(ar
, ce_ctrl_addr
+ DST_WATERMARK_ADDRESS
,
191 (addr
& ~DST_WATERMARK_HIGH_MASK
) |
192 DST_WATERMARK_HIGH_SET(n
));
195 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k
*ar
,
199 u32 addr
= ath10k_pci_read32(ar
, ce_ctrl_addr
+ DST_WATERMARK_ADDRESS
);
201 ath10k_pci_write32(ar
, ce_ctrl_addr
+ DST_WATERMARK_ADDRESS
,
202 (addr
& ~DST_WATERMARK_LOW_MASK
) |
203 DST_WATERMARK_LOW_SET(n
));
206 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k
*ar
,
209 u32 host_ie_addr
= ath10k_pci_read32(ar
,
210 ce_ctrl_addr
+ HOST_IE_ADDRESS
);
212 ath10k_pci_write32(ar
, ce_ctrl_addr
+ HOST_IE_ADDRESS
,
213 host_ie_addr
| HOST_IE_COPY_COMPLETE_MASK
);
216 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k
*ar
,
219 u32 host_ie_addr
= ath10k_pci_read32(ar
,
220 ce_ctrl_addr
+ HOST_IE_ADDRESS
);
222 ath10k_pci_write32(ar
, ce_ctrl_addr
+ HOST_IE_ADDRESS
,
223 host_ie_addr
& ~HOST_IE_COPY_COMPLETE_MASK
);
226 static inline void ath10k_ce_watermark_intr_disable(struct ath10k
*ar
,
229 u32 host_ie_addr
= ath10k_pci_read32(ar
,
230 ce_ctrl_addr
+ HOST_IE_ADDRESS
);
232 ath10k_pci_write32(ar
, ce_ctrl_addr
+ HOST_IE_ADDRESS
,
233 host_ie_addr
& ~CE_WATERMARK_MASK
);
236 static inline void ath10k_ce_error_intr_enable(struct ath10k
*ar
,
239 u32 misc_ie_addr
= ath10k_pci_read32(ar
,
240 ce_ctrl_addr
+ MISC_IE_ADDRESS
);
242 ath10k_pci_write32(ar
, ce_ctrl_addr
+ MISC_IE_ADDRESS
,
243 misc_ie_addr
| CE_ERROR_MASK
);
246 static inline void ath10k_ce_error_intr_disable(struct ath10k
*ar
,
249 u32 misc_ie_addr
= ath10k_pci_read32(ar
,
250 ce_ctrl_addr
+ MISC_IE_ADDRESS
);
252 ath10k_pci_write32(ar
, ce_ctrl_addr
+ MISC_IE_ADDRESS
,
253 misc_ie_addr
& ~CE_ERROR_MASK
);
256 static inline void ath10k_ce_engine_int_status_clear(struct ath10k
*ar
,
260 ath10k_pci_write32(ar
, ce_ctrl_addr
+ HOST_IS_ADDRESS
, mask
);
265 * Guts of ath10k_ce_send, used by both ath10k_ce_send and
266 * ath10k_ce_sendlist_send.
267 * The caller takes responsibility for any needed locking.
269 static int ath10k_ce_send_nolock(struct ath10k_ce_pipe
*ce_state
,
270 void *per_transfer_context
,
273 unsigned int transfer_id
,
276 struct ath10k
*ar
= ce_state
->ar
;
277 struct ath10k_ce_ring
*src_ring
= ce_state
->src_ring
;
278 struct ce_desc
*desc
, *sdesc
;
279 unsigned int nentries_mask
= src_ring
->nentries_mask
;
280 unsigned int sw_index
= src_ring
->sw_index
;
281 unsigned int write_index
= src_ring
->write_index
;
282 u32 ctrl_addr
= ce_state
->ctrl_addr
;
286 if (nbytes
> ce_state
->src_sz_max
)
287 ath10k_warn("%s: send more we can (nbytes: %d, max: %d)\n",
288 __func__
, nbytes
, ce_state
->src_sz_max
);
290 ret
= ath10k_pci_wake(ar
);
294 if (unlikely(CE_RING_DELTA(nentries_mask
,
295 write_index
, sw_index
- 1) <= 0)) {
300 desc
= CE_SRC_RING_TO_DESC(src_ring
->base_addr_owner_space
,
302 sdesc
= CE_SRC_RING_TO_DESC(src_ring
->shadow_base
, write_index
);
304 desc_flags
|= SM(transfer_id
, CE_DESC_FLAGS_META_DATA
);
306 if (flags
& CE_SEND_FLAG_GATHER
)
307 desc_flags
|= CE_DESC_FLAGS_GATHER
;
308 if (flags
& CE_SEND_FLAG_BYTE_SWAP
)
309 desc_flags
|= CE_DESC_FLAGS_BYTE_SWAP
;
311 sdesc
->addr
= __cpu_to_le32(buffer
);
312 sdesc
->nbytes
= __cpu_to_le16(nbytes
);
313 sdesc
->flags
= __cpu_to_le16(desc_flags
);
317 src_ring
->per_transfer_context
[write_index
] = per_transfer_context
;
319 /* Update Source Ring Write Index */
320 write_index
= CE_RING_IDX_INCR(nentries_mask
, write_index
);
323 if (!(flags
& CE_SEND_FLAG_GATHER
))
324 ath10k_ce_src_ring_write_index_set(ar
, ctrl_addr
, write_index
);
326 src_ring
->write_index
= write_index
;
328 ath10k_pci_sleep(ar
);
332 int ath10k_ce_send(struct ath10k_ce_pipe
*ce_state
,
333 void *per_transfer_context
,
336 unsigned int transfer_id
,
339 struct ath10k
*ar
= ce_state
->ar
;
340 struct ath10k_pci
*ar_pci
= ath10k_pci_priv(ar
);
343 spin_lock_bh(&ar_pci
->ce_lock
);
344 ret
= ath10k_ce_send_nolock(ce_state
, per_transfer_context
,
345 buffer
, nbytes
, transfer_id
, flags
);
346 spin_unlock_bh(&ar_pci
->ce_lock
);
351 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe
*pipe
)
353 struct ath10k
*ar
= pipe
->ar
;
354 struct ath10k_pci
*ar_pci
= ath10k_pci_priv(ar
);
357 spin_lock_bh(&ar_pci
->ce_lock
);
358 delta
= CE_RING_DELTA(pipe
->src_ring
->nentries_mask
,
359 pipe
->src_ring
->write_index
,
360 pipe
->src_ring
->sw_index
- 1);
361 spin_unlock_bh(&ar_pci
->ce_lock
);
366 int ath10k_ce_recv_buf_enqueue(struct ath10k_ce_pipe
*ce_state
,
367 void *per_recv_context
,
370 struct ath10k_ce_ring
*dest_ring
= ce_state
->dest_ring
;
371 u32 ctrl_addr
= ce_state
->ctrl_addr
;
372 struct ath10k
*ar
= ce_state
->ar
;
373 struct ath10k_pci
*ar_pci
= ath10k_pci_priv(ar
);
374 unsigned int nentries_mask
= dest_ring
->nentries_mask
;
375 unsigned int write_index
;
376 unsigned int sw_index
;
379 spin_lock_bh(&ar_pci
->ce_lock
);
380 write_index
= dest_ring
->write_index
;
381 sw_index
= dest_ring
->sw_index
;
383 ret
= ath10k_pci_wake(ar
);
387 if (CE_RING_DELTA(nentries_mask
, write_index
, sw_index
- 1) > 0) {
388 struct ce_desc
*base
= dest_ring
->base_addr_owner_space
;
389 struct ce_desc
*desc
= CE_DEST_RING_TO_DESC(base
, write_index
);
391 /* Update destination descriptor */
392 desc
->addr
= __cpu_to_le32(buffer
);
395 dest_ring
->per_transfer_context
[write_index
] =
398 /* Update Destination Ring Write Index */
399 write_index
= CE_RING_IDX_INCR(nentries_mask
, write_index
);
400 ath10k_ce_dest_ring_write_index_set(ar
, ctrl_addr
, write_index
);
401 dest_ring
->write_index
= write_index
;
406 ath10k_pci_sleep(ar
);
409 spin_unlock_bh(&ar_pci
->ce_lock
);
415 * Guts of ath10k_ce_completed_recv_next.
416 * The caller takes responsibility for any necessary locking.
418 static int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe
*ce_state
,
419 void **per_transfer_contextp
,
421 unsigned int *nbytesp
,
422 unsigned int *transfer_idp
,
423 unsigned int *flagsp
)
425 struct ath10k_ce_ring
*dest_ring
= ce_state
->dest_ring
;
426 unsigned int nentries_mask
= dest_ring
->nentries_mask
;
427 unsigned int sw_index
= dest_ring
->sw_index
;
429 struct ce_desc
*base
= dest_ring
->base_addr_owner_space
;
430 struct ce_desc
*desc
= CE_DEST_RING_TO_DESC(base
, sw_index
);
431 struct ce_desc sdesc
;
434 /* Copy in one go for performance reasons */
437 nbytes
= __le16_to_cpu(sdesc
.nbytes
);
440 * This closes a relatively unusual race where the Host
441 * sees the updated DRRI before the update to the
442 * corresponding descriptor has completed. We treat this
443 * as a descriptor that is not yet done.
450 /* Return data from completed destination descriptor */
451 *bufferp
= __le32_to_cpu(sdesc
.addr
);
453 *transfer_idp
= MS(__le16_to_cpu(sdesc
.flags
), CE_DESC_FLAGS_META_DATA
);
455 if (__le16_to_cpu(sdesc
.flags
) & CE_DESC_FLAGS_BYTE_SWAP
)
456 *flagsp
= CE_RECV_FLAG_SWAPPED
;
460 if (per_transfer_contextp
)
461 *per_transfer_contextp
=
462 dest_ring
->per_transfer_context
[sw_index
];
465 dest_ring
->per_transfer_context
[sw_index
] = NULL
;
467 /* Update sw_index */
468 sw_index
= CE_RING_IDX_INCR(nentries_mask
, sw_index
);
469 dest_ring
->sw_index
= sw_index
;
474 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe
*ce_state
,
475 void **per_transfer_contextp
,
477 unsigned int *nbytesp
,
478 unsigned int *transfer_idp
,
479 unsigned int *flagsp
)
481 struct ath10k
*ar
= ce_state
->ar
;
482 struct ath10k_pci
*ar_pci
= ath10k_pci_priv(ar
);
485 spin_lock_bh(&ar_pci
->ce_lock
);
486 ret
= ath10k_ce_completed_recv_next_nolock(ce_state
,
487 per_transfer_contextp
,
489 transfer_idp
, flagsp
);
490 spin_unlock_bh(&ar_pci
->ce_lock
);
495 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe
*ce_state
,
496 void **per_transfer_contextp
,
499 struct ath10k_ce_ring
*dest_ring
;
500 unsigned int nentries_mask
;
501 unsigned int sw_index
;
502 unsigned int write_index
;
505 struct ath10k_pci
*ar_pci
;
507 dest_ring
= ce_state
->dest_ring
;
513 ar_pci
= ath10k_pci_priv(ar
);
515 spin_lock_bh(&ar_pci
->ce_lock
);
517 nentries_mask
= dest_ring
->nentries_mask
;
518 sw_index
= dest_ring
->sw_index
;
519 write_index
= dest_ring
->write_index
;
520 if (write_index
!= sw_index
) {
521 struct ce_desc
*base
= dest_ring
->base_addr_owner_space
;
522 struct ce_desc
*desc
= CE_DEST_RING_TO_DESC(base
, sw_index
);
524 /* Return data from completed destination descriptor */
525 *bufferp
= __le32_to_cpu(desc
->addr
);
527 if (per_transfer_contextp
)
528 *per_transfer_contextp
=
529 dest_ring
->per_transfer_context
[sw_index
];
532 dest_ring
->per_transfer_context
[sw_index
] = NULL
;
534 /* Update sw_index */
535 sw_index
= CE_RING_IDX_INCR(nentries_mask
, sw_index
);
536 dest_ring
->sw_index
= sw_index
;
542 spin_unlock_bh(&ar_pci
->ce_lock
);
548 * Guts of ath10k_ce_completed_send_next.
549 * The caller takes responsibility for any necessary locking.
551 static int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe
*ce_state
,
552 void **per_transfer_contextp
,
554 unsigned int *nbytesp
,
555 unsigned int *transfer_idp
)
557 struct ath10k_ce_ring
*src_ring
= ce_state
->src_ring
;
558 u32 ctrl_addr
= ce_state
->ctrl_addr
;
559 struct ath10k
*ar
= ce_state
->ar
;
560 unsigned int nentries_mask
= src_ring
->nentries_mask
;
561 unsigned int sw_index
= src_ring
->sw_index
;
562 struct ce_desc
*sdesc
, *sbase
;
563 unsigned int read_index
;
566 if (src_ring
->hw_index
== sw_index
) {
568 * The SW completion index has caught up with the cached
569 * version of the HW completion index.
570 * Update the cached HW completion index to see whether
571 * the SW has really caught up to the HW, or if the cached
572 * value of the HW index has become stale.
575 ret
= ath10k_pci_wake(ar
);
580 ath10k_ce_src_ring_read_index_get(ar
, ctrl_addr
);
581 src_ring
->hw_index
&= nentries_mask
;
583 ath10k_pci_sleep(ar
);
586 read_index
= src_ring
->hw_index
;
588 if ((read_index
== sw_index
) || (read_index
== 0xffffffff))
591 sbase
= src_ring
->shadow_base
;
592 sdesc
= CE_SRC_RING_TO_DESC(sbase
, sw_index
);
594 /* Return data from completed source descriptor */
595 *bufferp
= __le32_to_cpu(sdesc
->addr
);
596 *nbytesp
= __le16_to_cpu(sdesc
->nbytes
);
597 *transfer_idp
= MS(__le16_to_cpu(sdesc
->flags
),
598 CE_DESC_FLAGS_META_DATA
);
600 if (per_transfer_contextp
)
601 *per_transfer_contextp
=
602 src_ring
->per_transfer_context
[sw_index
];
605 src_ring
->per_transfer_context
[sw_index
] = NULL
;
607 /* Update sw_index */
608 sw_index
= CE_RING_IDX_INCR(nentries_mask
, sw_index
);
609 src_ring
->sw_index
= sw_index
;
614 /* NB: Modeled after ath10k_ce_completed_send_next */
615 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe
*ce_state
,
616 void **per_transfer_contextp
,
618 unsigned int *nbytesp
,
619 unsigned int *transfer_idp
)
621 struct ath10k_ce_ring
*src_ring
;
622 unsigned int nentries_mask
;
623 unsigned int sw_index
;
624 unsigned int write_index
;
627 struct ath10k_pci
*ar_pci
;
629 src_ring
= ce_state
->src_ring
;
635 ar_pci
= ath10k_pci_priv(ar
);
637 spin_lock_bh(&ar_pci
->ce_lock
);
639 nentries_mask
= src_ring
->nentries_mask
;
640 sw_index
= src_ring
->sw_index
;
641 write_index
= src_ring
->write_index
;
643 if (write_index
!= sw_index
) {
644 struct ce_desc
*base
= src_ring
->base_addr_owner_space
;
645 struct ce_desc
*desc
= CE_SRC_RING_TO_DESC(base
, sw_index
);
647 /* Return data from completed source descriptor */
648 *bufferp
= __le32_to_cpu(desc
->addr
);
649 *nbytesp
= __le16_to_cpu(desc
->nbytes
);
650 *transfer_idp
= MS(__le16_to_cpu(desc
->flags
),
651 CE_DESC_FLAGS_META_DATA
);
653 if (per_transfer_contextp
)
654 *per_transfer_contextp
=
655 src_ring
->per_transfer_context
[sw_index
];
658 src_ring
->per_transfer_context
[sw_index
] = NULL
;
660 /* Update sw_index */
661 sw_index
= CE_RING_IDX_INCR(nentries_mask
, sw_index
);
662 src_ring
->sw_index
= sw_index
;
668 spin_unlock_bh(&ar_pci
->ce_lock
);
673 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe
*ce_state
,
674 void **per_transfer_contextp
,
676 unsigned int *nbytesp
,
677 unsigned int *transfer_idp
)
679 struct ath10k
*ar
= ce_state
->ar
;
680 struct ath10k_pci
*ar_pci
= ath10k_pci_priv(ar
);
683 spin_lock_bh(&ar_pci
->ce_lock
);
684 ret
= ath10k_ce_completed_send_next_nolock(ce_state
,
685 per_transfer_contextp
,
688 spin_unlock_bh(&ar_pci
->ce_lock
);
694 * Guts of interrupt handler for per-engine interrupts on a particular CE.
696 * Invokes registered callbacks for recv_complete,
697 * send_complete, and watermarks.
699 void ath10k_ce_per_engine_service(struct ath10k
*ar
, unsigned int ce_id
)
701 struct ath10k_pci
*ar_pci
= ath10k_pci_priv(ar
);
702 struct ath10k_ce_pipe
*ce_state
= &ar_pci
->ce_states
[ce_id
];
703 u32 ctrl_addr
= ce_state
->ctrl_addr
;
706 ret
= ath10k_pci_wake(ar
);
710 spin_lock_bh(&ar_pci
->ce_lock
);
712 /* Clear the copy-complete interrupts that will be handled here. */
713 ath10k_ce_engine_int_status_clear(ar
, ctrl_addr
,
714 HOST_IS_COPY_COMPLETE_MASK
);
716 spin_unlock_bh(&ar_pci
->ce_lock
);
718 if (ce_state
->recv_cb
)
719 ce_state
->recv_cb(ce_state
);
721 if (ce_state
->send_cb
)
722 ce_state
->send_cb(ce_state
);
724 spin_lock_bh(&ar_pci
->ce_lock
);
727 * Misc CE interrupts are not being handled, but still need
730 ath10k_ce_engine_int_status_clear(ar
, ctrl_addr
, CE_WATERMARK_MASK
);
732 spin_unlock_bh(&ar_pci
->ce_lock
);
733 ath10k_pci_sleep(ar
);
737 * Handler for per-engine interrupts on ALL active CEs.
738 * This is used in cases where the system is sharing a
739 * single interrput for all CEs
742 void ath10k_ce_per_engine_service_any(struct ath10k
*ar
)
747 ret
= ath10k_pci_wake(ar
);
751 intr_summary
= CE_INTERRUPT_SUMMARY(ar
);
753 for (ce_id
= 0; intr_summary
&& (ce_id
< CE_COUNT
); ce_id
++) {
754 if (intr_summary
& (1 << ce_id
))
755 intr_summary
&= ~(1 << ce_id
);
757 /* no intr pending on this CE */
760 ath10k_ce_per_engine_service(ar
, ce_id
);
763 ath10k_pci_sleep(ar
);
767 * Adjust interrupts for the copy complete handler.
768 * If it's needed for either send or recv, then unmask
769 * this interrupt; otherwise, mask it.
771 * Called with ce_lock held.
773 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe
*ce_state
,
774 int disable_copy_compl_intr
)
776 u32 ctrl_addr
= ce_state
->ctrl_addr
;
777 struct ath10k
*ar
= ce_state
->ar
;
780 ret
= ath10k_pci_wake(ar
);
784 if ((!disable_copy_compl_intr
) &&
785 (ce_state
->send_cb
|| ce_state
->recv_cb
))
786 ath10k_ce_copy_complete_inter_enable(ar
, ctrl_addr
);
788 ath10k_ce_copy_complete_intr_disable(ar
, ctrl_addr
);
790 ath10k_ce_watermark_intr_disable(ar
, ctrl_addr
);
792 ath10k_pci_sleep(ar
);
795 int ath10k_ce_enable_err_irq(struct ath10k
*ar
)
799 ret
= ath10k_pci_wake(ar
);
803 for (i
= 0; i
< CE_COUNT
; i
++)
804 ath10k_ce_error_intr_enable(ar
, ath10k_ce_base_address(i
));
806 ath10k_pci_sleep(ar
);
810 int ath10k_ce_disable_interrupts(struct ath10k
*ar
)
814 ret
= ath10k_pci_wake(ar
);
818 for (ce_id
= 0; ce_id
< CE_COUNT
; ce_id
++) {
819 u32 ctrl_addr
= ath10k_ce_base_address(ce_id
);
821 ath10k_ce_copy_complete_intr_disable(ar
, ctrl_addr
);
822 ath10k_ce_error_intr_disable(ar
, ctrl_addr
);
823 ath10k_ce_watermark_intr_disable(ar
, ctrl_addr
);
826 ath10k_pci_sleep(ar
);
831 void ath10k_ce_send_cb_register(struct ath10k_ce_pipe
*ce_state
,
832 void (*send_cb
)(struct ath10k_ce_pipe
*),
833 int disable_interrupts
)
835 struct ath10k
*ar
= ce_state
->ar
;
836 struct ath10k_pci
*ar_pci
= ath10k_pci_priv(ar
);
838 spin_lock_bh(&ar_pci
->ce_lock
);
839 ce_state
->send_cb
= send_cb
;
840 ath10k_ce_per_engine_handler_adjust(ce_state
, disable_interrupts
);
841 spin_unlock_bh(&ar_pci
->ce_lock
);
844 void ath10k_ce_recv_cb_register(struct ath10k_ce_pipe
*ce_state
,
845 void (*recv_cb
)(struct ath10k_ce_pipe
*))
847 struct ath10k
*ar
= ce_state
->ar
;
848 struct ath10k_pci
*ar_pci
= ath10k_pci_priv(ar
);
850 spin_lock_bh(&ar_pci
->ce_lock
);
851 ce_state
->recv_cb
= recv_cb
;
852 ath10k_ce_per_engine_handler_adjust(ce_state
, 0);
853 spin_unlock_bh(&ar_pci
->ce_lock
);
856 static int ath10k_ce_init_src_ring(struct ath10k
*ar
,
858 struct ath10k_ce_pipe
*ce_state
,
859 const struct ce_attr
*attr
)
861 struct ath10k_pci
*ar_pci
= ath10k_pci_priv(ar
);
862 struct ath10k_ce_ring
*src_ring
;
863 unsigned int nentries
= attr
->src_nentries
;
864 unsigned int ce_nbytes
;
865 u32 ctrl_addr
= ath10k_ce_base_address(ce_id
);
866 dma_addr_t base_addr
;
869 nentries
= roundup_pow_of_two(nentries
);
871 if (ce_state
->src_ring
) {
872 WARN_ON(ce_state
->src_ring
->nentries
!= nentries
);
876 ce_nbytes
= sizeof(struct ath10k_ce_ring
) + (nentries
* sizeof(void *));
877 ptr
= kzalloc(ce_nbytes
, GFP_KERNEL
);
881 ce_state
->src_ring
= (struct ath10k_ce_ring
*)ptr
;
882 src_ring
= ce_state
->src_ring
;
884 ptr
+= sizeof(struct ath10k_ce_ring
);
885 src_ring
->nentries
= nentries
;
886 src_ring
->nentries_mask
= nentries
- 1;
888 src_ring
->sw_index
= ath10k_ce_src_ring_read_index_get(ar
, ctrl_addr
);
889 src_ring
->sw_index
&= src_ring
->nentries_mask
;
890 src_ring
->hw_index
= src_ring
->sw_index
;
892 src_ring
->write_index
=
893 ath10k_ce_src_ring_write_index_get(ar
, ctrl_addr
);
894 src_ring
->write_index
&= src_ring
->nentries_mask
;
896 src_ring
->per_transfer_context
= (void **)ptr
;
899 * Legacy platforms that do not support cache
900 * coherent DMA are unsupported
902 src_ring
->base_addr_owner_space_unaligned
=
903 pci_alloc_consistent(ar_pci
->pdev
,
904 (nentries
* sizeof(struct ce_desc
) +
907 if (!src_ring
->base_addr_owner_space_unaligned
) {
908 kfree(ce_state
->src_ring
);
909 ce_state
->src_ring
= NULL
;
913 src_ring
->base_addr_ce_space_unaligned
= base_addr
;
915 src_ring
->base_addr_owner_space
= PTR_ALIGN(
916 src_ring
->base_addr_owner_space_unaligned
,
918 src_ring
->base_addr_ce_space
= ALIGN(
919 src_ring
->base_addr_ce_space_unaligned
,
923 * Also allocate a shadow src ring in regular
924 * mem to use for faster access.
926 src_ring
->shadow_base_unaligned
=
927 kmalloc((nentries
* sizeof(struct ce_desc
) +
928 CE_DESC_RING_ALIGN
), GFP_KERNEL
);
929 if (!src_ring
->shadow_base_unaligned
) {
930 pci_free_consistent(ar_pci
->pdev
,
931 (nentries
* sizeof(struct ce_desc
) +
933 src_ring
->base_addr_owner_space
,
934 src_ring
->base_addr_ce_space
);
935 kfree(ce_state
->src_ring
);
936 ce_state
->src_ring
= NULL
;
940 src_ring
->shadow_base
= PTR_ALIGN(
941 src_ring
->shadow_base_unaligned
,
944 ath10k_ce_src_ring_base_addr_set(ar
, ctrl_addr
,
945 src_ring
->base_addr_ce_space
);
946 ath10k_ce_src_ring_size_set(ar
, ctrl_addr
, nentries
);
947 ath10k_ce_src_ring_dmax_set(ar
, ctrl_addr
, attr
->src_sz_max
);
948 ath10k_ce_src_ring_byte_swap_set(ar
, ctrl_addr
, 0);
949 ath10k_ce_src_ring_lowmark_set(ar
, ctrl_addr
, 0);
950 ath10k_ce_src_ring_highmark_set(ar
, ctrl_addr
, nentries
);
952 ath10k_dbg(ATH10K_DBG_BOOT
,
953 "boot ce src ring id %d entries %d base_addr %p\n",
954 ce_id
, nentries
, src_ring
->base_addr_owner_space
);
959 static int ath10k_ce_init_dest_ring(struct ath10k
*ar
,
961 struct ath10k_ce_pipe
*ce_state
,
962 const struct ce_attr
*attr
)
964 struct ath10k_pci
*ar_pci
= ath10k_pci_priv(ar
);
965 struct ath10k_ce_ring
*dest_ring
;
966 unsigned int nentries
= attr
->dest_nentries
;
967 unsigned int ce_nbytes
;
968 u32 ctrl_addr
= ath10k_ce_base_address(ce_id
);
969 dma_addr_t base_addr
;
972 nentries
= roundup_pow_of_two(nentries
);
974 if (ce_state
->dest_ring
) {
975 WARN_ON(ce_state
->dest_ring
->nentries
!= nentries
);
979 ce_nbytes
= sizeof(struct ath10k_ce_ring
) + (nentries
* sizeof(void *));
980 ptr
= kzalloc(ce_nbytes
, GFP_KERNEL
);
984 ce_state
->dest_ring
= (struct ath10k_ce_ring
*)ptr
;
985 dest_ring
= ce_state
->dest_ring
;
987 ptr
+= sizeof(struct ath10k_ce_ring
);
988 dest_ring
->nentries
= nentries
;
989 dest_ring
->nentries_mask
= nentries
- 1;
991 dest_ring
->sw_index
= ath10k_ce_dest_ring_read_index_get(ar
, ctrl_addr
);
992 dest_ring
->sw_index
&= dest_ring
->nentries_mask
;
993 dest_ring
->write_index
=
994 ath10k_ce_dest_ring_write_index_get(ar
, ctrl_addr
);
995 dest_ring
->write_index
&= dest_ring
->nentries_mask
;
997 dest_ring
->per_transfer_context
= (void **)ptr
;
1000 * Legacy platforms that do not support cache
1001 * coherent DMA are unsupported
1003 dest_ring
->base_addr_owner_space_unaligned
=
1004 pci_alloc_consistent(ar_pci
->pdev
,
1005 (nentries
* sizeof(struct ce_desc
) +
1006 CE_DESC_RING_ALIGN
),
1008 if (!dest_ring
->base_addr_owner_space_unaligned
) {
1009 kfree(ce_state
->dest_ring
);
1010 ce_state
->dest_ring
= NULL
;
1014 dest_ring
->base_addr_ce_space_unaligned
= base_addr
;
1017 * Correctly initialize memory to 0 to prevent garbage
1018 * data crashing system when download firmware
1020 memset(dest_ring
->base_addr_owner_space_unaligned
, 0,
1021 nentries
* sizeof(struct ce_desc
) + CE_DESC_RING_ALIGN
);
1023 dest_ring
->base_addr_owner_space
= PTR_ALIGN(
1024 dest_ring
->base_addr_owner_space_unaligned
,
1025 CE_DESC_RING_ALIGN
);
1026 dest_ring
->base_addr_ce_space
= ALIGN(
1027 dest_ring
->base_addr_ce_space_unaligned
,
1028 CE_DESC_RING_ALIGN
);
1030 ath10k_ce_dest_ring_base_addr_set(ar
, ctrl_addr
,
1031 dest_ring
->base_addr_ce_space
);
1032 ath10k_ce_dest_ring_size_set(ar
, ctrl_addr
, nentries
);
1033 ath10k_ce_dest_ring_byte_swap_set(ar
, ctrl_addr
, 0);
1034 ath10k_ce_dest_ring_lowmark_set(ar
, ctrl_addr
, 0);
1035 ath10k_ce_dest_ring_highmark_set(ar
, ctrl_addr
, nentries
);
1037 ath10k_dbg(ATH10K_DBG_BOOT
,
1038 "boot ce dest ring id %d entries %d base_addr %p\n",
1039 ce_id
, nentries
, dest_ring
->base_addr_owner_space
);
1044 static struct ath10k_ce_pipe
*ath10k_ce_init_state(struct ath10k
*ar
,
1046 const struct ce_attr
*attr
)
1048 struct ath10k_pci
*ar_pci
= ath10k_pci_priv(ar
);
1049 struct ath10k_ce_pipe
*ce_state
= &ar_pci
->ce_states
[ce_id
];
1050 u32 ctrl_addr
= ath10k_ce_base_address(ce_id
);
1052 spin_lock_bh(&ar_pci
->ce_lock
);
1055 ce_state
->id
= ce_id
;
1056 ce_state
->ctrl_addr
= ctrl_addr
;
1057 ce_state
->attr_flags
= attr
->flags
;
1058 ce_state
->src_sz_max
= attr
->src_sz_max
;
1060 spin_unlock_bh(&ar_pci
->ce_lock
);
1066 * Initialize a Copy Engine based on caller-supplied attributes.
1067 * This may be called once to initialize both source and destination
1068 * rings or it may be called twice for separate source and destination
1069 * initialization. It may be that only one side or the other is
1070 * initialized by software/firmware.
1072 struct ath10k_ce_pipe
*ath10k_ce_init(struct ath10k
*ar
,
1074 const struct ce_attr
*attr
)
1076 struct ath10k_ce_pipe
*ce_state
;
1080 * Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1081 * additional TX locking checks.
1083 * For the lack of a better place do the check here.
1085 BUILD_BUG_ON(TARGET_NUM_MSDU_DESC
>
1086 (CE_HTT_H2T_MSG_SRC_NENTRIES
- 1));
1087 BUILD_BUG_ON(TARGET_10X_NUM_MSDU_DESC
>
1088 (CE_HTT_H2T_MSG_SRC_NENTRIES
- 1));
1090 ret
= ath10k_pci_wake(ar
);
1094 ce_state
= ath10k_ce_init_state(ar
, ce_id
, attr
);
1096 ath10k_err("Failed to initialize CE state for ID: %d\n", ce_id
);
1100 if (attr
->src_nentries
) {
1101 ret
= ath10k_ce_init_src_ring(ar
, ce_id
, ce_state
, attr
);
1103 ath10k_err("Failed to initialize CE src ring for ID: %d (%d)\n",
1105 ath10k_ce_deinit(ce_state
);
1111 if (attr
->dest_nentries
) {
1112 ret
= ath10k_ce_init_dest_ring(ar
, ce_id
, ce_state
, attr
);
1114 ath10k_err("Failed to initialize CE dest ring for ID: %d (%d)\n",
1116 ath10k_ce_deinit(ce_state
);
1123 ath10k_pci_sleep(ar
);
1127 void ath10k_ce_deinit(struct ath10k_ce_pipe
*ce_state
)
1129 struct ath10k
*ar
= ce_state
->ar
;
1130 struct ath10k_pci
*ar_pci
= ath10k_pci_priv(ar
);
1132 if (ce_state
->src_ring
) {
1133 kfree(ce_state
->src_ring
->shadow_base_unaligned
);
1134 pci_free_consistent(ar_pci
->pdev
,
1135 (ce_state
->src_ring
->nentries
*
1136 sizeof(struct ce_desc
) +
1137 CE_DESC_RING_ALIGN
),
1138 ce_state
->src_ring
->base_addr_owner_space
,
1139 ce_state
->src_ring
->base_addr_ce_space
);
1140 kfree(ce_state
->src_ring
);
1143 if (ce_state
->dest_ring
) {
1144 pci_free_consistent(ar_pci
->pdev
,
1145 (ce_state
->dest_ring
->nentries
*
1146 sizeof(struct ce_desc
) +
1147 CE_DESC_RING_ALIGN
),
1148 ce_state
->dest_ring
->base_addr_owner_space
,
1149 ce_state
->dest_ring
->base_addr_ce_space
);
1150 kfree(ce_state
->dest_ring
);
1153 ce_state
->src_ring
= NULL
;
1154 ce_state
->dest_ring
= NULL
;