Commit | Line | Data |
---|---|---|
5e3dd157 KV |
1 | /* |
2 | * Copyright (c) 2005-2011 Atheros Communications Inc. | |
3 | * Copyright (c) 2011-2013 Qualcomm Atheros, Inc. | |
4 | * | |
5 | * Permission to use, copy, modify, and/or distribute this software for any | |
6 | * purpose with or without fee is hereby granted, provided that the above | |
7 | * copyright notice and this permission notice appear in all copies. | |
8 | * | |
9 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES | |
10 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF | |
11 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR | |
12 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES | |
13 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN | |
14 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF | |
15 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. | |
16 | */ | |
17 | ||
18 | #include "hif.h" | |
19 | #include "pci.h" | |
20 | #include "ce.h" | |
21 | #include "debug.h" | |
22 | ||
23 | /* | |
24 | * Support for Copy Engine hardware, which is mainly used for | |
25 | * communication between Host and Target over a PCIe interconnect. | |
26 | */ | |
27 | ||
28 | /* | |
29 | * A single CopyEngine (CE) comprises two "rings": | |
30 | * a source ring | |
31 | * a destination ring | |
32 | * | |
33 | * Each ring consists of a number of descriptors which specify | |
34 | * an address, length, and meta-data. | |
35 | * | |
36 | * Typically, one side of the PCIe interconnect (Host or Target) | |
37 | * controls one ring and the other side controls the other ring. | |
38 | * The source side chooses when to initiate a transfer and it | |
39 | * chooses what to send (buffer address, length). The destination | |
40 | * side keeps a supply of "anonymous receive buffers" available and | |
41 | * it handles incoming data as it arrives (when the destination | |
42 | * recieves an interrupt). | |
43 | * | |
44 | * The sender may send a simple buffer (address/length) or it may | |
45 | * send a small list of buffers. When a small list is sent, hardware | |
46 | * "gathers" these and they end up in a single destination buffer | |
47 | * with a single interrupt. | |
48 | * | |
49 | * There are several "contexts" managed by this layer -- more, it | |
50 | * may seem -- than should be needed. These are provided mainly for | |
51 | * maximum flexibility and especially to facilitate a simpler HIF | |
52 | * implementation. There are per-CopyEngine recv, send, and watermark | |
53 | * contexts. These are supplied by the caller when a recv, send, | |
54 | * or watermark handler is established and they are echoed back to | |
55 | * the caller when the respective callbacks are invoked. There is | |
56 | * also a per-transfer context supplied by the caller when a buffer | |
57 | * (or sendlist) is sent and when a buffer is enqueued for recv. | |
58 | * These per-transfer contexts are echoed back to the caller when | |
59 | * the buffer is sent/received. | |
60 | */ | |
61 | ||
62 | static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, | |
63 | u32 ce_ctrl_addr, | |
64 | unsigned int n) | |
65 | { | |
66 | ath10k_pci_write32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS, n); | |
67 | } | |
68 | ||
69 | static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, | |
70 | u32 ce_ctrl_addr) | |
71 | { | |
72 | return ath10k_pci_read32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS); | |
73 | } | |
74 | ||
75 | static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, | |
76 | u32 ce_ctrl_addr, | |
77 | unsigned int n) | |
78 | { | |
57a8930a | 79 | ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n); |
5e3dd157 KV |
80 | } |
81 | ||
82 | static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, | |
83 | u32 ce_ctrl_addr) | |
84 | { | |
85 | return ath10k_pci_read32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS); | |
86 | } | |
87 | ||
88 | static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, | |
89 | u32 ce_ctrl_addr) | |
90 | { | |
91 | return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_SRRI_ADDRESS); | |
92 | } | |
93 | ||
94 | static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, | |
95 | u32 ce_ctrl_addr, | |
96 | unsigned int addr) | |
97 | { | |
98 | ath10k_pci_write32(ar, ce_ctrl_addr + SR_BA_ADDRESS, addr); | |
99 | } | |
100 | ||
101 | static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, | |
102 | u32 ce_ctrl_addr, | |
103 | unsigned int n) | |
104 | { | |
105 | ath10k_pci_write32(ar, ce_ctrl_addr + SR_SIZE_ADDRESS, n); | |
106 | } | |
107 | ||
108 | static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, | |
109 | u32 ce_ctrl_addr, | |
110 | unsigned int n) | |
111 | { | |
112 | u32 ctrl1_addr = ath10k_pci_read32((ar), | |
113 | (ce_ctrl_addr) + CE_CTRL1_ADDRESS); | |
114 | ||
115 | ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, | |
116 | (ctrl1_addr & ~CE_CTRL1_DMAX_LENGTH_MASK) | | |
117 | CE_CTRL1_DMAX_LENGTH_SET(n)); | |
118 | } | |
119 | ||
120 | static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, | |
121 | u32 ce_ctrl_addr, | |
122 | unsigned int n) | |
123 | { | |
124 | u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); | |
125 | ||
126 | ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, | |
127 | (ctrl1_addr & ~CE_CTRL1_SRC_RING_BYTE_SWAP_EN_MASK) | | |
128 | CE_CTRL1_SRC_RING_BYTE_SWAP_EN_SET(n)); | |
129 | } | |
130 | ||
131 | static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, | |
132 | u32 ce_ctrl_addr, | |
133 | unsigned int n) | |
134 | { | |
135 | u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); | |
136 | ||
137 | ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, | |
138 | (ctrl1_addr & ~CE_CTRL1_DST_RING_BYTE_SWAP_EN_MASK) | | |
139 | CE_CTRL1_DST_RING_BYTE_SWAP_EN_SET(n)); | |
140 | } | |
141 | ||
142 | static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, | |
143 | u32 ce_ctrl_addr) | |
144 | { | |
145 | return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_DRRI_ADDRESS); | |
146 | } | |
147 | ||
148 | static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, | |
149 | u32 ce_ctrl_addr, | |
150 | u32 addr) | |
151 | { | |
152 | ath10k_pci_write32(ar, ce_ctrl_addr + DR_BA_ADDRESS, addr); | |
153 | } | |
154 | ||
155 | static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, | |
156 | u32 ce_ctrl_addr, | |
157 | unsigned int n) | |
158 | { | |
159 | ath10k_pci_write32(ar, ce_ctrl_addr + DR_SIZE_ADDRESS, n); | |
160 | } | |
161 | ||
162 | static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, | |
163 | u32 ce_ctrl_addr, | |
164 | unsigned int n) | |
165 | { | |
166 | u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); | |
167 | ||
168 | ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, | |
169 | (addr & ~SRC_WATERMARK_HIGH_MASK) | | |
170 | SRC_WATERMARK_HIGH_SET(n)); | |
171 | } | |
172 | ||
173 | static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, | |
174 | u32 ce_ctrl_addr, | |
175 | unsigned int n) | |
176 | { | |
177 | u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); | |
178 | ||
179 | ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, | |
180 | (addr & ~SRC_WATERMARK_LOW_MASK) | | |
181 | SRC_WATERMARK_LOW_SET(n)); | |
182 | } | |
183 | ||
184 | static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, | |
185 | u32 ce_ctrl_addr, | |
186 | unsigned int n) | |
187 | { | |
188 | u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); | |
189 | ||
190 | ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, | |
191 | (addr & ~DST_WATERMARK_HIGH_MASK) | | |
192 | DST_WATERMARK_HIGH_SET(n)); | |
193 | } | |
194 | ||
195 | static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, | |
196 | u32 ce_ctrl_addr, | |
197 | unsigned int n) | |
198 | { | |
199 | u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); | |
200 | ||
201 | ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, | |
202 | (addr & ~DST_WATERMARK_LOW_MASK) | | |
203 | DST_WATERMARK_LOW_SET(n)); | |
204 | } | |
205 | ||
206 | static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, | |
207 | u32 ce_ctrl_addr) | |
208 | { | |
209 | u32 host_ie_addr = ath10k_pci_read32(ar, | |
210 | ce_ctrl_addr + HOST_IE_ADDRESS); | |
211 | ||
212 | ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, | |
213 | host_ie_addr | HOST_IE_COPY_COMPLETE_MASK); | |
214 | } | |
215 | ||
216 | static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, | |
217 | u32 ce_ctrl_addr) | |
218 | { | |
219 | u32 host_ie_addr = ath10k_pci_read32(ar, | |
220 | ce_ctrl_addr + HOST_IE_ADDRESS); | |
221 | ||
222 | ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, | |
223 | host_ie_addr & ~HOST_IE_COPY_COMPLETE_MASK); | |
224 | } | |
225 | ||
226 | static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, | |
227 | u32 ce_ctrl_addr) | |
228 | { | |
229 | u32 host_ie_addr = ath10k_pci_read32(ar, | |
230 | ce_ctrl_addr + HOST_IE_ADDRESS); | |
231 | ||
232 | ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, | |
233 | host_ie_addr & ~CE_WATERMARK_MASK); | |
234 | } | |
235 | ||
236 | static inline void ath10k_ce_error_intr_enable(struct ath10k *ar, | |
237 | u32 ce_ctrl_addr) | |
238 | { | |
239 | u32 misc_ie_addr = ath10k_pci_read32(ar, | |
240 | ce_ctrl_addr + MISC_IE_ADDRESS); | |
241 | ||
242 | ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS, | |
243 | misc_ie_addr | CE_ERROR_MASK); | |
244 | } | |
245 | ||
246 | static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, | |
247 | u32 ce_ctrl_addr, | |
248 | unsigned int mask) | |
249 | { | |
250 | ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IS_ADDRESS, mask); | |
251 | } | |
252 | ||
253 | ||
254 | /* | |
255 | * Guts of ath10k_ce_send, used by both ath10k_ce_send and | |
256 | * ath10k_ce_sendlist_send. | |
257 | * The caller takes responsibility for any needed locking. | |
258 | */ | |
259 | static int ath10k_ce_send_nolock(struct ce_state *ce_state, | |
260 | void *per_transfer_context, | |
261 | u32 buffer, | |
262 | unsigned int nbytes, | |
263 | unsigned int transfer_id, | |
264 | unsigned int flags) | |
265 | { | |
266 | struct ath10k *ar = ce_state->ar; | |
267 | struct ce_ring_state *src_ring = ce_state->src_ring; | |
268 | struct ce_desc *desc, *sdesc; | |
269 | unsigned int nentries_mask = src_ring->nentries_mask; | |
270 | unsigned int sw_index = src_ring->sw_index; | |
271 | unsigned int write_index = src_ring->write_index; | |
272 | u32 ctrl_addr = ce_state->ctrl_addr; | |
273 | u32 desc_flags = 0; | |
274 | int ret = 0; | |
275 | ||
276 | if (nbytes > ce_state->src_sz_max) | |
277 | ath10k_warn("%s: send more we can (nbytes: %d, max: %d)\n", | |
278 | __func__, nbytes, ce_state->src_sz_max); | |
279 | ||
280 | ath10k_pci_wake(ar); | |
281 | ||
282 | if (unlikely(CE_RING_DELTA(nentries_mask, | |
283 | write_index, sw_index - 1) <= 0)) { | |
284 | ret = -EIO; | |
285 | goto exit; | |
286 | } | |
287 | ||
288 | desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, | |
289 | write_index); | |
290 | sdesc = CE_SRC_RING_TO_DESC(src_ring->shadow_base, write_index); | |
291 | ||
292 | desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); | |
293 | ||
294 | if (flags & CE_SEND_FLAG_GATHER) | |
295 | desc_flags |= CE_DESC_FLAGS_GATHER; | |
296 | if (flags & CE_SEND_FLAG_BYTE_SWAP) | |
297 | desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; | |
298 | ||
299 | sdesc->addr = __cpu_to_le32(buffer); | |
300 | sdesc->nbytes = __cpu_to_le16(nbytes); | |
301 | sdesc->flags = __cpu_to_le16(desc_flags); | |
302 | ||
303 | *desc = *sdesc; | |
304 | ||
305 | src_ring->per_transfer_context[write_index] = per_transfer_context; | |
306 | ||
307 | /* Update Source Ring Write Index */ | |
308 | write_index = CE_RING_IDX_INCR(nentries_mask, write_index); | |
309 | ||
310 | /* WORKAROUND */ | |
311 | if (!(flags & CE_SEND_FLAG_GATHER)) | |
312 | ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); | |
313 | ||
314 | src_ring->write_index = write_index; | |
315 | exit: | |
316 | ath10k_pci_sleep(ar); | |
317 | return ret; | |
318 | } | |
319 | ||
320 | int ath10k_ce_send(struct ce_state *ce_state, | |
321 | void *per_transfer_context, | |
322 | u32 buffer, | |
323 | unsigned int nbytes, | |
324 | unsigned int transfer_id, | |
325 | unsigned int flags) | |
326 | { | |
327 | struct ath10k *ar = ce_state->ar; | |
328 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
329 | int ret; | |
330 | ||
331 | spin_lock_bh(&ar_pci->ce_lock); | |
332 | ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, | |
333 | buffer, nbytes, transfer_id, flags); | |
334 | spin_unlock_bh(&ar_pci->ce_lock); | |
335 | ||
336 | return ret; | |
337 | } | |
338 | ||
339 | void ath10k_ce_sendlist_buf_add(struct ce_sendlist *sendlist, u32 buffer, | |
340 | unsigned int nbytes, u32 flags) | |
341 | { | |
342 | unsigned int num_items = sendlist->num_items; | |
343 | struct ce_sendlist_item *item; | |
344 | ||
345 | item = &sendlist->item[num_items]; | |
346 | item->data = buffer; | |
347 | item->u.nbytes = nbytes; | |
348 | item->flags = flags; | |
349 | sendlist->num_items++; | |
350 | } | |
351 | ||
352 | int ath10k_ce_sendlist_send(struct ce_state *ce_state, | |
353 | void *per_transfer_context, | |
354 | struct ce_sendlist *sendlist, | |
355 | unsigned int transfer_id) | |
356 | { | |
357 | struct ce_ring_state *src_ring = ce_state->src_ring; | |
358 | struct ce_sendlist_item *item; | |
359 | struct ath10k *ar = ce_state->ar; | |
360 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
361 | unsigned int nentries_mask = src_ring->nentries_mask; | |
362 | unsigned int num_items = sendlist->num_items; | |
363 | unsigned int sw_index; | |
364 | unsigned int write_index; | |
365 | int i, delta, ret = -ENOMEM; | |
366 | ||
367 | spin_lock_bh(&ar_pci->ce_lock); | |
368 | ||
369 | sw_index = src_ring->sw_index; | |
370 | write_index = src_ring->write_index; | |
371 | ||
372 | delta = CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); | |
373 | ||
374 | if (delta >= num_items) { | |
375 | /* | |
376 | * Handle all but the last item uniformly. | |
377 | */ | |
378 | for (i = 0; i < num_items - 1; i++) { | |
379 | item = &sendlist->item[i]; | |
380 | ret = ath10k_ce_send_nolock(ce_state, | |
381 | CE_SENDLIST_ITEM_CTXT, | |
382 | (u32) item->data, | |
383 | item->u.nbytes, transfer_id, | |
384 | item->flags | | |
385 | CE_SEND_FLAG_GATHER); | |
386 | if (ret) | |
387 | ath10k_warn("CE send failed for item: %d\n", i); | |
388 | } | |
389 | /* | |
390 | * Provide valid context pointer for final item. | |
391 | */ | |
392 | item = &sendlist->item[i]; | |
393 | ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, | |
394 | (u32) item->data, item->u.nbytes, | |
395 | transfer_id, item->flags); | |
396 | if (ret) | |
397 | ath10k_warn("CE send failed for last item: %d\n", i); | |
398 | } | |
399 | ||
400 | spin_unlock_bh(&ar_pci->ce_lock); | |
401 | ||
402 | return ret; | |
403 | } | |
404 | ||
405 | int ath10k_ce_recv_buf_enqueue(struct ce_state *ce_state, | |
406 | void *per_recv_context, | |
407 | u32 buffer) | |
408 | { | |
409 | struct ce_ring_state *dest_ring = ce_state->dest_ring; | |
410 | u32 ctrl_addr = ce_state->ctrl_addr; | |
411 | struct ath10k *ar = ce_state->ar; | |
412 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
413 | unsigned int nentries_mask = dest_ring->nentries_mask; | |
414 | unsigned int write_index; | |
415 | unsigned int sw_index; | |
416 | int ret; | |
417 | ||
418 | spin_lock_bh(&ar_pci->ce_lock); | |
419 | write_index = dest_ring->write_index; | |
420 | sw_index = dest_ring->sw_index; | |
421 | ||
422 | ath10k_pci_wake(ar); | |
423 | ||
424 | if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) > 0) { | |
425 | struct ce_desc *base = dest_ring->base_addr_owner_space; | |
426 | struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); | |
427 | ||
428 | /* Update destination descriptor */ | |
429 | desc->addr = __cpu_to_le32(buffer); | |
430 | desc->nbytes = 0; | |
431 | ||
432 | dest_ring->per_transfer_context[write_index] = | |
433 | per_recv_context; | |
434 | ||
435 | /* Update Destination Ring Write Index */ | |
436 | write_index = CE_RING_IDX_INCR(nentries_mask, write_index); | |
437 | ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); | |
438 | dest_ring->write_index = write_index; | |
439 | ret = 0; | |
440 | } else { | |
441 | ret = -EIO; | |
442 | } | |
443 | ath10k_pci_sleep(ar); | |
444 | spin_unlock_bh(&ar_pci->ce_lock); | |
445 | ||
446 | return ret; | |
447 | } | |
448 | ||
449 | /* | |
450 | * Guts of ath10k_ce_completed_recv_next. | |
451 | * The caller takes responsibility for any necessary locking. | |
452 | */ | |
453 | static int ath10k_ce_completed_recv_next_nolock(struct ce_state *ce_state, | |
454 | void **per_transfer_contextp, | |
455 | u32 *bufferp, | |
456 | unsigned int *nbytesp, | |
457 | unsigned int *transfer_idp, | |
458 | unsigned int *flagsp) | |
459 | { | |
460 | struct ce_ring_state *dest_ring = ce_state->dest_ring; | |
461 | unsigned int nentries_mask = dest_ring->nentries_mask; | |
462 | unsigned int sw_index = dest_ring->sw_index; | |
463 | ||
464 | struct ce_desc *base = dest_ring->base_addr_owner_space; | |
465 | struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); | |
466 | struct ce_desc sdesc; | |
467 | u16 nbytes; | |
468 | ||
469 | /* Copy in one go for performance reasons */ | |
470 | sdesc = *desc; | |
471 | ||
472 | nbytes = __le16_to_cpu(sdesc.nbytes); | |
473 | if (nbytes == 0) { | |
474 | /* | |
475 | * This closes a relatively unusual race where the Host | |
476 | * sees the updated DRRI before the update to the | |
477 | * corresponding descriptor has completed. We treat this | |
478 | * as a descriptor that is not yet done. | |
479 | */ | |
480 | return -EIO; | |
481 | } | |
482 | ||
483 | desc->nbytes = 0; | |
484 | ||
485 | /* Return data from completed destination descriptor */ | |
486 | *bufferp = __le32_to_cpu(sdesc.addr); | |
487 | *nbytesp = nbytes; | |
488 | *transfer_idp = MS(__le16_to_cpu(sdesc.flags), CE_DESC_FLAGS_META_DATA); | |
489 | ||
490 | if (__le16_to_cpu(sdesc.flags) & CE_DESC_FLAGS_BYTE_SWAP) | |
491 | *flagsp = CE_RECV_FLAG_SWAPPED; | |
492 | else | |
493 | *flagsp = 0; | |
494 | ||
495 | if (per_transfer_contextp) | |
496 | *per_transfer_contextp = | |
497 | dest_ring->per_transfer_context[sw_index]; | |
498 | ||
499 | /* sanity */ | |
500 | dest_ring->per_transfer_context[sw_index] = NULL; | |
501 | ||
502 | /* Update sw_index */ | |
503 | sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); | |
504 | dest_ring->sw_index = sw_index; | |
505 | ||
506 | return 0; | |
507 | } | |
508 | ||
509 | int ath10k_ce_completed_recv_next(struct ce_state *ce_state, | |
510 | void **per_transfer_contextp, | |
511 | u32 *bufferp, | |
512 | unsigned int *nbytesp, | |
513 | unsigned int *transfer_idp, | |
514 | unsigned int *flagsp) | |
515 | { | |
516 | struct ath10k *ar = ce_state->ar; | |
517 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
518 | int ret; | |
519 | ||
520 | spin_lock_bh(&ar_pci->ce_lock); | |
521 | ret = ath10k_ce_completed_recv_next_nolock(ce_state, | |
522 | per_transfer_contextp, | |
523 | bufferp, nbytesp, | |
524 | transfer_idp, flagsp); | |
525 | spin_unlock_bh(&ar_pci->ce_lock); | |
526 | ||
527 | return ret; | |
528 | } | |
529 | ||
530 | int ath10k_ce_revoke_recv_next(struct ce_state *ce_state, | |
531 | void **per_transfer_contextp, | |
532 | u32 *bufferp) | |
533 | { | |
534 | struct ce_ring_state *dest_ring; | |
535 | unsigned int nentries_mask; | |
536 | unsigned int sw_index; | |
537 | unsigned int write_index; | |
538 | int ret; | |
539 | struct ath10k *ar; | |
540 | struct ath10k_pci *ar_pci; | |
541 | ||
542 | dest_ring = ce_state->dest_ring; | |
543 | ||
544 | if (!dest_ring) | |
545 | return -EIO; | |
546 | ||
547 | ar = ce_state->ar; | |
548 | ar_pci = ath10k_pci_priv(ar); | |
549 | ||
550 | spin_lock_bh(&ar_pci->ce_lock); | |
551 | ||
552 | nentries_mask = dest_ring->nentries_mask; | |
553 | sw_index = dest_ring->sw_index; | |
554 | write_index = dest_ring->write_index; | |
555 | if (write_index != sw_index) { | |
556 | struct ce_desc *base = dest_ring->base_addr_owner_space; | |
557 | struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); | |
558 | ||
559 | /* Return data from completed destination descriptor */ | |
560 | *bufferp = __le32_to_cpu(desc->addr); | |
561 | ||
562 | if (per_transfer_contextp) | |
563 | *per_transfer_contextp = | |
564 | dest_ring->per_transfer_context[sw_index]; | |
565 | ||
566 | /* sanity */ | |
567 | dest_ring->per_transfer_context[sw_index] = NULL; | |
568 | ||
569 | /* Update sw_index */ | |
570 | sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); | |
571 | dest_ring->sw_index = sw_index; | |
572 | ret = 0; | |
573 | } else { | |
574 | ret = -EIO; | |
575 | } | |
576 | ||
577 | spin_unlock_bh(&ar_pci->ce_lock); | |
578 | ||
579 | return ret; | |
580 | } | |
581 | ||
582 | /* | |
583 | * Guts of ath10k_ce_completed_send_next. | |
584 | * The caller takes responsibility for any necessary locking. | |
585 | */ | |
586 | static int ath10k_ce_completed_send_next_nolock(struct ce_state *ce_state, | |
587 | void **per_transfer_contextp, | |
588 | u32 *bufferp, | |
589 | unsigned int *nbytesp, | |
590 | unsigned int *transfer_idp) | |
591 | { | |
592 | struct ce_ring_state *src_ring = ce_state->src_ring; | |
593 | u32 ctrl_addr = ce_state->ctrl_addr; | |
594 | struct ath10k *ar = ce_state->ar; | |
595 | unsigned int nentries_mask = src_ring->nentries_mask; | |
596 | unsigned int sw_index = src_ring->sw_index; | |
597 | unsigned int read_index; | |
598 | int ret = -EIO; | |
599 | ||
600 | if (src_ring->hw_index == sw_index) { | |
601 | /* | |
602 | * The SW completion index has caught up with the cached | |
603 | * version of the HW completion index. | |
604 | * Update the cached HW completion index to see whether | |
605 | * the SW has really caught up to the HW, or if the cached | |
606 | * value of the HW index has become stale. | |
607 | */ | |
608 | ath10k_pci_wake(ar); | |
609 | src_ring->hw_index = | |
610 | ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); | |
432358ed | 611 | src_ring->hw_index &= nentries_mask; |
5e3dd157 KV |
612 | ath10k_pci_sleep(ar); |
613 | } | |
614 | read_index = src_ring->hw_index; | |
615 | ||
616 | if ((read_index != sw_index) && (read_index != 0xffffffff)) { | |
617 | struct ce_desc *sbase = src_ring->shadow_base; | |
618 | struct ce_desc *sdesc = CE_SRC_RING_TO_DESC(sbase, sw_index); | |
619 | ||
620 | /* Return data from completed source descriptor */ | |
621 | *bufferp = __le32_to_cpu(sdesc->addr); | |
622 | *nbytesp = __le16_to_cpu(sdesc->nbytes); | |
623 | *transfer_idp = MS(__le16_to_cpu(sdesc->flags), | |
624 | CE_DESC_FLAGS_META_DATA); | |
625 | ||
626 | if (per_transfer_contextp) | |
627 | *per_transfer_contextp = | |
628 | src_ring->per_transfer_context[sw_index]; | |
629 | ||
630 | /* sanity */ | |
631 | src_ring->per_transfer_context[sw_index] = NULL; | |
632 | ||
633 | /* Update sw_index */ | |
634 | sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); | |
635 | src_ring->sw_index = sw_index; | |
636 | ret = 0; | |
637 | } | |
638 | ||
639 | return ret; | |
640 | } | |
641 | ||
642 | /* NB: Modeled after ath10k_ce_completed_send_next */ | |
643 | int ath10k_ce_cancel_send_next(struct ce_state *ce_state, | |
644 | void **per_transfer_contextp, | |
645 | u32 *bufferp, | |
646 | unsigned int *nbytesp, | |
647 | unsigned int *transfer_idp) | |
648 | { | |
649 | struct ce_ring_state *src_ring; | |
650 | unsigned int nentries_mask; | |
651 | unsigned int sw_index; | |
652 | unsigned int write_index; | |
653 | int ret; | |
654 | struct ath10k *ar; | |
655 | struct ath10k_pci *ar_pci; | |
656 | ||
657 | src_ring = ce_state->src_ring; | |
658 | ||
659 | if (!src_ring) | |
660 | return -EIO; | |
661 | ||
662 | ar = ce_state->ar; | |
663 | ar_pci = ath10k_pci_priv(ar); | |
664 | ||
665 | spin_lock_bh(&ar_pci->ce_lock); | |
666 | ||
667 | nentries_mask = src_ring->nentries_mask; | |
668 | sw_index = src_ring->sw_index; | |
669 | write_index = src_ring->write_index; | |
670 | ||
671 | if (write_index != sw_index) { | |
672 | struct ce_desc *base = src_ring->base_addr_owner_space; | |
673 | struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); | |
674 | ||
675 | /* Return data from completed source descriptor */ | |
676 | *bufferp = __le32_to_cpu(desc->addr); | |
677 | *nbytesp = __le16_to_cpu(desc->nbytes); | |
678 | *transfer_idp = MS(__le16_to_cpu(desc->flags), | |
679 | CE_DESC_FLAGS_META_DATA); | |
680 | ||
681 | if (per_transfer_contextp) | |
682 | *per_transfer_contextp = | |
683 | src_ring->per_transfer_context[sw_index]; | |
684 | ||
685 | /* sanity */ | |
686 | src_ring->per_transfer_context[sw_index] = NULL; | |
687 | ||
688 | /* Update sw_index */ | |
689 | sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); | |
690 | src_ring->sw_index = sw_index; | |
691 | ret = 0; | |
692 | } else { | |
693 | ret = -EIO; | |
694 | } | |
695 | ||
696 | spin_unlock_bh(&ar_pci->ce_lock); | |
697 | ||
698 | return ret; | |
699 | } | |
700 | ||
701 | int ath10k_ce_completed_send_next(struct ce_state *ce_state, | |
702 | void **per_transfer_contextp, | |
703 | u32 *bufferp, | |
704 | unsigned int *nbytesp, | |
705 | unsigned int *transfer_idp) | |
706 | { | |
707 | struct ath10k *ar = ce_state->ar; | |
708 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
709 | int ret; | |
710 | ||
711 | spin_lock_bh(&ar_pci->ce_lock); | |
712 | ret = ath10k_ce_completed_send_next_nolock(ce_state, | |
713 | per_transfer_contextp, | |
714 | bufferp, nbytesp, | |
715 | transfer_idp); | |
716 | spin_unlock_bh(&ar_pci->ce_lock); | |
717 | ||
718 | return ret; | |
719 | } | |
720 | ||
721 | /* | |
722 | * Guts of interrupt handler for per-engine interrupts on a particular CE. | |
723 | * | |
724 | * Invokes registered callbacks for recv_complete, | |
725 | * send_complete, and watermarks. | |
726 | */ | |
727 | void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) | |
728 | { | |
729 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
730 | struct ce_state *ce_state = ar_pci->ce_id_to_state[ce_id]; | |
731 | u32 ctrl_addr = ce_state->ctrl_addr; | |
732 | void *transfer_context; | |
733 | u32 buf; | |
734 | unsigned int nbytes; | |
735 | unsigned int id; | |
736 | unsigned int flags; | |
737 | ||
738 | ath10k_pci_wake(ar); | |
739 | spin_lock_bh(&ar_pci->ce_lock); | |
740 | ||
741 | /* Clear the copy-complete interrupts that will be handled here. */ | |
742 | ath10k_ce_engine_int_status_clear(ar, ctrl_addr, | |
743 | HOST_IS_COPY_COMPLETE_MASK); | |
744 | ||
745 | if (ce_state->recv_cb) { | |
746 | /* | |
747 | * Pop completed recv buffers and call the registered | |
748 | * recv callback for each | |
749 | */ | |
750 | while (ath10k_ce_completed_recv_next_nolock(ce_state, | |
751 | &transfer_context, | |
752 | &buf, &nbytes, | |
753 | &id, &flags) == 0) { | |
754 | spin_unlock_bh(&ar_pci->ce_lock); | |
755 | ce_state->recv_cb(ce_state, transfer_context, buf, | |
756 | nbytes, id, flags); | |
757 | spin_lock_bh(&ar_pci->ce_lock); | |
758 | } | |
759 | } | |
760 | ||
761 | if (ce_state->send_cb) { | |
762 | /* | |
763 | * Pop completed send buffers and call the registered | |
764 | * send callback for each | |
765 | */ | |
766 | while (ath10k_ce_completed_send_next_nolock(ce_state, | |
767 | &transfer_context, | |
768 | &buf, | |
769 | &nbytes, | |
770 | &id) == 0) { | |
771 | spin_unlock_bh(&ar_pci->ce_lock); | |
772 | ce_state->send_cb(ce_state, transfer_context, | |
773 | buf, nbytes, id); | |
774 | spin_lock_bh(&ar_pci->ce_lock); | |
775 | } | |
776 | } | |
777 | ||
778 | /* | |
779 | * Misc CE interrupts are not being handled, but still need | |
780 | * to be cleared. | |
781 | */ | |
782 | ath10k_ce_engine_int_status_clear(ar, ctrl_addr, CE_WATERMARK_MASK); | |
783 | ||
784 | spin_unlock_bh(&ar_pci->ce_lock); | |
785 | ath10k_pci_sleep(ar); | |
786 | } | |
787 | ||
788 | /* | |
789 | * Handler for per-engine interrupts on ALL active CEs. | |
790 | * This is used in cases where the system is sharing a | |
791 | * single interrput for all CEs | |
792 | */ | |
793 | ||
794 | void ath10k_ce_per_engine_service_any(struct ath10k *ar) | |
795 | { | |
796 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
797 | int ce_id; | |
798 | u32 intr_summary; | |
799 | ||
800 | ath10k_pci_wake(ar); | |
801 | intr_summary = CE_INTERRUPT_SUMMARY(ar); | |
802 | ||
803 | for (ce_id = 0; intr_summary && (ce_id < ar_pci->ce_count); ce_id++) { | |
804 | if (intr_summary & (1 << ce_id)) | |
805 | intr_summary &= ~(1 << ce_id); | |
806 | else | |
807 | /* no intr pending on this CE */ | |
808 | continue; | |
809 | ||
810 | ath10k_ce_per_engine_service(ar, ce_id); | |
811 | } | |
812 | ||
813 | ath10k_pci_sleep(ar); | |
814 | } | |
815 | ||
816 | /* | |
817 | * Adjust interrupts for the copy complete handler. | |
818 | * If it's needed for either send or recv, then unmask | |
819 | * this interrupt; otherwise, mask it. | |
820 | * | |
821 | * Called with ce_lock held. | |
822 | */ | |
823 | static void ath10k_ce_per_engine_handler_adjust(struct ce_state *ce_state, | |
824 | int disable_copy_compl_intr) | |
825 | { | |
826 | u32 ctrl_addr = ce_state->ctrl_addr; | |
827 | struct ath10k *ar = ce_state->ar; | |
828 | ||
829 | ath10k_pci_wake(ar); | |
830 | ||
831 | if ((!disable_copy_compl_intr) && | |
832 | (ce_state->send_cb || ce_state->recv_cb)) | |
833 | ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); | |
834 | else | |
835 | ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); | |
836 | ||
837 | ath10k_ce_watermark_intr_disable(ar, ctrl_addr); | |
838 | ||
839 | ath10k_pci_sleep(ar); | |
840 | } | |
841 | ||
842 | void ath10k_ce_disable_interrupts(struct ath10k *ar) | |
843 | { | |
844 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
845 | int ce_id; | |
846 | ||
847 | ath10k_pci_wake(ar); | |
848 | for (ce_id = 0; ce_id < ar_pci->ce_count; ce_id++) { | |
849 | struct ce_state *ce_state = ar_pci->ce_id_to_state[ce_id]; | |
850 | u32 ctrl_addr = ce_state->ctrl_addr; | |
851 | ||
852 | ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); | |
853 | } | |
854 | ath10k_pci_sleep(ar); | |
855 | } | |
856 | ||
857 | void ath10k_ce_send_cb_register(struct ce_state *ce_state, | |
858 | void (*send_cb) (struct ce_state *ce_state, | |
859 | void *transfer_context, | |
860 | u32 buffer, | |
861 | unsigned int nbytes, | |
862 | unsigned int transfer_id), | |
863 | int disable_interrupts) | |
864 | { | |
865 | struct ath10k *ar = ce_state->ar; | |
866 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
867 | ||
868 | spin_lock_bh(&ar_pci->ce_lock); | |
869 | ce_state->send_cb = send_cb; | |
870 | ath10k_ce_per_engine_handler_adjust(ce_state, disable_interrupts); | |
871 | spin_unlock_bh(&ar_pci->ce_lock); | |
872 | } | |
873 | ||
874 | void ath10k_ce_recv_cb_register(struct ce_state *ce_state, | |
875 | void (*recv_cb) (struct ce_state *ce_state, | |
876 | void *transfer_context, | |
877 | u32 buffer, | |
878 | unsigned int nbytes, | |
879 | unsigned int transfer_id, | |
880 | unsigned int flags)) | |
881 | { | |
882 | struct ath10k *ar = ce_state->ar; | |
883 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
884 | ||
885 | spin_lock_bh(&ar_pci->ce_lock); | |
886 | ce_state->recv_cb = recv_cb; | |
887 | ath10k_ce_per_engine_handler_adjust(ce_state, 0); | |
888 | spin_unlock_bh(&ar_pci->ce_lock); | |
889 | } | |
890 | ||
891 | static int ath10k_ce_init_src_ring(struct ath10k *ar, | |
892 | unsigned int ce_id, | |
893 | struct ce_state *ce_state, | |
894 | const struct ce_attr *attr) | |
895 | { | |
896 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
897 | struct ce_ring_state *src_ring; | |
898 | unsigned int nentries = attr->src_nentries; | |
899 | unsigned int ce_nbytes; | |
900 | u32 ctrl_addr = ath10k_ce_base_address(ce_id); | |
901 | dma_addr_t base_addr; | |
902 | char *ptr; | |
903 | ||
904 | nentries = roundup_pow_of_two(nentries); | |
905 | ||
906 | if (ce_state->src_ring) { | |
907 | WARN_ON(ce_state->src_ring->nentries != nentries); | |
908 | return 0; | |
909 | } | |
910 | ||
911 | ce_nbytes = sizeof(struct ce_ring_state) + (nentries * sizeof(void *)); | |
912 | ptr = kzalloc(ce_nbytes, GFP_KERNEL); | |
913 | if (ptr == NULL) | |
914 | return -ENOMEM; | |
915 | ||
916 | ce_state->src_ring = (struct ce_ring_state *)ptr; | |
917 | src_ring = ce_state->src_ring; | |
918 | ||
919 | ptr += sizeof(struct ce_ring_state); | |
920 | src_ring->nentries = nentries; | |
921 | src_ring->nentries_mask = nentries - 1; | |
922 | ||
923 | ath10k_pci_wake(ar); | |
924 | src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); | |
432358ed | 925 | src_ring->sw_index &= src_ring->nentries_mask; |
5e3dd157 KV |
926 | src_ring->hw_index = src_ring->sw_index; |
927 | ||
928 | src_ring->write_index = | |
929 | ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); | |
432358ed | 930 | src_ring->write_index &= src_ring->nentries_mask; |
5e3dd157 KV |
931 | ath10k_pci_sleep(ar); |
932 | ||
933 | src_ring->per_transfer_context = (void **)ptr; | |
934 | ||
935 | /* | |
936 | * Legacy platforms that do not support cache | |
937 | * coherent DMA are unsupported | |
938 | */ | |
939 | src_ring->base_addr_owner_space_unaligned = | |
940 | pci_alloc_consistent(ar_pci->pdev, | |
941 | (nentries * sizeof(struct ce_desc) + | |
942 | CE_DESC_RING_ALIGN), | |
943 | &base_addr); | |
9c5ae691 JD |
944 | if (!src_ring->base_addr_owner_space_unaligned) { |
945 | kfree(ce_state->src_ring); | |
946 | ce_state->src_ring = NULL; | |
947 | return -ENOMEM; | |
948 | } | |
949 | ||
5e3dd157 KV |
950 | src_ring->base_addr_ce_space_unaligned = base_addr; |
951 | ||
952 | src_ring->base_addr_owner_space = PTR_ALIGN( | |
953 | src_ring->base_addr_owner_space_unaligned, | |
954 | CE_DESC_RING_ALIGN); | |
955 | src_ring->base_addr_ce_space = ALIGN( | |
956 | src_ring->base_addr_ce_space_unaligned, | |
957 | CE_DESC_RING_ALIGN); | |
958 | ||
959 | /* | |
960 | * Also allocate a shadow src ring in regular | |
961 | * mem to use for faster access. | |
962 | */ | |
963 | src_ring->shadow_base_unaligned = | |
964 | kmalloc((nentries * sizeof(struct ce_desc) + | |
965 | CE_DESC_RING_ALIGN), GFP_KERNEL); | |
9c5ae691 JD |
966 | if (!src_ring->shadow_base_unaligned) { |
967 | pci_free_consistent(ar_pci->pdev, | |
968 | (nentries * sizeof(struct ce_desc) + | |
969 | CE_DESC_RING_ALIGN), | |
970 | src_ring->base_addr_owner_space, | |
971 | src_ring->base_addr_ce_space); | |
972 | kfree(ce_state->src_ring); | |
973 | ce_state->src_ring = NULL; | |
974 | return -ENOMEM; | |
975 | } | |
5e3dd157 KV |
976 | |
977 | src_ring->shadow_base = PTR_ALIGN( | |
978 | src_ring->shadow_base_unaligned, | |
979 | CE_DESC_RING_ALIGN); | |
980 | ||
981 | ath10k_pci_wake(ar); | |
982 | ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, | |
983 | src_ring->base_addr_ce_space); | |
984 | ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); | |
985 | ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); | |
986 | ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); | |
987 | ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); | |
988 | ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); | |
989 | ath10k_pci_sleep(ar); | |
990 | ||
991 | return 0; | |
992 | } | |
993 | ||
994 | static int ath10k_ce_init_dest_ring(struct ath10k *ar, | |
995 | unsigned int ce_id, | |
996 | struct ce_state *ce_state, | |
997 | const struct ce_attr *attr) | |
998 | { | |
999 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
1000 | struct ce_ring_state *dest_ring; | |
1001 | unsigned int nentries = attr->dest_nentries; | |
1002 | unsigned int ce_nbytes; | |
1003 | u32 ctrl_addr = ath10k_ce_base_address(ce_id); | |
1004 | dma_addr_t base_addr; | |
1005 | char *ptr; | |
1006 | ||
1007 | nentries = roundup_pow_of_two(nentries); | |
1008 | ||
1009 | if (ce_state->dest_ring) { | |
1010 | WARN_ON(ce_state->dest_ring->nentries != nentries); | |
1011 | return 0; | |
1012 | } | |
1013 | ||
1014 | ce_nbytes = sizeof(struct ce_ring_state) + (nentries * sizeof(void *)); | |
1015 | ptr = kzalloc(ce_nbytes, GFP_KERNEL); | |
1016 | if (ptr == NULL) | |
1017 | return -ENOMEM; | |
1018 | ||
1019 | ce_state->dest_ring = (struct ce_ring_state *)ptr; | |
1020 | dest_ring = ce_state->dest_ring; | |
1021 | ||
1022 | ptr += sizeof(struct ce_ring_state); | |
1023 | dest_ring->nentries = nentries; | |
1024 | dest_ring->nentries_mask = nentries - 1; | |
1025 | ||
1026 | ath10k_pci_wake(ar); | |
1027 | dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); | |
432358ed | 1028 | dest_ring->sw_index &= dest_ring->nentries_mask; |
5e3dd157 KV |
1029 | dest_ring->write_index = |
1030 | ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); | |
432358ed | 1031 | dest_ring->write_index &= dest_ring->nentries_mask; |
5e3dd157 KV |
1032 | ath10k_pci_sleep(ar); |
1033 | ||
1034 | dest_ring->per_transfer_context = (void **)ptr; | |
1035 | ||
1036 | /* | |
1037 | * Legacy platforms that do not support cache | |
1038 | * coherent DMA are unsupported | |
1039 | */ | |
1040 | dest_ring->base_addr_owner_space_unaligned = | |
1041 | pci_alloc_consistent(ar_pci->pdev, | |
1042 | (nentries * sizeof(struct ce_desc) + | |
1043 | CE_DESC_RING_ALIGN), | |
1044 | &base_addr); | |
9c5ae691 JD |
1045 | if (!dest_ring->base_addr_owner_space_unaligned) { |
1046 | kfree(ce_state->dest_ring); | |
1047 | ce_state->dest_ring = NULL; | |
1048 | return -ENOMEM; | |
1049 | } | |
1050 | ||
5e3dd157 KV |
1051 | dest_ring->base_addr_ce_space_unaligned = base_addr; |
1052 | ||
1053 | /* | |
1054 | * Correctly initialize memory to 0 to prevent garbage | |
1055 | * data crashing system when download firmware | |
1056 | */ | |
1057 | memset(dest_ring->base_addr_owner_space_unaligned, 0, | |
1058 | nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN); | |
1059 | ||
1060 | dest_ring->base_addr_owner_space = PTR_ALIGN( | |
1061 | dest_ring->base_addr_owner_space_unaligned, | |
1062 | CE_DESC_RING_ALIGN); | |
1063 | dest_ring->base_addr_ce_space = ALIGN( | |
1064 | dest_ring->base_addr_ce_space_unaligned, | |
1065 | CE_DESC_RING_ALIGN); | |
1066 | ||
1067 | ath10k_pci_wake(ar); | |
1068 | ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, | |
1069 | dest_ring->base_addr_ce_space); | |
1070 | ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); | |
1071 | ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); | |
1072 | ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); | |
1073 | ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); | |
1074 | ath10k_pci_sleep(ar); | |
1075 | ||
1076 | return 0; | |
1077 | } | |
1078 | ||
1079 | static struct ce_state *ath10k_ce_init_state(struct ath10k *ar, | |
1080 | unsigned int ce_id, | |
1081 | const struct ce_attr *attr) | |
1082 | { | |
1083 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
1084 | struct ce_state *ce_state = NULL; | |
1085 | u32 ctrl_addr = ath10k_ce_base_address(ce_id); | |
1086 | ||
1087 | spin_lock_bh(&ar_pci->ce_lock); | |
1088 | ||
1089 | if (!ar_pci->ce_id_to_state[ce_id]) { | |
1090 | ce_state = kzalloc(sizeof(*ce_state), GFP_ATOMIC); | |
1091 | if (ce_state == NULL) { | |
1092 | spin_unlock_bh(&ar_pci->ce_lock); | |
1093 | return NULL; | |
1094 | } | |
1095 | ||
1096 | ar_pci->ce_id_to_state[ce_id] = ce_state; | |
1097 | ce_state->ar = ar; | |
1098 | ce_state->id = ce_id; | |
1099 | ce_state->ctrl_addr = ctrl_addr; | |
1100 | ce_state->state = CE_RUNNING; | |
1101 | /* Save attribute flags */ | |
1102 | ce_state->attr_flags = attr->flags; | |
1103 | ce_state->src_sz_max = attr->src_sz_max; | |
1104 | } | |
1105 | ||
1106 | spin_unlock_bh(&ar_pci->ce_lock); | |
1107 | ||
1108 | return ce_state; | |
1109 | } | |
1110 | ||
1111 | /* | |
1112 | * Initialize a Copy Engine based on caller-supplied attributes. | |
1113 | * This may be called once to initialize both source and destination | |
1114 | * rings or it may be called twice for separate source and destination | |
1115 | * initialization. It may be that only one side or the other is | |
1116 | * initialized by software/firmware. | |
1117 | */ | |
1118 | struct ce_state *ath10k_ce_init(struct ath10k *ar, | |
1119 | unsigned int ce_id, | |
1120 | const struct ce_attr *attr) | |
1121 | { | |
1122 | struct ce_state *ce_state; | |
1123 | u32 ctrl_addr = ath10k_ce_base_address(ce_id); | |
1124 | ||
1125 | ce_state = ath10k_ce_init_state(ar, ce_id, attr); | |
1126 | if (!ce_state) { | |
1127 | ath10k_err("Failed to initialize CE state for ID: %d\n", ce_id); | |
1128 | return NULL; | |
1129 | } | |
1130 | ||
1131 | if (attr->src_nentries) { | |
1132 | if (ath10k_ce_init_src_ring(ar, ce_id, ce_state, attr)) { | |
1133 | ath10k_err("Failed to initialize CE src ring for ID: %d\n", | |
1134 | ce_id); | |
1135 | ath10k_ce_deinit(ce_state); | |
1136 | return NULL; | |
1137 | } | |
1138 | } | |
1139 | ||
1140 | if (attr->dest_nentries) { | |
1141 | if (ath10k_ce_init_dest_ring(ar, ce_id, ce_state, attr)) { | |
1142 | ath10k_err("Failed to initialize CE dest ring for ID: %d\n", | |
1143 | ce_id); | |
1144 | ath10k_ce_deinit(ce_state); | |
1145 | return NULL; | |
1146 | } | |
1147 | } | |
1148 | ||
1149 | /* Enable CE error interrupts */ | |
1150 | ath10k_pci_wake(ar); | |
1151 | ath10k_ce_error_intr_enable(ar, ctrl_addr); | |
1152 | ath10k_pci_sleep(ar); | |
1153 | ||
1154 | return ce_state; | |
1155 | } | |
1156 | ||
1157 | void ath10k_ce_deinit(struct ce_state *ce_state) | |
1158 | { | |
1159 | unsigned int ce_id = ce_state->id; | |
1160 | struct ath10k *ar = ce_state->ar; | |
1161 | struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); | |
1162 | ||
1163 | ce_state->state = CE_UNUSED; | |
1164 | ar_pci->ce_id_to_state[ce_id] = NULL; | |
1165 | ||
1166 | if (ce_state->src_ring) { | |
1167 | kfree(ce_state->src_ring->shadow_base_unaligned); | |
1168 | pci_free_consistent(ar_pci->pdev, | |
1169 | (ce_state->src_ring->nentries * | |
1170 | sizeof(struct ce_desc) + | |
1171 | CE_DESC_RING_ALIGN), | |
1172 | ce_state->src_ring->base_addr_owner_space, | |
1173 | ce_state->src_ring->base_addr_ce_space); | |
1174 | kfree(ce_state->src_ring); | |
1175 | } | |
1176 | ||
1177 | if (ce_state->dest_ring) { | |
1178 | pci_free_consistent(ar_pci->pdev, | |
1179 | (ce_state->dest_ring->nentries * | |
1180 | sizeof(struct ce_desc) + | |
1181 | CE_DESC_RING_ALIGN), | |
1182 | ce_state->dest_ring->base_addr_owner_space, | |
1183 | ce_state->dest_ring->base_addr_ce_space); | |
1184 | kfree(ce_state->dest_ring); | |
1185 | } | |
1186 | kfree(ce_state); | |
1187 | } |