Commit | Line | Data |
---|---|---|
d1615ca2 SK |
1 | /* |
2 | * Qualcomm Technologies HIDMA DMA engine low level code | |
3 | * | |
4 | * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 and | |
8 | * only version 2 as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | */ | |
15 | ||
16 | #include <linux/dmaengine.h> | |
17 | #include <linux/slab.h> | |
18 | #include <linux/interrupt.h> | |
19 | #include <linux/mm.h> | |
20 | #include <linux/highmem.h> | |
21 | #include <linux/dma-mapping.h> | |
22 | #include <linux/delay.h> | |
23 | #include <linux/atomic.h> | |
24 | #include <linux/iopoll.h> | |
25 | #include <linux/kfifo.h> | |
26 | #include <linux/bitops.h> | |
27 | ||
28 | #include "hidma.h" | |
29 | ||
30 | #define HIDMA_EVRE_SIZE 16 /* each EVRE is 16 bytes */ | |
31 | ||
32 | #define HIDMA_TRCA_CTRLSTS_REG 0x000 | |
33 | #define HIDMA_TRCA_RING_LOW_REG 0x008 | |
34 | #define HIDMA_TRCA_RING_HIGH_REG 0x00C | |
35 | #define HIDMA_TRCA_RING_LEN_REG 0x010 | |
36 | #define HIDMA_TRCA_DOORBELL_REG 0x400 | |
37 | ||
38 | #define HIDMA_EVCA_CTRLSTS_REG 0x000 | |
39 | #define HIDMA_EVCA_INTCTRL_REG 0x004 | |
40 | #define HIDMA_EVCA_RING_LOW_REG 0x008 | |
41 | #define HIDMA_EVCA_RING_HIGH_REG 0x00C | |
42 | #define HIDMA_EVCA_RING_LEN_REG 0x010 | |
43 | #define HIDMA_EVCA_WRITE_PTR_REG 0x020 | |
44 | #define HIDMA_EVCA_DOORBELL_REG 0x400 | |
45 | ||
46 | #define HIDMA_EVCA_IRQ_STAT_REG 0x100 | |
47 | #define HIDMA_EVCA_IRQ_CLR_REG 0x108 | |
48 | #define HIDMA_EVCA_IRQ_EN_REG 0x110 | |
49 | ||
50 | #define HIDMA_EVRE_CFG_IDX 0 | |
51 | ||
52 | #define HIDMA_EVRE_ERRINFO_BIT_POS 24 | |
53 | #define HIDMA_EVRE_CODE_BIT_POS 28 | |
54 | ||
55 | #define HIDMA_EVRE_ERRINFO_MASK GENMASK(3, 0) | |
56 | #define HIDMA_EVRE_CODE_MASK GENMASK(3, 0) | |
57 | ||
58 | #define HIDMA_CH_CONTROL_MASK GENMASK(7, 0) | |
59 | #define HIDMA_CH_STATE_MASK GENMASK(7, 0) | |
60 | #define HIDMA_CH_STATE_BIT_POS 0x8 | |
61 | ||
62 | #define HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS 0 | |
63 | #define HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS 1 | |
64 | #define HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS 9 | |
65 | #define HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS 10 | |
66 | #define HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS 11 | |
67 | #define HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS 14 | |
68 | ||
69 | #define ENABLE_IRQS (BIT(HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS) | \ | |
70 | BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \ | |
71 | BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \ | |
72 | BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \ | |
73 | BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS) | \ | |
74 | BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS)) | |
75 | ||
76 | #define HIDMA_INCREMENT_ITERATOR(iter, size, ring_size) \ | |
77 | do { \ | |
78 | iter += size; \ | |
79 | if (iter >= ring_size) \ | |
80 | iter -= ring_size; \ | |
81 | } while (0) | |
82 | ||
83 | #define HIDMA_CH_STATE(val) \ | |
84 | ((val >> HIDMA_CH_STATE_BIT_POS) & HIDMA_CH_STATE_MASK) | |
85 | ||
86 | #define HIDMA_ERR_INT_MASK \ | |
87 | (BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS) | \ | |
88 | BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \ | |
89 | BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \ | |
90 | BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \ | |
91 | BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS)) | |
92 | ||
93 | enum ch_command { | |
94 | HIDMA_CH_DISABLE = 0, | |
95 | HIDMA_CH_ENABLE = 1, | |
96 | HIDMA_CH_SUSPEND = 2, | |
97 | HIDMA_CH_RESET = 9, | |
98 | }; | |
99 | ||
100 | enum ch_state { | |
101 | HIDMA_CH_DISABLED = 0, | |
102 | HIDMA_CH_ENABLED = 1, | |
103 | HIDMA_CH_RUNNING = 2, | |
104 | HIDMA_CH_SUSPENDED = 3, | |
105 | HIDMA_CH_STOPPED = 4, | |
106 | }; | |
107 | ||
108 | enum tre_type { | |
109 | HIDMA_TRE_MEMCPY = 3, | |
110 | }; | |
111 | ||
112 | enum err_code { | |
113 | HIDMA_EVRE_STATUS_COMPLETE = 1, | |
114 | HIDMA_EVRE_STATUS_ERROR = 4, | |
115 | }; | |
116 | ||
117 | static int hidma_is_chan_enabled(int state) | |
118 | { | |
119 | switch (state) { | |
120 | case HIDMA_CH_ENABLED: | |
121 | case HIDMA_CH_RUNNING: | |
122 | return true; | |
123 | default: | |
124 | return false; | |
125 | } | |
126 | } | |
127 | ||
128 | void hidma_ll_free(struct hidma_lldev *lldev, u32 tre_ch) | |
129 | { | |
130 | struct hidma_tre *tre; | |
131 | ||
132 | if (tre_ch >= lldev->nr_tres) { | |
133 | dev_err(lldev->dev, "invalid TRE number in free:%d", tre_ch); | |
134 | return; | |
135 | } | |
136 | ||
137 | tre = &lldev->trepool[tre_ch]; | |
138 | if (atomic_read(&tre->allocated) != true) { | |
139 | dev_err(lldev->dev, "trying to free an unused TRE:%d", tre_ch); | |
140 | return; | |
141 | } | |
142 | ||
143 | atomic_set(&tre->allocated, 0); | |
144 | } | |
145 | ||
146 | int hidma_ll_request(struct hidma_lldev *lldev, u32 sig, const char *dev_name, | |
147 | void (*callback)(void *data), void *data, u32 *tre_ch) | |
148 | { | |
149 | unsigned int i; | |
150 | struct hidma_tre *tre; | |
151 | u32 *tre_local; | |
152 | ||
153 | if (!tre_ch || !lldev) | |
154 | return -EINVAL; | |
155 | ||
156 | /* need to have at least one empty spot in the queue */ | |
157 | for (i = 0; i < lldev->nr_tres - 1; i++) { | |
158 | if (atomic_add_unless(&lldev->trepool[i].allocated, 1, 1)) | |
159 | break; | |
160 | } | |
161 | ||
162 | if (i == (lldev->nr_tres - 1)) | |
163 | return -ENOMEM; | |
164 | ||
165 | tre = &lldev->trepool[i]; | |
166 | tre->dma_sig = sig; | |
167 | tre->dev_name = dev_name; | |
168 | tre->callback = callback; | |
169 | tre->data = data; | |
170 | tre->idx = i; | |
171 | tre->status = 0; | |
172 | tre->queued = 0; | |
173 | tre->err_code = 0; | |
174 | tre->err_info = 0; | |
175 | tre->lldev = lldev; | |
176 | tre_local = &tre->tre_local[0]; | |
177 | tre_local[HIDMA_TRE_CFG_IDX] = HIDMA_TRE_MEMCPY; | |
178 | tre_local[HIDMA_TRE_CFG_IDX] |= (lldev->chidx & 0xFF) << 8; | |
179 | tre_local[HIDMA_TRE_CFG_IDX] |= BIT(16); /* set IEOB */ | |
180 | *tre_ch = i; | |
181 | if (callback) | |
182 | callback(data); | |
183 | return 0; | |
184 | } | |
185 | ||
186 | /* | |
187 | * Multiple TREs may be queued and waiting in the pending queue. | |
188 | */ | |
189 | static void hidma_ll_tre_complete(unsigned long arg) | |
190 | { | |
191 | struct hidma_lldev *lldev = (struct hidma_lldev *)arg; | |
192 | struct hidma_tre *tre; | |
193 | ||
194 | while (kfifo_out(&lldev->handoff_fifo, &tre, 1)) { | |
195 | /* call the user if it has been read by the hardware */ | |
196 | if (tre->callback) | |
197 | tre->callback(tre->data); | |
198 | } | |
199 | } | |
200 | ||
201 | static int hidma_post_completed(struct hidma_lldev *lldev, int tre_iterator, | |
202 | u8 err_info, u8 err_code) | |
203 | { | |
204 | struct hidma_tre *tre; | |
205 | unsigned long flags; | |
206 | ||
207 | spin_lock_irqsave(&lldev->lock, flags); | |
208 | tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE]; | |
209 | if (!tre) { | |
210 | spin_unlock_irqrestore(&lldev->lock, flags); | |
211 | dev_warn(lldev->dev, "tre_index [%d] and tre out of sync\n", | |
212 | tre_iterator / HIDMA_TRE_SIZE); | |
213 | return -EINVAL; | |
214 | } | |
215 | lldev->pending_tre_list[tre->tre_index] = NULL; | |
216 | ||
217 | /* | |
218 | * Keep track of pending TREs that SW is expecting to receive | |
219 | * from HW. We got one now. Decrement our counter. | |
220 | */ | |
221 | lldev->pending_tre_count--; | |
222 | if (lldev->pending_tre_count < 0) { | |
223 | dev_warn(lldev->dev, "tre count mismatch on completion"); | |
224 | lldev->pending_tre_count = 0; | |
225 | } | |
226 | ||
227 | spin_unlock_irqrestore(&lldev->lock, flags); | |
228 | ||
229 | tre->err_info = err_info; | |
230 | tre->err_code = err_code; | |
231 | tre->queued = 0; | |
232 | ||
233 | kfifo_put(&lldev->handoff_fifo, tre); | |
234 | tasklet_schedule(&lldev->task); | |
235 | ||
236 | return 0; | |
237 | } | |
238 | ||
239 | /* | |
240 | * Called to handle the interrupt for the channel. | |
241 | * Return a positive number if TRE or EVRE were consumed on this run. | |
242 | * Return a positive number if there are pending TREs or EVREs. | |
243 | * Return 0 if there is nothing to consume or no pending TREs/EVREs found. | |
244 | */ | |
245 | static int hidma_handle_tre_completion(struct hidma_lldev *lldev) | |
246 | { | |
247 | u32 evre_ring_size = lldev->evre_ring_size; | |
248 | u32 tre_ring_size = lldev->tre_ring_size; | |
249 | u32 err_info, err_code, evre_write_off; | |
250 | u32 tre_iterator, evre_iterator; | |
251 | u32 num_completed = 0; | |
252 | ||
253 | evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG); | |
254 | tre_iterator = lldev->tre_processed_off; | |
255 | evre_iterator = lldev->evre_processed_off; | |
256 | ||
257 | if ((evre_write_off > evre_ring_size) || | |
258 | (evre_write_off % HIDMA_EVRE_SIZE)) { | |
259 | dev_err(lldev->dev, "HW reports invalid EVRE write offset\n"); | |
260 | return 0; | |
261 | } | |
262 | ||
263 | /* | |
264 | * By the time control reaches here the number of EVREs and TREs | |
265 | * may not match. Only consume the ones that hardware told us. | |
266 | */ | |
267 | while ((evre_iterator != evre_write_off)) { | |
268 | u32 *current_evre = lldev->evre_ring + evre_iterator; | |
269 | u32 cfg; | |
270 | ||
271 | cfg = current_evre[HIDMA_EVRE_CFG_IDX]; | |
272 | err_info = cfg >> HIDMA_EVRE_ERRINFO_BIT_POS; | |
273 | err_info &= HIDMA_EVRE_ERRINFO_MASK; | |
274 | err_code = | |
275 | (cfg >> HIDMA_EVRE_CODE_BIT_POS) & HIDMA_EVRE_CODE_MASK; | |
276 | ||
277 | if (hidma_post_completed(lldev, tre_iterator, err_info, | |
278 | err_code)) | |
279 | break; | |
280 | ||
281 | HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE, | |
282 | tre_ring_size); | |
283 | HIDMA_INCREMENT_ITERATOR(evre_iterator, HIDMA_EVRE_SIZE, | |
284 | evre_ring_size); | |
285 | ||
286 | /* | |
287 | * Read the new event descriptor written by the HW. | |
288 | * As we are processing the delivered events, other events | |
289 | * get queued to the SW for processing. | |
290 | */ | |
291 | evre_write_off = | |
292 | readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG); | |
293 | num_completed++; | |
294 | } | |
295 | ||
296 | if (num_completed) { | |
297 | u32 evre_read_off = (lldev->evre_processed_off + | |
298 | HIDMA_EVRE_SIZE * num_completed); | |
299 | u32 tre_read_off = (lldev->tre_processed_off + | |
300 | HIDMA_TRE_SIZE * num_completed); | |
301 | ||
302 | evre_read_off = evre_read_off % evre_ring_size; | |
303 | tre_read_off = tre_read_off % tre_ring_size; | |
304 | ||
305 | writel(evre_read_off, lldev->evca + HIDMA_EVCA_DOORBELL_REG); | |
306 | ||
307 | /* record the last processed tre offset */ | |
308 | lldev->tre_processed_off = tre_read_off; | |
309 | lldev->evre_processed_off = evre_read_off; | |
310 | } | |
311 | ||
312 | return num_completed; | |
313 | } | |
314 | ||
315 | void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info, | |
316 | u8 err_code) | |
317 | { | |
318 | u32 tre_iterator; | |
319 | u32 tre_ring_size = lldev->tre_ring_size; | |
320 | int num_completed = 0; | |
321 | u32 tre_read_off; | |
322 | ||
323 | tre_iterator = lldev->tre_processed_off; | |
324 | while (lldev->pending_tre_count) { | |
325 | if (hidma_post_completed(lldev, tre_iterator, err_info, | |
326 | err_code)) | |
327 | break; | |
328 | HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE, | |
329 | tre_ring_size); | |
330 | num_completed++; | |
331 | } | |
332 | tre_read_off = (lldev->tre_processed_off + | |
333 | HIDMA_TRE_SIZE * num_completed); | |
334 | ||
335 | tre_read_off = tre_read_off % tre_ring_size; | |
336 | ||
337 | /* record the last processed tre offset */ | |
338 | lldev->tre_processed_off = tre_read_off; | |
339 | } | |
340 | ||
341 | static int hidma_ll_reset(struct hidma_lldev *lldev) | |
342 | { | |
343 | u32 val; | |
344 | int ret; | |
345 | ||
346 | val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); | |
347 | val &= ~(HIDMA_CH_CONTROL_MASK << 16); | |
348 | val |= HIDMA_CH_RESET << 16; | |
349 | writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG); | |
350 | ||
351 | /* | |
352 | * Delay 10ms after reset to allow DMA logic to quiesce. | |
353 | * Do a polled read up to 1ms and 10ms maximum. | |
354 | */ | |
355 | ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, | |
356 | HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED, | |
357 | 1000, 10000); | |
358 | if (ret) { | |
359 | dev_err(lldev->dev, "transfer channel did not reset\n"); | |
360 | return ret; | |
361 | } | |
362 | ||
363 | val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); | |
364 | val &= ~(HIDMA_CH_CONTROL_MASK << 16); | |
365 | val |= HIDMA_CH_RESET << 16; | |
366 | writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG); | |
367 | ||
368 | /* | |
369 | * Delay 10ms after reset to allow DMA logic to quiesce. | |
370 | * Do a polled read up to 1ms and 10ms maximum. | |
371 | */ | |
372 | ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, | |
373 | HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED, | |
374 | 1000, 10000); | |
375 | if (ret) | |
376 | return ret; | |
377 | ||
378 | lldev->trch_state = HIDMA_CH_DISABLED; | |
379 | lldev->evch_state = HIDMA_CH_DISABLED; | |
380 | return 0; | |
381 | } | |
382 | ||
d1615ca2 SK |
383 | /* |
384 | * The interrupt handler for HIDMA will try to consume as many pending | |
385 | * EVRE from the event queue as possible. Each EVRE has an associated | |
386 | * TRE that holds the user interface parameters. EVRE reports the | |
387 | * result of the transaction. Hardware guarantees ordering between EVREs | |
388 | * and TREs. We use last processed offset to figure out which TRE is | |
389 | * associated with which EVRE. If two TREs are consumed by HW, the EVREs | |
390 | * are in order in the event ring. | |
391 | * | |
392 | * This handler will do a one pass for consuming EVREs. Other EVREs may | |
393 | * be delivered while we are working. It will try to consume incoming | |
394 | * EVREs one more time and return. | |
395 | * | |
396 | * For unprocessed EVREs, hardware will trigger another interrupt until | |
397 | * all the interrupt bits are cleared. | |
398 | * | |
399 | * Hardware guarantees that by the time interrupt is observed, all data | |
400 | * transactions in flight are delivered to their respective places and | |
401 | * are visible to the CPU. | |
402 | * | |
403 | * On demand paging for IOMMU is only supported for PCIe via PRI | |
404 | * (Page Request Interface) not for HIDMA. All other hardware instances | |
405 | * including HIDMA work on pinned DMA addresses. | |
406 | * | |
407 | * HIDMA is not aware of IOMMU presence since it follows the DMA API. All | |
408 | * IOMMU latency will be built into the data movement time. By the time | |
409 | * interrupt happens, IOMMU lookups + data movement has already taken place. | |
410 | * | |
411 | * While the first read in a typical PCI endpoint ISR flushes all outstanding | |
412 | * requests traditionally to the destination, this concept does not apply | |
413 | * here for this HW. | |
414 | */ | |
415 | irqreturn_t hidma_ll_inthandler(int chirq, void *arg) | |
416 | { | |
417 | struct hidma_lldev *lldev = arg; | |
418 | u32 status; | |
419 | u32 enable; | |
420 | u32 cause; | |
421 | ||
422 | /* | |
423 | * Fine tuned for this HW... | |
424 | * | |
425 | * This ISR has been designed for this particular hardware. Relaxed | |
426 | * read and write accessors are used for performance reasons due to | |
427 | * interrupt delivery guarantees. Do not copy this code blindly and | |
428 | * expect that to work. | |
429 | */ | |
430 | status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); | |
431 | enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG); | |
432 | cause = status & enable; | |
433 | ||
434 | while (cause) { | |
435 | if (cause & HIDMA_ERR_INT_MASK) { | |
793ae66c | 436 | dev_err(lldev->dev, "error 0x%x, disabling...\n", |
d1615ca2 SK |
437 | cause); |
438 | ||
439 | /* Clear out pending interrupts */ | |
440 | writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); | |
441 | ||
793ae66c SK |
442 | /* No further submissions. */ |
443 | hidma_ll_disable(lldev); | |
444 | ||
445 | /* Driver completes the txn and intimates the client.*/ | |
446 | hidma_cleanup_pending_tre(lldev, 0xFF, | |
447 | HIDMA_EVRE_STATUS_ERROR); | |
d1615ca2 SK |
448 | goto out; |
449 | } | |
450 | ||
451 | /* | |
452 | * Try to consume as many EVREs as possible. | |
453 | */ | |
454 | hidma_handle_tre_completion(lldev); | |
455 | ||
456 | /* We consumed TREs or there are pending TREs or EVREs. */ | |
457 | writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); | |
458 | ||
459 | /* | |
460 | * Another interrupt might have arrived while we are | |
461 | * processing this one. Read the new cause. | |
462 | */ | |
463 | status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); | |
464 | enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG); | |
465 | cause = status & enable; | |
466 | } | |
467 | ||
468 | out: | |
469 | return IRQ_HANDLED; | |
470 | } | |
471 | ||
472 | int hidma_ll_enable(struct hidma_lldev *lldev) | |
473 | { | |
474 | u32 val; | |
475 | int ret; | |
476 | ||
477 | val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); | |
478 | val &= ~(HIDMA_CH_CONTROL_MASK << 16); | |
479 | val |= HIDMA_CH_ENABLE << 16; | |
480 | writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG); | |
481 | ||
482 | ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, | |
483 | hidma_is_chan_enabled(HIDMA_CH_STATE(val)), | |
484 | 1000, 10000); | |
485 | if (ret) { | |
486 | dev_err(lldev->dev, "event channel did not get enabled\n"); | |
487 | return ret; | |
488 | } | |
489 | ||
490 | val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); | |
491 | val &= ~(HIDMA_CH_CONTROL_MASK << 16); | |
492 | val |= HIDMA_CH_ENABLE << 16; | |
493 | writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG); | |
494 | ||
495 | ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, | |
496 | hidma_is_chan_enabled(HIDMA_CH_STATE(val)), | |
497 | 1000, 10000); | |
498 | if (ret) { | |
499 | dev_err(lldev->dev, "transfer channel did not get enabled\n"); | |
500 | return ret; | |
501 | } | |
502 | ||
503 | lldev->trch_state = HIDMA_CH_ENABLED; | |
504 | lldev->evch_state = HIDMA_CH_ENABLED; | |
505 | ||
506 | return 0; | |
507 | } | |
508 | ||
509 | void hidma_ll_start(struct hidma_lldev *lldev) | |
510 | { | |
511 | unsigned long irqflags; | |
512 | ||
513 | spin_lock_irqsave(&lldev->lock, irqflags); | |
514 | writel(lldev->tre_write_offset, lldev->trca + HIDMA_TRCA_DOORBELL_REG); | |
515 | spin_unlock_irqrestore(&lldev->lock, irqflags); | |
516 | } | |
517 | ||
518 | bool hidma_ll_isenabled(struct hidma_lldev *lldev) | |
519 | { | |
520 | u32 val; | |
521 | ||
522 | val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); | |
523 | lldev->trch_state = HIDMA_CH_STATE(val); | |
524 | val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); | |
525 | lldev->evch_state = HIDMA_CH_STATE(val); | |
526 | ||
527 | /* both channels have to be enabled before calling this function */ | |
528 | if (hidma_is_chan_enabled(lldev->trch_state) && | |
529 | hidma_is_chan_enabled(lldev->evch_state)) | |
530 | return true; | |
531 | ||
532 | return false; | |
533 | } | |
534 | ||
535 | void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch) | |
536 | { | |
537 | struct hidma_tre *tre; | |
538 | unsigned long flags; | |
539 | ||
540 | tre = &lldev->trepool[tre_ch]; | |
541 | ||
542 | /* copy the TRE into its location in the TRE ring */ | |
543 | spin_lock_irqsave(&lldev->lock, flags); | |
544 | tre->tre_index = lldev->tre_write_offset / HIDMA_TRE_SIZE; | |
545 | lldev->pending_tre_list[tre->tre_index] = tre; | |
546 | memcpy(lldev->tre_ring + lldev->tre_write_offset, | |
547 | &tre->tre_local[0], HIDMA_TRE_SIZE); | |
548 | tre->err_code = 0; | |
549 | tre->err_info = 0; | |
550 | tre->queued = 1; | |
551 | lldev->pending_tre_count++; | |
552 | lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE) | |
553 | % lldev->tre_ring_size; | |
554 | spin_unlock_irqrestore(&lldev->lock, flags); | |
555 | } | |
556 | ||
557 | /* | |
558 | * Note that even though we stop this channel if there is a pending transaction | |
559 | * in flight it will complete and follow the callback. This request will | |
560 | * prevent further requests to be made. | |
561 | */ | |
562 | int hidma_ll_disable(struct hidma_lldev *lldev) | |
563 | { | |
564 | u32 val; | |
565 | int ret; | |
566 | ||
567 | val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); | |
568 | lldev->evch_state = HIDMA_CH_STATE(val); | |
569 | val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); | |
570 | lldev->trch_state = HIDMA_CH_STATE(val); | |
571 | ||
572 | /* already suspended by this OS */ | |
573 | if ((lldev->trch_state == HIDMA_CH_SUSPENDED) || | |
574 | (lldev->evch_state == HIDMA_CH_SUSPENDED)) | |
575 | return 0; | |
576 | ||
577 | /* already stopped by the manager */ | |
578 | if ((lldev->trch_state == HIDMA_CH_STOPPED) || | |
579 | (lldev->evch_state == HIDMA_CH_STOPPED)) | |
580 | return 0; | |
581 | ||
582 | val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG); | |
583 | val &= ~(HIDMA_CH_CONTROL_MASK << 16); | |
584 | val |= HIDMA_CH_SUSPEND << 16; | |
585 | writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG); | |
586 | ||
587 | /* | |
588 | * Start the wait right after the suspend is confirmed. | |
589 | * Do a polled read up to 1ms and 10ms maximum. | |
590 | */ | |
591 | ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val, | |
592 | HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED, | |
593 | 1000, 10000); | |
594 | if (ret) | |
595 | return ret; | |
596 | ||
597 | val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG); | |
598 | val &= ~(HIDMA_CH_CONTROL_MASK << 16); | |
599 | val |= HIDMA_CH_SUSPEND << 16; | |
600 | writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG); | |
601 | ||
602 | /* | |
603 | * Start the wait right after the suspend is confirmed | |
604 | * Delay up to 10ms after reset to allow DMA logic to quiesce. | |
605 | */ | |
606 | ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val, | |
607 | HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED, | |
608 | 1000, 10000); | |
609 | if (ret) | |
610 | return ret; | |
611 | ||
612 | lldev->trch_state = HIDMA_CH_SUSPENDED; | |
613 | lldev->evch_state = HIDMA_CH_SUSPENDED; | |
614 | return 0; | |
615 | } | |
616 | ||
617 | void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch, | |
618 | dma_addr_t src, dma_addr_t dest, u32 len, | |
619 | u32 flags) | |
620 | { | |
621 | struct hidma_tre *tre; | |
622 | u32 *tre_local; | |
623 | ||
624 | if (tre_ch >= lldev->nr_tres) { | |
625 | dev_err(lldev->dev, "invalid TRE number in transfer params:%d", | |
626 | tre_ch); | |
627 | return; | |
628 | } | |
629 | ||
630 | tre = &lldev->trepool[tre_ch]; | |
631 | if (atomic_read(&tre->allocated) != true) { | |
632 | dev_err(lldev->dev, "trying to set params on an unused TRE:%d", | |
633 | tre_ch); | |
634 | return; | |
635 | } | |
636 | ||
637 | tre_local = &tre->tre_local[0]; | |
638 | tre_local[HIDMA_TRE_LEN_IDX] = len; | |
639 | tre_local[HIDMA_TRE_SRC_LOW_IDX] = lower_32_bits(src); | |
640 | tre_local[HIDMA_TRE_SRC_HI_IDX] = upper_32_bits(src); | |
641 | tre_local[HIDMA_TRE_DEST_LOW_IDX] = lower_32_bits(dest); | |
642 | tre_local[HIDMA_TRE_DEST_HI_IDX] = upper_32_bits(dest); | |
643 | tre->int_flags = flags; | |
644 | } | |
645 | ||
646 | /* | |
647 | * Called during initialization and after an error condition | |
648 | * to restore hardware state. | |
649 | */ | |
650 | int hidma_ll_setup(struct hidma_lldev *lldev) | |
651 | { | |
652 | int rc; | |
653 | u64 addr; | |
654 | u32 val; | |
655 | u32 nr_tres = lldev->nr_tres; | |
656 | ||
657 | lldev->pending_tre_count = 0; | |
658 | lldev->tre_processed_off = 0; | |
659 | lldev->evre_processed_off = 0; | |
660 | lldev->tre_write_offset = 0; | |
661 | ||
662 | /* disable interrupts */ | |
663 | writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); | |
664 | ||
665 | /* clear all pending interrupts */ | |
666 | val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); | |
667 | writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); | |
668 | ||
669 | rc = hidma_ll_reset(lldev); | |
670 | if (rc) | |
671 | return rc; | |
672 | ||
673 | /* | |
674 | * Clear all pending interrupts again. | |
675 | * Otherwise, we observe reset complete interrupts. | |
676 | */ | |
677 | val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); | |
678 | writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); | |
679 | ||
680 | /* disable interrupts again after reset */ | |
681 | writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); | |
682 | ||
683 | addr = lldev->tre_dma; | |
684 | writel(lower_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_LOW_REG); | |
685 | writel(upper_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_HIGH_REG); | |
686 | writel(lldev->tre_ring_size, lldev->trca + HIDMA_TRCA_RING_LEN_REG); | |
687 | ||
688 | addr = lldev->evre_dma; | |
689 | writel(lower_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_LOW_REG); | |
690 | writel(upper_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_HIGH_REG); | |
691 | writel(HIDMA_EVRE_SIZE * nr_tres, | |
692 | lldev->evca + HIDMA_EVCA_RING_LEN_REG); | |
693 | ||
694 | /* support IRQ only for now */ | |
695 | val = readl(lldev->evca + HIDMA_EVCA_INTCTRL_REG); | |
696 | val &= ~0xF; | |
697 | val |= 0x1; | |
698 | writel(val, lldev->evca + HIDMA_EVCA_INTCTRL_REG); | |
699 | ||
700 | /* clear all pending interrupts and enable them */ | |
701 | writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); | |
702 | writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); | |
703 | ||
704 | return hidma_ll_enable(lldev); | |
705 | } | |
706 | ||
707 | struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres, | |
708 | void __iomem *trca, void __iomem *evca, | |
709 | u8 chidx) | |
710 | { | |
711 | u32 required_bytes; | |
712 | struct hidma_lldev *lldev; | |
713 | int rc; | |
714 | size_t sz; | |
715 | ||
716 | if (!trca || !evca || !dev || !nr_tres) | |
717 | return NULL; | |
718 | ||
719 | /* need at least four TREs */ | |
720 | if (nr_tres < 4) | |
721 | return NULL; | |
722 | ||
723 | /* need an extra space */ | |
724 | nr_tres += 1; | |
725 | ||
726 | lldev = devm_kzalloc(dev, sizeof(struct hidma_lldev), GFP_KERNEL); | |
727 | if (!lldev) | |
728 | return NULL; | |
729 | ||
730 | lldev->evca = evca; | |
731 | lldev->trca = trca; | |
732 | lldev->dev = dev; | |
733 | sz = sizeof(struct hidma_tre); | |
734 | lldev->trepool = devm_kcalloc(lldev->dev, nr_tres, sz, GFP_KERNEL); | |
735 | if (!lldev->trepool) | |
736 | return NULL; | |
737 | ||
738 | required_bytes = sizeof(lldev->pending_tre_list[0]); | |
739 | lldev->pending_tre_list = devm_kcalloc(dev, nr_tres, required_bytes, | |
740 | GFP_KERNEL); | |
741 | if (!lldev->pending_tre_list) | |
742 | return NULL; | |
743 | ||
744 | sz = (HIDMA_TRE_SIZE + 1) * nr_tres; | |
745 | lldev->tre_ring = dmam_alloc_coherent(dev, sz, &lldev->tre_dma, | |
746 | GFP_KERNEL); | |
747 | if (!lldev->tre_ring) | |
748 | return NULL; | |
749 | ||
750 | memset(lldev->tre_ring, 0, (HIDMA_TRE_SIZE + 1) * nr_tres); | |
751 | lldev->tre_ring_size = HIDMA_TRE_SIZE * nr_tres; | |
752 | lldev->nr_tres = nr_tres; | |
753 | ||
754 | /* the TRE ring has to be TRE_SIZE aligned */ | |
755 | if (!IS_ALIGNED(lldev->tre_dma, HIDMA_TRE_SIZE)) { | |
756 | u8 tre_ring_shift; | |
757 | ||
758 | tre_ring_shift = lldev->tre_dma % HIDMA_TRE_SIZE; | |
759 | tre_ring_shift = HIDMA_TRE_SIZE - tre_ring_shift; | |
760 | lldev->tre_dma += tre_ring_shift; | |
761 | lldev->tre_ring += tre_ring_shift; | |
762 | } | |
763 | ||
764 | sz = (HIDMA_EVRE_SIZE + 1) * nr_tres; | |
765 | lldev->evre_ring = dmam_alloc_coherent(dev, sz, &lldev->evre_dma, | |
766 | GFP_KERNEL); | |
767 | if (!lldev->evre_ring) | |
768 | return NULL; | |
769 | ||
770 | memset(lldev->evre_ring, 0, (HIDMA_EVRE_SIZE + 1) * nr_tres); | |
771 | lldev->evre_ring_size = HIDMA_EVRE_SIZE * nr_tres; | |
772 | ||
773 | /* the EVRE ring has to be EVRE_SIZE aligned */ | |
774 | if (!IS_ALIGNED(lldev->evre_dma, HIDMA_EVRE_SIZE)) { | |
775 | u8 evre_ring_shift; | |
776 | ||
777 | evre_ring_shift = lldev->evre_dma % HIDMA_EVRE_SIZE; | |
778 | evre_ring_shift = HIDMA_EVRE_SIZE - evre_ring_shift; | |
779 | lldev->evre_dma += evre_ring_shift; | |
780 | lldev->evre_ring += evre_ring_shift; | |
781 | } | |
782 | lldev->nr_tres = nr_tres; | |
783 | lldev->chidx = chidx; | |
784 | ||
785 | sz = nr_tres * sizeof(struct hidma_tre *); | |
786 | rc = kfifo_alloc(&lldev->handoff_fifo, sz, GFP_KERNEL); | |
787 | if (rc) | |
788 | return NULL; | |
789 | ||
790 | rc = hidma_ll_setup(lldev); | |
791 | if (rc) | |
792 | return NULL; | |
793 | ||
794 | spin_lock_init(&lldev->lock); | |
d1615ca2 SK |
795 | tasklet_init(&lldev->task, hidma_ll_tre_complete, (unsigned long)lldev); |
796 | lldev->initialized = 1; | |
797 | writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); | |
798 | return lldev; | |
799 | } | |
800 | ||
801 | int hidma_ll_uninit(struct hidma_lldev *lldev) | |
802 | { | |
803 | u32 required_bytes; | |
804 | int rc = 0; | |
805 | u32 val; | |
806 | ||
807 | if (!lldev) | |
808 | return -ENODEV; | |
809 | ||
810 | if (!lldev->initialized) | |
811 | return 0; | |
812 | ||
813 | lldev->initialized = 0; | |
814 | ||
815 | required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres; | |
816 | tasklet_kill(&lldev->task); | |
817 | memset(lldev->trepool, 0, required_bytes); | |
818 | lldev->trepool = NULL; | |
819 | lldev->pending_tre_count = 0; | |
820 | lldev->tre_write_offset = 0; | |
821 | ||
822 | rc = hidma_ll_reset(lldev); | |
823 | ||
824 | /* | |
825 | * Clear all pending interrupts again. | |
826 | * Otherwise, we observe reset complete interrupts. | |
827 | */ | |
828 | val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG); | |
829 | writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG); | |
830 | writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG); | |
831 | return rc; | |
832 | } | |
833 | ||
834 | enum dma_status hidma_ll_status(struct hidma_lldev *lldev, u32 tre_ch) | |
835 | { | |
836 | enum dma_status ret = DMA_ERROR; | |
837 | struct hidma_tre *tre; | |
838 | unsigned long flags; | |
839 | u8 err_code; | |
840 | ||
841 | spin_lock_irqsave(&lldev->lock, flags); | |
842 | ||
843 | tre = &lldev->trepool[tre_ch]; | |
844 | err_code = tre->err_code; | |
845 | ||
846 | if (err_code & HIDMA_EVRE_STATUS_COMPLETE) | |
847 | ret = DMA_COMPLETE; | |
848 | else if (err_code & HIDMA_EVRE_STATUS_ERROR) | |
849 | ret = DMA_ERROR; | |
850 | else | |
851 | ret = DMA_IN_PROGRESS; | |
852 | spin_unlock_irqrestore(&lldev->lock, flags); | |
853 | ||
854 | return ret; | |
855 | } |