Commit | Line | Data |
---|---|---|
95b4ecbf SY |
1 | /* |
2 | * Intel MIC Platform Software Stack (MPSS) | |
3 | * | |
4 | * Copyright(c) 2014 Intel Corporation. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License, version 2, as | |
8 | * published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | * General Public License for more details. | |
14 | * | |
15 | * The full GNU General Public License is included in this distribution in | |
16 | * the file called "COPYING". | |
17 | * | |
18 | * Intel MIC X100 DMA Driver. | |
19 | * | |
20 | * Adapted from IOAT dma driver. | |
21 | */ | |
22 | #include <linux/module.h> | |
23 | #include <linux/io.h> | |
24 | #include <linux/seq_file.h> | |
d6472302 | 25 | #include <linux/vmalloc.h> |
95b4ecbf SY |
26 | |
27 | #include "mic_x100_dma.h" | |
28 | ||
29 | #define MIC_DMA_MAX_XFER_SIZE_CARD (1 * 1024 * 1024 -\ | |
30 | MIC_DMA_ALIGN_BYTES) | |
31 | #define MIC_DMA_MAX_XFER_SIZE_HOST (1 * 1024 * 1024 >> 1) | |
32 | #define MIC_DMA_DESC_TYPE_SHIFT 60 | |
33 | #define MIC_DMA_MEMCPY_LEN_SHIFT 46 | |
34 | #define MIC_DMA_STAT_INTR_SHIFT 59 | |
35 | ||
36 | /* high-water mark for pushing dma descriptors */ | |
37 | static int mic_dma_pending_level = 4; | |
38 | ||
39 | /* Status descriptor is used to write a 64 bit value to a memory location */ | |
40 | enum mic_dma_desc_format_type { | |
41 | MIC_DMA_MEMCPY = 1, | |
42 | MIC_DMA_STATUS, | |
43 | }; | |
44 | ||
45 | static inline u32 mic_dma_hw_ring_inc(u32 val) | |
46 | { | |
47 | return (val + 1) % MIC_DMA_DESC_RX_SIZE; | |
48 | } | |
49 | ||
50 | static inline u32 mic_dma_hw_ring_dec(u32 val) | |
51 | { | |
52 | return val ? val - 1 : MIC_DMA_DESC_RX_SIZE - 1; | |
53 | } | |
54 | ||
55 | static inline void mic_dma_hw_ring_inc_head(struct mic_dma_chan *ch) | |
56 | { | |
57 | ch->head = mic_dma_hw_ring_inc(ch->head); | |
58 | } | |
59 | ||
60 | /* Prepare a memcpy desc */ | |
61 | static inline void mic_dma_memcpy_desc(struct mic_dma_desc *desc, | |
62 | dma_addr_t src_phys, dma_addr_t dst_phys, u64 size) | |
63 | { | |
64 | u64 qw0, qw1; | |
65 | ||
66 | qw0 = src_phys; | |
67 | qw0 |= (size >> MIC_DMA_ALIGN_SHIFT) << MIC_DMA_MEMCPY_LEN_SHIFT; | |
68 | qw1 = MIC_DMA_MEMCPY; | |
69 | qw1 <<= MIC_DMA_DESC_TYPE_SHIFT; | |
70 | qw1 |= dst_phys; | |
71 | desc->qw0 = qw0; | |
72 | desc->qw1 = qw1; | |
73 | } | |
74 | ||
75 | /* Prepare a status desc. with @data to be written at @dst_phys */ | |
76 | static inline void mic_dma_prep_status_desc(struct mic_dma_desc *desc, u64 data, | |
77 | dma_addr_t dst_phys, bool generate_intr) | |
78 | { | |
79 | u64 qw0, qw1; | |
80 | ||
81 | qw0 = data; | |
82 | qw1 = (u64) MIC_DMA_STATUS << MIC_DMA_DESC_TYPE_SHIFT | dst_phys; | |
83 | if (generate_intr) | |
84 | qw1 |= (1ULL << MIC_DMA_STAT_INTR_SHIFT); | |
85 | desc->qw0 = qw0; | |
86 | desc->qw1 = qw1; | |
87 | } | |
88 | ||
89 | static void mic_dma_cleanup(struct mic_dma_chan *ch) | |
90 | { | |
91 | struct dma_async_tx_descriptor *tx; | |
92 | u32 tail; | |
93 | u32 last_tail; | |
94 | ||
95 | spin_lock(&ch->cleanup_lock); | |
96 | tail = mic_dma_read_cmp_cnt(ch); | |
97 | /* | |
98 | * This is the barrier pair for smp_wmb() in fn. | |
99 | * mic_dma_tx_submit_unlock. It's required so that we read the | |
100 | * updated cookie value from tx->cookie. | |
101 | */ | |
102 | smp_rmb(); | |
103 | for (last_tail = ch->last_tail; tail != last_tail;) { | |
104 | tx = &ch->tx_array[last_tail]; | |
105 | if (tx->cookie) { | |
106 | dma_cookie_complete(tx); | |
107 | if (tx->callback) { | |
108 | tx->callback(tx->callback_param); | |
109 | tx->callback = NULL; | |
110 | } | |
111 | } | |
112 | last_tail = mic_dma_hw_ring_inc(last_tail); | |
113 | } | |
114 | /* finish all completion callbacks before incrementing tail */ | |
115 | smp_mb(); | |
116 | ch->last_tail = last_tail; | |
117 | spin_unlock(&ch->cleanup_lock); | |
118 | } | |
119 | ||
120 | static u32 mic_dma_ring_count(u32 head, u32 tail) | |
121 | { | |
122 | u32 count; | |
123 | ||
124 | if (head >= tail) | |
125 | count = (tail - 0) + (MIC_DMA_DESC_RX_SIZE - head); | |
126 | else | |
127 | count = tail - head; | |
128 | return count - 1; | |
129 | } | |
130 | ||
131 | /* Returns the num. of free descriptors on success, -ENOMEM on failure */ | |
132 | static int mic_dma_avail_desc_ring_space(struct mic_dma_chan *ch, int required) | |
133 | { | |
134 | struct device *dev = mic_dma_ch_to_device(ch); | |
135 | u32 count; | |
136 | ||
137 | count = mic_dma_ring_count(ch->head, ch->last_tail); | |
138 | if (count < required) { | |
139 | mic_dma_cleanup(ch); | |
140 | count = mic_dma_ring_count(ch->head, ch->last_tail); | |
141 | } | |
142 | ||
143 | if (count < required) { | |
144 | dev_dbg(dev, "Not enough desc space"); | |
145 | dev_dbg(dev, "%s %d required=%u, avail=%u\n", | |
146 | __func__, __LINE__, required, count); | |
147 | return -ENOMEM; | |
148 | } else { | |
149 | return count; | |
150 | } | |
151 | } | |
152 | ||
153 | /* Program memcpy descriptors into the descriptor ring and update s/w head ptr*/ | |
154 | static int mic_dma_prog_memcpy_desc(struct mic_dma_chan *ch, dma_addr_t src, | |
155 | dma_addr_t dst, size_t len) | |
156 | { | |
157 | size_t current_transfer_len; | |
158 | size_t max_xfer_size = to_mic_dma_dev(ch)->max_xfer_size; | |
159 | /* 3 is added to make sure we have enough space for status desc */ | |
160 | int num_desc = len / max_xfer_size + 3; | |
161 | int ret; | |
162 | ||
163 | if (len % max_xfer_size) | |
164 | num_desc++; | |
165 | ||
166 | ret = mic_dma_avail_desc_ring_space(ch, num_desc); | |
167 | if (ret < 0) | |
168 | return ret; | |
169 | do { | |
170 | current_transfer_len = min(len, max_xfer_size); | |
171 | mic_dma_memcpy_desc(&ch->desc_ring[ch->head], | |
172 | src, dst, current_transfer_len); | |
173 | mic_dma_hw_ring_inc_head(ch); | |
174 | len -= current_transfer_len; | |
175 | dst = dst + current_transfer_len; | |
176 | src = src + current_transfer_len; | |
177 | } while (len > 0); | |
178 | return 0; | |
179 | } | |
180 | ||
181 | /* It's a h/w quirk and h/w needs 2 status descriptors for every status desc */ | |
182 | static void mic_dma_prog_intr(struct mic_dma_chan *ch) | |
183 | { | |
184 | mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0, | |
185 | ch->status_dest_micpa, false); | |
186 | mic_dma_hw_ring_inc_head(ch); | |
187 | mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0, | |
188 | ch->status_dest_micpa, true); | |
189 | mic_dma_hw_ring_inc_head(ch); | |
190 | } | |
191 | ||
192 | /* Wrapper function to program memcpy descriptors/status descriptors */ | |
193 | static int mic_dma_do_dma(struct mic_dma_chan *ch, int flags, dma_addr_t src, | |
194 | dma_addr_t dst, size_t len) | |
195 | { | |
ff39988a | 196 | if (len && -ENOMEM == mic_dma_prog_memcpy_desc(ch, src, dst, len)) { |
95b4ecbf | 197 | return -ENOMEM; |
ff39988a SY |
198 | } else { |
199 | /* 3 is the maximum number of status descriptors */ | |
200 | int ret = mic_dma_avail_desc_ring_space(ch, 3); | |
201 | ||
202 | if (ret < 0) | |
203 | return ret; | |
204 | } | |
205 | ||
95b4ecbf SY |
206 | /* Above mic_dma_prog_memcpy_desc() makes sure we have enough space */ |
207 | if (flags & DMA_PREP_FENCE) { | |
208 | mic_dma_prep_status_desc(&ch->desc_ring[ch->head], 0, | |
209 | ch->status_dest_micpa, false); | |
210 | mic_dma_hw_ring_inc_head(ch); | |
211 | } | |
212 | ||
213 | if (flags & DMA_PREP_INTERRUPT) | |
214 | mic_dma_prog_intr(ch); | |
215 | ||
216 | return 0; | |
217 | } | |
218 | ||
219 | static inline void mic_dma_issue_pending(struct dma_chan *ch) | |
220 | { | |
221 | struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch); | |
222 | ||
223 | spin_lock(&mic_ch->issue_lock); | |
224 | /* | |
225 | * Write to head triggers h/w to act on the descriptors. | |
226 | * On MIC, writing the same head value twice causes | |
227 | * a h/w error. On second write, h/w assumes we filled | |
228 | * the entire ring & overwrote some of the descriptors. | |
229 | */ | |
230 | if (mic_ch->issued == mic_ch->submitted) | |
231 | goto out; | |
232 | mic_ch->issued = mic_ch->submitted; | |
233 | /* | |
234 | * make descriptor updates visible before advancing head, | |
235 | * this is purposefully not smp_wmb() since we are also | |
236 | * publishing the descriptor updates to a dma device | |
237 | */ | |
238 | wmb(); | |
239 | mic_dma_write_reg(mic_ch, MIC_DMA_REG_DHPR, mic_ch->issued); | |
240 | out: | |
241 | spin_unlock(&mic_ch->issue_lock); | |
242 | } | |
243 | ||
244 | static inline void mic_dma_update_pending(struct mic_dma_chan *ch) | |
245 | { | |
246 | if (mic_dma_ring_count(ch->issued, ch->submitted) | |
247 | > mic_dma_pending_level) | |
248 | mic_dma_issue_pending(&ch->api_ch); | |
249 | } | |
250 | ||
251 | static dma_cookie_t mic_dma_tx_submit_unlock(struct dma_async_tx_descriptor *tx) | |
252 | { | |
253 | struct mic_dma_chan *mic_ch = to_mic_dma_chan(tx->chan); | |
254 | dma_cookie_t cookie; | |
255 | ||
256 | dma_cookie_assign(tx); | |
257 | cookie = tx->cookie; | |
258 | /* | |
259 | * We need an smp write barrier here because another CPU might see | |
260 | * an update to submitted and update h/w head even before we | |
261 | * assigned a cookie to this tx. | |
262 | */ | |
263 | smp_wmb(); | |
264 | mic_ch->submitted = mic_ch->head; | |
265 | spin_unlock(&mic_ch->prep_lock); | |
266 | mic_dma_update_pending(mic_ch); | |
267 | return cookie; | |
268 | } | |
269 | ||
270 | static inline struct dma_async_tx_descriptor * | |
271 | allocate_tx(struct mic_dma_chan *ch) | |
272 | { | |
273 | u32 idx = mic_dma_hw_ring_dec(ch->head); | |
274 | struct dma_async_tx_descriptor *tx = &ch->tx_array[idx]; | |
275 | ||
276 | dma_async_tx_descriptor_init(tx, &ch->api_ch); | |
277 | tx->tx_submit = mic_dma_tx_submit_unlock; | |
278 | return tx; | |
279 | } | |
280 | ||
ff39988a SY |
281 | /* Program a status descriptor with dst as address and value to be written */ |
282 | static struct dma_async_tx_descriptor * | |
283 | mic_dma_prep_status_lock(struct dma_chan *ch, dma_addr_t dst, u64 src_val, | |
284 | unsigned long flags) | |
285 | { | |
286 | struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch); | |
287 | int result; | |
288 | ||
289 | spin_lock(&mic_ch->prep_lock); | |
290 | result = mic_dma_avail_desc_ring_space(mic_ch, 4); | |
291 | if (result < 0) | |
292 | goto error; | |
293 | mic_dma_prep_status_desc(&mic_ch->desc_ring[mic_ch->head], src_val, dst, | |
294 | false); | |
295 | mic_dma_hw_ring_inc_head(mic_ch); | |
296 | result = mic_dma_do_dma(mic_ch, flags, 0, 0, 0); | |
297 | if (result < 0) | |
298 | goto error; | |
299 | ||
300 | return allocate_tx(mic_ch); | |
301 | error: | |
302 | dev_err(mic_dma_ch_to_device(mic_ch), | |
303 | "Error enqueueing dma status descriptor, error=%d\n", result); | |
304 | spin_unlock(&mic_ch->prep_lock); | |
305 | return NULL; | |
306 | } | |
307 | ||
95b4ecbf SY |
308 | /* |
309 | * Prepare a memcpy descriptor to be added to the ring. | |
310 | * Note that the temporary descriptor adds an extra overhead of copying the | |
311 | * descriptor to ring. So, we copy directly to the descriptor ring | |
312 | */ | |
313 | static struct dma_async_tx_descriptor * | |
314 | mic_dma_prep_memcpy_lock(struct dma_chan *ch, dma_addr_t dma_dest, | |
315 | dma_addr_t dma_src, size_t len, unsigned long flags) | |
316 | { | |
317 | struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch); | |
318 | struct device *dev = mic_dma_ch_to_device(mic_ch); | |
319 | int result; | |
320 | ||
321 | if (!len && !flags) | |
322 | return NULL; | |
323 | ||
324 | spin_lock(&mic_ch->prep_lock); | |
325 | result = mic_dma_do_dma(mic_ch, flags, dma_src, dma_dest, len); | |
326 | if (result >= 0) | |
16605e8d AD |
327 | return allocate_tx(mic_ch); |
328 | dev_err(dev, "Error enqueueing dma, error=%d\n", result); | |
95b4ecbf | 329 | spin_unlock(&mic_ch->prep_lock); |
16605e8d | 330 | return NULL; |
95b4ecbf SY |
331 | } |
332 | ||
333 | static struct dma_async_tx_descriptor * | |
334 | mic_dma_prep_interrupt_lock(struct dma_chan *ch, unsigned long flags) | |
335 | { | |
336 | struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch); | |
337 | int ret; | |
338 | ||
339 | spin_lock(&mic_ch->prep_lock); | |
340 | ret = mic_dma_do_dma(mic_ch, flags, 0, 0, 0); | |
341 | if (!ret) | |
16605e8d | 342 | return allocate_tx(mic_ch); |
95b4ecbf | 343 | spin_unlock(&mic_ch->prep_lock); |
16605e8d | 344 | return NULL; |
95b4ecbf SY |
345 | } |
346 | ||
347 | /* Return the status of the transaction */ | |
348 | static enum dma_status | |
349 | mic_dma_tx_status(struct dma_chan *ch, dma_cookie_t cookie, | |
350 | struct dma_tx_state *txstate) | |
351 | { | |
352 | struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch); | |
353 | ||
354 | if (DMA_COMPLETE != dma_cookie_status(ch, cookie, txstate)) | |
355 | mic_dma_cleanup(mic_ch); | |
356 | ||
357 | return dma_cookie_status(ch, cookie, txstate); | |
358 | } | |
359 | ||
360 | static irqreturn_t mic_dma_thread_fn(int irq, void *data) | |
361 | { | |
362 | mic_dma_cleanup((struct mic_dma_chan *)data); | |
363 | return IRQ_HANDLED; | |
364 | } | |
365 | ||
366 | static irqreturn_t mic_dma_intr_handler(int irq, void *data) | |
367 | { | |
368 | struct mic_dma_chan *ch = ((struct mic_dma_chan *)data); | |
369 | ||
370 | mic_dma_ack_interrupt(ch); | |
371 | return IRQ_WAKE_THREAD; | |
372 | } | |
373 | ||
374 | static int mic_dma_alloc_desc_ring(struct mic_dma_chan *ch) | |
375 | { | |
376 | u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring); | |
377 | struct device *dev = &to_mbus_device(ch)->dev; | |
378 | ||
379 | desc_ring_size = ALIGN(desc_ring_size, MIC_DMA_ALIGN_BYTES); | |
380 | ch->desc_ring = kzalloc(desc_ring_size, GFP_KERNEL); | |
381 | ||
382 | if (!ch->desc_ring) | |
383 | return -ENOMEM; | |
384 | ||
385 | ch->desc_ring_micpa = dma_map_single(dev, ch->desc_ring, | |
386 | desc_ring_size, DMA_BIDIRECTIONAL); | |
387 | if (dma_mapping_error(dev, ch->desc_ring_micpa)) | |
388 | goto map_error; | |
389 | ||
390 | ch->tx_array = vzalloc(MIC_DMA_DESC_RX_SIZE * sizeof(*ch->tx_array)); | |
391 | if (!ch->tx_array) | |
392 | goto tx_error; | |
393 | return 0; | |
394 | tx_error: | |
395 | dma_unmap_single(dev, ch->desc_ring_micpa, desc_ring_size, | |
396 | DMA_BIDIRECTIONAL); | |
397 | map_error: | |
398 | kfree(ch->desc_ring); | |
399 | return -ENOMEM; | |
400 | } | |
401 | ||
402 | static void mic_dma_free_desc_ring(struct mic_dma_chan *ch) | |
403 | { | |
404 | u64 desc_ring_size = MIC_DMA_DESC_RX_SIZE * sizeof(*ch->desc_ring); | |
405 | ||
406 | vfree(ch->tx_array); | |
407 | desc_ring_size = ALIGN(desc_ring_size, MIC_DMA_ALIGN_BYTES); | |
408 | dma_unmap_single(&to_mbus_device(ch)->dev, ch->desc_ring_micpa, | |
409 | desc_ring_size, DMA_BIDIRECTIONAL); | |
410 | kfree(ch->desc_ring); | |
411 | ch->desc_ring = NULL; | |
412 | } | |
413 | ||
414 | static void mic_dma_free_status_dest(struct mic_dma_chan *ch) | |
415 | { | |
416 | dma_unmap_single(&to_mbus_device(ch)->dev, ch->status_dest_micpa, | |
417 | L1_CACHE_BYTES, DMA_BIDIRECTIONAL); | |
418 | kfree(ch->status_dest); | |
419 | } | |
420 | ||
421 | static int mic_dma_alloc_status_dest(struct mic_dma_chan *ch) | |
422 | { | |
423 | struct device *dev = &to_mbus_device(ch)->dev; | |
424 | ||
425 | ch->status_dest = kzalloc(L1_CACHE_BYTES, GFP_KERNEL); | |
426 | if (!ch->status_dest) | |
427 | return -ENOMEM; | |
428 | ch->status_dest_micpa = dma_map_single(dev, ch->status_dest, | |
429 | L1_CACHE_BYTES, DMA_BIDIRECTIONAL); | |
430 | if (dma_mapping_error(dev, ch->status_dest_micpa)) { | |
431 | kfree(ch->status_dest); | |
432 | ch->status_dest = NULL; | |
433 | return -ENOMEM; | |
434 | } | |
435 | return 0; | |
436 | } | |
437 | ||
438 | static int mic_dma_check_chan(struct mic_dma_chan *ch) | |
439 | { | |
440 | if (mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR) || | |
441 | mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) & MIC_DMA_CHAN_QUIESCE) { | |
442 | mic_dma_disable_chan(ch); | |
443 | mic_dma_chan_mask_intr(ch); | |
444 | dev_err(mic_dma_ch_to_device(ch), | |
445 | "%s %d error setting up mic dma chan %d\n", | |
446 | __func__, __LINE__, ch->ch_num); | |
447 | return -EBUSY; | |
448 | } | |
449 | return 0; | |
450 | } | |
451 | ||
452 | static int mic_dma_chan_setup(struct mic_dma_chan *ch) | |
453 | { | |
454 | if (MIC_DMA_CHAN_MIC == ch->owner) | |
455 | mic_dma_chan_set_owner(ch); | |
456 | mic_dma_disable_chan(ch); | |
457 | mic_dma_chan_mask_intr(ch); | |
458 | mic_dma_write_reg(ch, MIC_DMA_REG_DCHERRMSK, 0); | |
459 | mic_dma_chan_set_desc_ring(ch); | |
460 | ch->last_tail = mic_dma_read_reg(ch, MIC_DMA_REG_DTPR); | |
461 | ch->head = ch->last_tail; | |
462 | ch->issued = 0; | |
463 | mic_dma_chan_unmask_intr(ch); | |
464 | mic_dma_enable_chan(ch); | |
465 | return mic_dma_check_chan(ch); | |
466 | } | |
467 | ||
468 | static void mic_dma_chan_destroy(struct mic_dma_chan *ch) | |
469 | { | |
470 | mic_dma_disable_chan(ch); | |
471 | mic_dma_chan_mask_intr(ch); | |
472 | } | |
473 | ||
474 | static void mic_dma_unregister_dma_device(struct mic_dma_device *mic_dma_dev) | |
475 | { | |
476 | dma_async_device_unregister(&mic_dma_dev->dma_dev); | |
477 | } | |
478 | ||
479 | static int mic_dma_setup_irq(struct mic_dma_chan *ch) | |
480 | { | |
481 | ch->cookie = | |
482 | to_mbus_hw_ops(ch)->request_threaded_irq(to_mbus_device(ch), | |
483 | mic_dma_intr_handler, mic_dma_thread_fn, | |
484 | "mic dma_channel", ch, ch->ch_num); | |
485 | if (IS_ERR(ch->cookie)) | |
486 | return IS_ERR(ch->cookie); | |
487 | return 0; | |
488 | } | |
489 | ||
490 | static inline void mic_dma_free_irq(struct mic_dma_chan *ch) | |
491 | { | |
492 | to_mbus_hw_ops(ch)->free_irq(to_mbus_device(ch), ch->cookie, ch); | |
493 | } | |
494 | ||
495 | static int mic_dma_chan_init(struct mic_dma_chan *ch) | |
496 | { | |
497 | int ret = mic_dma_alloc_desc_ring(ch); | |
498 | ||
499 | if (ret) | |
500 | goto ring_error; | |
501 | ret = mic_dma_alloc_status_dest(ch); | |
502 | if (ret) | |
503 | goto status_error; | |
504 | ret = mic_dma_chan_setup(ch); | |
505 | if (ret) | |
506 | goto chan_error; | |
507 | return ret; | |
508 | chan_error: | |
509 | mic_dma_free_status_dest(ch); | |
510 | status_error: | |
511 | mic_dma_free_desc_ring(ch); | |
512 | ring_error: | |
513 | return ret; | |
514 | } | |
515 | ||
516 | static int mic_dma_drain_chan(struct mic_dma_chan *ch) | |
517 | { | |
518 | struct dma_async_tx_descriptor *tx; | |
519 | int err = 0; | |
520 | dma_cookie_t cookie; | |
521 | ||
522 | tx = mic_dma_prep_memcpy_lock(&ch->api_ch, 0, 0, 0, DMA_PREP_FENCE); | |
523 | if (!tx) { | |
524 | err = -ENOMEM; | |
525 | goto error; | |
526 | } | |
527 | ||
528 | cookie = tx->tx_submit(tx); | |
529 | if (dma_submit_error(cookie)) | |
530 | err = -ENOMEM; | |
531 | else | |
532 | err = dma_sync_wait(&ch->api_ch, cookie); | |
533 | if (err) { | |
534 | dev_err(mic_dma_ch_to_device(ch), "%s %d TO chan 0x%x\n", | |
535 | __func__, __LINE__, ch->ch_num); | |
536 | err = -EIO; | |
537 | } | |
538 | error: | |
539 | mic_dma_cleanup(ch); | |
540 | return err; | |
541 | } | |
542 | ||
543 | static inline void mic_dma_chan_uninit(struct mic_dma_chan *ch) | |
544 | { | |
545 | mic_dma_chan_destroy(ch); | |
546 | mic_dma_cleanup(ch); | |
547 | mic_dma_free_status_dest(ch); | |
548 | mic_dma_free_desc_ring(ch); | |
549 | } | |
550 | ||
551 | static int mic_dma_init(struct mic_dma_device *mic_dma_dev, | |
552 | enum mic_dma_chan_owner owner) | |
553 | { | |
554 | int i, first_chan = mic_dma_dev->start_ch; | |
555 | struct mic_dma_chan *ch; | |
556 | int ret; | |
557 | ||
558 | for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) { | |
559 | unsigned long data; | |
560 | ch = &mic_dma_dev->mic_ch[i]; | |
561 | data = (unsigned long)ch; | |
562 | ch->ch_num = i; | |
563 | ch->owner = owner; | |
564 | spin_lock_init(&ch->cleanup_lock); | |
565 | spin_lock_init(&ch->prep_lock); | |
566 | spin_lock_init(&ch->issue_lock); | |
567 | ret = mic_dma_setup_irq(ch); | |
568 | if (ret) | |
569 | goto error; | |
570 | } | |
571 | return 0; | |
572 | error: | |
573 | for (i = i - 1; i >= first_chan; i--) | |
574 | mic_dma_free_irq(ch); | |
575 | return ret; | |
576 | } | |
577 | ||
578 | static void mic_dma_uninit(struct mic_dma_device *mic_dma_dev) | |
579 | { | |
580 | int i, first_chan = mic_dma_dev->start_ch; | |
581 | struct mic_dma_chan *ch; | |
582 | ||
583 | for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) { | |
584 | ch = &mic_dma_dev->mic_ch[i]; | |
585 | mic_dma_free_irq(ch); | |
586 | } | |
587 | } | |
588 | ||
589 | static int mic_dma_alloc_chan_resources(struct dma_chan *ch) | |
590 | { | |
591 | int ret = mic_dma_chan_init(to_mic_dma_chan(ch)); | |
592 | if (ret) | |
593 | return ret; | |
594 | return MIC_DMA_DESC_RX_SIZE; | |
595 | } | |
596 | ||
597 | static void mic_dma_free_chan_resources(struct dma_chan *ch) | |
598 | { | |
599 | struct mic_dma_chan *mic_ch = to_mic_dma_chan(ch); | |
600 | mic_dma_drain_chan(mic_ch); | |
601 | mic_dma_chan_uninit(mic_ch); | |
602 | } | |
603 | ||
604 | /* Set the fn. handlers and register the dma device with dma api */ | |
605 | static int mic_dma_register_dma_device(struct mic_dma_device *mic_dma_dev, | |
606 | enum mic_dma_chan_owner owner) | |
607 | { | |
608 | int i, first_chan = mic_dma_dev->start_ch; | |
609 | ||
610 | dma_cap_zero(mic_dma_dev->dma_dev.cap_mask); | |
611 | /* | |
612 | * This dma engine is not capable of host memory to host memory | |
613 | * transfers | |
614 | */ | |
615 | dma_cap_set(DMA_MEMCPY, mic_dma_dev->dma_dev.cap_mask); | |
616 | ||
617 | if (MIC_DMA_CHAN_HOST == owner) | |
618 | dma_cap_set(DMA_PRIVATE, mic_dma_dev->dma_dev.cap_mask); | |
619 | mic_dma_dev->dma_dev.device_alloc_chan_resources = | |
620 | mic_dma_alloc_chan_resources; | |
621 | mic_dma_dev->dma_dev.device_free_chan_resources = | |
622 | mic_dma_free_chan_resources; | |
623 | mic_dma_dev->dma_dev.device_tx_status = mic_dma_tx_status; | |
624 | mic_dma_dev->dma_dev.device_prep_dma_memcpy = mic_dma_prep_memcpy_lock; | |
ff39988a SY |
625 | mic_dma_dev->dma_dev.device_prep_dma_imm_data = |
626 | mic_dma_prep_status_lock; | |
95b4ecbf SY |
627 | mic_dma_dev->dma_dev.device_prep_dma_interrupt = |
628 | mic_dma_prep_interrupt_lock; | |
629 | mic_dma_dev->dma_dev.device_issue_pending = mic_dma_issue_pending; | |
630 | mic_dma_dev->dma_dev.copy_align = MIC_DMA_ALIGN_SHIFT; | |
631 | INIT_LIST_HEAD(&mic_dma_dev->dma_dev.channels); | |
632 | for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) { | |
633 | mic_dma_dev->mic_ch[i].api_ch.device = &mic_dma_dev->dma_dev; | |
634 | dma_cookie_init(&mic_dma_dev->mic_ch[i].api_ch); | |
635 | list_add_tail(&mic_dma_dev->mic_ch[i].api_ch.device_node, | |
636 | &mic_dma_dev->dma_dev.channels); | |
637 | } | |
638 | return dma_async_device_register(&mic_dma_dev->dma_dev); | |
639 | } | |
640 | ||
641 | /* | |
642 | * Initializes dma channels and registers the dma device with the | |
643 | * dma engine api. | |
644 | */ | |
645 | static struct mic_dma_device *mic_dma_dev_reg(struct mbus_device *mbdev, | |
646 | enum mic_dma_chan_owner owner) | |
647 | { | |
648 | struct mic_dma_device *mic_dma_dev; | |
649 | int ret; | |
650 | struct device *dev = &mbdev->dev; | |
651 | ||
652 | mic_dma_dev = kzalloc(sizeof(*mic_dma_dev), GFP_KERNEL); | |
653 | if (!mic_dma_dev) { | |
654 | ret = -ENOMEM; | |
655 | goto alloc_error; | |
656 | } | |
657 | mic_dma_dev->mbdev = mbdev; | |
658 | mic_dma_dev->dma_dev.dev = dev; | |
659 | mic_dma_dev->mmio = mbdev->mmio_va; | |
660 | if (MIC_DMA_CHAN_HOST == owner) { | |
661 | mic_dma_dev->start_ch = 0; | |
662 | mic_dma_dev->max_xfer_size = MIC_DMA_MAX_XFER_SIZE_HOST; | |
663 | } else { | |
664 | mic_dma_dev->start_ch = 4; | |
665 | mic_dma_dev->max_xfer_size = MIC_DMA_MAX_XFER_SIZE_CARD; | |
666 | } | |
667 | ret = mic_dma_init(mic_dma_dev, owner); | |
668 | if (ret) | |
669 | goto init_error; | |
670 | ret = mic_dma_register_dma_device(mic_dma_dev, owner); | |
671 | if (ret) | |
672 | goto reg_error; | |
673 | return mic_dma_dev; | |
674 | reg_error: | |
675 | mic_dma_uninit(mic_dma_dev); | |
676 | init_error: | |
677 | kfree(mic_dma_dev); | |
678 | mic_dma_dev = NULL; | |
679 | alloc_error: | |
680 | dev_err(dev, "Error at %s %d ret=%d\n", __func__, __LINE__, ret); | |
681 | return mic_dma_dev; | |
682 | } | |
683 | ||
684 | static void mic_dma_dev_unreg(struct mic_dma_device *mic_dma_dev) | |
685 | { | |
686 | mic_dma_unregister_dma_device(mic_dma_dev); | |
687 | mic_dma_uninit(mic_dma_dev); | |
688 | kfree(mic_dma_dev); | |
689 | } | |
690 | ||
691 | /* DEBUGFS CODE */ | |
692 | static int mic_dma_reg_seq_show(struct seq_file *s, void *pos) | |
693 | { | |
694 | struct mic_dma_device *mic_dma_dev = s->private; | |
695 | int i, chan_num, first_chan = mic_dma_dev->start_ch; | |
696 | struct mic_dma_chan *ch; | |
697 | ||
698 | seq_printf(s, "SBOX_DCR: %#x\n", | |
699 | mic_dma_mmio_read(&mic_dma_dev->mic_ch[first_chan], | |
700 | MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR)); | |
701 | seq_puts(s, "DMA Channel Registers\n"); | |
702 | seq_printf(s, "%-10s| %-10s %-10s %-10s %-10s %-10s", | |
703 | "Channel", "DCAR", "DTPR", "DHPR", "DRAR_HI", "DRAR_LO"); | |
704 | seq_printf(s, " %-11s %-14s %-10s\n", "DCHERR", "DCHERRMSK", "DSTAT"); | |
705 | for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) { | |
706 | ch = &mic_dma_dev->mic_ch[i]; | |
707 | chan_num = ch->ch_num; | |
708 | seq_printf(s, "%-10i| %-#10x %-#10x %-#10x %-#10x", | |
709 | chan_num, | |
710 | mic_dma_read_reg(ch, MIC_DMA_REG_DCAR), | |
711 | mic_dma_read_reg(ch, MIC_DMA_REG_DTPR), | |
712 | mic_dma_read_reg(ch, MIC_DMA_REG_DHPR), | |
713 | mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_HI)); | |
714 | seq_printf(s, " %-#10x %-#10x %-#14x %-#10x\n", | |
715 | mic_dma_read_reg(ch, MIC_DMA_REG_DRAR_LO), | |
716 | mic_dma_read_reg(ch, MIC_DMA_REG_DCHERR), | |
717 | mic_dma_read_reg(ch, MIC_DMA_REG_DCHERRMSK), | |
718 | mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT)); | |
719 | } | |
720 | return 0; | |
721 | } | |
722 | ||
723 | static int mic_dma_reg_debug_open(struct inode *inode, struct file *file) | |
724 | { | |
725 | return single_open(file, mic_dma_reg_seq_show, inode->i_private); | |
726 | } | |
727 | ||
728 | static int mic_dma_reg_debug_release(struct inode *inode, struct file *file) | |
729 | { | |
730 | return single_release(inode, file); | |
731 | } | |
732 | ||
733 | static const struct file_operations mic_dma_reg_ops = { | |
734 | .owner = THIS_MODULE, | |
735 | .open = mic_dma_reg_debug_open, | |
736 | .read = seq_read, | |
737 | .llseek = seq_lseek, | |
738 | .release = mic_dma_reg_debug_release | |
739 | }; | |
740 | ||
741 | /* Debugfs parent dir */ | |
742 | static struct dentry *mic_dma_dbg; | |
743 | ||
744 | static int mic_dma_driver_probe(struct mbus_device *mbdev) | |
745 | { | |
746 | struct mic_dma_device *mic_dma_dev; | |
747 | enum mic_dma_chan_owner owner; | |
748 | ||
749 | if (MBUS_DEV_DMA_MIC == mbdev->id.device) | |
750 | owner = MIC_DMA_CHAN_MIC; | |
751 | else | |
752 | owner = MIC_DMA_CHAN_HOST; | |
753 | ||
754 | mic_dma_dev = mic_dma_dev_reg(mbdev, owner); | |
755 | dev_set_drvdata(&mbdev->dev, mic_dma_dev); | |
756 | ||
757 | if (mic_dma_dbg) { | |
758 | mic_dma_dev->dbg_dir = debugfs_create_dir(dev_name(&mbdev->dev), | |
759 | mic_dma_dbg); | |
760 | if (mic_dma_dev->dbg_dir) | |
761 | debugfs_create_file("mic_dma_reg", 0444, | |
762 | mic_dma_dev->dbg_dir, mic_dma_dev, | |
763 | &mic_dma_reg_ops); | |
764 | } | |
765 | return 0; | |
766 | } | |
767 | ||
768 | static void mic_dma_driver_remove(struct mbus_device *mbdev) | |
769 | { | |
770 | struct mic_dma_device *mic_dma_dev; | |
771 | ||
772 | mic_dma_dev = dev_get_drvdata(&mbdev->dev); | |
773 | debugfs_remove_recursive(mic_dma_dev->dbg_dir); | |
774 | mic_dma_dev_unreg(mic_dma_dev); | |
775 | } | |
776 | ||
777 | static struct mbus_device_id id_table[] = { | |
778 | {MBUS_DEV_DMA_MIC, MBUS_DEV_ANY_ID}, | |
779 | {MBUS_DEV_DMA_HOST, MBUS_DEV_ANY_ID}, | |
780 | {0}, | |
781 | }; | |
782 | ||
783 | static struct mbus_driver mic_dma_driver = { | |
784 | .driver.name = KBUILD_MODNAME, | |
785 | .driver.owner = THIS_MODULE, | |
786 | .id_table = id_table, | |
787 | .probe = mic_dma_driver_probe, | |
788 | .remove = mic_dma_driver_remove, | |
789 | }; | |
790 | ||
791 | static int __init mic_x100_dma_init(void) | |
792 | { | |
793 | int rc = mbus_register_driver(&mic_dma_driver); | |
794 | if (rc) | |
795 | return rc; | |
796 | mic_dma_dbg = debugfs_create_dir(KBUILD_MODNAME, NULL); | |
797 | return 0; | |
798 | } | |
799 | ||
800 | static void __exit mic_x100_dma_exit(void) | |
801 | { | |
802 | debugfs_remove_recursive(mic_dma_dbg); | |
803 | mbus_unregister_driver(&mic_dma_driver); | |
804 | } | |
805 | ||
806 | module_init(mic_x100_dma_init); | |
807 | module_exit(mic_x100_dma_exit); | |
808 | ||
809 | MODULE_DEVICE_TABLE(mbus, id_table); | |
810 | MODULE_AUTHOR("Intel Corporation"); | |
811 | MODULE_DESCRIPTION("Intel(R) MIC X100 DMA Driver"); | |
812 | MODULE_LICENSE("GPL v2"); |