2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2015 Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * The full GNU General Public License is included in this distribution in
15 * the file called "COPYING".
20 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
24 #include <linux/init.h>
25 #include <linux/module.h>
26 #include <linux/slab.h>
27 #include <linux/pci.h>
28 #include <linux/interrupt.h>
29 #include <linux/dmaengine.h>
30 #include <linux/delay.h>
31 #include <linux/dma-mapping.h>
32 #include <linux/workqueue.h>
33 #include <linux/prefetch.h>
35 #include "registers.h"
38 #include "../dmaengine.h"
41 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
43 * @data: interrupt data
45 irqreturn_t
ioat_dma_do_interrupt(int irq
, void *data
)
47 struct ioatdma_device
*instance
= data
;
48 struct ioatdma_chan
*ioat_chan
;
49 unsigned long attnstatus
;
53 intrctrl
= readb(instance
->reg_base
+ IOAT_INTRCTRL_OFFSET
);
55 if (!(intrctrl
& IOAT_INTRCTRL_MASTER_INT_EN
))
58 if (!(intrctrl
& IOAT_INTRCTRL_INT_STATUS
)) {
59 writeb(intrctrl
, instance
->reg_base
+ IOAT_INTRCTRL_OFFSET
);
63 attnstatus
= readl(instance
->reg_base
+ IOAT_ATTNSTATUS_OFFSET
);
64 for_each_set_bit(bit
, &attnstatus
, BITS_PER_LONG
) {
65 ioat_chan
= ioat_chan_by_index(instance
, bit
);
66 if (test_bit(IOAT_RUN
, &ioat_chan
->state
))
67 tasklet_schedule(&ioat_chan
->cleanup_task
);
70 writeb(intrctrl
, instance
->reg_base
+ IOAT_INTRCTRL_OFFSET
);
75 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
77 * @data: interrupt data
79 irqreturn_t
ioat_dma_do_interrupt_msix(int irq
, void *data
)
81 struct ioatdma_chan
*ioat_chan
= data
;
83 if (test_bit(IOAT_RUN
, &ioat_chan
->state
))
84 tasklet_schedule(&ioat_chan
->cleanup_task
);
89 void ioat_stop(struct ioatdma_chan
*ioat_chan
)
91 struct ioatdma_device
*ioat_dma
= ioat_chan
->ioat_dma
;
92 struct pci_dev
*pdev
= ioat_dma
->pdev
;
93 int chan_id
= chan_num(ioat_chan
);
94 struct msix_entry
*msix
;
96 /* 1/ stop irq from firing tasklets
97 * 2/ stop the tasklet from re-arming irqs
99 clear_bit(IOAT_RUN
, &ioat_chan
->state
);
101 /* flush inflight interrupts */
102 switch (ioat_dma
->irq_mode
) {
104 msix
= &ioat_dma
->msix_entries
[chan_id
];
105 synchronize_irq(msix
->vector
);
109 synchronize_irq(pdev
->irq
);
115 /* flush inflight timers */
116 del_timer_sync(&ioat_chan
->timer
);
118 /* flush inflight tasklet runs */
119 tasklet_kill(&ioat_chan
->cleanup_task
);
121 /* final cleanup now that everything is quiesced and can't re-arm */
122 ioat_dma
->cleanup_fn((unsigned long)&ioat_chan
->dma_chan
);
125 dma_addr_t
ioat_get_current_completion(struct ioatdma_chan
*ioat_chan
)
127 dma_addr_t phys_complete
;
130 completion
= *ioat_chan
->completion
;
131 phys_complete
= ioat_chansts_to_addr(completion
);
133 dev_dbg(to_dev(ioat_chan
), "%s: phys_complete: %#llx\n", __func__
,
134 (unsigned long long) phys_complete
);
136 if (is_ioat_halted(completion
)) {
137 u32 chanerr
= readl(ioat_chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
139 dev_err(to_dev(ioat_chan
), "Channel halted, chanerr = %x\n",
142 /* TODO do something to salvage the situation */
145 return phys_complete
;
148 bool ioat_cleanup_preamble(struct ioatdma_chan
*ioat_chan
,
149 dma_addr_t
*phys_complete
)
151 *phys_complete
= ioat_get_current_completion(ioat_chan
);
152 if (*phys_complete
== ioat_chan
->last_completion
)
154 clear_bit(IOAT_COMPLETION_ACK
, &ioat_chan
->state
);
155 mod_timer(&ioat_chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
161 ioat_dma_tx_status(struct dma_chan
*c
, dma_cookie_t cookie
,
162 struct dma_tx_state
*txstate
)
164 struct ioatdma_chan
*ioat_chan
= to_ioat_chan(c
);
165 struct ioatdma_device
*ioat_dma
= ioat_chan
->ioat_dma
;
168 ret
= dma_cookie_status(c
, cookie
, txstate
);
169 if (ret
== DMA_COMPLETE
)
172 ioat_dma
->cleanup_fn((unsigned long) c
);
174 return dma_cookie_status(c
, cookie
, txstate
);
177 void __ioat_issue_pending(struct ioatdma_chan
*ioat_chan
)
179 ioat_chan
->dmacount
+= ioat_ring_pending(ioat_chan
);
180 ioat_chan
->issued
= ioat_chan
->head
;
181 writew(ioat_chan
->dmacount
,
182 ioat_chan
->reg_base
+ IOAT_CHAN_DMACOUNT_OFFSET
);
183 dev_dbg(to_dev(ioat_chan
),
184 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
185 __func__
, ioat_chan
->head
, ioat_chan
->tail
,
186 ioat_chan
->issued
, ioat_chan
->dmacount
);
189 void ioat_issue_pending(struct dma_chan
*c
)
191 struct ioatdma_chan
*ioat_chan
= to_ioat_chan(c
);
193 if (ioat_ring_pending(ioat_chan
)) {
194 spin_lock_bh(&ioat_chan
->prep_lock
);
195 __ioat_issue_pending(ioat_chan
);
196 spin_unlock_bh(&ioat_chan
->prep_lock
);
201 * ioat_update_pending - log pending descriptors
202 * @ioat: ioat+ channel
204 * Check if the number of unsubmitted descriptors has exceeded the
205 * watermark. Called with prep_lock held
207 static void ioat_update_pending(struct ioatdma_chan
*ioat_chan
)
209 if (ioat_ring_pending(ioat_chan
) > ioat_pending_level
)
210 __ioat_issue_pending(ioat_chan
);
213 static void __ioat_start_null_desc(struct ioatdma_chan
*ioat_chan
)
215 struct ioat_ring_ent
*desc
;
216 struct ioat_dma_descriptor
*hw
;
218 if (ioat_ring_space(ioat_chan
) < 1) {
219 dev_err(to_dev(ioat_chan
),
220 "Unable to start null desc - ring full\n");
224 dev_dbg(to_dev(ioat_chan
),
225 "%s: head: %#x tail: %#x issued: %#x\n",
226 __func__
, ioat_chan
->head
, ioat_chan
->tail
, ioat_chan
->issued
);
227 desc
= ioat_get_ring_ent(ioat_chan
, ioat_chan
->head
);
232 hw
->ctl_f
.int_en
= 1;
233 hw
->ctl_f
.compl_write
= 1;
234 /* set size to non-zero value (channel returns error when size is 0) */
235 hw
->size
= NULL_DESC_BUFFER_SIZE
;
238 async_tx_ack(&desc
->txd
);
239 ioat_set_chainaddr(ioat_chan
, desc
->txd
.phys
);
240 dump_desc_dbg(ioat_chan
, desc
);
241 /* make sure descriptors are written before we submit */
243 ioat_chan
->head
+= 1;
244 __ioat_issue_pending(ioat_chan
);
247 void ioat_start_null_desc(struct ioatdma_chan
*ioat_chan
)
249 spin_lock_bh(&ioat_chan
->prep_lock
);
250 __ioat_start_null_desc(ioat_chan
);
251 spin_unlock_bh(&ioat_chan
->prep_lock
);
254 void __ioat_restart_chan(struct ioatdma_chan
*ioat_chan
)
256 /* set the tail to be re-issued */
257 ioat_chan
->issued
= ioat_chan
->tail
;
258 ioat_chan
->dmacount
= 0;
259 set_bit(IOAT_COMPLETION_PENDING
, &ioat_chan
->state
);
260 mod_timer(&ioat_chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
262 dev_dbg(to_dev(ioat_chan
),
263 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
264 __func__
, ioat_chan
->head
, ioat_chan
->tail
,
265 ioat_chan
->issued
, ioat_chan
->dmacount
);
267 if (ioat_ring_pending(ioat_chan
)) {
268 struct ioat_ring_ent
*desc
;
270 desc
= ioat_get_ring_ent(ioat_chan
, ioat_chan
->tail
);
271 ioat_set_chainaddr(ioat_chan
, desc
->txd
.phys
);
272 __ioat_issue_pending(ioat_chan
);
274 __ioat_start_null_desc(ioat_chan
);
277 int ioat_quiesce(struct ioatdma_chan
*ioat_chan
, unsigned long tmo
)
279 unsigned long end
= jiffies
+ tmo
;
283 status
= ioat_chansts(ioat_chan
);
284 if (is_ioat_active(status
) || is_ioat_idle(status
))
285 ioat_suspend(ioat_chan
);
286 while (is_ioat_active(status
) || is_ioat_idle(status
)) {
287 if (tmo
&& time_after(jiffies
, end
)) {
291 status
= ioat_chansts(ioat_chan
);
298 int ioat_reset_sync(struct ioatdma_chan
*ioat_chan
, unsigned long tmo
)
300 unsigned long end
= jiffies
+ tmo
;
303 ioat_reset(ioat_chan
);
304 while (ioat_reset_pending(ioat_chan
)) {
305 if (end
&& time_after(jiffies
, end
)) {
315 static dma_cookie_t
ioat_tx_submit_unlock(struct dma_async_tx_descriptor
*tx
)
317 struct dma_chan
*c
= tx
->chan
;
318 struct ioatdma_chan
*ioat_chan
= to_ioat_chan(c
);
321 cookie
= dma_cookie_assign(tx
);
322 dev_dbg(to_dev(ioat_chan
), "%s: cookie: %d\n", __func__
, cookie
);
324 if (!test_and_set_bit(IOAT_CHAN_ACTIVE
, &ioat_chan
->state
))
325 mod_timer(&ioat_chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
327 /* make descriptor updates visible before advancing ioat->head,
328 * this is purposefully not smp_wmb() since we are also
329 * publishing the descriptor updates to a dma device
333 ioat_chan
->head
+= ioat_chan
->produce
;
335 ioat_update_pending(ioat_chan
);
336 spin_unlock_bh(&ioat_chan
->prep_lock
);
341 static struct ioat_ring_ent
*
342 ioat_alloc_ring_ent(struct dma_chan
*chan
, gfp_t flags
)
344 struct ioat_dma_descriptor
*hw
;
345 struct ioat_ring_ent
*desc
;
346 struct ioatdma_device
*ioat_dma
;
349 ioat_dma
= to_ioatdma_device(chan
->device
);
350 hw
= pci_pool_alloc(ioat_dma
->dma_pool
, flags
, &phys
);
353 memset(hw
, 0, sizeof(*hw
));
355 desc
= kmem_cache_zalloc(ioat_cache
, flags
);
357 pci_pool_free(ioat_dma
->dma_pool
, hw
, phys
);
361 dma_async_tx_descriptor_init(&desc
->txd
, chan
);
362 desc
->txd
.tx_submit
= ioat_tx_submit_unlock
;
364 desc
->txd
.phys
= phys
;
368 void ioat_free_ring_ent(struct ioat_ring_ent
*desc
, struct dma_chan
*chan
)
370 struct ioatdma_device
*ioat_dma
;
372 ioat_dma
= to_ioatdma_device(chan
->device
);
373 pci_pool_free(ioat_dma
->dma_pool
, desc
->hw
, desc
->txd
.phys
);
374 kmem_cache_free(ioat_cache
, desc
);
377 struct ioat_ring_ent
**
378 ioat_alloc_ring(struct dma_chan
*c
, int order
, gfp_t flags
)
380 struct ioat_ring_ent
**ring
;
381 int descs
= 1 << order
;
384 if (order
> ioat_get_max_alloc_order())
387 /* allocate the array to hold the software ring */
388 ring
= kcalloc(descs
, sizeof(*ring
), flags
);
391 for (i
= 0; i
< descs
; i
++) {
392 ring
[i
] = ioat_alloc_ring_ent(c
, flags
);
395 ioat_free_ring_ent(ring
[i
], c
);
399 set_desc_id(ring
[i
], i
);
403 for (i
= 0; i
< descs
-1; i
++) {
404 struct ioat_ring_ent
*next
= ring
[i
+1];
405 struct ioat_dma_descriptor
*hw
= ring
[i
]->hw
;
407 hw
->next
= next
->txd
.phys
;
409 ring
[i
]->hw
->next
= ring
[0]->txd
.phys
;
414 bool reshape_ring(struct ioatdma_chan
*ioat_chan
, int order
)
416 /* reshape differs from normal ring allocation in that we want
417 * to allocate a new software ring while only
418 * extending/truncating the hardware ring
420 struct dma_chan
*c
= &ioat_chan
->dma_chan
;
421 const u32 curr_size
= ioat_ring_size(ioat_chan
);
422 const u16 active
= ioat_ring_active(ioat_chan
);
423 const u32 new_size
= 1 << order
;
424 struct ioat_ring_ent
**ring
;
427 if (order
> ioat_get_max_alloc_order())
430 /* double check that we have at least 1 free descriptor */
431 if (active
== curr_size
)
434 /* when shrinking, verify that we can hold the current active
435 * set in the new ring
437 if (active
>= new_size
)
440 /* allocate the array to hold the software ring */
441 ring
= kcalloc(new_size
, sizeof(*ring
), GFP_NOWAIT
);
445 /* allocate/trim descriptors as needed */
446 if (new_size
> curr_size
) {
447 /* copy current descriptors to the new ring */
448 for (i
= 0; i
< curr_size
; i
++) {
449 u16 curr_idx
= (ioat_chan
->tail
+i
) & (curr_size
-1);
450 u16 new_idx
= (ioat_chan
->tail
+i
) & (new_size
-1);
452 ring
[new_idx
] = ioat_chan
->ring
[curr_idx
];
453 set_desc_id(ring
[new_idx
], new_idx
);
456 /* add new descriptors to the ring */
457 for (i
= curr_size
; i
< new_size
; i
++) {
458 u16 new_idx
= (ioat_chan
->tail
+i
) & (new_size
-1);
460 ring
[new_idx
] = ioat_alloc_ring_ent(c
, GFP_NOWAIT
);
461 if (!ring
[new_idx
]) {
463 u16 new_idx
= (ioat_chan
->tail
+i
) &
466 ioat_free_ring_ent(ring
[new_idx
], c
);
471 set_desc_id(ring
[new_idx
], new_idx
);
474 /* hw link new descriptors */
475 for (i
= curr_size
-1; i
< new_size
; i
++) {
476 u16 new_idx
= (ioat_chan
->tail
+i
) & (new_size
-1);
477 struct ioat_ring_ent
*next
=
478 ring
[(new_idx
+1) & (new_size
-1)];
479 struct ioat_dma_descriptor
*hw
= ring
[new_idx
]->hw
;
481 hw
->next
= next
->txd
.phys
;
484 struct ioat_dma_descriptor
*hw
;
485 struct ioat_ring_ent
*next
;
487 /* copy current descriptors to the new ring, dropping the
488 * removed descriptors
490 for (i
= 0; i
< new_size
; i
++) {
491 u16 curr_idx
= (ioat_chan
->tail
+i
) & (curr_size
-1);
492 u16 new_idx
= (ioat_chan
->tail
+i
) & (new_size
-1);
494 ring
[new_idx
] = ioat_chan
->ring
[curr_idx
];
495 set_desc_id(ring
[new_idx
], new_idx
);
498 /* free deleted descriptors */
499 for (i
= new_size
; i
< curr_size
; i
++) {
500 struct ioat_ring_ent
*ent
;
502 ent
= ioat_get_ring_ent(ioat_chan
, ioat_chan
->tail
+i
);
503 ioat_free_ring_ent(ent
, c
);
506 /* fix up hardware ring */
507 hw
= ring
[(ioat_chan
->tail
+new_size
-1) & (new_size
-1)]->hw
;
508 next
= ring
[(ioat_chan
->tail
+new_size
) & (new_size
-1)];
509 hw
->next
= next
->txd
.phys
;
512 dev_dbg(to_dev(ioat_chan
), "%s: allocated %d descriptors\n",
515 kfree(ioat_chan
->ring
);
516 ioat_chan
->ring
= ring
;
517 ioat_chan
->alloc_order
= order
;
523 * ioat_check_space_lock - verify space and grab ring producer lock
524 * @ioat: ioat,3 channel (ring) to operate on
525 * @num_descs: allocation length
527 int ioat_check_space_lock(struct ioatdma_chan
*ioat_chan
, int num_descs
)
532 spin_lock_bh(&ioat_chan
->prep_lock
);
533 /* never allow the last descriptor to be consumed, we need at
534 * least one free at all times to allow for on-the-fly ring
537 if (likely(ioat_ring_space(ioat_chan
) > num_descs
)) {
538 dev_dbg(to_dev(ioat_chan
), "%s: num_descs: %d (%x:%x:%x)\n",
539 __func__
, num_descs
, ioat_chan
->head
,
540 ioat_chan
->tail
, ioat_chan
->issued
);
541 ioat_chan
->produce
= num_descs
;
542 return 0; /* with ioat->prep_lock held */
544 retry
= test_and_set_bit(IOAT_RESHAPE_PENDING
, &ioat_chan
->state
);
545 spin_unlock_bh(&ioat_chan
->prep_lock
);
547 /* is another cpu already trying to expand the ring? */
551 spin_lock_bh(&ioat_chan
->cleanup_lock
);
552 spin_lock_bh(&ioat_chan
->prep_lock
);
553 retry
= reshape_ring(ioat_chan
, ioat_chan
->alloc_order
+ 1);
554 clear_bit(IOAT_RESHAPE_PENDING
, &ioat_chan
->state
);
555 spin_unlock_bh(&ioat_chan
->prep_lock
);
556 spin_unlock_bh(&ioat_chan
->cleanup_lock
);
558 /* if we were able to expand the ring retry the allocation */
562 dev_dbg_ratelimited(to_dev(ioat_chan
),
563 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
564 __func__
, num_descs
, ioat_chan
->head
,
565 ioat_chan
->tail
, ioat_chan
->issued
);
567 /* progress reclaim in the allocation failure case we may be
568 * called under bh_disabled so we need to trigger the timer
571 if (time_is_before_jiffies(ioat_chan
->timer
.expires
)
572 && timer_pending(&ioat_chan
->timer
)) {
573 struct ioatdma_device
*ioat_dma
= ioat_chan
->ioat_dma
;
575 mod_timer(&ioat_chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
576 ioat_dma
->timer_fn((unsigned long)ioat_chan
);
582 struct dma_async_tx_descriptor
*
583 ioat_dma_prep_memcpy_lock(struct dma_chan
*c
, dma_addr_t dma_dest
,
584 dma_addr_t dma_src
, size_t len
, unsigned long flags
)
586 struct ioatdma_chan
*ioat_chan
= to_ioat_chan(c
);
587 struct ioat_dma_descriptor
*hw
;
588 struct ioat_ring_ent
*desc
;
589 dma_addr_t dst
= dma_dest
;
590 dma_addr_t src
= dma_src
;
591 size_t total_len
= len
;
592 int num_descs
, idx
, i
;
594 num_descs
= ioat_xferlen_to_descs(ioat_chan
, len
);
595 if (likely(num_descs
) &&
596 ioat_check_space_lock(ioat_chan
, num_descs
) == 0)
597 idx
= ioat_chan
->head
;
602 size_t copy
= min_t(size_t, len
, 1 << ioat_chan
->xfercap_log
);
604 desc
= ioat_get_ring_ent(ioat_chan
, idx
+ i
);
615 dump_desc_dbg(ioat_chan
, desc
);
616 } while (++i
< num_descs
);
618 desc
->txd
.flags
= flags
;
619 desc
->len
= total_len
;
620 hw
->ctl_f
.int_en
= !!(flags
& DMA_PREP_INTERRUPT
);
621 hw
->ctl_f
.fence
= !!(flags
& DMA_PREP_FENCE
);
622 hw
->ctl_f
.compl_write
= 1;
623 dump_desc_dbg(ioat_chan
, desc
);
624 /* we leave the channel locked to ensure in order submission */