2 * xor offload engine api
4 * Copyright © 2006, Intel Corporation.
6 * Dan Williams <dan.j.williams@intel.com>
8 * with architecture considerations by:
9 * Neil Brown <neilb@suse.de>
10 * Jeff Garzik <jeff@garzik.org>
12 * This program is free software; you can redistribute it and/or modify it
13 * under the terms and conditions of the GNU General Public License,
14 * version 2, as published by the Free Software Foundation.
16 * This program is distributed in the hope it will be useful, but WITHOUT
17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
21 * You should have received a copy of the GNU General Public License along with
22 * this program; if not, write to the Free Software Foundation, Inc.,
23 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
26 #include <linux/kernel.h>
27 #include <linux/interrupt.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/raid/xor.h>
31 #include <linux/async_tx.h>
33 /* do_async_xor - dma map the pages and perform the xor with an engine.
34 * This routine is marked __always_inline so it can be compiled away
35 * when CONFIG_DMA_ENGINE=n
37 static __always_inline
void
38 do_async_xor(struct dma_async_tx_descriptor
*tx
, struct dma_device
*device
,
39 struct dma_chan
*chan
, struct page
*dest
, struct page
**src_list
,
40 unsigned int offset
, unsigned int src_cnt
, size_t len
,
41 enum async_tx_flags flags
, struct dma_async_tx_descriptor
*depend_tx
,
42 dma_async_tx_callback cb_fn
, void *cb_param
)
45 enum dma_data_direction dir
;
48 pr_debug("%s: len: %zu\n", __FUNCTION__
, len
);
50 dir
= (flags
& ASYNC_TX_ASSUME_COHERENT
) ?
51 DMA_NONE
: DMA_FROM_DEVICE
;
53 dma_addr
= dma_map_page(device
->dev
, dest
, offset
, len
, dir
);
54 tx
->tx_set_dest(dma_addr
, tx
, 0);
56 dir
= (flags
& ASYNC_TX_ASSUME_COHERENT
) ?
57 DMA_NONE
: DMA_TO_DEVICE
;
59 for (i
= 0; i
< src_cnt
; i
++) {
60 dma_addr
= dma_map_page(device
->dev
, src_list
[i
],
62 tx
->tx_set_src(dma_addr
, tx
, i
);
65 async_tx_submit(chan
, tx
, flags
, depend_tx
, cb_fn
, cb_param
);
69 do_sync_xor(struct page
*dest
, struct page
**src_list
, unsigned int offset
,
70 unsigned int src_cnt
, size_t len
, enum async_tx_flags flags
,
71 struct dma_async_tx_descriptor
*depend_tx
,
72 dma_async_tx_callback cb_fn
, void *cb_param
)
77 pr_debug("%s: len: %zu\n", __FUNCTION__
, len
);
79 /* reuse the 'src_list' array to convert to buffer pointers */
80 for (i
= 0; i
< src_cnt
; i
++)
81 src_list
[i
] = (struct page
*)
82 (page_address(src_list
[i
]) + offset
);
84 /* set destination address */
85 _dest
= page_address(dest
) + offset
;
87 if (flags
& ASYNC_TX_XOR_ZERO_DST
)
88 memset(_dest
, 0, len
);
90 xor_blocks(src_cnt
, len
, _dest
,
93 async_tx_sync_epilog(flags
, depend_tx
, cb_fn
, cb_param
);
97 * async_xor - attempt to xor a set of blocks with a dma engine.
98 * xor_blocks always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST
99 * flag must be set to not include dest data in the calculation. The
100 * assumption with dma eninges is that they only use the destination
101 * buffer as a source when it is explicity specified in the source list.
102 * @dest: destination page
103 * @src_list: array of source pages (if the dest is also a source it must be
104 * at index zero). The contents of this array may be overwritten.
105 * @offset: offset in pages to start transaction
106 * @src_cnt: number of source pages
107 * @len: length in bytes
108 * @flags: ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DEST,
109 * ASYNC_TX_ASSUME_COHERENT, ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
110 * @depend_tx: xor depends on the result of this transaction.
111 * @cb_fn: function to call when the xor completes
112 * @cb_param: parameter to pass to the callback routine
114 struct dma_async_tx_descriptor
*
115 async_xor(struct page
*dest
, struct page
**src_list
, unsigned int offset
,
116 int src_cnt
, size_t len
, enum async_tx_flags flags
,
117 struct dma_async_tx_descriptor
*depend_tx
,
118 dma_async_tx_callback cb_fn
, void *cb_param
)
120 struct dma_chan
*chan
= async_tx_find_channel(depend_tx
, DMA_XOR
);
121 struct dma_device
*device
= chan
? chan
->device
: NULL
;
122 struct dma_async_tx_descriptor
*tx
= NULL
;
123 dma_async_tx_callback _cb_fn
;
125 unsigned long local_flags
;
127 int i
= 0, src_off
= 0, int_en
;
129 BUG_ON(src_cnt
<= 1);
133 if (device
) { /* run the xor asynchronously */
134 xor_src_cnt
= min(src_cnt
, device
->max_xor
);
135 /* if we are submitting additional xors
136 * only set the callback on the last transaction
138 if (src_cnt
> xor_src_cnt
) {
139 local_flags
&= ~ASYNC_TX_ACK
;
144 _cb_param
= cb_param
;
147 int_en
= _cb_fn
? 1 : 0;
149 tx
= device
->device_prep_dma_xor(
150 chan
, xor_src_cnt
, len
, int_en
);
153 do_async_xor(tx
, device
, chan
, dest
,
154 &src_list
[src_off
], offset
, xor_src_cnt
, len
,
155 local_flags
, depend_tx
, _cb_fn
,
157 } else /* fall through */
159 } else { /* run the xor synchronously */
161 /* in the sync case the dest is an implied source
162 * (assumes the dest is at the src_off index)
164 if (flags
& ASYNC_TX_XOR_DROP_DST
) {
169 /* process up to 'MAX_XOR_BLOCKS' sources */
170 xor_src_cnt
= min(src_cnt
, MAX_XOR_BLOCKS
);
172 /* if we are submitting additional xors
173 * only set the callback on the last transaction
175 if (src_cnt
> xor_src_cnt
) {
176 local_flags
&= ~ASYNC_TX_ACK
;
181 _cb_param
= cb_param
;
184 /* wait for any prerequisite operations */
186 /* if ack is already set then we cannot be sure
187 * we are referring to the correct operation
189 BUG_ON(depend_tx
->ack
);
190 if (dma_wait_for_async_tx(depend_tx
) ==
192 panic("%s: DMA_ERROR waiting for "
197 do_sync_xor(dest
, &src_list
[src_off
], offset
,
198 xor_src_cnt
, len
, local_flags
, depend_tx
,
202 /* the previous tx is hidden from the client,
206 async_tx_ack(depend_tx
);
210 if (src_cnt
> xor_src_cnt
) {
211 /* drop completed sources */
212 src_cnt
-= xor_src_cnt
;
213 src_off
+= xor_src_cnt
;
215 /* unconditionally preserve the destination */
216 flags
&= ~ASYNC_TX_XOR_ZERO_DST
;
218 /* use the intermediate result a source, but remember
219 * it's dropped, because it's implied, in the sync case
221 src_list
[--src_off
] = dest
;
223 flags
|= ASYNC_TX_XOR_DROP_DST
;
231 EXPORT_SYMBOL_GPL(async_xor
);
233 static int page_is_zero(struct page
*p
, unsigned int offset
, size_t len
)
235 char *a
= page_address(p
) + offset
;
236 return ((*(u32
*) a
) == 0 &&
237 memcmp(a
, a
+ 4, len
- 4) == 0);
241 * async_xor_zero_sum - attempt a xor parity check with a dma engine.
242 * @dest: destination page used if the xor is performed synchronously
243 * @src_list: array of source pages. The dest page must be listed as a source
244 * at index zero. The contents of this array may be overwritten.
245 * @offset: offset in pages to start transaction
246 * @src_cnt: number of source pages
247 * @len: length in bytes
248 * @result: 0 if sum == 0 else non-zero
249 * @flags: ASYNC_TX_ASSUME_COHERENT, ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
250 * @depend_tx: xor depends on the result of this transaction.
251 * @cb_fn: function to call when the xor completes
252 * @cb_param: parameter to pass to the callback routine
254 struct dma_async_tx_descriptor
*
255 async_xor_zero_sum(struct page
*dest
, struct page
**src_list
,
256 unsigned int offset
, int src_cnt
, size_t len
,
257 u32
*result
, enum async_tx_flags flags
,
258 struct dma_async_tx_descriptor
*depend_tx
,
259 dma_async_tx_callback cb_fn
, void *cb_param
)
261 struct dma_chan
*chan
= async_tx_find_channel(depend_tx
, DMA_ZERO_SUM
);
262 struct dma_device
*device
= chan
? chan
->device
: NULL
;
263 int int_en
= cb_fn
? 1 : 0;
264 struct dma_async_tx_descriptor
*tx
= device
?
265 device
->device_prep_dma_zero_sum(chan
, src_cnt
, len
, result
,
269 BUG_ON(src_cnt
<= 1);
273 enum dma_data_direction dir
;
275 pr_debug("%s: (async) len: %zu\n", __FUNCTION__
, len
);
277 dir
= (flags
& ASYNC_TX_ASSUME_COHERENT
) ?
278 DMA_NONE
: DMA_TO_DEVICE
;
280 for (i
= 0; i
< src_cnt
; i
++) {
281 dma_addr
= dma_map_page(device
->dev
, src_list
[i
],
283 tx
->tx_set_src(dma_addr
, tx
, i
);
286 async_tx_submit(chan
, tx
, flags
, depend_tx
, cb_fn
, cb_param
);
288 unsigned long xor_flags
= flags
;
290 pr_debug("%s: (sync) len: %zu\n", __FUNCTION__
, len
);
292 xor_flags
|= ASYNC_TX_XOR_DROP_DST
;
293 xor_flags
&= ~ASYNC_TX_ACK
;
295 tx
= async_xor(dest
, src_list
, offset
, src_cnt
, len
, xor_flags
,
296 depend_tx
, NULL
, NULL
);
299 if (dma_wait_for_async_tx(tx
) == DMA_ERROR
)
300 panic("%s: DMA_ERROR waiting for tx\n",
305 *result
= page_is_zero(dest
, offset
, len
) ? 0 : 1;
309 async_tx_sync_epilog(flags
, depend_tx
, cb_fn
, cb_param
);
314 EXPORT_SYMBOL_GPL(async_xor_zero_sum
);
316 static int __init
async_xor_init(void)
321 static void __exit
async_xor_exit(void)
326 module_init(async_xor_init
);
327 module_exit(async_xor_exit
);
329 MODULE_AUTHOR("Intel Corporation");
330 MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api");
331 MODULE_LICENSE("GPL");