2 * bio-integrity.c - bio data integrity extensions
4 * Copyright (C) 2007, 2008, 2009 Oracle Corporation
5 * Written by: Martin K. Petersen <martin.petersen@oracle.com>
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License version
9 * 2 as published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; see the file COPYING. If not, write to
18 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
23 #include <linux/blkdev.h>
24 #include <linux/mempool.h>
25 #include <linux/export.h>
26 #include <linux/bio.h>
27 #include <linux/workqueue.h>
28 #include <linux/slab.h>
30 #define BIP_INLINE_VECS 4
32 static struct kmem_cache
*bip_slab
;
33 static struct workqueue_struct
*kintegrityd_wq
;
36 * bio_integrity_alloc - Allocate integrity payload and attach it to bio
37 * @bio: bio to attach integrity metadata to
38 * @gfp_mask: Memory allocation mask
39 * @nr_vecs: Number of integrity metadata scatter-gather elements
41 * Description: This function prepares a bio for attaching integrity
42 * metadata. nr_vecs specifies the maximum number of pages containing
43 * integrity metadata that can be attached.
45 struct bio_integrity_payload
*bio_integrity_alloc(struct bio
*bio
,
49 struct bio_integrity_payload
*bip
;
50 struct bio_set
*bs
= bio
->bi_pool
;
51 unsigned long idx
= BIO_POOL_NONE
;
55 bip
= kmalloc(sizeof(struct bio_integrity_payload
) +
56 sizeof(struct bio_vec
) * nr_vecs
, gfp_mask
);
57 inline_vecs
= nr_vecs
;
59 bip
= mempool_alloc(bs
->bio_integrity_pool
, gfp_mask
);
60 inline_vecs
= BIP_INLINE_VECS
;
66 memset(bip
, 0, sizeof(*bip
));
68 if (nr_vecs
> inline_vecs
) {
69 bip
->bip_vec
= bvec_alloc(gfp_mask
, nr_vecs
, &idx
,
70 bs
->bvec_integrity_pool
);
73 bip
->bip_max_vcnt
= bvec_nr_vecs(idx
);
75 bip
->bip_vec
= bip
->bip_inline_vecs
;
76 bip
->bip_max_vcnt
= inline_vecs
;
81 bio
->bi_integrity
= bip
;
82 bio
->bi_rw
|= REQ_INTEGRITY
;
86 mempool_free(bip
, bs
->bio_integrity_pool
);
89 EXPORT_SYMBOL(bio_integrity_alloc
);
92 * bio_integrity_free - Free bio integrity payload
93 * @bio: bio containing bip to be freed
95 * Description: Used to free the integrity portion of a bio. Usually
96 * called from bio_free().
98 void bio_integrity_free(struct bio
*bio
)
100 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
101 struct bio_set
*bs
= bio
->bi_pool
;
103 if (bip
->bip_owns_buf
)
104 kfree(page_address(bip
->bip_vec
->bv_page
) +
105 bip
->bip_vec
->bv_offset
);
108 if (bip
->bip_slab
!= BIO_POOL_NONE
)
109 bvec_free(bs
->bvec_integrity_pool
, bip
->bip_vec
,
112 mempool_free(bip
, bs
->bio_integrity_pool
);
117 bio
->bi_integrity
= NULL
;
119 EXPORT_SYMBOL(bio_integrity_free
);
122 * bio_integrity_add_page - Attach integrity metadata
123 * @bio: bio to update
124 * @page: page containing integrity metadata
125 * @len: number of bytes of integrity metadata in page
126 * @offset: start offset within page
128 * Description: Attach a page containing integrity metadata to bio.
130 int bio_integrity_add_page(struct bio
*bio
, struct page
*page
,
131 unsigned int len
, unsigned int offset
)
133 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
136 if (bip
->bip_vcnt
>= bip
->bip_max_vcnt
) {
137 printk(KERN_ERR
"%s: bip_vec full\n", __func__
);
141 iv
= bip
->bip_vec
+ bip
->bip_vcnt
;
145 iv
->bv_offset
= offset
;
150 EXPORT_SYMBOL(bio_integrity_add_page
);
153 * bio_integrity_enabled - Check whether integrity can be passed
156 * Description: Determines whether bio_integrity_prep() can be called
157 * on this bio or not. bio data direction and target device must be
158 * set prior to calling. The functions honors the write_generate and
159 * read_verify flags in sysfs.
161 bool bio_integrity_enabled(struct bio
*bio
)
163 struct blk_integrity
*bi
= bdev_get_integrity(bio
->bi_bdev
);
168 /* Already protected? */
169 if (bio_integrity(bio
))
175 if (bio_data_dir(bio
) == READ
&& bi
->verify_fn
!= NULL
&&
176 (bi
->flags
& INTEGRITY_FLAG_READ
))
179 if (bio_data_dir(bio
) == WRITE
&& bi
->generate_fn
!= NULL
&&
180 (bi
->flags
& INTEGRITY_FLAG_WRITE
))
185 EXPORT_SYMBOL(bio_integrity_enabled
);
188 * bio_integrity_hw_sectors - Convert 512b sectors to hardware ditto
189 * @bi: blk_integrity profile for device
190 * @sectors: Number of 512 sectors to convert
192 * Description: The block layer calculates everything in 512 byte
193 * sectors but integrity metadata is done in terms of the hardware
194 * sector size of the storage device. Convert the block layer sectors
195 * to physical sectors.
197 static inline unsigned int bio_integrity_hw_sectors(struct blk_integrity
*bi
,
198 unsigned int sectors
)
200 /* At this point there are only 512b or 4096b DIF/EPP devices */
201 if (bi
->sector_size
== 4096)
202 return sectors
>>= 3;
207 static inline unsigned int bio_integrity_bytes(struct blk_integrity
*bi
,
208 unsigned int sectors
)
210 return bio_integrity_hw_sectors(bi
, sectors
) * bi
->tuple_size
;
214 * bio_integrity_generate_verify - Generate/verify integrity metadata for a bio
215 * @bio: bio to generate/verify integrity metadata for
216 * @operate: operate number, 1 for generate, 0 for verify
218 static int bio_integrity_generate_verify(struct bio
*bio
, int operate
)
220 struct blk_integrity
*bi
= bdev_get_integrity(bio
->bi_bdev
);
221 struct blk_integrity_exchg bix
;
223 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
225 unsigned int sectors
, ret
= 0, i
;
226 void *prot_buf
= page_address(bip
->bip_vec
->bv_page
) +
227 bip
->bip_vec
->bv_offset
;
230 sector
= bio
->bi_iter
.bi_sector
;
232 sector
= bip
->bip_iter
.bi_sector
;
234 bix
.disk_name
= bio
->bi_bdev
->bd_disk
->disk_name
;
235 bix
.sector_size
= bi
->sector_size
;
237 bio_for_each_segment_all(bv
, bio
, i
) {
238 void *kaddr
= kmap_atomic(bv
->bv_page
);
239 bix
.data_buf
= kaddr
+ bv
->bv_offset
;
240 bix
.data_size
= bv
->bv_len
;
241 bix
.prot_buf
= prot_buf
;
245 bi
->generate_fn(&bix
);
247 ret
= bi
->verify_fn(&bix
);
249 kunmap_atomic(kaddr
);
254 sectors
= bv
->bv_len
/ bi
->sector_size
;
256 prot_buf
+= sectors
* bi
->tuple_size
;
258 kunmap_atomic(kaddr
);
264 * bio_integrity_generate - Generate integrity metadata for a bio
265 * @bio: bio to generate integrity metadata for
267 * Description: Generates integrity metadata for a bio by calling the
268 * block device's generation callback function. The bio must have a
269 * bip attached with enough room to accommodate the generated
270 * integrity metadata.
272 static void bio_integrity_generate(struct bio
*bio
)
274 bio_integrity_generate_verify(bio
, 1);
278 * bio_integrity_prep - Prepare bio for integrity I/O
279 * @bio: bio to prepare
281 * Description: Allocates a buffer for integrity metadata, maps the
282 * pages and attaches them to a bio. The bio must have data
283 * direction, target device and start sector set priot to calling. In
284 * the WRITE case, integrity metadata will be generated using the
285 * block device's integrity function. In the READ case, the buffer
286 * will be prepared for DMA and a suitable end_io handler set up.
288 int bio_integrity_prep(struct bio
*bio
)
290 struct bio_integrity_payload
*bip
;
291 struct blk_integrity
*bi
;
292 struct request_queue
*q
;
294 unsigned long start
, end
;
295 unsigned int len
, nr_pages
;
296 unsigned int bytes
, offset
, i
;
297 unsigned int sectors
;
299 bi
= bdev_get_integrity(bio
->bi_bdev
);
300 q
= bdev_get_queue(bio
->bi_bdev
);
302 BUG_ON(bio_integrity(bio
));
304 sectors
= bio_integrity_hw_sectors(bi
, bio_sectors(bio
));
306 /* Allocate kernel buffer for protection data */
307 len
= sectors
* bi
->tuple_size
;
308 buf
= kmalloc(len
, GFP_NOIO
| q
->bounce_gfp
);
309 if (unlikely(buf
== NULL
)) {
310 printk(KERN_ERR
"could not allocate integrity buffer\n");
314 end
= (((unsigned long) buf
) + len
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
315 start
= ((unsigned long) buf
) >> PAGE_SHIFT
;
316 nr_pages
= end
- start
;
318 /* Allocate bio integrity payload and integrity vectors */
319 bip
= bio_integrity_alloc(bio
, GFP_NOIO
, nr_pages
);
320 if (unlikely(bip
== NULL
)) {
321 printk(KERN_ERR
"could not allocate data integrity bioset\n");
326 bip
->bip_owns_buf
= 1;
327 bip
->bip_iter
.bi_size
= len
;
328 bip
->bip_iter
.bi_sector
= bio
->bi_iter
.bi_sector
;
331 offset
= offset_in_page(buf
);
332 for (i
= 0 ; i
< nr_pages
; i
++) {
334 bytes
= PAGE_SIZE
- offset
;
342 ret
= bio_integrity_add_page(bio
, virt_to_page(buf
),
356 /* Install custom I/O completion handler if read verify is enabled */
357 if (bio_data_dir(bio
) == READ
) {
358 bip
->bip_end_io
= bio
->bi_end_io
;
359 bio
->bi_end_io
= bio_integrity_endio
;
362 /* Auto-generate integrity metadata if this is a write */
363 if (bio_data_dir(bio
) == WRITE
)
364 bio_integrity_generate(bio
);
368 EXPORT_SYMBOL(bio_integrity_prep
);
371 * bio_integrity_verify - Verify integrity metadata for a bio
372 * @bio: bio to verify
374 * Description: This function is called to verify the integrity of a
375 * bio. The data in the bio io_vec is compared to the integrity
376 * metadata returned by the HBA.
378 static int bio_integrity_verify(struct bio
*bio
)
380 return bio_integrity_generate_verify(bio
, 0);
384 * bio_integrity_verify_fn - Integrity I/O completion worker
385 * @work: Work struct stored in bio to be verified
387 * Description: This workqueue function is called to complete a READ
388 * request. The function verifies the transferred integrity metadata
389 * and then calls the original bio end_io function.
391 static void bio_integrity_verify_fn(struct work_struct
*work
)
393 struct bio_integrity_payload
*bip
=
394 container_of(work
, struct bio_integrity_payload
, bip_work
);
395 struct bio
*bio
= bip
->bip_bio
;
398 error
= bio_integrity_verify(bio
);
400 /* Restore original bio completion handler */
401 bio
->bi_end_io
= bip
->bip_end_io
;
402 bio_endio_nodec(bio
, error
);
406 * bio_integrity_endio - Integrity I/O completion function
407 * @bio: Protected bio
408 * @error: Pointer to errno
410 * Description: Completion for integrity I/O
412 * Normally I/O completion is done in interrupt context. However,
413 * verifying I/O integrity is a time-consuming task which must be run
414 * in process context. This function postpones completion
417 void bio_integrity_endio(struct bio
*bio
, int error
)
419 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
421 BUG_ON(bip
->bip_bio
!= bio
);
423 /* In case of an I/O error there is no point in verifying the
424 * integrity metadata. Restore original bio end_io handler
428 bio
->bi_end_io
= bip
->bip_end_io
;
429 bio_endio_nodec(bio
, error
);
434 INIT_WORK(&bip
->bip_work
, bio_integrity_verify_fn
);
435 queue_work(kintegrityd_wq
, &bip
->bip_work
);
437 EXPORT_SYMBOL(bio_integrity_endio
);
440 * bio_integrity_advance - Advance integrity vector
441 * @bio: bio whose integrity vector to update
442 * @bytes_done: number of data bytes that have been completed
444 * Description: This function calculates how many integrity bytes the
445 * number of completed data bytes correspond to and advances the
446 * integrity vector accordingly.
448 void bio_integrity_advance(struct bio
*bio
, unsigned int bytes_done
)
450 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
451 struct blk_integrity
*bi
= bdev_get_integrity(bio
->bi_bdev
);
452 unsigned bytes
= bio_integrity_bytes(bi
, bytes_done
>> 9);
454 bvec_iter_advance(bip
->bip_vec
, &bip
->bip_iter
, bytes
);
456 EXPORT_SYMBOL(bio_integrity_advance
);
459 * bio_integrity_trim - Trim integrity vector
460 * @bio: bio whose integrity vector to update
461 * @offset: offset to first data sector
462 * @sectors: number of data sectors
464 * Description: Used to trim the integrity vector in a cloned bio.
465 * The ivec will be advanced corresponding to 'offset' data sectors
466 * and the length will be truncated corresponding to 'len' data
469 void bio_integrity_trim(struct bio
*bio
, unsigned int offset
,
470 unsigned int sectors
)
472 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
473 struct blk_integrity
*bi
= bdev_get_integrity(bio
->bi_bdev
);
475 bio_integrity_advance(bio
, offset
<< 9);
476 bip
->bip_iter
.bi_size
= bio_integrity_bytes(bi
, sectors
);
478 EXPORT_SYMBOL(bio_integrity_trim
);
481 * bio_integrity_clone - Callback for cloning bios with integrity metadata
483 * @bio_src: Original bio
484 * @gfp_mask: Memory allocation mask
486 * Description: Called to allocate a bip when cloning a bio
488 int bio_integrity_clone(struct bio
*bio
, struct bio
*bio_src
,
491 struct bio_integrity_payload
*bip_src
= bio_integrity(bio_src
);
492 struct bio_integrity_payload
*bip
;
494 BUG_ON(bip_src
== NULL
);
496 bip
= bio_integrity_alloc(bio
, gfp_mask
, bip_src
->bip_vcnt
);
501 memcpy(bip
->bip_vec
, bip_src
->bip_vec
,
502 bip_src
->bip_vcnt
* sizeof(struct bio_vec
));
504 bip
->bip_vcnt
= bip_src
->bip_vcnt
;
505 bip
->bip_iter
= bip_src
->bip_iter
;
509 EXPORT_SYMBOL(bio_integrity_clone
);
511 int bioset_integrity_create(struct bio_set
*bs
, int pool_size
)
513 if (bs
->bio_integrity_pool
)
516 bs
->bio_integrity_pool
= mempool_create_slab_pool(pool_size
, bip_slab
);
517 if (!bs
->bio_integrity_pool
)
520 bs
->bvec_integrity_pool
= biovec_create_pool(pool_size
);
521 if (!bs
->bvec_integrity_pool
) {
522 mempool_destroy(bs
->bio_integrity_pool
);
528 EXPORT_SYMBOL(bioset_integrity_create
);
530 void bioset_integrity_free(struct bio_set
*bs
)
532 if (bs
->bio_integrity_pool
)
533 mempool_destroy(bs
->bio_integrity_pool
);
535 if (bs
->bvec_integrity_pool
)
536 mempool_destroy(bs
->bvec_integrity_pool
);
538 EXPORT_SYMBOL(bioset_integrity_free
);
540 void __init
bio_integrity_init(void)
543 * kintegrityd won't block much but may burn a lot of CPU cycles.
544 * Make it highpri CPU intensive wq with max concurrency of 1.
546 kintegrityd_wq
= alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM
|
547 WQ_HIGHPRI
| WQ_CPU_INTENSIVE
, 1);
549 panic("Failed to create kintegrityd\n");
551 bip_slab
= kmem_cache_create("bio_integrity_payload",
552 sizeof(struct bio_integrity_payload
) +
553 sizeof(struct bio_vec
) * BIP_INLINE_VECS
,
554 0, SLAB_HWCACHE_ALIGN
|SLAB_PANIC
, NULL
);
556 panic("Failed to create slab\n");
This page took 0.042776 seconds and 5 git commands to generate.