2 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
4 * bitmap_create - sets up the bitmap structure
5 * bitmap_destroy - destroys the bitmap structure
7 * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
8 * - added disk storage for bitmap
9 * - changes to allow various bitmap chunk sizes
10 * - added bitmap daemon (to asynchronously clear bitmap bits from disk)
16 * flush after percent set rather than just time based. (maybe both).
17 * wait if count gets too high, wake when it drops to half.
18 * allow bitmap to be mirrored with superblock (before or after...)
19 * allow hot-add to re-instate a current device.
20 * allow hot-add of bitmap after quiessing device
23 #include <linux/module.h>
24 #include <linux/version.h>
25 #include <linux/errno.h>
26 #include <linux/slab.h>
27 #include <linux/init.h>
28 #include <linux/config.h>
29 #include <linux/timer.h>
30 #include <linux/sched.h>
31 #include <linux/list.h>
32 #include <linux/file.h>
33 #include <linux/mount.h>
34 #include <linux/buffer_head.h>
35 #include <linux/raid/md.h>
36 #include <linux/raid/bitmap.h>
43 /* these are for debugging purposes only! */
45 /* define one and only one of these */
46 #define INJECT_FAULTS_1 0 /* cause bitmap_alloc_page to fail always */
47 #define INJECT_FAULTS_2 0 /* cause bitmap file to be kicked when first bit set*/
48 #define INJECT_FAULTS_3 0 /* treat bitmap file as kicked at init time */
49 #define INJECT_FAULTS_4 0 /* undef */
50 #define INJECT_FAULTS_5 0 /* undef */
51 #define INJECT_FAULTS_6 0
53 /* if these are defined, the driver will fail! debug only */
54 #define INJECT_FATAL_FAULT_1 0 /* fail kmalloc, causing bitmap_create to fail */
55 #define INJECT_FATAL_FAULT_2 0 /* undef */
56 #define INJECT_FATAL_FAULT_3 0 /* undef */
59 //#define DPRINTK PRINTK /* set this NULL to avoid verbose debug output */
60 #define DPRINTK(x...) do { } while(0)
64 # define PRINTK(x...) printk(KERN_DEBUG x)
70 static inline char * bmname(struct bitmap
*bitmap
)
72 return bitmap
->mddev
? mdname(bitmap
->mddev
) : "mdX";
77 * test if the bitmap is active
79 int bitmap_active(struct bitmap
*bitmap
)
86 spin_lock_irqsave(&bitmap
->lock
, flags
);
87 res
= bitmap
->flags
& BITMAP_ACTIVE
;
88 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
92 #define WRITE_POOL_SIZE 256
93 /* mempool for queueing pending writes on the bitmap file */
94 static void *write_pool_alloc(unsigned int gfp_flags
, void *data
)
96 return kmalloc(sizeof(struct page_list
), gfp_flags
);
99 static void write_pool_free(void *ptr
, void *data
)
105 * just a placeholder - calls kmalloc for bitmap pages
107 static unsigned char *bitmap_alloc_page(struct bitmap
*bitmap
)
114 page
= kmalloc(PAGE_SIZE
, GFP_NOIO
);
117 printk("%s: bitmap_alloc_page FAILED\n", bmname(bitmap
));
119 printk("%s: bitmap_alloc_page: allocated page at %p\n",
120 bmname(bitmap
), page
);
125 * for now just a placeholder -- just calls kfree for bitmap pages
127 static void bitmap_free_page(struct bitmap
*bitmap
, unsigned char *page
)
129 PRINTK("%s: bitmap_free_page: free page %p\n", bmname(bitmap
), page
);
134 * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
136 * 1) check to see if this page is allocated, if it's not then try to alloc
137 * 2) if the alloc fails, set the page's hijacked flag so we'll use the
138 * page pointer directly as a counter
140 * if we find our page, we increment the page's refcount so that it stays
141 * allocated while we're using it
143 static int bitmap_checkpage(struct bitmap
*bitmap
, unsigned long page
, int create
)
145 unsigned char *mappage
;
147 if (page
>= bitmap
->pages
) {
149 "%s: invalid bitmap page request: %lu (> %lu)\n",
150 bmname(bitmap
), page
, bitmap
->pages
-1);
155 if (bitmap
->bp
[page
].hijacked
) /* it's hijacked, don't try to alloc */
158 if (bitmap
->bp
[page
].map
) /* page is already allocated, just return */
164 spin_unlock_irq(&bitmap
->lock
);
166 /* this page has not been allocated yet */
168 if ((mappage
= bitmap_alloc_page(bitmap
)) == NULL
) {
169 PRINTK("%s: bitmap map page allocation failed, hijacking\n",
171 /* failed - set the hijacked flag so that we can use the
172 * pointer as a counter */
173 spin_lock_irq(&bitmap
->lock
);
174 if (!bitmap
->bp
[page
].map
)
175 bitmap
->bp
[page
].hijacked
= 1;
181 spin_lock_irq(&bitmap
->lock
);
183 /* recheck the page */
185 if (bitmap
->bp
[page
].map
|| bitmap
->bp
[page
].hijacked
) {
186 /* somebody beat us to getting the page */
187 bitmap_free_page(bitmap
, mappage
);
191 /* no page was in place and we have one, so install it */
193 memset(mappage
, 0, PAGE_SIZE
);
194 bitmap
->bp
[page
].map
= mappage
;
195 bitmap
->missing_pages
--;
201 /* if page is completely empty, put it back on the free list, or dealloc it */
202 /* if page was hijacked, unmark the flag so it might get alloced next time */
203 /* Note: lock should be held when calling this */
204 static inline void bitmap_checkfree(struct bitmap
*bitmap
, unsigned long page
)
208 if (bitmap
->bp
[page
].count
) /* page is still busy */
211 /* page is no longer in use, it can be released */
213 if (bitmap
->bp
[page
].hijacked
) { /* page was hijacked, undo this now */
214 bitmap
->bp
[page
].hijacked
= 0;
215 bitmap
->bp
[page
].map
= NULL
;
219 /* normal case, free the page */
222 /* actually ... let's not. We will probably need the page again exactly when
223 * memory is tight and we are flusing to disk
227 ptr
= bitmap
->bp
[page
].map
;
228 bitmap
->bp
[page
].map
= NULL
;
229 bitmap
->missing_pages
++;
230 bitmap_free_page(bitmap
, ptr
);
237 * bitmap file handling - read and write the bitmap file and its superblock
240 /* copy the pathname of a file to a buffer */
241 char *file_path(struct file
*file
, char *buf
, int count
)
252 buf
= d_path(d
, v
, buf
, count
);
254 return IS_ERR(buf
) ? NULL
: buf
;
258 * basic page I/O operations
264 static int write_page(struct page
*page
, int wait
)
270 if (page
->mapping
== NULL
)
272 else if (i_size_read(page
->mapping
->host
) < page
->index
<< PAGE_SHIFT
) {
277 ret
= page
->mapping
->a_ops
->prepare_write(NULL
, page
, 0, PAGE_SIZE
);
279 ret
= page
->mapping
->a_ops
->commit_write(NULL
, page
, 0,
287 set_page_dirty(page
); /* force it to be written out */
288 return write_one_page(page
, wait
);
291 /* read a page from a file, pinning it into cache, and return bytes_read */
292 static struct page
*read_page(struct file
*file
, unsigned long index
,
293 unsigned long *bytes_read
)
295 struct inode
*inode
= file
->f_mapping
->host
;
296 struct page
*page
= NULL
;
297 loff_t isize
= i_size_read(inode
);
298 unsigned long end_index
= isize
>> PAGE_CACHE_SHIFT
;
300 PRINTK("read bitmap file (%dB @ %Lu)\n", (int)PAGE_CACHE_SIZE
,
301 (unsigned long long)index
<< PAGE_CACHE_SHIFT
);
303 page
= read_cache_page(inode
->i_mapping
, index
,
304 (filler_t
*)inode
->i_mapping
->a_ops
->readpage
, file
);
307 wait_on_page_locked(page
);
308 if (!PageUptodate(page
) || PageError(page
)) {
309 page_cache_release(page
);
310 page
= ERR_PTR(-EIO
);
314 if (index
> end_index
) /* we have read beyond EOF */
316 else if (index
== end_index
) /* possible short read */
317 *bytes_read
= isize
& ~PAGE_CACHE_MASK
;
319 *bytes_read
= PAGE_CACHE_SIZE
; /* got a full page */
322 printk(KERN_ALERT
"md: bitmap read error: (%dB @ %Lu): %ld\n",
323 (int)PAGE_CACHE_SIZE
,
324 (unsigned long long)index
<< PAGE_CACHE_SHIFT
,
330 * bitmap file superblock operations
333 /* update the event counter and sync the superblock to disk */
334 int bitmap_update_sb(struct bitmap
*bitmap
)
339 if (!bitmap
|| !bitmap
->mddev
) /* no bitmap for this array */
341 spin_lock_irqsave(&bitmap
->lock
, flags
);
342 if (!bitmap
->sb_page
) { /* no superblock */
343 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
346 page_cache_get(bitmap
->sb_page
);
347 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
348 sb
= (bitmap_super_t
*)kmap(bitmap
->sb_page
);
349 sb
->events
= cpu_to_le64(bitmap
->mddev
->events
);
350 if (!bitmap
->mddev
->degraded
)
351 sb
->events_cleared
= cpu_to_le64(bitmap
->mddev
->events
);
352 kunmap(bitmap
->sb_page
);
353 write_page(bitmap
->sb_page
, 0);
357 /* print out the bitmap file superblock */
358 void bitmap_print_sb(struct bitmap
*bitmap
)
362 if (!bitmap
|| !bitmap
->sb_page
)
364 sb
= (bitmap_super_t
*)kmap(bitmap
->sb_page
);
365 printk(KERN_DEBUG
"%s: bitmap file superblock:\n", bmname(bitmap
));
366 printk(KERN_DEBUG
" magic: %08x\n", le32_to_cpu(sb
->magic
));
367 printk(KERN_DEBUG
" version: %d\n", le32_to_cpu(sb
->version
));
368 printk(KERN_DEBUG
" uuid: %08x.%08x.%08x.%08x\n",
369 *(__u32
*)(sb
->uuid
+0),
370 *(__u32
*)(sb
->uuid
+4),
371 *(__u32
*)(sb
->uuid
+8),
372 *(__u32
*)(sb
->uuid
+12));
373 printk(KERN_DEBUG
" events: %llu\n",
374 (unsigned long long) le64_to_cpu(sb
->events
));
375 printk(KERN_DEBUG
"events_clred: %llu\n",
376 (unsigned long long) le64_to_cpu(sb
->events_cleared
));
377 printk(KERN_DEBUG
" state: %08x\n", le32_to_cpu(sb
->state
));
378 printk(KERN_DEBUG
" chunksize: %d B\n", le32_to_cpu(sb
->chunksize
));
379 printk(KERN_DEBUG
"daemon sleep: %ds\n", le32_to_cpu(sb
->daemon_sleep
));
380 printk(KERN_DEBUG
" sync size: %llu KB\n", le64_to_cpu(sb
->sync_size
));
381 kunmap(bitmap
->sb_page
);
384 /* read the superblock from the bitmap file and initialize some bitmap fields */
385 static int bitmap_read_sb(struct bitmap
*bitmap
)
389 unsigned long chunksize
, daemon_sleep
;
390 unsigned long bytes_read
;
391 unsigned long long events
;
394 /* page 0 is the superblock, read it... */
395 bitmap
->sb_page
= read_page(bitmap
->file
, 0, &bytes_read
);
396 if (IS_ERR(bitmap
->sb_page
)) {
397 err
= PTR_ERR(bitmap
->sb_page
);
398 bitmap
->sb_page
= NULL
;
402 sb
= (bitmap_super_t
*)kmap(bitmap
->sb_page
);
404 if (bytes_read
< sizeof(*sb
)) { /* short read */
405 printk(KERN_INFO
"%s: bitmap file superblock truncated\n",
411 chunksize
= le32_to_cpu(sb
->chunksize
);
412 daemon_sleep
= le32_to_cpu(sb
->daemon_sleep
);
414 /* verify that the bitmap-specific fields are valid */
415 if (sb
->magic
!= cpu_to_le32(BITMAP_MAGIC
))
416 reason
= "bad magic";
417 else if (sb
->version
!= cpu_to_le32(BITMAP_MAJOR
))
418 reason
= "unrecognized superblock version";
419 else if (chunksize
< 512 || chunksize
> (1024 * 1024 * 4))
420 reason
= "bitmap chunksize out of range (512B - 4MB)";
421 else if ((1 << ffz(~chunksize
)) != chunksize
)
422 reason
= "bitmap chunksize not a power of 2";
423 else if (daemon_sleep
< 1 || daemon_sleep
> 15)
424 reason
= "daemon sleep period out of range";
426 printk(KERN_INFO
"%s: invalid bitmap file superblock: %s\n",
427 bmname(bitmap
), reason
);
431 /* keep the array size field of the bitmap superblock up to date */
432 sb
->sync_size
= cpu_to_le64(bitmap
->mddev
->resync_max_sectors
);
434 if (!bitmap
->mddev
->persistent
)
438 * if we have a persistent array superblock, compare the
439 * bitmap's UUID and event counter to the mddev's
441 if (memcmp(sb
->uuid
, bitmap
->mddev
->uuid
, 16)) {
442 printk(KERN_INFO
"%s: bitmap superblock UUID mismatch\n",
446 events
= le64_to_cpu(sb
->events
);
447 if (events
< bitmap
->mddev
->events
) {
448 printk(KERN_INFO
"%s: bitmap file is out of date (%llu < %llu) "
449 "-- forcing full recovery\n", bmname(bitmap
), events
,
450 (unsigned long long) bitmap
->mddev
->events
);
451 sb
->state
|= BITMAP_STALE
;
454 /* assign fields using values from superblock */
455 bitmap
->chunksize
= chunksize
;
456 bitmap
->daemon_sleep
= daemon_sleep
;
457 bitmap
->flags
|= sb
->state
;
458 bitmap
->events_cleared
= le64_to_cpu(sb
->events_cleared
);
461 kunmap(bitmap
->sb_page
);
463 bitmap_print_sb(bitmap
);
467 enum bitmap_mask_op
{
472 /* record the state of the bitmap in the superblock */
473 static void bitmap_mask_state(struct bitmap
*bitmap
, enum bitmap_state bits
,
474 enum bitmap_mask_op op
)
479 spin_lock_irqsave(&bitmap
->lock
, flags
);
480 if (!bitmap
|| !bitmap
->sb_page
) { /* can't set the state */
481 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
484 page_cache_get(bitmap
->sb_page
);
485 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
486 sb
= (bitmap_super_t
*)kmap(bitmap
->sb_page
);
488 case MASK_SET
: sb
->state
|= bits
;
490 case MASK_UNSET
: sb
->state
&= ~bits
;
494 kunmap(bitmap
->sb_page
);
495 page_cache_release(bitmap
->sb_page
);
499 * general bitmap file operations
502 /* calculate the index of the page that contains this bit */
503 static inline unsigned long file_page_index(unsigned long chunk
)
505 return CHUNK_BIT_OFFSET(chunk
) >> PAGE_BIT_SHIFT
;
508 /* calculate the (bit) offset of this bit within a page */
509 static inline unsigned long file_page_offset(unsigned long chunk
)
511 return CHUNK_BIT_OFFSET(chunk
) & (PAGE_BITS
- 1);
515 * return a pointer to the page in the filemap that contains the given bit
517 * this lookup is complicated by the fact that the bitmap sb might be exactly
518 * 1 page (e.g., x86) or less than 1 page -- so the bitmap might start on page
521 static inline struct page
*filemap_get_page(struct bitmap
*bitmap
,
524 return bitmap
->filemap
[file_page_index(chunk
) - file_page_index(0)];
528 static void bitmap_file_unmap(struct bitmap
*bitmap
)
530 struct page
**map
, *sb_page
;
535 spin_lock_irqsave(&bitmap
->lock
, flags
);
536 map
= bitmap
->filemap
;
537 bitmap
->filemap
= NULL
;
538 attr
= bitmap
->filemap_attr
;
539 bitmap
->filemap_attr
= NULL
;
540 pages
= bitmap
->file_pages
;
541 bitmap
->file_pages
= 0;
542 sb_page
= bitmap
->sb_page
;
543 bitmap
->sb_page
= NULL
;
544 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
547 if (map
[pages
]->index
!= 0) /* 0 is sb_page, release it below */
548 page_cache_release(map
[pages
]);
553 page_cache_release(sb_page
);
556 static void bitmap_stop_daemons(struct bitmap
*bitmap
);
558 /* dequeue the next item in a page list -- don't call from irq context */
559 static struct page_list
*dequeue_page(struct bitmap
*bitmap
,
560 struct list_head
*head
)
562 struct page_list
*item
= NULL
;
564 spin_lock(&bitmap
->write_lock
);
565 if (list_empty(head
))
567 item
= list_entry(head
->prev
, struct page_list
, list
);
568 list_del(head
->prev
);
570 spin_unlock(&bitmap
->write_lock
);
574 static void drain_write_queues(struct bitmap
*bitmap
)
576 struct list_head
*queues
[] = { &bitmap
->complete_pages
, NULL
};
577 struct list_head
*head
;
578 struct page_list
*item
;
581 for (i
= 0; queues
[i
]; i
++) {
583 while ((item
= dequeue_page(bitmap
, head
))) {
584 page_cache_release(item
->page
);
585 mempool_free(item
, bitmap
->write_pool
);
589 spin_lock(&bitmap
->write_lock
);
590 bitmap
->writes_pending
= 0; /* make sure waiters continue */
591 wake_up(&bitmap
->write_wait
);
592 spin_unlock(&bitmap
->write_lock
);
595 static void bitmap_file_put(struct bitmap
*bitmap
)
601 spin_lock_irqsave(&bitmap
->lock
, flags
);
604 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
606 bitmap_stop_daemons(bitmap
);
608 drain_write_queues(bitmap
);
610 bitmap_file_unmap(bitmap
);
613 inode
= file
->f_mapping
->host
;
614 spin_lock(&inode
->i_lock
);
615 atomic_set(&inode
->i_writecount
, 1); /* allow writes again */
616 spin_unlock(&inode
->i_lock
);
623 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
624 * then it is no longer reliable, so we stop using it and we mark the file
625 * as failed in the superblock
627 static void bitmap_file_kick(struct bitmap
*bitmap
)
629 char *path
, *ptr
= NULL
;
631 bitmap_mask_state(bitmap
, BITMAP_STALE
, MASK_SET
);
632 bitmap_update_sb(bitmap
);
634 path
= kmalloc(PAGE_SIZE
, GFP_KERNEL
);
636 ptr
= file_path(bitmap
->file
, path
, PAGE_SIZE
);
638 printk(KERN_ALERT
"%s: kicking failed bitmap file %s from array!\n",
639 bmname(bitmap
), ptr
? ptr
: "");
643 bitmap_file_put(bitmap
);
648 enum bitmap_page_attr
{
649 BITMAP_PAGE_DIRTY
= 1, // there are set bits that need to be synced
650 BITMAP_PAGE_CLEAN
= 2, // there are bits that might need to be cleared
651 BITMAP_PAGE_NEEDWRITE
=4, // there are cleared bits that need to be synced
654 static inline void set_page_attr(struct bitmap
*bitmap
, struct page
*page
,
655 enum bitmap_page_attr attr
)
657 bitmap
->filemap_attr
[page
->index
] |= attr
;
660 static inline void clear_page_attr(struct bitmap
*bitmap
, struct page
*page
,
661 enum bitmap_page_attr attr
)
663 bitmap
->filemap_attr
[page
->index
] &= ~attr
;
666 static inline unsigned long get_page_attr(struct bitmap
*bitmap
, struct page
*page
)
668 return bitmap
->filemap_attr
[page
->index
];
672 * bitmap_file_set_bit -- called before performing a write to the md device
673 * to set (and eventually sync) a particular bit in the bitmap file
675 * we set the bit immediately, then we record the page number so that
676 * when an unplug occurs, we can flush the dirty pages out to disk
678 static void bitmap_file_set_bit(struct bitmap
*bitmap
, sector_t block
)
683 unsigned long chunk
= block
>> CHUNK_BLOCK_SHIFT(bitmap
);
685 if (!bitmap
->file
|| !bitmap
->filemap
) {
689 page
= filemap_get_page(bitmap
, chunk
);
690 bit
= file_page_offset(chunk
);
693 /* make sure the page stays cached until it gets written out */
694 if (! (get_page_attr(bitmap
, page
) & BITMAP_PAGE_DIRTY
))
695 page_cache_get(page
);
698 kaddr
= kmap_atomic(page
, KM_USER0
);
700 kunmap_atomic(kaddr
, KM_USER0
);
701 PRINTK("set file bit %lu page %lu\n", bit
, page
->index
);
703 /* record page number so it gets flushed to disk when unplug occurs */
704 set_page_attr(bitmap
, page
, BITMAP_PAGE_DIRTY
);
708 /* this gets called when the md device is ready to unplug its underlying
709 * (slave) device queues -- before we let any writes go down, we need to
710 * sync the dirty pages of the bitmap file to disk */
711 int bitmap_unplug(struct bitmap
*bitmap
)
713 unsigned long i
, attr
, flags
;
720 /* look at each page to see if there are any set bits that need to be
721 * flushed out to disk */
722 for (i
= 0; i
< bitmap
->file_pages
; i
++) {
723 spin_lock_irqsave(&bitmap
->lock
, flags
);
724 if (!bitmap
->file
|| !bitmap
->filemap
) {
725 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
728 page
= bitmap
->filemap
[i
];
729 attr
= get_page_attr(bitmap
, page
);
730 clear_page_attr(bitmap
, page
, BITMAP_PAGE_DIRTY
);
731 clear_page_attr(bitmap
, page
, BITMAP_PAGE_NEEDWRITE
);
732 if ((attr
& BITMAP_PAGE_DIRTY
))
734 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
736 if (attr
& (BITMAP_PAGE_DIRTY
| BITMAP_PAGE_NEEDWRITE
))
739 if (wait
) { /* if any writes were performed, we need to wait on them */
740 spin_lock_irq(&bitmap
->write_lock
);
741 wait_event_lock_irq(bitmap
->write_wait
,
742 bitmap
->writes_pending
== 0, bitmap
->write_lock
,
743 wake_up_process(bitmap
->writeback_daemon
->tsk
));
744 spin_unlock_irq(&bitmap
->write_lock
);
749 static void bitmap_set_memory_bits(struct bitmap
*bitmap
, sector_t offset
,
750 unsigned long sectors
, int set
);
751 /* * bitmap_init_from_disk -- called at bitmap_create time to initialize
752 * the in-memory bitmap from the on-disk bitmap -- also, sets up the
753 * memory mapping of the bitmap file
755 * if there's no bitmap file, or if the bitmap file had been
756 * previously kicked from the array, we mark all the bits as
757 * 1's in order to cause a full resync.
759 static int bitmap_init_from_disk(struct bitmap
*bitmap
)
761 unsigned long i
, chunks
, index
, oldindex
, bit
;
762 struct page
*page
= NULL
, *oldpage
= NULL
;
763 unsigned long num_pages
, bit_cnt
= 0;
765 unsigned long bytes
, offset
, dummy
;
769 chunks
= bitmap
->chunks
;
777 outofdate
= bitmap
->flags
& BITMAP_STALE
;
780 printk(KERN_INFO
"%s: bitmap file is out of date, doing full "
781 "recovery\n", bmname(bitmap
));
783 bytes
= (chunks
+ 7) / 8;
784 num_pages
= (bytes
+ PAGE_SIZE
- 1) / PAGE_SIZE
;
785 if (i_size_read(file
->f_mapping
->host
) < bytes
+ sizeof(bitmap_super_t
)) {
786 printk(KERN_INFO
"%s: bitmap file too short %lu < %lu\n",
788 (unsigned long) i_size_read(file
->f_mapping
->host
),
789 bytes
+ sizeof(bitmap_super_t
));
793 bitmap
->filemap
= kmalloc(sizeof(struct page
*) * num_pages
, GFP_KERNEL
);
794 if (!bitmap
->filemap
) {
799 bitmap
->filemap_attr
= kmalloc(sizeof(long) * num_pages
, GFP_KERNEL
);
800 if (!bitmap
->filemap_attr
) {
805 memset(bitmap
->filemap_attr
, 0, sizeof(long) * num_pages
);
809 for (i
= 0; i
< chunks
; i
++) {
810 index
= file_page_index(i
);
811 bit
= file_page_offset(i
);
812 if (index
!= oldindex
) { /* this is a new page, read it in */
813 /* unmap the old page, we're done with it */
818 * if we're here then the superblock page
819 * contains some bits (PAGE_SIZE != sizeof sb)
820 * we've already read it in, so just use it
822 page
= bitmap
->sb_page
;
823 offset
= sizeof(bitmap_super_t
);
825 page
= read_page(file
, index
, &dummy
);
826 if (IS_ERR(page
)) { /* read error */
838 * if bitmap is out of date, dirty the
839 * whole page and write it out
841 memset(page_address(page
) + offset
, 0xff,
843 ret
= write_page(page
, 1);
846 /* release, page not in filemap yet */
847 page_cache_release(page
);
852 bitmap
->filemap
[bitmap
->file_pages
++] = page
;
854 if (test_bit(bit
, page_address(page
))) {
855 /* if the disk bit is set, set the memory bit */
856 bitmap_set_memory_bits(bitmap
,
857 i
<< CHUNK_BLOCK_SHIFT(bitmap
), 1, 1);
862 bitmap_set_memory_bits(bitmap
,
863 i
<< CHUNK_BLOCK_SHIFT(bitmap
), 1, 0);
867 /* everything went OK */
869 bitmap_mask_state(bitmap
, BITMAP_STALE
, MASK_UNSET
);
871 if (page
) /* unmap the last page */
874 if (bit_cnt
) { /* Kick recovery if any bits were set */
875 set_bit(MD_RECOVERY_NEEDED
, &bitmap
->mddev
->recovery
);
876 md_wakeup_thread(bitmap
->mddev
->thread
);
880 printk(KERN_INFO
"%s: bitmap initialized from disk: "
881 "read %lu/%lu pages, set %lu bits, status: %d\n",
882 bmname(bitmap
), bitmap
->file_pages
, num_pages
, bit_cnt
, ret
);
888 static void bitmap_count_page(struct bitmap
*bitmap
, sector_t offset
, int inc
)
890 sector_t chunk
= offset
>> CHUNK_BLOCK_SHIFT(bitmap
);
891 unsigned long page
= chunk
>> PAGE_COUNTER_SHIFT
;
892 bitmap
->bp
[page
].count
+= inc
;
894 if (page == 0) printk("count page 0, offset %llu: %d gives %d\n",
895 (unsigned long long)offset, inc, bitmap->bp[page].count);
897 bitmap_checkfree(bitmap
, page
);
899 static bitmap_counter_t
*bitmap_get_counter(struct bitmap
*bitmap
,
900 sector_t offset
, int *blocks
,
904 * bitmap daemon -- periodically wakes up to clean bits and flush pages
908 int bitmap_daemon_work(struct bitmap
*bitmap
)
910 unsigned long bit
, j
;
912 struct page
*page
= NULL
, *lastpage
= NULL
;
919 if (time_before(jiffies
, bitmap
->daemon_lastrun
+ bitmap
->daemon_sleep
*HZ
))
921 bitmap
->daemon_lastrun
= jiffies
;
923 for (j
= 0; j
< bitmap
->chunks
; j
++) {
924 bitmap_counter_t
*bmc
;
925 spin_lock_irqsave(&bitmap
->lock
, flags
);
926 if (!bitmap
->file
|| !bitmap
->filemap
) {
927 /* error or shutdown */
928 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
932 page
= filemap_get_page(bitmap
, j
);
933 /* skip this page unless it's marked as needing cleaning */
934 if (!((attr
=get_page_attr(bitmap
, page
)) & BITMAP_PAGE_CLEAN
)) {
935 if (attr
& BITMAP_PAGE_NEEDWRITE
) {
936 page_cache_get(page
);
937 clear_page_attr(bitmap
, page
, BITMAP_PAGE_NEEDWRITE
);
939 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
940 if (attr
& BITMAP_PAGE_NEEDWRITE
) {
941 if (write_page(page
, 0))
942 bitmap_file_kick(bitmap
);
943 page_cache_release(page
);
948 bit
= file_page_offset(j
);
950 if (page
!= lastpage
) {
951 /* grab the new page, sync and release the old */
952 page_cache_get(page
);
953 if (lastpage
!= NULL
) {
954 if (get_page_attr(bitmap
, lastpage
) & BITMAP_PAGE_NEEDWRITE
) {
955 clear_page_attr(bitmap
, lastpage
, BITMAP_PAGE_NEEDWRITE
);
956 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
957 write_page(lastpage
, 0);
959 set_page_attr(bitmap
, lastpage
, BITMAP_PAGE_NEEDWRITE
);
960 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
963 page_cache_release(lastpage
);
965 bitmap_file_kick(bitmap
);
967 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
971 printk("bitmap clean at page %lu\n", j);
973 spin_lock_irqsave(&bitmap
->lock
, flags
);
974 clear_page_attr(bitmap
, page
, BITMAP_PAGE_CLEAN
);
976 bmc
= bitmap_get_counter(bitmap
, j
<< CHUNK_BLOCK_SHIFT(bitmap
),
980 if (j < 100) printk("bitmap: j=%lu, *bmc = 0x%x\n", j, *bmc);
983 *bmc
=1; /* maybe clear the bit next time */
984 set_page_attr(bitmap
, page
, BITMAP_PAGE_CLEAN
);
985 } else if (*bmc
== 1) {
986 /* we can clear the bit */
988 bitmap_count_page(bitmap
, j
<< CHUNK_BLOCK_SHIFT(bitmap
),
992 clear_bit(bit
, page_address(page
));
995 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
998 /* now sync the final page */
999 if (lastpage
!= NULL
) {
1001 spin_lock_irqsave(&bitmap
->lock
, flags
);
1002 if (get_page_attr(bitmap
, lastpage
) &BITMAP_PAGE_NEEDWRITE
) {
1003 clear_page_attr(bitmap
, lastpage
, BITMAP_PAGE_NEEDWRITE
);
1004 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1005 write_page(lastpage
, 0);
1007 set_page_attr(bitmap
, lastpage
, BITMAP_PAGE_NEEDWRITE
);
1008 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1011 page_cache_release(lastpage
);
1017 static void daemon_exit(struct bitmap
*bitmap
, mdk_thread_t
**daemon
)
1020 unsigned long flags
;
1022 /* if no one is waiting on us, we'll free the md thread struct
1023 * and exit, otherwise we let the waiter clean things up */
1024 spin_lock_irqsave(&bitmap
->lock
, flags
);
1025 if ((dmn
= *daemon
)) { /* no one is waiting, cleanup and exit */
1027 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1029 complete_and_exit(NULL
, 0); /* do_exit not exported */
1031 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1034 static void bitmap_writeback_daemon(mddev_t
*mddev
)
1036 struct bitmap
*bitmap
= mddev
->bitmap
;
1038 struct page_list
*item
;
1042 PRINTK("%s: bitmap writeback daemon waiting...\n", bmname(bitmap
));
1043 down_interruptible(&bitmap
->write_done
);
1044 if (signal_pending(current
)) {
1046 "%s: bitmap writeback daemon got signal, exiting...\n",
1051 PRINTK("%s: bitmap writeback daemon woke up...\n", bmname(bitmap
));
1052 /* wait on bitmap page writebacks */
1053 while ((item
= dequeue_page(bitmap
, &bitmap
->complete_pages
))) {
1055 mempool_free(item
, bitmap
->write_pool
);
1056 PRINTK("wait on page writeback: %p %lu\n", page
, bitmap
->writes_pending
);
1057 wait_on_page_writeback(page
);
1058 PRINTK("finished page writeback: %p %lu\n", page
, bitmap
->writes_pending
);
1059 spin_lock(&bitmap
->write_lock
);
1060 if (!--bitmap
->writes_pending
)
1061 wake_up(&bitmap
->write_wait
);
1062 spin_unlock(&bitmap
->write_lock
);
1063 err
= PageError(page
);
1064 page_cache_release(page
);
1066 printk(KERN_WARNING
"%s: bitmap file writeback "
1067 "failed (page %lu): %d\n",
1068 bmname(bitmap
), page
->index
, err
);
1069 bitmap_file_kick(bitmap
);
1076 printk(KERN_INFO
"%s: bitmap writeback daemon exiting (%d)\n",
1077 bmname(bitmap
), err
);
1078 daemon_exit(bitmap
, &bitmap
->writeback_daemon
);
1083 static int bitmap_start_daemon(struct bitmap
*bitmap
, mdk_thread_t
**ptr
,
1084 void (*func
)(mddev_t
*), char *name
)
1086 mdk_thread_t
*daemon
;
1087 unsigned long flags
;
1090 spin_lock_irqsave(&bitmap
->lock
, flags
);
1092 if (!bitmap
->file
) /* no need for daemon if there's no backing file */
1095 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1097 #if INJECT_FATAL_FAULT_2
1100 sprintf(namebuf
, "%%s_%s", name
);
1101 daemon
= md_register_thread(func
, bitmap
->mddev
, namebuf
);
1104 printk(KERN_ERR
"%s: failed to start bitmap daemon\n",
1109 spin_lock_irqsave(&bitmap
->lock
, flags
);
1112 md_wakeup_thread(daemon
); /* start it running */
1114 PRINTK("%s: %s daemon (pid %d) started...\n",
1115 bmname(bitmap
), name
, bitmap
->daemon
->tsk
->pid
);
1117 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1121 static int bitmap_start_daemons(struct bitmap
*bitmap
)
1123 int err
= bitmap_start_daemon(bitmap
, &bitmap
->writeback_daemon
,
1124 bitmap_writeback_daemon
, "bitmap_wb");
1128 static void bitmap_stop_daemon(struct bitmap
*bitmap
, mdk_thread_t
**ptr
)
1130 mdk_thread_t
*daemon
;
1131 unsigned long flags
;
1133 spin_lock_irqsave(&bitmap
->lock
, flags
);
1136 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1138 md_unregister_thread(daemon
); /* destroy the thread */
1141 static void bitmap_stop_daemons(struct bitmap
*bitmap
)
1143 /* the daemons can't stop themselves... they'll just exit instead... */
1144 if (bitmap
->writeback_daemon
&&
1145 current
->pid
!= bitmap
->writeback_daemon
->tsk
->pid
)
1146 bitmap_stop_daemon(bitmap
, &bitmap
->writeback_daemon
);
1149 static bitmap_counter_t
*bitmap_get_counter(struct bitmap
*bitmap
,
1150 sector_t offset
, int *blocks
,
1153 /* If 'create', we might release the lock and reclaim it.
1154 * The lock must have been taken with interrupts enabled.
1155 * If !create, we don't release the lock.
1157 sector_t chunk
= offset
>> CHUNK_BLOCK_SHIFT(bitmap
);
1158 unsigned long page
= chunk
>> PAGE_COUNTER_SHIFT
;
1159 unsigned long pageoff
= (chunk
& PAGE_COUNTER_MASK
) << COUNTER_BYTE_SHIFT
;
1162 if (bitmap_checkpage(bitmap
, page
, create
) < 0) {
1163 csize
= ((sector_t
)1) << (CHUNK_BLOCK_SHIFT(bitmap
));
1164 *blocks
= csize
- (offset
& (csize
- 1));
1167 /* now locked ... */
1169 if (bitmap
->bp
[page
].hijacked
) { /* hijacked pointer */
1170 /* should we use the first or second counter field
1171 * of the hijacked pointer? */
1172 int hi
= (pageoff
> PAGE_COUNTER_MASK
);
1173 csize
= ((sector_t
)1) << (CHUNK_BLOCK_SHIFT(bitmap
) +
1174 PAGE_COUNTER_SHIFT
- 1);
1175 *blocks
= csize
- (offset
& (csize
- 1));
1176 return &((bitmap_counter_t
*)
1177 &bitmap
->bp
[page
].map
)[hi
];
1178 } else { /* page is allocated */
1179 csize
= ((sector_t
)1) << (CHUNK_BLOCK_SHIFT(bitmap
));
1180 *blocks
= csize
- (offset
& (csize
- 1));
1181 return (bitmap_counter_t
*)
1182 &(bitmap
->bp
[page
].map
[pageoff
]);
1186 int bitmap_startwrite(struct bitmap
*bitmap
, sector_t offset
, unsigned long sectors
)
1188 if (!bitmap
) return 0;
1191 bitmap_counter_t
*bmc
;
1193 spin_lock_irq(&bitmap
->lock
);
1194 bmc
= bitmap_get_counter(bitmap
, offset
, &blocks
, 1);
1196 spin_unlock_irq(&bitmap
->lock
);
1202 bitmap_file_set_bit(bitmap
, offset
);
1203 bitmap_count_page(bitmap
,offset
, 1);
1204 blk_plug_device(bitmap
->mddev
->queue
);
1209 if ((*bmc
& COUNTER_MAX
) == COUNTER_MAX
) BUG();
1212 spin_unlock_irq(&bitmap
->lock
);
1215 if (sectors
> blocks
)
1222 void bitmap_endwrite(struct bitmap
*bitmap
, sector_t offset
, unsigned long sectors
,
1225 if (!bitmap
) return;
1228 unsigned long flags
;
1229 bitmap_counter_t
*bmc
;
1231 spin_lock_irqsave(&bitmap
->lock
, flags
);
1232 bmc
= bitmap_get_counter(bitmap
, offset
, &blocks
, 0);
1234 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1238 if (!success
&& ! (*bmc
& NEEDED_MASK
))
1239 *bmc
|= NEEDED_MASK
;
1243 set_page_attr(bitmap
,
1244 filemap_get_page(bitmap
, offset
>> CHUNK_BLOCK_SHIFT(bitmap
)),
1247 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1249 if (sectors
> blocks
)
1255 int bitmap_start_sync(struct bitmap
*bitmap
, sector_t offset
, int *blocks
)
1257 bitmap_counter_t
*bmc
;
1259 if (bitmap
== NULL
) {/* FIXME or bitmap set as 'failed' */
1261 return 1; /* always resync if no bitmap */
1263 spin_lock_irq(&bitmap
->lock
);
1264 bmc
= bitmap_get_counter(bitmap
, offset
, blocks
, 0);
1270 else if (NEEDED(*bmc
)) {
1272 *bmc
|= RESYNC_MASK
;
1273 *bmc
&= ~NEEDED_MASK
;
1276 spin_unlock_irq(&bitmap
->lock
);
1280 void bitmap_end_sync(struct bitmap
*bitmap
, sector_t offset
, int *blocks
, int aborted
)
1282 bitmap_counter_t
*bmc
;
1283 unsigned long flags
;
1285 if (offset == 0) printk("bitmap_end_sync 0 (%d)\n", aborted);
1286 */ if (bitmap
== NULL
) {
1290 spin_lock_irqsave(&bitmap
->lock
, flags
);
1291 bmc
= bitmap_get_counter(bitmap
, offset
, blocks
, 0);
1296 if (offset == 0) printk("bitmap_end sync found 0x%x, blocks %d\n", *bmc, *blocks);
1299 *bmc
&= ~RESYNC_MASK
;
1301 if (!NEEDED(*bmc
) && aborted
)
1302 *bmc
|= NEEDED_MASK
;
1305 set_page_attr(bitmap
,
1306 filemap_get_page(bitmap
, offset
>> CHUNK_BLOCK_SHIFT(bitmap
)),
1312 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1315 void bitmap_close_sync(struct bitmap
*bitmap
)
1317 /* Sync has finished, and any bitmap chunks that weren't synced
1318 * properly have been aborted. It remains to us to clear the
1319 * RESYNC bit wherever it is still on
1321 sector_t sector
= 0;
1323 if (!bitmap
) return;
1324 while (sector
< bitmap
->mddev
->resync_max_sectors
) {
1325 bitmap_end_sync(bitmap
, sector
, &blocks
, 0);
1327 if (sector < 500) printk("bitmap_close_sync: sec %llu blks %d\n",
1328 (unsigned long long)sector, blocks);
1329 */ sector
+= blocks
;
1333 static void bitmap_set_memory_bits(struct bitmap
*bitmap
, sector_t offset
,
1334 unsigned long sectors
, int set
)
1336 /* For each chunk covered by any of these sectors, set the
1337 * resync needed bit, and the counter to 1. They should all
1338 * be 0 at this point
1342 bitmap_counter_t
*bmc
;
1343 spin_lock_irq(&bitmap
->lock
);
1344 bmc
= bitmap_get_counter(bitmap
, offset
, &secs
, 1);
1346 spin_unlock_irq(&bitmap
->lock
);
1349 if (set
&& !NEEDED(*bmc
)) {
1351 *bmc
= NEEDED_MASK
| 1;
1352 bitmap_count_page(bitmap
, offset
, 1);
1354 spin_unlock_irq(&bitmap
->lock
);
1362 /* dirty the entire bitmap */
1363 int bitmap_setallbits(struct bitmap
*bitmap
)
1365 unsigned long flags
;
1368 /* dirty the in-memory bitmap */
1369 bitmap_set_memory_bits(bitmap
, 0, bitmap
->chunks
<< CHUNK_BLOCK_SHIFT(bitmap
), 1);
1371 /* dirty the bitmap file */
1372 for (j
= 0; j
< bitmap
->file_pages
; j
++) {
1373 struct page
*page
= bitmap
->filemap
[j
];
1375 spin_lock_irqsave(&bitmap
->lock
, flags
);
1376 page_cache_get(page
);
1377 spin_unlock_irqrestore(&bitmap
->lock
, flags
);
1378 memset(kmap(page
), 0xff, PAGE_SIZE
);
1380 write_page(page
, 0);
1387 * free memory that was allocated
1389 void bitmap_destroy(mddev_t
*mddev
)
1391 unsigned long k
, pages
;
1392 struct bitmap_page
*bp
;
1393 struct bitmap
*bitmap
= mddev
->bitmap
;
1395 if (!bitmap
) /* there was no bitmap */
1398 mddev
->bitmap
= NULL
; /* disconnect from the md device */
1400 /* release the bitmap file and kill the daemon */
1401 bitmap_file_put(bitmap
);
1404 pages
= bitmap
->pages
;
1406 /* free all allocated memory */
1408 mempool_destroy(bitmap
->write_pool
);
1410 if (bp
) /* deallocate the page memory */
1411 for (k
= 0; k
< pages
; k
++)
1412 if (bp
[k
].map
&& !bp
[k
].hijacked
)
1419 * initialize the bitmap structure
1420 * if this returns an error, bitmap_destroy must be called to do clean up
1422 int bitmap_create(mddev_t
*mddev
)
1424 struct bitmap
*bitmap
;
1425 unsigned long blocks
= mddev
->resync_max_sectors
;
1426 unsigned long chunks
;
1427 unsigned long pages
;
1428 struct file
*file
= mddev
->bitmap_file
;
1431 BUG_ON(sizeof(bitmap_super_t
) != 256);
1433 if (!file
) /* bitmap disabled, nothing to do */
1436 bitmap
= kmalloc(sizeof(*bitmap
), GFP_KERNEL
);
1440 memset(bitmap
, 0, sizeof(*bitmap
));
1442 spin_lock_init(&bitmap
->lock
);
1443 bitmap
->mddev
= mddev
;
1444 mddev
->bitmap
= bitmap
;
1446 spin_lock_init(&bitmap
->write_lock
);
1447 init_MUTEX_LOCKED(&bitmap
->write_done
);
1448 INIT_LIST_HEAD(&bitmap
->complete_pages
);
1449 init_waitqueue_head(&bitmap
->write_wait
);
1450 bitmap
->write_pool
= mempool_create(WRITE_POOL_SIZE
, write_pool_alloc
,
1451 write_pool_free
, NULL
);
1452 if (!bitmap
->write_pool
)
1455 bitmap
->file
= file
;
1457 /* read superblock from bitmap file (this sets bitmap->chunksize) */
1458 err
= bitmap_read_sb(bitmap
);
1462 bitmap
->chunkshift
= find_first_bit(&bitmap
->chunksize
,
1463 sizeof(bitmap
->chunksize
));
1465 /* now that chunksize and chunkshift are set, we can use these macros */
1466 chunks
= (blocks
+ CHUNK_BLOCK_RATIO(bitmap
) - 1) /
1467 CHUNK_BLOCK_RATIO(bitmap
);
1468 pages
= (chunks
+ PAGE_COUNTER_RATIO
- 1) / PAGE_COUNTER_RATIO
;
1472 bitmap
->chunks
= chunks
;
1473 bitmap
->pages
= pages
;
1474 bitmap
->missing_pages
= pages
;
1475 bitmap
->counter_bits
= COUNTER_BITS
;
1477 bitmap
->syncchunk
= ~0UL;
1479 #if INJECT_FATAL_FAULT_1
1482 bitmap
->bp
= kmalloc(pages
* sizeof(*bitmap
->bp
), GFP_KERNEL
);
1486 memset(bitmap
->bp
, 0, pages
* sizeof(*bitmap
->bp
));
1488 bitmap
->flags
|= BITMAP_ACTIVE
;
1490 /* now that we have some pages available, initialize the in-memory
1491 * bitmap from the on-disk bitmap */
1492 err
= bitmap_init_from_disk(bitmap
);
1496 printk(KERN_INFO
"created bitmap (%lu pages) for device %s\n",
1497 pages
, bmname(bitmap
));
1499 /* kick off the bitmap daemons */
1500 err
= bitmap_start_daemons(bitmap
);
1503 return bitmap_update_sb(bitmap
);
1506 /* the bitmap API -- for raid personalities */
1507 EXPORT_SYMBOL(bitmap_startwrite
);
1508 EXPORT_SYMBOL(bitmap_endwrite
);
1509 EXPORT_SYMBOL(bitmap_start_sync
);
1510 EXPORT_SYMBOL(bitmap_end_sync
);
1511 EXPORT_SYMBOL(bitmap_unplug
);
1512 EXPORT_SYMBOL(bitmap_close_sync
);
1513 EXPORT_SYMBOL(bitmap_daemon_work
);
This page took 0.06097 seconds and 6 git commands to generate.