2 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
3 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
5 * This copyrighted material is made available to anyone wishing to use,
6 * modify, copy, or redistribute it subject to the terms and conditions
7 * of the GNU General Public License version 2.
10 #include <linux/sched.h>
11 #include <linux/slab.h>
12 #include <linux/spinlock.h>
13 #include <linux/completion.h>
14 #include <linux/buffer_head.h>
15 #include <linux/pagemap.h>
16 #include <linux/pagevec.h>
17 #include <linux/mpage.h>
19 #include <linux/gfs2_ondisk.h>
22 #include "lm_interface.h"
29 #include "ops_address.h"
38 static void gfs2_page_add_databufs(struct gfs2_inode
*ip
, struct page
*page
,
39 unsigned int from
, unsigned int to
)
41 struct buffer_head
*head
= page_buffers(page
);
42 unsigned int bsize
= head
->b_size
;
43 struct buffer_head
*bh
;
44 unsigned int start
, end
;
46 for (bh
= head
, start
= 0; bh
!= head
|| !start
;
47 bh
= bh
->b_this_page
, start
= end
) {
49 if (end
<= from
|| start
>= to
)
51 gfs2_trans_add_bh(ip
->i_gl
, bh
, 0);
56 * gfs2_get_block - Fills in a buffer head with details about a block
58 * @lblock: The block number to look up
59 * @bh_result: The buffer head to return the result in
60 * @create: Non-zero if we may add block to the file
65 int gfs2_get_block(struct inode
*inode
, sector_t lblock
,
66 struct buffer_head
*bh_result
, int create
)
73 error
= gfs2_block_map(inode
, lblock
, &new, &dblock
, &boundary
);
80 map_bh(bh_result
, inode
->i_sb
, dblock
);
82 set_buffer_new(bh_result
);
84 set_buffer_boundary(bh_result
);
90 * get_block_noalloc - Fills in a buffer head with details about a block
92 * @lblock: The block number to look up
93 * @bh_result: The buffer head to return the result in
94 * @create: Non-zero if we may add block to the file
99 static int get_block_noalloc(struct inode
*inode
, sector_t lblock
,
100 struct buffer_head
*bh_result
, int create
)
107 error
= gfs2_block_map(inode
, lblock
, &new, &dblock
, &boundary
);
112 map_bh(bh_result
, inode
->i_sb
, dblock
);
113 else if (gfs2_assert_withdraw(GFS2_SB(inode
), !create
))
116 set_buffer_boundary(bh_result
);
121 static int get_block_direct(struct inode
*inode
, sector_t lblock
,
122 struct buffer_head
*bh_result
, int create
)
128 error
= gfs2_block_map(inode
, lblock
, &new, &dblock
, &boundary
);
133 map_bh(bh_result
, inode
->i_sb
, dblock
);
135 set_buffer_boundary(bh_result
);
141 * gfs2_writepage - Write complete page
142 * @page: Page to write
146 * Some of this is copied from block_write_full_page() although we still
147 * call it to do most of the work.
150 static int gfs2_writepage(struct page
*page
, struct writeback_control
*wbc
)
152 struct inode
*inode
= page
->mapping
->host
;
153 struct gfs2_inode
*ip
= GFS2_I(inode
);
154 struct gfs2_sbd
*sdp
= GFS2_SB(inode
);
155 loff_t i_size
= i_size_read(inode
);
156 pgoff_t end_index
= i_size
>> PAGE_CACHE_SHIFT
;
161 if (gfs2_assert_withdraw(sdp
, gfs2_glock_is_held_excl(ip
->i_gl
))) {
165 if (current
->journal_info
)
168 /* Is the page fully outside i_size? (truncate in progress) */
169 offset
= i_size
& (PAGE_CACHE_SIZE
-1);
170 if (page
->index
> end_index
|| (page
->index
== end_index
&& !offset
)) {
171 page
->mapping
->a_ops
->invalidatepage(page
, 0);
173 return 0; /* don't care */
176 if (sdp
->sd_args
.ar_data
== GFS2_DATA_ORDERED
|| gfs2_is_jdata(ip
)) {
177 error
= gfs2_trans_begin(sdp
, RES_DINODE
+ 1, 0);
180 if (!page_has_buffers(page
)) {
181 create_empty_buffers(page
, inode
->i_sb
->s_blocksize
,
182 (1 << BH_Dirty
)|(1 << BH_Uptodate
));
184 gfs2_page_add_databufs(ip
, page
, 0, sdp
->sd_vfs
->s_blocksize
-1);
187 error
= block_write_full_page(page
, get_block_noalloc
, wbc
);
190 gfs2_meta_cache_flush(ip
);
194 redirty_page_for_writepage(wbc
, page
);
199 static int zero_readpage(struct page
*page
)
203 kaddr
= kmap_atomic(page
, KM_USER0
);
204 memset(kaddr
, 0, PAGE_CACHE_SIZE
);
205 kunmap_atomic(page
, KM_USER0
);
207 SetPageUptodate(page
);
213 * stuffed_readpage - Fill in a Linux page with stuffed file data
220 static int stuffed_readpage(struct gfs2_inode
*ip
, struct page
*page
)
222 struct buffer_head
*dibh
;
226 /* Only the first page of a stuffed file might contain data */
227 if (unlikely(page
->index
))
228 return zero_readpage(page
);
230 error
= gfs2_meta_inode_buffer(ip
, &dibh
);
234 kaddr
= kmap_atomic(page
, KM_USER0
);
235 memcpy(kaddr
, dibh
->b_data
+ sizeof(struct gfs2_dinode
),
237 memset(kaddr
+ ip
->i_di
.di_size
, 0, PAGE_CACHE_SIZE
- ip
->i_di
.di_size
);
238 kunmap_atomic(page
, KM_USER0
);
242 SetPageUptodate(page
);
249 * gfs2_readpage - readpage with locking
250 * @file: The file to read a page for. N.B. This may be NULL if we are
251 * reading an internal file.
252 * @page: The page to read
257 static int gfs2_readpage(struct file
*file
, struct page
*page
)
259 struct gfs2_inode
*ip
= GFS2_I(page
->mapping
->host
);
260 struct gfs2_sbd
*sdp
= GFS2_SB(page
->mapping
->host
);
261 struct gfs2_holder gh
;
265 if (likely(file
!= &gfs2_internal_file_sentinal
)) {
267 struct gfs2_file
*gf
= file
->private_data
;
268 if (test_bit(GFF_EXLOCK
, &gf
->f_flags
))
271 gfs2_holder_init(ip
->i_gl
, LM_ST_SHARED
, GL_ATIME
|GL_AOP
, &gh
);
273 error
= gfs2_glock_nq_m_atime(1, &gh
);
279 if (gfs2_is_stuffed(ip
)) {
280 error
= stuffed_readpage(ip
, page
);
283 error
= mpage_readpage(page
, gfs2_get_block
);
285 if (unlikely(test_bit(SDF_SHUTDOWN
, &sdp
->sd_flags
)))
288 if (file
!= &gfs2_internal_file_sentinal
) {
289 gfs2_glock_dq_m(1, &gh
);
290 gfs2_holder_uninit(&gh
);
297 gfs2_holder_uninit(&gh
);
302 * gfs2_readpages - Read a bunch of pages at once
305 * 1. This is only for readahead, so we can simply ignore any things
306 * which are slightly inconvenient (such as locking conflicts between
307 * the page lock and the glock) and return having done no I/O. Its
308 * obviously not something we'd want to do on too regular a basis.
309 * Any I/O we ignore at this time will be done via readpage later.
310 * 2. We have to handle stuffed files here too.
311 * 3. mpage_readpages() does most of the heavy lifting in the common case.
312 * 4. gfs2_get_block() is relied upon to set BH_Boundary in the right places.
313 * 5. We use LM_FLAG_TRY_1CB here, effectively we then have lock-ahead as
314 * well as read-ahead.
316 static int gfs2_readpages(struct file
*file
, struct address_space
*mapping
,
317 struct list_head
*pages
, unsigned nr_pages
)
319 struct inode
*inode
= mapping
->host
;
320 struct gfs2_inode
*ip
= GFS2_I(inode
);
321 struct gfs2_sbd
*sdp
= GFS2_SB(inode
);
322 struct gfs2_holder gh
;
327 if (likely(file
!= &gfs2_internal_file_sentinal
)) {
329 struct gfs2_file
*gf
= file
->private_data
;
330 if (test_bit(GFF_EXLOCK
, &gf
->f_flags
))
333 gfs2_holder_init(ip
->i_gl
, LM_ST_SHARED
,
334 LM_FLAG_TRY_1CB
|GL_ATIME
|GL_AOP
, &gh
);
336 ret
= gfs2_glock_nq_m_atime(1, &gh
);
337 if (ret
== GLR_TRYFAILED
)
343 if (gfs2_is_stuffed(ip
)) {
344 struct pagevec lru_pvec
;
345 pagevec_init(&lru_pvec
, 0);
346 for (page_idx
= 0; page_idx
< nr_pages
; page_idx
++) {
347 struct page
*page
= list_entry(pages
->prev
, struct page
, lru
);
348 prefetchw(&page
->flags
);
349 list_del(&page
->lru
);
350 if (!add_to_page_cache(page
, mapping
,
351 page
->index
, GFP_KERNEL
)) {
352 ret
= stuffed_readpage(ip
, page
);
354 if (!pagevec_add(&lru_pvec
, page
))
355 __pagevec_lru_add(&lru_pvec
);
357 page_cache_release(page
);
360 pagevec_lru_add(&lru_pvec
);
363 /* What we really want to do .... */
364 ret
= mpage_readpages(mapping
, pages
, nr_pages
, gfs2_get_block
);
368 gfs2_glock_dq_m(1, &gh
);
369 gfs2_holder_uninit(&gh
);
372 if (unlikely(test_bit(SDF_SHUTDOWN
, &sdp
->sd_flags
)))
378 /* unlock all pages, we can't do any I/O right now */
379 for (page_idx
= 0; page_idx
< nr_pages
; page_idx
++) {
380 struct page
*page
= list_entry(pages
->prev
, struct page
, lru
);
381 list_del(&page
->lru
);
383 page_cache_release(page
);
386 gfs2_holder_uninit(&gh
);
391 * gfs2_prepare_write - Prepare to write a page to a file
392 * @file: The file to write to
393 * @page: The page which is to be prepared for writing
394 * @from: From (byte range within page)
395 * @to: To (byte range within page)
400 static int gfs2_prepare_write(struct file
*file
, struct page
*page
,
401 unsigned from
, unsigned to
)
403 struct gfs2_inode
*ip
= GFS2_I(page
->mapping
->host
);
404 struct gfs2_sbd
*sdp
= GFS2_SB(page
->mapping
->host
);
405 unsigned int data_blocks
, ind_blocks
, rblocks
;
408 loff_t pos
= ((loff_t
)page
->index
<< PAGE_CACHE_SHIFT
) + from
;
409 loff_t end
= ((loff_t
)page
->index
<< PAGE_CACHE_SHIFT
) + to
;
410 struct gfs2_alloc
*al
;
412 gfs2_holder_init(ip
->i_gl
, LM_ST_EXCLUSIVE
, GL_ATIME
|GL_AOP
, &ip
->i_gh
);
413 error
= gfs2_glock_nq_m_atime(1, &ip
->i_gh
);
417 gfs2_write_calc_reserv(ip
, to
- from
, &data_blocks
, &ind_blocks
);
419 error
= gfs2_write_alloc_required(ip
, pos
, from
- to
, &alloc_required
);
424 if (alloc_required
) {
425 al
= gfs2_alloc_get(ip
);
427 error
= gfs2_quota_lock(ip
, NO_QUOTA_CHANGE
, NO_QUOTA_CHANGE
);
431 error
= gfs2_quota_check(ip
, ip
->i_di
.di_uid
, ip
->i_di
.di_gid
);
435 al
->al_requested
= data_blocks
+ ind_blocks
;
436 error
= gfs2_inplace_reserve(ip
);
441 rblocks
= RES_DINODE
+ ind_blocks
;
442 if (gfs2_is_jdata(ip
))
443 rblocks
+= data_blocks
? data_blocks
: 1;
444 if (ind_blocks
|| data_blocks
)
445 rblocks
+= RES_STATFS
+ RES_QUOTA
;
447 error
= gfs2_trans_begin(sdp
, rblocks
, 0);
451 if (gfs2_is_stuffed(ip
)) {
452 if (end
> sdp
->sd_sb
.sb_bsize
- sizeof(struct gfs2_dinode
)) {
453 error
= gfs2_unstuff_dinode(ip
, page
);
456 } else if (!PageUptodate(page
))
457 error
= stuffed_readpage(ip
, page
);
462 error
= block_prepare_write(page
, from
, to
, gfs2_get_block
);
467 if (alloc_required
) {
468 gfs2_inplace_release(ip
);
470 gfs2_quota_unlock(ip
);
475 gfs2_glock_dq_m(1, &ip
->i_gh
);
477 gfs2_holder_uninit(&ip
->i_gh
);
484 * gfs2_commit_write - Commit write to a file
485 * @file: The file to write to
486 * @page: The page containing the data
487 * @from: From (byte range within page)
488 * @to: To (byte range within page)
493 static int gfs2_commit_write(struct file
*file
, struct page
*page
,
494 unsigned from
, unsigned to
)
496 struct inode
*inode
= page
->mapping
->host
;
497 struct gfs2_inode
*ip
= GFS2_I(inode
);
498 struct gfs2_sbd
*sdp
= GFS2_SB(inode
);
499 int error
= -EOPNOTSUPP
;
500 struct buffer_head
*dibh
;
501 struct gfs2_alloc
*al
= &ip
->i_alloc
;;
503 if (gfs2_assert_withdraw(sdp
, gfs2_glock_is_locked_by_me(ip
->i_gl
)))
506 error
= gfs2_meta_inode_buffer(ip
, &dibh
);
510 gfs2_trans_add_bh(ip
->i_gl
, dibh
, 1);
512 if (gfs2_is_stuffed(ip
)) {
516 file_size
= ((u64
)page
->index
<< PAGE_CACHE_SHIFT
) + to
;
518 kaddr
= kmap_atomic(page
, KM_USER0
);
519 memcpy(dibh
->b_data
+ sizeof(struct gfs2_dinode
) + from
,
520 (char *)kaddr
+ from
, to
- from
);
521 kunmap_atomic(page
, KM_USER0
);
523 SetPageUptodate(page
);
525 if (inode
->i_size
< file_size
)
526 i_size_write(inode
, file_size
);
528 if (sdp
->sd_args
.ar_data
== GFS2_DATA_ORDERED
||
530 gfs2_page_add_databufs(ip
, page
, from
, to
);
531 error
= generic_commit_write(file
, page
, from
, to
);
536 if (ip
->i_di
.di_size
< inode
->i_size
)
537 ip
->i_di
.di_size
= inode
->i_size
;
539 gfs2_dinode_out(&ip
->i_di
, dibh
->b_data
);
542 if (al
->al_requested
) {
543 gfs2_inplace_release(ip
);
544 gfs2_quota_unlock(ip
);
547 gfs2_glock_dq_m(1, &ip
->i_gh
);
548 gfs2_holder_uninit(&ip
->i_gh
);
555 if (al
->al_requested
) {
556 gfs2_inplace_release(ip
);
557 gfs2_quota_unlock(ip
);
560 gfs2_glock_dq_m(1, &ip
->i_gh
);
561 gfs2_holder_uninit(&ip
->i_gh
);
563 ClearPageUptodate(page
);
568 * gfs2_bmap - Block map function
569 * @mapping: Address space info
570 * @lblock: The block to map
572 * Returns: The disk address for the block or 0 on hole or error
575 static sector_t
gfs2_bmap(struct address_space
*mapping
, sector_t lblock
)
577 struct gfs2_inode
*ip
= GFS2_I(mapping
->host
);
578 struct gfs2_holder i_gh
;
582 error
= gfs2_glock_nq_init(ip
->i_gl
, LM_ST_SHARED
, LM_FLAG_ANY
, &i_gh
);
586 if (!gfs2_is_stuffed(ip
))
587 dblock
= generic_block_bmap(mapping
, lblock
, gfs2_get_block
);
589 gfs2_glock_dq_uninit(&i_gh
);
594 static void discard_buffer(struct gfs2_sbd
*sdp
, struct buffer_head
*bh
)
596 struct gfs2_bufdata
*bd
;
602 bh
->b_private
= NULL
;
604 gfs2_log_unlock(sdp
);
607 clear_buffer_dirty(bh
);
609 clear_buffer_mapped(bh
);
610 clear_buffer_req(bh
);
611 clear_buffer_new(bh
);
612 clear_buffer_delay(bh
);
616 static void gfs2_invalidatepage(struct page
*page
, unsigned long offset
)
618 struct gfs2_sbd
*sdp
= GFS2_SB(page
->mapping
->host
);
619 struct buffer_head
*head
, *bh
, *next
;
620 unsigned int curr_off
= 0;
622 BUG_ON(!PageLocked(page
));
623 if (!page_has_buffers(page
))
626 bh
= head
= page_buffers(page
);
628 unsigned int next_off
= curr_off
+ bh
->b_size
;
629 next
= bh
->b_this_page
;
631 if (offset
<= curr_off
)
632 discard_buffer(sdp
, bh
);
636 } while (bh
!= head
);
639 try_to_release_page(page
, 0);
644 static ssize_t
gfs2_direct_IO(int rw
, struct kiocb
*iocb
,
645 const struct iovec
*iov
, loff_t offset
,
646 unsigned long nr_segs
)
648 struct file
*file
= iocb
->ki_filp
;
649 struct inode
*inode
= file
->f_mapping
->host
;
650 struct gfs2_inode
*ip
= GFS2_I(inode
);
651 struct gfs2_holder gh
;
655 mutex_lock(&inode
->i_mutex
);
657 * Shared lock, even if its a write, since we do no allocation
658 * on this path. All we need change is atime.
660 gfs2_holder_init(ip
->i_gl
, LM_ST_SHARED
, GL_ATIME
, &gh
);
661 rv
= gfs2_glock_nq_m_atime(1, &gh
);
665 if (offset
> i_size_read(inode
))
669 * Should we return an error here? I can't see that O_DIRECT for
670 * a journaled file makes any sense. For now we'll silently fall
671 * back to buffered I/O, likewise we do the same for stuffed
672 * files since they are (a) small and (b) unaligned.
674 if (gfs2_is_jdata(ip
))
677 if (gfs2_is_stuffed(ip
))
680 rv
= blockdev_direct_IO_own_locking(rw
, iocb
, inode
,
682 iov
, offset
, nr_segs
,
683 get_block_direct
, NULL
);
685 gfs2_glock_dq_m(1, &gh
);
686 gfs2_holder_uninit(&gh
);
688 mutex_unlock(&inode
->i_mutex
);
694 * stuck_releasepage - We're stuck in gfs2_releasepage(). Print stuff out.
695 * @bh: the buffer we're stuck on
699 static void stuck_releasepage(struct buffer_head
*bh
)
701 struct inode
*inode
= bh
->b_page
->mapping
->host
;
702 struct gfs2_sbd
*sdp
= inode
->i_sb
->s_fs_info
;
703 struct gfs2_bufdata
*bd
= bh
->b_private
;
704 struct gfs2_glock
*gl
;
705 static unsigned limit
= 0;
710 fs_warn(sdp
, "stuck in gfs2_releasepage() %p\n", inode
);
711 fs_warn(sdp
, "blkno = %llu, bh->b_count = %d\n",
712 (unsigned long long)bh
->b_blocknr
, atomic_read(&bh
->b_count
));
713 fs_warn(sdp
, "pinned = %u\n", buffer_pinned(bh
));
714 fs_warn(sdp
, "bh->b_private = %s\n", (bd
) ? "!NULL" : "NULL");
721 fs_warn(sdp
, "gl = (%u, %llu)\n",
722 gl
->gl_name
.ln_type
, (unsigned long long)gl
->gl_name
.ln_number
);
724 fs_warn(sdp
, "bd_list_tr = %s, bd_le.le_list = %s\n",
725 (list_empty(&bd
->bd_list_tr
)) ? "no" : "yes",
726 (list_empty(&bd
->bd_le
.le_list
)) ? "no" : "yes");
728 if (gl
->gl_ops
== &gfs2_inode_glops
) {
729 struct gfs2_inode
*ip
= gl
->gl_object
;
735 fs_warn(sdp
, "ip = %llu %llu\n",
736 (unsigned long long)ip
->i_num
.no_formal_ino
,
737 (unsigned long long)ip
->i_num
.no_addr
);
739 for (x
= 0; x
< GFS2_MAX_META_HEIGHT
; x
++)
740 fs_warn(sdp
, "ip->i_cache[%u] = %s\n",
741 x
, (ip
->i_cache
[x
]) ? "!NULL" : "NULL");
746 * gfs2_releasepage - free the metadata associated with a page
747 * @page: the page that's being released
748 * @gfp_mask: passed from Linux VFS, ignored by us
750 * Call try_to_free_buffers() if the buffers in this page can be
756 int gfs2_releasepage(struct page
*page
, gfp_t gfp_mask
)
758 struct inode
*aspace
= page
->mapping
->host
;
759 struct gfs2_sbd
*sdp
= aspace
->i_sb
->s_fs_info
;
760 struct buffer_head
*bh
, *head
;
761 struct gfs2_bufdata
*bd
;
762 unsigned long t
= jiffies
+ gfs2_tune_get(sdp
, gt_stall_secs
) * HZ
;
764 if (!page_has_buffers(page
))
767 head
= bh
= page_buffers(page
);
769 while (atomic_read(&bh
->b_count
)) {
770 if (!atomic_read(&aspace
->i_writecount
))
773 if (time_after_eq(jiffies
, t
)) {
774 stuck_releasepage(bh
);
775 /* should we withdraw here? */
782 gfs2_assert_warn(sdp
, !buffer_pinned(bh
));
783 gfs2_assert_warn(sdp
, !buffer_dirty(bh
));
788 gfs2_assert_warn(sdp
, bd
->bd_bh
== bh
);
789 gfs2_assert_warn(sdp
, list_empty(&bd
->bd_list_tr
));
790 gfs2_assert_warn(sdp
, !bd
->bd_ail
);
792 if (!list_empty(&bd
->bd_le
.le_list
))
794 bh
->b_private
= NULL
;
796 gfs2_log_unlock(sdp
);
798 kmem_cache_free(gfs2_bufdata_cachep
, bd
);
800 bh
= bh
->b_this_page
;
801 } while (bh
!= head
);
804 return try_to_free_buffers(page
);
807 const struct address_space_operations gfs2_file_aops
= {
808 .writepage
= gfs2_writepage
,
809 .readpage
= gfs2_readpage
,
810 .readpages
= gfs2_readpages
,
811 .sync_page
= block_sync_page
,
812 .prepare_write
= gfs2_prepare_write
,
813 .commit_write
= gfs2_commit_write
,
815 .invalidatepage
= gfs2_invalidatepage
,
816 .releasepage
= gfs2_releasepage
,
817 .direct_IO
= gfs2_direct_IO
,