2 * fs/dax.c - Direct Access filesystem code
3 * Copyright (c) 2013-2014 Intel Corporation
4 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
5 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 #include <linux/atomic.h>
18 #include <linux/blkdev.h>
19 #include <linux/buffer_head.h>
21 #include <linux/genhd.h>
22 #include <linux/highmem.h>
23 #include <linux/memcontrol.h>
25 #include <linux/mutex.h>
26 #include <linux/sched.h>
27 #include <linux/uio.h>
28 #include <linux/vmstat.h>
30 int dax_clear_blocks(struct inode
*inode
, sector_t block
, long size
)
32 struct block_device
*bdev
= inode
->i_sb
->s_bdev
;
33 sector_t sector
= block
<< (inode
->i_blkbits
- 9);
41 count
= bdev_direct_access(bdev
, sector
, &addr
, &pfn
, size
);
46 unsigned pgsz
= PAGE_SIZE
- offset_in_page(addr
);
50 memset(addr
, 0, pgsz
);
64 EXPORT_SYMBOL_GPL(dax_clear_blocks
);
66 static long dax_get_addr(struct buffer_head
*bh
, void **addr
, unsigned blkbits
)
69 sector_t sector
= bh
->b_blocknr
<< (blkbits
- 9);
70 return bdev_direct_access(bh
->b_bdev
, sector
, addr
, &pfn
, bh
->b_size
);
73 static void dax_new_buf(void *addr
, unsigned size
, unsigned first
, loff_t pos
,
76 loff_t final
= end
- pos
+ first
; /* The final byte of the buffer */
79 memset(addr
, 0, first
);
81 memset(addr
+ final
, 0, size
- final
);
84 static bool buffer_written(struct buffer_head
*bh
)
86 return buffer_mapped(bh
) && !buffer_unwritten(bh
);
90 * When ext4 encounters a hole, it returns without modifying the buffer_head
91 * which means that we can't trust b_size. To cope with this, we set b_state
92 * to 0 before calling get_block and, if any bit is set, we know we can trust
93 * b_size. Unfortunate, really, since ext4 knows precisely how long a hole is
94 * and would save us time calling get_block repeatedly.
96 static bool buffer_size_valid(struct buffer_head
*bh
)
98 return bh
->b_state
!= 0;
101 static ssize_t
dax_io(struct inode
*inode
, struct iov_iter
*iter
,
102 loff_t start
, loff_t end
, get_block_t get_block
,
103 struct buffer_head
*bh
)
108 loff_t bh_max
= start
;
112 if (iov_iter_rw(iter
) != WRITE
)
113 end
= min(end
, i_size_read(inode
));
118 unsigned blkbits
= inode
->i_blkbits
;
119 sector_t block
= pos
>> blkbits
;
120 unsigned first
= pos
- (block
<< blkbits
);
124 bh
->b_size
= PAGE_ALIGN(end
- pos
);
126 retval
= get_block(inode
, block
, bh
,
127 iov_iter_rw(iter
) == WRITE
);
130 if (!buffer_size_valid(bh
))
131 bh
->b_size
= 1 << blkbits
;
132 bh_max
= pos
- first
+ bh
->b_size
;
134 unsigned done
= bh
->b_size
-
135 (bh_max
- (pos
- first
));
136 bh
->b_blocknr
+= done
>> blkbits
;
140 hole
= iov_iter_rw(iter
) != WRITE
&& !buffer_written(bh
);
143 size
= bh
->b_size
- first
;
145 retval
= dax_get_addr(bh
, &addr
, blkbits
);
148 if (buffer_unwritten(bh
) || buffer_new(bh
))
149 dax_new_buf(addr
, retval
, first
, pos
,
152 size
= retval
- first
;
154 max
= min(pos
+ size
, end
);
157 if (iov_iter_rw(iter
) == WRITE
)
158 len
= copy_from_iter_nocache(addr
, max
- pos
, iter
);
160 len
= copy_to_iter(addr
, max
- pos
, iter
);
162 len
= iov_iter_zero(max
- pos
, iter
);
171 return (pos
== start
) ? retval
: pos
- start
;
175 * dax_do_io - Perform I/O to a DAX file
176 * @iocb: The control block for this I/O
177 * @inode: The file which the I/O is directed at
178 * @iter: The addresses to do I/O from or to
179 * @pos: The file offset where the I/O starts
180 * @get_block: The filesystem method used to translate file offsets to blocks
181 * @end_io: A filesystem callback for I/O completion
184 * This function uses the same locking scheme as do_blockdev_direct_IO:
185 * If @flags has DIO_LOCKING set, we assume that the i_mutex is held by the
186 * caller for writes. For reads, we take and release the i_mutex ourselves.
187 * If DIO_LOCKING is not set, the filesystem takes care of its own locking.
188 * As with do_blockdev_direct_IO(), we increment i_dio_count while the I/O
191 ssize_t
dax_do_io(struct kiocb
*iocb
, struct inode
*inode
,
192 struct iov_iter
*iter
, loff_t pos
, get_block_t get_block
,
193 dio_iodone_t end_io
, int flags
)
195 struct buffer_head bh
;
196 ssize_t retval
= -EINVAL
;
197 loff_t end
= pos
+ iov_iter_count(iter
);
199 memset(&bh
, 0, sizeof(bh
));
201 if ((flags
& DIO_LOCKING
) && iov_iter_rw(iter
) == READ
) {
202 struct address_space
*mapping
= inode
->i_mapping
;
203 mutex_lock(&inode
->i_mutex
);
204 retval
= filemap_write_and_wait_range(mapping
, pos
, end
- 1);
206 mutex_unlock(&inode
->i_mutex
);
211 /* Protects against truncate */
212 if (!(flags
& DIO_SKIP_DIO_COUNT
))
213 inode_dio_begin(inode
);
215 retval
= dax_io(inode
, iter
, pos
, end
, get_block
, &bh
);
217 if ((flags
& DIO_LOCKING
) && iov_iter_rw(iter
) == READ
)
218 mutex_unlock(&inode
->i_mutex
);
220 if ((retval
> 0) && end_io
)
221 end_io(iocb
, pos
, retval
, bh
.b_private
);
223 if (!(flags
& DIO_SKIP_DIO_COUNT
))
224 inode_dio_end(inode
);
228 EXPORT_SYMBOL_GPL(dax_do_io
);
231 * The user has performed a load from a hole in the file. Allocating
232 * a new page in the file would cause excessive storage usage for
233 * workloads with sparse files. We allocate a page cache page instead.
234 * We'll kick it out of the page cache if it's ever written to,
235 * otherwise it will simply fall out of the page cache under memory
236 * pressure without ever having been dirtied.
238 static int dax_load_hole(struct address_space
*mapping
, struct page
*page
,
239 struct vm_fault
*vmf
)
242 struct inode
*inode
= mapping
->host
;
244 page
= find_or_create_page(mapping
, vmf
->pgoff
,
245 GFP_KERNEL
| __GFP_ZERO
);
248 /* Recheck i_size under page lock to avoid truncate race */
249 size
= (i_size_read(inode
) + PAGE_SIZE
- 1) >> PAGE_SHIFT
;
250 if (vmf
->pgoff
>= size
) {
252 page_cache_release(page
);
253 return VM_FAULT_SIGBUS
;
257 return VM_FAULT_LOCKED
;
260 static int copy_user_bh(struct page
*to
, struct buffer_head
*bh
,
261 unsigned blkbits
, unsigned long vaddr
)
264 if (dax_get_addr(bh
, &vfrom
, blkbits
) < 0)
266 vto
= kmap_atomic(to
);
267 copy_user_page(vto
, vfrom
, vaddr
, to
);
272 static int dax_insert_mapping(struct inode
*inode
, struct buffer_head
*bh
,
273 struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
275 sector_t sector
= bh
->b_blocknr
<< (inode
->i_blkbits
- 9);
276 unsigned long vaddr
= (unsigned long)vmf
->virtual_address
;
283 * Check truncate didn't happen while we were allocating a block.
284 * If it did, this block may or may not be still allocated to the
285 * file. We can't tell the filesystem to free it because we can't
286 * take i_mutex here. In the worst case, the file still has blocks
287 * allocated past the end of the file.
289 size
= (i_size_read(inode
) + PAGE_SIZE
- 1) >> PAGE_SHIFT
;
290 if (unlikely(vmf
->pgoff
>= size
)) {
295 error
= bdev_direct_access(bh
->b_bdev
, sector
, &addr
, &pfn
, bh
->b_size
);
298 if (error
< PAGE_SIZE
) {
303 if (buffer_unwritten(bh
) || buffer_new(bh
))
306 error
= vm_insert_mixed(vma
, vaddr
, pfn
);
313 * __dax_fault - handle a page fault on a DAX file
314 * @vma: The virtual memory area where the fault occurred
315 * @vmf: The description of the fault
316 * @get_block: The filesystem method used to translate file offsets to blocks
317 * @complete_unwritten: The filesystem method used to convert unwritten blocks
318 * to written so the data written to them is exposed. This is required for
319 * required by write faults for filesystems that will return unwritten
320 * extent mappings from @get_block, but it is optional for reads as
321 * dax_insert_mapping() will always zero unwritten blocks. If the fs does
322 * not support unwritten extents, the it should pass NULL.
324 * When a page fault occurs, filesystems may call this helper in their
325 * fault handler for DAX files. __dax_fault() assumes the caller has done all
326 * the necessary locking for the page fault to proceed successfully.
328 int __dax_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
,
329 get_block_t get_block
, dax_iodone_t complete_unwritten
)
331 struct file
*file
= vma
->vm_file
;
332 struct address_space
*mapping
= file
->f_mapping
;
333 struct inode
*inode
= mapping
->host
;
335 struct buffer_head bh
;
336 unsigned long vaddr
= (unsigned long)vmf
->virtual_address
;
337 unsigned blkbits
= inode
->i_blkbits
;
343 size
= (i_size_read(inode
) + PAGE_SIZE
- 1) >> PAGE_SHIFT
;
344 if (vmf
->pgoff
>= size
)
345 return VM_FAULT_SIGBUS
;
347 memset(&bh
, 0, sizeof(bh
));
348 block
= (sector_t
)vmf
->pgoff
<< (PAGE_SHIFT
- blkbits
);
349 bh
.b_size
= PAGE_SIZE
;
352 page
= find_get_page(mapping
, vmf
->pgoff
);
354 if (!lock_page_or_retry(page
, vma
->vm_mm
, vmf
->flags
)) {
355 page_cache_release(page
);
356 return VM_FAULT_RETRY
;
358 if (unlikely(page
->mapping
!= mapping
)) {
360 page_cache_release(page
);
363 size
= (i_size_read(inode
) + PAGE_SIZE
- 1) >> PAGE_SHIFT
;
364 if (unlikely(vmf
->pgoff
>= size
)) {
366 * We have a struct page covering a hole in the file
367 * from a read fault and we've raced with a truncate
373 i_mmap_lock_write(mapping
);
376 error
= get_block(inode
, block
, &bh
, 0);
377 if (!error
&& (bh
.b_size
< PAGE_SIZE
))
378 error
= -EIO
; /* fs corruption? */
382 if (!buffer_mapped(&bh
) && !buffer_unwritten(&bh
) && !vmf
->cow_page
) {
383 if (vmf
->flags
& FAULT_FLAG_WRITE
) {
384 error
= get_block(inode
, block
, &bh
, 1);
385 count_vm_event(PGMAJFAULT
);
386 mem_cgroup_count_vm_event(vma
->vm_mm
, PGMAJFAULT
);
387 major
= VM_FAULT_MAJOR
;
388 if (!error
&& (bh
.b_size
< PAGE_SIZE
))
393 i_mmap_unlock_write(mapping
);
394 return dax_load_hole(mapping
, page
, vmf
);
399 struct page
*new_page
= vmf
->cow_page
;
400 if (buffer_written(&bh
))
401 error
= copy_user_bh(new_page
, &bh
, blkbits
, vaddr
);
403 clear_user_highpage(new_page
, vaddr
);
408 /* Check we didn't race with truncate */
409 size
= (i_size_read(inode
) + PAGE_SIZE
- 1) >>
411 if (vmf
->pgoff
>= size
) {
416 return VM_FAULT_LOCKED
;
419 /* Check we didn't race with a read fault installing a new page */
421 page
= find_lock_page(mapping
, vmf
->pgoff
);
424 unmap_mapping_range(mapping
, vmf
->pgoff
<< PAGE_SHIFT
,
426 delete_from_page_cache(page
);
428 page_cache_release(page
);
432 * If we successfully insert the new mapping over an unwritten extent,
433 * we need to ensure we convert the unwritten extent. If there is an
434 * error inserting the mapping, the filesystem needs to leave it as
435 * unwritten to prevent exposure of the stale underlying data to
436 * userspace, but we still need to call the completion function so
437 * the private resources on the mapping buffer can be released. We
438 * indicate what the callback should do via the uptodate variable, same
439 * as for normal BH based IO completions.
441 error
= dax_insert_mapping(inode
, &bh
, vma
, vmf
);
442 if (buffer_unwritten(&bh
)) {
443 if (complete_unwritten
)
444 complete_unwritten(&bh
, !error
);
446 WARN_ON_ONCE(!(vmf
->flags
& FAULT_FLAG_WRITE
));
450 i_mmap_unlock_write(mapping
);
452 if (error
== -ENOMEM
)
453 return VM_FAULT_OOM
| major
;
454 /* -EBUSY is fine, somebody else faulted on the same PTE */
455 if ((error
< 0) && (error
!= -EBUSY
))
456 return VM_FAULT_SIGBUS
| major
;
457 return VM_FAULT_NOPAGE
| major
;
462 page_cache_release(page
);
464 i_mmap_unlock_write(mapping
);
469 EXPORT_SYMBOL(__dax_fault
);
472 * dax_fault - handle a page fault on a DAX file
473 * @vma: The virtual memory area where the fault occurred
474 * @vmf: The description of the fault
475 * @get_block: The filesystem method used to translate file offsets to blocks
477 * When a page fault occurs, filesystems may call this helper in their
478 * fault handler for DAX files.
480 int dax_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
,
481 get_block_t get_block
, dax_iodone_t complete_unwritten
)
484 struct super_block
*sb
= file_inode(vma
->vm_file
)->i_sb
;
486 if (vmf
->flags
& FAULT_FLAG_WRITE
) {
487 sb_start_pagefault(sb
);
488 file_update_time(vma
->vm_file
);
490 result
= __dax_fault(vma
, vmf
, get_block
, complete_unwritten
);
491 if (vmf
->flags
& FAULT_FLAG_WRITE
)
492 sb_end_pagefault(sb
);
496 EXPORT_SYMBOL_GPL(dax_fault
);
498 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
500 * The 'colour' (ie low bits) within a PMD of a page offset. This comes up
501 * more often than one might expect in the below function.
503 #define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)
505 int __dax_pmd_fault(struct vm_area_struct
*vma
, unsigned long address
,
506 pmd_t
*pmd
, unsigned int flags
, get_block_t get_block
,
507 dax_iodone_t complete_unwritten
)
509 struct file
*file
= vma
->vm_file
;
510 struct address_space
*mapping
= file
->f_mapping
;
511 struct inode
*inode
= mapping
->host
;
512 struct buffer_head bh
;
513 unsigned blkbits
= inode
->i_blkbits
;
514 unsigned long pmd_addr
= address
& PMD_MASK
;
515 bool write
= flags
& FAULT_FLAG_WRITE
;
519 sector_t block
, sector
;
523 /* Fall back to PTEs if we're going to COW */
524 if (write
&& !(vma
->vm_flags
& VM_SHARED
))
525 return VM_FAULT_FALLBACK
;
526 /* If the PMD would extend outside the VMA */
527 if (pmd_addr
< vma
->vm_start
)
528 return VM_FAULT_FALLBACK
;
529 if ((pmd_addr
+ PMD_SIZE
) > vma
->vm_end
)
530 return VM_FAULT_FALLBACK
;
532 pgoff
= ((pmd_addr
- vma
->vm_start
) >> PAGE_SHIFT
) + vma
->vm_pgoff
;
533 size
= (i_size_read(inode
) + PAGE_SIZE
- 1) >> PAGE_SHIFT
;
535 return VM_FAULT_SIGBUS
;
536 /* If the PMD would cover blocks out of the file */
537 if ((pgoff
| PG_PMD_COLOUR
) >= size
)
538 return VM_FAULT_FALLBACK
;
540 memset(&bh
, 0, sizeof(bh
));
541 block
= (sector_t
)pgoff
<< (PAGE_SHIFT
- blkbits
);
543 bh
.b_size
= PMD_SIZE
;
544 i_mmap_lock_write(mapping
);
545 length
= get_block(inode
, block
, &bh
, write
);
547 return VM_FAULT_SIGBUS
;
550 * If the filesystem isn't willing to tell us the length of a hole,
551 * just fall back to PTEs. Calling get_block 512 times in a loop
554 if (!buffer_size_valid(&bh
) || bh
.b_size
< PMD_SIZE
)
558 * If a truncate happened while we were allocating blocks, we may
559 * leave blocks allocated to the file that are beyond EOF. We can't
560 * take i_mutex here, so just leave them hanging; they'll be freed
561 * when the file is deleted.
563 size
= (i_size_read(inode
) + PAGE_SIZE
- 1) >> PAGE_SHIFT
;
565 result
= VM_FAULT_SIGBUS
;
568 if ((pgoff
| PG_PMD_COLOUR
) >= size
)
571 if (is_huge_zero_pmd(*pmd
))
572 unmap_mapping_range(mapping
, pgoff
<< PAGE_SHIFT
, PMD_SIZE
, 0);
574 if (!write
&& !buffer_mapped(&bh
) && buffer_uptodate(&bh
)) {
577 struct page
*zero_page
= get_huge_zero_page();
579 if (unlikely(!zero_page
))
582 ptl
= pmd_lock(vma
->vm_mm
, pmd
);
583 if (!pmd_none(*pmd
)) {
588 entry
= mk_pmd(zero_page
, vma
->vm_page_prot
);
589 entry
= pmd_mkhuge(entry
);
590 set_pmd_at(vma
->vm_mm
, pmd_addr
, pmd
, entry
);
591 result
= VM_FAULT_NOPAGE
;
594 sector
= bh
.b_blocknr
<< (blkbits
- 9);
595 length
= bdev_direct_access(bh
.b_bdev
, sector
, &kaddr
, &pfn
,
598 result
= VM_FAULT_SIGBUS
;
601 if ((length
< PMD_SIZE
) || (pfn
& PG_PMD_COLOUR
))
604 if (buffer_unwritten(&bh
) || buffer_new(&bh
)) {
606 for (i
= 0; i
< PTRS_PER_PMD
; i
++)
607 clear_page(kaddr
+ i
* PAGE_SIZE
);
608 count_vm_event(PGMAJFAULT
);
609 mem_cgroup_count_vm_event(vma
->vm_mm
, PGMAJFAULT
);
610 result
|= VM_FAULT_MAJOR
;
613 result
|= vmf_insert_pfn_pmd(vma
, address
, pmd
, pfn
, write
);
617 if (buffer_unwritten(&bh
))
618 complete_unwritten(&bh
, !(result
& VM_FAULT_ERROR
));
620 i_mmap_unlock_write(mapping
);
625 count_vm_event(THP_FAULT_FALLBACK
);
626 result
= VM_FAULT_FALLBACK
;
629 EXPORT_SYMBOL_GPL(__dax_pmd_fault
);
632 * dax_pmd_fault - handle a PMD fault on a DAX file
633 * @vma: The virtual memory area where the fault occurred
634 * @vmf: The description of the fault
635 * @get_block: The filesystem method used to translate file offsets to blocks
637 * When a page fault occurs, filesystems may call this helper in their
638 * pmd_fault handler for DAX files.
640 int dax_pmd_fault(struct vm_area_struct
*vma
, unsigned long address
,
641 pmd_t
*pmd
, unsigned int flags
, get_block_t get_block
,
642 dax_iodone_t complete_unwritten
)
645 struct super_block
*sb
= file_inode(vma
->vm_file
)->i_sb
;
647 if (flags
& FAULT_FLAG_WRITE
) {
648 sb_start_pagefault(sb
);
649 file_update_time(vma
->vm_file
);
651 result
= __dax_pmd_fault(vma
, address
, pmd
, flags
, get_block
,
653 if (flags
& FAULT_FLAG_WRITE
)
654 sb_end_pagefault(sb
);
658 EXPORT_SYMBOL_GPL(dax_pmd_fault
);
659 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
662 * dax_pfn_mkwrite - handle first write to DAX page
663 * @vma: The virtual memory area where the fault occurred
664 * @vmf: The description of the fault
667 int dax_pfn_mkwrite(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
669 struct super_block
*sb
= file_inode(vma
->vm_file
)->i_sb
;
671 sb_start_pagefault(sb
);
672 file_update_time(vma
->vm_file
);
673 sb_end_pagefault(sb
);
674 return VM_FAULT_NOPAGE
;
676 EXPORT_SYMBOL_GPL(dax_pfn_mkwrite
);
679 * dax_zero_page_range - zero a range within a page of a DAX file
680 * @inode: The file being truncated
681 * @from: The file offset that is being truncated to
682 * @length: The number of bytes to zero
683 * @get_block: The filesystem method used to translate file offsets to blocks
685 * This function can be called by a filesystem when it is zeroing part of a
686 * page in a DAX file. This is intended for hole-punch operations. If
687 * you are truncating a file, the helper function dax_truncate_page() may be
690 * We work in terms of PAGE_CACHE_SIZE here for commonality with
691 * block_truncate_page(), but we could go down to PAGE_SIZE if the filesystem
692 * took care of disposing of the unnecessary blocks. Even if the filesystem
693 * block size is smaller than PAGE_SIZE, we have to zero the rest of the page
694 * since the file might be mmapped.
696 int dax_zero_page_range(struct inode
*inode
, loff_t from
, unsigned length
,
697 get_block_t get_block
)
699 struct buffer_head bh
;
700 pgoff_t index
= from
>> PAGE_CACHE_SHIFT
;
701 unsigned offset
= from
& (PAGE_CACHE_SIZE
-1);
704 /* Block boundary? Nothing to do */
707 BUG_ON((offset
+ length
) > PAGE_CACHE_SIZE
);
709 memset(&bh
, 0, sizeof(bh
));
710 bh
.b_size
= PAGE_CACHE_SIZE
;
711 err
= get_block(inode
, index
, &bh
, 0);
714 if (buffer_written(&bh
)) {
716 err
= dax_get_addr(&bh
, &addr
, inode
->i_blkbits
);
719 memset(addr
+ offset
, 0, length
);
724 EXPORT_SYMBOL_GPL(dax_zero_page_range
);
727 * dax_truncate_page - handle a partial page being truncated in a DAX file
728 * @inode: The file being truncated
729 * @from: The file offset that is being truncated to
730 * @get_block: The filesystem method used to translate file offsets to blocks
732 * Similar to block_truncate_page(), this function can be called by a
733 * filesystem when it is truncating a DAX file to handle the partial page.
735 * We work in terms of PAGE_CACHE_SIZE here for commonality with
736 * block_truncate_page(), but we could go down to PAGE_SIZE if the filesystem
737 * took care of disposing of the unnecessary blocks. Even if the filesystem
738 * block size is smaller than PAGE_SIZE, we have to zero the rest of the page
739 * since the file might be mmapped.
741 int dax_truncate_page(struct inode
*inode
, loff_t from
, get_block_t get_block
)
743 unsigned length
= PAGE_CACHE_ALIGN(from
) - from
;
744 return dax_zero_page_range(inode
, from
, length
, get_block
);
746 EXPORT_SYMBOL_GPL(dax_truncate_page
);