Commit | Line | Data |
---|---|---|
d475c634 MW |
1 | /* |
2 | * fs/dax.c - Direct Access filesystem code | |
3 | * Copyright (c) 2013-2014 Intel Corporation | |
4 | * Author: Matthew Wilcox <matthew.r.wilcox@intel.com> | |
5 | * Author: Ross Zwisler <ross.zwisler@linux.intel.com> | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify it | |
8 | * under the terms and conditions of the GNU General Public License, | |
9 | * version 2, as published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope it will be useful, but WITHOUT | |
12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
14 | * more details. | |
15 | */ | |
16 | ||
17 | #include <linux/atomic.h> | |
18 | #include <linux/blkdev.h> | |
19 | #include <linux/buffer_head.h> | |
d77e92e2 | 20 | #include <linux/dax.h> |
d475c634 MW |
21 | #include <linux/fs.h> |
22 | #include <linux/genhd.h> | |
f7ca90b1 MW |
23 | #include <linux/highmem.h> |
24 | #include <linux/memcontrol.h> | |
25 | #include <linux/mm.h> | |
d475c634 | 26 | #include <linux/mutex.h> |
2765cfbb | 27 | #include <linux/pmem.h> |
289c6aed | 28 | #include <linux/sched.h> |
d475c634 | 29 | #include <linux/uio.h> |
f7ca90b1 | 30 | #include <linux/vmstat.h> |
0e749e54 | 31 | #include <linux/sizes.h> |
d475c634 | 32 | |
b2e0d162 DW |
33 | static long dax_map_atomic(struct block_device *bdev, struct blk_dax_ctl *dax) |
34 | { | |
35 | struct request_queue *q = bdev->bd_queue; | |
36 | long rc = -EIO; | |
37 | ||
38 | dax->addr = (void __pmem *) ERR_PTR(-EIO); | |
39 | if (blk_queue_enter(q, true) != 0) | |
40 | return rc; | |
41 | ||
42 | rc = bdev_direct_access(bdev, dax); | |
43 | if (rc < 0) { | |
44 | dax->addr = (void __pmem *) ERR_PTR(rc); | |
45 | blk_queue_exit(q); | |
46 | return rc; | |
47 | } | |
48 | return rc; | |
49 | } | |
50 | ||
51 | static void dax_unmap_atomic(struct block_device *bdev, | |
52 | const struct blk_dax_ctl *dax) | |
53 | { | |
54 | if (IS_ERR(dax->addr)) | |
55 | return; | |
56 | blk_queue_exit(bdev->bd_queue); | |
57 | } | |
58 | ||
1ca19157 DC |
59 | /* |
60 | * dax_clear_blocks() is called from within transaction context from XFS, | |
61 | * and hence this means the stack from this point must follow GFP_NOFS | |
62 | * semantics for all operations. | |
63 | */ | |
b2e0d162 | 64 | int dax_clear_blocks(struct inode *inode, sector_t block, long _size) |
289c6aed MW |
65 | { |
66 | struct block_device *bdev = inode->i_sb->s_bdev; | |
b2e0d162 DW |
67 | struct blk_dax_ctl dax = { |
68 | .sector = block << (inode->i_blkbits - 9), | |
69 | .size = _size, | |
70 | }; | |
289c6aed MW |
71 | |
72 | might_sleep(); | |
73 | do { | |
0e749e54 | 74 | long count, sz; |
289c6aed | 75 | |
b2e0d162 | 76 | count = dax_map_atomic(bdev, &dax); |
289c6aed MW |
77 | if (count < 0) |
78 | return count; | |
0e749e54 | 79 | sz = min_t(long, count, SZ_128K); |
b2e0d162 DW |
80 | clear_pmem(dax.addr, sz); |
81 | dax.size -= sz; | |
82 | dax.sector += sz / 512; | |
83 | dax_unmap_atomic(bdev, &dax); | |
0e749e54 | 84 | cond_resched(); |
b2e0d162 | 85 | } while (dax.size); |
289c6aed | 86 | |
2765cfbb | 87 | wmb_pmem(); |
289c6aed MW |
88 | return 0; |
89 | } | |
90 | EXPORT_SYMBOL_GPL(dax_clear_blocks); | |
91 | ||
2765cfbb | 92 | /* the clear_pmem() calls are ordered by a wmb_pmem() in the caller */ |
e2e05394 RZ |
93 | static void dax_new_buf(void __pmem *addr, unsigned size, unsigned first, |
94 | loff_t pos, loff_t end) | |
d475c634 MW |
95 | { |
96 | loff_t final = end - pos + first; /* The final byte of the buffer */ | |
97 | ||
98 | if (first > 0) | |
e2e05394 | 99 | clear_pmem(addr, first); |
d475c634 | 100 | if (final < size) |
e2e05394 | 101 | clear_pmem(addr + final, size - final); |
d475c634 MW |
102 | } |
103 | ||
104 | static bool buffer_written(struct buffer_head *bh) | |
105 | { | |
106 | return buffer_mapped(bh) && !buffer_unwritten(bh); | |
107 | } | |
108 | ||
109 | /* | |
110 | * When ext4 encounters a hole, it returns without modifying the buffer_head | |
111 | * which means that we can't trust b_size. To cope with this, we set b_state | |
112 | * to 0 before calling get_block and, if any bit is set, we know we can trust | |
113 | * b_size. Unfortunate, really, since ext4 knows precisely how long a hole is | |
114 | * and would save us time calling get_block repeatedly. | |
115 | */ | |
116 | static bool buffer_size_valid(struct buffer_head *bh) | |
117 | { | |
118 | return bh->b_state != 0; | |
119 | } | |
120 | ||
b2e0d162 DW |
121 | |
122 | static sector_t to_sector(const struct buffer_head *bh, | |
123 | const struct inode *inode) | |
124 | { | |
125 | sector_t sector = bh->b_blocknr << (inode->i_blkbits - 9); | |
126 | ||
127 | return sector; | |
128 | } | |
129 | ||
a95cd631 OS |
130 | static ssize_t dax_io(struct inode *inode, struct iov_iter *iter, |
131 | loff_t start, loff_t end, get_block_t get_block, | |
132 | struct buffer_head *bh) | |
d475c634 | 133 | { |
b2e0d162 DW |
134 | loff_t pos = start, max = start, bh_max = start; |
135 | bool hole = false, need_wmb = false; | |
136 | struct block_device *bdev = NULL; | |
137 | int rw = iov_iter_rw(iter), rc; | |
138 | long map_len = 0; | |
139 | struct blk_dax_ctl dax = { | |
140 | .addr = (void __pmem *) ERR_PTR(-EIO), | |
141 | }; | |
142 | ||
143 | if (rw == READ) | |
d475c634 MW |
144 | end = min(end, i_size_read(inode)); |
145 | ||
146 | while (pos < end) { | |
2765cfbb | 147 | size_t len; |
d475c634 MW |
148 | if (pos == max) { |
149 | unsigned blkbits = inode->i_blkbits; | |
e94f5a22 JM |
150 | long page = pos >> PAGE_SHIFT; |
151 | sector_t block = page << (PAGE_SHIFT - blkbits); | |
d475c634 MW |
152 | unsigned first = pos - (block << blkbits); |
153 | long size; | |
154 | ||
155 | if (pos == bh_max) { | |
156 | bh->b_size = PAGE_ALIGN(end - pos); | |
157 | bh->b_state = 0; | |
b2e0d162 DW |
158 | rc = get_block(inode, block, bh, rw == WRITE); |
159 | if (rc) | |
d475c634 MW |
160 | break; |
161 | if (!buffer_size_valid(bh)) | |
162 | bh->b_size = 1 << blkbits; | |
163 | bh_max = pos - first + bh->b_size; | |
b2e0d162 | 164 | bdev = bh->b_bdev; |
d475c634 MW |
165 | } else { |
166 | unsigned done = bh->b_size - | |
167 | (bh_max - (pos - first)); | |
168 | bh->b_blocknr += done >> blkbits; | |
169 | bh->b_size -= done; | |
170 | } | |
171 | ||
b2e0d162 | 172 | hole = rw == READ && !buffer_written(bh); |
d475c634 | 173 | if (hole) { |
d475c634 MW |
174 | size = bh->b_size - first; |
175 | } else { | |
b2e0d162 DW |
176 | dax_unmap_atomic(bdev, &dax); |
177 | dax.sector = to_sector(bh, inode); | |
178 | dax.size = bh->b_size; | |
179 | map_len = dax_map_atomic(bdev, &dax); | |
180 | if (map_len < 0) { | |
181 | rc = map_len; | |
d475c634 | 182 | break; |
b2e0d162 | 183 | } |
2765cfbb | 184 | if (buffer_unwritten(bh) || buffer_new(bh)) { |
b2e0d162 DW |
185 | dax_new_buf(dax.addr, map_len, first, |
186 | pos, end); | |
2765cfbb RZ |
187 | need_wmb = true; |
188 | } | |
b2e0d162 DW |
189 | dax.addr += first; |
190 | size = map_len - first; | |
d475c634 MW |
191 | } |
192 | max = min(pos + size, end); | |
193 | } | |
194 | ||
2765cfbb | 195 | if (iov_iter_rw(iter) == WRITE) { |
b2e0d162 | 196 | len = copy_from_iter_pmem(dax.addr, max - pos, iter); |
2765cfbb RZ |
197 | need_wmb = true; |
198 | } else if (!hole) | |
b2e0d162 | 199 | len = copy_to_iter((void __force *) dax.addr, max - pos, |
e2e05394 | 200 | iter); |
d475c634 MW |
201 | else |
202 | len = iov_iter_zero(max - pos, iter); | |
203 | ||
cadfbb6e | 204 | if (!len) { |
b2e0d162 | 205 | rc = -EFAULT; |
d475c634 | 206 | break; |
cadfbb6e | 207 | } |
d475c634 MW |
208 | |
209 | pos += len; | |
b2e0d162 DW |
210 | if (!IS_ERR(dax.addr)) |
211 | dax.addr += len; | |
d475c634 MW |
212 | } |
213 | ||
2765cfbb RZ |
214 | if (need_wmb) |
215 | wmb_pmem(); | |
b2e0d162 | 216 | dax_unmap_atomic(bdev, &dax); |
2765cfbb | 217 | |
b2e0d162 | 218 | return (pos == start) ? rc : pos - start; |
d475c634 MW |
219 | } |
220 | ||
221 | /** | |
222 | * dax_do_io - Perform I/O to a DAX file | |
d475c634 MW |
223 | * @iocb: The control block for this I/O |
224 | * @inode: The file which the I/O is directed at | |
225 | * @iter: The addresses to do I/O from or to | |
226 | * @pos: The file offset where the I/O starts | |
227 | * @get_block: The filesystem method used to translate file offsets to blocks | |
228 | * @end_io: A filesystem callback for I/O completion | |
229 | * @flags: See below | |
230 | * | |
231 | * This function uses the same locking scheme as do_blockdev_direct_IO: | |
232 | * If @flags has DIO_LOCKING set, we assume that the i_mutex is held by the | |
233 | * caller for writes. For reads, we take and release the i_mutex ourselves. | |
234 | * If DIO_LOCKING is not set, the filesystem takes care of its own locking. | |
235 | * As with do_blockdev_direct_IO(), we increment i_dio_count while the I/O | |
236 | * is in progress. | |
237 | */ | |
a95cd631 OS |
238 | ssize_t dax_do_io(struct kiocb *iocb, struct inode *inode, |
239 | struct iov_iter *iter, loff_t pos, get_block_t get_block, | |
240 | dio_iodone_t end_io, int flags) | |
d475c634 MW |
241 | { |
242 | struct buffer_head bh; | |
243 | ssize_t retval = -EINVAL; | |
244 | loff_t end = pos + iov_iter_count(iter); | |
245 | ||
246 | memset(&bh, 0, sizeof(bh)); | |
247 | ||
a95cd631 | 248 | if ((flags & DIO_LOCKING) && iov_iter_rw(iter) == READ) { |
d475c634 MW |
249 | struct address_space *mapping = inode->i_mapping; |
250 | mutex_lock(&inode->i_mutex); | |
251 | retval = filemap_write_and_wait_range(mapping, pos, end - 1); | |
252 | if (retval) { | |
253 | mutex_unlock(&inode->i_mutex); | |
254 | goto out; | |
255 | } | |
256 | } | |
257 | ||
258 | /* Protects against truncate */ | |
bbab37dd MW |
259 | if (!(flags & DIO_SKIP_DIO_COUNT)) |
260 | inode_dio_begin(inode); | |
d475c634 | 261 | |
a95cd631 | 262 | retval = dax_io(inode, iter, pos, end, get_block, &bh); |
d475c634 | 263 | |
a95cd631 | 264 | if ((flags & DIO_LOCKING) && iov_iter_rw(iter) == READ) |
d475c634 MW |
265 | mutex_unlock(&inode->i_mutex); |
266 | ||
267 | if ((retval > 0) && end_io) | |
268 | end_io(iocb, pos, retval, bh.b_private); | |
269 | ||
bbab37dd MW |
270 | if (!(flags & DIO_SKIP_DIO_COUNT)) |
271 | inode_dio_end(inode); | |
d475c634 MW |
272 | out: |
273 | return retval; | |
274 | } | |
275 | EXPORT_SYMBOL_GPL(dax_do_io); | |
f7ca90b1 MW |
276 | |
277 | /* | |
278 | * The user has performed a load from a hole in the file. Allocating | |
279 | * a new page in the file would cause excessive storage usage for | |
280 | * workloads with sparse files. We allocate a page cache page instead. | |
281 | * We'll kick it out of the page cache if it's ever written to, | |
282 | * otherwise it will simply fall out of the page cache under memory | |
283 | * pressure without ever having been dirtied. | |
284 | */ | |
285 | static int dax_load_hole(struct address_space *mapping, struct page *page, | |
286 | struct vm_fault *vmf) | |
287 | { | |
288 | unsigned long size; | |
289 | struct inode *inode = mapping->host; | |
290 | if (!page) | |
291 | page = find_or_create_page(mapping, vmf->pgoff, | |
292 | GFP_KERNEL | __GFP_ZERO); | |
293 | if (!page) | |
294 | return VM_FAULT_OOM; | |
295 | /* Recheck i_size under page lock to avoid truncate race */ | |
296 | size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; | |
297 | if (vmf->pgoff >= size) { | |
298 | unlock_page(page); | |
299 | page_cache_release(page); | |
300 | return VM_FAULT_SIGBUS; | |
301 | } | |
302 | ||
303 | vmf->page = page; | |
304 | return VM_FAULT_LOCKED; | |
305 | } | |
306 | ||
b2e0d162 DW |
307 | static int copy_user_bh(struct page *to, struct inode *inode, |
308 | struct buffer_head *bh, unsigned long vaddr) | |
f7ca90b1 | 309 | { |
b2e0d162 DW |
310 | struct blk_dax_ctl dax = { |
311 | .sector = to_sector(bh, inode), | |
312 | .size = bh->b_size, | |
313 | }; | |
314 | struct block_device *bdev = bh->b_bdev; | |
e2e05394 RZ |
315 | void *vto; |
316 | ||
b2e0d162 DW |
317 | if (dax_map_atomic(bdev, &dax) < 0) |
318 | return PTR_ERR(dax.addr); | |
f7ca90b1 | 319 | vto = kmap_atomic(to); |
b2e0d162 | 320 | copy_user_page(vto, (void __force *)dax.addr, vaddr, to); |
f7ca90b1 | 321 | kunmap_atomic(vto); |
b2e0d162 | 322 | dax_unmap_atomic(bdev, &dax); |
f7ca90b1 MW |
323 | return 0; |
324 | } | |
325 | ||
326 | static int dax_insert_mapping(struct inode *inode, struct buffer_head *bh, | |
327 | struct vm_area_struct *vma, struct vm_fault *vmf) | |
328 | { | |
f7ca90b1 | 329 | unsigned long vaddr = (unsigned long)vmf->virtual_address; |
b2e0d162 DW |
330 | struct address_space *mapping = inode->i_mapping; |
331 | struct block_device *bdev = bh->b_bdev; | |
332 | struct blk_dax_ctl dax = { | |
333 | .sector = to_sector(bh, inode), | |
334 | .size = bh->b_size, | |
335 | }; | |
f7ca90b1 MW |
336 | pgoff_t size; |
337 | int error; | |
338 | ||
0f90cc66 RZ |
339 | i_mmap_lock_read(mapping); |
340 | ||
f7ca90b1 MW |
341 | /* |
342 | * Check truncate didn't happen while we were allocating a block. | |
343 | * If it did, this block may or may not be still allocated to the | |
344 | * file. We can't tell the filesystem to free it because we can't | |
345 | * take i_mutex here. In the worst case, the file still has blocks | |
346 | * allocated past the end of the file. | |
347 | */ | |
348 | size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; | |
349 | if (unlikely(vmf->pgoff >= size)) { | |
350 | error = -EIO; | |
351 | goto out; | |
352 | } | |
353 | ||
b2e0d162 DW |
354 | if (dax_map_atomic(bdev, &dax) < 0) { |
355 | error = PTR_ERR(dax.addr); | |
f7ca90b1 MW |
356 | goto out; |
357 | } | |
358 | ||
2765cfbb | 359 | if (buffer_unwritten(bh) || buffer_new(bh)) { |
b2e0d162 | 360 | clear_pmem(dax.addr, PAGE_SIZE); |
2765cfbb RZ |
361 | wmb_pmem(); |
362 | } | |
b2e0d162 | 363 | dax_unmap_atomic(bdev, &dax); |
f7ca90b1 | 364 | |
b2e0d162 | 365 | error = vm_insert_mixed(vma, vaddr, dax.pfn); |
f7ca90b1 MW |
366 | |
367 | out: | |
0f90cc66 RZ |
368 | i_mmap_unlock_read(mapping); |
369 | ||
f7ca90b1 MW |
370 | return error; |
371 | } | |
372 | ||
ce5c5d55 DC |
373 | /** |
374 | * __dax_fault - handle a page fault on a DAX file | |
375 | * @vma: The virtual memory area where the fault occurred | |
376 | * @vmf: The description of the fault | |
377 | * @get_block: The filesystem method used to translate file offsets to blocks | |
b2442c5a DC |
378 | * @complete_unwritten: The filesystem method used to convert unwritten blocks |
379 | * to written so the data written to them is exposed. This is required for | |
380 | * required by write faults for filesystems that will return unwritten | |
381 | * extent mappings from @get_block, but it is optional for reads as | |
382 | * dax_insert_mapping() will always zero unwritten blocks. If the fs does | |
383 | * not support unwritten extents, the it should pass NULL. | |
ce5c5d55 DC |
384 | * |
385 | * When a page fault occurs, filesystems may call this helper in their | |
386 | * fault handler for DAX files. __dax_fault() assumes the caller has done all | |
387 | * the necessary locking for the page fault to proceed successfully. | |
388 | */ | |
389 | int __dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, | |
e842f290 | 390 | get_block_t get_block, dax_iodone_t complete_unwritten) |
f7ca90b1 MW |
391 | { |
392 | struct file *file = vma->vm_file; | |
393 | struct address_space *mapping = file->f_mapping; | |
394 | struct inode *inode = mapping->host; | |
395 | struct page *page; | |
396 | struct buffer_head bh; | |
397 | unsigned long vaddr = (unsigned long)vmf->virtual_address; | |
398 | unsigned blkbits = inode->i_blkbits; | |
399 | sector_t block; | |
400 | pgoff_t size; | |
401 | int error; | |
402 | int major = 0; | |
403 | ||
404 | size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; | |
405 | if (vmf->pgoff >= size) | |
406 | return VM_FAULT_SIGBUS; | |
407 | ||
408 | memset(&bh, 0, sizeof(bh)); | |
409 | block = (sector_t)vmf->pgoff << (PAGE_SHIFT - blkbits); | |
410 | bh.b_size = PAGE_SIZE; | |
411 | ||
412 | repeat: | |
413 | page = find_get_page(mapping, vmf->pgoff); | |
414 | if (page) { | |
415 | if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags)) { | |
416 | page_cache_release(page); | |
417 | return VM_FAULT_RETRY; | |
418 | } | |
419 | if (unlikely(page->mapping != mapping)) { | |
420 | unlock_page(page); | |
421 | page_cache_release(page); | |
422 | goto repeat; | |
423 | } | |
424 | size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; | |
425 | if (unlikely(vmf->pgoff >= size)) { | |
426 | /* | |
427 | * We have a struct page covering a hole in the file | |
428 | * from a read fault and we've raced with a truncate | |
429 | */ | |
430 | error = -EIO; | |
0f90cc66 | 431 | goto unlock_page; |
f7ca90b1 MW |
432 | } |
433 | } | |
434 | ||
435 | error = get_block(inode, block, &bh, 0); | |
436 | if (!error && (bh.b_size < PAGE_SIZE)) | |
437 | error = -EIO; /* fs corruption? */ | |
438 | if (error) | |
0f90cc66 | 439 | goto unlock_page; |
f7ca90b1 MW |
440 | |
441 | if (!buffer_mapped(&bh) && !buffer_unwritten(&bh) && !vmf->cow_page) { | |
442 | if (vmf->flags & FAULT_FLAG_WRITE) { | |
443 | error = get_block(inode, block, &bh, 1); | |
444 | count_vm_event(PGMAJFAULT); | |
445 | mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); | |
446 | major = VM_FAULT_MAJOR; | |
447 | if (!error && (bh.b_size < PAGE_SIZE)) | |
448 | error = -EIO; | |
449 | if (error) | |
0f90cc66 | 450 | goto unlock_page; |
f7ca90b1 MW |
451 | } else { |
452 | return dax_load_hole(mapping, page, vmf); | |
453 | } | |
454 | } | |
455 | ||
456 | if (vmf->cow_page) { | |
457 | struct page *new_page = vmf->cow_page; | |
458 | if (buffer_written(&bh)) | |
b2e0d162 | 459 | error = copy_user_bh(new_page, inode, &bh, vaddr); |
f7ca90b1 MW |
460 | else |
461 | clear_user_highpage(new_page, vaddr); | |
462 | if (error) | |
0f90cc66 | 463 | goto unlock_page; |
f7ca90b1 MW |
464 | vmf->page = page; |
465 | if (!page) { | |
0f90cc66 | 466 | i_mmap_lock_read(mapping); |
f7ca90b1 MW |
467 | /* Check we didn't race with truncate */ |
468 | size = (i_size_read(inode) + PAGE_SIZE - 1) >> | |
469 | PAGE_SHIFT; | |
470 | if (vmf->pgoff >= size) { | |
0f90cc66 | 471 | i_mmap_unlock_read(mapping); |
f7ca90b1 | 472 | error = -EIO; |
0f90cc66 | 473 | goto out; |
f7ca90b1 MW |
474 | } |
475 | } | |
476 | return VM_FAULT_LOCKED; | |
477 | } | |
478 | ||
479 | /* Check we didn't race with a read fault installing a new page */ | |
480 | if (!page && major) | |
481 | page = find_lock_page(mapping, vmf->pgoff); | |
482 | ||
483 | if (page) { | |
484 | unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT, | |
485 | PAGE_CACHE_SIZE, 0); | |
486 | delete_from_page_cache(page); | |
487 | unlock_page(page); | |
488 | page_cache_release(page); | |
489 | } | |
490 | ||
e842f290 DC |
491 | /* |
492 | * If we successfully insert the new mapping over an unwritten extent, | |
493 | * we need to ensure we convert the unwritten extent. If there is an | |
494 | * error inserting the mapping, the filesystem needs to leave it as | |
495 | * unwritten to prevent exposure of the stale underlying data to | |
496 | * userspace, but we still need to call the completion function so | |
497 | * the private resources on the mapping buffer can be released. We | |
498 | * indicate what the callback should do via the uptodate variable, same | |
499 | * as for normal BH based IO completions. | |
500 | */ | |
f7ca90b1 | 501 | error = dax_insert_mapping(inode, &bh, vma, vmf); |
b2442c5a DC |
502 | if (buffer_unwritten(&bh)) { |
503 | if (complete_unwritten) | |
504 | complete_unwritten(&bh, !error); | |
505 | else | |
506 | WARN_ON_ONCE(!(vmf->flags & FAULT_FLAG_WRITE)); | |
507 | } | |
f7ca90b1 MW |
508 | |
509 | out: | |
510 | if (error == -ENOMEM) | |
511 | return VM_FAULT_OOM | major; | |
512 | /* -EBUSY is fine, somebody else faulted on the same PTE */ | |
513 | if ((error < 0) && (error != -EBUSY)) | |
514 | return VM_FAULT_SIGBUS | major; | |
515 | return VM_FAULT_NOPAGE | major; | |
516 | ||
0f90cc66 | 517 | unlock_page: |
f7ca90b1 MW |
518 | if (page) { |
519 | unlock_page(page); | |
520 | page_cache_release(page); | |
521 | } | |
522 | goto out; | |
523 | } | |
ce5c5d55 | 524 | EXPORT_SYMBOL(__dax_fault); |
f7ca90b1 MW |
525 | |
526 | /** | |
527 | * dax_fault - handle a page fault on a DAX file | |
528 | * @vma: The virtual memory area where the fault occurred | |
529 | * @vmf: The description of the fault | |
530 | * @get_block: The filesystem method used to translate file offsets to blocks | |
531 | * | |
532 | * When a page fault occurs, filesystems may call this helper in their | |
533 | * fault handler for DAX files. | |
534 | */ | |
535 | int dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, | |
e842f290 | 536 | get_block_t get_block, dax_iodone_t complete_unwritten) |
f7ca90b1 MW |
537 | { |
538 | int result; | |
539 | struct super_block *sb = file_inode(vma->vm_file)->i_sb; | |
540 | ||
541 | if (vmf->flags & FAULT_FLAG_WRITE) { | |
542 | sb_start_pagefault(sb); | |
543 | file_update_time(vma->vm_file); | |
544 | } | |
ce5c5d55 | 545 | result = __dax_fault(vma, vmf, get_block, complete_unwritten); |
f7ca90b1 MW |
546 | if (vmf->flags & FAULT_FLAG_WRITE) |
547 | sb_end_pagefault(sb); | |
548 | ||
549 | return result; | |
550 | } | |
551 | EXPORT_SYMBOL_GPL(dax_fault); | |
4c0ccfef | 552 | |
844f35db MW |
553 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
554 | /* | |
555 | * The 'colour' (ie low bits) within a PMD of a page offset. This comes up | |
556 | * more often than one might expect in the below function. | |
557 | */ | |
558 | #define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1) | |
559 | ||
560 | int __dax_pmd_fault(struct vm_area_struct *vma, unsigned long address, | |
561 | pmd_t *pmd, unsigned int flags, get_block_t get_block, | |
562 | dax_iodone_t complete_unwritten) | |
563 | { | |
564 | struct file *file = vma->vm_file; | |
565 | struct address_space *mapping = file->f_mapping; | |
566 | struct inode *inode = mapping->host; | |
567 | struct buffer_head bh; | |
568 | unsigned blkbits = inode->i_blkbits; | |
569 | unsigned long pmd_addr = address & PMD_MASK; | |
570 | bool write = flags & FAULT_FLAG_WRITE; | |
b2e0d162 | 571 | struct block_device *bdev; |
844f35db | 572 | pgoff_t size, pgoff; |
b2e0d162 | 573 | sector_t block; |
844f35db MW |
574 | int result = 0; |
575 | ||
ee82c9ed DW |
576 | /* dax pmd mappings are broken wrt gup and fork */ |
577 | if (!IS_ENABLED(CONFIG_FS_DAX_PMD)) | |
578 | return VM_FAULT_FALLBACK; | |
579 | ||
844f35db | 580 | /* Fall back to PTEs if we're going to COW */ |
59bf4fb9 TK |
581 | if (write && !(vma->vm_flags & VM_SHARED)) { |
582 | split_huge_pmd(vma, pmd, address); | |
844f35db | 583 | return VM_FAULT_FALLBACK; |
59bf4fb9 | 584 | } |
844f35db MW |
585 | /* If the PMD would extend outside the VMA */ |
586 | if (pmd_addr < vma->vm_start) | |
587 | return VM_FAULT_FALLBACK; | |
588 | if ((pmd_addr + PMD_SIZE) > vma->vm_end) | |
589 | return VM_FAULT_FALLBACK; | |
590 | ||
3fdd1b47 | 591 | pgoff = linear_page_index(vma, pmd_addr); |
844f35db MW |
592 | size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; |
593 | if (pgoff >= size) | |
594 | return VM_FAULT_SIGBUS; | |
595 | /* If the PMD would cover blocks out of the file */ | |
596 | if ((pgoff | PG_PMD_COLOUR) >= size) | |
597 | return VM_FAULT_FALLBACK; | |
598 | ||
599 | memset(&bh, 0, sizeof(bh)); | |
600 | block = (sector_t)pgoff << (PAGE_SHIFT - blkbits); | |
601 | ||
602 | bh.b_size = PMD_SIZE; | |
b2e0d162 | 603 | if (get_block(inode, block, &bh, write) != 0) |
844f35db | 604 | return VM_FAULT_SIGBUS; |
b2e0d162 | 605 | bdev = bh.b_bdev; |
0f90cc66 | 606 | i_mmap_lock_read(mapping); |
844f35db MW |
607 | |
608 | /* | |
609 | * If the filesystem isn't willing to tell us the length of a hole, | |
610 | * just fall back to PTEs. Calling get_block 512 times in a loop | |
611 | * would be silly. | |
612 | */ | |
613 | if (!buffer_size_valid(&bh) || bh.b_size < PMD_SIZE) | |
614 | goto fallback; | |
615 | ||
46c043ed KS |
616 | /* |
617 | * If we allocated new storage, make sure no process has any | |
618 | * zero pages covering this hole | |
619 | */ | |
620 | if (buffer_new(&bh)) { | |
0f90cc66 | 621 | i_mmap_unlock_read(mapping); |
46c043ed | 622 | unmap_mapping_range(mapping, pgoff << PAGE_SHIFT, PMD_SIZE, 0); |
0f90cc66 | 623 | i_mmap_lock_read(mapping); |
46c043ed KS |
624 | } |
625 | ||
84c4e5e6 MW |
626 | /* |
627 | * If a truncate happened while we were allocating blocks, we may | |
628 | * leave blocks allocated to the file that are beyond EOF. We can't | |
629 | * take i_mutex here, so just leave them hanging; they'll be freed | |
630 | * when the file is deleted. | |
631 | */ | |
844f35db MW |
632 | size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; |
633 | if (pgoff >= size) { | |
634 | result = VM_FAULT_SIGBUS; | |
635 | goto out; | |
636 | } | |
637 | if ((pgoff | PG_PMD_COLOUR) >= size) | |
638 | goto fallback; | |
639 | ||
844f35db | 640 | if (!write && !buffer_mapped(&bh) && buffer_uptodate(&bh)) { |
844f35db | 641 | spinlock_t *ptl; |
d295e341 | 642 | pmd_t entry; |
844f35db | 643 | struct page *zero_page = get_huge_zero_page(); |
d295e341 | 644 | |
844f35db MW |
645 | if (unlikely(!zero_page)) |
646 | goto fallback; | |
647 | ||
d295e341 KS |
648 | ptl = pmd_lock(vma->vm_mm, pmd); |
649 | if (!pmd_none(*pmd)) { | |
650 | spin_unlock(ptl); | |
651 | goto fallback; | |
652 | } | |
653 | ||
654 | entry = mk_pmd(zero_page, vma->vm_page_prot); | |
655 | entry = pmd_mkhuge(entry); | |
656 | set_pmd_at(vma->vm_mm, pmd_addr, pmd, entry); | |
844f35db | 657 | result = VM_FAULT_NOPAGE; |
d295e341 | 658 | spin_unlock(ptl); |
844f35db | 659 | } else { |
b2e0d162 DW |
660 | struct blk_dax_ctl dax = { |
661 | .sector = to_sector(&bh, inode), | |
662 | .size = PMD_SIZE, | |
663 | }; | |
664 | long length = dax_map_atomic(bdev, &dax); | |
665 | ||
844f35db MW |
666 | if (length < 0) { |
667 | result = VM_FAULT_SIGBUS; | |
668 | goto out; | |
669 | } | |
b2e0d162 DW |
670 | if ((length < PMD_SIZE) || (dax.pfn & PG_PMD_COLOUR)) { |
671 | dax_unmap_atomic(bdev, &dax); | |
844f35db | 672 | goto fallback; |
b2e0d162 | 673 | } |
844f35db | 674 | |
152d7bd8 DW |
675 | /* |
676 | * TODO: teach vmf_insert_pfn_pmd() to support | |
677 | * 'pte_special' for pmds | |
678 | */ | |
b2e0d162 DW |
679 | if (pfn_valid(dax.pfn)) { |
680 | dax_unmap_atomic(bdev, &dax); | |
152d7bd8 | 681 | goto fallback; |
b2e0d162 | 682 | } |
152d7bd8 | 683 | |
0f90cc66 | 684 | if (buffer_unwritten(&bh) || buffer_new(&bh)) { |
b2e0d162 | 685 | clear_pmem(dax.addr, PMD_SIZE); |
0f90cc66 RZ |
686 | wmb_pmem(); |
687 | count_vm_event(PGMAJFAULT); | |
688 | mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); | |
689 | result |= VM_FAULT_MAJOR; | |
690 | } | |
b2e0d162 | 691 | dax_unmap_atomic(bdev, &dax); |
0f90cc66 | 692 | |
b2e0d162 | 693 | result |= vmf_insert_pfn_pmd(vma, address, pmd, dax.pfn, write); |
844f35db MW |
694 | } |
695 | ||
696 | out: | |
0f90cc66 RZ |
697 | i_mmap_unlock_read(mapping); |
698 | ||
844f35db MW |
699 | if (buffer_unwritten(&bh)) |
700 | complete_unwritten(&bh, !(result & VM_FAULT_ERROR)); | |
701 | ||
702 | return result; | |
703 | ||
704 | fallback: | |
705 | count_vm_event(THP_FAULT_FALLBACK); | |
706 | result = VM_FAULT_FALLBACK; | |
707 | goto out; | |
708 | } | |
709 | EXPORT_SYMBOL_GPL(__dax_pmd_fault); | |
710 | ||
711 | /** | |
712 | * dax_pmd_fault - handle a PMD fault on a DAX file | |
713 | * @vma: The virtual memory area where the fault occurred | |
714 | * @vmf: The description of the fault | |
715 | * @get_block: The filesystem method used to translate file offsets to blocks | |
716 | * | |
717 | * When a page fault occurs, filesystems may call this helper in their | |
718 | * pmd_fault handler for DAX files. | |
719 | */ | |
720 | int dax_pmd_fault(struct vm_area_struct *vma, unsigned long address, | |
721 | pmd_t *pmd, unsigned int flags, get_block_t get_block, | |
722 | dax_iodone_t complete_unwritten) | |
723 | { | |
724 | int result; | |
725 | struct super_block *sb = file_inode(vma->vm_file)->i_sb; | |
726 | ||
727 | if (flags & FAULT_FLAG_WRITE) { | |
728 | sb_start_pagefault(sb); | |
729 | file_update_time(vma->vm_file); | |
730 | } | |
731 | result = __dax_pmd_fault(vma, address, pmd, flags, get_block, | |
732 | complete_unwritten); | |
733 | if (flags & FAULT_FLAG_WRITE) | |
734 | sb_end_pagefault(sb); | |
735 | ||
736 | return result; | |
737 | } | |
738 | EXPORT_SYMBOL_GPL(dax_pmd_fault); | |
dd8a2b6c | 739 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
844f35db | 740 | |
0e3b210c BH |
741 | /** |
742 | * dax_pfn_mkwrite - handle first write to DAX page | |
743 | * @vma: The virtual memory area where the fault occurred | |
744 | * @vmf: The description of the fault | |
745 | * | |
746 | */ | |
747 | int dax_pfn_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) | |
748 | { | |
749 | struct super_block *sb = file_inode(vma->vm_file)->i_sb; | |
750 | ||
751 | sb_start_pagefault(sb); | |
752 | file_update_time(vma->vm_file); | |
753 | sb_end_pagefault(sb); | |
754 | return VM_FAULT_NOPAGE; | |
755 | } | |
756 | EXPORT_SYMBOL_GPL(dax_pfn_mkwrite); | |
757 | ||
4c0ccfef | 758 | /** |
25726bc1 | 759 | * dax_zero_page_range - zero a range within a page of a DAX file |
4c0ccfef MW |
760 | * @inode: The file being truncated |
761 | * @from: The file offset that is being truncated to | |
25726bc1 | 762 | * @length: The number of bytes to zero |
4c0ccfef MW |
763 | * @get_block: The filesystem method used to translate file offsets to blocks |
764 | * | |
25726bc1 MW |
765 | * This function can be called by a filesystem when it is zeroing part of a |
766 | * page in a DAX file. This is intended for hole-punch operations. If | |
767 | * you are truncating a file, the helper function dax_truncate_page() may be | |
768 | * more convenient. | |
4c0ccfef MW |
769 | * |
770 | * We work in terms of PAGE_CACHE_SIZE here for commonality with | |
771 | * block_truncate_page(), but we could go down to PAGE_SIZE if the filesystem | |
772 | * took care of disposing of the unnecessary blocks. Even if the filesystem | |
773 | * block size is smaller than PAGE_SIZE, we have to zero the rest of the page | |
25726bc1 | 774 | * since the file might be mmapped. |
4c0ccfef | 775 | */ |
25726bc1 MW |
776 | int dax_zero_page_range(struct inode *inode, loff_t from, unsigned length, |
777 | get_block_t get_block) | |
4c0ccfef MW |
778 | { |
779 | struct buffer_head bh; | |
780 | pgoff_t index = from >> PAGE_CACHE_SHIFT; | |
781 | unsigned offset = from & (PAGE_CACHE_SIZE-1); | |
4c0ccfef MW |
782 | int err; |
783 | ||
784 | /* Block boundary? Nothing to do */ | |
785 | if (!length) | |
786 | return 0; | |
25726bc1 | 787 | BUG_ON((offset + length) > PAGE_CACHE_SIZE); |
4c0ccfef MW |
788 | |
789 | memset(&bh, 0, sizeof(bh)); | |
790 | bh.b_size = PAGE_CACHE_SIZE; | |
791 | err = get_block(inode, index, &bh, 0); | |
792 | if (err < 0) | |
793 | return err; | |
794 | if (buffer_written(&bh)) { | |
b2e0d162 DW |
795 | struct block_device *bdev = bh.b_bdev; |
796 | struct blk_dax_ctl dax = { | |
797 | .sector = to_sector(&bh, inode), | |
798 | .size = PAGE_CACHE_SIZE, | |
799 | }; | |
800 | ||
801 | if (dax_map_atomic(bdev, &dax) < 0) | |
802 | return PTR_ERR(dax.addr); | |
803 | clear_pmem(dax.addr + offset, length); | |
2765cfbb | 804 | wmb_pmem(); |
b2e0d162 | 805 | dax_unmap_atomic(bdev, &dax); |
4c0ccfef MW |
806 | } |
807 | ||
808 | return 0; | |
809 | } | |
25726bc1 MW |
810 | EXPORT_SYMBOL_GPL(dax_zero_page_range); |
811 | ||
812 | /** | |
813 | * dax_truncate_page - handle a partial page being truncated in a DAX file | |
814 | * @inode: The file being truncated | |
815 | * @from: The file offset that is being truncated to | |
816 | * @get_block: The filesystem method used to translate file offsets to blocks | |
817 | * | |
818 | * Similar to block_truncate_page(), this function can be called by a | |
819 | * filesystem when it is truncating a DAX file to handle the partial page. | |
820 | * | |
821 | * We work in terms of PAGE_CACHE_SIZE here for commonality with | |
822 | * block_truncate_page(), but we could go down to PAGE_SIZE if the filesystem | |
823 | * took care of disposing of the unnecessary blocks. Even if the filesystem | |
824 | * block size is smaller than PAGE_SIZE, we have to zero the rest of the page | |
825 | * since the file might be mmapped. | |
826 | */ | |
827 | int dax_truncate_page(struct inode *inode, loff_t from, get_block_t get_block) | |
828 | { | |
829 | unsigned length = PAGE_CACHE_ALIGN(from) - from; | |
830 | return dax_zero_page_range(inode, from, length, get_block); | |
831 | } | |
4c0ccfef | 832 | EXPORT_SYMBOL_GPL(dax_truncate_page); |