2 * linux/fs/nfs/pagelist.c
4 * A set of helper functions for managing NFS read and write requests.
5 * The main purpose of these routines is to provide support for the
6 * coalescing of several requests into a single RPC call.
8 * Copyright 2000, 2001 (c) Trond Myklebust <trond.myklebust@fys.uio.no>
12 #include <linux/slab.h>
13 #include <linux/file.h>
14 #include <linux/sched.h>
15 #include <linux/sunrpc/clnt.h>
16 #include <linux/nfs.h>
17 #include <linux/nfs3.h>
18 #include <linux/nfs4.h>
19 #include <linux/nfs_page.h>
20 #include <linux/nfs_fs.h>
21 #include <linux/nfs_mount.h>
22 #include <linux/export.h>
27 #define NFSDBG_FACILITY NFSDBG_PAGECACHE
29 static struct kmem_cache
*nfs_page_cachep
;
30 static const struct rpc_call_ops nfs_pgio_common_ops
;
32 static bool nfs_pgarray_set(struct nfs_page_array
*p
, unsigned int pagecount
)
34 p
->npages
= pagecount
;
35 if (pagecount
<= ARRAY_SIZE(p
->page_array
))
36 p
->pagevec
= p
->page_array
;
38 p
->pagevec
= kcalloc(pagecount
, sizeof(struct page
*), GFP_KERNEL
);
42 return p
->pagevec
!= NULL
;
45 void nfs_pgheader_init(struct nfs_pageio_descriptor
*desc
,
46 struct nfs_pgio_header
*hdr
,
47 void (*release
)(struct nfs_pgio_header
*hdr
))
49 hdr
->req
= nfs_list_entry(desc
->pg_list
.next
);
50 hdr
->inode
= desc
->pg_inode
;
51 hdr
->cred
= hdr
->req
->wb_context
->cred
;
52 hdr
->io_start
= req_offset(hdr
->req
);
53 hdr
->good_bytes
= desc
->pg_count
;
54 hdr
->dreq
= desc
->pg_dreq
;
55 hdr
->layout_private
= desc
->pg_layout_private
;
56 hdr
->release
= release
;
57 hdr
->completion_ops
= desc
->pg_completion_ops
;
58 if (hdr
->completion_ops
->init_hdr
)
59 hdr
->completion_ops
->init_hdr(hdr
);
61 EXPORT_SYMBOL_GPL(nfs_pgheader_init
);
63 void nfs_set_pgio_error(struct nfs_pgio_header
*hdr
, int error
, loff_t pos
)
65 spin_lock(&hdr
->lock
);
66 if (pos
< hdr
->io_start
+ hdr
->good_bytes
) {
67 set_bit(NFS_IOHDR_ERROR
, &hdr
->flags
);
68 clear_bit(NFS_IOHDR_EOF
, &hdr
->flags
);
69 hdr
->good_bytes
= pos
- hdr
->io_start
;
72 spin_unlock(&hdr
->lock
);
75 static inline struct nfs_page
*
78 struct nfs_page
*p
= kmem_cache_zalloc(nfs_page_cachep
, GFP_NOIO
);
80 INIT_LIST_HEAD(&p
->wb_list
);
85 nfs_page_free(struct nfs_page
*p
)
87 kmem_cache_free(nfs_page_cachep
, p
);
91 nfs_iocounter_inc(struct nfs_io_counter
*c
)
93 atomic_inc(&c
->io_count
);
97 nfs_iocounter_dec(struct nfs_io_counter
*c
)
99 if (atomic_dec_and_test(&c
->io_count
)) {
100 clear_bit(NFS_IO_INPROGRESS
, &c
->flags
);
101 smp_mb__after_clear_bit();
102 wake_up_bit(&c
->flags
, NFS_IO_INPROGRESS
);
107 __nfs_iocounter_wait(struct nfs_io_counter
*c
)
109 wait_queue_head_t
*wq
= bit_waitqueue(&c
->flags
, NFS_IO_INPROGRESS
);
110 DEFINE_WAIT_BIT(q
, &c
->flags
, NFS_IO_INPROGRESS
);
114 prepare_to_wait(wq
, &q
.wait
, TASK_KILLABLE
);
115 set_bit(NFS_IO_INPROGRESS
, &c
->flags
);
116 if (atomic_read(&c
->io_count
) == 0)
118 ret
= nfs_wait_bit_killable(&c
->flags
);
119 } while (atomic_read(&c
->io_count
) != 0);
120 finish_wait(wq
, &q
.wait
);
125 * nfs_iocounter_wait - wait for i/o to complete
126 * @c: nfs_io_counter to use
128 * returns -ERESTARTSYS if interrupted by a fatal signal.
129 * Otherwise returns 0 once the io_count hits 0.
132 nfs_iocounter_wait(struct nfs_io_counter
*c
)
134 if (atomic_read(&c
->io_count
) == 0)
136 return __nfs_iocounter_wait(c
);
140 * nfs_create_request - Create an NFS read/write request.
141 * @ctx: open context to use
142 * @inode: inode to which the request is attached
143 * @page: page to write
144 * @offset: starting offset within the page for the write
145 * @count: number of bytes to read/write
147 * The page must be locked by the caller. This makes sure we never
148 * create two different requests for the same page.
149 * User should ensure it is safe to sleep in this function.
152 nfs_create_request(struct nfs_open_context
*ctx
, struct inode
*inode
,
154 unsigned int offset
, unsigned int count
)
156 struct nfs_page
*req
;
157 struct nfs_lock_context
*l_ctx
;
159 if (test_bit(NFS_CONTEXT_BAD
, &ctx
->flags
))
160 return ERR_PTR(-EBADF
);
161 /* try to allocate the request struct */
162 req
= nfs_page_alloc();
164 return ERR_PTR(-ENOMEM
);
166 /* get lock context early so we can deal with alloc failures */
167 l_ctx
= nfs_get_lock_context(ctx
);
170 return ERR_CAST(l_ctx
);
172 req
->wb_lock_context
= l_ctx
;
173 nfs_iocounter_inc(&l_ctx
->io_count
);
175 /* Initialize the request struct. Initially, we assume a
176 * long write-back delay. This will be adjusted in
177 * update_nfs_request below if the region is not locked. */
179 req
->wb_index
= page_file_index(page
);
180 page_cache_get(page
);
181 req
->wb_offset
= offset
;
182 req
->wb_pgbase
= offset
;
183 req
->wb_bytes
= count
;
184 req
->wb_context
= get_nfs_open_context(ctx
);
185 kref_init(&req
->wb_kref
);
190 * nfs_unlock_request - Unlock request and wake up sleepers.
193 void nfs_unlock_request(struct nfs_page
*req
)
195 if (!NFS_WBACK_BUSY(req
)) {
196 printk(KERN_ERR
"NFS: Invalid unlock attempted\n");
199 smp_mb__before_clear_bit();
200 clear_bit(PG_BUSY
, &req
->wb_flags
);
201 smp_mb__after_clear_bit();
202 wake_up_bit(&req
->wb_flags
, PG_BUSY
);
206 * nfs_unlock_and_release_request - Unlock request and release the nfs_page
209 void nfs_unlock_and_release_request(struct nfs_page
*req
)
211 nfs_unlock_request(req
);
212 nfs_release_request(req
);
216 * nfs_clear_request - Free up all resources allocated to the request
219 * Release page and open context resources associated with a read/write
220 * request after it has completed.
222 static void nfs_clear_request(struct nfs_page
*req
)
224 struct page
*page
= req
->wb_page
;
225 struct nfs_open_context
*ctx
= req
->wb_context
;
226 struct nfs_lock_context
*l_ctx
= req
->wb_lock_context
;
229 page_cache_release(page
);
233 nfs_iocounter_dec(&l_ctx
->io_count
);
234 nfs_put_lock_context(l_ctx
);
235 req
->wb_lock_context
= NULL
;
238 put_nfs_open_context(ctx
);
239 req
->wb_context
= NULL
;
245 * nfs_release_request - Release the count on an NFS read/write request
246 * @req: request to release
248 * Note: Should never be called with the spinlock held!
250 static void nfs_free_request(struct kref
*kref
)
252 struct nfs_page
*req
= container_of(kref
, struct nfs_page
, wb_kref
);
254 /* Release struct file and open context */
255 nfs_clear_request(req
);
259 void nfs_release_request(struct nfs_page
*req
)
261 kref_put(&req
->wb_kref
, nfs_free_request
);
264 static int nfs_wait_bit_uninterruptible(void *word
)
271 * nfs_wait_on_request - Wait for a request to complete.
272 * @req: request to wait upon.
274 * Interruptible by fatal signals only.
275 * The user is responsible for holding a count on the request.
278 nfs_wait_on_request(struct nfs_page
*req
)
280 return wait_on_bit(&req
->wb_flags
, PG_BUSY
,
281 nfs_wait_bit_uninterruptible
,
282 TASK_UNINTERRUPTIBLE
);
285 bool nfs_generic_pg_test(struct nfs_pageio_descriptor
*desc
, struct nfs_page
*prev
, struct nfs_page
*req
)
288 * FIXME: ideally we should be able to coalesce all requests
289 * that are not block boundary aligned, but currently this
290 * is problematic for the case of bsize < PAGE_CACHE_SIZE,
291 * since nfs_flush_multi and nfs_pagein_multi assume you
292 * can have only one struct nfs_page.
294 if (desc
->pg_bsize
< PAGE_SIZE
)
297 return desc
->pg_count
+ req
->wb_bytes
<= desc
->pg_bsize
;
299 EXPORT_SYMBOL_GPL(nfs_generic_pg_test
);
301 static inline struct nfs_rw_header
*NFS_RW_HEADER(struct nfs_pgio_header
*hdr
)
303 return container_of(hdr
, struct nfs_rw_header
, header
);
307 * nfs_rw_header_alloc - Allocate a header for a read or write
308 * @ops: Read or write function vector
310 struct nfs_rw_header
*nfs_rw_header_alloc(const struct nfs_rw_ops
*ops
)
312 struct nfs_rw_header
*header
= ops
->rw_alloc_header();
315 struct nfs_pgio_header
*hdr
= &header
->header
;
317 INIT_LIST_HEAD(&hdr
->pages
);
318 INIT_LIST_HEAD(&hdr
->rpc_list
);
319 spin_lock_init(&hdr
->lock
);
320 atomic_set(&hdr
->refcnt
, 0);
325 EXPORT_SYMBOL_GPL(nfs_rw_header_alloc
);
328 * nfs_rw_header_free - Free a read or write header
329 * @hdr: The header to free
331 void nfs_rw_header_free(struct nfs_pgio_header
*hdr
)
333 hdr
->rw_ops
->rw_free_header(NFS_RW_HEADER(hdr
));
335 EXPORT_SYMBOL_GPL(nfs_rw_header_free
);
338 * nfs_pgio_data_alloc - Allocate pageio data
339 * @hdr: The header making a request
340 * @pagecount: Number of pages to create
342 static struct nfs_pgio_data
*nfs_pgio_data_alloc(struct nfs_pgio_header
*hdr
,
343 unsigned int pagecount
)
345 struct nfs_pgio_data
*data
, *prealloc
;
347 prealloc
= &NFS_RW_HEADER(hdr
)->rpc_data
;
348 if (prealloc
->header
== NULL
)
351 data
= kzalloc(sizeof(*data
), GFP_KERNEL
);
355 if (nfs_pgarray_set(&data
->pages
, pagecount
)) {
357 atomic_inc(&hdr
->refcnt
);
359 if (data
!= prealloc
)
368 * nfs_pgio_data_release - Properly free pageio data
369 * @data: The data to release
371 void nfs_pgio_data_release(struct nfs_pgio_data
*data
)
373 struct nfs_pgio_header
*hdr
= data
->header
;
374 struct nfs_rw_header
*pageio_header
= NFS_RW_HEADER(hdr
);
376 put_nfs_open_context(data
->args
.context
);
377 if (data
->pages
.pagevec
!= data
->pages
.page_array
)
378 kfree(data
->pages
.pagevec
);
379 if (data
== &pageio_header
->rpc_data
) {
383 if (atomic_dec_and_test(&hdr
->refcnt
))
384 hdr
->completion_ops
->completion(hdr
);
385 /* Note: we only free the rpc_task after callbacks are done.
386 * See the comment in rpc_free_task() for why
390 EXPORT_SYMBOL_GPL(nfs_pgio_data_release
);
393 * nfs_pgio_rpcsetup - Set up arguments for a pageio call
394 * @data: The pageio data
395 * @count: Number of bytes to read
396 * @offset: Initial offset
397 * @how: How to commit data (writes only)
398 * @cinfo: Commit information for the call (writes only)
400 static void nfs_pgio_rpcsetup(struct nfs_pgio_data
*data
,
401 unsigned int count
, unsigned int offset
,
402 int how
, struct nfs_commit_info
*cinfo
)
404 struct nfs_page
*req
= data
->header
->req
;
406 /* Set up the RPC argument and reply structs
407 * NB: take care not to mess about with data->commit et al. */
409 data
->args
.fh
= NFS_FH(data
->header
->inode
);
410 data
->args
.offset
= req_offset(req
) + offset
;
411 /* pnfs_set_layoutcommit needs this */
412 data
->mds_offset
= data
->args
.offset
;
413 data
->args
.pgbase
= req
->wb_pgbase
+ offset
;
414 data
->args
.pages
= data
->pages
.pagevec
;
415 data
->args
.count
= count
;
416 data
->args
.context
= get_nfs_open_context(req
->wb_context
);
417 data
->args
.lock_context
= req
->wb_lock_context
;
418 data
->args
.stable
= NFS_UNSTABLE
;
419 switch (how
& (FLUSH_STABLE
| FLUSH_COND_STABLE
)) {
422 case FLUSH_COND_STABLE
:
423 if (nfs_reqs_to_commit(cinfo
))
426 data
->args
.stable
= NFS_FILE_SYNC
;
429 data
->res
.fattr
= &data
->fattr
;
430 data
->res
.count
= count
;
432 data
->res
.verf
= &data
->verf
;
433 nfs_fattr_init(&data
->fattr
);
437 * nfs_pgio_prepare - Prepare pageio data to go over the wire
438 * @task: The current task
439 * @calldata: pageio data to prepare
441 static void nfs_pgio_prepare(struct rpc_task
*task
, void *calldata
)
443 struct nfs_pgio_data
*data
= calldata
;
445 err
= NFS_PROTO(data
->header
->inode
)->pgio_rpc_prepare(task
, data
);
450 int nfs_initiate_pgio(struct rpc_clnt
*clnt
, struct nfs_pgio_data
*data
,
451 const struct rpc_call_ops
*call_ops
, int how
, int flags
)
453 struct rpc_task
*task
;
454 struct rpc_message msg
= {
455 .rpc_argp
= &data
->args
,
456 .rpc_resp
= &data
->res
,
457 .rpc_cred
= data
->header
->cred
,
459 struct rpc_task_setup task_setup_data
= {
463 .callback_ops
= call_ops
,
464 .callback_data
= data
,
465 .workqueue
= nfsiod_workqueue
,
466 .flags
= RPC_TASK_ASYNC
| flags
,
470 data
->header
->rw_ops
->rw_initiate(data
, &msg
, &task_setup_data
, how
);
472 dprintk("NFS: %5u initiated pgio call "
473 "(req %s/%llu, %u bytes @ offset %llu)\n",
475 data
->header
->inode
->i_sb
->s_id
,
476 (unsigned long long)NFS_FILEID(data
->header
->inode
),
478 (unsigned long long)data
->args
.offset
);
480 task
= rpc_run_task(&task_setup_data
);
485 if (how
& FLUSH_SYNC
) {
486 ret
= rpc_wait_for_completion_task(task
);
488 ret
= task
->tk_status
;
494 EXPORT_SYMBOL_GPL(nfs_initiate_pgio
);
496 int nfs_do_multiple_pgios(struct list_head
*head
,
497 const struct rpc_call_ops
*call_ops
,
500 struct nfs_pgio_data
*data
;
503 while (!list_empty(head
)) {
506 data
= list_first_entry(head
, struct nfs_pgio_data
, list
);
507 list_del_init(&data
->list
);
509 ret2
= nfs_initiate_pgio(NFS_CLIENT(data
->header
->inode
),
510 data
, call_ops
, how
, 0);
518 * nfs_pgio_error - Clean up from a pageio error
519 * @desc: IO descriptor
520 * @hdr: pageio header
522 static int nfs_pgio_error(struct nfs_pageio_descriptor
*desc
,
523 struct nfs_pgio_header
*hdr
)
525 struct nfs_pgio_data
*data
;
527 set_bit(NFS_IOHDR_REDO
, &hdr
->flags
);
528 while (!list_empty(&hdr
->rpc_list
)) {
529 data
= list_first_entry(&hdr
->rpc_list
, struct nfs_pgio_data
, list
);
530 list_del(&data
->list
);
531 nfs_pgio_data_release(data
);
533 desc
->pg_completion_ops
->error_cleanup(&desc
->pg_list
);
538 * nfs_pgio_release - Release pageio data
539 * @calldata: The pageio data to release
541 static void nfs_pgio_release(void *calldata
)
543 struct nfs_pgio_data
*data
= calldata
;
544 if (data
->header
->rw_ops
->rw_release
)
545 data
->header
->rw_ops
->rw_release(data
);
546 nfs_pgio_data_release(data
);
550 * nfs_pageio_init - initialise a page io descriptor
551 * @desc: pointer to descriptor
552 * @inode: pointer to inode
553 * @doio: pointer to io function
554 * @bsize: io block size
555 * @io_flags: extra parameters for the io function
557 void nfs_pageio_init(struct nfs_pageio_descriptor
*desc
,
559 const struct nfs_pageio_ops
*pg_ops
,
560 const struct nfs_pgio_completion_ops
*compl_ops
,
561 const struct nfs_rw_ops
*rw_ops
,
565 INIT_LIST_HEAD(&desc
->pg_list
);
566 desc
->pg_bytes_written
= 0;
568 desc
->pg_bsize
= bsize
;
571 desc
->pg_recoalesce
= 0;
572 desc
->pg_inode
= inode
;
573 desc
->pg_ops
= pg_ops
;
574 desc
->pg_completion_ops
= compl_ops
;
575 desc
->pg_rw_ops
= rw_ops
;
576 desc
->pg_ioflags
= io_flags
;
578 desc
->pg_lseg
= NULL
;
579 desc
->pg_dreq
= NULL
;
580 desc
->pg_layout_private
= NULL
;
582 EXPORT_SYMBOL_GPL(nfs_pageio_init
);
585 * nfs_pgio_result - Basic pageio error handling
586 * @task: The task that ran
587 * @calldata: Pageio data to check
589 static void nfs_pgio_result(struct rpc_task
*task
, void *calldata
)
591 struct nfs_pgio_data
*data
= calldata
;
592 struct inode
*inode
= data
->header
->inode
;
594 dprintk("NFS: %s: %5u, (status %d)\n", __func__
,
595 task
->tk_pid
, task
->tk_status
);
597 if (data
->header
->rw_ops
->rw_done(task
, data
, inode
) != 0)
599 if (task
->tk_status
< 0)
600 nfs_set_pgio_error(data
->header
, task
->tk_status
, data
->args
.offset
);
602 data
->header
->rw_ops
->rw_result(task
, data
);
606 * Generate multiple small requests to read or write a single
607 * contiguous dirty on one page.
609 static int nfs_pgio_multi(struct nfs_pageio_descriptor
*desc
,
610 struct nfs_pgio_header
*hdr
)
612 struct nfs_page
*req
= hdr
->req
;
613 struct page
*page
= req
->wb_page
;
614 struct nfs_pgio_data
*data
;
615 size_t wsize
= desc
->pg_bsize
, nbytes
;
618 struct nfs_commit_info cinfo
;
620 nfs_init_cinfo(&cinfo
, desc
->pg_inode
, desc
->pg_dreq
);
622 if ((desc
->pg_ioflags
& FLUSH_COND_STABLE
) &&
623 (desc
->pg_moreio
|| nfs_reqs_to_commit(&cinfo
) ||
624 desc
->pg_count
> wsize
))
625 desc
->pg_ioflags
&= ~FLUSH_COND_STABLE
;
628 nbytes
= desc
->pg_count
;
630 size_t len
= min(nbytes
, wsize
);
632 data
= nfs_pgio_data_alloc(hdr
, 1);
634 return nfs_pgio_error(desc
, hdr
);
635 data
->pages
.pagevec
[0] = page
;
636 nfs_pgio_rpcsetup(data
, len
, offset
, desc
->pg_ioflags
, &cinfo
);
637 list_add(&data
->list
, &hdr
->rpc_list
);
641 } while (nbytes
!= 0);
643 nfs_list_remove_request(req
);
644 nfs_list_add_request(req
, &hdr
->pages
);
645 desc
->pg_rpc_callops
= &nfs_pgio_common_ops
;
650 * Create an RPC task for the given read or write request and kick it.
651 * The page must have been locked by the caller.
653 * It may happen that the page we're passed is not marked dirty.
654 * This is the case if nfs_updatepage detects a conflicting request
655 * that has been written but not committed.
657 static int nfs_pgio_one(struct nfs_pageio_descriptor
*desc
,
658 struct nfs_pgio_header
*hdr
)
660 struct nfs_page
*req
;
662 struct nfs_pgio_data
*data
;
663 struct list_head
*head
= &desc
->pg_list
;
664 struct nfs_commit_info cinfo
;
666 data
= nfs_pgio_data_alloc(hdr
, nfs_page_array_len(desc
->pg_base
,
669 return nfs_pgio_error(desc
, hdr
);
671 nfs_init_cinfo(&cinfo
, desc
->pg_inode
, desc
->pg_dreq
);
672 pages
= data
->pages
.pagevec
;
673 while (!list_empty(head
)) {
674 req
= nfs_list_entry(head
->next
);
675 nfs_list_remove_request(req
);
676 nfs_list_add_request(req
, &hdr
->pages
);
677 *pages
++ = req
->wb_page
;
680 if ((desc
->pg_ioflags
& FLUSH_COND_STABLE
) &&
681 (desc
->pg_moreio
|| nfs_reqs_to_commit(&cinfo
)))
682 desc
->pg_ioflags
&= ~FLUSH_COND_STABLE
;
684 /* Set up the argument struct */
685 nfs_pgio_rpcsetup(data
, desc
->pg_count
, 0, desc
->pg_ioflags
, &cinfo
);
686 list_add(&data
->list
, &hdr
->rpc_list
);
687 desc
->pg_rpc_callops
= &nfs_pgio_common_ops
;
691 int nfs_generic_pgio(struct nfs_pageio_descriptor
*desc
,
692 struct nfs_pgio_header
*hdr
)
694 if (desc
->pg_bsize
< PAGE_CACHE_SIZE
)
695 return nfs_pgio_multi(desc
, hdr
);
696 return nfs_pgio_one(desc
, hdr
);
698 EXPORT_SYMBOL_GPL(nfs_generic_pgio
);
700 static bool nfs_match_open_context(const struct nfs_open_context
*ctx1
,
701 const struct nfs_open_context
*ctx2
)
703 return ctx1
->cred
== ctx2
->cred
&& ctx1
->state
== ctx2
->state
;
706 static bool nfs_match_lock_context(const struct nfs_lock_context
*l1
,
707 const struct nfs_lock_context
*l2
)
709 return l1
->lockowner
.l_owner
== l2
->lockowner
.l_owner
710 && l1
->lockowner
.l_pid
== l2
->lockowner
.l_pid
;
714 * nfs_can_coalesce_requests - test two requests for compatibility
715 * @prev: pointer to nfs_page
716 * @req: pointer to nfs_page
718 * The nfs_page structures 'prev' and 'req' are compared to ensure that the
719 * page data area they describe is contiguous, and that their RPC
720 * credentials, NFSv4 open state, and lockowners are the same.
722 * Return 'true' if this is the case, else return 'false'.
724 static bool nfs_can_coalesce_requests(struct nfs_page
*prev
,
725 struct nfs_page
*req
,
726 struct nfs_pageio_descriptor
*pgio
)
728 if (!nfs_match_open_context(req
->wb_context
, prev
->wb_context
))
730 if (req
->wb_context
->dentry
->d_inode
->i_flock
!= NULL
&&
731 !nfs_match_lock_context(req
->wb_lock_context
, prev
->wb_lock_context
))
733 if (req
->wb_pgbase
!= 0)
735 if (prev
->wb_pgbase
+ prev
->wb_bytes
!= PAGE_CACHE_SIZE
)
737 if (req_offset(req
) != req_offset(prev
) + prev
->wb_bytes
)
739 return pgio
->pg_ops
->pg_test(pgio
, prev
, req
);
743 * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
744 * @desc: destination io descriptor
747 * Returns true if the request 'req' was successfully coalesced into the
748 * existing list of pages 'desc'.
750 static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor
*desc
,
751 struct nfs_page
*req
)
753 if (desc
->pg_count
!= 0) {
754 struct nfs_page
*prev
;
756 prev
= nfs_list_entry(desc
->pg_list
.prev
);
757 if (!nfs_can_coalesce_requests(prev
, req
, desc
))
760 if (desc
->pg_ops
->pg_init
)
761 desc
->pg_ops
->pg_init(desc
, req
);
762 desc
->pg_base
= req
->wb_pgbase
;
764 nfs_list_remove_request(req
);
765 nfs_list_add_request(req
, &desc
->pg_list
);
766 desc
->pg_count
+= req
->wb_bytes
;
771 * Helper for nfs_pageio_add_request and nfs_pageio_complete
773 static void nfs_pageio_doio(struct nfs_pageio_descriptor
*desc
)
775 if (!list_empty(&desc
->pg_list
)) {
776 int error
= desc
->pg_ops
->pg_doio(desc
);
778 desc
->pg_error
= error
;
780 desc
->pg_bytes_written
+= desc
->pg_count
;
782 if (list_empty(&desc
->pg_list
)) {
789 * nfs_pageio_add_request - Attempt to coalesce a request into a page list.
790 * @desc: destination io descriptor
793 * Returns true if the request 'req' was successfully coalesced into the
794 * existing list of pages 'desc'.
796 static int __nfs_pageio_add_request(struct nfs_pageio_descriptor
*desc
,
797 struct nfs_page
*req
)
799 while (!nfs_pageio_do_add_request(desc
, req
)) {
801 nfs_pageio_doio(desc
);
802 if (desc
->pg_error
< 0)
805 if (desc
->pg_recoalesce
)
811 static int nfs_do_recoalesce(struct nfs_pageio_descriptor
*desc
)
816 list_splice_init(&desc
->pg_list
, &head
);
817 desc
->pg_bytes_written
-= desc
->pg_count
;
820 desc
->pg_recoalesce
= 0;
822 while (!list_empty(&head
)) {
823 struct nfs_page
*req
;
825 req
= list_first_entry(&head
, struct nfs_page
, wb_list
);
826 nfs_list_remove_request(req
);
827 if (__nfs_pageio_add_request(desc
, req
))
829 if (desc
->pg_error
< 0)
833 } while (desc
->pg_recoalesce
);
837 int nfs_pageio_add_request(struct nfs_pageio_descriptor
*desc
,
838 struct nfs_page
*req
)
843 ret
= __nfs_pageio_add_request(desc
, req
);
846 if (desc
->pg_error
< 0)
848 ret
= nfs_do_recoalesce(desc
);
852 EXPORT_SYMBOL_GPL(nfs_pageio_add_request
);
855 * nfs_pageio_complete - Complete I/O on an nfs_pageio_descriptor
856 * @desc: pointer to io descriptor
858 void nfs_pageio_complete(struct nfs_pageio_descriptor
*desc
)
861 nfs_pageio_doio(desc
);
862 if (!desc
->pg_recoalesce
)
864 if (!nfs_do_recoalesce(desc
))
868 EXPORT_SYMBOL_GPL(nfs_pageio_complete
);
871 * nfs_pageio_cond_complete - Conditional I/O completion
872 * @desc: pointer to io descriptor
875 * It is important to ensure that processes don't try to take locks
876 * on non-contiguous ranges of pages as that might deadlock. This
877 * function should be called before attempting to wait on a locked
878 * nfs_page. It will complete the I/O if the page index 'index'
879 * is not contiguous with the existing list of pages in 'desc'.
881 void nfs_pageio_cond_complete(struct nfs_pageio_descriptor
*desc
, pgoff_t index
)
883 if (!list_empty(&desc
->pg_list
)) {
884 struct nfs_page
*prev
= nfs_list_entry(desc
->pg_list
.prev
);
885 if (index
!= prev
->wb_index
+ 1)
886 nfs_pageio_complete(desc
);
890 int __init
nfs_init_nfspagecache(void)
892 nfs_page_cachep
= kmem_cache_create("nfs_page",
893 sizeof(struct nfs_page
),
894 0, SLAB_HWCACHE_ALIGN
,
896 if (nfs_page_cachep
== NULL
)
902 void nfs_destroy_nfspagecache(void)
904 kmem_cache_destroy(nfs_page_cachep
);
907 static const struct rpc_call_ops nfs_pgio_common_ops
= {
908 .rpc_call_prepare
= nfs_pgio_prepare
,
909 .rpc_call_done
= nfs_pgio_result
,
910 .rpc_release
= nfs_pgio_release
,