1 #include <linux/export.h>
3 #include <linux/pagemap.h>
4 #include <linux/slab.h>
5 #include <linux/vmalloc.h>
7 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
11 __v.iov_len = min(n, __p->iov_len - skip); \
12 if (likely(__v.iov_len)) { \
13 __v.iov_base = __p->iov_base + skip; \
15 __v.iov_len -= left; \
16 skip += __v.iov_len; \
21 while (unlikely(!left && n)) { \
23 __v.iov_len = min(n, __p->iov_len); \
24 if (unlikely(!__v.iov_len)) \
26 __v.iov_base = __p->iov_base; \
28 __v.iov_len -= left; \
35 #define iterate_bvec(i, n, __v, __p, skip, STEP) { \
38 __v.bv_len = min_t(size_t, n, __p->bv_len - skip); \
39 if (likely(__v.bv_len)) { \
40 __v.bv_page = __p->bv_page; \
41 __v.bv_offset = __p->bv_offset + skip; \
46 while (unlikely(n)) { \
48 __v.bv_len = min_t(size_t, n, __p->bv_len); \
49 if (unlikely(!__v.bv_len)) \
51 __v.bv_page = __p->bv_page; \
52 __v.bv_offset = __p->bv_offset; \
60 #define iterate_all_kinds(i, n, v, I, B) { \
61 size_t skip = i->iov_offset; \
62 if (unlikely(i->type & ITER_BVEC)) { \
63 const struct bio_vec *bvec; \
65 iterate_bvec(i, n, v, bvec, skip, (B)) \
67 const struct iovec *iov; \
69 iterate_iovec(i, n, v, iov, skip, (I)) \
73 #define iterate_and_advance(i, n, v, I, B) { \
74 size_t skip = i->iov_offset; \
75 if (unlikely(i->type & ITER_BVEC)) { \
76 const struct bio_vec *bvec; \
78 iterate_bvec(i, n, v, bvec, skip, (B)) \
79 if (skip == bvec->bv_len) { \
83 i->nr_segs -= bvec - i->bvec; \
86 const struct iovec *iov; \
88 iterate_iovec(i, n, v, iov, skip, (I)) \
89 if (skip == iov->iov_len) { \
93 i->nr_segs -= iov - i->iov; \
97 i->iov_offset = skip; \
100 static size_t copy_to_iter_iovec(void *from
, size_t bytes
, struct iov_iter
*i
)
102 size_t skip
, copy
, left
, wanted
;
103 const struct iovec
*iov
;
106 if (unlikely(bytes
> i
->count
))
109 if (unlikely(!bytes
))
114 skip
= i
->iov_offset
;
115 buf
= iov
->iov_base
+ skip
;
116 copy
= min(bytes
, iov
->iov_len
- skip
);
118 left
= __copy_to_user(buf
, from
, copy
);
123 while (unlikely(!left
&& bytes
)) {
126 copy
= min(bytes
, iov
->iov_len
);
127 left
= __copy_to_user(buf
, from
, copy
);
134 if (skip
== iov
->iov_len
) {
138 i
->count
-= wanted
- bytes
;
139 i
->nr_segs
-= iov
- i
->iov
;
141 i
->iov_offset
= skip
;
142 return wanted
- bytes
;
145 static size_t copy_from_iter_iovec(void *to
, size_t bytes
, struct iov_iter
*i
)
147 size_t skip
, copy
, left
, wanted
;
148 const struct iovec
*iov
;
151 if (unlikely(bytes
> i
->count
))
154 if (unlikely(!bytes
))
159 skip
= i
->iov_offset
;
160 buf
= iov
->iov_base
+ skip
;
161 copy
= min(bytes
, iov
->iov_len
- skip
);
163 left
= __copy_from_user(to
, buf
, copy
);
168 while (unlikely(!left
&& bytes
)) {
171 copy
= min(bytes
, iov
->iov_len
);
172 left
= __copy_from_user(to
, buf
, copy
);
179 if (skip
== iov
->iov_len
) {
183 i
->count
-= wanted
- bytes
;
184 i
->nr_segs
-= iov
- i
->iov
;
186 i
->iov_offset
= skip
;
187 return wanted
- bytes
;
190 static size_t copy_page_to_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
193 size_t skip
, copy
, left
, wanted
;
194 const struct iovec
*iov
;
198 if (unlikely(bytes
> i
->count
))
201 if (unlikely(!bytes
))
206 skip
= i
->iov_offset
;
207 buf
= iov
->iov_base
+ skip
;
208 copy
= min(bytes
, iov
->iov_len
- skip
);
210 if (!fault_in_pages_writeable(buf
, copy
)) {
211 kaddr
= kmap_atomic(page
);
212 from
= kaddr
+ offset
;
214 /* first chunk, usually the only one */
215 left
= __copy_to_user_inatomic(buf
, from
, copy
);
221 while (unlikely(!left
&& bytes
)) {
224 copy
= min(bytes
, iov
->iov_len
);
225 left
= __copy_to_user_inatomic(buf
, from
, copy
);
231 if (likely(!bytes
)) {
232 kunmap_atomic(kaddr
);
235 offset
= from
- kaddr
;
237 kunmap_atomic(kaddr
);
238 copy
= min(bytes
, iov
->iov_len
- skip
);
240 /* Too bad - revert to non-atomic kmap */
242 from
= kaddr
+ offset
;
243 left
= __copy_to_user(buf
, from
, copy
);
248 while (unlikely(!left
&& bytes
)) {
251 copy
= min(bytes
, iov
->iov_len
);
252 left
= __copy_to_user(buf
, from
, copy
);
260 if (skip
== iov
->iov_len
) {
264 i
->count
-= wanted
- bytes
;
265 i
->nr_segs
-= iov
- i
->iov
;
267 i
->iov_offset
= skip
;
268 return wanted
- bytes
;
271 static size_t copy_page_from_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
274 size_t skip
, copy
, left
, wanted
;
275 const struct iovec
*iov
;
279 if (unlikely(bytes
> i
->count
))
282 if (unlikely(!bytes
))
287 skip
= i
->iov_offset
;
288 buf
= iov
->iov_base
+ skip
;
289 copy
= min(bytes
, iov
->iov_len
- skip
);
291 if (!fault_in_pages_readable(buf
, copy
)) {
292 kaddr
= kmap_atomic(page
);
295 /* first chunk, usually the only one */
296 left
= __copy_from_user_inatomic(to
, buf
, copy
);
302 while (unlikely(!left
&& bytes
)) {
305 copy
= min(bytes
, iov
->iov_len
);
306 left
= __copy_from_user_inatomic(to
, buf
, copy
);
312 if (likely(!bytes
)) {
313 kunmap_atomic(kaddr
);
318 kunmap_atomic(kaddr
);
319 copy
= min(bytes
, iov
->iov_len
- skip
);
321 /* Too bad - revert to non-atomic kmap */
324 left
= __copy_from_user(to
, buf
, copy
);
329 while (unlikely(!left
&& bytes
)) {
332 copy
= min(bytes
, iov
->iov_len
);
333 left
= __copy_from_user(to
, buf
, copy
);
341 if (skip
== iov
->iov_len
) {
345 i
->count
-= wanted
- bytes
;
346 i
->nr_segs
-= iov
- i
->iov
;
348 i
->iov_offset
= skip
;
349 return wanted
- bytes
;
352 static size_t zero_iovec(size_t bytes
, struct iov_iter
*i
)
354 size_t skip
, copy
, left
, wanted
;
355 const struct iovec
*iov
;
358 if (unlikely(bytes
> i
->count
))
361 if (unlikely(!bytes
))
366 skip
= i
->iov_offset
;
367 buf
= iov
->iov_base
+ skip
;
368 copy
= min(bytes
, iov
->iov_len
- skip
);
370 left
= __clear_user(buf
, copy
);
375 while (unlikely(!left
&& bytes
)) {
378 copy
= min(bytes
, iov
->iov_len
);
379 left
= __clear_user(buf
, copy
);
385 if (skip
== iov
->iov_len
) {
389 i
->count
-= wanted
- bytes
;
390 i
->nr_segs
-= iov
- i
->iov
;
392 i
->iov_offset
= skip
;
393 return wanted
- bytes
;
397 * Fault in the first iovec of the given iov_iter, to a maximum length
398 * of bytes. Returns 0 on success, or non-zero if the memory could not be
399 * accessed (ie. because it is an invalid address).
401 * writev-intensive code may want this to prefault several iovecs -- that
402 * would be possible (callers must not rely on the fact that _only_ the
403 * first iovec will be faulted with the current implementation).
405 int iov_iter_fault_in_readable(struct iov_iter
*i
, size_t bytes
)
407 if (!(i
->type
& ITER_BVEC
)) {
408 char __user
*buf
= i
->iov
->iov_base
+ i
->iov_offset
;
409 bytes
= min(bytes
, i
->iov
->iov_len
- i
->iov_offset
);
410 return fault_in_pages_readable(buf
, bytes
);
414 EXPORT_SYMBOL(iov_iter_fault_in_readable
);
416 void iov_iter_init(struct iov_iter
*i
, int direction
,
417 const struct iovec
*iov
, unsigned long nr_segs
,
420 /* It will get better. Eventually... */
421 if (segment_eq(get_fs(), KERNEL_DS
))
422 direction
|= ITER_KVEC
;
425 i
->nr_segs
= nr_segs
;
429 EXPORT_SYMBOL(iov_iter_init
);
431 static ssize_t
get_pages_iovec(struct iov_iter
*i
,
432 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
435 size_t offset
= i
->iov_offset
;
436 const struct iovec
*iov
= i
->iov
;
442 len
= iov
->iov_len
- offset
;
447 addr
= (unsigned long)iov
->iov_base
+ offset
;
448 len
+= *start
= addr
& (PAGE_SIZE
- 1);
449 if (len
> maxpages
* PAGE_SIZE
)
450 len
= maxpages
* PAGE_SIZE
;
451 addr
&= ~(PAGE_SIZE
- 1);
452 n
= (len
+ PAGE_SIZE
- 1) / PAGE_SIZE
;
453 res
= get_user_pages_fast(addr
, n
, (i
->type
& WRITE
) != WRITE
, pages
);
454 if (unlikely(res
< 0))
456 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
459 static ssize_t
get_pages_alloc_iovec(struct iov_iter
*i
,
460 struct page
***pages
, size_t maxsize
,
463 size_t offset
= i
->iov_offset
;
464 const struct iovec
*iov
= i
->iov
;
471 len
= iov
->iov_len
- offset
;
476 addr
= (unsigned long)iov
->iov_base
+ offset
;
477 len
+= *start
= addr
& (PAGE_SIZE
- 1);
478 addr
&= ~(PAGE_SIZE
- 1);
479 n
= (len
+ PAGE_SIZE
- 1) / PAGE_SIZE
;
481 p
= kmalloc(n
* sizeof(struct page
*), GFP_KERNEL
);
483 p
= vmalloc(n
* sizeof(struct page
*));
487 res
= get_user_pages_fast(addr
, n
, (i
->type
& WRITE
) != WRITE
, p
);
488 if (unlikely(res
< 0)) {
493 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
496 static void memcpy_from_page(char *to
, struct page
*page
, size_t offset
, size_t len
)
498 char *from
= kmap_atomic(page
);
499 memcpy(to
, from
+ offset
, len
);
503 static void memcpy_to_page(struct page
*page
, size_t offset
, char *from
, size_t len
)
505 char *to
= kmap_atomic(page
);
506 memcpy(to
+ offset
, from
, len
);
510 static void memzero_page(struct page
*page
, size_t offset
, size_t len
)
512 char *addr
= kmap_atomic(page
);
513 memset(addr
+ offset
, 0, len
);
517 static size_t copy_to_iter_bvec(void *from
, size_t bytes
, struct iov_iter
*i
)
519 size_t skip
, copy
, wanted
;
520 const struct bio_vec
*bvec
;
522 if (unlikely(bytes
> i
->count
))
525 if (unlikely(!bytes
))
530 skip
= i
->iov_offset
;
531 copy
= min_t(size_t, bytes
, bvec
->bv_len
- skip
);
533 memcpy_to_page(bvec
->bv_page
, skip
+ bvec
->bv_offset
, from
, copy
);
539 copy
= min(bytes
, (size_t)bvec
->bv_len
);
540 memcpy_to_page(bvec
->bv_page
, bvec
->bv_offset
, from
, copy
);
545 if (skip
== bvec
->bv_len
) {
549 i
->count
-= wanted
- bytes
;
550 i
->nr_segs
-= bvec
- i
->bvec
;
552 i
->iov_offset
= skip
;
553 return wanted
- bytes
;
556 static size_t copy_from_iter_bvec(void *to
, size_t bytes
, struct iov_iter
*i
)
558 size_t skip
, copy
, wanted
;
559 const struct bio_vec
*bvec
;
561 if (unlikely(bytes
> i
->count
))
564 if (unlikely(!bytes
))
569 skip
= i
->iov_offset
;
571 copy
= min(bytes
, bvec
->bv_len
- skip
);
573 memcpy_from_page(to
, bvec
->bv_page
, bvec
->bv_offset
+ skip
, copy
);
581 copy
= min(bytes
, (size_t)bvec
->bv_len
);
582 memcpy_from_page(to
, bvec
->bv_page
, bvec
->bv_offset
, copy
);
587 if (skip
== bvec
->bv_len
) {
592 i
->nr_segs
-= bvec
- i
->bvec
;
594 i
->iov_offset
= skip
;
598 static size_t copy_page_to_iter_bvec(struct page
*page
, size_t offset
,
599 size_t bytes
, struct iov_iter
*i
)
601 void *kaddr
= kmap_atomic(page
);
602 size_t wanted
= copy_to_iter_bvec(kaddr
+ offset
, bytes
, i
);
603 kunmap_atomic(kaddr
);
607 static size_t copy_page_from_iter_bvec(struct page
*page
, size_t offset
,
608 size_t bytes
, struct iov_iter
*i
)
610 void *kaddr
= kmap_atomic(page
);
611 size_t wanted
= copy_from_iter_bvec(kaddr
+ offset
, bytes
, i
);
612 kunmap_atomic(kaddr
);
616 static size_t zero_bvec(size_t bytes
, struct iov_iter
*i
)
618 size_t skip
, copy
, wanted
;
619 const struct bio_vec
*bvec
;
621 if (unlikely(bytes
> i
->count
))
624 if (unlikely(!bytes
))
629 skip
= i
->iov_offset
;
630 copy
= min_t(size_t, bytes
, bvec
->bv_len
- skip
);
632 memzero_page(bvec
->bv_page
, skip
+ bvec
->bv_offset
, copy
);
637 copy
= min(bytes
, (size_t)bvec
->bv_len
);
638 memzero_page(bvec
->bv_page
, bvec
->bv_offset
, copy
);
642 if (skip
== bvec
->bv_len
) {
646 i
->count
-= wanted
- bytes
;
647 i
->nr_segs
-= bvec
- i
->bvec
;
649 i
->iov_offset
= skip
;
650 return wanted
- bytes
;
653 static ssize_t
get_pages_bvec(struct iov_iter
*i
,
654 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
657 const struct bio_vec
*bvec
= i
->bvec
;
658 size_t len
= bvec
->bv_len
- i
->iov_offset
;
663 /* can't be more than PAGE_SIZE */
664 *start
= bvec
->bv_offset
+ i
->iov_offset
;
666 get_page(*pages
= bvec
->bv_page
);
671 static ssize_t
get_pages_alloc_bvec(struct iov_iter
*i
,
672 struct page
***pages
, size_t maxsize
,
675 const struct bio_vec
*bvec
= i
->bvec
;
676 size_t len
= bvec
->bv_len
- i
->iov_offset
;
681 *start
= bvec
->bv_offset
+ i
->iov_offset
;
683 *pages
= kmalloc(sizeof(struct page
*), GFP_KERNEL
);
687 get_page(**pages
= bvec
->bv_page
);
692 size_t copy_page_to_iter(struct page
*page
, size_t offset
, size_t bytes
,
695 if (i
->type
& ITER_BVEC
)
696 return copy_page_to_iter_bvec(page
, offset
, bytes
, i
);
698 return copy_page_to_iter_iovec(page
, offset
, bytes
, i
);
700 EXPORT_SYMBOL(copy_page_to_iter
);
702 size_t copy_page_from_iter(struct page
*page
, size_t offset
, size_t bytes
,
705 if (i
->type
& ITER_BVEC
)
706 return copy_page_from_iter_bvec(page
, offset
, bytes
, i
);
708 return copy_page_from_iter_iovec(page
, offset
, bytes
, i
);
710 EXPORT_SYMBOL(copy_page_from_iter
);
712 size_t copy_to_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
714 if (i
->type
& ITER_BVEC
)
715 return copy_to_iter_bvec(addr
, bytes
, i
);
717 return copy_to_iter_iovec(addr
, bytes
, i
);
719 EXPORT_SYMBOL(copy_to_iter
);
721 size_t copy_from_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
723 if (i
->type
& ITER_BVEC
)
724 return copy_from_iter_bvec(addr
, bytes
, i
);
726 return copy_from_iter_iovec(addr
, bytes
, i
);
728 EXPORT_SYMBOL(copy_from_iter
);
730 size_t iov_iter_zero(size_t bytes
, struct iov_iter
*i
)
732 if (i
->type
& ITER_BVEC
) {
733 return zero_bvec(bytes
, i
);
735 return zero_iovec(bytes
, i
);
738 EXPORT_SYMBOL(iov_iter_zero
);
740 size_t iov_iter_copy_from_user_atomic(struct page
*page
,
741 struct iov_iter
*i
, unsigned long offset
, size_t bytes
)
743 char *kaddr
= kmap_atomic(page
), *p
= kaddr
+ offset
;
744 iterate_all_kinds(i
, bytes
, v
,
745 __copy_from_user_inatomic((p
+= v
.iov_len
) - v
.iov_len
,
746 v
.iov_base
, v
.iov_len
),
747 memcpy_from_page((p
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
748 v
.bv_offset
, v
.bv_len
)
750 kunmap_atomic(kaddr
);
753 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic
);
755 void iov_iter_advance(struct iov_iter
*i
, size_t size
)
757 iterate_and_advance(i
, size
, v
, 0, 0)
759 EXPORT_SYMBOL(iov_iter_advance
);
762 * Return the count of just the current iov_iter segment.
764 size_t iov_iter_single_seg_count(const struct iov_iter
*i
)
768 else if (i
->type
& ITER_BVEC
)
769 return min(i
->count
, i
->bvec
->bv_len
- i
->iov_offset
);
771 return min(i
->count
, i
->iov
->iov_len
- i
->iov_offset
);
773 EXPORT_SYMBOL(iov_iter_single_seg_count
);
775 unsigned long iov_iter_alignment(const struct iov_iter
*i
)
777 unsigned long res
= 0;
778 size_t size
= i
->count
;
783 iterate_all_kinds(i
, size
, v
,
784 (res
|= (unsigned long)v
.iov_base
| v
.iov_len
, 0),
785 res
|= v
.bv_offset
| v
.bv_len
789 EXPORT_SYMBOL(iov_iter_alignment
);
791 ssize_t
iov_iter_get_pages(struct iov_iter
*i
,
792 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
795 if (i
->type
& ITER_BVEC
)
796 return get_pages_bvec(i
, pages
, maxsize
, maxpages
, start
);
798 return get_pages_iovec(i
, pages
, maxsize
, maxpages
, start
);
800 EXPORT_SYMBOL(iov_iter_get_pages
);
802 ssize_t
iov_iter_get_pages_alloc(struct iov_iter
*i
,
803 struct page
***pages
, size_t maxsize
,
806 if (i
->type
& ITER_BVEC
)
807 return get_pages_alloc_bvec(i
, pages
, maxsize
, start
);
809 return get_pages_alloc_iovec(i
, pages
, maxsize
, start
);
811 EXPORT_SYMBOL(iov_iter_get_pages_alloc
);
813 int iov_iter_npages(const struct iov_iter
*i
, int maxpages
)
815 size_t size
= i
->count
;
821 iterate_all_kinds(i
, size
, v
, ({
822 unsigned long p
= (unsigned long)v
.iov_base
;
823 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
825 if (npages
>= maxpages
)
829 if (npages
>= maxpages
)
835 EXPORT_SYMBOL(iov_iter_npages
);