1 #include <linux/export.h>
3 #include <linux/pagemap.h>
4 #include <linux/slab.h>
5 #include <linux/vmalloc.h>
6 #include <net/checksum.h>
8 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
12 __v.iov_len = min(n, __p->iov_len - skip); \
13 if (likely(__v.iov_len)) { \
14 __v.iov_base = __p->iov_base + skip; \
16 __v.iov_len -= left; \
17 skip += __v.iov_len; \
22 while (unlikely(!left && n)) { \
24 __v.iov_len = min(n, __p->iov_len); \
25 if (unlikely(!__v.iov_len)) \
27 __v.iov_base = __p->iov_base; \
29 __v.iov_len -= left; \
36 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
39 __v.iov_len = min(n, __p->iov_len - skip); \
40 if (likely(__v.iov_len)) { \
41 __v.iov_base = __p->iov_base + skip; \
43 skip += __v.iov_len; \
46 while (unlikely(n)) { \
48 __v.iov_len = min(n, __p->iov_len); \
49 if (unlikely(!__v.iov_len)) \
51 __v.iov_base = __p->iov_base; \
59 #define iterate_bvec(i, n, __v, __p, skip, STEP) { \
62 __v.bv_len = min_t(size_t, n, __p->bv_len - skip); \
63 if (likely(__v.bv_len)) { \
64 __v.bv_page = __p->bv_page; \
65 __v.bv_offset = __p->bv_offset + skip; \
70 while (unlikely(n)) { \
72 __v.bv_len = min_t(size_t, n, __p->bv_len); \
73 if (unlikely(!__v.bv_len)) \
75 __v.bv_page = __p->bv_page; \
76 __v.bv_offset = __p->bv_offset; \
84 #define iterate_all_kinds(i, n, v, I, B, K) { \
85 size_t skip = i->iov_offset; \
86 if (unlikely(i->type & ITER_BVEC)) { \
87 const struct bio_vec *bvec; \
89 iterate_bvec(i, n, v, bvec, skip, (B)) \
90 } else if (unlikely(i->type & ITER_KVEC)) { \
91 const struct kvec *kvec; \
93 iterate_kvec(i, n, v, kvec, skip, (K)) \
95 const struct iovec *iov; \
97 iterate_iovec(i, n, v, iov, skip, (I)) \
101 #define iterate_and_advance(i, n, v, I, B, K) { \
102 size_t skip = i->iov_offset; \
103 if (unlikely(i->type & ITER_BVEC)) { \
104 const struct bio_vec *bvec; \
106 iterate_bvec(i, n, v, bvec, skip, (B)) \
107 if (skip == bvec->bv_len) { \
111 i->nr_segs -= bvec - i->bvec; \
113 } else if (unlikely(i->type & ITER_KVEC)) { \
114 const struct kvec *kvec; \
116 iterate_kvec(i, n, v, kvec, skip, (K)) \
117 if (skip == kvec->iov_len) { \
121 i->nr_segs -= kvec - i->kvec; \
124 const struct iovec *iov; \
126 iterate_iovec(i, n, v, iov, skip, (I)) \
127 if (skip == iov->iov_len) { \
131 i->nr_segs -= iov - i->iov; \
135 i->iov_offset = skip; \
138 static size_t copy_page_to_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
141 size_t skip
, copy
, left
, wanted
;
142 const struct iovec
*iov
;
146 if (unlikely(bytes
> i
->count
))
149 if (unlikely(!bytes
))
154 skip
= i
->iov_offset
;
155 buf
= iov
->iov_base
+ skip
;
156 copy
= min(bytes
, iov
->iov_len
- skip
);
158 if (!fault_in_pages_writeable(buf
, copy
)) {
159 kaddr
= kmap_atomic(page
);
160 from
= kaddr
+ offset
;
162 /* first chunk, usually the only one */
163 left
= __copy_to_user_inatomic(buf
, from
, copy
);
169 while (unlikely(!left
&& bytes
)) {
172 copy
= min(bytes
, iov
->iov_len
);
173 left
= __copy_to_user_inatomic(buf
, from
, copy
);
179 if (likely(!bytes
)) {
180 kunmap_atomic(kaddr
);
183 offset
= from
- kaddr
;
185 kunmap_atomic(kaddr
);
186 copy
= min(bytes
, iov
->iov_len
- skip
);
188 /* Too bad - revert to non-atomic kmap */
190 from
= kaddr
+ offset
;
191 left
= __copy_to_user(buf
, from
, copy
);
196 while (unlikely(!left
&& bytes
)) {
199 copy
= min(bytes
, iov
->iov_len
);
200 left
= __copy_to_user(buf
, from
, copy
);
208 if (skip
== iov
->iov_len
) {
212 i
->count
-= wanted
- bytes
;
213 i
->nr_segs
-= iov
- i
->iov
;
215 i
->iov_offset
= skip
;
216 return wanted
- bytes
;
219 static size_t copy_page_from_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
222 size_t skip
, copy
, left
, wanted
;
223 const struct iovec
*iov
;
227 if (unlikely(bytes
> i
->count
))
230 if (unlikely(!bytes
))
235 skip
= i
->iov_offset
;
236 buf
= iov
->iov_base
+ skip
;
237 copy
= min(bytes
, iov
->iov_len
- skip
);
239 if (!fault_in_pages_readable(buf
, copy
)) {
240 kaddr
= kmap_atomic(page
);
243 /* first chunk, usually the only one */
244 left
= __copy_from_user_inatomic(to
, buf
, copy
);
250 while (unlikely(!left
&& bytes
)) {
253 copy
= min(bytes
, iov
->iov_len
);
254 left
= __copy_from_user_inatomic(to
, buf
, copy
);
260 if (likely(!bytes
)) {
261 kunmap_atomic(kaddr
);
266 kunmap_atomic(kaddr
);
267 copy
= min(bytes
, iov
->iov_len
- skip
);
269 /* Too bad - revert to non-atomic kmap */
272 left
= __copy_from_user(to
, buf
, copy
);
277 while (unlikely(!left
&& bytes
)) {
280 copy
= min(bytes
, iov
->iov_len
);
281 left
= __copy_from_user(to
, buf
, copy
);
289 if (skip
== iov
->iov_len
) {
293 i
->count
-= wanted
- bytes
;
294 i
->nr_segs
-= iov
- i
->iov
;
296 i
->iov_offset
= skip
;
297 return wanted
- bytes
;
301 * Fault in the first iovec of the given iov_iter, to a maximum length
302 * of bytes. Returns 0 on success, or non-zero if the memory could not be
303 * accessed (ie. because it is an invalid address).
305 * writev-intensive code may want this to prefault several iovecs -- that
306 * would be possible (callers must not rely on the fact that _only_ the
307 * first iovec will be faulted with the current implementation).
309 int iov_iter_fault_in_readable(struct iov_iter
*i
, size_t bytes
)
311 if (!(i
->type
& (ITER_BVEC
|ITER_KVEC
))) {
312 char __user
*buf
= i
->iov
->iov_base
+ i
->iov_offset
;
313 bytes
= min(bytes
, i
->iov
->iov_len
- i
->iov_offset
);
314 return fault_in_pages_readable(buf
, bytes
);
318 EXPORT_SYMBOL(iov_iter_fault_in_readable
);
320 void iov_iter_init(struct iov_iter
*i
, int direction
,
321 const struct iovec
*iov
, unsigned long nr_segs
,
324 /* It will get better. Eventually... */
325 if (segment_eq(get_fs(), KERNEL_DS
)) {
326 direction
|= ITER_KVEC
;
328 i
->kvec
= (struct kvec
*)iov
;
333 i
->nr_segs
= nr_segs
;
337 EXPORT_SYMBOL(iov_iter_init
);
339 static void memcpy_from_page(char *to
, struct page
*page
, size_t offset
, size_t len
)
341 char *from
= kmap_atomic(page
);
342 memcpy(to
, from
+ offset
, len
);
346 static void memcpy_to_page(struct page
*page
, size_t offset
, char *from
, size_t len
)
348 char *to
= kmap_atomic(page
);
349 memcpy(to
+ offset
, from
, len
);
353 static void memzero_page(struct page
*page
, size_t offset
, size_t len
)
355 char *addr
= kmap_atomic(page
);
356 memset(addr
+ offset
, 0, len
);
360 size_t copy_to_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
363 if (unlikely(bytes
> i
->count
))
366 if (unlikely(!bytes
))
369 iterate_and_advance(i
, bytes
, v
,
370 __copy_to_user(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
,
372 memcpy_to_page(v
.bv_page
, v
.bv_offset
,
373 (from
+= v
.bv_len
) - v
.bv_len
, v
.bv_len
),
374 memcpy(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
, v
.iov_len
)
379 EXPORT_SYMBOL(copy_to_iter
);
381 size_t copy_from_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
384 if (unlikely(bytes
> i
->count
))
387 if (unlikely(!bytes
))
390 iterate_and_advance(i
, bytes
, v
,
391 __copy_from_user((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
,
393 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
394 v
.bv_offset
, v
.bv_len
),
395 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
400 EXPORT_SYMBOL(copy_from_iter
);
402 size_t copy_page_to_iter(struct page
*page
, size_t offset
, size_t bytes
,
405 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
406 void *kaddr
= kmap_atomic(page
);
407 size_t wanted
= copy_to_iter(kaddr
+ offset
, bytes
, i
);
408 kunmap_atomic(kaddr
);
411 return copy_page_to_iter_iovec(page
, offset
, bytes
, i
);
413 EXPORT_SYMBOL(copy_page_to_iter
);
415 size_t copy_page_from_iter(struct page
*page
, size_t offset
, size_t bytes
,
418 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
419 void *kaddr
= kmap_atomic(page
);
420 size_t wanted
= copy_from_iter(kaddr
+ offset
, bytes
, i
);
421 kunmap_atomic(kaddr
);
424 return copy_page_from_iter_iovec(page
, offset
, bytes
, i
);
426 EXPORT_SYMBOL(copy_page_from_iter
);
428 size_t iov_iter_zero(size_t bytes
, struct iov_iter
*i
)
430 if (unlikely(bytes
> i
->count
))
433 if (unlikely(!bytes
))
436 iterate_and_advance(i
, bytes
, v
,
437 __clear_user(v
.iov_base
, v
.iov_len
),
438 memzero_page(v
.bv_page
, v
.bv_offset
, v
.bv_len
),
439 memset(v
.iov_base
, 0, v
.iov_len
)
444 EXPORT_SYMBOL(iov_iter_zero
);
446 size_t iov_iter_copy_from_user_atomic(struct page
*page
,
447 struct iov_iter
*i
, unsigned long offset
, size_t bytes
)
449 char *kaddr
= kmap_atomic(page
), *p
= kaddr
+ offset
;
450 iterate_all_kinds(i
, bytes
, v
,
451 __copy_from_user_inatomic((p
+= v
.iov_len
) - v
.iov_len
,
452 v
.iov_base
, v
.iov_len
),
453 memcpy_from_page((p
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
454 v
.bv_offset
, v
.bv_len
),
455 memcpy((p
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
457 kunmap_atomic(kaddr
);
460 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic
);
462 void iov_iter_advance(struct iov_iter
*i
, size_t size
)
464 iterate_and_advance(i
, size
, v
, 0, 0, 0)
466 EXPORT_SYMBOL(iov_iter_advance
);
469 * Return the count of just the current iov_iter segment.
471 size_t iov_iter_single_seg_count(const struct iov_iter
*i
)
475 else if (i
->type
& ITER_BVEC
)
476 return min(i
->count
, i
->bvec
->bv_len
- i
->iov_offset
);
478 return min(i
->count
, i
->iov
->iov_len
- i
->iov_offset
);
480 EXPORT_SYMBOL(iov_iter_single_seg_count
);
482 void iov_iter_kvec(struct iov_iter
*i
, int direction
,
483 const struct kvec
*iov
, unsigned long nr_segs
,
486 BUG_ON(!(direction
& ITER_KVEC
));
488 i
->kvec
= (struct kvec
*)iov
;
489 i
->nr_segs
= nr_segs
;
493 EXPORT_SYMBOL(iov_iter_kvec
);
495 unsigned long iov_iter_alignment(const struct iov_iter
*i
)
497 unsigned long res
= 0;
498 size_t size
= i
->count
;
503 iterate_all_kinds(i
, size
, v
,
504 (res
|= (unsigned long)v
.iov_base
| v
.iov_len
, 0),
505 res
|= v
.bv_offset
| v
.bv_len
,
506 res
|= (unsigned long)v
.iov_base
| v
.iov_len
510 EXPORT_SYMBOL(iov_iter_alignment
);
512 ssize_t
iov_iter_get_pages(struct iov_iter
*i
,
513 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
516 if (maxsize
> i
->count
)
522 iterate_all_kinds(i
, maxsize
, v
, ({
523 unsigned long addr
= (unsigned long)v
.iov_base
;
524 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
528 if (len
> maxpages
* PAGE_SIZE
)
529 len
= maxpages
* PAGE_SIZE
;
530 addr
&= ~(PAGE_SIZE
- 1);
531 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
532 res
= get_user_pages_fast(addr
, n
, (i
->type
& WRITE
) != WRITE
, pages
);
533 if (unlikely(res
< 0))
535 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
537 /* can't be more than PAGE_SIZE */
538 *start
= v
.bv_offset
;
539 get_page(*pages
= v
.bv_page
);
547 EXPORT_SYMBOL(iov_iter_get_pages
);
549 static struct page
**get_pages_array(size_t n
)
551 struct page
**p
= kmalloc(n
* sizeof(struct page
*), GFP_KERNEL
);
553 p
= vmalloc(n
* sizeof(struct page
*));
557 ssize_t
iov_iter_get_pages_alloc(struct iov_iter
*i
,
558 struct page
***pages
, size_t maxsize
,
563 if (maxsize
> i
->count
)
569 iterate_all_kinds(i
, maxsize
, v
, ({
570 unsigned long addr
= (unsigned long)v
.iov_base
;
571 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
575 addr
&= ~(PAGE_SIZE
- 1);
576 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
577 p
= get_pages_array(n
);
580 res
= get_user_pages_fast(addr
, n
, (i
->type
& WRITE
) != WRITE
, p
);
581 if (unlikely(res
< 0)) {
586 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
588 /* can't be more than PAGE_SIZE */
589 *start
= v
.bv_offset
;
590 *pages
= p
= get_pages_array(1);
593 get_page(*p
= v
.bv_page
);
601 EXPORT_SYMBOL(iov_iter_get_pages_alloc
);
603 size_t csum_and_copy_from_iter(void *addr
, size_t bytes
, __wsum
*csum
,
609 if (unlikely(bytes
> i
->count
))
612 if (unlikely(!bytes
))
616 iterate_and_advance(i
, bytes
, v
, ({
618 next
= csum_and_copy_from_user(v
.iov_base
,
619 (to
+= v
.iov_len
) - v
.iov_len
,
622 sum
= csum_block_add(sum
, next
, off
);
627 char *p
= kmap_atomic(v
.bv_page
);
628 next
= csum_partial_copy_nocheck(p
+ v
.bv_offset
,
629 (to
+= v
.bv_len
) - v
.bv_len
,
632 sum
= csum_block_add(sum
, next
, off
);
635 next
= csum_partial_copy_nocheck(v
.iov_base
,
636 (to
+= v
.iov_len
) - v
.iov_len
,
638 sum
= csum_block_add(sum
, next
, off
);
645 EXPORT_SYMBOL(csum_and_copy_from_iter
);
647 size_t csum_and_copy_to_iter(void *addr
, size_t bytes
, __wsum
*csum
,
653 if (unlikely(bytes
> i
->count
))
656 if (unlikely(!bytes
))
660 iterate_and_advance(i
, bytes
, v
, ({
662 next
= csum_and_copy_to_user((from
+= v
.iov_len
) - v
.iov_len
,
666 sum
= csum_block_add(sum
, next
, off
);
671 char *p
= kmap_atomic(v
.bv_page
);
672 next
= csum_partial_copy_nocheck((from
+= v
.bv_len
) - v
.bv_len
,
676 sum
= csum_block_add(sum
, next
, off
);
679 next
= csum_partial_copy_nocheck((from
+= v
.iov_len
) - v
.iov_len
,
682 sum
= csum_block_add(sum
, next
, off
);
689 EXPORT_SYMBOL(csum_and_copy_to_iter
);
691 int iov_iter_npages(const struct iov_iter
*i
, int maxpages
)
693 size_t size
= i
->count
;
699 iterate_all_kinds(i
, size
, v
, ({
700 unsigned long p
= (unsigned long)v
.iov_base
;
701 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
703 if (npages
>= maxpages
)
707 if (npages
>= maxpages
)
710 unsigned long p
= (unsigned long)v
.iov_base
;
711 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
713 if (npages
>= maxpages
)
719 EXPORT_SYMBOL(iov_iter_npages
);