1 #include <linux/export.h>
3 #include <linux/pagemap.h>
4 #include <linux/slab.h>
5 #include <linux/vmalloc.h>
7 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
11 __v.iov_len = min(n, __p->iov_len - skip); \
12 if (likely(__v.iov_len)) { \
13 __v.iov_base = __p->iov_base + skip; \
15 __v.iov_len -= left; \
16 skip += __v.iov_len; \
21 while (unlikely(!left && n)) { \
23 __v.iov_len = min(n, __p->iov_len); \
24 if (unlikely(!__v.iov_len)) \
26 __v.iov_base = __p->iov_base; \
28 __v.iov_len -= left; \
35 #define iterate_bvec(i, n, __v, __p, skip, STEP) { \
38 __v.bv_len = min_t(size_t, n, __p->bv_len - skip); \
39 if (likely(__v.bv_len)) { \
40 __v.bv_page = __p->bv_page; \
41 __v.bv_offset = __p->bv_offset + skip; \
46 while (unlikely(n)) { \
48 __v.bv_len = min_t(size_t, n, __p->bv_len); \
49 if (unlikely(!__v.bv_len)) \
51 __v.bv_page = __p->bv_page; \
52 __v.bv_offset = __p->bv_offset; \
60 #define iterate_all_kinds(i, n, v, I, B) { \
61 size_t skip = i->iov_offset; \
62 if (unlikely(i->type & ITER_BVEC)) { \
63 const struct bio_vec *bvec; \
65 iterate_bvec(i, n, v, bvec, skip, (B)) \
67 const struct iovec *iov; \
69 iterate_iovec(i, n, v, iov, skip, (I)) \
73 #define iterate_and_advance(i, n, v, I, B) { \
74 size_t skip = i->iov_offset; \
75 if (unlikely(i->type & ITER_BVEC)) { \
76 const struct bio_vec *bvec; \
78 iterate_bvec(i, n, v, bvec, skip, (B)) \
79 if (skip == bvec->bv_len) { \
83 i->nr_segs -= bvec - i->bvec; \
86 const struct iovec *iov; \
88 iterate_iovec(i, n, v, iov, skip, (I)) \
89 if (skip == iov->iov_len) { \
93 i->nr_segs -= iov - i->iov; \
97 i->iov_offset = skip; \
100 static size_t copy_to_iter_iovec(void *from
, size_t bytes
, struct iov_iter
*i
)
102 size_t skip
, copy
, left
, wanted
;
103 const struct iovec
*iov
;
106 if (unlikely(bytes
> i
->count
))
109 if (unlikely(!bytes
))
114 skip
= i
->iov_offset
;
115 buf
= iov
->iov_base
+ skip
;
116 copy
= min(bytes
, iov
->iov_len
- skip
);
118 left
= __copy_to_user(buf
, from
, copy
);
123 while (unlikely(!left
&& bytes
)) {
126 copy
= min(bytes
, iov
->iov_len
);
127 left
= __copy_to_user(buf
, from
, copy
);
134 if (skip
== iov
->iov_len
) {
138 i
->count
-= wanted
- bytes
;
139 i
->nr_segs
-= iov
- i
->iov
;
141 i
->iov_offset
= skip
;
142 return wanted
- bytes
;
145 static size_t copy_page_to_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
148 size_t skip
, copy
, left
, wanted
;
149 const struct iovec
*iov
;
153 if (unlikely(bytes
> i
->count
))
156 if (unlikely(!bytes
))
161 skip
= i
->iov_offset
;
162 buf
= iov
->iov_base
+ skip
;
163 copy
= min(bytes
, iov
->iov_len
- skip
);
165 if (!fault_in_pages_writeable(buf
, copy
)) {
166 kaddr
= kmap_atomic(page
);
167 from
= kaddr
+ offset
;
169 /* first chunk, usually the only one */
170 left
= __copy_to_user_inatomic(buf
, from
, copy
);
176 while (unlikely(!left
&& bytes
)) {
179 copy
= min(bytes
, iov
->iov_len
);
180 left
= __copy_to_user_inatomic(buf
, from
, copy
);
186 if (likely(!bytes
)) {
187 kunmap_atomic(kaddr
);
190 offset
= from
- kaddr
;
192 kunmap_atomic(kaddr
);
193 copy
= min(bytes
, iov
->iov_len
- skip
);
195 /* Too bad - revert to non-atomic kmap */
197 from
= kaddr
+ offset
;
198 left
= __copy_to_user(buf
, from
, copy
);
203 while (unlikely(!left
&& bytes
)) {
206 copy
= min(bytes
, iov
->iov_len
);
207 left
= __copy_to_user(buf
, from
, copy
);
215 if (skip
== iov
->iov_len
) {
219 i
->count
-= wanted
- bytes
;
220 i
->nr_segs
-= iov
- i
->iov
;
222 i
->iov_offset
= skip
;
223 return wanted
- bytes
;
226 static size_t copy_page_from_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
229 size_t skip
, copy
, left
, wanted
;
230 const struct iovec
*iov
;
234 if (unlikely(bytes
> i
->count
))
237 if (unlikely(!bytes
))
242 skip
= i
->iov_offset
;
243 buf
= iov
->iov_base
+ skip
;
244 copy
= min(bytes
, iov
->iov_len
- skip
);
246 if (!fault_in_pages_readable(buf
, copy
)) {
247 kaddr
= kmap_atomic(page
);
250 /* first chunk, usually the only one */
251 left
= __copy_from_user_inatomic(to
, buf
, copy
);
257 while (unlikely(!left
&& bytes
)) {
260 copy
= min(bytes
, iov
->iov_len
);
261 left
= __copy_from_user_inatomic(to
, buf
, copy
);
267 if (likely(!bytes
)) {
268 kunmap_atomic(kaddr
);
273 kunmap_atomic(kaddr
);
274 copy
= min(bytes
, iov
->iov_len
- skip
);
276 /* Too bad - revert to non-atomic kmap */
279 left
= __copy_from_user(to
, buf
, copy
);
284 while (unlikely(!left
&& bytes
)) {
287 copy
= min(bytes
, iov
->iov_len
);
288 left
= __copy_from_user(to
, buf
, copy
);
296 if (skip
== iov
->iov_len
) {
300 i
->count
-= wanted
- bytes
;
301 i
->nr_segs
-= iov
- i
->iov
;
303 i
->iov_offset
= skip
;
304 return wanted
- bytes
;
308 * Fault in the first iovec of the given iov_iter, to a maximum length
309 * of bytes. Returns 0 on success, or non-zero if the memory could not be
310 * accessed (ie. because it is an invalid address).
312 * writev-intensive code may want this to prefault several iovecs -- that
313 * would be possible (callers must not rely on the fact that _only_ the
314 * first iovec will be faulted with the current implementation).
316 int iov_iter_fault_in_readable(struct iov_iter
*i
, size_t bytes
)
318 if (!(i
->type
& ITER_BVEC
)) {
319 char __user
*buf
= i
->iov
->iov_base
+ i
->iov_offset
;
320 bytes
= min(bytes
, i
->iov
->iov_len
- i
->iov_offset
);
321 return fault_in_pages_readable(buf
, bytes
);
325 EXPORT_SYMBOL(iov_iter_fault_in_readable
);
327 void iov_iter_init(struct iov_iter
*i
, int direction
,
328 const struct iovec
*iov
, unsigned long nr_segs
,
331 /* It will get better. Eventually... */
332 if (segment_eq(get_fs(), KERNEL_DS
))
333 direction
|= ITER_KVEC
;
336 i
->nr_segs
= nr_segs
;
340 EXPORT_SYMBOL(iov_iter_init
);
342 static void memcpy_from_page(char *to
, struct page
*page
, size_t offset
, size_t len
)
344 char *from
= kmap_atomic(page
);
345 memcpy(to
, from
+ offset
, len
);
349 static void memcpy_to_page(struct page
*page
, size_t offset
, char *from
, size_t len
)
351 char *to
= kmap_atomic(page
);
352 memcpy(to
+ offset
, from
, len
);
356 static void memzero_page(struct page
*page
, size_t offset
, size_t len
)
358 char *addr
= kmap_atomic(page
);
359 memset(addr
+ offset
, 0, len
);
363 static size_t copy_to_iter_bvec(void *from
, size_t bytes
, struct iov_iter
*i
)
365 size_t skip
, copy
, wanted
;
366 const struct bio_vec
*bvec
;
368 if (unlikely(bytes
> i
->count
))
371 if (unlikely(!bytes
))
376 skip
= i
->iov_offset
;
377 copy
= min_t(size_t, bytes
, bvec
->bv_len
- skip
);
379 memcpy_to_page(bvec
->bv_page
, skip
+ bvec
->bv_offset
, from
, copy
);
385 copy
= min(bytes
, (size_t)bvec
->bv_len
);
386 memcpy_to_page(bvec
->bv_page
, bvec
->bv_offset
, from
, copy
);
391 if (skip
== bvec
->bv_len
) {
395 i
->count
-= wanted
- bytes
;
396 i
->nr_segs
-= bvec
- i
->bvec
;
398 i
->iov_offset
= skip
;
399 return wanted
- bytes
;
402 size_t copy_to_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
404 if (i
->type
& ITER_BVEC
)
405 return copy_to_iter_bvec(addr
, bytes
, i
);
407 return copy_to_iter_iovec(addr
, bytes
, i
);
409 EXPORT_SYMBOL(copy_to_iter
);
411 size_t copy_from_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
414 if (unlikely(bytes
> i
->count
))
417 if (unlikely(!bytes
))
420 iterate_and_advance(i
, bytes
, v
,
421 __copy_from_user((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
,
423 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
424 v
.bv_offset
, v
.bv_len
)
429 EXPORT_SYMBOL(copy_from_iter
);
431 size_t copy_page_to_iter(struct page
*page
, size_t offset
, size_t bytes
,
434 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
435 void *kaddr
= kmap_atomic(page
);
436 size_t wanted
= copy_to_iter(kaddr
+ offset
, bytes
, i
);
437 kunmap_atomic(kaddr
);
440 return copy_page_to_iter_iovec(page
, offset
, bytes
, i
);
442 EXPORT_SYMBOL(copy_page_to_iter
);
444 size_t copy_page_from_iter(struct page
*page
, size_t offset
, size_t bytes
,
447 if (i
->type
& ITER_BVEC
) {
448 void *kaddr
= kmap_atomic(page
);
449 size_t wanted
= copy_from_iter(kaddr
+ offset
, bytes
, i
);
450 kunmap_atomic(kaddr
);
453 return copy_page_from_iter_iovec(page
, offset
, bytes
, i
);
455 EXPORT_SYMBOL(copy_page_from_iter
);
457 size_t iov_iter_zero(size_t bytes
, struct iov_iter
*i
)
459 if (unlikely(bytes
> i
->count
))
462 if (unlikely(!bytes
))
465 iterate_and_advance(i
, bytes
, v
,
466 __clear_user(v
.iov_base
, v
.iov_len
),
467 memzero_page(v
.bv_page
, v
.bv_offset
, v
.bv_len
)
472 EXPORT_SYMBOL(iov_iter_zero
);
474 size_t iov_iter_copy_from_user_atomic(struct page
*page
,
475 struct iov_iter
*i
, unsigned long offset
, size_t bytes
)
477 char *kaddr
= kmap_atomic(page
), *p
= kaddr
+ offset
;
478 iterate_all_kinds(i
, bytes
, v
,
479 __copy_from_user_inatomic((p
+= v
.iov_len
) - v
.iov_len
,
480 v
.iov_base
, v
.iov_len
),
481 memcpy_from_page((p
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
482 v
.bv_offset
, v
.bv_len
)
484 kunmap_atomic(kaddr
);
487 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic
);
489 void iov_iter_advance(struct iov_iter
*i
, size_t size
)
491 iterate_and_advance(i
, size
, v
, 0, 0)
493 EXPORT_SYMBOL(iov_iter_advance
);
496 * Return the count of just the current iov_iter segment.
498 size_t iov_iter_single_seg_count(const struct iov_iter
*i
)
502 else if (i
->type
& ITER_BVEC
)
503 return min(i
->count
, i
->bvec
->bv_len
- i
->iov_offset
);
505 return min(i
->count
, i
->iov
->iov_len
- i
->iov_offset
);
507 EXPORT_SYMBOL(iov_iter_single_seg_count
);
509 unsigned long iov_iter_alignment(const struct iov_iter
*i
)
511 unsigned long res
= 0;
512 size_t size
= i
->count
;
517 iterate_all_kinds(i
, size
, v
,
518 (res
|= (unsigned long)v
.iov_base
| v
.iov_len
, 0),
519 res
|= v
.bv_offset
| v
.bv_len
523 EXPORT_SYMBOL(iov_iter_alignment
);
525 ssize_t
iov_iter_get_pages(struct iov_iter
*i
,
526 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
529 if (maxsize
> i
->count
)
535 iterate_all_kinds(i
, maxsize
, v
, ({
536 unsigned long addr
= (unsigned long)v
.iov_base
;
537 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
541 if (len
> maxpages
* PAGE_SIZE
)
542 len
= maxpages
* PAGE_SIZE
;
543 addr
&= ~(PAGE_SIZE
- 1);
544 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
545 res
= get_user_pages_fast(addr
, n
, (i
->type
& WRITE
) != WRITE
, pages
);
546 if (unlikely(res
< 0))
548 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
550 /* can't be more than PAGE_SIZE */
551 *start
= v
.bv_offset
;
552 get_page(*pages
= v
.bv_page
);
558 EXPORT_SYMBOL(iov_iter_get_pages
);
560 static struct page
**get_pages_array(size_t n
)
562 struct page
**p
= kmalloc(n
* sizeof(struct page
*), GFP_KERNEL
);
564 p
= vmalloc(n
* sizeof(struct page
*));
568 ssize_t
iov_iter_get_pages_alloc(struct iov_iter
*i
,
569 struct page
***pages
, size_t maxsize
,
574 if (maxsize
> i
->count
)
580 iterate_all_kinds(i
, maxsize
, v
, ({
581 unsigned long addr
= (unsigned long)v
.iov_base
;
582 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
586 addr
&= ~(PAGE_SIZE
- 1);
587 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
588 p
= get_pages_array(n
);
591 res
= get_user_pages_fast(addr
, n
, (i
->type
& WRITE
) != WRITE
, p
);
592 if (unlikely(res
< 0)) {
597 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
599 /* can't be more than PAGE_SIZE */
600 *start
= v
.bv_offset
;
601 *pages
= p
= get_pages_array(1);
604 get_page(*p
= v
.bv_page
);
610 EXPORT_SYMBOL(iov_iter_get_pages_alloc
);
612 int iov_iter_npages(const struct iov_iter
*i
, int maxpages
)
614 size_t size
= i
->count
;
620 iterate_all_kinds(i
, size
, v
, ({
621 unsigned long p
= (unsigned long)v
.iov_base
;
622 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
624 if (npages
>= maxpages
)
628 if (npages
>= maxpages
)
634 EXPORT_SYMBOL(iov_iter_npages
);