1 #include <linux/export.h>
3 #include <linux/pagemap.h>
4 #include <linux/slab.h>
5 #include <linux/vmalloc.h>
7 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
11 __v.iov_len = min(n, __p->iov_len - skip); \
12 if (likely(__v.iov_len)) { \
13 __v.iov_base = __p->iov_base + skip; \
15 __v.iov_len -= left; \
16 skip += __v.iov_len; \
21 while (unlikely(!left && n)) { \
23 __v.iov_len = min(n, __p->iov_len); \
24 if (unlikely(!__v.iov_len)) \
26 __v.iov_base = __p->iov_base; \
28 __v.iov_len -= left; \
35 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
38 __v.iov_len = min(n, __p->iov_len - skip); \
39 if (likely(__v.iov_len)) { \
40 __v.iov_base = __p->iov_base + skip; \
42 skip += __v.iov_len; \
45 while (unlikely(n)) { \
47 __v.iov_len = min(n, __p->iov_len); \
48 if (unlikely(!__v.iov_len)) \
50 __v.iov_base = __p->iov_base; \
58 #define iterate_bvec(i, n, __v, __p, skip, STEP) { \
61 __v.bv_len = min_t(size_t, n, __p->bv_len - skip); \
62 if (likely(__v.bv_len)) { \
63 __v.bv_page = __p->bv_page; \
64 __v.bv_offset = __p->bv_offset + skip; \
69 while (unlikely(n)) { \
71 __v.bv_len = min_t(size_t, n, __p->bv_len); \
72 if (unlikely(!__v.bv_len)) \
74 __v.bv_page = __p->bv_page; \
75 __v.bv_offset = __p->bv_offset; \
83 #define iterate_all_kinds(i, n, v, I, B, K) { \
84 size_t skip = i->iov_offset; \
85 if (unlikely(i->type & ITER_BVEC)) { \
86 const struct bio_vec *bvec; \
88 iterate_bvec(i, n, v, bvec, skip, (B)) \
89 } else if (unlikely(i->type & ITER_KVEC)) { \
90 const struct kvec *kvec; \
92 iterate_kvec(i, n, v, kvec, skip, (K)) \
94 const struct iovec *iov; \
96 iterate_iovec(i, n, v, iov, skip, (I)) \
100 #define iterate_and_advance(i, n, v, I, B, K) { \
101 size_t skip = i->iov_offset; \
102 if (unlikely(i->type & ITER_BVEC)) { \
103 const struct bio_vec *bvec; \
105 iterate_bvec(i, n, v, bvec, skip, (B)) \
106 if (skip == bvec->bv_len) { \
110 i->nr_segs -= bvec - i->bvec; \
112 } else if (unlikely(i->type & ITER_KVEC)) { \
113 const struct kvec *kvec; \
115 iterate_kvec(i, n, v, kvec, skip, (K)) \
116 if (skip == kvec->iov_len) { \
120 i->nr_segs -= kvec - i->kvec; \
123 const struct iovec *iov; \
125 iterate_iovec(i, n, v, iov, skip, (I)) \
126 if (skip == iov->iov_len) { \
130 i->nr_segs -= iov - i->iov; \
134 i->iov_offset = skip; \
137 static size_t copy_page_to_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
140 size_t skip
, copy
, left
, wanted
;
141 const struct iovec
*iov
;
145 if (unlikely(bytes
> i
->count
))
148 if (unlikely(!bytes
))
153 skip
= i
->iov_offset
;
154 buf
= iov
->iov_base
+ skip
;
155 copy
= min(bytes
, iov
->iov_len
- skip
);
157 if (!fault_in_pages_writeable(buf
, copy
)) {
158 kaddr
= kmap_atomic(page
);
159 from
= kaddr
+ offset
;
161 /* first chunk, usually the only one */
162 left
= __copy_to_user_inatomic(buf
, from
, copy
);
168 while (unlikely(!left
&& bytes
)) {
171 copy
= min(bytes
, iov
->iov_len
);
172 left
= __copy_to_user_inatomic(buf
, from
, copy
);
178 if (likely(!bytes
)) {
179 kunmap_atomic(kaddr
);
182 offset
= from
- kaddr
;
184 kunmap_atomic(kaddr
);
185 copy
= min(bytes
, iov
->iov_len
- skip
);
187 /* Too bad - revert to non-atomic kmap */
189 from
= kaddr
+ offset
;
190 left
= __copy_to_user(buf
, from
, copy
);
195 while (unlikely(!left
&& bytes
)) {
198 copy
= min(bytes
, iov
->iov_len
);
199 left
= __copy_to_user(buf
, from
, copy
);
207 if (skip
== iov
->iov_len
) {
211 i
->count
-= wanted
- bytes
;
212 i
->nr_segs
-= iov
- i
->iov
;
214 i
->iov_offset
= skip
;
215 return wanted
- bytes
;
218 static size_t copy_page_from_iter_iovec(struct page
*page
, size_t offset
, size_t bytes
,
221 size_t skip
, copy
, left
, wanted
;
222 const struct iovec
*iov
;
226 if (unlikely(bytes
> i
->count
))
229 if (unlikely(!bytes
))
234 skip
= i
->iov_offset
;
235 buf
= iov
->iov_base
+ skip
;
236 copy
= min(bytes
, iov
->iov_len
- skip
);
238 if (!fault_in_pages_readable(buf
, copy
)) {
239 kaddr
= kmap_atomic(page
);
242 /* first chunk, usually the only one */
243 left
= __copy_from_user_inatomic(to
, buf
, copy
);
249 while (unlikely(!left
&& bytes
)) {
252 copy
= min(bytes
, iov
->iov_len
);
253 left
= __copy_from_user_inatomic(to
, buf
, copy
);
259 if (likely(!bytes
)) {
260 kunmap_atomic(kaddr
);
265 kunmap_atomic(kaddr
);
266 copy
= min(bytes
, iov
->iov_len
- skip
);
268 /* Too bad - revert to non-atomic kmap */
271 left
= __copy_from_user(to
, buf
, copy
);
276 while (unlikely(!left
&& bytes
)) {
279 copy
= min(bytes
, iov
->iov_len
);
280 left
= __copy_from_user(to
, buf
, copy
);
288 if (skip
== iov
->iov_len
) {
292 i
->count
-= wanted
- bytes
;
293 i
->nr_segs
-= iov
- i
->iov
;
295 i
->iov_offset
= skip
;
296 return wanted
- bytes
;
300 * Fault in the first iovec of the given iov_iter, to a maximum length
301 * of bytes. Returns 0 on success, or non-zero if the memory could not be
302 * accessed (ie. because it is an invalid address).
304 * writev-intensive code may want this to prefault several iovecs -- that
305 * would be possible (callers must not rely on the fact that _only_ the
306 * first iovec will be faulted with the current implementation).
308 int iov_iter_fault_in_readable(struct iov_iter
*i
, size_t bytes
)
310 if (!(i
->type
& (ITER_BVEC
|ITER_KVEC
))) {
311 char __user
*buf
= i
->iov
->iov_base
+ i
->iov_offset
;
312 bytes
= min(bytes
, i
->iov
->iov_len
- i
->iov_offset
);
313 return fault_in_pages_readable(buf
, bytes
);
317 EXPORT_SYMBOL(iov_iter_fault_in_readable
);
319 void iov_iter_init(struct iov_iter
*i
, int direction
,
320 const struct iovec
*iov
, unsigned long nr_segs
,
323 /* It will get better. Eventually... */
324 if (segment_eq(get_fs(), KERNEL_DS
)) {
325 direction
|= ITER_KVEC
;
327 i
->kvec
= (struct kvec
*)iov
;
332 i
->nr_segs
= nr_segs
;
336 EXPORT_SYMBOL(iov_iter_init
);
338 static void memcpy_from_page(char *to
, struct page
*page
, size_t offset
, size_t len
)
340 char *from
= kmap_atomic(page
);
341 memcpy(to
, from
+ offset
, len
);
345 static void memcpy_to_page(struct page
*page
, size_t offset
, char *from
, size_t len
)
347 char *to
= kmap_atomic(page
);
348 memcpy(to
+ offset
, from
, len
);
352 static void memzero_page(struct page
*page
, size_t offset
, size_t len
)
354 char *addr
= kmap_atomic(page
);
355 memset(addr
+ offset
, 0, len
);
359 size_t copy_to_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
362 if (unlikely(bytes
> i
->count
))
365 if (unlikely(!bytes
))
368 iterate_and_advance(i
, bytes
, v
,
369 __copy_to_user(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
,
371 memcpy_to_page(v
.bv_page
, v
.bv_offset
,
372 (from
+= v
.bv_len
) - v
.bv_len
, v
.bv_len
),
373 memcpy(v
.iov_base
, (from
+= v
.iov_len
) - v
.iov_len
, v
.iov_len
)
378 EXPORT_SYMBOL(copy_to_iter
);
380 size_t copy_from_iter(void *addr
, size_t bytes
, struct iov_iter
*i
)
383 if (unlikely(bytes
> i
->count
))
386 if (unlikely(!bytes
))
389 iterate_and_advance(i
, bytes
, v
,
390 __copy_from_user((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
,
392 memcpy_from_page((to
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
393 v
.bv_offset
, v
.bv_len
),
394 memcpy((to
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
399 EXPORT_SYMBOL(copy_from_iter
);
401 size_t copy_page_to_iter(struct page
*page
, size_t offset
, size_t bytes
,
404 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
405 void *kaddr
= kmap_atomic(page
);
406 size_t wanted
= copy_to_iter(kaddr
+ offset
, bytes
, i
);
407 kunmap_atomic(kaddr
);
410 return copy_page_to_iter_iovec(page
, offset
, bytes
, i
);
412 EXPORT_SYMBOL(copy_page_to_iter
);
414 size_t copy_page_from_iter(struct page
*page
, size_t offset
, size_t bytes
,
417 if (i
->type
& (ITER_BVEC
|ITER_KVEC
)) {
418 void *kaddr
= kmap_atomic(page
);
419 size_t wanted
= copy_from_iter(kaddr
+ offset
, bytes
, i
);
420 kunmap_atomic(kaddr
);
423 return copy_page_from_iter_iovec(page
, offset
, bytes
, i
);
425 EXPORT_SYMBOL(copy_page_from_iter
);
427 size_t iov_iter_zero(size_t bytes
, struct iov_iter
*i
)
429 if (unlikely(bytes
> i
->count
))
432 if (unlikely(!bytes
))
435 iterate_and_advance(i
, bytes
, v
,
436 __clear_user(v
.iov_base
, v
.iov_len
),
437 memzero_page(v
.bv_page
, v
.bv_offset
, v
.bv_len
),
438 memset(v
.iov_base
, 0, v
.iov_len
)
443 EXPORT_SYMBOL(iov_iter_zero
);
445 size_t iov_iter_copy_from_user_atomic(struct page
*page
,
446 struct iov_iter
*i
, unsigned long offset
, size_t bytes
)
448 char *kaddr
= kmap_atomic(page
), *p
= kaddr
+ offset
;
449 iterate_all_kinds(i
, bytes
, v
,
450 __copy_from_user_inatomic((p
+= v
.iov_len
) - v
.iov_len
,
451 v
.iov_base
, v
.iov_len
),
452 memcpy_from_page((p
+= v
.bv_len
) - v
.bv_len
, v
.bv_page
,
453 v
.bv_offset
, v
.bv_len
),
454 memcpy((p
+= v
.iov_len
) - v
.iov_len
, v
.iov_base
, v
.iov_len
)
456 kunmap_atomic(kaddr
);
459 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic
);
461 void iov_iter_advance(struct iov_iter
*i
, size_t size
)
463 iterate_and_advance(i
, size
, v
, 0, 0, 0)
465 EXPORT_SYMBOL(iov_iter_advance
);
468 * Return the count of just the current iov_iter segment.
470 size_t iov_iter_single_seg_count(const struct iov_iter
*i
)
474 else if (i
->type
& ITER_BVEC
)
475 return min(i
->count
, i
->bvec
->bv_len
- i
->iov_offset
);
477 return min(i
->count
, i
->iov
->iov_len
- i
->iov_offset
);
479 EXPORT_SYMBOL(iov_iter_single_seg_count
);
481 unsigned long iov_iter_alignment(const struct iov_iter
*i
)
483 unsigned long res
= 0;
484 size_t size
= i
->count
;
489 iterate_all_kinds(i
, size
, v
,
490 (res
|= (unsigned long)v
.iov_base
| v
.iov_len
, 0),
491 res
|= v
.bv_offset
| v
.bv_len
,
492 res
|= (unsigned long)v
.iov_base
| v
.iov_len
496 EXPORT_SYMBOL(iov_iter_alignment
);
498 ssize_t
iov_iter_get_pages(struct iov_iter
*i
,
499 struct page
**pages
, size_t maxsize
, unsigned maxpages
,
502 if (maxsize
> i
->count
)
508 iterate_all_kinds(i
, maxsize
, v
, ({
509 unsigned long addr
= (unsigned long)v
.iov_base
;
510 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
514 if (len
> maxpages
* PAGE_SIZE
)
515 len
= maxpages
* PAGE_SIZE
;
516 addr
&= ~(PAGE_SIZE
- 1);
517 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
518 res
= get_user_pages_fast(addr
, n
, (i
->type
& WRITE
) != WRITE
, pages
);
519 if (unlikely(res
< 0))
521 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
523 /* can't be more than PAGE_SIZE */
524 *start
= v
.bv_offset
;
525 get_page(*pages
= v
.bv_page
);
533 EXPORT_SYMBOL(iov_iter_get_pages
);
535 static struct page
**get_pages_array(size_t n
)
537 struct page
**p
= kmalloc(n
* sizeof(struct page
*), GFP_KERNEL
);
539 p
= vmalloc(n
* sizeof(struct page
*));
543 ssize_t
iov_iter_get_pages_alloc(struct iov_iter
*i
,
544 struct page
***pages
, size_t maxsize
,
549 if (maxsize
> i
->count
)
555 iterate_all_kinds(i
, maxsize
, v
, ({
556 unsigned long addr
= (unsigned long)v
.iov_base
;
557 size_t len
= v
.iov_len
+ (*start
= addr
& (PAGE_SIZE
- 1));
561 addr
&= ~(PAGE_SIZE
- 1);
562 n
= DIV_ROUND_UP(len
, PAGE_SIZE
);
563 p
= get_pages_array(n
);
566 res
= get_user_pages_fast(addr
, n
, (i
->type
& WRITE
) != WRITE
, p
);
567 if (unlikely(res
< 0)) {
572 return (res
== n
? len
: res
* PAGE_SIZE
) - *start
;
574 /* can't be more than PAGE_SIZE */
575 *start
= v
.bv_offset
;
576 *pages
= p
= get_pages_array(1);
579 get_page(*p
= v
.bv_page
);
587 EXPORT_SYMBOL(iov_iter_get_pages_alloc
);
589 int iov_iter_npages(const struct iov_iter
*i
, int maxpages
)
591 size_t size
= i
->count
;
597 iterate_all_kinds(i
, size
, v
, ({
598 unsigned long p
= (unsigned long)v
.iov_base
;
599 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
601 if (npages
>= maxpages
)
605 if (npages
>= maxpages
)
608 unsigned long p
= (unsigned long)v
.iov_base
;
609 npages
+= DIV_ROUND_UP(p
+ v
.iov_len
, PAGE_SIZE
)
611 if (npages
>= maxpages
)
617 EXPORT_SYMBOL(iov_iter_npages
);