projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'akpm' (patches from Andrew)
[deliverable/linux.git] / lib / iov_iter.c
1 #include <linux/export.h>
2 #include <linux/uio.h>
3 #include <linux/pagemap.h>
4 #include <linux/slab.h>
5 #include <linux/vmalloc.h>
6 #include <net/checksum.h>
7
8 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
9 size_t left; \
10 size_t wanted = n; \
11 __p = i->iov; \
12 __v.iov_len = min(n, __p->iov_len - skip); \
13 if (likely(__v.iov_len)) { \
14 __v.iov_base = __p->iov_base + skip; \
15 left = (STEP); \
16 __v.iov_len -= left; \
17 skip += __v.iov_len; \
18 n -= __v.iov_len; \
19 } else { \
20 left = 0; \
21 } \
22 while (unlikely(!left && n)) { \
23 __p++; \
24 __v.iov_len = min(n, __p->iov_len); \
25 if (unlikely(!__v.iov_len)) \
26 continue; \
27 __v.iov_base = __p->iov_base; \
28 left = (STEP); \
29 __v.iov_len -= left; \
30 skip = __v.iov_len; \
31 n -= __v.iov_len; \
32 } \
33 n = wanted - n; \
34 }
35
36 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
37 size_t wanted = n; \
38 __p = i->kvec; \
39 __v.iov_len = min(n, __p->iov_len - skip); \
40 if (likely(__v.iov_len)) { \
41 __v.iov_base = __p->iov_base + skip; \
42 (void)(STEP); \
43 skip += __v.iov_len; \
44 n -= __v.iov_len; \
45 } \
46 while (unlikely(n)) { \
47 __p++; \
48 __v.iov_len = min(n, __p->iov_len); \
49 if (unlikely(!__v.iov_len)) \
50 continue; \
51 __v.iov_base = __p->iov_base; \
52 (void)(STEP); \
53 skip = __v.iov_len; \
54 n -= __v.iov_len; \
55 } \
56 n = wanted; \
57 }
58
59 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
60 struct bvec_iter __start; \
61 __start.bi_size = n; \
62 __start.bi_bvec_done = skip; \
63 __start.bi_idx = 0; \
64 for_each_bvec(__v, i->bvec, __bi, __start) { \
65 if (!__v.bv_len) \
66 continue; \
67 (void)(STEP); \
68 } \
69 }
70
71 #define iterate_all_kinds(i, n, v, I, B, K) { \
72 size_t skip = i->iov_offset; \
73 if (unlikely(i->type & ITER_BVEC)) { \
74 struct bio_vec v; \
75 struct bvec_iter __bi; \
76 iterate_bvec(i, n, v, __bi, skip, (B)) \
77 } else if (unlikely(i->type & ITER_KVEC)) { \
78 const struct kvec *kvec; \
79 struct kvec v; \
80 iterate_kvec(i, n, v, kvec, skip, (K)) \
81 } else { \
82 const struct iovec *iov; \
83 struct iovec v; \
84 iterate_iovec(i, n, v, iov, skip, (I)) \
85 } \
86 }
87
88 #define iterate_and_advance(i, n, v, I, B, K) { \
89 if (unlikely(i->count < n)) \
90 n = i->count; \
91 if (i->count) { \
92 size_t skip = i->iov_offset; \
93 if (unlikely(i->type & ITER_BVEC)) { \
94 const struct bio_vec *bvec = i->bvec; \
95 struct bio_vec v; \
96 struct bvec_iter __bi; \
97 iterate_bvec(i, n, v, __bi, skip, (B)) \
98 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
99 i->nr_segs -= i->bvec - bvec; \
100 skip = __bi.bi_bvec_done; \
101 } else if (unlikely(i->type & ITER_KVEC)) { \
102 const struct kvec *kvec; \
103 struct kvec v; \
104 iterate_kvec(i, n, v, kvec, skip, (K)) \
105 if (skip == kvec->iov_len) { \
106 kvec++; \
107 skip = 0; \
108 } \
109 i->nr_segs -= kvec - i->kvec; \
110 i->kvec = kvec; \
111 } else { \
112 const struct iovec *iov; \
113 struct iovec v; \
114 iterate_iovec(i, n, v, iov, skip, (I)) \
115 if (skip == iov->iov_len) { \
116 iov++; \
117 skip = 0; \
118 } \
119 i->nr_segs -= iov - i->iov; \
120 i->iov = iov; \
121 } \
122 i->count -= n; \
123 i->iov_offset = skip; \
124 } \
125 }
126
127 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
128 struct iov_iter *i)
129 {
130 size_t skip, copy, left, wanted;
131 const struct iovec *iov;
132 char __user *buf;
133 void *kaddr, *from;
134
135 if (unlikely(bytes > i->count))
136 bytes = i->count;
137
138 if (unlikely(!bytes))
139 return 0;
140
141 wanted = bytes;
142 iov = i->iov;
143 skip = i->iov_offset;
144 buf = iov->iov_base + skip;
145 copy = min(bytes, iov->iov_len - skip);
146
147 if (!fault_in_pages_writeable(buf, copy)) {
148 kaddr = kmap_atomic(page);
149 from = kaddr + offset;
150
151 /* first chunk, usually the only one */
152 left = __copy_to_user_inatomic(buf, from, copy);
153 copy -= left;
154 skip += copy;
155 from += copy;
156 bytes -= copy;
157
158 while (unlikely(!left && bytes)) {
159 iov++;
160 buf = iov->iov_base;
161 copy = min(bytes, iov->iov_len);
162 left = __copy_to_user_inatomic(buf, from, copy);
163 copy -= left;
164 skip = copy;
165 from += copy;
166 bytes -= copy;
167 }
168 if (likely(!bytes)) {
169 kunmap_atomic(kaddr);
170 goto done;
171 }
172 offset = from - kaddr;
173 buf += copy;
174 kunmap_atomic(kaddr);
175 copy = min(bytes, iov->iov_len - skip);
176 }
177 /* Too bad - revert to non-atomic kmap */
178 kaddr = kmap(page);
179 from = kaddr + offset;
180 left = __copy_to_user(buf, from, copy);
181 copy -= left;
182 skip += copy;
183 from += copy;
184 bytes -= copy;
185 while (unlikely(!left && bytes)) {
186 iov++;
187 buf = iov->iov_base;
188 copy = min(bytes, iov->iov_len);
189 left = __copy_to_user(buf, from, copy);
190 copy -= left;
191 skip = copy;
192 from += copy;
193 bytes -= copy;
194 }
195 kunmap(page);
196 done:
197 if (skip == iov->iov_len) {
198 iov++;
199 skip = 0;
200 }
201 i->count -= wanted - bytes;
202 i->nr_segs -= iov - i->iov;
203 i->iov = iov;
204 i->iov_offset = skip;
205 return wanted - bytes;
206 }
207
208 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
209 struct iov_iter *i)
210 {
211 size_t skip, copy, left, wanted;
212 const struct iovec *iov;
213 char __user *buf;
214 void *kaddr, *to;
215
216 if (unlikely(bytes > i->count))
217 bytes = i->count;
218
219 if (unlikely(!bytes))
220 return 0;
221
222 wanted = bytes;
223 iov = i->iov;
224 skip = i->iov_offset;
225 buf = iov->iov_base + skip;
226 copy = min(bytes, iov->iov_len - skip);
227
228 if (!fault_in_pages_readable(buf, copy)) {
229 kaddr = kmap_atomic(page);
230 to = kaddr + offset;
231
232 /* first chunk, usually the only one */
233 left = __copy_from_user_inatomic(to, buf, copy);
234 copy -= left;
235 skip += copy;
236 to += copy;
237 bytes -= copy;
238
239 while (unlikely(!left && bytes)) {
240 iov++;
241 buf = iov->iov_base;
242 copy = min(bytes, iov->iov_len);
243 left = __copy_from_user_inatomic(to, buf, copy);
244 copy -= left;
245 skip = copy;
246 to += copy;
247 bytes -= copy;
248 }
249 if (likely(!bytes)) {
250 kunmap_atomic(kaddr);
251 goto done;
252 }
253 offset = to - kaddr;
254 buf += copy;
255 kunmap_atomic(kaddr);
256 copy = min(bytes, iov->iov_len - skip);
257 }
258 /* Too bad - revert to non-atomic kmap */
259 kaddr = kmap(page);
260 to = kaddr + offset;
261 left = __copy_from_user(to, buf, copy);
262 copy -= left;
263 skip += copy;
264 to += copy;
265 bytes -= copy;
266 while (unlikely(!left && bytes)) {
267 iov++;
268 buf = iov->iov_base;
269 copy = min(bytes, iov->iov_len);
270 left = __copy_from_user(to, buf, copy);
271 copy -= left;
272 skip = copy;
273 to += copy;
274 bytes -= copy;
275 }
276 kunmap(page);
277 done:
278 if (skip == iov->iov_len) {
279 iov++;
280 skip = 0;
281 }
282 i->count -= wanted - bytes;
283 i->nr_segs -= iov - i->iov;
284 i->iov = iov;
285 i->iov_offset = skip;
286 return wanted - bytes;
287 }
288
289 /*
290 * Fault in the first iovec of the given iov_iter, to a maximum length
291 * of bytes. Returns 0 on success, or non-zero if the memory could not be
292 * accessed (ie. because it is an invalid address).
293 *
294 * writev-intensive code may want this to prefault several iovecs -- that
295 * would be possible (callers must not rely on the fact that _only_ the
296 * first iovec will be faulted with the current implementation).
297 */
298 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
299 {
300 if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
301 char __user *buf = i->iov->iov_base + i->iov_offset;
302 bytes = min(bytes, i->iov->iov_len - i->iov_offset);
303 return fault_in_pages_readable(buf, bytes);
304 }
305 return 0;
306 }
307 EXPORT_SYMBOL(iov_iter_fault_in_readable);
308
309 /*
310 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
311 * bytes. For each iovec, fault in each page that constitutes the iovec.
312 *
313 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
314 * because it is an invalid address).
315 */
316 int iov_iter_fault_in_multipages_readable(struct iov_iter *i, size_t bytes)
317 {
318 size_t skip = i->iov_offset;
319 const struct iovec *iov;
320 int err;
321 struct iovec v;
322
323 if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
324 iterate_iovec(i, bytes, v, iov, skip, ({
325 err = fault_in_multipages_readable(v.iov_base,
326 v.iov_len);
327 if (unlikely(err))
328 return err;
329 0;}))
330 }
331 return 0;
332 }
333 EXPORT_SYMBOL(iov_iter_fault_in_multipages_readable);
334
335 void iov_iter_init(struct iov_iter *i, int direction,
336 const struct iovec *iov, unsigned long nr_segs,
337 size_t count)
338 {
339 /* It will get better. Eventually... */
340 if (segment_eq(get_fs(), KERNEL_DS)) {
341 direction |= ITER_KVEC;
342 i->type = direction;
343 i->kvec = (struct kvec *)iov;
344 } else {
345 i->type = direction;
346 i->iov = iov;
347 }
348 i->nr_segs = nr_segs;
349 i->iov_offset = 0;
350 i->count = count;
351 }
352 EXPORT_SYMBOL(iov_iter_init);
353
354 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
355 {
356 char *from = kmap_atomic(page);
357 memcpy(to, from + offset, len);
358 kunmap_atomic(from);
359 }
360
361 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
362 {
363 char *to = kmap_atomic(page);
364 memcpy(to + offset, from, len);
365 kunmap_atomic(to);
366 }
367
368 static void memzero_page(struct page *page, size_t offset, size_t len)
369 {
370 char *addr = kmap_atomic(page);
371 memset(addr + offset, 0, len);
372 kunmap_atomic(addr);
373 }
374
375 size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
376 {
377 const char *from = addr;
378 iterate_and_advance(i, bytes, v,
379 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
380 v.iov_len),
381 memcpy_to_page(v.bv_page, v.bv_offset,
382 (from += v.bv_len) - v.bv_len, v.bv_len),
383 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
384 )
385
386 return bytes;
387 }
388 EXPORT_SYMBOL(copy_to_iter);
389
390 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
391 {
392 char *to = addr;
393 iterate_and_advance(i, bytes, v,
394 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
395 v.iov_len),
396 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
397 v.bv_offset, v.bv_len),
398 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
399 )
400
401 return bytes;
402 }
403 EXPORT_SYMBOL(copy_from_iter);
404
405 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
406 {
407 char *to = addr;
408 iterate_and_advance(i, bytes, v,
409 __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
410 v.iov_base, v.iov_len),
411 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
412 v.bv_offset, v.bv_len),
413 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
414 )
415
416 return bytes;
417 }
418 EXPORT_SYMBOL(copy_from_iter_nocache);
419
420 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
421 struct iov_iter *i)
422 {
423 if (i->type & (ITER_BVEC|ITER_KVEC)) {
424 void *kaddr = kmap_atomic(page);
425 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
426 kunmap_atomic(kaddr);
427 return wanted;
428 } else
429 return copy_page_to_iter_iovec(page, offset, bytes, i);
430 }
431 EXPORT_SYMBOL(copy_page_to_iter);
432
433 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
434 struct iov_iter *i)
435 {
436 if (i->type & (ITER_BVEC|ITER_KVEC)) {
437 void *kaddr = kmap_atomic(page);
438 size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
439 kunmap_atomic(kaddr);
440 return wanted;
441 } else
442 return copy_page_from_iter_iovec(page, offset, bytes, i);
443 }
444 EXPORT_SYMBOL(copy_page_from_iter);
445
446 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
447 {
448 iterate_and_advance(i, bytes, v,
449 __clear_user(v.iov_base, v.iov_len),
450 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
451 memset(v.iov_base, 0, v.iov_len)
452 )
453
454 return bytes;
455 }
456 EXPORT_SYMBOL(iov_iter_zero);
457
458 size_t iov_iter_copy_from_user_atomic(struct page *page,
459 struct iov_iter *i, unsigned long offset, size_t bytes)
460 {
461 char *kaddr = kmap_atomic(page), *p = kaddr + offset;
462 iterate_all_kinds(i, bytes, v,
463 __copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
464 v.iov_base, v.iov_len),
465 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
466 v.bv_offset, v.bv_len),
467 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
468 )
469 kunmap_atomic(kaddr);
470 return bytes;
471 }
472 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
473
474 void iov_iter_advance(struct iov_iter *i, size_t size)
475 {
476 iterate_and_advance(i, size, v, 0, 0, 0)
477 }
478 EXPORT_SYMBOL(iov_iter_advance);
479
480 /*
481 * Return the count of just the current iov_iter segment.
482 */
483 size_t iov_iter_single_seg_count(const struct iov_iter *i)
484 {
485 if (i->nr_segs == 1)
486 return i->count;
487 else if (i->type & ITER_BVEC)
488 return min(i->count, i->bvec->bv_len - i->iov_offset);
489 else
490 return min(i->count, i->iov->iov_len - i->iov_offset);
491 }
492 EXPORT_SYMBOL(iov_iter_single_seg_count);
493
494 void iov_iter_kvec(struct iov_iter *i, int direction,
495 const struct kvec *kvec, unsigned long nr_segs,
496 size_t count)
497 {
498 BUG_ON(!(direction & ITER_KVEC));
499 i->type = direction;
500 i->kvec = kvec;
501 i->nr_segs = nr_segs;
502 i->iov_offset = 0;
503 i->count = count;
504 }
505 EXPORT_SYMBOL(iov_iter_kvec);
506
507 void iov_iter_bvec(struct iov_iter *i, int direction,
508 const struct bio_vec *bvec, unsigned long nr_segs,
509 size_t count)
510 {
511 BUG_ON(!(direction & ITER_BVEC));
512 i->type = direction;
513 i->bvec = bvec;
514 i->nr_segs = nr_segs;
515 i->iov_offset = 0;
516 i->count = count;
517 }
518 EXPORT_SYMBOL(iov_iter_bvec);
519
520 unsigned long iov_iter_alignment(const struct iov_iter *i)
521 {
522 unsigned long res = 0;
523 size_t size = i->count;
524
525 if (!size)
526 return 0;
527
528 iterate_all_kinds(i, size, v,
529 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
530 res |= v.bv_offset | v.bv_len,
531 res |= (unsigned long)v.iov_base | v.iov_len
532 )
533 return res;
534 }
535 EXPORT_SYMBOL(iov_iter_alignment);
536
537 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
538 {
539 unsigned long res = 0;
540 size_t size = i->count;
541 if (!size)
542 return 0;
543
544 iterate_all_kinds(i, size, v,
545 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
546 (size != v.iov_len ? size : 0), 0),
547 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
548 (size != v.bv_len ? size : 0)),
549 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
550 (size != v.iov_len ? size : 0))
551 );
552 return res;
553 }
554 EXPORT_SYMBOL(iov_iter_gap_alignment);
555
556 ssize_t iov_iter_get_pages(struct iov_iter *i,
557 struct page **pages, size_t maxsize, unsigned maxpages,
558 size_t *start)
559 {
560 if (maxsize > i->count)
561 maxsize = i->count;
562
563 if (!maxsize)
564 return 0;
565
566 iterate_all_kinds(i, maxsize, v, ({
567 unsigned long addr = (unsigned long)v.iov_base;
568 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
569 int n;
570 int res;
571
572 if (len > maxpages * PAGE_SIZE)
573 len = maxpages * PAGE_SIZE;
574 addr &= ~(PAGE_SIZE - 1);
575 n = DIV_ROUND_UP(len, PAGE_SIZE);
576 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
577 if (unlikely(res < 0))
578 return res;
579 return (res == n ? len : res * PAGE_SIZE) - *start;
580 0;}),({
581 /* can't be more than PAGE_SIZE */
582 *start = v.bv_offset;
583 get_page(*pages = v.bv_page);
584 return v.bv_len;
585 }),({
586 return -EFAULT;
587 })
588 )
589 return 0;
590 }
591 EXPORT_SYMBOL(iov_iter_get_pages);
592
593 static struct page **get_pages_array(size_t n)
594 {
595 struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
596 if (!p)
597 p = vmalloc(n * sizeof(struct page *));
598 return p;
599 }
600
601 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
602 struct page ***pages, size_t maxsize,
603 size_t *start)
604 {
605 struct page **p;
606
607 if (maxsize > i->count)
608 maxsize = i->count;
609
610 if (!maxsize)
611 return 0;
612
613 iterate_all_kinds(i, maxsize, v, ({
614 unsigned long addr = (unsigned long)v.iov_base;
615 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
616 int n;
617 int res;
618
619 addr &= ~(PAGE_SIZE - 1);
620 n = DIV_ROUND_UP(len, PAGE_SIZE);
621 p = get_pages_array(n);
622 if (!p)
623 return -ENOMEM;
624 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
625 if (unlikely(res < 0)) {
626 kvfree(p);
627 return res;
628 }
629 *pages = p;
630 return (res == n ? len : res * PAGE_SIZE) - *start;
631 0;}),({
632 /* can't be more than PAGE_SIZE */
633 *start = v.bv_offset;
634 *pages = p = get_pages_array(1);
635 if (!p)
636 return -ENOMEM;
637 get_page(*p = v.bv_page);
638 return v.bv_len;
639 }),({
640 return -EFAULT;
641 })
642 )
643 return 0;
644 }
645 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
646
647 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
648 struct iov_iter *i)
649 {
650 char *to = addr;
651 __wsum sum, next;
652 size_t off = 0;
653 sum = *csum;
654 iterate_and_advance(i, bytes, v, ({
655 int err = 0;
656 next = csum_and_copy_from_user(v.iov_base,
657 (to += v.iov_len) - v.iov_len,
658 v.iov_len, 0, &err);
659 if (!err) {
660 sum = csum_block_add(sum, next, off);
661 off += v.iov_len;
662 }
663 err ? v.iov_len : 0;
664 }), ({
665 char *p = kmap_atomic(v.bv_page);
666 next = csum_partial_copy_nocheck(p + v.bv_offset,
667 (to += v.bv_len) - v.bv_len,
668 v.bv_len, 0);
669 kunmap_atomic(p);
670 sum = csum_block_add(sum, next, off);
671 off += v.bv_len;
672 }),({
673 next = csum_partial_copy_nocheck(v.iov_base,
674 (to += v.iov_len) - v.iov_len,
675 v.iov_len, 0);
676 sum = csum_block_add(sum, next, off);
677 off += v.iov_len;
678 })
679 )
680 *csum = sum;
681 return bytes;
682 }
683 EXPORT_SYMBOL(csum_and_copy_from_iter);
684
685 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
686 struct iov_iter *i)
687 {
688 const char *from = addr;
689 __wsum sum, next;
690 size_t off = 0;
691 sum = *csum;
692 iterate_and_advance(i, bytes, v, ({
693 int err = 0;
694 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
695 v.iov_base,
696 v.iov_len, 0, &err);
697 if (!err) {
698 sum = csum_block_add(sum, next, off);
699 off += v.iov_len;
700 }
701 err ? v.iov_len : 0;
702 }), ({
703 char *p = kmap_atomic(v.bv_page);
704 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
705 p + v.bv_offset,
706 v.bv_len, 0);
707 kunmap_atomic(p);
708 sum = csum_block_add(sum, next, off);
709 off += v.bv_len;
710 }),({
711 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
712 v.iov_base,
713 v.iov_len, 0);
714 sum = csum_block_add(sum, next, off);
715 off += v.iov_len;
716 })
717 )
718 *csum = sum;
719 return bytes;
720 }
721 EXPORT_SYMBOL(csum_and_copy_to_iter);
722
723 int iov_iter_npages(const struct iov_iter *i, int maxpages)
724 {
725 size_t size = i->count;
726 int npages = 0;
727
728 if (!size)
729 return 0;
730
731 iterate_all_kinds(i, size, v, ({
732 unsigned long p = (unsigned long)v.iov_base;
733 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
734 - p / PAGE_SIZE;
735 if (npages >= maxpages)
736 return maxpages;
737 0;}),({
738 npages++;
739 if (npages >= maxpages)
740 return maxpages;
741 }),({
742 unsigned long p = (unsigned long)v.iov_base;
743 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
744 - p / PAGE_SIZE;
745 if (npages >= maxpages)
746 return maxpages;
747 })
748 )
749 return npages;
750 }
751 EXPORT_SYMBOL(iov_iter_npages);
752
753 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
754 {
755 *new = *old;
756 if (new->type & ITER_BVEC)
757 return new->bvec = kmemdup(new->bvec,
758 new->nr_segs * sizeof(struct bio_vec),
759 flags);
760 else
761 /* iovec and kvec have identical layout */
762 return new->iov = kmemdup(new->iov,
763 new->nr_segs * sizeof(struct iovec),
764 flags);
765 }
766 EXPORT_SYMBOL(dup_iter);
767
768 int import_iovec(int type, const struct iovec __user * uvector,
769 unsigned nr_segs, unsigned fast_segs,
770 struct iovec **iov, struct iov_iter *i)
771 {
772 ssize_t n;
773 struct iovec *p;
774 n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
775 *iov, &p);
776 if (n < 0) {
777 if (p != *iov)
778 kfree(p);
779 *iov = NULL;
780 return n;
781 }
782 iov_iter_init(i, type, p, nr_segs, n);
783 *iov = p == *iov ? NULL : p;
784 return 0;
785 }
786 EXPORT_SYMBOL(import_iovec);
787
788 #ifdef CONFIG_COMPAT
789 #include <linux/compat.h>
790
791 int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
792 unsigned nr_segs, unsigned fast_segs,
793 struct iovec **iov, struct iov_iter *i)
794 {
795 ssize_t n;
796 struct iovec *p;
797 n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
798 *iov, &p);
799 if (n < 0) {
800 if (p != *iov)
801 kfree(p);
802 *iov = NULL;
803 return n;
804 }
805 iov_iter_init(i, type, p, nr_segs, n);
806 *iov = p == *iov ? NULL : p;
807 return 0;
808 }
809 #endif
810
811 int import_single_range(int rw, void __user *buf, size_t len,
812 struct iovec *iov, struct iov_iter *i)
813 {
814 if (len > MAX_RW_COUNT)
815 len = MAX_RW_COUNT;
816 if (unlikely(!access_ok(!rw, buf, len)))
817 return -EFAULT;
818
819 iov->iov_base = buf;
820 iov->iov_len = len;
821 iov_iter_init(i, rw, iov, 1, len);
822 return 0;
823 }
824 EXPORT_SYMBOL(import_single_range);
Linux kernel - Deliverables
This page took 0.066696 seconds and 6 git commands to generate.