Commit | Line | Data |
---|---|---|
8025e5dd JK |
1 | #include <linux/kernel.h> |
2 | #include <linux/errno.h> | |
3 | #include <linux/err.h> | |
4 | #include <linux/mm.h> | |
5 | #include <linux/slab.h> | |
6 | #include <linux/vmalloc.h> | |
7 | #include <linux/pagemap.h> | |
8 | #include <linux/sched.h> | |
9 | ||
61f9ec1d | 10 | /** |
8025e5dd JK |
11 | * get_vaddr_frames() - map virtual addresses to pfns |
12 | * @start: starting user address | |
13 | * @nr_frames: number of pages / pfns from start to map | |
14 | * @write: whether pages will be written to by the caller | |
15 | * @force: whether to force write access even if user mapping is | |
16 | * readonly. See description of the same argument of | |
17 | get_user_pages(). | |
18 | * @vec: structure which receives pages / pfns of the addresses mapped. | |
19 | * It should have space for at least nr_frames entries. | |
20 | * | |
21 | * This function maps virtual addresses from @start and fills @vec structure | |
22 | * with page frame numbers or page pointers to corresponding pages (choice | |
23 | * depends on the type of the vma underlying the virtual address). If @start | |
24 | * belongs to a normal vma, the function grabs reference to each of the pages | |
25 | * to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't | |
26 | * touch page structures and the caller must make sure pfns aren't reused for | |
27 | * anything else while he is using them. | |
28 | * | |
29 | * The function returns number of pages mapped which may be less than | |
30 | * @nr_frames. In particular we stop mapping if there are more vmas of | |
31 | * different type underlying the specified range of virtual addresses. | |
32 | * When the function isn't able to map a single page, it returns error. | |
33 | * | |
34 | * This function takes care of grabbing mmap_sem as necessary. | |
35 | */ | |
36 | int get_vaddr_frames(unsigned long start, unsigned int nr_frames, | |
37 | bool write, bool force, struct frame_vector *vec) | |
38 | { | |
39 | struct mm_struct *mm = current->mm; | |
40 | struct vm_area_struct *vma; | |
41 | int ret = 0; | |
42 | int err; | |
43 | int locked; | |
44 | ||
45 | if (nr_frames == 0) | |
46 | return 0; | |
47 | ||
48 | if (WARN_ON_ONCE(nr_frames > vec->nr_allocated)) | |
49 | nr_frames = vec->nr_allocated; | |
50 | ||
51 | down_read(&mm->mmap_sem); | |
52 | locked = 1; | |
53 | vma = find_vma_intersection(mm, start, start + 1); | |
54 | if (!vma) { | |
55 | ret = -EFAULT; | |
56 | goto out; | |
57 | } | |
58 | if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) { | |
59 | vec->got_ref = true; | |
60 | vec->is_pfns = false; | |
61 | ret = get_user_pages_locked(current, mm, start, nr_frames, | |
62 | write, force, (struct page **)(vec->ptrs), &locked); | |
63 | goto out; | |
64 | } | |
65 | ||
66 | vec->got_ref = false; | |
67 | vec->is_pfns = true; | |
68 | do { | |
69 | unsigned long *nums = frame_vector_pfns(vec); | |
70 | ||
71 | while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) { | |
72 | err = follow_pfn(vma, start, &nums[ret]); | |
73 | if (err) { | |
74 | if (ret == 0) | |
75 | ret = err; | |
76 | goto out; | |
77 | } | |
78 | start += PAGE_SIZE; | |
79 | ret++; | |
80 | } | |
81 | /* | |
82 | * We stop if we have enough pages or if VMA doesn't completely | |
83 | * cover the tail page. | |
84 | */ | |
85 | if (ret >= nr_frames || start < vma->vm_end) | |
86 | break; | |
87 | vma = find_vma_intersection(mm, start, start + 1); | |
88 | } while (vma && vma->vm_flags & (VM_IO | VM_PFNMAP)); | |
89 | out: | |
90 | if (locked) | |
91 | up_read(&mm->mmap_sem); | |
92 | if (!ret) | |
93 | ret = -EFAULT; | |
94 | if (ret > 0) | |
95 | vec->nr_frames = ret; | |
96 | return ret; | |
97 | } | |
98 | EXPORT_SYMBOL(get_vaddr_frames); | |
99 | ||
100 | /** | |
101 | * put_vaddr_frames() - drop references to pages if get_vaddr_frames() acquired | |
102 | * them | |
103 | * @vec: frame vector to put | |
104 | * | |
105 | * Drop references to pages if get_vaddr_frames() acquired them. We also | |
106 | * invalidate the frame vector so that it is prepared for the next call into | |
107 | * get_vaddr_frames(). | |
108 | */ | |
109 | void put_vaddr_frames(struct frame_vector *vec) | |
110 | { | |
111 | int i; | |
112 | struct page **pages; | |
113 | ||
114 | if (!vec->got_ref) | |
115 | goto out; | |
116 | pages = frame_vector_pages(vec); | |
117 | /* | |
118 | * frame_vector_pages() might needed to do a conversion when | |
119 | * get_vaddr_frames() got pages but vec was later converted to pfns. | |
120 | * But it shouldn't really fail to convert pfns back... | |
121 | */ | |
122 | if (WARN_ON(IS_ERR(pages))) | |
123 | goto out; | |
124 | for (i = 0; i < vec->nr_frames; i++) | |
125 | put_page(pages[i]); | |
126 | vec->got_ref = false; | |
127 | out: | |
128 | vec->nr_frames = 0; | |
129 | } | |
130 | EXPORT_SYMBOL(put_vaddr_frames); | |
131 | ||
132 | /** | |
133 | * frame_vector_to_pages - convert frame vector to contain page pointers | |
134 | * @vec: frame vector to convert | |
135 | * | |
136 | * Convert @vec to contain array of page pointers. If the conversion is | |
137 | * successful, return 0. Otherwise return an error. Note that we do not grab | |
138 | * page references for the page structures. | |
139 | */ | |
140 | int frame_vector_to_pages(struct frame_vector *vec) | |
141 | { | |
142 | int i; | |
143 | unsigned long *nums; | |
144 | struct page **pages; | |
145 | ||
146 | if (!vec->is_pfns) | |
147 | return 0; | |
148 | nums = frame_vector_pfns(vec); | |
149 | for (i = 0; i < vec->nr_frames; i++) | |
150 | if (!pfn_valid(nums[i])) | |
151 | return -EINVAL; | |
152 | pages = (struct page **)nums; | |
153 | for (i = 0; i < vec->nr_frames; i++) | |
154 | pages[i] = pfn_to_page(nums[i]); | |
155 | vec->is_pfns = false; | |
156 | return 0; | |
157 | } | |
158 | EXPORT_SYMBOL(frame_vector_to_pages); | |
159 | ||
160 | /** | |
161 | * frame_vector_to_pfns - convert frame vector to contain pfns | |
162 | * @vec: frame vector to convert | |
163 | * | |
164 | * Convert @vec to contain array of pfns. | |
165 | */ | |
166 | void frame_vector_to_pfns(struct frame_vector *vec) | |
167 | { | |
168 | int i; | |
169 | unsigned long *nums; | |
170 | struct page **pages; | |
171 | ||
172 | if (vec->is_pfns) | |
173 | return; | |
174 | pages = (struct page **)(vec->ptrs); | |
175 | nums = (unsigned long *)pages; | |
176 | for (i = 0; i < vec->nr_frames; i++) | |
177 | nums[i] = page_to_pfn(pages[i]); | |
178 | vec->is_pfns = true; | |
179 | } | |
180 | EXPORT_SYMBOL(frame_vector_to_pfns); | |
181 | ||
182 | /** | |
183 | * frame_vector_create() - allocate & initialize structure for pinned pfns | |
184 | * @nr_frames: number of pfns slots we should reserve | |
185 | * | |
186 | * Allocate and initialize struct pinned_pfns to be able to hold @nr_pfns | |
187 | * pfns. | |
188 | */ | |
189 | struct frame_vector *frame_vector_create(unsigned int nr_frames) | |
190 | { | |
191 | struct frame_vector *vec; | |
192 | int size = sizeof(struct frame_vector) + sizeof(void *) * nr_frames; | |
193 | ||
194 | if (WARN_ON_ONCE(nr_frames == 0)) | |
195 | return NULL; | |
196 | /* | |
197 | * This is absurdly high. It's here just to avoid strange effects when | |
198 | * arithmetics overflows. | |
199 | */ | |
200 | if (WARN_ON_ONCE(nr_frames > INT_MAX / sizeof(void *) / 2)) | |
201 | return NULL; | |
202 | /* | |
203 | * Avoid higher order allocations, use vmalloc instead. It should | |
204 | * be rare anyway. | |
205 | */ | |
206 | if (size <= PAGE_SIZE) | |
207 | vec = kmalloc(size, GFP_KERNEL); | |
208 | else | |
209 | vec = vmalloc(size); | |
210 | if (!vec) | |
211 | return NULL; | |
212 | vec->nr_allocated = nr_frames; | |
213 | vec->nr_frames = 0; | |
214 | return vec; | |
215 | } | |
216 | EXPORT_SYMBOL(frame_vector_create); | |
217 | ||
218 | /** | |
219 | * frame_vector_destroy() - free memory allocated to carry frame vector | |
220 | * @vec: Frame vector to free | |
221 | * | |
222 | * Free structure allocated by frame_vector_create() to carry frames. | |
223 | */ | |
224 | void frame_vector_destroy(struct frame_vector *vec) | |
225 | { | |
226 | /* Make sure put_vaddr_frames() got called properly... */ | |
227 | VM_BUG_ON(vec->nr_frames > 0); | |
228 | kvfree(vec); | |
229 | } | |
230 | EXPORT_SYMBOL(frame_vector_destroy); |