Commit | Line | Data |
---|---|---|
de5506e1 CL |
1 | /* |
2 | * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. | |
3 | * Portions based on net/core/datagram.c and copyrighted by their authors. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms of the GNU General Public License as published by the Free | |
7 | * Software Foundation; either version 2 of the License, or (at your option) | |
8 | * any later version. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
13 | * more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License along with | |
16 | * this program; if not, write to the Free Software Foundation, Inc., 59 | |
17 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
18 | * | |
19 | * The full GNU General Public License is included in this distribution in the | |
20 | * file called COPYING. | |
21 | */ | |
22 | ||
23 | /* | |
24 | * This code allows the net stack to make use of a DMA engine for | |
25 | * skb to iovec copies. | |
26 | */ | |
27 | ||
28 | #include <linux/dmaengine.h> | |
29 | #include <linux/pagemap.h> | |
30 | #include <net/tcp.h> /* for memcpy_toiovec */ | |
31 | #include <asm/io.h> | |
32 | #include <asm/uaccess.h> | |
33 | ||
56e0873b | 34 | static int num_pages_spanned(struct iovec *iov) |
de5506e1 CL |
35 | { |
36 | return | |
37 | ((PAGE_ALIGN((unsigned long)iov->iov_base + iov->iov_len) - | |
38 | ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT); | |
39 | } | |
40 | ||
41 | /* | |
42 | * Pin down all the iovec pages needed for len bytes. | |
43 | * Return a struct dma_pinned_list to keep track of pages pinned down. | |
44 | * | |
45 | * We are allocating a single chunk of memory, and then carving it up into | |
46 | * 3 sections, the latter 2 whose size depends on the number of iovecs and the | |
47 | * total number of pages, respectively. | |
48 | */ | |
49 | struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len) | |
50 | { | |
51 | struct dma_pinned_list *local_list; | |
52 | struct page **pages; | |
53 | int i; | |
54 | int ret; | |
55 | int nr_iovecs = 0; | |
56 | int iovec_len_used = 0; | |
57 | int iovec_pages_used = 0; | |
58 | long err; | |
59 | ||
60 | /* don't pin down non-user-based iovecs */ | |
61 | if (segment_eq(get_fs(), KERNEL_DS)) | |
62 | return NULL; | |
63 | ||
64 | /* determine how many iovecs/pages there are, up front */ | |
65 | do { | |
66 | iovec_len_used += iov[nr_iovecs].iov_len; | |
67 | iovec_pages_used += num_pages_spanned(&iov[nr_iovecs]); | |
68 | nr_iovecs++; | |
69 | } while (iovec_len_used < len); | |
70 | ||
71 | /* single kmalloc for pinned list, page_list[], and the page arrays */ | |
72 | local_list = kmalloc(sizeof(*local_list) | |
73 | + (nr_iovecs * sizeof (struct dma_page_list)) | |
74 | + (iovec_pages_used * sizeof (struct page*)), GFP_KERNEL); | |
75 | if (!local_list) { | |
76 | err = -ENOMEM; | |
77 | goto out; | |
78 | } | |
79 | ||
80 | /* list of pages starts right after the page list array */ | |
81 | pages = (struct page **) &local_list->page_list[nr_iovecs]; | |
82 | ||
83 | for (i = 0; i < nr_iovecs; i++) { | |
84 | struct dma_page_list *page_list = &local_list->page_list[i]; | |
85 | ||
86 | len -= iov[i].iov_len; | |
87 | ||
88 | if (!access_ok(VERIFY_WRITE, iov[i].iov_base, iov[i].iov_len)) { | |
89 | err = -EFAULT; | |
90 | goto unpin; | |
91 | } | |
92 | ||
93 | page_list->nr_pages = num_pages_spanned(&iov[i]); | |
94 | page_list->base_address = iov[i].iov_base; | |
95 | ||
96 | page_list->pages = pages; | |
97 | pages += page_list->nr_pages; | |
98 | ||
99 | /* pin pages down */ | |
100 | down_read(¤t->mm->mmap_sem); | |
101 | ret = get_user_pages( | |
102 | current, | |
103 | current->mm, | |
104 | (unsigned long) iov[i].iov_base, | |
105 | page_list->nr_pages, | |
106 | 1, /* write */ | |
107 | 0, /* force */ | |
108 | page_list->pages, | |
109 | NULL); | |
110 | up_read(¤t->mm->mmap_sem); | |
111 | ||
112 | if (ret != page_list->nr_pages) { | |
113 | err = -ENOMEM; | |
114 | goto unpin; | |
115 | } | |
116 | ||
117 | local_list->nr_iovecs = i + 1; | |
118 | } | |
119 | ||
120 | return local_list; | |
121 | ||
122 | unpin: | |
123 | dma_unpin_iovec_pages(local_list); | |
124 | out: | |
125 | return ERR_PTR(err); | |
126 | } | |
127 | ||
128 | void dma_unpin_iovec_pages(struct dma_pinned_list *pinned_list) | |
129 | { | |
130 | int i, j; | |
131 | ||
132 | if (!pinned_list) | |
133 | return; | |
134 | ||
135 | for (i = 0; i < pinned_list->nr_iovecs; i++) { | |
136 | struct dma_page_list *page_list = &pinned_list->page_list[i]; | |
137 | for (j = 0; j < page_list->nr_pages; j++) { | |
138 | set_page_dirty_lock(page_list->pages[j]); | |
139 | page_cache_release(page_list->pages[j]); | |
140 | } | |
141 | } | |
142 | ||
143 | kfree(pinned_list); | |
144 | } | |
145 | ||
146 | static dma_cookie_t dma_memcpy_to_kernel_iovec(struct dma_chan *chan, struct | |
147 | iovec *iov, unsigned char *kdata, size_t len) | |
148 | { | |
149 | dma_cookie_t dma_cookie = 0; | |
150 | ||
151 | while (len > 0) { | |
152 | if (iov->iov_len) { | |
153 | int copy = min_t(unsigned int, iov->iov_len, len); | |
154 | dma_cookie = dma_async_memcpy_buf_to_buf( | |
155 | chan, | |
156 | iov->iov_base, | |
157 | kdata, | |
158 | copy); | |
159 | kdata += copy; | |
160 | len -= copy; | |
161 | iov->iov_len -= copy; | |
162 | iov->iov_base += copy; | |
163 | } | |
164 | iov++; | |
165 | } | |
166 | ||
167 | return dma_cookie; | |
168 | } | |
169 | ||
170 | /* | |
171 | * We have already pinned down the pages we will be using in the iovecs. | |
172 | * Each entry in iov array has corresponding entry in pinned_list->page_list. | |
173 | * Using array indexing to keep iov[] and page_list[] in sync. | |
174 | * Initial elements in iov array's iov->iov_len will be 0 if already copied into | |
175 | * by another call. | |
176 | * iov array length remaining guaranteed to be bigger than len. | |
177 | */ | |
178 | dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov, | |
179 | struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len) | |
180 | { | |
181 | int iov_byte_offset; | |
182 | int copy; | |
183 | dma_cookie_t dma_cookie = 0; | |
184 | int iovec_idx; | |
185 | int page_idx; | |
186 | ||
187 | if (!chan) | |
188 | return memcpy_toiovec(iov, kdata, len); | |
189 | ||
190 | /* -> kernel copies (e.g. smbfs) */ | |
191 | if (!pinned_list) | |
192 | return dma_memcpy_to_kernel_iovec(chan, iov, kdata, len); | |
193 | ||
194 | iovec_idx = 0; | |
195 | while (iovec_idx < pinned_list->nr_iovecs) { | |
196 | struct dma_page_list *page_list; | |
197 | ||
198 | /* skip already used-up iovecs */ | |
199 | while (!iov[iovec_idx].iov_len) | |
200 | iovec_idx++; | |
201 | ||
202 | page_list = &pinned_list->page_list[iovec_idx]; | |
203 | ||
204 | iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); | |
205 | page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) | |
206 | - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; | |
207 | ||
208 | /* break up copies to not cross page boundary */ | |
209 | while (iov[iovec_idx].iov_len) { | |
210 | copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); | |
211 | copy = min_t(int, copy, iov[iovec_idx].iov_len); | |
212 | ||
213 | dma_cookie = dma_async_memcpy_buf_to_pg(chan, | |
214 | page_list->pages[page_idx], | |
215 | iov_byte_offset, | |
216 | kdata, | |
217 | copy); | |
218 | ||
219 | len -= copy; | |
220 | iov[iovec_idx].iov_len -= copy; | |
221 | iov[iovec_idx].iov_base += copy; | |
222 | ||
223 | if (!len) | |
224 | return dma_cookie; | |
225 | ||
226 | kdata += copy; | |
227 | iov_byte_offset = 0; | |
228 | page_idx++; | |
229 | } | |
230 | iovec_idx++; | |
231 | } | |
232 | ||
233 | /* really bad if we ever run out of iovecs */ | |
234 | BUG(); | |
235 | return -EFAULT; | |
236 | } | |
237 | ||
238 | dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov, | |
239 | struct dma_pinned_list *pinned_list, struct page *page, | |
240 | unsigned int offset, size_t len) | |
241 | { | |
242 | int iov_byte_offset; | |
243 | int copy; | |
244 | dma_cookie_t dma_cookie = 0; | |
245 | int iovec_idx; | |
246 | int page_idx; | |
247 | int err; | |
248 | ||
249 | /* this needs as-yet-unimplemented buf-to-buff, so punt. */ | |
250 | /* TODO: use dma for this */ | |
251 | if (!chan || !pinned_list) { | |
252 | u8 *vaddr = kmap(page); | |
253 | err = memcpy_toiovec(iov, vaddr + offset, len); | |
254 | kunmap(page); | |
255 | return err; | |
256 | } | |
257 | ||
258 | iovec_idx = 0; | |
259 | while (iovec_idx < pinned_list->nr_iovecs) { | |
260 | struct dma_page_list *page_list; | |
261 | ||
262 | /* skip already used-up iovecs */ | |
263 | while (!iov[iovec_idx].iov_len) | |
264 | iovec_idx++; | |
265 | ||
266 | page_list = &pinned_list->page_list[iovec_idx]; | |
267 | ||
268 | iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); | |
269 | page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) | |
270 | - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; | |
271 | ||
272 | /* break up copies to not cross page boundary */ | |
273 | while (iov[iovec_idx].iov_len) { | |
274 | copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); | |
275 | copy = min_t(int, copy, iov[iovec_idx].iov_len); | |
276 | ||
277 | dma_cookie = dma_async_memcpy_pg_to_pg(chan, | |
278 | page_list->pages[page_idx], | |
279 | iov_byte_offset, | |
280 | page, | |
281 | offset, | |
282 | copy); | |
283 | ||
284 | len -= copy; | |
285 | iov[iovec_idx].iov_len -= copy; | |
286 | iov[iovec_idx].iov_base += copy; | |
287 | ||
288 | if (!len) | |
289 | return dma_cookie; | |
290 | ||
291 | offset += copy; | |
292 | iov_byte_offset = 0; | |
293 | page_idx++; | |
294 | } | |
295 | iovec_idx++; | |
296 | } | |
297 | ||
298 | /* really bad if we ever run out of iovecs */ | |
299 | BUG(); | |
300 | return -EFAULT; | |
301 | } |