Commit | Line | Data |
---|---|---|
c06b540a | 1 | /* |
0bf48289 | 2 | * Copyright (c) 2014 Open Grid Computing, Inc. All rights reserved. |
c06b540a TT |
3 | * Copyright (c) 2005-2006 Network Appliance, Inc. All rights reserved. |
4 | * | |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the BSD-type | |
9 | * license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or without | |
12 | * modification, are permitted provided that the following conditions | |
13 | * are met: | |
14 | * | |
15 | * Redistributions of source code must retain the above copyright | |
16 | * notice, this list of conditions and the following disclaimer. | |
17 | * | |
18 | * Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials provided | |
21 | * with the distribution. | |
22 | * | |
23 | * Neither the name of the Network Appliance, Inc. nor the names of | |
24 | * its contributors may be used to endorse or promote products | |
25 | * derived from this software without specific prior written | |
26 | * permission. | |
27 | * | |
28 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
29 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
30 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
31 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
32 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
33 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
34 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
35 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
36 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
37 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
38 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
39 | * | |
40 | * Author: Tom Tucker <tom@opengridcomputing.com> | |
41 | */ | |
42 | ||
43 | #include <linux/sunrpc/debug.h> | |
44 | #include <linux/sunrpc/rpc_rdma.h> | |
45 | #include <linux/spinlock.h> | |
46 | #include <asm/unaligned.h> | |
47 | #include <rdma/ib_verbs.h> | |
48 | #include <rdma/rdma_cm.h> | |
49 | #include <linux/sunrpc/svc_rdma.h> | |
50 | ||
51 | #define RPCDBG_FACILITY RPCDBG_SVCXPRT | |
52 | ||
afd566ea TT |
53 | static int map_xdr(struct svcxprt_rdma *xprt, |
54 | struct xdr_buf *xdr, | |
55 | struct svc_rdma_req_map *vec) | |
c06b540a | 56 | { |
c06b540a | 57 | int sge_no; |
c06b540a TT |
58 | u32 sge_bytes; |
59 | u32 page_bytes; | |
34d16e42 | 60 | u32 page_off; |
c06b540a TT |
61 | int page_no; |
62 | ||
3fe04ee9 CL |
63 | if (xdr->len != |
64 | (xdr->head[0].iov_len + xdr->page_len + xdr->tail[0].iov_len)) { | |
65 | pr_err("svcrdma: map_xdr: XDR buffer length error\n"); | |
66 | return -EIO; | |
67 | } | |
34d16e42 | 68 | |
c06b540a TT |
69 | /* Skip the first sge, this is for the RPCRDMA header */ |
70 | sge_no = 1; | |
71 | ||
72 | /* Head SGE */ | |
34d16e42 TT |
73 | vec->sge[sge_no].iov_base = xdr->head[0].iov_base; |
74 | vec->sge[sge_no].iov_len = xdr->head[0].iov_len; | |
c06b540a TT |
75 | sge_no++; |
76 | ||
77 | /* pages SGE */ | |
78 | page_no = 0; | |
79 | page_bytes = xdr->page_len; | |
80 | page_off = xdr->page_base; | |
34d16e42 TT |
81 | while (page_bytes) { |
82 | vec->sge[sge_no].iov_base = | |
83 | page_address(xdr->pages[page_no]) + page_off; | |
84 | sge_bytes = min_t(u32, page_bytes, (PAGE_SIZE - page_off)); | |
c06b540a | 85 | page_bytes -= sge_bytes; |
34d16e42 | 86 | vec->sge[sge_no].iov_len = sge_bytes; |
c06b540a TT |
87 | |
88 | sge_no++; | |
89 | page_no++; | |
90 | page_off = 0; /* reset for next time through loop */ | |
91 | } | |
92 | ||
93 | /* Tail SGE */ | |
34d16e42 TT |
94 | if (xdr->tail[0].iov_len) { |
95 | vec->sge[sge_no].iov_base = xdr->tail[0].iov_base; | |
96 | vec->sge[sge_no].iov_len = xdr->tail[0].iov_len; | |
c06b540a TT |
97 | sge_no++; |
98 | } | |
99 | ||
b1e1e158 | 100 | dprintk("svcrdma: map_xdr: sge_no %d page_no %d " |
2e3c230b | 101 | "page_base %u page_len %u head_len %zu tail_len %zu\n", |
b1e1e158 TT |
102 | sge_no, page_no, xdr->page_base, xdr->page_len, |
103 | xdr->head[0].iov_len, xdr->tail[0].iov_len); | |
104 | ||
34d16e42 | 105 | vec->count = sge_no; |
afd566ea | 106 | return 0; |
c06b540a TT |
107 | } |
108 | ||
b432e6b3 TT |
109 | static dma_addr_t dma_map_xdr(struct svcxprt_rdma *xprt, |
110 | struct xdr_buf *xdr, | |
111 | u32 xdr_off, size_t len, int dir) | |
112 | { | |
113 | struct page *page; | |
114 | dma_addr_t dma_addr; | |
115 | if (xdr_off < xdr->head[0].iov_len) { | |
116 | /* This offset is in the head */ | |
117 | xdr_off += (unsigned long)xdr->head[0].iov_base & ~PAGE_MASK; | |
118 | page = virt_to_page(xdr->head[0].iov_base); | |
119 | } else { | |
120 | xdr_off -= xdr->head[0].iov_len; | |
121 | if (xdr_off < xdr->page_len) { | |
122 | /* This offset is in the page list */ | |
3cbe01a9 | 123 | xdr_off += xdr->page_base; |
b432e6b3 TT |
124 | page = xdr->pages[xdr_off >> PAGE_SHIFT]; |
125 | xdr_off &= ~PAGE_MASK; | |
126 | } else { | |
127 | /* This offset is in the tail */ | |
128 | xdr_off -= xdr->page_len; | |
129 | xdr_off += (unsigned long) | |
130 | xdr->tail[0].iov_base & ~PAGE_MASK; | |
131 | page = virt_to_page(xdr->tail[0].iov_base); | |
132 | } | |
133 | } | |
134 | dma_addr = ib_dma_map_page(xprt->sc_cm_id->device, page, xdr_off, | |
135 | min_t(size_t, PAGE_SIZE, len), dir); | |
136 | return dma_addr; | |
137 | } | |
138 | ||
c06b540a TT |
139 | /* Assumptions: |
140 | * - The specified write_len can be represented in sc_max_sge * PAGE_SIZE | |
141 | */ | |
142 | static int send_write(struct svcxprt_rdma *xprt, struct svc_rqst *rqstp, | |
143 | u32 rmr, u64 to, | |
144 | u32 xdr_off, int write_len, | |
34d16e42 | 145 | struct svc_rdma_req_map *vec) |
c06b540a | 146 | { |
c06b540a TT |
147 | struct ib_send_wr write_wr; |
148 | struct ib_sge *sge; | |
149 | int xdr_sge_no; | |
150 | int sge_no; | |
151 | int sge_bytes; | |
152 | int sge_off; | |
153 | int bc; | |
154 | struct svc_rdma_op_ctxt *ctxt; | |
c06b540a | 155 | |
3fe04ee9 CL |
156 | if (vec->count > RPCSVC_MAXPAGES) { |
157 | pr_err("svcrdma: Too many pages (%lu)\n", vec->count); | |
158 | return -EIO; | |
159 | } | |
160 | ||
c06b540a | 161 | dprintk("svcrdma: RDMA_WRITE rmr=%x, to=%llx, xdr_off=%d, " |
34d16e42 | 162 | "write_len=%d, vec->sge=%p, vec->count=%lu\n", |
bb50c801 | 163 | rmr, (unsigned long long)to, xdr_off, |
34d16e42 | 164 | write_len, vec->sge, vec->count); |
c06b540a TT |
165 | |
166 | ctxt = svc_rdma_get_context(xprt); | |
34d16e42 TT |
167 | ctxt->direction = DMA_TO_DEVICE; |
168 | sge = ctxt->sge; | |
c06b540a TT |
169 | |
170 | /* Find the SGE associated with xdr_off */ | |
34d16e42 | 171 | for (bc = xdr_off, xdr_sge_no = 1; bc && xdr_sge_no < vec->count; |
c06b540a | 172 | xdr_sge_no++) { |
34d16e42 | 173 | if (vec->sge[xdr_sge_no].iov_len > bc) |
c06b540a | 174 | break; |
34d16e42 | 175 | bc -= vec->sge[xdr_sge_no].iov_len; |
c06b540a TT |
176 | } |
177 | ||
178 | sge_off = bc; | |
179 | bc = write_len; | |
180 | sge_no = 0; | |
181 | ||
182 | /* Copy the remaining SGE */ | |
afd566ea TT |
183 | while (bc != 0) { |
184 | sge_bytes = min_t(size_t, | |
185 | bc, vec->sge[xdr_sge_no].iov_len-sge_off); | |
c06b540a | 186 | sge[sge_no].length = sge_bytes; |
0bf48289 SW |
187 | sge[sge_no].addr = |
188 | dma_map_xdr(xprt, &rqstp->rq_res, xdr_off, | |
189 | sge_bytes, DMA_TO_DEVICE); | |
190 | xdr_off += sge_bytes; | |
191 | if (ib_dma_mapping_error(xprt->sc_cm_id->device, | |
192 | sge[sge_no].addr)) | |
193 | goto err; | |
194 | atomic_inc(&xprt->sc_dma_used); | |
195 | sge[sge_no].lkey = xprt->sc_dma_lkey; | |
afd566ea | 196 | ctxt->count++; |
c06b540a TT |
197 | sge_off = 0; |
198 | sge_no++; | |
199 | xdr_sge_no++; | |
3fe04ee9 CL |
200 | if (xdr_sge_no > vec->count) { |
201 | pr_err("svcrdma: Too many sges (%d)\n", xdr_sge_no); | |
202 | goto err; | |
203 | } | |
c06b540a | 204 | bc -= sge_bytes; |
25594290 SW |
205 | if (sge_no == xprt->sc_max_sge) |
206 | break; | |
c06b540a TT |
207 | } |
208 | ||
c06b540a TT |
209 | /* Prepare WRITE WR */ |
210 | memset(&write_wr, 0, sizeof write_wr); | |
211 | ctxt->wr_op = IB_WR_RDMA_WRITE; | |
212 | write_wr.wr_id = (unsigned long)ctxt; | |
213 | write_wr.sg_list = &sge[0]; | |
214 | write_wr.num_sge = sge_no; | |
215 | write_wr.opcode = IB_WR_RDMA_WRITE; | |
216 | write_wr.send_flags = IB_SEND_SIGNALED; | |
217 | write_wr.wr.rdma.rkey = rmr; | |
218 | write_wr.wr.rdma.remote_addr = to; | |
219 | ||
220 | /* Post It */ | |
221 | atomic_inc(&rdma_stat_write); | |
34d16e42 TT |
222 | if (svc_rdma_send(xprt, &write_wr)) |
223 | goto err; | |
25594290 | 224 | return write_len - bc; |
34d16e42 | 225 | err: |
4a84386f | 226 | svc_rdma_unmap_dma(ctxt); |
34d16e42 TT |
227 | svc_rdma_put_context(ctxt, 0); |
228 | /* Fatal error, close transport */ | |
229 | return -EIO; | |
c06b540a TT |
230 | } |
231 | ||
232 | static int send_write_chunks(struct svcxprt_rdma *xprt, | |
233 | struct rpcrdma_msg *rdma_argp, | |
234 | struct rpcrdma_msg *rdma_resp, | |
235 | struct svc_rqst *rqstp, | |
34d16e42 | 236 | struct svc_rdma_req_map *vec) |
c06b540a TT |
237 | { |
238 | u32 xfer_len = rqstp->rq_res.page_len + rqstp->rq_res.tail[0].iov_len; | |
239 | int write_len; | |
c06b540a TT |
240 | u32 xdr_off; |
241 | int chunk_off; | |
242 | int chunk_no; | |
70747c25 | 243 | int nchunks; |
c06b540a TT |
244 | struct rpcrdma_write_array *arg_ary; |
245 | struct rpcrdma_write_array *res_ary; | |
246 | int ret; | |
247 | ||
248 | arg_ary = svc_rdma_get_write_array(rdma_argp); | |
249 | if (!arg_ary) | |
250 | return 0; | |
251 | res_ary = (struct rpcrdma_write_array *) | |
252 | &rdma_resp->rm_body.rm_chunks[1]; | |
253 | ||
c06b540a | 254 | /* Write chunks start at the pagelist */ |
70747c25 | 255 | nchunks = be32_to_cpu(arg_ary->wc_nchunks); |
c06b540a | 256 | for (xdr_off = rqstp->rq_res.head[0].iov_len, chunk_no = 0; |
70747c25 | 257 | xfer_len && chunk_no < nchunks; |
c06b540a TT |
258 | chunk_no++) { |
259 | struct rpcrdma_segment *arg_ch; | |
260 | u64 rs_offset; | |
261 | ||
262 | arg_ch = &arg_ary->wc_array[chunk_no].wc_target; | |
70747c25 | 263 | write_len = min(xfer_len, be32_to_cpu(arg_ch->rs_length)); |
c06b540a TT |
264 | |
265 | /* Prepare the response chunk given the length actually | |
266 | * written */ | |
cec56c8f | 267 | xdr_decode_hyper((__be32 *)&arg_ch->rs_offset, &rs_offset); |
c06b540a | 268 | svc_rdma_xdr_encode_array_chunk(res_ary, chunk_no, |
cec56c8f TT |
269 | arg_ch->rs_handle, |
270 | arg_ch->rs_offset, | |
271 | write_len); | |
c06b540a TT |
272 | chunk_off = 0; |
273 | while (write_len) { | |
c06b540a | 274 | ret = send_write(xprt, rqstp, |
70747c25 | 275 | be32_to_cpu(arg_ch->rs_handle), |
c06b540a TT |
276 | rs_offset + chunk_off, |
277 | xdr_off, | |
25594290 | 278 | write_len, |
34d16e42 | 279 | vec); |
25594290 | 280 | if (ret <= 0) { |
c06b540a TT |
281 | dprintk("svcrdma: RDMA_WRITE failed, ret=%d\n", |
282 | ret); | |
283 | return -EIO; | |
284 | } | |
25594290 SW |
285 | chunk_off += ret; |
286 | xdr_off += ret; | |
287 | xfer_len -= ret; | |
288 | write_len -= ret; | |
c06b540a TT |
289 | } |
290 | } | |
291 | /* Update the req with the number of chunks actually used */ | |
292 | svc_rdma_xdr_encode_write_list(rdma_resp, chunk_no); | |
293 | ||
294 | return rqstp->rq_res.page_len + rqstp->rq_res.tail[0].iov_len; | |
295 | } | |
296 | ||
297 | static int send_reply_chunks(struct svcxprt_rdma *xprt, | |
298 | struct rpcrdma_msg *rdma_argp, | |
299 | struct rpcrdma_msg *rdma_resp, | |
300 | struct svc_rqst *rqstp, | |
34d16e42 | 301 | struct svc_rdma_req_map *vec) |
c06b540a TT |
302 | { |
303 | u32 xfer_len = rqstp->rq_res.len; | |
304 | int write_len; | |
c06b540a TT |
305 | u32 xdr_off; |
306 | int chunk_no; | |
307 | int chunk_off; | |
cec56c8f | 308 | int nchunks; |
c06b540a TT |
309 | struct rpcrdma_segment *ch; |
310 | struct rpcrdma_write_array *arg_ary; | |
311 | struct rpcrdma_write_array *res_ary; | |
312 | int ret; | |
313 | ||
314 | arg_ary = svc_rdma_get_reply_array(rdma_argp); | |
315 | if (!arg_ary) | |
316 | return 0; | |
317 | /* XXX: need to fix when reply lists occur with read-list and or | |
318 | * write-list */ | |
319 | res_ary = (struct rpcrdma_write_array *) | |
320 | &rdma_resp->rm_body.rm_chunks[2]; | |
321 | ||
c06b540a | 322 | /* xdr offset starts at RPC message */ |
70747c25 | 323 | nchunks = be32_to_cpu(arg_ary->wc_nchunks); |
c06b540a | 324 | for (xdr_off = 0, chunk_no = 0; |
cec56c8f | 325 | xfer_len && chunk_no < nchunks; |
c06b540a TT |
326 | chunk_no++) { |
327 | u64 rs_offset; | |
328 | ch = &arg_ary->wc_array[chunk_no].wc_target; | |
70747c25 | 329 | write_len = min(xfer_len, be32_to_cpu(ch->rs_length)); |
c06b540a | 330 | |
c06b540a TT |
331 | /* Prepare the reply chunk given the length actually |
332 | * written */ | |
cec56c8f | 333 | xdr_decode_hyper((__be32 *)&ch->rs_offset, &rs_offset); |
c06b540a | 334 | svc_rdma_xdr_encode_array_chunk(res_ary, chunk_no, |
cec56c8f TT |
335 | ch->rs_handle, ch->rs_offset, |
336 | write_len); | |
c06b540a TT |
337 | chunk_off = 0; |
338 | while (write_len) { | |
c06b540a | 339 | ret = send_write(xprt, rqstp, |
70747c25 | 340 | be32_to_cpu(ch->rs_handle), |
c06b540a TT |
341 | rs_offset + chunk_off, |
342 | xdr_off, | |
25594290 | 343 | write_len, |
34d16e42 | 344 | vec); |
25594290 | 345 | if (ret <= 0) { |
c06b540a TT |
346 | dprintk("svcrdma: RDMA_WRITE failed, ret=%d\n", |
347 | ret); | |
348 | return -EIO; | |
349 | } | |
25594290 SW |
350 | chunk_off += ret; |
351 | xdr_off += ret; | |
352 | xfer_len -= ret; | |
353 | write_len -= ret; | |
c06b540a TT |
354 | } |
355 | } | |
356 | /* Update the req with the number of chunks actually used */ | |
357 | svc_rdma_xdr_encode_reply_array(res_ary, chunk_no); | |
358 | ||
359 | return rqstp->rq_res.len; | |
360 | } | |
361 | ||
362 | /* This function prepares the portion of the RPCRDMA message to be | |
363 | * sent in the RDMA_SEND. This function is called after data sent via | |
364 | * RDMA has already been transmitted. There are three cases: | |
365 | * - The RPCRDMA header, RPC header, and payload are all sent in a | |
366 | * single RDMA_SEND. This is the "inline" case. | |
367 | * - The RPCRDMA header and some portion of the RPC header and data | |
368 | * are sent via this RDMA_SEND and another portion of the data is | |
369 | * sent via RDMA. | |
370 | * - The RPCRDMA header [NOMSG] is sent in this RDMA_SEND and the RPC | |
371 | * header and data are all transmitted via RDMA. | |
372 | * In all three cases, this function prepares the RPCRDMA header in | |
373 | * sge[0], the 'type' parameter indicates the type to place in the | |
374 | * RPCRDMA header, and the 'byte_count' field indicates how much of | |
b432e6b3 TT |
375 | * the XDR to include in this RDMA_SEND. NB: The offset of the payload |
376 | * to send is zero in the XDR. | |
c06b540a TT |
377 | */ |
378 | static int send_reply(struct svcxprt_rdma *rdma, | |
379 | struct svc_rqst *rqstp, | |
380 | struct page *page, | |
381 | struct rpcrdma_msg *rdma_resp, | |
382 | struct svc_rdma_op_ctxt *ctxt, | |
34d16e42 | 383 | struct svc_rdma_req_map *vec, |
c06b540a TT |
384 | int byte_count) |
385 | { | |
386 | struct ib_send_wr send_wr; | |
387 | int sge_no; | |
388 | int sge_bytes; | |
389 | int page_no; | |
afc59400 | 390 | int pages; |
c06b540a TT |
391 | int ret; |
392 | ||
0e7f011a TT |
393 | /* Post a recv buffer to handle another request. */ |
394 | ret = svc_rdma_post_recv(rdma); | |
395 | if (ret) { | |
396 | printk(KERN_INFO | |
397 | "svcrdma: could not post a receive buffer, err=%d." | |
398 | "Closing transport %p.\n", ret, rdma); | |
399 | set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags); | |
5ac461a6 TT |
400 | svc_rdma_put_context(ctxt, 0); |
401 | return -ENOTCONN; | |
0e7f011a TT |
402 | } |
403 | ||
c06b540a TT |
404 | /* Prepare the context */ |
405 | ctxt->pages[0] = page; | |
406 | ctxt->count = 1; | |
407 | ||
408 | /* Prepare the SGE for the RPCRDMA Header */ | |
98779be8 SW |
409 | ctxt->sge[0].lkey = rdma->sc_dma_lkey; |
410 | ctxt->sge[0].length = svc_rdma_xdr_get_reply_hdr_len(rdma_resp); | |
c06b540a | 411 | ctxt->sge[0].addr = |
b432e6b3 TT |
412 | ib_dma_map_page(rdma->sc_cm_id->device, page, 0, |
413 | ctxt->sge[0].length, DMA_TO_DEVICE); | |
afd566ea TT |
414 | if (ib_dma_mapping_error(rdma->sc_cm_id->device, ctxt->sge[0].addr)) |
415 | goto err; | |
416 | atomic_inc(&rdma->sc_dma_used); | |
417 | ||
c06b540a | 418 | ctxt->direction = DMA_TO_DEVICE; |
afd566ea | 419 | |
b432e6b3 | 420 | /* Map the payload indicated by 'byte_count' */ |
34d16e42 | 421 | for (sge_no = 1; byte_count && sge_no < vec->count; sge_no++) { |
b432e6b3 | 422 | int xdr_off = 0; |
34d16e42 | 423 | sge_bytes = min_t(size_t, vec->sge[sge_no].iov_len, byte_count); |
c06b540a | 424 | byte_count -= sge_bytes; |
0bf48289 SW |
425 | ctxt->sge[sge_no].addr = |
426 | dma_map_xdr(rdma, &rqstp->rq_res, xdr_off, | |
427 | sge_bytes, DMA_TO_DEVICE); | |
428 | xdr_off += sge_bytes; | |
429 | if (ib_dma_mapping_error(rdma->sc_cm_id->device, | |
430 | ctxt->sge[sge_no].addr)) | |
431 | goto err; | |
432 | atomic_inc(&rdma->sc_dma_used); | |
433 | ctxt->sge[sge_no].lkey = rdma->sc_dma_lkey; | |
34d16e42 | 434 | ctxt->sge[sge_no].length = sge_bytes; |
c06b540a | 435 | } |
3fe04ee9 CL |
436 | if (byte_count != 0) { |
437 | pr_err("svcrdma: Could not map %d bytes\n", byte_count); | |
438 | goto err; | |
439 | } | |
c06b540a TT |
440 | |
441 | /* Save all respages in the ctxt and remove them from the | |
442 | * respages array. They are our pages until the I/O | |
443 | * completes. | |
444 | */ | |
afc59400 BF |
445 | pages = rqstp->rq_next_page - rqstp->rq_respages; |
446 | for (page_no = 0; page_no < pages; page_no++) { | |
c06b540a TT |
447 | ctxt->pages[page_no+1] = rqstp->rq_respages[page_no]; |
448 | ctxt->count++; | |
449 | rqstp->rq_respages[page_no] = NULL; | |
afd566ea TT |
450 | /* |
451 | * If there are more pages than SGE, terminate SGE | |
452 | * list so that svc_rdma_unmap_dma doesn't attempt to | |
453 | * unmap garbage. | |
454 | */ | |
34d16e42 TT |
455 | if (page_no+1 >= sge_no) |
456 | ctxt->sge[page_no+1].length = 0; | |
c06b540a | 457 | } |
7e4359e2 | 458 | rqstp->rq_next_page = rqstp->rq_respages + 1; |
0bf48289 | 459 | |
3fe04ee9 CL |
460 | if (sge_no > rdma->sc_max_sge) { |
461 | pr_err("svcrdma: Too many sges (%d)\n", sge_no); | |
462 | goto err; | |
463 | } | |
c06b540a TT |
464 | memset(&send_wr, 0, sizeof send_wr); |
465 | ctxt->wr_op = IB_WR_SEND; | |
466 | send_wr.wr_id = (unsigned long)ctxt; | |
467 | send_wr.sg_list = ctxt->sge; | |
468 | send_wr.num_sge = sge_no; | |
469 | send_wr.opcode = IB_WR_SEND; | |
470 | send_wr.send_flags = IB_SEND_SIGNALED; | |
471 | ||
472 | ret = svc_rdma_send(rdma, &send_wr); | |
473 | if (ret) | |
afd566ea | 474 | goto err; |
c06b540a | 475 | |
afd566ea TT |
476 | return 0; |
477 | ||
478 | err: | |
21515e46 | 479 | svc_rdma_unmap_dma(ctxt); |
afd566ea TT |
480 | svc_rdma_put_context(ctxt, 1); |
481 | return -EIO; | |
c06b540a TT |
482 | } |
483 | ||
484 | void svc_rdma_prep_reply_hdr(struct svc_rqst *rqstp) | |
485 | { | |
486 | } | |
487 | ||
c06b540a TT |
488 | int svc_rdma_sendto(struct svc_rqst *rqstp) |
489 | { | |
490 | struct svc_xprt *xprt = rqstp->rq_xprt; | |
491 | struct svcxprt_rdma *rdma = | |
492 | container_of(xprt, struct svcxprt_rdma, sc_xprt); | |
493 | struct rpcrdma_msg *rdma_argp; | |
494 | struct rpcrdma_msg *rdma_resp; | |
495 | struct rpcrdma_write_array *reply_ary; | |
496 | enum rpcrdma_proc reply_type; | |
497 | int ret; | |
498 | int inline_bytes; | |
c06b540a TT |
499 | struct page *res_page; |
500 | struct svc_rdma_op_ctxt *ctxt; | |
34d16e42 | 501 | struct svc_rdma_req_map *vec; |
c06b540a TT |
502 | |
503 | dprintk("svcrdma: sending response for rqstp=%p\n", rqstp); | |
504 | ||
e5523bd2 CL |
505 | /* Get the RDMA request header. The receive logic always |
506 | * places this at the start of page 0. | |
507 | */ | |
508 | rdma_argp = page_address(rqstp->rq_pages[0]); | |
c06b540a | 509 | |
34d16e42 | 510 | /* Build an req vec for the XDR */ |
c06b540a TT |
511 | ctxt = svc_rdma_get_context(rdma); |
512 | ctxt->direction = DMA_TO_DEVICE; | |
34d16e42 | 513 | vec = svc_rdma_get_req_map(); |
afd566ea TT |
514 | ret = map_xdr(rdma, &rqstp->rq_res, vec); |
515 | if (ret) | |
516 | goto err0; | |
c06b540a TT |
517 | inline_bytes = rqstp->rq_res.len; |
518 | ||
519 | /* Create the RDMA response header */ | |
b7e0b9a9 | 520 | res_page = alloc_page(GFP_KERNEL | __GFP_NOFAIL); |
c06b540a TT |
521 | rdma_resp = page_address(res_page); |
522 | reply_ary = svc_rdma_get_reply_array(rdma_argp); | |
523 | if (reply_ary) | |
524 | reply_type = RDMA_NOMSG; | |
525 | else | |
526 | reply_type = RDMA_MSG; | |
527 | svc_rdma_xdr_encode_reply_header(rdma, rdma_argp, | |
528 | rdma_resp, reply_type); | |
529 | ||
530 | /* Send any write-chunk data and build resp write-list */ | |
531 | ret = send_write_chunks(rdma, rdma_argp, rdma_resp, | |
34d16e42 | 532 | rqstp, vec); |
c06b540a TT |
533 | if (ret < 0) { |
534 | printk(KERN_ERR "svcrdma: failed to send write chunks, rc=%d\n", | |
535 | ret); | |
afd566ea | 536 | goto err1; |
c06b540a TT |
537 | } |
538 | inline_bytes -= ret; | |
539 | ||
540 | /* Send any reply-list data and update resp reply-list */ | |
541 | ret = send_reply_chunks(rdma, rdma_argp, rdma_resp, | |
34d16e42 | 542 | rqstp, vec); |
c06b540a TT |
543 | if (ret < 0) { |
544 | printk(KERN_ERR "svcrdma: failed to send reply chunks, rc=%d\n", | |
545 | ret); | |
afd566ea | 546 | goto err1; |
c06b540a TT |
547 | } |
548 | inline_bytes -= ret; | |
549 | ||
34d16e42 | 550 | ret = send_reply(rdma, rqstp, res_page, rdma_resp, ctxt, vec, |
c06b540a | 551 | inline_bytes); |
34d16e42 | 552 | svc_rdma_put_req_map(vec); |
c06b540a TT |
553 | dprintk("svcrdma: send_reply returns %d\n", ret); |
554 | return ret; | |
afd566ea TT |
555 | |
556 | err1: | |
557 | put_page(res_page); | |
558 | err0: | |
34d16e42 | 559 | svc_rdma_put_req_map(vec); |
c06b540a | 560 | svc_rdma_put_context(ctxt, 0); |
c06b540a TT |
561 | return ret; |
562 | } |