Commit | Line | Data |
---|---|---|
d21b05f1 TT |
1 | /* |
2 | * Copyright (c) 2005-2006 Network Appliance, Inc. All rights reserved. | |
3 | * | |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the BSD-type | |
8 | * license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or without | |
11 | * modification, are permitted provided that the following conditions | |
12 | * are met: | |
13 | * | |
14 | * Redistributions of source code must retain the above copyright | |
15 | * notice, this list of conditions and the following disclaimer. | |
16 | * | |
17 | * Redistributions in binary form must reproduce the above | |
18 | * copyright notice, this list of conditions and the following | |
19 | * disclaimer in the documentation and/or other materials provided | |
20 | * with the distribution. | |
21 | * | |
22 | * Neither the name of the Network Appliance, Inc. nor the names of | |
23 | * its contributors may be used to endorse or promote products | |
24 | * derived from this software without specific prior written | |
25 | * permission. | |
26 | * | |
27 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
28 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
29 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
30 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
31 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
32 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
33 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
34 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
35 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
36 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
37 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
38 | * | |
39 | * Author: Tom Tucker <tom@opengridcomputing.com> | |
40 | */ | |
41 | ||
42 | #ifndef SVC_RDMA_H | |
43 | #define SVC_RDMA_H | |
44 | #include <linux/sunrpc/xdr.h> | |
45 | #include <linux/sunrpc/svcsock.h> | |
46 | #include <linux/sunrpc/rpc_rdma.h> | |
47 | #include <rdma/ib_verbs.h> | |
48 | #include <rdma/rdma_cm.h> | |
49 | #define SVCRDMA_DEBUG | |
50 | ||
51 | /* RPC/RDMA parameters and stats */ | |
52 | extern unsigned int svcrdma_ord; | |
53 | extern unsigned int svcrdma_max_requests; | |
03fe9931 | 54 | extern unsigned int svcrdma_max_bc_requests; |
d21b05f1 TT |
55 | extern unsigned int svcrdma_max_req_size; |
56 | ||
57 | extern atomic_t rdma_stat_recv; | |
58 | extern atomic_t rdma_stat_read; | |
59 | extern atomic_t rdma_stat_write; | |
60 | extern atomic_t rdma_stat_sq_starve; | |
61 | extern atomic_t rdma_stat_rq_starve; | |
62 | extern atomic_t rdma_stat_rq_poll; | |
63 | extern atomic_t rdma_stat_rq_prod; | |
64 | extern atomic_t rdma_stat_sq_poll; | |
65 | extern atomic_t rdma_stat_sq_prod; | |
66 | ||
d21b05f1 TT |
67 | /* |
68 | * Contexts are built when an RDMA request is created and are a | |
69 | * record of the resources that can be recovered when the request | |
70 | * completes. | |
71 | */ | |
72 | struct svc_rdma_op_ctxt { | |
cc886c9f | 73 | struct list_head free; |
02e7452d | 74 | struct svc_rdma_op_ctxt *read_hdr; |
0d3ebb9a | 75 | struct svc_rdma_fastreg_mr *frmr; |
f820c57e | 76 | int hdr_count; |
d21b05f1 TT |
77 | struct xdr_buf arg; |
78 | struct list_head dto_q; | |
79 | enum ib_wr_opcode wr_op; | |
80 | enum ib_wc_status wc_status; | |
81 | u32 byte_len; | |
0b056c22 | 82 | u32 position; |
d21b05f1 TT |
83 | struct svcxprt_rdma *xprt; |
84 | unsigned long flags; | |
85 | enum dma_data_direction direction; | |
86 | int count; | |
87 | struct ib_sge sge[RPCSVC_MAXPAGES]; | |
88 | struct page *pages[RPCSVC_MAXPAGES]; | |
89 | }; | |
90 | ||
ab96dddb TT |
91 | /* |
92 | * NFS_ requests are mapped on the client side by the chunk lists in | |
93 | * the RPCRDMA header. During the fetching of the RPC from the client | |
94 | * and the writing of the reply to the client, the memory in the | |
95 | * client and the memory in the server must be mapped as contiguous | |
96 | * vaddr/len for access by the hardware. These data strucures keep | |
97 | * these mappings. | |
98 | * | |
99 | * For an RDMA_WRITE, the 'sge' maps the RPC REPLY. For RDMA_READ, the | |
100 | * 'sge' in the svc_rdma_req_map maps the server side RPC reply and the | |
101 | * 'ch' field maps the read-list of the RPCRDMA header to the 'sge' | |
102 | * mapping of the reply. | |
103 | */ | |
104 | struct svc_rdma_chunk_sge { | |
105 | int start; /* sge no for this chunk */ | |
106 | int count; /* sge count for this chunk */ | |
107 | }; | |
0d3ebb9a TT |
108 | struct svc_rdma_fastreg_mr { |
109 | struct ib_mr *mr; | |
412a15c0 SG |
110 | struct scatterlist *sg; |
111 | int sg_nents; | |
0d3ebb9a | 112 | unsigned long access_flags; |
0d3ebb9a TT |
113 | enum dma_data_direction direction; |
114 | struct list_head frmr_list; | |
115 | }; | |
ab96dddb | 116 | struct svc_rdma_req_map { |
2fe81b23 | 117 | struct list_head free; |
ab96dddb TT |
118 | unsigned long count; |
119 | union { | |
120 | struct kvec sge[RPCSVC_MAXPAGES]; | |
121 | struct svc_rdma_chunk_sge ch[RPCSVC_MAXPAGES]; | |
0bf48289 | 122 | unsigned long lkey[RPCSVC_MAXPAGES]; |
ab96dddb TT |
123 | }; |
124 | }; | |
d21b05f1 TT |
125 | #define RDMACTXT_F_LAST_CTXT 2 |
126 | ||
0d3ebb9a TT |
127 | #define SVCRDMA_DEVCAP_FAST_REG 1 /* fast mr registration */ |
128 | #define SVCRDMA_DEVCAP_READ_W_INV 2 /* read w/ invalidate */ | |
129 | ||
d21b05f1 TT |
130 | struct svcxprt_rdma { |
131 | struct svc_xprt sc_xprt; /* SVC transport structure */ | |
132 | struct rdma_cm_id *sc_cm_id; /* RDMA connection id */ | |
133 | struct list_head sc_accept_q; /* Conn. waiting accept */ | |
134 | int sc_ord; /* RDMA read limit */ | |
d21b05f1 | 135 | int sc_max_sge; |
bc3fe2e3 | 136 | int sc_max_sge_rd; /* max sge for read target */ |
d21b05f1 | 137 | |
d21b05f1 | 138 | atomic_t sc_sq_count; /* Number of SQ WR on queue */ |
03fe9931 CL |
139 | unsigned int sc_sq_depth; /* Depth of SQ */ |
140 | unsigned int sc_rq_depth; /* Depth of RQ */ | |
141 | u32 sc_max_requests; /* Forward credits */ | |
142 | u32 sc_max_bc_requests;/* Backward credits */ | |
d21b05f1 TT |
143 | int sc_max_req_size; /* Size of each RQ WR buf */ |
144 | ||
145 | struct ib_pd *sc_pd; | |
146 | ||
87295b6c | 147 | atomic_t sc_dma_used; |
cc886c9f CL |
148 | spinlock_t sc_ctxt_lock; |
149 | struct list_head sc_ctxts; | |
150 | int sc_ctxt_used; | |
2fe81b23 CL |
151 | spinlock_t sc_map_lock; |
152 | struct list_head sc_maps; | |
cc886c9f | 153 | |
d21b05f1 TT |
154 | struct list_head sc_rq_dto_q; |
155 | spinlock_t sc_rq_dto_lock; | |
156 | struct ib_qp *sc_qp; | |
157 | struct ib_cq *sc_rq_cq; | |
158 | struct ib_cq *sc_sq_cq; | |
e5452411 CL |
159 | int (*sc_reader)(struct svcxprt_rdma *, |
160 | struct svc_rqst *, | |
161 | struct svc_rdma_op_ctxt *, | |
162 | int *, u32 *, u32, u32, u64, bool); | |
0d3ebb9a | 163 | u32 sc_dev_caps; /* distilled device caps */ |
0d3ebb9a TT |
164 | unsigned int sc_frmr_pg_list_len; |
165 | struct list_head sc_frmr_q; | |
166 | spinlock_t sc_frmr_q_lock; | |
d21b05f1 TT |
167 | |
168 | spinlock_t sc_lock; /* transport lock */ | |
169 | ||
170 | wait_queue_head_t sc_send_wait; /* SQ exhaustion waitlist */ | |
171 | unsigned long sc_flags; | |
172 | struct list_head sc_dto_q; /* DTO tasklet I/O pending Q */ | |
173 | struct list_head sc_read_complete_q; | |
8da91ea8 | 174 | struct work_struct sc_work; |
d21b05f1 TT |
175 | }; |
176 | /* sc_flags */ | |
177 | #define RDMAXPRT_RQ_PENDING 1 | |
178 | #define RDMAXPRT_SQ_PENDING 2 | |
179 | #define RDMAXPRT_CONN_PENDING 3 | |
180 | ||
181 | #define RPCRDMA_LISTEN_BACKLOG 10 | |
182 | /* The default ORD value is based on two outstanding full-size writes with a | |
183 | * page size of 4k, or 32k * 2 ops / 4k = 16 outstanding RDMA_READ. */ | |
184 | #define RPCRDMA_ORD (64/4) | |
185 | #define RPCRDMA_SQ_DEPTH_MULT 8 | |
d9bb5a43 | 186 | #define RPCRDMA_MAX_REQUESTS 32 |
d21b05f1 TT |
187 | #define RPCRDMA_MAX_REQ_SIZE 4096 |
188 | ||
03fe9931 CL |
189 | /* Typical ULP usage of BC requests is NFSv4.1 backchannel. Our |
190 | * current NFSv4.1 implementation supports one backchannel slot. | |
191 | */ | |
192 | #define RPCRDMA_MAX_BC_REQUESTS 2 | |
193 | ||
cc9a903d CL |
194 | #define RPCSVC_MAXPAYLOAD_RDMA RPCSVC_MAXPAYLOAD |
195 | ||
5d252f90 CL |
196 | /* svc_rdma_backchannel.c */ |
197 | extern int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, | |
198 | struct rpcrdma_msg *rmsgp, | |
199 | struct xdr_buf *rcvbuf); | |
200 | ||
d21b05f1 | 201 | /* svc_rdma_marshal.c */ |
f3ea53fb | 202 | extern int svc_rdma_xdr_decode_req(struct rpcrdma_msg *, struct svc_rqst *); |
d21b05f1 TT |
203 | extern int svc_rdma_xdr_encode_error(struct svcxprt_rdma *, |
204 | struct rpcrdma_msg *, | |
30b7e246 | 205 | enum rpcrdma_errcode, __be32 *); |
d21b05f1 TT |
206 | extern void svc_rdma_xdr_encode_write_list(struct rpcrdma_msg *, int); |
207 | extern void svc_rdma_xdr_encode_reply_array(struct rpcrdma_write_array *, int); | |
208 | extern void svc_rdma_xdr_encode_array_chunk(struct rpcrdma_write_array *, int, | |
1fa9c444 | 209 | __be32, __be64, u32); |
d21b05f1 TT |
210 | extern void svc_rdma_xdr_encode_reply_header(struct svcxprt_rdma *, |
211 | struct rpcrdma_msg *, | |
212 | struct rpcrdma_msg *, | |
213 | enum rpcrdma_proc); | |
214 | extern int svc_rdma_xdr_get_reply_hdr_len(struct rpcrdma_msg *); | |
215 | ||
216 | /* svc_rdma_recvfrom.c */ | |
217 | extern int svc_rdma_recvfrom(struct svc_rqst *); | |
e5452411 CL |
218 | extern int rdma_read_chunk_lcl(struct svcxprt_rdma *, struct svc_rqst *, |
219 | struct svc_rdma_op_ctxt *, int *, u32 *, | |
220 | u32, u32, u64, bool); | |
221 | extern int rdma_read_chunk_frmr(struct svcxprt_rdma *, struct svc_rqst *, | |
222 | struct svc_rdma_op_ctxt *, int *, u32 *, | |
223 | u32, u32, u64, bool); | |
d21b05f1 TT |
224 | |
225 | /* svc_rdma_sendto.c */ | |
ba986c96 | 226 | extern int svc_rdma_map_xdr(struct svcxprt_rdma *, struct xdr_buf *, |
f6763c29 | 227 | struct svc_rdma_req_map *, bool); |
d21b05f1 | 228 | extern int svc_rdma_sendto(struct svc_rqst *); |
10dc4512 CL |
229 | extern struct rpcrdma_read_chunk * |
230 | svc_rdma_get_read_chunk(struct rpcrdma_msg *); | |
a6081b82 CL |
231 | extern void svc_rdma_send_error(struct svcxprt_rdma *, struct rpcrdma_msg *, |
232 | int); | |
d21b05f1 TT |
233 | |
234 | /* svc_rdma_transport.c */ | |
235 | extern int svc_rdma_send(struct svcxprt_rdma *, struct ib_send_wr *); | |
39b09a1a | 236 | extern int svc_rdma_post_recv(struct svcxprt_rdma *, gfp_t); |
bf36387a | 237 | extern int svc_rdma_repost_recv(struct svcxprt_rdma *, gfp_t); |
d21b05f1 TT |
238 | extern int svc_rdma_create_listen(struct svc_serv *, int, struct sockaddr *); |
239 | extern struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *); | |
240 | extern void svc_rdma_put_context(struct svc_rdma_op_ctxt *, int); | |
146b6df6 | 241 | extern void svc_rdma_unmap_dma(struct svc_rdma_op_ctxt *ctxt); |
2fe81b23 CL |
242 | extern struct svc_rdma_req_map *svc_rdma_get_req_map(struct svcxprt_rdma *); |
243 | extern void svc_rdma_put_req_map(struct svcxprt_rdma *, | |
244 | struct svc_rdma_req_map *); | |
64be8608 TT |
245 | extern struct svc_rdma_fastreg_mr *svc_rdma_get_frmr(struct svcxprt_rdma *); |
246 | extern void svc_rdma_put_frmr(struct svcxprt_rdma *, | |
247 | struct svc_rdma_fastreg_mr *); | |
d21b05f1 TT |
248 | extern void svc_sq_reap(struct svcxprt_rdma *); |
249 | extern void svc_rq_reap(struct svcxprt_rdma *); | |
d21b05f1 TT |
250 | extern void svc_rdma_prep_reply_hdr(struct svc_rqst *); |
251 | ||
94684319 CL |
252 | extern struct svc_xprt_class svc_rdma_class; |
253 | #ifdef CONFIG_SUNRPC_BACKCHANNEL | |
254 | extern struct svc_xprt_class svc_rdma_bc_class; | |
255 | #endif | |
256 | ||
d21b05f1 | 257 | /* svc_rdma.c */ |
71810ef3 | 258 | extern struct workqueue_struct *svc_rdma_wq; |
d21b05f1 TT |
259 | extern int svc_rdma_init(void); |
260 | extern void svc_rdma_cleanup(void); | |
261 | ||
d21b05f1 | 262 | #endif |