2 * Copyright (c) 2015 Oracle. All rights reserved.
3 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
6 /* Lightweight memory registration using Fast Memory Regions (FMR).
7 * Referred to sometimes as MTHCAFMR mode.
9 * FMR uses synchronous memory registration and deregistration.
10 * FMR registration is known to be fast, but FMR deregistration
11 * can take tens of usecs to complete.
16 * A Memory Region is prepared for RDMA READ or WRITE using the
17 * ib_map_phys_fmr verb (fmr_op_map). When the RDMA operation is
18 * finished, the Memory Region is unmapped using the ib_unmap_fmr
19 * verb (fmr_op_unmap).
24 * After a transport reconnect, fmr_op_map re-uses the MR already
25 * allocated for the RPC, but generates a fresh rkey then maps the
26 * MR again. This process is synchronous.
29 #include "xprt_rdma.h"
31 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
32 # define RPCDBG_FACILITY RPCDBG_TRANS
35 /* Maximum scatter/gather per FMR */
36 #define RPCRDMA_MAX_FMR_SGES (64)
39 fmr_op_open(struct rpcrdma_ia
*ia
, struct rpcrdma_ep
*ep
,
40 struct rpcrdma_create_data_internal
*cdata
)
45 /* FMR mode conveys up to 64 pages of payload per chunk segment.
48 fmr_op_maxpages(struct rpcrdma_xprt
*r_xprt
)
50 return min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS
,
51 rpcrdma_max_segments(r_xprt
) * RPCRDMA_MAX_FMR_SGES
);
55 fmr_op_init(struct rpcrdma_xprt
*r_xprt
)
57 struct rpcrdma_buffer
*buf
= &r_xprt
->rx_buf
;
58 int mr_access_flags
= IB_ACCESS_REMOTE_WRITE
| IB_ACCESS_REMOTE_READ
;
59 struct ib_fmr_attr fmr_attr
= {
60 .max_pages
= RPCRDMA_MAX_FMR_SGES
,
62 .page_shift
= PAGE_SHIFT
64 struct ib_pd
*pd
= r_xprt
->rx_ia
.ri_pd
;
68 INIT_LIST_HEAD(&buf
->rb_mws
);
69 INIT_LIST_HEAD(&buf
->rb_all
);
71 i
= (buf
->rb_max_requests
+ 1) * RPCRDMA_MAX_SEGS
;
72 dprintk("RPC: %s: initializing %d FMRs\n", __func__
, i
);
75 r
= kzalloc(sizeof(*r
), GFP_KERNEL
);
79 r
->r
.fmr
= ib_alloc_fmr(pd
, mr_access_flags
, &fmr_attr
);
83 list_add(&r
->mw_list
, &buf
->rb_mws
);
84 list_add(&r
->mw_all
, &buf
->rb_all
);
89 rc
= PTR_ERR(r
->r
.fmr
);
90 dprintk("RPC: %s: ib_alloc_fmr status %i\n", __func__
, rc
);
96 __fmr_unmap(struct rpcrdma_mw
*r
)
100 list_add(&r
->r
.fmr
->list
, &l
);
101 return ib_unmap_fmr(&l
);
104 /* Use the ib_map_phys_fmr() verb to register a memory region
105 * for remote access via RDMA READ or RDMA WRITE.
108 fmr_op_map(struct rpcrdma_xprt
*r_xprt
, struct rpcrdma_mr_seg
*seg
,
109 int nsegs
, bool writing
)
111 struct rpcrdma_ia
*ia
= &r_xprt
->rx_ia
;
112 struct ib_device
*device
= ia
->ri_device
;
113 enum dma_data_direction direction
= rpcrdma_data_dir(writing
);
114 struct rpcrdma_mr_seg
*seg1
= seg
;
115 u64 physaddrs
[RPCRDMA_MAX_DATA_SEGS
];
116 int len
, pageoff
, i
, rc
;
117 struct rpcrdma_mw
*mw
;
122 mw
= rpcrdma_get_mw(r_xprt
);
126 /* this is a retransmit; generate a fresh rkey */
127 rc
= __fmr_unmap(mw
);
132 pageoff
= offset_in_page(seg1
->mr_offset
);
133 seg1
->mr_offset
-= pageoff
; /* start of page */
134 seg1
->mr_len
+= pageoff
;
136 if (nsegs
> RPCRDMA_MAX_FMR_SGES
)
137 nsegs
= RPCRDMA_MAX_FMR_SGES
;
138 for (i
= 0; i
< nsegs
;) {
139 rpcrdma_map_one(device
, seg
, direction
);
140 physaddrs
[i
] = seg
->mr_dma
;
144 /* Check for holes */
145 if ((i
< nsegs
&& offset_in_page(seg
->mr_offset
)) ||
146 offset_in_page((seg
-1)->mr_offset
+ (seg
-1)->mr_len
))
150 rc
= ib_map_phys_fmr(mw
->r
.fmr
, physaddrs
, i
, seg1
->mr_dma
);
155 seg1
->mr_rkey
= mw
->r
.fmr
->rkey
;
156 seg1
->mr_base
= seg1
->mr_dma
+ pageoff
;
162 dprintk("RPC: %s: ib_map_phys_fmr %u@0x%llx+%i (%d) status %i\n",
163 __func__
, len
, (unsigned long long)seg1
->mr_dma
,
166 rpcrdma_unmap_one(device
, --seg
);
170 /* Use the ib_unmap_fmr() verb to prevent further remote
171 * access via RDMA READ or RDMA WRITE.
174 fmr_op_unmap(struct rpcrdma_xprt
*r_xprt
, struct rpcrdma_mr_seg
*seg
)
176 struct rpcrdma_ia
*ia
= &r_xprt
->rx_ia
;
177 struct rpcrdma_mr_seg
*seg1
= seg
;
178 struct rpcrdma_mw
*mw
= seg1
->rl_mw
;
179 int rc
, nsegs
= seg
->mr_nsegs
;
181 dprintk("RPC: %s: FMR %p\n", __func__
, mw
);
184 while (seg1
->mr_nsegs
--)
185 rpcrdma_unmap_one(ia
->ri_device
, seg
++);
186 rc
= __fmr_unmap(mw
);
189 rpcrdma_put_mw(r_xprt
, mw
);
193 /* The FMR is abandoned, but remains in rb_all. fmr_op_destroy
194 * will attempt to release it when the transport is destroyed.
196 dprintk("RPC: %s: ib_unmap_fmr status %i\n", __func__
, rc
);
201 fmr_op_destroy(struct rpcrdma_buffer
*buf
)
203 struct rpcrdma_mw
*r
;
206 while (!list_empty(&buf
->rb_all
)) {
207 r
= list_entry(buf
->rb_all
.next
, struct rpcrdma_mw
, mw_all
);
208 list_del(&r
->mw_all
);
209 rc
= ib_dealloc_fmr(r
->r
.fmr
);
211 dprintk("RPC: %s: ib_dealloc_fmr failed %i\n",
217 const struct rpcrdma_memreg_ops rpcrdma_fmr_memreg_ops
= {
218 .ro_map
= fmr_op_map
,
219 .ro_unmap
= fmr_op_unmap
,
220 .ro_open
= fmr_op_open
,
221 .ro_maxpages
= fmr_op_maxpages
,
222 .ro_init
= fmr_op_init
,
223 .ro_destroy
= fmr_op_destroy
,
224 .ro_displayname
= "fmr",