Commit | Line | Data |
---|---|---|
cfdda9d7 SW |
1 | /* |
2 | * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. | |
3 | * | |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | */ | |
32 | ||
42b6a949 VP |
33 | #include <linux/module.h> |
34 | #include <linux/moduleparam.h> | |
cfdda9d7 | 35 | #include <rdma/ib_umem.h> |
60063497 | 36 | #include <linux/atomic.h> |
cfdda9d7 SW |
37 | |
38 | #include "iw_cxgb4.h" | |
39 | ||
96bb2706 | 40 | int use_dsgl = 0; |
42b6a949 | 41 | module_param(use_dsgl, int, 0644); |
96bb2706 | 42 | MODULE_PARM_DESC(use_dsgl, "Use DSGL for PBL/FastReg (default=0)"); |
42b6a949 | 43 | |
cfdda9d7 SW |
44 | #define T4_ULPTX_MIN_IO 32 |
45 | #define C4IW_MAX_INLINE_SIZE 96 | |
42b6a949 VP |
46 | #define T4_ULPTX_MAX_DMA 1024 |
47 | #define C4IW_INLINE_THRESHOLD 128 | |
cfdda9d7 | 48 | |
42b6a949 VP |
49 | static int inline_threshold = C4IW_INLINE_THRESHOLD; |
50 | module_param(inline_threshold, int, 0644); | |
51 | MODULE_PARM_DESC(inline_threshold, "inline vs dsgl threshold (default=128)"); | |
52 | ||
2550a88d HS |
53 | static int mr_exceeds_hw_limits(struct c4iw_dev *dev, u64 length) |
54 | { | |
55 | return (is_t4(dev->rdev.lldi.adapter_type) || | |
56 | is_t5(dev->rdev.lldi.adapter_type)) && | |
57 | length >= 8*1024*1024*1024ULL; | |
58 | } | |
59 | ||
42b6a949 | 60 | static int _c4iw_write_mem_dma_aligned(struct c4iw_rdev *rdev, u32 addr, |
0e5eca79 | 61 | u32 len, dma_addr_t data, int wait) |
42b6a949 VP |
62 | { |
63 | struct sk_buff *skb; | |
64 | struct ulp_mem_io *req; | |
65 | struct ulptx_sgl *sgl; | |
66 | u8 wr_len; | |
67 | int ret = 0; | |
68 | struct c4iw_wr_wait wr_wait; | |
69 | ||
70 | addr &= 0x7FFFFFF; | |
71 | ||
72 | if (wait) | |
73 | c4iw_init_wr_wait(&wr_wait); | |
74 | wr_len = roundup(sizeof(*req) + sizeof(*sgl), 16); | |
75 | ||
f72f116a | 76 | skb = alloc_skb(wr_len, GFP_KERNEL); |
42b6a949 VP |
77 | if (!skb) |
78 | return -ENOMEM; | |
79 | set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); | |
80 | ||
81 | req = (struct ulp_mem_io *)__skb_put(skb, wr_len); | |
82 | memset(req, 0, wr_len); | |
83 | INIT_ULPTX_WR(req, wr_len, 0, 0); | |
e2ac9628 HS |
84 | req->wr.wr_hi = cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) | |
85 | (wait ? FW_WR_COMPL_F : 0)); | |
298589b1 | 86 | req->wr.wr_lo = wait ? (__force __be64)(unsigned long) &wr_wait : 0L; |
e2ac9628 | 87 | req->wr.wr_mid = cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(wr_len, 16))); |
d7990b0c | 88 | req->cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE)); |
bdc590b9 | 89 | req->cmd |= cpu_to_be32(T5_ULP_MEMIO_ORDER_V(1)); |
d7990b0c | 90 | req->dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V(len>>5)); |
42b6a949 | 91 | req->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(req->wr), 16)); |
d7990b0c | 92 | req->lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(addr)); |
42b6a949 VP |
93 | |
94 | sgl = (struct ulptx_sgl *)(req + 1); | |
d7990b0c | 95 | sgl->cmd_nsge = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_DSGL) | |
bdc590b9 | 96 | ULPTX_NSGE_V(1)); |
42b6a949 | 97 | sgl->len0 = cpu_to_be32(len); |
0e5eca79 | 98 | sgl->addr0 = cpu_to_be64(data); |
42b6a949 VP |
99 | |
100 | ret = c4iw_ofld_send(rdev, skb); | |
101 | if (ret) | |
102 | return ret; | |
103 | if (wait) | |
104 | ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__); | |
105 | return ret; | |
106 | } | |
107 | ||
108 | static int _c4iw_write_mem_inline(struct c4iw_rdev *rdev, u32 addr, u32 len, | |
109 | void *data) | |
cfdda9d7 SW |
110 | { |
111 | struct sk_buff *skb; | |
112 | struct ulp_mem_io *req; | |
113 | struct ulptx_idata *sc; | |
114 | u8 wr_len, *to_dp, *from_dp; | |
115 | int copy_len, num_wqe, i, ret = 0; | |
116 | struct c4iw_wr_wait wr_wait; | |
d7990b0c | 117 | __be32 cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE)); |
42b6a949 VP |
118 | |
119 | if (is_t4(rdev->lldi.adapter_type)) | |
d7990b0c | 120 | cmd |= cpu_to_be32(ULP_MEMIO_ORDER_F); |
42b6a949 | 121 | else |
d7990b0c | 122 | cmd |= cpu_to_be32(T5_ULP_MEMIO_IMM_F); |
cfdda9d7 SW |
123 | |
124 | addr &= 0x7FFFFFF; | |
125 | PDBG("%s addr 0x%x len %u\n", __func__, addr, len); | |
126 | num_wqe = DIV_ROUND_UP(len, C4IW_MAX_INLINE_SIZE); | |
127 | c4iw_init_wr_wait(&wr_wait); | |
128 | for (i = 0; i < num_wqe; i++) { | |
129 | ||
130 | copy_len = len > C4IW_MAX_INLINE_SIZE ? C4IW_MAX_INLINE_SIZE : | |
131 | len; | |
132 | wr_len = roundup(sizeof *req + sizeof *sc + | |
133 | roundup(copy_len, T4_ULPTX_MIN_IO), 16); | |
134 | ||
d3c814e8 | 135 | skb = alloc_skb(wr_len, GFP_KERNEL); |
cfdda9d7 SW |
136 | if (!skb) |
137 | return -ENOMEM; | |
138 | set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); | |
139 | ||
140 | req = (struct ulp_mem_io *)__skb_put(skb, wr_len); | |
141 | memset(req, 0, wr_len); | |
142 | INIT_ULPTX_WR(req, wr_len, 0, 0); | |
143 | ||
144 | if (i == (num_wqe-1)) { | |
e2ac9628 HS |
145 | req->wr.wr_hi = cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) | |
146 | FW_WR_COMPL_F); | |
b61e564a | 147 | req->wr.wr_lo = (__force __be64)(unsigned long)&wr_wait; |
cfdda9d7 | 148 | } else |
e2ac9628 | 149 | req->wr.wr_hi = cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR)); |
cfdda9d7 | 150 | req->wr.wr_mid = cpu_to_be32( |
e2ac9628 | 151 | FW_WR_LEN16_V(DIV_ROUND_UP(wr_len, 16))); |
cfdda9d7 | 152 | |
42b6a949 | 153 | req->cmd = cmd; |
d7990b0c | 154 | req->dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V( |
cfdda9d7 SW |
155 | DIV_ROUND_UP(copy_len, T4_ULPTX_MIN_IO))); |
156 | req->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(req->wr), | |
157 | 16)); | |
d7990b0c | 158 | req->lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(addr + i * 3)); |
cfdda9d7 SW |
159 | |
160 | sc = (struct ulptx_idata *)(req + 1); | |
d7990b0c | 161 | sc->cmd_more = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM)); |
cfdda9d7 SW |
162 | sc->len = cpu_to_be32(roundup(copy_len, T4_ULPTX_MIN_IO)); |
163 | ||
164 | to_dp = (u8 *)(sc + 1); | |
165 | from_dp = (u8 *)data + i * C4IW_MAX_INLINE_SIZE; | |
166 | if (data) | |
167 | memcpy(to_dp, from_dp, copy_len); | |
168 | else | |
169 | memset(to_dp, 0, copy_len); | |
170 | if (copy_len % T4_ULPTX_MIN_IO) | |
171 | memset(to_dp + copy_len, 0, T4_ULPTX_MIN_IO - | |
172 | (copy_len % T4_ULPTX_MIN_IO)); | |
173 | ret = c4iw_ofld_send(rdev, skb); | |
174 | if (ret) | |
175 | return ret; | |
176 | len -= C4IW_MAX_INLINE_SIZE; | |
177 | } | |
178 | ||
aadc4df3 | 179 | ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__); |
cfdda9d7 SW |
180 | return ret; |
181 | } | |
182 | ||
c00850dd | 183 | static int _c4iw_write_mem_dma(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data) |
42b6a949 VP |
184 | { |
185 | u32 remain = len; | |
186 | u32 dmalen; | |
187 | int ret = 0; | |
0e5eca79 VP |
188 | dma_addr_t daddr; |
189 | dma_addr_t save; | |
190 | ||
191 | daddr = dma_map_single(&rdev->lldi.pdev->dev, data, len, DMA_TO_DEVICE); | |
192 | if (dma_mapping_error(&rdev->lldi.pdev->dev, daddr)) | |
193 | return -1; | |
194 | save = daddr; | |
42b6a949 VP |
195 | |
196 | while (remain > inline_threshold) { | |
197 | if (remain < T4_ULPTX_MAX_DMA) { | |
198 | if (remain & ~T4_ULPTX_MIN_IO) | |
199 | dmalen = remain & ~(T4_ULPTX_MIN_IO-1); | |
200 | else | |
201 | dmalen = remain; | |
202 | } else | |
203 | dmalen = T4_ULPTX_MAX_DMA; | |
204 | remain -= dmalen; | |
0e5eca79 | 205 | ret = _c4iw_write_mem_dma_aligned(rdev, addr, dmalen, daddr, |
42b6a949 VP |
206 | !remain); |
207 | if (ret) | |
208 | goto out; | |
209 | addr += dmalen >> 5; | |
210 | data += dmalen; | |
0e5eca79 | 211 | daddr += dmalen; |
42b6a949 VP |
212 | } |
213 | if (remain) | |
214 | ret = _c4iw_write_mem_inline(rdev, addr, remain, data); | |
215 | out: | |
0e5eca79 | 216 | dma_unmap_single(&rdev->lldi.pdev->dev, save, len, DMA_TO_DEVICE); |
42b6a949 VP |
217 | return ret; |
218 | } | |
219 | ||
220 | /* | |
221 | * write len bytes of data into addr (32B aligned address) | |
222 | * If data is NULL, clear len byte of memory to zero. | |
223 | */ | |
224 | static int write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, | |
225 | void *data) | |
226 | { | |
227 | if (is_t5(rdev->lldi.adapter_type) && use_dsgl) { | |
0e5eca79 VP |
228 | if (len > inline_threshold) { |
229 | if (_c4iw_write_mem_dma(rdev, addr, len, data)) { | |
230 | printk_ratelimited(KERN_WARNING | |
231 | "%s: dma map" | |
232 | " failure (non fatal)\n", | |
233 | pci_name(rdev->lldi.pdev)); | |
234 | return _c4iw_write_mem_inline(rdev, addr, len, | |
235 | data); | |
236 | } else | |
237 | return 0; | |
238 | } else | |
42b6a949 VP |
239 | return _c4iw_write_mem_inline(rdev, addr, len, data); |
240 | } else | |
241 | return _c4iw_write_mem_inline(rdev, addr, len, data); | |
242 | } | |
243 | ||
cfdda9d7 SW |
244 | /* |
245 | * Build and write a TPT entry. | |
246 | * IN: stag key, pdid, perm, bind_enabled, zbva, to, len, page_size, | |
247 | * pbl_size and pbl_addr | |
248 | * OUT: stag index | |
249 | */ | |
250 | static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, | |
251 | u32 *stag, u8 stag_state, u32 pdid, | |
252 | enum fw_ri_stag_type type, enum fw_ri_mem_perms perm, | |
253 | int bind_enabled, u32 zbva, u64 to, | |
254 | u64 len, u8 page_size, u32 pbl_size, u32 pbl_addr) | |
255 | { | |
256 | int err; | |
257 | struct fw_ri_tpte tpt; | |
258 | u32 stag_idx; | |
259 | static atomic_t key; | |
260 | ||
261 | if (c4iw_fatal_error(rdev)) | |
262 | return -EIO; | |
263 | ||
264 | stag_state = stag_state > 0; | |
265 | stag_idx = (*stag) >> 8; | |
266 | ||
267 | if ((!reset_tpt_entry) && (*stag == T4_STAG_UNSET)) { | |
ec3eead2 | 268 | stag_idx = c4iw_get_resource(&rdev->resource.tpt_table); |
98a3e879 SW |
269 | if (!stag_idx) { |
270 | mutex_lock(&rdev->stats.lock); | |
271 | rdev->stats.stag.fail++; | |
272 | mutex_unlock(&rdev->stats.lock); | |
cfdda9d7 | 273 | return -ENOMEM; |
98a3e879 | 274 | } |
8d81ef34 VP |
275 | mutex_lock(&rdev->stats.lock); |
276 | rdev->stats.stag.cur += 32; | |
277 | if (rdev->stats.stag.cur > rdev->stats.stag.max) | |
278 | rdev->stats.stag.max = rdev->stats.stag.cur; | |
279 | mutex_unlock(&rdev->stats.lock); | |
cfdda9d7 SW |
280 | *stag = (stag_idx << 8) | (atomic_inc_return(&key) & 0xff); |
281 | } | |
282 | PDBG("%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n", | |
283 | __func__, stag_state, type, pdid, stag_idx); | |
284 | ||
285 | /* write TPT entry */ | |
286 | if (reset_tpt_entry) | |
287 | memset(&tpt, 0, sizeof(tpt)); | |
288 | else { | |
cf7fe64a HS |
289 | tpt.valid_to_pdid = cpu_to_be32(FW_RI_TPTE_VALID_F | |
290 | FW_RI_TPTE_STAGKEY_V((*stag & FW_RI_TPTE_STAGKEY_M)) | | |
291 | FW_RI_TPTE_STAGSTATE_V(stag_state) | | |
292 | FW_RI_TPTE_STAGTYPE_V(type) | FW_RI_TPTE_PDID_V(pdid)); | |
293 | tpt.locread_to_qpid = cpu_to_be32(FW_RI_TPTE_PERM_V(perm) | | |
294 | (bind_enabled ? FW_RI_TPTE_MWBINDEN_F : 0) | | |
295 | FW_RI_TPTE_ADDRTYPE_V((zbva ? FW_RI_ZERO_BASED_TO : | |
cfdda9d7 | 296 | FW_RI_VA_BASED_TO))| |
cf7fe64a | 297 | FW_RI_TPTE_PS_V(page_size)); |
cfdda9d7 | 298 | tpt.nosnoop_pbladdr = !pbl_size ? 0 : cpu_to_be32( |
cf7fe64a | 299 | FW_RI_TPTE_PBLADDR_V(PBL_OFF(rdev, pbl_addr)>>3)); |
cfdda9d7 SW |
300 | tpt.len_lo = cpu_to_be32((u32)(len & 0xffffffffUL)); |
301 | tpt.va_hi = cpu_to_be32((u32)(to >> 32)); | |
302 | tpt.va_lo_fbo = cpu_to_be32((u32)(to & 0xffffffffUL)); | |
303 | tpt.dca_mwbcnt_pstag = cpu_to_be32(0); | |
304 | tpt.len_hi = cpu_to_be32((u32)(len >> 32)); | |
305 | } | |
306 | err = write_adapter_mem(rdev, stag_idx + | |
307 | (rdev->lldi.vr->stag.start >> 5), | |
308 | sizeof(tpt), &tpt); | |
309 | ||
8d81ef34 | 310 | if (reset_tpt_entry) { |
ec3eead2 | 311 | c4iw_put_resource(&rdev->resource.tpt_table, stag_idx); |
8d81ef34 VP |
312 | mutex_lock(&rdev->stats.lock); |
313 | rdev->stats.stag.cur -= 32; | |
314 | mutex_unlock(&rdev->stats.lock); | |
315 | } | |
cfdda9d7 SW |
316 | return err; |
317 | } | |
318 | ||
319 | static int write_pbl(struct c4iw_rdev *rdev, __be64 *pbl, | |
320 | u32 pbl_addr, u32 pbl_size) | |
321 | { | |
322 | int err; | |
323 | ||
324 | PDBG("%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d\n", | |
325 | __func__, pbl_addr, rdev->lldi.vr->pbl.start, | |
326 | pbl_size); | |
327 | ||
328 | err = write_adapter_mem(rdev, pbl_addr >> 5, pbl_size << 3, pbl); | |
329 | return err; | |
330 | } | |
331 | ||
332 | static int dereg_mem(struct c4iw_rdev *rdev, u32 stag, u32 pbl_size, | |
333 | u32 pbl_addr) | |
334 | { | |
335 | return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, | |
336 | pbl_size, pbl_addr); | |
337 | } | |
338 | ||
339 | static int allocate_window(struct c4iw_rdev *rdev, u32 * stag, u32 pdid) | |
340 | { | |
341 | *stag = T4_STAG_UNSET; | |
342 | return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_MW, 0, 0, 0, | |
343 | 0UL, 0, 0, 0, 0); | |
344 | } | |
345 | ||
346 | static int deallocate_window(struct c4iw_rdev *rdev, u32 stag) | |
347 | { | |
348 | return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 0, | |
349 | 0); | |
350 | } | |
351 | ||
352 | static int allocate_stag(struct c4iw_rdev *rdev, u32 *stag, u32 pdid, | |
353 | u32 pbl_size, u32 pbl_addr) | |
354 | { | |
355 | *stag = T4_STAG_UNSET; | |
356 | return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_NSMR, 0, 0, 0, | |
357 | 0UL, 0, 0, pbl_size, pbl_addr); | |
358 | } | |
359 | ||
360 | static int finish_mem_reg(struct c4iw_mr *mhp, u32 stag) | |
361 | { | |
362 | u32 mmid; | |
363 | ||
364 | mhp->attr.state = 1; | |
365 | mhp->attr.stag = stag; | |
366 | mmid = stag >> 8; | |
367 | mhp->ibmr.rkey = mhp->ibmr.lkey = stag; | |
368 | PDBG("%s mmid 0x%x mhp %p\n", __func__, mmid, mhp); | |
369 | return insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid); | |
370 | } | |
371 | ||
372 | static int register_mem(struct c4iw_dev *rhp, struct c4iw_pd *php, | |
373 | struct c4iw_mr *mhp, int shift) | |
374 | { | |
375 | u32 stag = T4_STAG_UNSET; | |
376 | int ret; | |
377 | ||
378 | ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid, | |
123bc2a2 PK |
379 | FW_RI_STAG_NSMR, mhp->attr.len ? |
380 | mhp->attr.perms : 0, | |
cfdda9d7 | 381 | mhp->attr.mw_bind_enable, mhp->attr.zbva, |
123bc2a2 PK |
382 | mhp->attr.va_fbo, mhp->attr.len ? |
383 | mhp->attr.len : -1, shift - 12, | |
cfdda9d7 SW |
384 | mhp->attr.pbl_size, mhp->attr.pbl_addr); |
385 | if (ret) | |
386 | return ret; | |
387 | ||
388 | ret = finish_mem_reg(mhp, stag); | |
389 | if (ret) | |
390 | dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, | |
391 | mhp->attr.pbl_addr); | |
392 | return ret; | |
393 | } | |
394 | ||
cfdda9d7 SW |
395 | static int alloc_pbl(struct c4iw_mr *mhp, int npages) |
396 | { | |
397 | mhp->attr.pbl_addr = c4iw_pblpool_alloc(&mhp->rhp->rdev, | |
398 | npages << 3); | |
399 | ||
400 | if (!mhp->attr.pbl_addr) | |
401 | return -ENOMEM; | |
402 | ||
403 | mhp->attr.pbl_size = npages; | |
404 | ||
405 | return 0; | |
406 | } | |
407 | ||
cfdda9d7 SW |
408 | struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc) |
409 | { | |
410 | struct c4iw_dev *rhp; | |
411 | struct c4iw_pd *php; | |
412 | struct c4iw_mr *mhp; | |
413 | int ret; | |
414 | u32 stag = T4_STAG_UNSET; | |
415 | ||
416 | PDBG("%s ib_pd %p\n", __func__, pd); | |
417 | php = to_c4iw_pd(pd); | |
418 | rhp = php->rhp; | |
419 | ||
420 | mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); | |
421 | if (!mhp) | |
422 | return ERR_PTR(-ENOMEM); | |
423 | ||
424 | mhp->rhp = rhp; | |
425 | mhp->attr.pdid = php->pdid; | |
426 | mhp->attr.perms = c4iw_ib_to_tpt_access(acc); | |
427 | mhp->attr.mw_bind_enable = (acc&IB_ACCESS_MW_BIND) == IB_ACCESS_MW_BIND; | |
428 | mhp->attr.zbva = 0; | |
429 | mhp->attr.va_fbo = 0; | |
430 | mhp->attr.page_size = 0; | |
6198dd8d | 431 | mhp->attr.len = ~0ULL; |
cfdda9d7 SW |
432 | mhp->attr.pbl_size = 0; |
433 | ||
434 | ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, php->pdid, | |
435 | FW_RI_STAG_NSMR, mhp->attr.perms, | |
6198dd8d | 436 | mhp->attr.mw_bind_enable, 0, 0, ~0ULL, 0, 0, 0); |
cfdda9d7 SW |
437 | if (ret) |
438 | goto err1; | |
439 | ||
440 | ret = finish_mem_reg(mhp, stag); | |
441 | if (ret) | |
442 | goto err2; | |
443 | return &mhp->ibmr; | |
444 | err2: | |
445 | dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, | |
446 | mhp->attr.pbl_addr); | |
447 | err1: | |
448 | kfree(mhp); | |
449 | return ERR_PTR(ret); | |
450 | } | |
451 | ||
452 | struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, | |
453 | u64 virt, int acc, struct ib_udata *udata) | |
454 | { | |
455 | __be64 *pages; | |
456 | int shift, n, len; | |
eeb8461e | 457 | int i, k, entry; |
cfdda9d7 | 458 | int err = 0; |
eeb8461e | 459 | struct scatterlist *sg; |
cfdda9d7 SW |
460 | struct c4iw_dev *rhp; |
461 | struct c4iw_pd *php; | |
462 | struct c4iw_mr *mhp; | |
463 | ||
464 | PDBG("%s ib_pd %p\n", __func__, pd); | |
465 | ||
466 | if (length == ~0ULL) | |
467 | return ERR_PTR(-EINVAL); | |
468 | ||
469 | if ((length + start) < start) | |
470 | return ERR_PTR(-EINVAL); | |
471 | ||
472 | php = to_c4iw_pd(pd); | |
473 | rhp = php->rhp; | |
2550a88d HS |
474 | |
475 | if (mr_exceeds_hw_limits(rhp, length)) | |
476 | return ERR_PTR(-EINVAL); | |
477 | ||
cfdda9d7 SW |
478 | mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); |
479 | if (!mhp) | |
480 | return ERR_PTR(-ENOMEM); | |
481 | ||
482 | mhp->rhp = rhp; | |
483 | ||
484 | mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0); | |
485 | if (IS_ERR(mhp->umem)) { | |
486 | err = PTR_ERR(mhp->umem); | |
487 | kfree(mhp); | |
488 | return ERR_PTR(err); | |
489 | } | |
490 | ||
491 | shift = ffs(mhp->umem->page_size) - 1; | |
492 | ||
eeb8461e | 493 | n = mhp->umem->nmap; |
cfdda9d7 SW |
494 | err = alloc_pbl(mhp, n); |
495 | if (err) | |
496 | goto err; | |
497 | ||
498 | pages = (__be64 *) __get_free_page(GFP_KERNEL); | |
499 | if (!pages) { | |
500 | err = -ENOMEM; | |
501 | goto err_pbl; | |
502 | } | |
503 | ||
504 | i = n = 0; | |
505 | ||
eeb8461e YH |
506 | for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) { |
507 | len = sg_dma_len(sg) >> shift; | |
508 | for (k = 0; k < len; ++k) { | |
509 | pages[i++] = cpu_to_be64(sg_dma_address(sg) + | |
510 | mhp->umem->page_size * k); | |
511 | if (i == PAGE_SIZE / sizeof *pages) { | |
512 | err = write_pbl(&mhp->rhp->rdev, | |
513 | pages, | |
514 | mhp->attr.pbl_addr + (n << 3), i); | |
515 | if (err) | |
516 | goto pbl_done; | |
517 | n += i; | |
518 | i = 0; | |
cfdda9d7 SW |
519 | } |
520 | } | |
eeb8461e | 521 | } |
cfdda9d7 SW |
522 | |
523 | if (i) | |
524 | err = write_pbl(&mhp->rhp->rdev, pages, | |
525 | mhp->attr.pbl_addr + (n << 3), i); | |
526 | ||
527 | pbl_done: | |
528 | free_page((unsigned long) pages); | |
529 | if (err) | |
530 | goto err_pbl; | |
531 | ||
532 | mhp->attr.pdid = php->pdid; | |
533 | mhp->attr.zbva = 0; | |
534 | mhp->attr.perms = c4iw_ib_to_tpt_access(acc); | |
535 | mhp->attr.va_fbo = virt; | |
536 | mhp->attr.page_size = shift - 12; | |
301c2c3f | 537 | mhp->attr.len = length; |
cfdda9d7 SW |
538 | |
539 | err = register_mem(rhp, php, mhp, shift); | |
540 | if (err) | |
541 | goto err_pbl; | |
542 | ||
543 | return &mhp->ibmr; | |
544 | ||
545 | err_pbl: | |
546 | c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, | |
547 | mhp->attr.pbl_size << 3); | |
548 | ||
549 | err: | |
550 | ib_umem_release(mhp->umem); | |
551 | kfree(mhp); | |
552 | return ERR_PTR(err); | |
553 | } | |
554 | ||
7083e42e | 555 | struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type) |
cfdda9d7 SW |
556 | { |
557 | struct c4iw_dev *rhp; | |
558 | struct c4iw_pd *php; | |
559 | struct c4iw_mw *mhp; | |
560 | u32 mmid; | |
561 | u32 stag = 0; | |
562 | int ret; | |
563 | ||
7083e42e SM |
564 | if (type != IB_MW_TYPE_1) |
565 | return ERR_PTR(-EINVAL); | |
566 | ||
cfdda9d7 SW |
567 | php = to_c4iw_pd(pd); |
568 | rhp = php->rhp; | |
569 | mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); | |
570 | if (!mhp) | |
571 | return ERR_PTR(-ENOMEM); | |
572 | ret = allocate_window(&rhp->rdev, &stag, php->pdid); | |
573 | if (ret) { | |
574 | kfree(mhp); | |
575 | return ERR_PTR(ret); | |
576 | } | |
577 | mhp->rhp = rhp; | |
578 | mhp->attr.pdid = php->pdid; | |
579 | mhp->attr.type = FW_RI_STAG_MW; | |
580 | mhp->attr.stag = stag; | |
581 | mmid = (stag) >> 8; | |
582 | mhp->ibmw.rkey = stag; | |
583 | if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { | |
584 | deallocate_window(&rhp->rdev, mhp->attr.stag); | |
585 | kfree(mhp); | |
586 | return ERR_PTR(-ENOMEM); | |
587 | } | |
588 | PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag); | |
589 | return &(mhp->ibmw); | |
590 | } | |
591 | ||
592 | int c4iw_dealloc_mw(struct ib_mw *mw) | |
593 | { | |
594 | struct c4iw_dev *rhp; | |
595 | struct c4iw_mw *mhp; | |
596 | u32 mmid; | |
597 | ||
598 | mhp = to_c4iw_mw(mw); | |
599 | rhp = mhp->rhp; | |
600 | mmid = (mw->rkey) >> 8; | |
cfdda9d7 | 601 | remove_handle(rhp, &rhp->mmidr, mmid); |
ec3eead2 | 602 | deallocate_window(&rhp->rdev, mhp->attr.stag); |
cfdda9d7 SW |
603 | kfree(mhp); |
604 | PDBG("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp); | |
605 | return 0; | |
606 | } | |
607 | ||
a2164034 SG |
608 | struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, |
609 | enum ib_mr_type mr_type, | |
610 | u32 max_num_sg) | |
cfdda9d7 SW |
611 | { |
612 | struct c4iw_dev *rhp; | |
613 | struct c4iw_pd *php; | |
614 | struct c4iw_mr *mhp; | |
615 | u32 mmid; | |
616 | u32 stag = 0; | |
617 | int ret = 0; | |
8376b86d | 618 | int length = roundup(max_num_sg * sizeof(u64), 32); |
cfdda9d7 | 619 | |
ee30f7d5 H |
620 | php = to_c4iw_pd(pd); |
621 | rhp = php->rhp; | |
622 | ||
a2164034 | 623 | if (mr_type != IB_MR_TYPE_MEM_REG || |
ee30f7d5 H |
624 | max_num_sg > t4_max_fr_depth(&rhp->rdev.lldi.ulptx_memwrite_dsgl && |
625 | use_dsgl)) | |
a2164034 SG |
626 | return ERR_PTR(-EINVAL); |
627 | ||
cfdda9d7 | 628 | mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); |
841dba9a SW |
629 | if (!mhp) { |
630 | ret = -ENOMEM; | |
cfdda9d7 | 631 | goto err; |
841dba9a | 632 | } |
cfdda9d7 | 633 | |
8376b86d SG |
634 | mhp->mpl = dma_alloc_coherent(&rhp->rdev.lldi.pdev->dev, |
635 | length, &mhp->mpl_addr, GFP_KERNEL); | |
636 | if (!mhp->mpl) { | |
637 | ret = -ENOMEM; | |
638 | goto err_mpl; | |
639 | } | |
640 | mhp->max_mpl_len = length; | |
641 | ||
cfdda9d7 | 642 | mhp->rhp = rhp; |
a2164034 | 643 | ret = alloc_pbl(mhp, max_num_sg); |
cfdda9d7 SW |
644 | if (ret) |
645 | goto err1; | |
a2164034 | 646 | mhp->attr.pbl_size = max_num_sg; |
cfdda9d7 SW |
647 | ret = allocate_stag(&rhp->rdev, &stag, php->pdid, |
648 | mhp->attr.pbl_size, mhp->attr.pbl_addr); | |
649 | if (ret) | |
650 | goto err2; | |
651 | mhp->attr.pdid = php->pdid; | |
652 | mhp->attr.type = FW_RI_STAG_NSMR; | |
653 | mhp->attr.stag = stag; | |
654 | mhp->attr.state = 1; | |
655 | mmid = (stag) >> 8; | |
656 | mhp->ibmr.rkey = mhp->ibmr.lkey = stag; | |
841dba9a SW |
657 | if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) { |
658 | ret = -ENOMEM; | |
cfdda9d7 | 659 | goto err3; |
841dba9a | 660 | } |
cfdda9d7 SW |
661 | |
662 | PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag); | |
663 | return &(mhp->ibmr); | |
664 | err3: | |
665 | dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size, | |
666 | mhp->attr.pbl_addr); | |
667 | err2: | |
668 | c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, | |
669 | mhp->attr.pbl_size << 3); | |
670 | err1: | |
8376b86d SG |
671 | dma_free_coherent(&mhp->rhp->rdev.lldi.pdev->dev, |
672 | mhp->max_mpl_len, mhp->mpl, mhp->mpl_addr); | |
673 | err_mpl: | |
cfdda9d7 SW |
674 | kfree(mhp); |
675 | err: | |
676 | return ERR_PTR(ret); | |
677 | } | |
678 | ||
8376b86d SG |
679 | static int c4iw_set_page(struct ib_mr *ibmr, u64 addr) |
680 | { | |
681 | struct c4iw_mr *mhp = to_c4iw_mr(ibmr); | |
682 | ||
683 | if (unlikely(mhp->mpl_len == mhp->max_mpl_len)) | |
684 | return -ENOMEM; | |
685 | ||
686 | mhp->mpl[mhp->mpl_len++] = addr; | |
687 | ||
688 | return 0; | |
689 | } | |
690 | ||
691 | int c4iw_map_mr_sg(struct ib_mr *ibmr, | |
692 | struct scatterlist *sg, | |
693 | int sg_nents) | |
694 | { | |
695 | struct c4iw_mr *mhp = to_c4iw_mr(ibmr); | |
696 | ||
697 | mhp->mpl_len = 0; | |
698 | ||
699 | return ib_sg_to_pages(ibmr, sg, sg_nents, c4iw_set_page); | |
700 | } | |
701 | ||
cfdda9d7 SW |
702 | int c4iw_dereg_mr(struct ib_mr *ib_mr) |
703 | { | |
704 | struct c4iw_dev *rhp; | |
705 | struct c4iw_mr *mhp; | |
706 | u32 mmid; | |
707 | ||
708 | PDBG("%s ib_mr %p\n", __func__, ib_mr); | |
cfdda9d7 SW |
709 | |
710 | mhp = to_c4iw_mr(ib_mr); | |
711 | rhp = mhp->rhp; | |
712 | mmid = mhp->attr.stag >> 8; | |
ec3eead2 | 713 | remove_handle(rhp, &rhp->mmidr, mmid); |
8376b86d SG |
714 | if (mhp->mpl) |
715 | dma_free_coherent(&mhp->rhp->rdev.lldi.pdev->dev, | |
716 | mhp->max_mpl_len, mhp->mpl, mhp->mpl_addr); | |
cfdda9d7 SW |
717 | dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size, |
718 | mhp->attr.pbl_addr); | |
719 | if (mhp->attr.pbl_size) | |
720 | c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr, | |
721 | mhp->attr.pbl_size << 3); | |
cfdda9d7 SW |
722 | if (mhp->kva) |
723 | kfree((void *) (unsigned long) mhp->kva); | |
724 | if (mhp->umem) | |
725 | ib_umem_release(mhp->umem); | |
726 | PDBG("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp); | |
727 | kfree(mhp); | |
728 | return 0; | |
729 | } |