IB/iser: Make fastreg pool cache friendly
[deliverable/linux.git] / drivers / infiniband / ulp / iser / iser_memory.c
CommitLineData
6461f64a
OG
1/*
2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
3ee07d27 3 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
6461f64a
OG
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
6461f64a
OG
32 */
33#include <linux/module.h>
34#include <linux/kernel.h>
35#include <linux/slab.h>
36#include <linux/mm.h>
a1f8e7f7 37#include <linux/highmem.h>
6461f64a
OG
38#include <linux/scatterlist.h>
39
40#include "iscsi_iser.h"
41
42#define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */
8dfa0876 43
bd8b944e
SG
44struct fast_reg_descriptor *
45iser_reg_desc_get(struct ib_conn *ib_conn)
46{
47 struct fast_reg_descriptor *desc;
48 unsigned long flags;
49
50 spin_lock_irqsave(&ib_conn->lock, flags);
51 desc = list_first_entry(&ib_conn->fastreg.pool,
52 struct fast_reg_descriptor, list);
53 list_del(&desc->list);
54 spin_unlock_irqrestore(&ib_conn->lock, flags);
55
56 return desc;
57}
58
59void
60iser_reg_desc_put(struct ib_conn *ib_conn,
61 struct fast_reg_descriptor *desc)
62{
63 unsigned long flags;
64
65 spin_lock_irqsave(&ib_conn->lock, flags);
8b95aa2c 66 list_add(&desc->list, &ib_conn->fastreg.pool);
bd8b944e
SG
67 spin_unlock_irqrestore(&ib_conn->lock, flags);
68}
69
6461f64a
OG
70/**
71 * iser_start_rdma_unaligned_sg
72 */
2261ec3d 73static int iser_start_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
5f588e3d 74 struct iser_data_buf *data,
41179e2d 75 enum iser_data_dir cmd_dir)
6461f64a 76{
a4ee3539 77 struct ib_device *dev = iser_task->iser_conn->ib_conn.device->ib_device;
e3784bd1 78 struct scatterlist *sgl = data->sg;
5f588e3d 79 struct scatterlist *sg;
6461f64a 80 char *mem = NULL;
ecc3993a 81 unsigned long cmd_data_len = data->data_len;
5f588e3d
SG
82 int dma_nents, i;
83
6461f64a 84 if (cmd_data_len > ISER_KMALLOC_THRESHOLD)
528f4e8c 85 mem = (void *)__get_free_pages(GFP_ATOMIC,
f0d1b0b3 86 ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT);
6461f64a 87 else
528f4e8c 88 mem = kmalloc(cmd_data_len, GFP_ATOMIC);
6461f64a
OG
89
90 if (mem == NULL) {
91 iser_err("Failed to allocate mem size %d %d for copying sglist\n",
5f588e3d 92 data->size, (int)cmd_data_len);
6461f64a
OG
93 return -ENOMEM;
94 }
95
96 if (cmd_dir == ISER_DIR_OUT) {
97 /* copy the unaligned sg the buffer which is used for RDMA */
6461f64a
OG
98 char *p, *from;
99
e3784bd1 100 sgl = data->sg;
53d412fc
JA
101 p = mem;
102 for_each_sg(sgl, sg, data->size, i) {
2a156d09 103 from = kmap_atomic(sg_page(sg));
6461f64a 104 memcpy(p,
53d412fc
JA
105 from + sg->offset,
106 sg->length);
2a156d09 107 kunmap_atomic(from);
53d412fc 108 p += sg->length;
6461f64a
OG
109 }
110 }
111
e3784bd1
SG
112 sg_init_one(&data->sg_single, mem, cmd_data_len);
113 data->orig_sg = data->sg;
114 data->sg = &data->sg_single;
115 data->copy_buf = mem;
116 dma_nents = ib_dma_map_sg(dev, data->sg, 1,
5180311f
RC
117 (cmd_dir == ISER_DIR_OUT) ?
118 DMA_TO_DEVICE : DMA_FROM_DEVICE);
6461f64a
OG
119 BUG_ON(dma_nents == 0);
120
e3784bd1 121 data->dma_nents = dma_nents;
5f588e3d 122
6461f64a
OG
123 return 0;
124}
125
126/**
127 * iser_finalize_rdma_unaligned_sg
128 */
9a8b08fa 129
2261ec3d 130void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
9a8b08fa 131 struct iser_data_buf *data,
9a8b08fa 132 enum iser_data_dir cmd_dir)
6461f64a 133{
5180311f 134 struct ib_device *dev;
6461f64a
OG
135 unsigned long cmd_data_len;
136
a4ee3539 137 dev = iser_task->iser_conn->ib_conn.device->ib_device;
6461f64a 138
e3784bd1 139 ib_dma_unmap_sg(dev, data->sg, 1,
5180311f
RC
140 (cmd_dir == ISER_DIR_OUT) ?
141 DMA_TO_DEVICE : DMA_FROM_DEVICE);
6461f64a
OG
142
143 if (cmd_dir == ISER_DIR_IN) {
144 char *mem;
53d412fc 145 struct scatterlist *sgl, *sg;
6461f64a
OG
146 unsigned char *p, *to;
147 unsigned int sg_size;
148 int i;
149
150 /* copy back read RDMA to unaligned sg */
e3784bd1 151 mem = data->copy_buf;
6461f64a 152
e3784bd1 153 sgl = data->sg;
9a8b08fa 154 sg_size = data->size;
6461f64a 155
53d412fc
JA
156 p = mem;
157 for_each_sg(sgl, sg, sg_size, i) {
2a156d09 158 to = kmap_atomic(sg_page(sg));
53d412fc 159 memcpy(to + sg->offset,
6461f64a 160 p,
53d412fc 161 sg->length);
2a156d09 162 kunmap_atomic(to);
53d412fc 163 p += sg->length;
6461f64a
OG
164 }
165 }
166
9a8b08fa 167 cmd_data_len = data->data_len;
6461f64a
OG
168
169 if (cmd_data_len > ISER_KMALLOC_THRESHOLD)
e3784bd1 170 free_pages((unsigned long)data->copy_buf,
f0d1b0b3 171 ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT);
6461f64a 172 else
e3784bd1 173 kfree(data->copy_buf);
6461f64a 174
e3784bd1 175 data->copy_buf = NULL;
6461f64a
OG
176}
177
c1ccaf24
OG
178#define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & ~MASK_4K) == 0)
179
6461f64a
OG
180/**
181 * iser_sg_to_page_vec - Translates scatterlist entries to physical addresses
182 * and returns the length of resulting physical address array (may be less than
183 * the original due to possible compaction).
184 *
185 * we build a "page vec" under the assumption that the SG meets the RDMA
186 * alignment requirements. Other then the first and last SG elements, all
187 * the "internal" elements can be compacted into a list whose elements are
188 * dma addresses of physical pages. The code supports also the weird case
189 * where --few fragments of the same page-- are present in the SG as
190 * consecutive elements. Also, it handles one entry SG.
191 */
c1ccaf24 192
6461f64a 193static int iser_sg_to_page_vec(struct iser_data_buf *data,
919fc274
SG
194 struct ib_device *ibdev, u64 *pages,
195 int *offset, int *data_size)
6461f64a 196{
e3784bd1 197 struct scatterlist *sg, *sgl = data->sg;
c1ccaf24 198 u64 start_addr, end_addr, page, chunk_start = 0;
6461f64a 199 unsigned long total_sz = 0;
c1ccaf24
OG
200 unsigned int dma_len;
201 int i, new_chunk, cur_page, last_ent = data->dma_nents - 1;
6461f64a
OG
202
203 /* compute the offset of first element */
919fc274 204 *offset = (u64) sgl[0].offset & ~MASK_4K;
6461f64a 205
c1ccaf24
OG
206 new_chunk = 1;
207 cur_page = 0;
53d412fc 208 for_each_sg(sgl, sg, data->dma_nents, i) {
c1ccaf24
OG
209 start_addr = ib_sg_dma_address(ibdev, sg);
210 if (new_chunk)
211 chunk_start = start_addr;
212 dma_len = ib_sg_dma_len(ibdev, sg);
213 end_addr = start_addr + dma_len;
5180311f 214 total_sz += dma_len;
6461f64a 215
c1ccaf24
OG
216 /* collect page fragments until aligned or end of SG list */
217 if (!IS_4K_ALIGNED(end_addr) && i < last_ent) {
218 new_chunk = 0;
219 continue;
6461f64a 220 }
c1ccaf24
OG
221 new_chunk = 1;
222
223 /* address of the first page in the contiguous chunk;
224 masking relevant for the very first SG entry,
225 which might be unaligned */
226 page = chunk_start & MASK_4K;
227 do {
919fc274 228 pages[cur_page++] = page;
8dfa0876 229 page += SIZE_4K;
c1ccaf24 230 } while (page < end_addr);
6461f64a 231 }
c1ccaf24 232
919fc274
SG
233 *data_size = total_sz;
234 iser_dbg("page_vec->data_size:%d cur_page %d\n",
235 *data_size, cur_page);
6461f64a
OG
236 return cur_page;
237}
238
6461f64a
OG
239
240/**
241 * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned
242 * for RDMA sub-list of a scatter-gather list of memory buffers, and returns
243 * the number of entries which are aligned correctly. Supports the case where
244 * consecutive SG elements are actually fragments of the same physcial page.
245 */
c1ccaf24
OG
246static int iser_data_buf_aligned_len(struct iser_data_buf *data,
247 struct ib_device *ibdev)
6461f64a 248{
e3784bd1 249 struct scatterlist *sg, *sgl, *next_sg = NULL;
c1ccaf24
OG
250 u64 start_addr, end_addr;
251 int i, ret_len, start_check = 0;
252
253 if (data->dma_nents == 1)
254 return 1;
6461f64a 255
e3784bd1 256 sgl = data->sg;
c1ccaf24 257 start_addr = ib_sg_dma_address(ibdev, sgl);
6461f64a 258
53d412fc 259 for_each_sg(sgl, sg, data->dma_nents, i) {
c1ccaf24
OG
260 if (start_check && !IS_4K_ALIGNED(start_addr))
261 break;
262
263 next_sg = sg_next(sg);
264 if (!next_sg)
265 break;
266
267 end_addr = start_addr + ib_sg_dma_len(ibdev, sg);
268 start_addr = ib_sg_dma_address(ibdev, next_sg);
269
270 if (end_addr == start_addr) {
271 start_check = 0;
272 continue;
273 } else
274 start_check = 1;
275
276 if (!IS_4K_ALIGNED(end_addr))
277 break;
6461f64a 278 }
c1ccaf24 279 ret_len = (next_sg) ? i : i+1;
6461f64a
OG
280 iser_dbg("Found %d aligned entries out of %d in sg:0x%p\n",
281 ret_len, data->dma_nents, data);
282 return ret_len;
283}
284
5180311f
RC
285static void iser_data_buf_dump(struct iser_data_buf *data,
286 struct ib_device *ibdev)
6461f64a 287{
53d412fc 288 struct scatterlist *sg;
6461f64a
OG
289 int i;
290
e3784bd1 291 for_each_sg(data->sg, sg, data->dma_nents, i)
f91424cf 292 iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p "
e981f1d4 293 "off:0x%x sz:0x%x dma_len:0x%x\n",
53d412fc 294 i, (unsigned long)ib_sg_dma_address(ibdev, sg),
45711f1a 295 sg_page(sg), sg->offset,
53d412fc 296 sg->length, ib_sg_dma_len(ibdev, sg));
6461f64a
OG
297}
298
299static void iser_dump_page_vec(struct iser_page_vec *page_vec)
300{
301 int i;
302
303 iser_err("page vec length %d data size %d\n",
304 page_vec->length, page_vec->data_size);
305 for (i = 0; i < page_vec->length; i++)
306 iser_err("%d %lx\n",i,(unsigned long)page_vec->pages[i]);
307}
308
2261ec3d
MC
309int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
310 struct iser_data_buf *data,
311 enum iser_data_dir iser_dir,
312 enum dma_data_direction dma_dir)
74a20780 313{
5180311f 314 struct ib_device *dev;
74a20780 315
2261ec3d 316 iser_task->dir[iser_dir] = 1;
a4ee3539 317 dev = iser_task->iser_conn->ib_conn.device->ib_device;
74a20780 318
e3784bd1 319 data->dma_nents = ib_dma_map_sg(dev, data->sg, data->size, dma_dir);
74a20780
EZ
320 if (data->dma_nents == 0) {
321 iser_err("dma_map_sg failed!!!\n");
322 return -EINVAL;
323 }
324 return 0;
325}
326
9a8b08fa 327void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
c6c95ef4
RD
328 struct iser_data_buf *data,
329 enum dma_data_direction dir)
74a20780 330{
5180311f 331 struct ib_device *dev;
74a20780 332
a4ee3539 333 dev = iser_task->iser_conn->ib_conn.device->ib_device;
e3784bd1 334 ib_dma_unmap_sg(dev, data->sg, data->size, dir);
74a20780
EZ
335}
336
919fc274 337static int fall_to_bounce_buf(struct iscsi_iser_task *iser_task,
5f588e3d 338 struct iser_data_buf *mem,
919fc274
SG
339 enum iser_data_dir cmd_dir,
340 int aligned_len)
341{
56408325
SG
342 struct iscsi_conn *iscsi_conn = iser_task->iser_conn->iscsi_conn;
343 struct iser_device *device = iser_task->iser_conn->ib_conn.device;
919fc274
SG
344
345 iscsi_conn->fmr_unalign_cnt++;
346 iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n",
347 aligned_len, mem->size);
348
349 if (iser_debug_level > 0)
56408325 350 iser_data_buf_dump(mem, device->ib_device);
919fc274
SG
351
352 /* unmap the command data before accessing it */
c6c95ef4
RD
353 iser_dma_unmap_task_data(iser_task, mem,
354 (cmd_dir == ISER_DIR_OUT) ?
355 DMA_TO_DEVICE : DMA_FROM_DEVICE);
919fc274
SG
356
357 /* allocate copy buf, if we are writing, copy the */
358 /* unaligned scatterlist, dma map the copy */
e3784bd1 359 if (iser_start_rdma_unaligned_sg(iser_task, mem, cmd_dir) != 0)
5f588e3d 360 return -ENOMEM;
919fc274
SG
361
362 return 0;
363}
364
d03e61d0
SG
365/**
366 * iser_reg_page_vec - Register physical memory
367 *
368 * returns: 0 on success, errno code on failure
369 */
370static
f0e35c27
SG
371int iser_reg_page_vec(struct iscsi_iser_task *iser_task,
372 struct iser_data_buf *mem,
d03e61d0 373 struct iser_page_vec *page_vec,
f0e35c27 374 struct iser_mem_reg *mem_reg)
d03e61d0 375{
f0e35c27
SG
376 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
377 struct iser_device *device = ib_conn->device;
378 struct ib_pool_fmr *fmr;
379 int ret, plen;
380
381 plen = iser_sg_to_page_vec(mem, device->ib_device,
382 page_vec->pages,
383 &page_vec->offset,
384 &page_vec->data_size);
385 page_vec->length = plen;
386 if (plen * SIZE_4K < page_vec->data_size) {
387 iser_err("page vec too short to hold this SG\n");
388 iser_data_buf_dump(mem, device->ib_device);
389 iser_dump_page_vec(page_vec);
390 return -EINVAL;
391 }
d03e61d0 392
f0e35c27
SG
393 fmr = ib_fmr_pool_map_phys(ib_conn->fmr.pool,
394 page_vec->pages,
d03e61d0 395 page_vec->length,
f0e35c27
SG
396 page_vec->pages[0]);
397 if (IS_ERR(fmr)) {
398 ret = PTR_ERR(fmr);
399 iser_err("ib_fmr_pool_map_phys failed: %d\n", ret);
400 return ret;
d03e61d0
SG
401 }
402
f0e35c27
SG
403 mem_reg->lkey = fmr->fmr->lkey;
404 mem_reg->rkey = fmr->fmr->rkey;
405 mem_reg->va = page_vec->pages[0] + page_vec->offset;
406 mem_reg->len = page_vec->data_size;
407 mem_reg->mem_h = fmr;
408
d03e61d0
SG
409 return 0;
410}
411
412/**
413 * Unregister (previosuly registered using FMR) memory.
414 * If memory is non-FMR does nothing.
415 */
416void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
417 enum iser_data_dir cmd_dir)
418{
b130eded 419 struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir];
d03e61d0
SG
420 int ret;
421
422 if (!reg->mem_h)
423 return;
424
425 iser_dbg("PHYSICAL Mem.Unregister mem_h %p\n", reg->mem_h);
426
427 ret = ib_fmr_pool_unmap((struct ib_pool_fmr *)reg->mem_h);
428 if (ret)
429 iser_err("ib_fmr_pool_unmap failed %d\n", ret);
430
431 reg->mem_h = NULL;
432}
433
434void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
435 enum iser_data_dir cmd_dir)
436{
b130eded 437 struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir];
d03e61d0 438
bd8b944e 439 if (!reg->mem_h)
d03e61d0
SG
440 return;
441
bd8b944e
SG
442 iser_reg_desc_put(&iser_task->iser_conn->ib_conn,
443 reg->mem_h);
d03e61d0 444 reg->mem_h = NULL;
d03e61d0
SG
445}
446
6461f64a 447/**
e657571b
SG
448 * iser_reg_rdma_mem_fmr - Registers memory intended for RDMA,
449 * using FMR (if possible) obtaining rkey and va
6461f64a
OG
450 *
451 * returns 0 on success, errno code on failure
452 */
e657571b
SG
453int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task,
454 enum iser_data_dir cmd_dir)
6461f64a 455{
a4ee3539
SG
456 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
457 struct iser_device *device = ib_conn->device;
5180311f 458 struct ib_device *ibdev = device->ib_device;
2261ec3d 459 struct iser_data_buf *mem = &iser_task->data[cmd_dir];
b130eded 460 struct iser_mem_reg *mem_reg;
6461f64a
OG
461 int aligned_len;
462 int err;
e981f1d4 463 int i;
d8111028 464 struct scatterlist *sg;
6461f64a 465
b130eded 466 mem_reg = &iser_task->rdma_reg[cmd_dir];
6461f64a 467
5180311f 468 aligned_len = iser_data_buf_aligned_len(mem, ibdev);
5587856c 469 if (aligned_len != mem->dma_nents) {
56408325 470 err = fall_to_bounce_buf(iser_task, mem,
919fc274
SG
471 cmd_dir, aligned_len);
472 if (err) {
473 iser_err("failed to allocate bounce buffer\n");
474 return err;
475 }
6461f64a
OG
476 }
477
d8111028
EZ
478 /* if there a single dma entry, FMR is not needed */
479 if (mem->dma_nents == 1) {
e3784bd1 480 sg = mem->sg;
d8111028 481
b130eded
SG
482 mem_reg->lkey = device->mr->lkey;
483 mem_reg->rkey = device->mr->rkey;
484 mem_reg->len = ib_sg_dma_len(ibdev, &sg[0]);
485 mem_reg->va = ib_sg_dma_address(ibdev, &sg[0]);
d8111028
EZ
486
487 iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X "
488 "va: 0x%08lX sz: %ld]\n",
b130eded
SG
489 (unsigned int)mem_reg->lkey,
490 (unsigned int)mem_reg->rkey,
491 (unsigned long)mem_reg->va,
492 (unsigned long)mem_reg->len);
d8111028 493 } else { /* use FMR for multiple dma entries */
f0e35c27 494 err = iser_reg_page_vec(iser_task, mem, ib_conn->fmr.page_vec,
b130eded 495 mem_reg);
819a0873 496 if (err && err != -EAGAIN) {
5180311f 497 iser_data_buf_dump(mem, ibdev);
2261ec3d
MC
498 iser_err("mem->dma_nents = %d (dlength = 0x%x)\n",
499 mem->dma_nents,
500 ntoh24(iser_task->desc.iscsi_header.dlength));
d8111028 501 iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n",
a4ee3539
SG
502 ib_conn->fmr.page_vec->data_size,
503 ib_conn->fmr.page_vec->length,
504 ib_conn->fmr.page_vec->offset);
505 for (i = 0; i < ib_conn->fmr.page_vec->length; i++)
d8111028 506 iser_err("page_vec[%d] = 0x%llx\n", i,
a4ee3539 507 (unsigned long long)ib_conn->fmr.page_vec->pages[i]);
e981f1d4 508 }
450d1e40
OG
509 if (err)
510 return err;
e981f1d4 511 }
6461f64a
OG
512 return 0;
513}
5587856c 514
5bb6e543 515static void
92792c0a
SG
516iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs,
517 struct ib_sig_domain *domain)
518{
78eda2bb 519 domain->sig_type = IB_SIG_TYPE_T10_DIF;
5bb6e543
SG
520 domain->sig.dif.pi_interval = scsi_prot_interval(sc);
521 domain->sig.dif.ref_tag = scsi_prot_ref_tag(sc);
78eda2bb
SG
522 /*
523 * At the moment we hard code those, but in the future
524 * we will take them from sc.
525 */
526 domain->sig.dif.apptag_check_mask = 0xffff;
527 domain->sig.dif.app_escape = true;
528 domain->sig.dif.ref_escape = true;
5bb6e543 529 if (sc->prot_flags & SCSI_PROT_REF_INCREMENT)
78eda2bb 530 domain->sig.dif.ref_remap = true;
92792c0a 531};
177e31bd
SG
532
533static int
534iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs)
535{
177e31bd
SG
536 switch (scsi_get_prot_op(sc)) {
537 case SCSI_PROT_WRITE_INSERT:
538 case SCSI_PROT_READ_STRIP:
78eda2bb 539 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
92792c0a 540 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire);
177e31bd 541 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;
177e31bd
SG
542 break;
543 case SCSI_PROT_READ_INSERT:
544 case SCSI_PROT_WRITE_STRIP:
78eda2bb 545 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
92792c0a 546 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem);
5bb6e543
SG
547 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ?
548 IB_T10DIF_CSUM : IB_T10DIF_CRC;
177e31bd
SG
549 break;
550 case SCSI_PROT_READ_PASS:
551 case SCSI_PROT_WRITE_PASS:
92792c0a 552 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire);
177e31bd 553 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;
92792c0a 554 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem);
5bb6e543
SG
555 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ?
556 IB_T10DIF_CSUM : IB_T10DIF_CRC;
177e31bd
SG
557 break;
558 default:
559 iser_err("Unsupported PI operation %d\n",
560 scsi_get_prot_op(sc));
561 return -EINVAL;
562 }
78eda2bb 563
177e31bd
SG
564 return 0;
565}
566
5bb6e543 567static inline void
177e31bd
SG
568iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask)
569{
5bb6e543
SG
570 *mask = 0;
571 if (sc->prot_flags & SCSI_PROT_REF_CHECK)
572 *mask |= ISER_CHECK_REFTAG;
573 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK)
574 *mask |= ISER_CHECK_GUARD;
177e31bd
SG
575}
576
a11b3e69
SG
577static void
578iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr)
579{
580 u32 rkey;
581
582 memset(inv_wr, 0, sizeof(*inv_wr));
583 inv_wr->opcode = IB_WR_LOCAL_INV;
584 inv_wr->wr_id = ISER_FASTREG_LI_WRID;
585 inv_wr->ex.invalidate_rkey = mr->rkey;
586
587 rkey = ib_inc_rkey(mr->rkey);
588 ib_update_fast_reg_key(mr, rkey);
589}
590
177e31bd
SG
591static int
592iser_reg_sig_mr(struct iscsi_iser_task *iser_task,
593 struct fast_reg_descriptor *desc, struct ib_sge *data_sge,
594 struct ib_sge *prot_sge, struct ib_sge *sig_sge)
595{
a4ee3539 596 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
177e31bd
SG
597 struct iser_pi_context *pi_ctx = desc->pi_ctx;
598 struct ib_send_wr sig_wr, inv_wr;
599 struct ib_send_wr *bad_wr, *wr = NULL;
600 struct ib_sig_attrs sig_attrs;
601 int ret;
177e31bd
SG
602
603 memset(&sig_attrs, 0, sizeof(sig_attrs));
604 ret = iser_set_sig_attrs(iser_task->sc, &sig_attrs);
605 if (ret)
606 goto err;
607
5bb6e543 608 iser_set_prot_checks(iser_task->sc, &sig_attrs.check_mask);
177e31bd
SG
609
610 if (!(desc->reg_indicators & ISER_SIG_KEY_VALID)) {
a11b3e69 611 iser_inv_rkey(&inv_wr, pi_ctx->sig_mr);
177e31bd 612 wr = &inv_wr;
177e31bd
SG
613 }
614
615 memset(&sig_wr, 0, sizeof(sig_wr));
616 sig_wr.opcode = IB_WR_REG_SIG_MR;
617 sig_wr.wr_id = ISER_FASTREG_LI_WRID;
618 sig_wr.sg_list = data_sge;
619 sig_wr.num_sge = 1;
620 sig_wr.wr.sig_handover.sig_attrs = &sig_attrs;
621 sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr;
622 if (scsi_prot_sg_count(iser_task->sc))
623 sig_wr.wr.sig_handover.prot = prot_sge;
624 sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE |
625 IB_ACCESS_REMOTE_READ |
626 IB_ACCESS_REMOTE_WRITE;
627
628 if (!wr)
629 wr = &sig_wr;
630 else
631 wr->next = &sig_wr;
632
a4ee3539 633 ret = ib_post_send(ib_conn->qp, wr, &bad_wr);
177e31bd
SG
634 if (ret) {
635 iser_err("reg_sig_mr failed, ret:%d\n", ret);
636 goto err;
637 }
638 desc->reg_indicators &= ~ISER_SIG_KEY_VALID;
639
640 sig_sge->lkey = pi_ctx->sig_mr->lkey;
641 sig_sge->addr = 0;
5bb6e543 642 sig_sge->length = scsi_transfer_length(iser_task->sc);
177e31bd
SG
643
644 iser_dbg("sig_sge: addr: 0x%llx length: %u lkey: 0x%x\n",
645 sig_sge->addr, sig_sge->length,
646 sig_sge->lkey);
647err:
648 return ret;
649}
650
d11ec4ec 651static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
b130eded 652 struct iser_mem_reg *mem_reg,
d11ec4ec 653 struct iser_data_buf *mem,
177e31bd 654 enum iser_reg_indicator ind,
d11ec4ec 655 struct ib_sge *sge)
5587856c 656{
b130eded 657 struct fast_reg_descriptor *desc = mem_reg->mem_h;
a4ee3539
SG
658 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
659 struct iser_device *device = ib_conn->device;
d11ec4ec 660 struct ib_device *ibdev = device->ib_device;
177e31bd
SG
661 struct ib_mr *mr;
662 struct ib_fast_reg_page_list *frpl;
5587856c
SG
663 struct ib_send_wr fastreg_wr, inv_wr;
664 struct ib_send_wr *bad_wr, *wr = NULL;
d11ec4ec
SG
665 int ret, offset, size, plen;
666
667 /* if there a single dma entry, dma mr suffices */
668 if (mem->dma_nents == 1) {
e3784bd1 669 struct scatterlist *sg = mem->sg;
d11ec4ec
SG
670
671 sge->lkey = device->mr->lkey;
672 sge->addr = ib_sg_dma_address(ibdev, &sg[0]);
673 sge->length = ib_sg_dma_len(ibdev, &sg[0]);
674
675 iser_dbg("Single DMA entry: lkey=0x%x, addr=0x%llx, length=0x%x\n",
676 sge->lkey, sge->addr, sge->length);
677 return 0;
678 }
679
177e31bd
SG
680 if (ind == ISER_DATA_KEY_VALID) {
681 mr = desc->data_mr;
682 frpl = desc->data_frpl;
683 } else {
684 mr = desc->pi_ctx->prot_mr;
685 frpl = desc->pi_ctx->prot_frpl;
686 }
687
688 plen = iser_sg_to_page_vec(mem, device->ib_device, frpl->page_list,
d11ec4ec
SG
689 &offset, &size);
690 if (plen * SIZE_4K < size) {
691 iser_err("fast reg page_list too short to hold this SG\n");
692 return -EINVAL;
693 }
5587856c 694
177e31bd 695 if (!(desc->reg_indicators & ind)) {
a11b3e69 696 iser_inv_rkey(&inv_wr, mr);
5587856c 697 wr = &inv_wr;
5587856c
SG
698 }
699
700 /* Prepare FASTREG WR */
701 memset(&fastreg_wr, 0, sizeof(fastreg_wr));
7306b8fa 702 fastreg_wr.wr_id = ISER_FASTREG_LI_WRID;
5587856c 703 fastreg_wr.opcode = IB_WR_FAST_REG_MR;
177e31bd
SG
704 fastreg_wr.wr.fast_reg.iova_start = frpl->page_list[0] + offset;
705 fastreg_wr.wr.fast_reg.page_list = frpl;
d11ec4ec 706 fastreg_wr.wr.fast_reg.page_list_len = plen;
5587856c 707 fastreg_wr.wr.fast_reg.page_shift = SHIFT_4K;
d11ec4ec 708 fastreg_wr.wr.fast_reg.length = size;
177e31bd 709 fastreg_wr.wr.fast_reg.rkey = mr->rkey;
5587856c
SG
710 fastreg_wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE |
711 IB_ACCESS_REMOTE_WRITE |
712 IB_ACCESS_REMOTE_READ);
713
db523b8d 714 if (!wr)
5587856c 715 wr = &fastreg_wr;
db523b8d 716 else
5587856c 717 wr->next = &fastreg_wr;
5587856c 718
a4ee3539 719 ret = ib_post_send(ib_conn->qp, wr, &bad_wr);
5587856c 720 if (ret) {
5587856c
SG
721 iser_err("fast registration failed, ret:%d\n", ret);
722 return ret;
723 }
177e31bd 724 desc->reg_indicators &= ~ind;
5587856c 725
177e31bd
SG
726 sge->lkey = mr->lkey;
727 sge->addr = frpl->page_list[0] + offset;
d11ec4ec 728 sge->length = size;
5587856c
SG
729
730 return ret;
731}
732
733/**
7306b8fa 734 * iser_reg_rdma_mem_fastreg - Registers memory intended for RDMA,
5587856c
SG
735 * using Fast Registration WR (if possible) obtaining rkey and va
736 *
737 * returns 0 on success, errno code on failure
738 */
7306b8fa
SG
739int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *iser_task,
740 enum iser_data_dir cmd_dir)
5587856c 741{
a4ee3539
SG
742 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
743 struct iser_device *device = ib_conn->device;
5587856c
SG
744 struct ib_device *ibdev = device->ib_device;
745 struct iser_data_buf *mem = &iser_task->data[cmd_dir];
b130eded 746 struct iser_mem_reg *mem_reg = &iser_task->rdma_reg[cmd_dir];
d11ec4ec
SG
747 struct fast_reg_descriptor *desc = NULL;
748 struct ib_sge data_sge;
5587856c 749 int err, aligned_len;
5587856c
SG
750
751 aligned_len = iser_data_buf_aligned_len(mem, ibdev);
752 if (aligned_len != mem->dma_nents) {
56408325 753 err = fall_to_bounce_buf(iser_task, mem,
5587856c
SG
754 cmd_dir, aligned_len);
755 if (err) {
756 iser_err("failed to allocate bounce buffer\n");
757 return err;
758 }
5587856c
SG
759 }
760
177e31bd
SG
761 if (mem->dma_nents != 1 ||
762 scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) {
bd8b944e 763 desc = iser_reg_desc_get(ib_conn);
b130eded 764 mem_reg->mem_h = desc;
d11ec4ec 765 }
5587856c 766
b130eded 767 err = iser_fast_reg_mr(iser_task, mem_reg, mem,
177e31bd 768 ISER_DATA_KEY_VALID, &data_sge);
d11ec4ec
SG
769 if (err)
770 goto err_reg;
771
177e31bd
SG
772 if (scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) {
773 struct ib_sge prot_sge, sig_sge;
774
775 memset(&prot_sge, 0, sizeof(prot_sge));
776 if (scsi_prot_sg_count(iser_task->sc)) {
777 mem = &iser_task->prot[cmd_dir];
778 aligned_len = iser_data_buf_aligned_len(mem, ibdev);
779 if (aligned_len != mem->dma_nents) {
56408325 780 err = fall_to_bounce_buf(iser_task, mem,
177e31bd
SG
781 cmd_dir, aligned_len);
782 if (err) {
783 iser_err("failed to allocate bounce buffer\n");
784 return err;
785 }
177e31bd
SG
786 }
787
b130eded 788 err = iser_fast_reg_mr(iser_task, mem_reg, mem,
177e31bd
SG
789 ISER_PROT_KEY_VALID, &prot_sge);
790 if (err)
791 goto err_reg;
792 }
793
794 err = iser_reg_sig_mr(iser_task, desc, &data_sge,
795 &prot_sge, &sig_sge);
796 if (err) {
797 iser_err("Failed to register signature mr\n");
798 return err;
799 }
800 desc->reg_indicators |= ISER_FASTREG_PROTECTED;
801
b130eded
SG
802 mem_reg->lkey = sig_sge.lkey;
803 mem_reg->rkey = desc->pi_ctx->sig_mr->rkey;
804 mem_reg->va = sig_sge.addr;
805 mem_reg->len = sig_sge.length;
d11ec4ec 806 } else {
7e1fd4d1 807 if (desc)
b130eded 808 mem_reg->rkey = desc->data_mr->rkey;
7e1fd4d1 809 else
b130eded 810 mem_reg->rkey = device->mr->rkey;
5587856c 811
b130eded
SG
812 mem_reg->lkey = data_sge.lkey;
813 mem_reg->va = data_sge.addr;
814 mem_reg->len = data_sge.length;
177e31bd 815 }
d11ec4ec 816
5587856c
SG
817 return 0;
818err_reg:
bd8b944e
SG
819 if (desc)
820 iser_reg_desc_put(ib_conn, desc);
d11ec4ec 821
5587856c
SG
822 return err;
823}
This page took 0.685677 seconds and 5 git commands to generate.