2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * $Id: iser_initiator.c 6964 2006-05-07 11:11:43Z ogerlitz $
34 #include <linux/kernel.h>
35 #include <linux/slab.h>
38 #include <asm/scatterlist.h>
39 #include <linux/scatterlist.h>
40 #include <linux/kfifo.h>
41 #include <scsi/scsi_cmnd.h>
42 #include <scsi/scsi_host.h>
44 #include "iscsi_iser.h"
46 /* Constant PDU lengths calculations */
47 #define ISER_TOTAL_HEADERS_LEN (sizeof (struct iser_hdr) + \
48 sizeof (struct iscsi_hdr))
50 /* iser_dto_add_regd_buff - increments the reference count for *
51 * the registered buffer & adds it to the DTO object */
52 static void iser_dto_add_regd_buff(struct iser_dto
*dto
,
53 struct iser_regd_buf
*regd_buf
,
54 unsigned long use_offset
,
55 unsigned long use_size
)
59 atomic_inc(®d_buf
->ref_count
);
61 add_idx
= dto
->regd_vector_len
;
62 dto
->regd
[add_idx
] = regd_buf
;
63 dto
->used_sz
[add_idx
] = use_size
;
64 dto
->offset
[add_idx
] = use_offset
;
66 dto
->regd_vector_len
++;
69 /* Register user buffer memory and initialize passive rdma
70 * dto descriptor. Total data size is stored in
71 * iser_ctask->data[ISER_DIR_IN].data_len
73 static int iser_prepare_read_cmd(struct iscsi_cmd_task
*ctask
,
77 struct iscsi_iser_cmd_task
*iser_ctask
= ctask
->dd_data
;
78 struct iser_regd_buf
*regd_buf
;
80 struct iser_hdr
*hdr
= &iser_ctask
->desc
.iser_header
;
81 struct iser_data_buf
*buf_in
= &iser_ctask
->data
[ISER_DIR_IN
];
83 err
= iser_dma_map_task_data(iser_ctask
,
90 if (edtl
> iser_ctask
->data
[ISER_DIR_IN
].data_len
) {
91 iser_err("Total data length: %ld, less than EDTL: "
92 "%d, in READ cmd BHS itt: %d, conn: 0x%p\n",
93 iser_ctask
->data
[ISER_DIR_IN
].data_len
, edtl
,
94 ctask
->itt
, iser_ctask
->iser_conn
);
98 err
= iser_reg_rdma_mem(iser_ctask
,ISER_DIR_IN
);
100 iser_err("Failed to set up Data-IN RDMA\n");
103 regd_buf
= &iser_ctask
->rdma_regd
[ISER_DIR_IN
];
105 hdr
->flags
|= ISER_RSV
;
106 hdr
->read_stag
= cpu_to_be32(regd_buf
->reg
.rkey
);
107 hdr
->read_va
= cpu_to_be64(regd_buf
->reg
.va
);
109 iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n",
110 ctask
->itt
, regd_buf
->reg
.rkey
,
111 (unsigned long long)regd_buf
->reg
.va
);
116 /* Register user buffer memory and initialize passive rdma
117 * dto descriptor. Total data size is stored in
118 * ctask->data[ISER_DIR_OUT].data_len
121 iser_prepare_write_cmd(struct iscsi_cmd_task
*ctask
,
123 unsigned int unsol_sz
,
126 struct iscsi_iser_cmd_task
*iser_ctask
= ctask
->dd_data
;
127 struct iser_regd_buf
*regd_buf
;
129 struct iser_dto
*send_dto
= &iser_ctask
->desc
.dto
;
130 struct iser_hdr
*hdr
= &iser_ctask
->desc
.iser_header
;
131 struct iser_data_buf
*buf_out
= &iser_ctask
->data
[ISER_DIR_OUT
];
133 err
= iser_dma_map_task_data(iser_ctask
,
140 if (edtl
> iser_ctask
->data
[ISER_DIR_OUT
].data_len
) {
141 iser_err("Total data length: %ld, less than EDTL: %d, "
142 "in WRITE cmd BHS itt: %d, conn: 0x%p\n",
143 iser_ctask
->data
[ISER_DIR_OUT
].data_len
,
144 edtl
, ctask
->itt
, ctask
->conn
);
148 err
= iser_reg_rdma_mem(iser_ctask
,ISER_DIR_OUT
);
150 iser_err("Failed to register write cmd RDMA mem\n");
154 regd_buf
= &iser_ctask
->rdma_regd
[ISER_DIR_OUT
];
156 if (unsol_sz
< edtl
) {
157 hdr
->flags
|= ISER_WSV
;
158 hdr
->write_stag
= cpu_to_be32(regd_buf
->reg
.rkey
);
159 hdr
->write_va
= cpu_to_be64(regd_buf
->reg
.va
+ unsol_sz
);
161 iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X "
162 "VA:%#llX + unsol:%d\n",
163 ctask
->itt
, regd_buf
->reg
.rkey
,
164 (unsigned long long)regd_buf
->reg
.va
, unsol_sz
);
168 iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n",
170 iser_dto_add_regd_buff(send_dto
,
180 * iser_post_receive_control - allocates, initializes and posts receive DTO.
182 static int iser_post_receive_control(struct iscsi_conn
*conn
)
184 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
185 struct iser_desc
*rx_desc
;
186 struct iser_regd_buf
*regd_hdr
;
187 struct iser_regd_buf
*regd_data
;
188 struct iser_dto
*recv_dto
= NULL
;
189 struct iser_device
*device
= iser_conn
->ib_conn
->device
;
190 int rx_data_size
, err
= 0;
192 rx_desc
= kmem_cache_alloc(ig
.desc_cache
, GFP_NOIO
);
193 if (rx_desc
== NULL
) {
194 iser_err("Failed to alloc desc for post recv\n");
197 rx_desc
->type
= ISCSI_RX
;
199 /* for the login sequence we must support rx of upto 8K; login is done
200 * after conn create/bind (connect) and conn stop/bind (reconnect),
201 * what's common for both schemes is that the connection is not started
203 if (conn
->c_stage
!= ISCSI_CONN_STARTED
)
204 rx_data_size
= ISCSI_DEF_MAX_RECV_SEG_LEN
;
205 else /* FIXME till user space sets conn->max_recv_dlength correctly */
208 rx_desc
->data
= kmalloc(rx_data_size
, GFP_NOIO
);
209 if (rx_desc
->data
== NULL
) {
210 iser_err("Failed to alloc data buf for post recv\n");
212 goto post_rx_kmalloc_failure
;
215 recv_dto
= &rx_desc
->dto
;
216 recv_dto
->ib_conn
= iser_conn
->ib_conn
;
217 recv_dto
->regd_vector_len
= 0;
219 regd_hdr
= &rx_desc
->hdr_regd_buf
;
220 memset(regd_hdr
, 0, sizeof(struct iser_regd_buf
));
221 regd_hdr
->device
= device
;
222 regd_hdr
->virt_addr
= rx_desc
; /* == &rx_desc->iser_header */
223 regd_hdr
->data_size
= ISER_TOTAL_HEADERS_LEN
;
225 iser_reg_single(device
, regd_hdr
, DMA_FROM_DEVICE
);
227 iser_dto_add_regd_buff(recv_dto
, regd_hdr
, 0, 0);
229 regd_data
= &rx_desc
->data_regd_buf
;
230 memset(regd_data
, 0, sizeof(struct iser_regd_buf
));
231 regd_data
->device
= device
;
232 regd_data
->virt_addr
= rx_desc
->data
;
233 regd_data
->data_size
= rx_data_size
;
235 iser_reg_single(device
, regd_data
, DMA_FROM_DEVICE
);
237 iser_dto_add_regd_buff(recv_dto
, regd_data
, 0, 0);
239 err
= iser_post_recv(rx_desc
);
243 /* iser_post_recv failed */
244 iser_dto_buffs_release(recv_dto
);
245 kfree(rx_desc
->data
);
246 post_rx_kmalloc_failure
:
247 kmem_cache_free(ig
.desc_cache
, rx_desc
);
251 /* creates a new tx descriptor and adds header regd buffer */
252 static void iser_create_send_desc(struct iscsi_iser_conn
*iser_conn
,
253 struct iser_desc
*tx_desc
)
255 struct iser_regd_buf
*regd_hdr
= &tx_desc
->hdr_regd_buf
;
256 struct iser_dto
*send_dto
= &tx_desc
->dto
;
258 memset(regd_hdr
, 0, sizeof(struct iser_regd_buf
));
259 regd_hdr
->device
= iser_conn
->ib_conn
->device
;
260 regd_hdr
->virt_addr
= tx_desc
; /* == &tx_desc->iser_header */
261 regd_hdr
->data_size
= ISER_TOTAL_HEADERS_LEN
;
263 send_dto
->ib_conn
= iser_conn
->ib_conn
;
264 send_dto
->notify_enable
= 1;
265 send_dto
->regd_vector_len
= 0;
267 memset(&tx_desc
->iser_header
, 0, sizeof(struct iser_hdr
));
268 tx_desc
->iser_header
.flags
= ISER_VER
;
270 iser_dto_add_regd_buff(send_dto
, regd_hdr
, 0, 0);
274 * iser_conn_set_full_featured_mode - (iSER API)
276 int iser_conn_set_full_featured_mode(struct iscsi_conn
*conn
)
278 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
281 /* no need to keep it in a var, we are after login so if this should
282 * be negotiated, by now the result should be available here */
283 int initial_post_recv_bufs_num
= ISER_MAX_RX_MISC_PDUS
;
285 iser_dbg("Initially post: %d\n", initial_post_recv_bufs_num
);
287 /* Check that there is no posted recv or send buffers left - */
288 /* they must be consumed during the login phase */
289 BUG_ON(atomic_read(&iser_conn
->ib_conn
->post_recv_buf_count
) != 0);
290 BUG_ON(atomic_read(&iser_conn
->ib_conn
->post_send_buf_count
) != 0);
292 /* Initial post receive buffers */
293 for (i
= 0; i
< initial_post_recv_bufs_num
; i
++) {
294 if (iser_post_receive_control(conn
) != 0) {
295 iser_err("Failed to post recv bufs at:%d conn:0x%p\n",
300 iser_dbg("Posted %d post recv bufs, conn:0x%p\n", i
, conn
);
305 iser_check_xmit(struct iscsi_conn
*conn
, void *task
)
307 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
309 if (atomic_read(&iser_conn
->ib_conn
->post_send_buf_count
) ==
310 ISER_QP_MAX_REQ_DTOS
) {
311 iser_dbg("%ld can't xmit task %p\n",jiffies
,task
);
319 * iser_send_command - send command PDU
321 int iser_send_command(struct iscsi_conn
*conn
,
322 struct iscsi_cmd_task
*ctask
)
324 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
325 struct iscsi_iser_cmd_task
*iser_ctask
= ctask
->dd_data
;
326 struct iser_dto
*send_dto
= NULL
;
329 struct iser_data_buf
*data_buf
;
331 struct iscsi_cmd
*hdr
= ctask
->hdr
;
332 struct scsi_cmnd
*sc
= ctask
->sc
;
334 if (!iser_conn_state_comp(iser_conn
->ib_conn
, ISER_CONN_UP
)) {
335 iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn
->ib_conn
);
338 if (iser_check_xmit(conn
, ctask
))
341 edtl
= ntohl(hdr
->data_length
);
343 /* build the tx desc regd header and add it to the tx desc dto */
344 iser_ctask
->desc
.type
= ISCSI_TX_SCSI_COMMAND
;
345 send_dto
= &iser_ctask
->desc
.dto
;
346 send_dto
->ctask
= iser_ctask
;
347 iser_create_send_desc(iser_conn
, &iser_ctask
->desc
);
349 if (hdr
->flags
& ISCSI_FLAG_CMD_READ
)
350 data_buf
= &iser_ctask
->data
[ISER_DIR_IN
];
352 data_buf
= &iser_ctask
->data
[ISER_DIR_OUT
];
354 if (scsi_sg_count(sc
)) { /* using a scatter list */
355 data_buf
->buf
= scsi_sglist(sc
);
356 data_buf
->size
= scsi_sg_count(sc
);
359 data_buf
->data_len
= scsi_bufflen(sc
);
361 if (hdr
->flags
& ISCSI_FLAG_CMD_READ
) {
362 err
= iser_prepare_read_cmd(ctask
, edtl
);
364 goto send_command_error
;
366 if (hdr
->flags
& ISCSI_FLAG_CMD_WRITE
) {
367 err
= iser_prepare_write_cmd(ctask
,
373 goto send_command_error
;
376 iser_reg_single(iser_conn
->ib_conn
->device
,
377 send_dto
->regd
[0], DMA_TO_DEVICE
);
379 if (iser_post_receive_control(conn
) != 0) {
380 iser_err("post_recv failed!\n");
382 goto send_command_error
;
385 iser_ctask
->status
= ISER_TASK_STATUS_STARTED
;
387 err
= iser_post_send(&iser_ctask
->desc
);
392 iser_dto_buffs_release(send_dto
);
393 iser_err("conn %p failed ctask->itt %d err %d\n",conn
, ctask
->itt
, err
);
398 * iser_send_data_out - send data out PDU
400 int iser_send_data_out(struct iscsi_conn
*conn
,
401 struct iscsi_cmd_task
*ctask
,
402 struct iscsi_data
*hdr
)
404 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
405 struct iscsi_iser_cmd_task
*iser_ctask
= ctask
->dd_data
;
406 struct iser_desc
*tx_desc
= NULL
;
407 struct iser_dto
*send_dto
= NULL
;
408 unsigned long buf_offset
;
409 unsigned long data_seg_len
;
413 if (!iser_conn_state_comp(iser_conn
->ib_conn
, ISER_CONN_UP
)) {
414 iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn
->ib_conn
);
418 if (iser_check_xmit(conn
, ctask
))
421 itt
= ntohl(hdr
->itt
);
422 data_seg_len
= ntoh24(hdr
->dlength
);
423 buf_offset
= ntohl(hdr
->offset
);
425 iser_dbg("%s itt %d dseg_len %d offset %d\n",
426 __func__
,(int)itt
,(int)data_seg_len
,(int)buf_offset
);
428 tx_desc
= kmem_cache_alloc(ig
.desc_cache
, GFP_NOIO
);
429 if (tx_desc
== NULL
) {
430 iser_err("Failed to alloc desc for post dataout\n");
434 tx_desc
->type
= ISCSI_TX_DATAOUT
;
435 memcpy(&tx_desc
->iscsi_header
, hdr
, sizeof(struct iscsi_hdr
));
437 /* build the tx desc regd header and add it to the tx desc dto */
438 send_dto
= &tx_desc
->dto
;
439 send_dto
->ctask
= iser_ctask
;
440 iser_create_send_desc(iser_conn
, tx_desc
);
442 iser_reg_single(iser_conn
->ib_conn
->device
,
443 send_dto
->regd
[0], DMA_TO_DEVICE
);
445 /* all data was registered for RDMA, we can use the lkey */
446 iser_dto_add_regd_buff(send_dto
,
447 &iser_ctask
->rdma_regd
[ISER_DIR_OUT
],
451 if (buf_offset
+ data_seg_len
> iser_ctask
->data
[ISER_DIR_OUT
].data_len
) {
452 iser_err("Offset:%ld & DSL:%ld in Data-Out "
453 "inconsistent with total len:%ld, itt:%d\n",
454 buf_offset
, data_seg_len
,
455 iser_ctask
->data
[ISER_DIR_OUT
].data_len
, itt
);
457 goto send_data_out_error
;
459 iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n",
460 itt
, buf_offset
, data_seg_len
);
463 err
= iser_post_send(tx_desc
);
468 iser_dto_buffs_release(send_dto
);
469 kmem_cache_free(ig
.desc_cache
, tx_desc
);
470 iser_err("conn %p failed err %d\n",conn
, err
);
474 int iser_send_control(struct iscsi_conn
*conn
,
475 struct iscsi_mgmt_task
*mtask
)
477 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
478 struct iser_desc
*mdesc
= mtask
->dd_data
;
479 struct iser_dto
*send_dto
= NULL
;
480 unsigned long data_seg_len
;
482 struct iser_regd_buf
*regd_buf
;
483 struct iser_device
*device
;
485 if (!iser_conn_state_comp(iser_conn
->ib_conn
, ISER_CONN_UP
)) {
486 iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn
->ib_conn
);
490 if (iser_check_xmit(conn
,mtask
))
493 /* build the tx desc regd header and add it to the tx desc dto */
494 mdesc
->type
= ISCSI_TX_CONTROL
;
495 send_dto
= &mdesc
->dto
;
496 send_dto
->ctask
= NULL
;
497 iser_create_send_desc(iser_conn
, mdesc
);
499 device
= iser_conn
->ib_conn
->device
;
501 iser_reg_single(device
, send_dto
->regd
[0], DMA_TO_DEVICE
);
503 data_seg_len
= ntoh24(mtask
->hdr
->dlength
);
505 if (data_seg_len
> 0) {
506 regd_buf
= &mdesc
->data_regd_buf
;
507 memset(regd_buf
, 0, sizeof(struct iser_regd_buf
));
508 regd_buf
->device
= device
;
509 regd_buf
->virt_addr
= mtask
->data
;
510 regd_buf
->data_size
= mtask
->data_count
;
511 iser_reg_single(device
, regd_buf
,
513 iser_dto_add_regd_buff(send_dto
, regd_buf
,
518 if (iser_post_receive_control(conn
) != 0) {
519 iser_err("post_rcv_buff failed!\n");
521 goto send_control_error
;
524 err
= iser_post_send(mdesc
);
529 iser_dto_buffs_release(send_dto
);
530 iser_err("conn %p failed err %d\n",conn
, err
);
535 * iser_rcv_dto_completion - recv DTO completion
537 void iser_rcv_completion(struct iser_desc
*rx_desc
,
538 unsigned long dto_xfer_len
)
540 struct iser_dto
*dto
= &rx_desc
->dto
;
541 struct iscsi_iser_conn
*conn
= dto
->ib_conn
->iser_conn
;
542 struct iscsi_session
*session
= conn
->iscsi_conn
->session
;
543 struct iscsi_cmd_task
*ctask
;
544 struct iscsi_iser_cmd_task
*iser_ctask
;
545 struct iscsi_hdr
*hdr
;
546 char *rx_data
= NULL
;
549 unsigned char opcode
;
551 hdr
= &rx_desc
->iscsi_header
;
553 iser_dbg("op 0x%x itt 0x%x\n", hdr
->opcode
,hdr
->itt
);
555 if (dto_xfer_len
> ISER_TOTAL_HEADERS_LEN
) { /* we have data */
556 rx_data_len
= dto_xfer_len
- ISER_TOTAL_HEADERS_LEN
;
557 rx_data
= dto
->regd
[1]->virt_addr
;
558 rx_data
+= dto
->offset
[1];
561 opcode
= hdr
->opcode
& ISCSI_OPCODE_MASK
;
563 if (opcode
== ISCSI_OP_SCSI_CMD_RSP
) {
564 itt
= get_itt(hdr
->itt
); /* mask out cid and age bits */
565 if (!(itt
< session
->cmds_max
))
566 iser_err("itt can't be matched to task!!!"
567 "conn %p opcode %d cmds_max %d itt %d\n",
568 conn
->iscsi_conn
,opcode
,session
->cmds_max
,itt
);
569 /* use the mapping given with the cmds array indexed by itt */
570 ctask
= (struct iscsi_cmd_task
*)session
->cmds
[itt
];
571 iser_ctask
= ctask
->dd_data
;
572 iser_dbg("itt %d ctask %p\n",itt
,ctask
);
573 iser_ctask
->status
= ISER_TASK_STATUS_COMPLETED
;
574 iser_ctask_rdma_finalize(iser_ctask
);
577 iser_dto_buffs_release(dto
);
579 iscsi_iser_recv(conn
->iscsi_conn
, hdr
, rx_data
, rx_data_len
);
581 kfree(rx_desc
->data
);
582 kmem_cache_free(ig
.desc_cache
, rx_desc
);
584 /* decrementing conn->post_recv_buf_count only --after-- freeing the *
585 * task eliminates the need to worry on tasks which are completed in *
586 * parallel to the execution of iser_conn_term. So the code that waits *
587 * for the posted rx bufs refcount to become zero handles everything */
588 atomic_dec(&conn
->ib_conn
->post_recv_buf_count
);
591 void iser_snd_completion(struct iser_desc
*tx_desc
)
593 struct iser_dto
*dto
= &tx_desc
->dto
;
594 struct iser_conn
*ib_conn
= dto
->ib_conn
;
595 struct iscsi_iser_conn
*iser_conn
= ib_conn
->iser_conn
;
596 struct iscsi_conn
*conn
= iser_conn
->iscsi_conn
;
597 struct iscsi_mgmt_task
*mtask
;
600 iser_dbg("Initiator, Data sent dto=0x%p\n", dto
);
602 iser_dto_buffs_release(dto
);
604 if (tx_desc
->type
== ISCSI_TX_DATAOUT
)
605 kmem_cache_free(ig
.desc_cache
, tx_desc
);
607 if (atomic_read(&iser_conn
->ib_conn
->post_send_buf_count
) ==
608 ISER_QP_MAX_REQ_DTOS
)
611 atomic_dec(&ib_conn
->post_send_buf_count
);
614 iser_dbg("%ld resuming tx\n",jiffies
);
615 scsi_queue_work(conn
->session
->host
, &conn
->xmitwork
);
618 if (tx_desc
->type
== ISCSI_TX_CONTROL
) {
619 /* this arithmetic is legal by libiscsi dd_data allocation */
620 mtask
= (void *) ((long)(void *)tx_desc
-
621 sizeof(struct iscsi_mgmt_task
));
622 if (mtask
->hdr
->itt
== RESERVED_ITT
) {
623 struct iscsi_session
*session
= conn
->session
;
625 spin_lock(&conn
->session
->lock
);
626 list_del(&mtask
->running
);
627 __kfifo_put(session
->mgmtpool
.queue
, (void*)&mtask
,
629 spin_unlock(&session
->lock
);
634 void iser_ctask_rdma_init(struct iscsi_iser_cmd_task
*iser_ctask
)
637 iser_ctask
->status
= ISER_TASK_STATUS_INIT
;
639 iser_ctask
->dir
[ISER_DIR_IN
] = 0;
640 iser_ctask
->dir
[ISER_DIR_OUT
] = 0;
642 iser_ctask
->data
[ISER_DIR_IN
].data_len
= 0;
643 iser_ctask
->data
[ISER_DIR_OUT
].data_len
= 0;
645 memset(&iser_ctask
->rdma_regd
[ISER_DIR_IN
], 0,
646 sizeof(struct iser_regd_buf
));
647 memset(&iser_ctask
->rdma_regd
[ISER_DIR_OUT
], 0,
648 sizeof(struct iser_regd_buf
));
651 void iser_ctask_rdma_finalize(struct iscsi_iser_cmd_task
*iser_ctask
)
654 int is_rdma_aligned
= 1;
655 struct iser_regd_buf
*regd
;
657 /* if we were reading, copy back to unaligned sglist,
658 * anyway dma_unmap and free the copy
660 if (iser_ctask
->data_copy
[ISER_DIR_IN
].copy_buf
!= NULL
) {
662 iser_finalize_rdma_unaligned_sg(iser_ctask
, ISER_DIR_IN
);
664 if (iser_ctask
->data_copy
[ISER_DIR_OUT
].copy_buf
!= NULL
) {
666 iser_finalize_rdma_unaligned_sg(iser_ctask
, ISER_DIR_OUT
);
669 if (iser_ctask
->dir
[ISER_DIR_IN
]) {
670 regd
= &iser_ctask
->rdma_regd
[ISER_DIR_IN
];
671 deferred
= iser_regd_buff_release(regd
);
673 iser_err("%d references remain for BUF-IN rdma reg\n",
674 atomic_read(®d
->ref_count
));
678 if (iser_ctask
->dir
[ISER_DIR_OUT
]) {
679 regd
= &iser_ctask
->rdma_regd
[ISER_DIR_OUT
];
680 deferred
= iser_regd_buff_release(regd
);
682 iser_err("%d references remain for BUF-OUT rdma reg\n",
683 atomic_read(®d
->ref_count
));
687 /* if the data was unaligned, it was already unmapped and then copied */
689 iser_dma_unmap_task_data(iser_ctask
);
692 void iser_dto_buffs_release(struct iser_dto
*dto
)
696 for (i
= 0; i
< dto
->regd_vector_len
; i
++)
697 iser_regd_buff_release(dto
->regd
[i
]);