1 /*******************************************************************************
2 * This file contains iSCSI extentions for RDMA (iSER) Verbs
4 * (c) Copyright 2013 Datera, Inc.
6 * Nicholas A. Bellinger <nab@linux-iscsi.org>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 ****************************************************************************/
19 #include <linux/string.h>
20 #include <linux/module.h>
21 #include <linux/scatterlist.h>
22 #include <linux/socket.h>
24 #include <linux/in6.h>
25 #include <linux/llist.h>
26 #include <rdma/ib_verbs.h>
27 #include <rdma/rdma_cm.h>
28 #include <target/target_core_base.h>
29 #include <target/target_core_fabric.h>
30 #include <target/iscsi/iscsi_transport.h>
32 #include "isert_proto.h"
35 #define ISERT_MAX_CONN 8
36 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
37 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN)
39 static DEFINE_MUTEX(device_list_mutex
);
40 static LIST_HEAD(device_list
);
41 static struct workqueue_struct
*isert_rx_wq
;
42 static struct workqueue_struct
*isert_comp_wq
;
45 isert_unmap_cmd(struct isert_cmd
*isert_cmd
, struct isert_conn
*isert_conn
);
47 isert_map_rdma(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
,
48 struct isert_rdma_wr
*wr
);
50 isert_unreg_rdma(struct isert_cmd
*isert_cmd
, struct isert_conn
*isert_conn
);
52 isert_reg_rdma(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
,
53 struct isert_rdma_wr
*wr
);
56 isert_qp_event_callback(struct ib_event
*e
, void *context
)
58 struct isert_conn
*isert_conn
= (struct isert_conn
*)context
;
60 pr_err("isert_qp_event_callback event: %d\n", e
->event
);
62 case IB_EVENT_COMM_EST
:
63 rdma_notify(isert_conn
->conn_cm_id
, IB_EVENT_COMM_EST
);
65 case IB_EVENT_QP_LAST_WQE_REACHED
:
66 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n");
74 isert_query_device(struct ib_device
*ib_dev
, struct ib_device_attr
*devattr
)
78 ret
= ib_query_device(ib_dev
, devattr
);
80 pr_err("ib_query_device() failed: %d\n", ret
);
83 pr_debug("devattr->max_sge: %d\n", devattr
->max_sge
);
84 pr_debug("devattr->max_sge_rd: %d\n", devattr
->max_sge_rd
);
90 isert_conn_setup_qp(struct isert_conn
*isert_conn
, struct rdma_cm_id
*cma_id
)
92 struct isert_device
*device
= isert_conn
->conn_device
;
93 struct ib_qp_init_attr attr
;
94 int ret
, index
, min_index
= 0;
96 mutex_lock(&device_list_mutex
);
97 for (index
= 0; index
< device
->cqs_used
; index
++)
98 if (device
->cq_active_qps
[index
] <
99 device
->cq_active_qps
[min_index
])
101 device
->cq_active_qps
[min_index
]++;
102 pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index
);
103 mutex_unlock(&device_list_mutex
);
105 memset(&attr
, 0, sizeof(struct ib_qp_init_attr
));
106 attr
.event_handler
= isert_qp_event_callback
;
107 attr
.qp_context
= isert_conn
;
108 attr
.send_cq
= device
->dev_tx_cq
[min_index
];
109 attr
.recv_cq
= device
->dev_rx_cq
[min_index
];
110 attr
.cap
.max_send_wr
= ISERT_QP_MAX_REQ_DTOS
;
111 attr
.cap
.max_recv_wr
= ISERT_QP_MAX_RECV_DTOS
;
113 * FIXME: Use devattr.max_sge - 2 for max_send_sge as
114 * work-around for RDMA_READ..
116 attr
.cap
.max_send_sge
= device
->dev_attr
.max_sge
- 2;
117 isert_conn
->max_sge
= attr
.cap
.max_send_sge
;
119 attr
.cap
.max_recv_sge
= 1;
120 attr
.sq_sig_type
= IB_SIGNAL_REQ_WR
;
121 attr
.qp_type
= IB_QPT_RC
;
123 pr_debug("isert_conn_setup_qp cma_id->device: %p\n",
125 pr_debug("isert_conn_setup_qp conn_pd->device: %p\n",
126 isert_conn
->conn_pd
->device
);
128 ret
= rdma_create_qp(cma_id
, isert_conn
->conn_pd
, &attr
);
130 pr_err("rdma_create_qp failed for cma_id %d\n", ret
);
133 isert_conn
->conn_qp
= cma_id
->qp
;
134 pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n");
140 isert_cq_event_callback(struct ib_event
*e
, void *context
)
142 pr_debug("isert_cq_event_callback event: %d\n", e
->event
);
146 isert_alloc_rx_descriptors(struct isert_conn
*isert_conn
)
148 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
149 struct iser_rx_desc
*rx_desc
;
150 struct ib_sge
*rx_sg
;
154 isert_conn
->conn_rx_descs
= kzalloc(ISERT_QP_MAX_RECV_DTOS
*
155 sizeof(struct iser_rx_desc
), GFP_KERNEL
);
156 if (!isert_conn
->conn_rx_descs
)
159 rx_desc
= isert_conn
->conn_rx_descs
;
161 for (i
= 0; i
< ISERT_QP_MAX_RECV_DTOS
; i
++, rx_desc
++) {
162 dma_addr
= ib_dma_map_single(ib_dev
, (void *)rx_desc
,
163 ISER_RX_PAYLOAD_SIZE
, DMA_FROM_DEVICE
);
164 if (ib_dma_mapping_error(ib_dev
, dma_addr
))
167 rx_desc
->dma_addr
= dma_addr
;
169 rx_sg
= &rx_desc
->rx_sg
;
170 rx_sg
->addr
= rx_desc
->dma_addr
;
171 rx_sg
->length
= ISER_RX_PAYLOAD_SIZE
;
172 rx_sg
->lkey
= isert_conn
->conn_mr
->lkey
;
175 isert_conn
->conn_rx_desc_head
= 0;
179 rx_desc
= isert_conn
->conn_rx_descs
;
180 for (j
= 0; j
< i
; j
++, rx_desc
++) {
181 ib_dma_unmap_single(ib_dev
, rx_desc
->dma_addr
,
182 ISER_RX_PAYLOAD_SIZE
, DMA_FROM_DEVICE
);
184 kfree(isert_conn
->conn_rx_descs
);
185 isert_conn
->conn_rx_descs
= NULL
;
191 isert_free_rx_descriptors(struct isert_conn
*isert_conn
)
193 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
194 struct iser_rx_desc
*rx_desc
;
197 if (!isert_conn
->conn_rx_descs
)
200 rx_desc
= isert_conn
->conn_rx_descs
;
201 for (i
= 0; i
< ISERT_QP_MAX_RECV_DTOS
; i
++, rx_desc
++) {
202 ib_dma_unmap_single(ib_dev
, rx_desc
->dma_addr
,
203 ISER_RX_PAYLOAD_SIZE
, DMA_FROM_DEVICE
);
206 kfree(isert_conn
->conn_rx_descs
);
207 isert_conn
->conn_rx_descs
= NULL
;
210 static void isert_cq_tx_work(struct work_struct
*);
211 static void isert_cq_tx_callback(struct ib_cq
*, void *);
212 static void isert_cq_rx_work(struct work_struct
*);
213 static void isert_cq_rx_callback(struct ib_cq
*, void *);
216 isert_create_device_ib_res(struct isert_device
*device
)
218 struct ib_device
*ib_dev
= device
->ib_device
;
219 struct isert_cq_desc
*cq_desc
;
220 struct ib_device_attr
*dev_attr
;
223 dev_attr
= &device
->dev_attr
;
224 ret
= isert_query_device(ib_dev
, dev_attr
);
228 /* asign function handlers */
229 if (dev_attr
->device_cap_flags
& IB_DEVICE_MEM_MGT_EXTENSIONS
) {
230 device
->use_fastreg
= 1;
231 device
->reg_rdma_mem
= isert_reg_rdma
;
232 device
->unreg_rdma_mem
= isert_unreg_rdma
;
234 device
->use_fastreg
= 0;
235 device
->reg_rdma_mem
= isert_map_rdma
;
236 device
->unreg_rdma_mem
= isert_unmap_cmd
;
239 device
->cqs_used
= min_t(int, num_online_cpus(),
240 device
->ib_device
->num_comp_vectors
);
241 device
->cqs_used
= min(ISERT_MAX_CQ
, device
->cqs_used
);
242 pr_debug("Using %d CQs, device %s supports %d vectors support "
243 "Fast registration %d\n",
244 device
->cqs_used
, device
->ib_device
->name
,
245 device
->ib_device
->num_comp_vectors
, device
->use_fastreg
);
246 device
->cq_desc
= kzalloc(sizeof(struct isert_cq_desc
) *
247 device
->cqs_used
, GFP_KERNEL
);
248 if (!device
->cq_desc
) {
249 pr_err("Unable to allocate device->cq_desc\n");
252 cq_desc
= device
->cq_desc
;
254 for (i
= 0; i
< device
->cqs_used
; i
++) {
255 cq_desc
[i
].device
= device
;
256 cq_desc
[i
].cq_index
= i
;
258 INIT_WORK(&cq_desc
[i
].cq_rx_work
, isert_cq_rx_work
);
259 device
->dev_rx_cq
[i
] = ib_create_cq(device
->ib_device
,
260 isert_cq_rx_callback
,
261 isert_cq_event_callback
,
263 ISER_MAX_RX_CQ_LEN
, i
);
264 if (IS_ERR(device
->dev_rx_cq
[i
])) {
265 ret
= PTR_ERR(device
->dev_rx_cq
[i
]);
266 device
->dev_rx_cq
[i
] = NULL
;
270 INIT_WORK(&cq_desc
[i
].cq_tx_work
, isert_cq_tx_work
);
271 device
->dev_tx_cq
[i
] = ib_create_cq(device
->ib_device
,
272 isert_cq_tx_callback
,
273 isert_cq_event_callback
,
275 ISER_MAX_TX_CQ_LEN
, i
);
276 if (IS_ERR(device
->dev_tx_cq
[i
])) {
277 ret
= PTR_ERR(device
->dev_tx_cq
[i
]);
278 device
->dev_tx_cq
[i
] = NULL
;
282 ret
= ib_req_notify_cq(device
->dev_rx_cq
[i
], IB_CQ_NEXT_COMP
);
286 ret
= ib_req_notify_cq(device
->dev_tx_cq
[i
], IB_CQ_NEXT_COMP
);
294 for (j
= 0; j
< i
; j
++) {
295 cq_desc
= &device
->cq_desc
[j
];
297 if (device
->dev_rx_cq
[j
]) {
298 cancel_work_sync(&cq_desc
->cq_rx_work
);
299 ib_destroy_cq(device
->dev_rx_cq
[j
]);
301 if (device
->dev_tx_cq
[j
]) {
302 cancel_work_sync(&cq_desc
->cq_tx_work
);
303 ib_destroy_cq(device
->dev_tx_cq
[j
]);
306 kfree(device
->cq_desc
);
312 isert_free_device_ib_res(struct isert_device
*device
)
314 struct isert_cq_desc
*cq_desc
;
317 for (i
= 0; i
< device
->cqs_used
; i
++) {
318 cq_desc
= &device
->cq_desc
[i
];
320 cancel_work_sync(&cq_desc
->cq_rx_work
);
321 cancel_work_sync(&cq_desc
->cq_tx_work
);
322 ib_destroy_cq(device
->dev_rx_cq
[i
]);
323 ib_destroy_cq(device
->dev_tx_cq
[i
]);
324 device
->dev_rx_cq
[i
] = NULL
;
325 device
->dev_tx_cq
[i
] = NULL
;
328 kfree(device
->cq_desc
);
332 isert_device_try_release(struct isert_device
*device
)
334 mutex_lock(&device_list_mutex
);
336 if (!device
->refcount
) {
337 isert_free_device_ib_res(device
);
338 list_del(&device
->dev_node
);
341 mutex_unlock(&device_list_mutex
);
344 static struct isert_device
*
345 isert_device_find_by_ib_dev(struct rdma_cm_id
*cma_id
)
347 struct isert_device
*device
;
350 mutex_lock(&device_list_mutex
);
351 list_for_each_entry(device
, &device_list
, dev_node
) {
352 if (device
->ib_device
->node_guid
== cma_id
->device
->node_guid
) {
354 mutex_unlock(&device_list_mutex
);
359 device
= kzalloc(sizeof(struct isert_device
), GFP_KERNEL
);
361 mutex_unlock(&device_list_mutex
);
362 return ERR_PTR(-ENOMEM
);
365 INIT_LIST_HEAD(&device
->dev_node
);
367 device
->ib_device
= cma_id
->device
;
368 ret
= isert_create_device_ib_res(device
);
371 mutex_unlock(&device_list_mutex
);
376 list_add_tail(&device
->dev_node
, &device_list
);
377 mutex_unlock(&device_list_mutex
);
383 isert_conn_free_fastreg_pool(struct isert_conn
*isert_conn
)
385 struct fast_reg_descriptor
*fr_desc
, *tmp
;
388 if (list_empty(&isert_conn
->conn_fr_pool
))
391 pr_debug("Freeing conn %p fastreg pool", isert_conn
);
393 list_for_each_entry_safe(fr_desc
, tmp
,
394 &isert_conn
->conn_fr_pool
, list
) {
395 list_del(&fr_desc
->list
);
396 ib_free_fast_reg_page_list(fr_desc
->data_frpl
);
397 ib_dereg_mr(fr_desc
->data_mr
);
402 if (i
< isert_conn
->conn_fr_pool_size
)
403 pr_warn("Pool still has %d regions registered\n",
404 isert_conn
->conn_fr_pool_size
- i
);
408 isert_create_fr_desc(struct ib_device
*ib_device
, struct ib_pd
*pd
,
409 struct fast_reg_descriptor
*fr_desc
)
411 fr_desc
->data_frpl
= ib_alloc_fast_reg_page_list(ib_device
,
412 ISCSI_ISER_SG_TABLESIZE
);
413 if (IS_ERR(fr_desc
->data_frpl
)) {
414 pr_err("Failed to allocate data frpl err=%ld\n",
415 PTR_ERR(fr_desc
->data_frpl
));
416 return PTR_ERR(fr_desc
->data_frpl
);
419 fr_desc
->data_mr
= ib_alloc_fast_reg_mr(pd
, ISCSI_ISER_SG_TABLESIZE
);
420 if (IS_ERR(fr_desc
->data_mr
)) {
421 pr_err("Failed to allocate data frmr err=%ld\n",
422 PTR_ERR(fr_desc
->data_mr
));
423 ib_free_fast_reg_page_list(fr_desc
->data_frpl
);
424 return PTR_ERR(fr_desc
->data_mr
);
426 pr_debug("Create fr_desc %p page_list %p\n",
427 fr_desc
, fr_desc
->data_frpl
->page_list
);
429 fr_desc
->valid
= true;
435 isert_conn_create_fastreg_pool(struct isert_conn
*isert_conn
)
437 struct fast_reg_descriptor
*fr_desc
;
438 struct isert_device
*device
= isert_conn
->conn_device
;
441 INIT_LIST_HEAD(&isert_conn
->conn_fr_pool
);
442 isert_conn
->conn_fr_pool_size
= 0;
443 for (i
= 0; i
< ISCSI_DEF_XMIT_CMDS_MAX
; i
++) {
444 fr_desc
= kzalloc(sizeof(*fr_desc
), GFP_KERNEL
);
446 pr_err("Failed to allocate fast_reg descriptor\n");
451 ret
= isert_create_fr_desc(device
->ib_device
,
452 isert_conn
->conn_pd
, fr_desc
);
454 pr_err("Failed to create fastreg descriptor err=%d\n",
460 list_add_tail(&fr_desc
->list
, &isert_conn
->conn_fr_pool
);
461 isert_conn
->conn_fr_pool_size
++;
464 pr_debug("Creating conn %p fastreg pool size=%d",
465 isert_conn
, isert_conn
->conn_fr_pool_size
);
470 isert_conn_free_fastreg_pool(isert_conn
);
475 isert_connect_request(struct rdma_cm_id
*cma_id
, struct rdma_cm_event
*event
)
477 struct iscsi_np
*np
= cma_id
->context
;
478 struct isert_np
*isert_np
= np
->np_context
;
479 struct isert_conn
*isert_conn
;
480 struct isert_device
*device
;
481 struct ib_device
*ib_dev
= cma_id
->device
;
484 pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n",
485 cma_id
, cma_id
->context
);
487 isert_conn
= kzalloc(sizeof(struct isert_conn
), GFP_KERNEL
);
489 pr_err("Unable to allocate isert_conn\n");
492 isert_conn
->state
= ISER_CONN_INIT
;
493 INIT_LIST_HEAD(&isert_conn
->conn_accept_node
);
494 init_completion(&isert_conn
->conn_login_comp
);
495 init_waitqueue_head(&isert_conn
->conn_wait
);
496 init_waitqueue_head(&isert_conn
->conn_wait_comp_err
);
497 kref_init(&isert_conn
->conn_kref
);
498 kref_get(&isert_conn
->conn_kref
);
499 mutex_init(&isert_conn
->conn_mutex
);
500 mutex_init(&isert_conn
->conn_comp_mutex
);
501 spin_lock_init(&isert_conn
->conn_lock
);
503 cma_id
->context
= isert_conn
;
504 isert_conn
->conn_cm_id
= cma_id
;
505 isert_conn
->responder_resources
= event
->param
.conn
.responder_resources
;
506 isert_conn
->initiator_depth
= event
->param
.conn
.initiator_depth
;
507 pr_debug("Using responder_resources: %u initiator_depth: %u\n",
508 isert_conn
->responder_resources
, isert_conn
->initiator_depth
);
510 isert_conn
->login_buf
= kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN
+
511 ISER_RX_LOGIN_SIZE
, GFP_KERNEL
);
512 if (!isert_conn
->login_buf
) {
513 pr_err("Unable to allocate isert_conn->login_buf\n");
518 isert_conn
->login_req_buf
= isert_conn
->login_buf
;
519 isert_conn
->login_rsp_buf
= isert_conn
->login_buf
+
520 ISCSI_DEF_MAX_RECV_SEG_LEN
;
521 pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n",
522 isert_conn
->login_buf
, isert_conn
->login_req_buf
,
523 isert_conn
->login_rsp_buf
);
525 isert_conn
->login_req_dma
= ib_dma_map_single(ib_dev
,
526 (void *)isert_conn
->login_req_buf
,
527 ISCSI_DEF_MAX_RECV_SEG_LEN
, DMA_FROM_DEVICE
);
529 ret
= ib_dma_mapping_error(ib_dev
, isert_conn
->login_req_dma
);
531 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n",
533 isert_conn
->login_req_dma
= 0;
537 isert_conn
->login_rsp_dma
= ib_dma_map_single(ib_dev
,
538 (void *)isert_conn
->login_rsp_buf
,
539 ISER_RX_LOGIN_SIZE
, DMA_TO_DEVICE
);
541 ret
= ib_dma_mapping_error(ib_dev
, isert_conn
->login_rsp_dma
);
543 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n",
545 isert_conn
->login_rsp_dma
= 0;
546 goto out_req_dma_map
;
549 device
= isert_device_find_by_ib_dev(cma_id
);
550 if (IS_ERR(device
)) {
551 ret
= PTR_ERR(device
);
552 goto out_rsp_dma_map
;
555 isert_conn
->conn_device
= device
;
556 isert_conn
->conn_pd
= ib_alloc_pd(isert_conn
->conn_device
->ib_device
);
557 if (IS_ERR(isert_conn
->conn_pd
)) {
558 ret
= PTR_ERR(isert_conn
->conn_pd
);
559 pr_err("ib_alloc_pd failed for conn %p: ret=%d\n",
564 isert_conn
->conn_mr
= ib_get_dma_mr(isert_conn
->conn_pd
,
565 IB_ACCESS_LOCAL_WRITE
);
566 if (IS_ERR(isert_conn
->conn_mr
)) {
567 ret
= PTR_ERR(isert_conn
->conn_mr
);
568 pr_err("ib_get_dma_mr failed for conn %p: ret=%d\n",
573 if (device
->use_fastreg
) {
574 ret
= isert_conn_create_fastreg_pool(isert_conn
);
576 pr_err("Conn: %p failed to create fastreg pool\n",
582 ret
= isert_conn_setup_qp(isert_conn
, cma_id
);
586 mutex_lock(&isert_np
->np_accept_mutex
);
587 list_add_tail(&isert_np
->np_accept_list
, &isert_conn
->conn_accept_node
);
588 mutex_unlock(&isert_np
->np_accept_mutex
);
590 pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np
);
591 wake_up(&isert_np
->np_accept_wq
);
595 if (device
->use_fastreg
)
596 isert_conn_free_fastreg_pool(isert_conn
);
598 ib_dereg_mr(isert_conn
->conn_mr
);
600 ib_dealloc_pd(isert_conn
->conn_pd
);
602 isert_device_try_release(device
);
604 ib_dma_unmap_single(ib_dev
, isert_conn
->login_rsp_dma
,
605 ISER_RX_LOGIN_SIZE
, DMA_TO_DEVICE
);
607 ib_dma_unmap_single(ib_dev
, isert_conn
->login_req_dma
,
608 ISCSI_DEF_MAX_RECV_SEG_LEN
, DMA_FROM_DEVICE
);
610 kfree(isert_conn
->login_buf
);
617 isert_connect_release(struct isert_conn
*isert_conn
)
619 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
620 struct isert_device
*device
= isert_conn
->conn_device
;
623 pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
625 if (device
&& device
->use_fastreg
)
626 isert_conn_free_fastreg_pool(isert_conn
);
628 if (isert_conn
->conn_qp
) {
629 cq_index
= ((struct isert_cq_desc
*)
630 isert_conn
->conn_qp
->recv_cq
->cq_context
)->cq_index
;
631 pr_debug("isert_connect_release: cq_index: %d\n", cq_index
);
632 isert_conn
->conn_device
->cq_active_qps
[cq_index
]--;
634 rdma_destroy_qp(isert_conn
->conn_cm_id
);
637 isert_free_rx_descriptors(isert_conn
);
638 rdma_destroy_id(isert_conn
->conn_cm_id
);
640 ib_dereg_mr(isert_conn
->conn_mr
);
641 ib_dealloc_pd(isert_conn
->conn_pd
);
643 if (isert_conn
->login_buf
) {
644 ib_dma_unmap_single(ib_dev
, isert_conn
->login_rsp_dma
,
645 ISER_RX_LOGIN_SIZE
, DMA_TO_DEVICE
);
646 ib_dma_unmap_single(ib_dev
, isert_conn
->login_req_dma
,
647 ISCSI_DEF_MAX_RECV_SEG_LEN
,
649 kfree(isert_conn
->login_buf
);
654 isert_device_try_release(device
);
656 pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n");
660 isert_connected_handler(struct rdma_cm_id
*cma_id
)
666 isert_release_conn_kref(struct kref
*kref
)
668 struct isert_conn
*isert_conn
= container_of(kref
,
669 struct isert_conn
, conn_kref
);
671 pr_debug("Calling isert_connect_release for final kref %s/%d\n",
672 current
->comm
, current
->pid
);
674 isert_connect_release(isert_conn
);
678 isert_put_conn(struct isert_conn
*isert_conn
)
680 kref_put(&isert_conn
->conn_kref
, isert_release_conn_kref
);
684 isert_disconnect_work(struct work_struct
*work
)
686 struct isert_conn
*isert_conn
= container_of(work
,
687 struct isert_conn
, conn_logout_work
);
689 pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
690 mutex_lock(&isert_conn
->conn_mutex
);
691 isert_conn
->state
= ISER_CONN_DOWN
;
693 if (isert_conn
->post_recv_buf_count
== 0 &&
694 atomic_read(&isert_conn
->post_send_buf_count
) == 0) {
695 pr_debug("Calling wake_up(&isert_conn->conn_wait);\n");
696 mutex_unlock(&isert_conn
->conn_mutex
);
699 if (!isert_conn
->conn_cm_id
) {
700 mutex_unlock(&isert_conn
->conn_mutex
);
701 isert_put_conn(isert_conn
);
704 if (!isert_conn
->logout_posted
) {
705 pr_debug("Calling rdma_disconnect for !logout_posted from"
706 " isert_disconnect_work\n");
707 rdma_disconnect(isert_conn
->conn_cm_id
);
708 mutex_unlock(&isert_conn
->conn_mutex
);
709 iscsit_cause_connection_reinstatement(isert_conn
->conn
, 0);
712 mutex_unlock(&isert_conn
->conn_mutex
);
715 wake_up(&isert_conn
->conn_wait
);
716 isert_put_conn(isert_conn
);
720 isert_disconnected_handler(struct rdma_cm_id
*cma_id
)
722 struct isert_conn
*isert_conn
= (struct isert_conn
*)cma_id
->context
;
724 INIT_WORK(&isert_conn
->conn_logout_work
, isert_disconnect_work
);
725 schedule_work(&isert_conn
->conn_logout_work
);
729 isert_cma_handler(struct rdma_cm_id
*cma_id
, struct rdma_cm_event
*event
)
733 pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n",
734 event
->event
, event
->status
, cma_id
->context
, cma_id
);
736 switch (event
->event
) {
737 case RDMA_CM_EVENT_CONNECT_REQUEST
:
738 pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n");
739 ret
= isert_connect_request(cma_id
, event
);
741 case RDMA_CM_EVENT_ESTABLISHED
:
742 pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n");
743 isert_connected_handler(cma_id
);
745 case RDMA_CM_EVENT_DISCONNECTED
:
746 pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n");
747 isert_disconnected_handler(cma_id
);
749 case RDMA_CM_EVENT_DEVICE_REMOVAL
:
750 case RDMA_CM_EVENT_ADDR_CHANGE
:
752 case RDMA_CM_EVENT_CONNECT_ERROR
:
754 pr_err("Unknown RDMA CMA event: %d\n", event
->event
);
759 pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n",
768 isert_post_recv(struct isert_conn
*isert_conn
, u32 count
)
770 struct ib_recv_wr
*rx_wr
, *rx_wr_failed
;
772 unsigned int rx_head
= isert_conn
->conn_rx_desc_head
;
773 struct iser_rx_desc
*rx_desc
;
775 for (rx_wr
= isert_conn
->conn_rx_wr
, i
= 0; i
< count
; i
++, rx_wr
++) {
776 rx_desc
= &isert_conn
->conn_rx_descs
[rx_head
];
777 rx_wr
->wr_id
= (unsigned long)rx_desc
;
778 rx_wr
->sg_list
= &rx_desc
->rx_sg
;
780 rx_wr
->next
= rx_wr
+ 1;
781 rx_head
= (rx_head
+ 1) & (ISERT_QP_MAX_RECV_DTOS
- 1);
785 rx_wr
->next
= NULL
; /* mark end of work requests list */
787 isert_conn
->post_recv_buf_count
+= count
;
788 ret
= ib_post_recv(isert_conn
->conn_qp
, isert_conn
->conn_rx_wr
,
791 pr_err("ib_post_recv() failed with ret: %d\n", ret
);
792 isert_conn
->post_recv_buf_count
-= count
;
794 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count
);
795 isert_conn
->conn_rx_desc_head
= rx_head
;
801 isert_post_send(struct isert_conn
*isert_conn
, struct iser_tx_desc
*tx_desc
)
803 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
804 struct ib_send_wr send_wr
, *send_wr_failed
;
807 ib_dma_sync_single_for_device(ib_dev
, tx_desc
->dma_addr
,
808 ISER_HEADERS_LEN
, DMA_TO_DEVICE
);
811 send_wr
.wr_id
= (unsigned long)tx_desc
;
812 send_wr
.sg_list
= tx_desc
->tx_sg
;
813 send_wr
.num_sge
= tx_desc
->num_sge
;
814 send_wr
.opcode
= IB_WR_SEND
;
815 send_wr
.send_flags
= IB_SEND_SIGNALED
;
817 atomic_inc(&isert_conn
->post_send_buf_count
);
819 ret
= ib_post_send(isert_conn
->conn_qp
, &send_wr
, &send_wr_failed
);
821 pr_err("ib_post_send() failed, ret: %d\n", ret
);
822 atomic_dec(&isert_conn
->post_send_buf_count
);
829 isert_create_send_desc(struct isert_conn
*isert_conn
,
830 struct isert_cmd
*isert_cmd
,
831 struct iser_tx_desc
*tx_desc
)
833 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
835 ib_dma_sync_single_for_cpu(ib_dev
, tx_desc
->dma_addr
,
836 ISER_HEADERS_LEN
, DMA_TO_DEVICE
);
838 memset(&tx_desc
->iser_header
, 0, sizeof(struct iser_hdr
));
839 tx_desc
->iser_header
.flags
= ISER_VER
;
841 tx_desc
->num_sge
= 1;
842 tx_desc
->isert_cmd
= isert_cmd
;
844 if (tx_desc
->tx_sg
[0].lkey
!= isert_conn
->conn_mr
->lkey
) {
845 tx_desc
->tx_sg
[0].lkey
= isert_conn
->conn_mr
->lkey
;
846 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc
);
851 isert_init_tx_hdrs(struct isert_conn
*isert_conn
,
852 struct iser_tx_desc
*tx_desc
)
854 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
857 dma_addr
= ib_dma_map_single(ib_dev
, (void *)tx_desc
,
858 ISER_HEADERS_LEN
, DMA_TO_DEVICE
);
859 if (ib_dma_mapping_error(ib_dev
, dma_addr
)) {
860 pr_err("ib_dma_mapping_error() failed\n");
864 tx_desc
->dma_addr
= dma_addr
;
865 tx_desc
->tx_sg
[0].addr
= tx_desc
->dma_addr
;
866 tx_desc
->tx_sg
[0].length
= ISER_HEADERS_LEN
;
867 tx_desc
->tx_sg
[0].lkey
= isert_conn
->conn_mr
->lkey
;
869 pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u"
870 " lkey: 0x%08x\n", tx_desc
->tx_sg
[0].addr
,
871 tx_desc
->tx_sg
[0].length
, tx_desc
->tx_sg
[0].lkey
);
877 isert_init_send_wr(struct isert_conn
*isert_conn
, struct isert_cmd
*isert_cmd
,
878 struct ib_send_wr
*send_wr
, bool coalesce
)
880 struct iser_tx_desc
*tx_desc
= &isert_cmd
->tx_desc
;
882 isert_cmd
->rdma_wr
.iser_ib_op
= ISER_IB_SEND
;
883 send_wr
->wr_id
= (unsigned long)&isert_cmd
->tx_desc
;
884 send_wr
->opcode
= IB_WR_SEND
;
885 send_wr
->sg_list
= &tx_desc
->tx_sg
[0];
886 send_wr
->num_sge
= isert_cmd
->tx_desc
.num_sge
;
888 * Coalesce send completion interrupts by only setting IB_SEND_SIGNALED
889 * bit for every ISERT_COMP_BATCH_COUNT number of ib_post_send() calls.
891 mutex_lock(&isert_conn
->conn_comp_mutex
);
893 ++isert_conn
->conn_comp_batch
< ISERT_COMP_BATCH_COUNT
) {
894 llist_add(&tx_desc
->comp_llnode
, &isert_conn
->conn_comp_llist
);
895 mutex_unlock(&isert_conn
->conn_comp_mutex
);
898 isert_conn
->conn_comp_batch
= 0;
899 tx_desc
->comp_llnode_batch
= llist_del_all(&isert_conn
->conn_comp_llist
);
900 mutex_unlock(&isert_conn
->conn_comp_mutex
);
902 send_wr
->send_flags
= IB_SEND_SIGNALED
;
906 isert_rdma_post_recvl(struct isert_conn
*isert_conn
)
908 struct ib_recv_wr rx_wr
, *rx_wr_fail
;
912 memset(&sge
, 0, sizeof(struct ib_sge
));
913 sge
.addr
= isert_conn
->login_req_dma
;
914 sge
.length
= ISER_RX_LOGIN_SIZE
;
915 sge
.lkey
= isert_conn
->conn_mr
->lkey
;
917 pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n",
918 sge
.addr
, sge
.length
, sge
.lkey
);
920 memset(&rx_wr
, 0, sizeof(struct ib_recv_wr
));
921 rx_wr
.wr_id
= (unsigned long)isert_conn
->login_req_buf
;
922 rx_wr
.sg_list
= &sge
;
925 isert_conn
->post_recv_buf_count
++;
926 ret
= ib_post_recv(isert_conn
->conn_qp
, &rx_wr
, &rx_wr_fail
);
928 pr_err("ib_post_recv() failed: %d\n", ret
);
929 isert_conn
->post_recv_buf_count
--;
932 pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n");
937 isert_put_login_tx(struct iscsi_conn
*conn
, struct iscsi_login
*login
,
940 struct isert_conn
*isert_conn
= conn
->context
;
941 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
942 struct iser_tx_desc
*tx_desc
= &isert_conn
->conn_login_tx_desc
;
945 isert_create_send_desc(isert_conn
, NULL
, tx_desc
);
947 memcpy(&tx_desc
->iscsi_header
, &login
->rsp
[0],
948 sizeof(struct iscsi_hdr
));
950 isert_init_tx_hdrs(isert_conn
, tx_desc
);
953 struct ib_sge
*tx_dsg
= &tx_desc
->tx_sg
[1];
955 ib_dma_sync_single_for_cpu(ib_dev
, isert_conn
->login_rsp_dma
,
956 length
, DMA_TO_DEVICE
);
958 memcpy(isert_conn
->login_rsp_buf
, login
->rsp_buf
, length
);
960 ib_dma_sync_single_for_device(ib_dev
, isert_conn
->login_rsp_dma
,
961 length
, DMA_TO_DEVICE
);
963 tx_dsg
->addr
= isert_conn
->login_rsp_dma
;
964 tx_dsg
->length
= length
;
965 tx_dsg
->lkey
= isert_conn
->conn_mr
->lkey
;
966 tx_desc
->num_sge
= 2;
968 if (!login
->login_failed
) {
969 if (login
->login_complete
) {
970 ret
= isert_alloc_rx_descriptors(isert_conn
);
974 ret
= isert_post_recv(isert_conn
, ISERT_MIN_POSTED_RX
);
978 isert_conn
->state
= ISER_CONN_UP
;
982 ret
= isert_rdma_post_recvl(isert_conn
);
987 ret
= isert_post_send(isert_conn
, tx_desc
);
995 isert_rx_login_req(struct iser_rx_desc
*rx_desc
, int rx_buflen
,
996 struct isert_conn
*isert_conn
)
998 struct iscsi_conn
*conn
= isert_conn
->conn
;
999 struct iscsi_login
*login
= conn
->conn_login
;
1003 pr_err("conn->conn_login is NULL\n");
1008 if (login
->first_request
) {
1009 struct iscsi_login_req
*login_req
=
1010 (struct iscsi_login_req
*)&rx_desc
->iscsi_header
;
1012 * Setup the initial iscsi_login values from the leading
1013 * login request PDU.
1015 login
->leading_connection
= (!login_req
->tsih
) ? 1 : 0;
1016 login
->current_stage
=
1017 (login_req
->flags
& ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK
)
1019 login
->version_min
= login_req
->min_version
;
1020 login
->version_max
= login_req
->max_version
;
1021 memcpy(login
->isid
, login_req
->isid
, 6);
1022 login
->cmd_sn
= be32_to_cpu(login_req
->cmdsn
);
1023 login
->init_task_tag
= login_req
->itt
;
1024 login
->initial_exp_statsn
= be32_to_cpu(login_req
->exp_statsn
);
1025 login
->cid
= be16_to_cpu(login_req
->cid
);
1026 login
->tsih
= be16_to_cpu(login_req
->tsih
);
1029 memcpy(&login
->req
[0], (void *)&rx_desc
->iscsi_header
, ISCSI_HDR_LEN
);
1031 size
= min(rx_buflen
, MAX_KEY_VALUE_PAIRS
);
1032 pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n",
1033 size
, rx_buflen
, MAX_KEY_VALUE_PAIRS
);
1034 memcpy(login
->req_buf
, &rx_desc
->data
[0], size
);
1036 if (login
->first_request
) {
1037 complete(&isert_conn
->conn_login_comp
);
1040 schedule_delayed_work(&conn
->login_work
, 0);
1043 static struct iscsi_cmd
1044 *isert_allocate_cmd(struct iscsi_conn
*conn
)
1046 struct isert_conn
*isert_conn
= (struct isert_conn
*)conn
->context
;
1047 struct isert_cmd
*isert_cmd
;
1048 struct iscsi_cmd
*cmd
;
1050 cmd
= iscsit_allocate_cmd(conn
, TASK_INTERRUPTIBLE
);
1052 pr_err("Unable to allocate iscsi_cmd + isert_cmd\n");
1055 isert_cmd
= iscsit_priv_cmd(cmd
);
1056 isert_cmd
->conn
= isert_conn
;
1057 isert_cmd
->iscsi_cmd
= cmd
;
1063 isert_handle_scsi_cmd(struct isert_conn
*isert_conn
,
1064 struct isert_cmd
*isert_cmd
, struct iscsi_cmd
*cmd
,
1065 struct iser_rx_desc
*rx_desc
, unsigned char *buf
)
1067 struct iscsi_conn
*conn
= isert_conn
->conn
;
1068 struct iscsi_scsi_req
*hdr
= (struct iscsi_scsi_req
*)buf
;
1069 struct scatterlist
*sg
;
1070 int imm_data
, imm_data_len
, unsol_data
, sg_nents
, rc
;
1071 bool dump_payload
= false;
1073 rc
= iscsit_setup_scsi_cmd(conn
, cmd
, buf
);
1077 imm_data
= cmd
->immediate_data
;
1078 imm_data_len
= cmd
->first_burst_len
;
1079 unsol_data
= cmd
->unsolicited_data
;
1081 rc
= iscsit_process_scsi_cmd(conn
, cmd
, hdr
);
1084 } else if (rc
> 0) {
1085 dump_payload
= true;
1092 sg
= &cmd
->se_cmd
.t_data_sg
[0];
1093 sg_nents
= max(1UL, DIV_ROUND_UP(imm_data_len
, PAGE_SIZE
));
1095 pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n",
1096 sg
, sg_nents
, &rx_desc
->data
[0], imm_data_len
);
1098 sg_copy_from_buffer(sg
, sg_nents
, &rx_desc
->data
[0], imm_data_len
);
1100 cmd
->write_data_done
+= imm_data_len
;
1102 if (cmd
->write_data_done
== cmd
->se_cmd
.data_length
) {
1103 spin_lock_bh(&cmd
->istate_lock
);
1104 cmd
->cmd_flags
|= ICF_GOT_LAST_DATAOUT
;
1105 cmd
->i_state
= ISTATE_RECEIVED_LAST_DATAOUT
;
1106 spin_unlock_bh(&cmd
->istate_lock
);
1110 rc
= iscsit_sequence_cmd(conn
, cmd
, buf
, hdr
->cmdsn
);
1112 if (!rc
&& dump_payload
== false && unsol_data
)
1113 iscsit_set_unsoliticed_dataout(cmd
);
1119 isert_handle_iscsi_dataout(struct isert_conn
*isert_conn
,
1120 struct iser_rx_desc
*rx_desc
, unsigned char *buf
)
1122 struct scatterlist
*sg_start
;
1123 struct iscsi_conn
*conn
= isert_conn
->conn
;
1124 struct iscsi_cmd
*cmd
= NULL
;
1125 struct iscsi_data
*hdr
= (struct iscsi_data
*)buf
;
1126 u32 unsol_data_len
= ntoh24(hdr
->dlength
);
1127 int rc
, sg_nents
, sg_off
, page_off
;
1129 rc
= iscsit_check_dataout_hdr(conn
, buf
, &cmd
);
1135 * FIXME: Unexpected unsolicited_data out
1137 if (!cmd
->unsolicited_data
) {
1138 pr_err("Received unexpected solicited data payload\n");
1143 pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n",
1144 unsol_data_len
, cmd
->write_data_done
, cmd
->se_cmd
.data_length
);
1146 sg_off
= cmd
->write_data_done
/ PAGE_SIZE
;
1147 sg_start
= &cmd
->se_cmd
.t_data_sg
[sg_off
];
1148 sg_nents
= max(1UL, DIV_ROUND_UP(unsol_data_len
, PAGE_SIZE
));
1149 page_off
= cmd
->write_data_done
% PAGE_SIZE
;
1151 * FIXME: Non page-aligned unsolicited_data out
1154 pr_err("Received unexpected non-page aligned data payload\n");
1158 pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n",
1159 sg_start
, sg_off
, sg_nents
, &rx_desc
->data
[0], unsol_data_len
);
1161 sg_copy_from_buffer(sg_start
, sg_nents
, &rx_desc
->data
[0],
1164 rc
= iscsit_check_dataout_payload(cmd
, hdr
, false);
1172 isert_handle_nop_out(struct isert_conn
*isert_conn
, struct isert_cmd
*isert_cmd
,
1173 struct iscsi_cmd
*cmd
, struct iser_rx_desc
*rx_desc
,
1176 struct iscsi_conn
*conn
= isert_conn
->conn
;
1177 struct iscsi_nopout
*hdr
= (struct iscsi_nopout
*)buf
;
1180 rc
= iscsit_setup_nop_out(conn
, cmd
, hdr
);
1184 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload
1187 return iscsit_process_nop_out(conn
, cmd
, hdr
);
1191 isert_handle_text_cmd(struct isert_conn
*isert_conn
, struct isert_cmd
*isert_cmd
,
1192 struct iscsi_cmd
*cmd
, struct iser_rx_desc
*rx_desc
,
1193 struct iscsi_text
*hdr
)
1195 struct iscsi_conn
*conn
= isert_conn
->conn
;
1196 u32 payload_length
= ntoh24(hdr
->dlength
);
1198 unsigned char *text_in
;
1200 rc
= iscsit_setup_text_cmd(conn
, cmd
, hdr
);
1204 text_in
= kzalloc(payload_length
, GFP_KERNEL
);
1206 pr_err("Unable to allocate text_in of payload_length: %u\n",
1210 cmd
->text_in_ptr
= text_in
;
1212 memcpy(cmd
->text_in_ptr
, &rx_desc
->data
[0], payload_length
);
1214 return iscsit_process_text_cmd(conn
, cmd
, hdr
);
1218 isert_rx_opcode(struct isert_conn
*isert_conn
, struct iser_rx_desc
*rx_desc
,
1219 uint32_t read_stag
, uint64_t read_va
,
1220 uint32_t write_stag
, uint64_t write_va
)
1222 struct iscsi_hdr
*hdr
= &rx_desc
->iscsi_header
;
1223 struct iscsi_conn
*conn
= isert_conn
->conn
;
1224 struct iscsi_session
*sess
= conn
->sess
;
1225 struct iscsi_cmd
*cmd
;
1226 struct isert_cmd
*isert_cmd
;
1228 u8 opcode
= (hdr
->opcode
& ISCSI_OPCODE_MASK
);
1230 if (sess
->sess_ops
->SessionType
&&
1231 (!(opcode
& ISCSI_OP_TEXT
) || !(opcode
& ISCSI_OP_LOGOUT
))) {
1232 pr_err("Got illegal opcode: 0x%02x in SessionType=Discovery,"
1233 " ignoring\n", opcode
);
1238 case ISCSI_OP_SCSI_CMD
:
1239 cmd
= isert_allocate_cmd(conn
);
1243 isert_cmd
= iscsit_priv_cmd(cmd
);
1244 isert_cmd
->read_stag
= read_stag
;
1245 isert_cmd
->read_va
= read_va
;
1246 isert_cmd
->write_stag
= write_stag
;
1247 isert_cmd
->write_va
= write_va
;
1249 ret
= isert_handle_scsi_cmd(isert_conn
, isert_cmd
, cmd
,
1250 rx_desc
, (unsigned char *)hdr
);
1252 case ISCSI_OP_NOOP_OUT
:
1253 cmd
= isert_allocate_cmd(conn
);
1257 isert_cmd
= iscsit_priv_cmd(cmd
);
1258 ret
= isert_handle_nop_out(isert_conn
, isert_cmd
, cmd
,
1259 rx_desc
, (unsigned char *)hdr
);
1261 case ISCSI_OP_SCSI_DATA_OUT
:
1262 ret
= isert_handle_iscsi_dataout(isert_conn
, rx_desc
,
1263 (unsigned char *)hdr
);
1265 case ISCSI_OP_SCSI_TMFUNC
:
1266 cmd
= isert_allocate_cmd(conn
);
1270 ret
= iscsit_handle_task_mgt_cmd(conn
, cmd
,
1271 (unsigned char *)hdr
);
1273 case ISCSI_OP_LOGOUT
:
1274 cmd
= isert_allocate_cmd(conn
);
1278 ret
= iscsit_handle_logout_cmd(conn
, cmd
, (unsigned char *)hdr
);
1280 wait_for_completion_timeout(&conn
->conn_logout_comp
,
1281 SECONDS_FOR_LOGOUT_COMP
*
1285 cmd
= isert_allocate_cmd(conn
);
1289 isert_cmd
= iscsit_priv_cmd(cmd
);
1290 ret
= isert_handle_text_cmd(isert_conn
, isert_cmd
, cmd
,
1291 rx_desc
, (struct iscsi_text
*)hdr
);
1294 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode
);
1303 isert_rx_do_work(struct iser_rx_desc
*rx_desc
, struct isert_conn
*isert_conn
)
1305 struct iser_hdr
*iser_hdr
= &rx_desc
->iser_header
;
1306 uint64_t read_va
= 0, write_va
= 0;
1307 uint32_t read_stag
= 0, write_stag
= 0;
1310 switch (iser_hdr
->flags
& 0xF0) {
1312 if (iser_hdr
->flags
& ISER_RSV
) {
1313 read_stag
= be32_to_cpu(iser_hdr
->read_stag
);
1314 read_va
= be64_to_cpu(iser_hdr
->read_va
);
1315 pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n",
1316 read_stag
, (unsigned long long)read_va
);
1318 if (iser_hdr
->flags
& ISER_WSV
) {
1319 write_stag
= be32_to_cpu(iser_hdr
->write_stag
);
1320 write_va
= be64_to_cpu(iser_hdr
->write_va
);
1321 pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n",
1322 write_stag
, (unsigned long long)write_va
);
1325 pr_debug("ISER ISCSI_CTRL PDU\n");
1328 pr_err("iSER Hello message\n");
1331 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr
->flags
);
1335 rc
= isert_rx_opcode(isert_conn
, rx_desc
,
1336 read_stag
, read_va
, write_stag
, write_va
);
1340 isert_rx_completion(struct iser_rx_desc
*desc
, struct isert_conn
*isert_conn
,
1341 unsigned long xfer_len
)
1343 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1344 struct iscsi_hdr
*hdr
;
1346 int rx_buflen
, outstanding
;
1348 if ((char *)desc
== isert_conn
->login_req_buf
) {
1349 rx_dma
= isert_conn
->login_req_dma
;
1350 rx_buflen
= ISER_RX_LOGIN_SIZE
;
1351 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1354 rx_dma
= desc
->dma_addr
;
1355 rx_buflen
= ISER_RX_PAYLOAD_SIZE
;
1356 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1360 ib_dma_sync_single_for_cpu(ib_dev
, rx_dma
, rx_buflen
, DMA_FROM_DEVICE
);
1362 hdr
= &desc
->iscsi_header
;
1363 pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
1364 hdr
->opcode
, hdr
->itt
, hdr
->flags
,
1365 (int)(xfer_len
- ISER_HEADERS_LEN
));
1367 if ((char *)desc
== isert_conn
->login_req_buf
)
1368 isert_rx_login_req(desc
, xfer_len
- ISER_HEADERS_LEN
,
1371 isert_rx_do_work(desc
, isert_conn
);
1373 ib_dma_sync_single_for_device(ib_dev
, rx_dma
, rx_buflen
,
1376 isert_conn
->post_recv_buf_count
--;
1377 pr_debug("iSERT: Decremented post_recv_buf_count: %d\n",
1378 isert_conn
->post_recv_buf_count
);
1380 if ((char *)desc
== isert_conn
->login_req_buf
)
1383 outstanding
= isert_conn
->post_recv_buf_count
;
1384 if (outstanding
+ ISERT_MIN_POSTED_RX
<= ISERT_QP_MAX_RECV_DTOS
) {
1385 int err
, count
= min(ISERT_QP_MAX_RECV_DTOS
- outstanding
,
1386 ISERT_MIN_POSTED_RX
);
1387 err
= isert_post_recv(isert_conn
, count
);
1389 pr_err("isert_post_recv() count: %d failed, %d\n",
1396 isert_unmap_cmd(struct isert_cmd
*isert_cmd
, struct isert_conn
*isert_conn
)
1398 struct isert_rdma_wr
*wr
= &isert_cmd
->rdma_wr
;
1399 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1401 pr_debug("isert_unmap_cmd: %p\n", isert_cmd
);
1403 pr_debug("isert_unmap_cmd: %p unmap_sg op\n", isert_cmd
);
1404 ib_dma_unmap_sg(ib_dev
, wr
->sge
, wr
->num_sge
,
1405 (wr
->iser_ib_op
== ISER_IB_RDMA_WRITE
) ?
1406 DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
1411 pr_debug("isert_unmap_cmd: %p free send_wr\n", isert_cmd
);
1417 pr_debug("isert_unmap_cmd: %p free ib_sge\n", isert_cmd
);
1424 isert_unreg_rdma(struct isert_cmd
*isert_cmd
, struct isert_conn
*isert_conn
)
1426 struct isert_rdma_wr
*wr
= &isert_cmd
->rdma_wr
;
1427 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1428 LIST_HEAD(unmap_list
);
1430 pr_debug("unreg_fastreg_cmd: %p\n", isert_cmd
);
1433 pr_debug("unreg_fastreg_cmd: %p free fr_desc %p\n",
1434 isert_cmd
, wr
->fr_desc
);
1435 spin_lock_bh(&isert_conn
->conn_lock
);
1436 list_add_tail(&wr
->fr_desc
->list
, &isert_conn
->conn_fr_pool
);
1437 spin_unlock_bh(&isert_conn
->conn_lock
);
1442 pr_debug("unreg_fastreg_cmd: %p unmap_sg op\n", isert_cmd
);
1443 ib_dma_unmap_sg(ib_dev
, wr
->sge
, wr
->num_sge
,
1444 (wr
->iser_ib_op
== ISER_IB_RDMA_WRITE
) ?
1445 DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
1454 isert_put_cmd(struct isert_cmd
*isert_cmd
)
1456 struct iscsi_cmd
*cmd
= isert_cmd
->iscsi_cmd
;
1457 struct isert_conn
*isert_conn
= isert_cmd
->conn
;
1458 struct iscsi_conn
*conn
= isert_conn
->conn
;
1459 struct isert_device
*device
= isert_conn
->conn_device
;
1461 pr_debug("Entering isert_put_cmd: %p\n", isert_cmd
);
1463 switch (cmd
->iscsi_opcode
) {
1464 case ISCSI_OP_SCSI_CMD
:
1465 spin_lock_bh(&conn
->cmd_lock
);
1466 if (!list_empty(&cmd
->i_conn_node
))
1467 list_del(&cmd
->i_conn_node
);
1468 spin_unlock_bh(&conn
->cmd_lock
);
1470 if (cmd
->data_direction
== DMA_TO_DEVICE
)
1471 iscsit_stop_dataout_timer(cmd
);
1473 device
->unreg_rdma_mem(isert_cmd
, isert_conn
);
1474 transport_generic_free_cmd(&cmd
->se_cmd
, 0);
1476 case ISCSI_OP_SCSI_TMFUNC
:
1477 spin_lock_bh(&conn
->cmd_lock
);
1478 if (!list_empty(&cmd
->i_conn_node
))
1479 list_del(&cmd
->i_conn_node
);
1480 spin_unlock_bh(&conn
->cmd_lock
);
1482 transport_generic_free_cmd(&cmd
->se_cmd
, 0);
1484 case ISCSI_OP_REJECT
:
1485 case ISCSI_OP_NOOP_OUT
:
1487 spin_lock_bh(&conn
->cmd_lock
);
1488 if (!list_empty(&cmd
->i_conn_node
))
1489 list_del(&cmd
->i_conn_node
);
1490 spin_unlock_bh(&conn
->cmd_lock
);
1493 * Handle special case for REJECT when iscsi_add_reject*() has
1494 * overwritten the original iscsi_opcode assignment, and the
1495 * associated cmd->se_cmd needs to be released.
1497 if (cmd
->se_cmd
.se_tfo
!= NULL
) {
1498 pr_debug("Calling transport_generic_free_cmd from"
1499 " isert_put_cmd for 0x%02x\n",
1501 transport_generic_free_cmd(&cmd
->se_cmd
, 0);
1508 iscsit_release_cmd(cmd
);
1514 isert_unmap_tx_desc(struct iser_tx_desc
*tx_desc
, struct ib_device
*ib_dev
)
1516 if (tx_desc
->dma_addr
!= 0) {
1517 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n");
1518 ib_dma_unmap_single(ib_dev
, tx_desc
->dma_addr
,
1519 ISER_HEADERS_LEN
, DMA_TO_DEVICE
);
1520 tx_desc
->dma_addr
= 0;
1525 isert_completion_put(struct iser_tx_desc
*tx_desc
, struct isert_cmd
*isert_cmd
,
1526 struct ib_device
*ib_dev
)
1528 if (isert_cmd
->pdu_buf_dma
!= 0) {
1529 pr_debug("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n");
1530 ib_dma_unmap_single(ib_dev
, isert_cmd
->pdu_buf_dma
,
1531 isert_cmd
->pdu_buf_len
, DMA_TO_DEVICE
);
1532 isert_cmd
->pdu_buf_dma
= 0;
1535 isert_unmap_tx_desc(tx_desc
, ib_dev
);
1536 isert_put_cmd(isert_cmd
);
1540 isert_completion_rdma_read(struct iser_tx_desc
*tx_desc
,
1541 struct isert_cmd
*isert_cmd
)
1543 struct isert_rdma_wr
*wr
= &isert_cmd
->rdma_wr
;
1544 struct iscsi_cmd
*cmd
= isert_cmd
->iscsi_cmd
;
1545 struct se_cmd
*se_cmd
= &cmd
->se_cmd
;
1546 struct isert_conn
*isert_conn
= isert_cmd
->conn
;
1547 struct isert_device
*device
= isert_conn
->conn_device
;
1549 iscsit_stop_dataout_timer(cmd
);
1550 device
->unreg_rdma_mem(isert_cmd
, isert_conn
);
1551 cmd
->write_data_done
= wr
->cur_rdma_length
;
1553 pr_debug("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd
);
1554 spin_lock_bh(&cmd
->istate_lock
);
1555 cmd
->cmd_flags
|= ICF_GOT_LAST_DATAOUT
;
1556 cmd
->i_state
= ISTATE_RECEIVED_LAST_DATAOUT
;
1557 spin_unlock_bh(&cmd
->istate_lock
);
1559 target_execute_cmd(se_cmd
);
1563 isert_do_control_comp(struct work_struct
*work
)
1565 struct isert_cmd
*isert_cmd
= container_of(work
,
1566 struct isert_cmd
, comp_work
);
1567 struct isert_conn
*isert_conn
= isert_cmd
->conn
;
1568 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1569 struct iscsi_cmd
*cmd
= isert_cmd
->iscsi_cmd
;
1571 switch (cmd
->i_state
) {
1572 case ISTATE_SEND_TASKMGTRSP
:
1573 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n");
1575 atomic_dec(&isert_conn
->post_send_buf_count
);
1576 iscsit_tmr_post_handler(cmd
, cmd
->conn
);
1578 cmd
->i_state
= ISTATE_SENT_STATUS
;
1579 isert_completion_put(&isert_cmd
->tx_desc
, isert_cmd
, ib_dev
);
1581 case ISTATE_SEND_REJECT
:
1582 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n");
1583 atomic_dec(&isert_conn
->post_send_buf_count
);
1585 cmd
->i_state
= ISTATE_SENT_STATUS
;
1586 isert_completion_put(&isert_cmd
->tx_desc
, isert_cmd
, ib_dev
);
1588 case ISTATE_SEND_LOGOUTRSP
:
1589 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n");
1591 * Call atomic_dec(&isert_conn->post_send_buf_count)
1592 * from isert_free_conn()
1594 isert_conn
->logout_posted
= true;
1595 iscsit_logout_post_handler(cmd
, cmd
->conn
);
1597 case ISTATE_SEND_TEXTRSP
:
1598 atomic_dec(&isert_conn
->post_send_buf_count
);
1599 cmd
->i_state
= ISTATE_SENT_STATUS
;
1600 isert_completion_put(&isert_cmd
->tx_desc
, isert_cmd
, ib_dev
);
1603 pr_err("Unknown do_control_comp i_state %d\n", cmd
->i_state
);
1610 isert_response_completion(struct iser_tx_desc
*tx_desc
,
1611 struct isert_cmd
*isert_cmd
,
1612 struct isert_conn
*isert_conn
,
1613 struct ib_device
*ib_dev
)
1615 struct iscsi_cmd
*cmd
= isert_cmd
->iscsi_cmd
;
1617 if (cmd
->i_state
== ISTATE_SEND_TASKMGTRSP
||
1618 cmd
->i_state
== ISTATE_SEND_LOGOUTRSP
||
1619 cmd
->i_state
== ISTATE_SEND_REJECT
||
1620 cmd
->i_state
== ISTATE_SEND_TEXTRSP
) {
1621 isert_unmap_tx_desc(tx_desc
, ib_dev
);
1623 INIT_WORK(&isert_cmd
->comp_work
, isert_do_control_comp
);
1624 queue_work(isert_comp_wq
, &isert_cmd
->comp_work
);
1627 atomic_dec(&isert_conn
->post_send_buf_count
);
1629 cmd
->i_state
= ISTATE_SENT_STATUS
;
1630 isert_completion_put(tx_desc
, isert_cmd
, ib_dev
);
1634 __isert_send_completion(struct iser_tx_desc
*tx_desc
,
1635 struct isert_conn
*isert_conn
)
1637 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1638 struct isert_cmd
*isert_cmd
= tx_desc
->isert_cmd
;
1639 struct isert_rdma_wr
*wr
;
1642 atomic_dec(&isert_conn
->post_send_buf_count
);
1643 isert_unmap_tx_desc(tx_desc
, ib_dev
);
1646 wr
= &isert_cmd
->rdma_wr
;
1648 switch (wr
->iser_ib_op
) {
1650 pr_err("isert_send_completion: Got ISER_IB_RECV\n");
1654 pr_debug("isert_send_completion: Got ISER_IB_SEND\n");
1655 isert_response_completion(tx_desc
, isert_cmd
,
1656 isert_conn
, ib_dev
);
1658 case ISER_IB_RDMA_WRITE
:
1659 pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n");
1662 case ISER_IB_RDMA_READ
:
1663 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n");
1665 atomic_dec(&isert_conn
->post_send_buf_count
);
1666 isert_completion_rdma_read(tx_desc
, isert_cmd
);
1669 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr
->iser_ib_op
);
1676 isert_send_completion(struct iser_tx_desc
*tx_desc
,
1677 struct isert_conn
*isert_conn
)
1679 struct llist_node
*llnode
= tx_desc
->comp_llnode_batch
;
1680 struct iser_tx_desc
*t
;
1682 * Drain coalesced completion llist starting from comp_llnode_batch
1683 * setup in isert_init_send_wr(), and then complete trailing tx_desc.
1686 t
= llist_entry(llnode
, struct iser_tx_desc
, comp_llnode
);
1687 llnode
= llist_next(llnode
);
1688 __isert_send_completion(t
, isert_conn
);
1690 __isert_send_completion(tx_desc
, isert_conn
);
1694 isert_cq_comp_err(struct iser_tx_desc
*tx_desc
, struct isert_conn
*isert_conn
)
1696 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1699 struct isert_cmd
*isert_cmd
= tx_desc
->isert_cmd
;
1702 isert_unmap_tx_desc(tx_desc
, ib_dev
);
1704 isert_completion_put(tx_desc
, isert_cmd
, ib_dev
);
1707 if (isert_conn
->post_recv_buf_count
== 0 &&
1708 atomic_read(&isert_conn
->post_send_buf_count
) == 0) {
1709 pr_debug("isert_cq_comp_err >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
1710 pr_debug("Calling wake_up from isert_cq_comp_err\n");
1712 mutex_lock(&isert_conn
->conn_mutex
);
1713 if (isert_conn
->state
!= ISER_CONN_DOWN
)
1714 isert_conn
->state
= ISER_CONN_TERMINATING
;
1715 mutex_unlock(&isert_conn
->conn_mutex
);
1717 wake_up(&isert_conn
->conn_wait_comp_err
);
1722 isert_cq_tx_work(struct work_struct
*work
)
1724 struct isert_cq_desc
*cq_desc
= container_of(work
,
1725 struct isert_cq_desc
, cq_tx_work
);
1726 struct isert_device
*device
= cq_desc
->device
;
1727 int cq_index
= cq_desc
->cq_index
;
1728 struct ib_cq
*tx_cq
= device
->dev_tx_cq
[cq_index
];
1729 struct isert_conn
*isert_conn
;
1730 struct iser_tx_desc
*tx_desc
;
1733 while (ib_poll_cq(tx_cq
, 1, &wc
) == 1) {
1734 tx_desc
= (struct iser_tx_desc
*)(unsigned long)wc
.wr_id
;
1735 isert_conn
= wc
.qp
->qp_context
;
1737 if (wc
.status
== IB_WC_SUCCESS
) {
1738 isert_send_completion(tx_desc
, isert_conn
);
1740 pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
1741 pr_debug("TX wc.status: 0x%08x\n", wc
.status
);
1742 pr_debug("TX wc.vendor_err: 0x%08x\n", wc
.vendor_err
);
1743 atomic_dec(&isert_conn
->post_send_buf_count
);
1744 isert_cq_comp_err(tx_desc
, isert_conn
);
1748 ib_req_notify_cq(tx_cq
, IB_CQ_NEXT_COMP
);
1752 isert_cq_tx_callback(struct ib_cq
*cq
, void *context
)
1754 struct isert_cq_desc
*cq_desc
= (struct isert_cq_desc
*)context
;
1756 queue_work(isert_comp_wq
, &cq_desc
->cq_tx_work
);
1760 isert_cq_rx_work(struct work_struct
*work
)
1762 struct isert_cq_desc
*cq_desc
= container_of(work
,
1763 struct isert_cq_desc
, cq_rx_work
);
1764 struct isert_device
*device
= cq_desc
->device
;
1765 int cq_index
= cq_desc
->cq_index
;
1766 struct ib_cq
*rx_cq
= device
->dev_rx_cq
[cq_index
];
1767 struct isert_conn
*isert_conn
;
1768 struct iser_rx_desc
*rx_desc
;
1770 unsigned long xfer_len
;
1772 while (ib_poll_cq(rx_cq
, 1, &wc
) == 1) {
1773 rx_desc
= (struct iser_rx_desc
*)(unsigned long)wc
.wr_id
;
1774 isert_conn
= wc
.qp
->qp_context
;
1776 if (wc
.status
== IB_WC_SUCCESS
) {
1777 xfer_len
= (unsigned long)wc
.byte_len
;
1778 isert_rx_completion(rx_desc
, isert_conn
, xfer_len
);
1780 pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
1781 if (wc
.status
!= IB_WC_WR_FLUSH_ERR
) {
1782 pr_debug("RX wc.status: 0x%08x\n", wc
.status
);
1783 pr_debug("RX wc.vendor_err: 0x%08x\n",
1786 isert_conn
->post_recv_buf_count
--;
1787 isert_cq_comp_err(NULL
, isert_conn
);
1791 ib_req_notify_cq(rx_cq
, IB_CQ_NEXT_COMP
);
1795 isert_cq_rx_callback(struct ib_cq
*cq
, void *context
)
1797 struct isert_cq_desc
*cq_desc
= (struct isert_cq_desc
*)context
;
1799 queue_work(isert_rx_wq
, &cq_desc
->cq_rx_work
);
1803 isert_post_response(struct isert_conn
*isert_conn
, struct isert_cmd
*isert_cmd
)
1805 struct ib_send_wr
*wr_failed
;
1808 atomic_inc(&isert_conn
->post_send_buf_count
);
1810 ret
= ib_post_send(isert_conn
->conn_qp
, &isert_cmd
->tx_desc
.send_wr
,
1813 pr_err("ib_post_send failed with %d\n", ret
);
1814 atomic_dec(&isert_conn
->post_send_buf_count
);
1821 isert_put_response(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
)
1823 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
1824 struct isert_conn
*isert_conn
= (struct isert_conn
*)conn
->context
;
1825 struct ib_send_wr
*send_wr
= &isert_cmd
->tx_desc
.send_wr
;
1826 struct iscsi_scsi_rsp
*hdr
= (struct iscsi_scsi_rsp
*)
1827 &isert_cmd
->tx_desc
.iscsi_header
;
1829 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
1830 iscsit_build_rsp_pdu(cmd
, conn
, true, hdr
);
1831 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
1833 * Attach SENSE DATA payload to iSCSI Response PDU
1835 if (cmd
->se_cmd
.sense_buffer
&&
1836 ((cmd
->se_cmd
.se_cmd_flags
& SCF_TRANSPORT_TASK_SENSE
) ||
1837 (cmd
->se_cmd
.se_cmd_flags
& SCF_EMULATED_TASK_SENSE
))) {
1838 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1839 struct ib_sge
*tx_dsg
= &isert_cmd
->tx_desc
.tx_sg
[1];
1840 u32 padding
, pdu_len
;
1842 put_unaligned_be16(cmd
->se_cmd
.scsi_sense_length
,
1844 cmd
->se_cmd
.scsi_sense_length
+= sizeof(__be16
);
1846 padding
= -(cmd
->se_cmd
.scsi_sense_length
) & 3;
1847 hton24(hdr
->dlength
, (u32
)cmd
->se_cmd
.scsi_sense_length
);
1848 pdu_len
= cmd
->se_cmd
.scsi_sense_length
+ padding
;
1850 isert_cmd
->pdu_buf_dma
= ib_dma_map_single(ib_dev
,
1851 (void *)cmd
->sense_buffer
, pdu_len
,
1854 isert_cmd
->pdu_buf_len
= pdu_len
;
1855 tx_dsg
->addr
= isert_cmd
->pdu_buf_dma
;
1856 tx_dsg
->length
= pdu_len
;
1857 tx_dsg
->lkey
= isert_conn
->conn_mr
->lkey
;
1858 isert_cmd
->tx_desc
.num_sge
= 2;
1861 isert_init_send_wr(isert_conn
, isert_cmd
, send_wr
, true);
1863 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1865 return isert_post_response(isert_conn
, isert_cmd
);
1869 isert_put_nopin(struct iscsi_cmd
*cmd
, struct iscsi_conn
*conn
,
1870 bool nopout_response
)
1872 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
1873 struct isert_conn
*isert_conn
= (struct isert_conn
*)conn
->context
;
1874 struct ib_send_wr
*send_wr
= &isert_cmd
->tx_desc
.send_wr
;
1876 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
1877 iscsit_build_nopin_rsp(cmd
, conn
, (struct iscsi_nopin
*)
1878 &isert_cmd
->tx_desc
.iscsi_header
,
1880 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
1881 isert_init_send_wr(isert_conn
, isert_cmd
, send_wr
, false);
1883 pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1885 return isert_post_response(isert_conn
, isert_cmd
);
1889 isert_put_logout_rsp(struct iscsi_cmd
*cmd
, struct iscsi_conn
*conn
)
1891 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
1892 struct isert_conn
*isert_conn
= (struct isert_conn
*)conn
->context
;
1893 struct ib_send_wr
*send_wr
= &isert_cmd
->tx_desc
.send_wr
;
1895 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
1896 iscsit_build_logout_rsp(cmd
, conn
, (struct iscsi_logout_rsp
*)
1897 &isert_cmd
->tx_desc
.iscsi_header
);
1898 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
1899 isert_init_send_wr(isert_conn
, isert_cmd
, send_wr
, false);
1901 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1903 return isert_post_response(isert_conn
, isert_cmd
);
1907 isert_put_tm_rsp(struct iscsi_cmd
*cmd
, struct iscsi_conn
*conn
)
1909 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
1910 struct isert_conn
*isert_conn
= (struct isert_conn
*)conn
->context
;
1911 struct ib_send_wr
*send_wr
= &isert_cmd
->tx_desc
.send_wr
;
1913 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
1914 iscsit_build_task_mgt_rsp(cmd
, conn
, (struct iscsi_tm_rsp
*)
1915 &isert_cmd
->tx_desc
.iscsi_header
);
1916 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
1917 isert_init_send_wr(isert_conn
, isert_cmd
, send_wr
, false);
1919 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1921 return isert_post_response(isert_conn
, isert_cmd
);
1925 isert_put_reject(struct iscsi_cmd
*cmd
, struct iscsi_conn
*conn
)
1927 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
1928 struct isert_conn
*isert_conn
= (struct isert_conn
*)conn
->context
;
1929 struct ib_send_wr
*send_wr
= &isert_cmd
->tx_desc
.send_wr
;
1930 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1931 struct ib_sge
*tx_dsg
= &isert_cmd
->tx_desc
.tx_sg
[1];
1932 struct iscsi_reject
*hdr
=
1933 (struct iscsi_reject
*)&isert_cmd
->tx_desc
.iscsi_header
;
1935 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
1936 iscsit_build_reject(cmd
, conn
, hdr
);
1937 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
1939 hton24(hdr
->dlength
, ISCSI_HDR_LEN
);
1940 isert_cmd
->pdu_buf_dma
= ib_dma_map_single(ib_dev
,
1941 (void *)cmd
->buf_ptr
, ISCSI_HDR_LEN
,
1943 isert_cmd
->pdu_buf_len
= ISCSI_HDR_LEN
;
1944 tx_dsg
->addr
= isert_cmd
->pdu_buf_dma
;
1945 tx_dsg
->length
= ISCSI_HDR_LEN
;
1946 tx_dsg
->lkey
= isert_conn
->conn_mr
->lkey
;
1947 isert_cmd
->tx_desc
.num_sge
= 2;
1949 isert_init_send_wr(isert_conn
, isert_cmd
, send_wr
, false);
1951 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1953 return isert_post_response(isert_conn
, isert_cmd
);
1957 isert_put_text_rsp(struct iscsi_cmd
*cmd
, struct iscsi_conn
*conn
)
1959 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
1960 struct isert_conn
*isert_conn
= (struct isert_conn
*)conn
->context
;
1961 struct ib_send_wr
*send_wr
= &isert_cmd
->tx_desc
.send_wr
;
1962 struct iscsi_text_rsp
*hdr
=
1963 (struct iscsi_text_rsp
*)&isert_cmd
->tx_desc
.iscsi_header
;
1967 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
1968 rc
= iscsit_build_text_rsp(cmd
, conn
, hdr
);
1973 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
1976 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
1977 struct ib_sge
*tx_dsg
= &isert_cmd
->tx_desc
.tx_sg
[1];
1978 void *txt_rsp_buf
= cmd
->buf_ptr
;
1980 isert_cmd
->pdu_buf_dma
= ib_dma_map_single(ib_dev
,
1981 txt_rsp_buf
, txt_rsp_len
, DMA_TO_DEVICE
);
1983 isert_cmd
->pdu_buf_len
= txt_rsp_len
;
1984 tx_dsg
->addr
= isert_cmd
->pdu_buf_dma
;
1985 tx_dsg
->length
= txt_rsp_len
;
1986 tx_dsg
->lkey
= isert_conn
->conn_mr
->lkey
;
1987 isert_cmd
->tx_desc
.num_sge
= 2;
1989 isert_init_send_wr(isert_conn
, isert_cmd
, send_wr
, false);
1991 pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1993 return isert_post_response(isert_conn
, isert_cmd
);
1997 isert_build_rdma_wr(struct isert_conn
*isert_conn
, struct isert_cmd
*isert_cmd
,
1998 struct ib_sge
*ib_sge
, struct ib_send_wr
*send_wr
,
1999 u32 data_left
, u32 offset
)
2001 struct iscsi_cmd
*cmd
= isert_cmd
->iscsi_cmd
;
2002 struct scatterlist
*sg_start
, *tmp_sg
;
2003 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
2004 u32 sg_off
, page_off
;
2005 int i
= 0, sg_nents
;
2007 sg_off
= offset
/ PAGE_SIZE
;
2008 sg_start
= &cmd
->se_cmd
.t_data_sg
[sg_off
];
2009 sg_nents
= min(cmd
->se_cmd
.t_data_nents
- sg_off
, isert_conn
->max_sge
);
2010 page_off
= offset
% PAGE_SIZE
;
2012 send_wr
->sg_list
= ib_sge
;
2013 send_wr
->num_sge
= sg_nents
;
2014 send_wr
->wr_id
= (unsigned long)&isert_cmd
->tx_desc
;
2016 * Perform mapping of TCM scatterlist memory ib_sge dma_addr.
2018 for_each_sg(sg_start
, tmp_sg
, sg_nents
, i
) {
2019 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n",
2020 (unsigned long long)tmp_sg
->dma_address
,
2021 tmp_sg
->length
, page_off
);
2023 ib_sge
->addr
= ib_sg_dma_address(ib_dev
, tmp_sg
) + page_off
;
2024 ib_sge
->length
= min_t(u32
, data_left
,
2025 ib_sg_dma_len(ib_dev
, tmp_sg
) - page_off
);
2026 ib_sge
->lkey
= isert_conn
->conn_mr
->lkey
;
2028 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n",
2029 ib_sge
->addr
, ib_sge
->length
, ib_sge
->lkey
);
2031 data_left
-= ib_sge
->length
;
2033 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge
);
2036 pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n",
2037 send_wr
->sg_list
, send_wr
->num_sge
);
2043 isert_map_rdma(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
,
2044 struct isert_rdma_wr
*wr
)
2046 struct se_cmd
*se_cmd
= &cmd
->se_cmd
;
2047 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2048 struct isert_conn
*isert_conn
= (struct isert_conn
*)conn
->context
;
2049 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
2050 struct ib_send_wr
*send_wr
;
2051 struct ib_sge
*ib_sge
;
2052 struct scatterlist
*sg_start
;
2053 u32 sg_off
= 0, sg_nents
;
2054 u32 offset
= 0, data_len
, data_left
, rdma_write_max
, va_offset
= 0;
2055 int ret
= 0, count
, i
, ib_sge_cnt
;
2057 if (wr
->iser_ib_op
== ISER_IB_RDMA_WRITE
) {
2058 data_left
= se_cmd
->data_length
;
2060 sg_off
= cmd
->write_data_done
/ PAGE_SIZE
;
2061 data_left
= se_cmd
->data_length
- cmd
->write_data_done
;
2062 offset
= cmd
->write_data_done
;
2063 isert_cmd
->tx_desc
.isert_cmd
= isert_cmd
;
2066 sg_start
= &cmd
->se_cmd
.t_data_sg
[sg_off
];
2067 sg_nents
= se_cmd
->t_data_nents
- sg_off
;
2069 count
= ib_dma_map_sg(ib_dev
, sg_start
, sg_nents
,
2070 (wr
->iser_ib_op
== ISER_IB_RDMA_WRITE
) ?
2071 DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
2072 if (unlikely(!count
)) {
2073 pr_err("Cmd: %p unrable to map SGs\n", isert_cmd
);
2077 wr
->num_sge
= sg_nents
;
2078 wr
->cur_rdma_length
= data_left
;
2079 pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n",
2080 isert_cmd
, count
, sg_start
, sg_nents
, data_left
);
2082 ib_sge
= kzalloc(sizeof(struct ib_sge
) * sg_nents
, GFP_KERNEL
);
2084 pr_warn("Unable to allocate ib_sge\n");
2088 wr
->ib_sge
= ib_sge
;
2090 wr
->send_wr_num
= DIV_ROUND_UP(sg_nents
, isert_conn
->max_sge
);
2091 wr
->send_wr
= kzalloc(sizeof(struct ib_send_wr
) * wr
->send_wr_num
,
2094 pr_debug("Unable to allocate wr->send_wr\n");
2099 wr
->isert_cmd
= isert_cmd
;
2100 rdma_write_max
= isert_conn
->max_sge
* PAGE_SIZE
;
2102 for (i
= 0; i
< wr
->send_wr_num
; i
++) {
2103 send_wr
= &isert_cmd
->rdma_wr
.send_wr
[i
];
2104 data_len
= min(data_left
, rdma_write_max
);
2106 send_wr
->send_flags
= 0;
2107 if (wr
->iser_ib_op
== ISER_IB_RDMA_WRITE
) {
2108 send_wr
->opcode
= IB_WR_RDMA_WRITE
;
2109 send_wr
->wr
.rdma
.remote_addr
= isert_cmd
->read_va
+ offset
;
2110 send_wr
->wr
.rdma
.rkey
= isert_cmd
->read_stag
;
2111 if (i
+ 1 == wr
->send_wr_num
)
2112 send_wr
->next
= &isert_cmd
->tx_desc
.send_wr
;
2114 send_wr
->next
= &wr
->send_wr
[i
+ 1];
2116 send_wr
->opcode
= IB_WR_RDMA_READ
;
2117 send_wr
->wr
.rdma
.remote_addr
= isert_cmd
->write_va
+ va_offset
;
2118 send_wr
->wr
.rdma
.rkey
= isert_cmd
->write_stag
;
2119 if (i
+ 1 == wr
->send_wr_num
)
2120 send_wr
->send_flags
= IB_SEND_SIGNALED
;
2122 send_wr
->next
= &wr
->send_wr
[i
+ 1];
2125 ib_sge_cnt
= isert_build_rdma_wr(isert_conn
, isert_cmd
, ib_sge
,
2126 send_wr
, data_len
, offset
);
2127 ib_sge
+= ib_sge_cnt
;
2130 va_offset
+= data_len
;
2131 data_left
-= data_len
;
2136 ib_dma_unmap_sg(ib_dev
, sg_start
, sg_nents
,
2137 (wr
->iser_ib_op
== ISER_IB_RDMA_WRITE
) ?
2138 DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
2143 isert_map_fr_pagelist(struct ib_device
*ib_dev
,
2144 struct scatterlist
*sg_start
, int sg_nents
, u64
*fr_pl
)
2146 u64 start_addr
, end_addr
, page
, chunk_start
= 0;
2147 struct scatterlist
*tmp_sg
;
2148 int i
= 0, new_chunk
, last_ent
, n_pages
;
2152 last_ent
= sg_nents
- 1;
2153 for_each_sg(sg_start
, tmp_sg
, sg_nents
, i
) {
2154 start_addr
= ib_sg_dma_address(ib_dev
, tmp_sg
);
2156 chunk_start
= start_addr
;
2157 end_addr
= start_addr
+ ib_sg_dma_len(ib_dev
, tmp_sg
);
2159 pr_debug("SGL[%d] dma_addr: 0x%16llx len: %u\n",
2160 i
, (unsigned long long)tmp_sg
->dma_address
,
2163 if ((end_addr
& ~PAGE_MASK
) && i
< last_ent
) {
2169 page
= chunk_start
& PAGE_MASK
;
2171 fr_pl
[n_pages
++] = page
;
2172 pr_debug("Mapped page_list[%d] page_addr: 0x%16llx\n",
2175 } while (page
< end_addr
);
2182 isert_fast_reg_mr(struct fast_reg_descriptor
*fr_desc
,
2183 struct isert_conn
*isert_conn
, struct scatterlist
*sg_start
,
2184 struct ib_sge
*ib_sge
, u32 sg_nents
, u32 offset
,
2185 unsigned int data_len
)
2187 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
2188 struct ib_send_wr fr_wr
, inv_wr
;
2189 struct ib_send_wr
*bad_wr
, *wr
= NULL
;
2190 int ret
, pagelist_len
;
2194 sg_nents
= min_t(unsigned int, sg_nents
, ISCSI_ISER_SG_TABLESIZE
);
2195 page_off
= offset
% PAGE_SIZE
;
2197 pr_debug("Use fr_desc %p sg_nents %d offset %u\n",
2198 fr_desc
, sg_nents
, offset
);
2200 pagelist_len
= isert_map_fr_pagelist(ib_dev
, sg_start
, sg_nents
,
2201 &fr_desc
->data_frpl
->page_list
[0]);
2203 if (!fr_desc
->valid
) {
2204 memset(&inv_wr
, 0, sizeof(inv_wr
));
2205 inv_wr
.opcode
= IB_WR_LOCAL_INV
;
2206 inv_wr
.ex
.invalidate_rkey
= fr_desc
->data_mr
->rkey
;
2209 key
= (u8
)(fr_desc
->data_mr
->rkey
& 0x000000FF);
2210 ib_update_fast_reg_key(fr_desc
->data_mr
, ++key
);
2213 /* Prepare FASTREG WR */
2214 memset(&fr_wr
, 0, sizeof(fr_wr
));
2215 fr_wr
.opcode
= IB_WR_FAST_REG_MR
;
2216 fr_wr
.wr
.fast_reg
.iova_start
=
2217 fr_desc
->data_frpl
->page_list
[0] + page_off
;
2218 fr_wr
.wr
.fast_reg
.page_list
= fr_desc
->data_frpl
;
2219 fr_wr
.wr
.fast_reg
.page_list_len
= pagelist_len
;
2220 fr_wr
.wr
.fast_reg
.page_shift
= PAGE_SHIFT
;
2221 fr_wr
.wr
.fast_reg
.length
= data_len
;
2222 fr_wr
.wr
.fast_reg
.rkey
= fr_desc
->data_mr
->rkey
;
2223 fr_wr
.wr
.fast_reg
.access_flags
= IB_ACCESS_LOCAL_WRITE
;
2230 ret
= ib_post_send(isert_conn
->conn_qp
, wr
, &bad_wr
);
2232 pr_err("fast registration failed, ret:%d\n", ret
);
2235 fr_desc
->valid
= false;
2237 ib_sge
->lkey
= fr_desc
->data_mr
->lkey
;
2238 ib_sge
->addr
= fr_desc
->data_frpl
->page_list
[0] + page_off
;
2239 ib_sge
->length
= data_len
;
2241 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n",
2242 ib_sge
->addr
, ib_sge
->length
, ib_sge
->lkey
);
2248 isert_reg_rdma(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
,
2249 struct isert_rdma_wr
*wr
)
2251 struct se_cmd
*se_cmd
= &cmd
->se_cmd
;
2252 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2253 struct isert_conn
*isert_conn
= (struct isert_conn
*)conn
->context
;
2254 struct ib_device
*ib_dev
= isert_conn
->conn_cm_id
->device
;
2255 struct ib_send_wr
*send_wr
;
2256 struct ib_sge
*ib_sge
;
2257 struct scatterlist
*sg_start
;
2258 struct fast_reg_descriptor
*fr_desc
;
2259 u32 sg_off
= 0, sg_nents
;
2260 u32 offset
= 0, data_len
, data_left
, rdma_write_max
;
2262 unsigned long flags
;
2264 if (wr
->iser_ib_op
== ISER_IB_RDMA_WRITE
) {
2265 data_left
= se_cmd
->data_length
;
2267 offset
= cmd
->write_data_done
;
2268 sg_off
= offset
/ PAGE_SIZE
;
2269 data_left
= se_cmd
->data_length
- cmd
->write_data_done
;
2270 isert_cmd
->tx_desc
.isert_cmd
= isert_cmd
;
2273 sg_start
= &cmd
->se_cmd
.t_data_sg
[sg_off
];
2274 sg_nents
= se_cmd
->t_data_nents
- sg_off
;
2276 count
= ib_dma_map_sg(ib_dev
, sg_start
, sg_nents
,
2277 (wr
->iser_ib_op
== ISER_IB_RDMA_WRITE
) ?
2278 DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
2279 if (unlikely(!count
)) {
2280 pr_err("Cmd: %p unrable to map SGs\n", isert_cmd
);
2284 wr
->num_sge
= sg_nents
;
2285 pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n",
2286 isert_cmd
, count
, sg_start
, sg_nents
, data_left
);
2288 memset(&wr
->s_ib_sge
, 0, sizeof(*ib_sge
));
2289 ib_sge
= &wr
->s_ib_sge
;
2290 wr
->ib_sge
= ib_sge
;
2292 wr
->send_wr_num
= 1;
2293 memset(&wr
->s_send_wr
, 0, sizeof(*send_wr
));
2294 wr
->send_wr
= &wr
->s_send_wr
;
2296 wr
->isert_cmd
= isert_cmd
;
2297 rdma_write_max
= ISCSI_ISER_SG_TABLESIZE
* PAGE_SIZE
;
2299 send_wr
= &isert_cmd
->rdma_wr
.s_send_wr
;
2300 send_wr
->sg_list
= ib_sge
;
2301 send_wr
->num_sge
= 1;
2302 send_wr
->wr_id
= (unsigned long)&isert_cmd
->tx_desc
;
2303 if (wr
->iser_ib_op
== ISER_IB_RDMA_WRITE
) {
2304 send_wr
->opcode
= IB_WR_RDMA_WRITE
;
2305 send_wr
->wr
.rdma
.remote_addr
= isert_cmd
->read_va
;
2306 send_wr
->wr
.rdma
.rkey
= isert_cmd
->read_stag
;
2307 send_wr
->send_flags
= 0;
2308 send_wr
->next
= &isert_cmd
->tx_desc
.send_wr
;
2310 send_wr
->opcode
= IB_WR_RDMA_READ
;
2311 send_wr
->wr
.rdma
.remote_addr
= isert_cmd
->write_va
;
2312 send_wr
->wr
.rdma
.rkey
= isert_cmd
->write_stag
;
2313 send_wr
->send_flags
= IB_SEND_SIGNALED
;
2316 data_len
= min(data_left
, rdma_write_max
);
2317 wr
->cur_rdma_length
= data_len
;
2319 /* if there is a single dma entry, dma mr is sufficient */
2321 ib_sge
->addr
= ib_sg_dma_address(ib_dev
, &sg_start
[0]);
2322 ib_sge
->length
= ib_sg_dma_len(ib_dev
, &sg_start
[0]);
2323 ib_sge
->lkey
= isert_conn
->conn_mr
->lkey
;
2326 spin_lock_irqsave(&isert_conn
->conn_lock
, flags
);
2327 fr_desc
= list_first_entry(&isert_conn
->conn_fr_pool
,
2328 struct fast_reg_descriptor
, list
);
2329 list_del(&fr_desc
->list
);
2330 spin_unlock_irqrestore(&isert_conn
->conn_lock
, flags
);
2331 wr
->fr_desc
= fr_desc
;
2333 ret
= isert_fast_reg_mr(fr_desc
, isert_conn
, sg_start
,
2334 ib_sge
, sg_nents
, offset
, data_len
);
2336 list_add_tail(&fr_desc
->list
, &isert_conn
->conn_fr_pool
);
2344 ib_dma_unmap_sg(ib_dev
, sg_start
, sg_nents
,
2345 (wr
->iser_ib_op
== ISER_IB_RDMA_WRITE
) ?
2346 DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
2351 isert_put_datain(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
)
2353 struct se_cmd
*se_cmd
= &cmd
->se_cmd
;
2354 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2355 struct isert_rdma_wr
*wr
= &isert_cmd
->rdma_wr
;
2356 struct isert_conn
*isert_conn
= (struct isert_conn
*)conn
->context
;
2357 struct isert_device
*device
= isert_conn
->conn_device
;
2358 struct ib_send_wr
*wr_failed
;
2361 pr_debug("Cmd: %p RDMA_WRITE data_length: %u\n",
2362 isert_cmd
, se_cmd
->data_length
);
2363 wr
->iser_ib_op
= ISER_IB_RDMA_WRITE
;
2364 rc
= device
->reg_rdma_mem(conn
, cmd
, wr
);
2366 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd
);
2371 * Build isert_conn->tx_desc for iSCSI response PDU and attach
2373 isert_create_send_desc(isert_conn
, isert_cmd
, &isert_cmd
->tx_desc
);
2374 iscsit_build_rsp_pdu(cmd
, conn
, true, (struct iscsi_scsi_rsp
*)
2375 &isert_cmd
->tx_desc
.iscsi_header
);
2376 isert_init_tx_hdrs(isert_conn
, &isert_cmd
->tx_desc
);
2377 isert_init_send_wr(isert_conn
, isert_cmd
,
2378 &isert_cmd
->tx_desc
.send_wr
, true);
2380 atomic_inc(&isert_conn
->post_send_buf_count
);
2382 rc
= ib_post_send(isert_conn
->conn_qp
, wr
->send_wr
, &wr_failed
);
2384 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n");
2385 atomic_dec(&isert_conn
->post_send_buf_count
);
2387 pr_debug("Cmd: %p posted RDMA_WRITE + Response for iSER Data READ\n",
2394 isert_get_dataout(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
, bool recovery
)
2396 struct se_cmd
*se_cmd
= &cmd
->se_cmd
;
2397 struct isert_cmd
*isert_cmd
= iscsit_priv_cmd(cmd
);
2398 struct isert_rdma_wr
*wr
= &isert_cmd
->rdma_wr
;
2399 struct isert_conn
*isert_conn
= (struct isert_conn
*)conn
->context
;
2400 struct isert_device
*device
= isert_conn
->conn_device
;
2401 struct ib_send_wr
*wr_failed
;
2404 pr_debug("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n",
2405 isert_cmd
, se_cmd
->data_length
, cmd
->write_data_done
);
2406 wr
->iser_ib_op
= ISER_IB_RDMA_READ
;
2407 rc
= device
->reg_rdma_mem(conn
, cmd
, wr
);
2409 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd
);
2413 atomic_inc(&isert_conn
->post_send_buf_count
);
2415 rc
= ib_post_send(isert_conn
->conn_qp
, wr
->send_wr
, &wr_failed
);
2417 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n");
2418 atomic_dec(&isert_conn
->post_send_buf_count
);
2420 pr_debug("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n",
2427 isert_immediate_queue(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
, int state
)
2432 case ISTATE_SEND_NOPIN_WANT_RESPONSE
:
2433 ret
= isert_put_nopin(cmd
, conn
, false);
2436 pr_err("Unknown immediate state: 0x%02x\n", state
);
2445 isert_response_queue(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
, int state
)
2450 case ISTATE_SEND_LOGOUTRSP
:
2451 ret
= isert_put_logout_rsp(cmd
, conn
);
2453 pr_debug("Returning iSER Logout -EAGAIN\n");
2457 case ISTATE_SEND_NOPIN
:
2458 ret
= isert_put_nopin(cmd
, conn
, true);
2460 case ISTATE_SEND_TASKMGTRSP
:
2461 ret
= isert_put_tm_rsp(cmd
, conn
);
2463 case ISTATE_SEND_REJECT
:
2464 ret
= isert_put_reject(cmd
, conn
);
2466 case ISTATE_SEND_TEXTRSP
:
2467 ret
= isert_put_text_rsp(cmd
, conn
);
2469 case ISTATE_SEND_STATUS
:
2471 * Special case for sending non GOOD SCSI status from TX thread
2472 * context during pre se_cmd excecution failure.
2474 ret
= isert_put_response(conn
, cmd
);
2477 pr_err("Unknown response state: 0x%02x\n", state
);
2486 isert_setup_np(struct iscsi_np
*np
,
2487 struct __kernel_sockaddr_storage
*ksockaddr
)
2489 struct isert_np
*isert_np
;
2490 struct rdma_cm_id
*isert_lid
;
2491 struct sockaddr
*sa
;
2494 isert_np
= kzalloc(sizeof(struct isert_np
), GFP_KERNEL
);
2496 pr_err("Unable to allocate struct isert_np\n");
2499 init_waitqueue_head(&isert_np
->np_accept_wq
);
2500 mutex_init(&isert_np
->np_accept_mutex
);
2501 INIT_LIST_HEAD(&isert_np
->np_accept_list
);
2502 init_completion(&isert_np
->np_login_comp
);
2504 sa
= (struct sockaddr
*)ksockaddr
;
2505 pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr
, sa
);
2507 * Setup the np->np_sockaddr from the passed sockaddr setup
2508 * in iscsi_target_configfs.c code..
2510 memcpy(&np
->np_sockaddr
, ksockaddr
,
2511 sizeof(struct __kernel_sockaddr_storage
));
2513 isert_lid
= rdma_create_id(isert_cma_handler
, np
, RDMA_PS_TCP
,
2515 if (IS_ERR(isert_lid
)) {
2516 pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n",
2517 PTR_ERR(isert_lid
));
2518 ret
= PTR_ERR(isert_lid
);
2522 ret
= rdma_bind_addr(isert_lid
, sa
);
2524 pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret
);
2528 ret
= rdma_listen(isert_lid
, ISERT_RDMA_LISTEN_BACKLOG
);
2530 pr_err("rdma_listen() for isert_lid failed: %d\n", ret
);
2534 isert_np
->np_cm_id
= isert_lid
;
2535 np
->np_context
= isert_np
;
2536 pr_debug("Setup isert_lid->context: %p\n", isert_lid
->context
);
2541 rdma_destroy_id(isert_lid
);
2548 isert_check_accept_queue(struct isert_np
*isert_np
)
2552 mutex_lock(&isert_np
->np_accept_mutex
);
2553 empty
= list_empty(&isert_np
->np_accept_list
);
2554 mutex_unlock(&isert_np
->np_accept_mutex
);
2560 isert_rdma_accept(struct isert_conn
*isert_conn
)
2562 struct rdma_cm_id
*cm_id
= isert_conn
->conn_cm_id
;
2563 struct rdma_conn_param cp
;
2566 memset(&cp
, 0, sizeof(struct rdma_conn_param
));
2567 cp
.responder_resources
= isert_conn
->responder_resources
;
2568 cp
.initiator_depth
= isert_conn
->initiator_depth
;
2570 cp
.rnr_retry_count
= 7;
2572 pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n");
2574 ret
= rdma_accept(cm_id
, &cp
);
2576 pr_err("rdma_accept() failed with: %d\n", ret
);
2580 pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n");
2586 isert_get_login_rx(struct iscsi_conn
*conn
, struct iscsi_login
*login
)
2588 struct isert_conn
*isert_conn
= (struct isert_conn
*)conn
->context
;
2591 pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn
);
2593 * For login requests after the first PDU, isert_rx_login_req() will
2594 * kick schedule_delayed_work(&conn->login_work) as the packet is
2595 * received, which turns this callback from iscsi_target_do_login_rx()
2598 if (!login
->first_request
)
2601 ret
= wait_for_completion_interruptible(&isert_conn
->conn_login_comp
);
2605 pr_debug("isert_get_login_rx processing login->req: %p\n", login
->req
);
2610 isert_set_conn_info(struct iscsi_np
*np
, struct iscsi_conn
*conn
,
2611 struct isert_conn
*isert_conn
)
2613 struct rdma_cm_id
*cm_id
= isert_conn
->conn_cm_id
;
2614 struct rdma_route
*cm_route
= &cm_id
->route
;
2615 struct sockaddr_in
*sock_in
;
2616 struct sockaddr_in6
*sock_in6
;
2618 conn
->login_family
= np
->np_sockaddr
.ss_family
;
2620 if (np
->np_sockaddr
.ss_family
== AF_INET6
) {
2621 sock_in6
= (struct sockaddr_in6
*)&cm_route
->addr
.dst_addr
;
2622 snprintf(conn
->login_ip
, sizeof(conn
->login_ip
), "%pI6c",
2623 &sock_in6
->sin6_addr
.in6_u
);
2624 conn
->login_port
= ntohs(sock_in6
->sin6_port
);
2626 sock_in6
= (struct sockaddr_in6
*)&cm_route
->addr
.src_addr
;
2627 snprintf(conn
->local_ip
, sizeof(conn
->local_ip
), "%pI6c",
2628 &sock_in6
->sin6_addr
.in6_u
);
2629 conn
->local_port
= ntohs(sock_in6
->sin6_port
);
2631 sock_in
= (struct sockaddr_in
*)&cm_route
->addr
.dst_addr
;
2632 sprintf(conn
->login_ip
, "%pI4",
2633 &sock_in
->sin_addr
.s_addr
);
2634 conn
->login_port
= ntohs(sock_in
->sin_port
);
2636 sock_in
= (struct sockaddr_in
*)&cm_route
->addr
.src_addr
;
2637 sprintf(conn
->local_ip
, "%pI4",
2638 &sock_in
->sin_addr
.s_addr
);
2639 conn
->local_port
= ntohs(sock_in
->sin_port
);
2644 isert_accept_np(struct iscsi_np
*np
, struct iscsi_conn
*conn
)
2646 struct isert_np
*isert_np
= (struct isert_np
*)np
->np_context
;
2647 struct isert_conn
*isert_conn
;
2648 int max_accept
= 0, ret
;
2651 ret
= wait_event_interruptible(isert_np
->np_accept_wq
,
2652 !isert_check_accept_queue(isert_np
) ||
2653 np
->np_thread_state
== ISCSI_NP_THREAD_RESET
);
2657 spin_lock_bh(&np
->np_thread_lock
);
2658 if (np
->np_thread_state
== ISCSI_NP_THREAD_RESET
) {
2659 spin_unlock_bh(&np
->np_thread_lock
);
2660 pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n");
2663 spin_unlock_bh(&np
->np_thread_lock
);
2665 mutex_lock(&isert_np
->np_accept_mutex
);
2666 if (list_empty(&isert_np
->np_accept_list
)) {
2667 mutex_unlock(&isert_np
->np_accept_mutex
);
2671 isert_conn
= list_first_entry(&isert_np
->np_accept_list
,
2672 struct isert_conn
, conn_accept_node
);
2673 list_del_init(&isert_conn
->conn_accept_node
);
2674 mutex_unlock(&isert_np
->np_accept_mutex
);
2676 conn
->context
= isert_conn
;
2677 isert_conn
->conn
= conn
;
2680 ret
= isert_rdma_post_recvl(isert_conn
);
2684 ret
= isert_rdma_accept(isert_conn
);
2688 isert_set_conn_info(np
, conn
, isert_conn
);
2690 pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn
);
2695 isert_free_np(struct iscsi_np
*np
)
2697 struct isert_np
*isert_np
= (struct isert_np
*)np
->np_context
;
2699 rdma_destroy_id(isert_np
->np_cm_id
);
2701 np
->np_context
= NULL
;
2705 static int isert_check_state(struct isert_conn
*isert_conn
, int state
)
2709 mutex_lock(&isert_conn
->conn_mutex
);
2710 ret
= (isert_conn
->state
== state
);
2711 mutex_unlock(&isert_conn
->conn_mutex
);
2716 static void isert_free_conn(struct iscsi_conn
*conn
)
2718 struct isert_conn
*isert_conn
= conn
->context
;
2720 pr_debug("isert_free_conn: Starting \n");
2722 * Decrement post_send_buf_count for special case when called
2723 * from isert_do_control_comp() -> iscsit_logout_post_handler()
2725 mutex_lock(&isert_conn
->conn_mutex
);
2726 if (isert_conn
->logout_posted
)
2727 atomic_dec(&isert_conn
->post_send_buf_count
);
2729 if (isert_conn
->conn_cm_id
&& isert_conn
->state
!= ISER_CONN_DOWN
) {
2730 pr_debug("Calling rdma_disconnect from isert_free_conn\n");
2731 rdma_disconnect(isert_conn
->conn_cm_id
);
2734 * Only wait for conn_wait_comp_err if the isert_conn made it
2735 * into full feature phase..
2737 if (isert_conn
->state
== ISER_CONN_UP
) {
2738 pr_debug("isert_free_conn: Before wait_event comp_err %d\n",
2740 mutex_unlock(&isert_conn
->conn_mutex
);
2742 wait_event(isert_conn
->conn_wait_comp_err
,
2743 (isert_check_state(isert_conn
, ISER_CONN_TERMINATING
)));
2745 wait_event(isert_conn
->conn_wait
,
2746 (isert_check_state(isert_conn
, ISER_CONN_DOWN
)));
2748 isert_put_conn(isert_conn
);
2751 if (isert_conn
->state
== ISER_CONN_INIT
) {
2752 mutex_unlock(&isert_conn
->conn_mutex
);
2753 isert_put_conn(isert_conn
);
2756 pr_debug("isert_free_conn: wait_event conn_wait %d\n",
2758 mutex_unlock(&isert_conn
->conn_mutex
);
2760 wait_event(isert_conn
->conn_wait
,
2761 (isert_check_state(isert_conn
, ISER_CONN_DOWN
)));
2763 isert_put_conn(isert_conn
);
2766 static struct iscsit_transport iser_target_transport
= {
2768 .transport_type
= ISCSI_INFINIBAND
,
2769 .priv_size
= sizeof(struct isert_cmd
),
2770 .owner
= THIS_MODULE
,
2771 .iscsit_setup_np
= isert_setup_np
,
2772 .iscsit_accept_np
= isert_accept_np
,
2773 .iscsit_free_np
= isert_free_np
,
2774 .iscsit_free_conn
= isert_free_conn
,
2775 .iscsit_get_login_rx
= isert_get_login_rx
,
2776 .iscsit_put_login_tx
= isert_put_login_tx
,
2777 .iscsit_immediate_queue
= isert_immediate_queue
,
2778 .iscsit_response_queue
= isert_response_queue
,
2779 .iscsit_get_dataout
= isert_get_dataout
,
2780 .iscsit_queue_data_in
= isert_put_datain
,
2781 .iscsit_queue_status
= isert_put_response
,
2784 static int __init
isert_init(void)
2788 isert_rx_wq
= alloc_workqueue("isert_rx_wq", 0, 0);
2790 pr_err("Unable to allocate isert_rx_wq\n");
2794 isert_comp_wq
= alloc_workqueue("isert_comp_wq", 0, 0);
2795 if (!isert_comp_wq
) {
2796 pr_err("Unable to allocate isert_comp_wq\n");
2801 iscsit_register_transport(&iser_target_transport
);
2802 pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n");
2806 destroy_workqueue(isert_rx_wq
);
2810 static void __exit
isert_exit(void)
2812 destroy_workqueue(isert_comp_wq
);
2813 destroy_workqueue(isert_rx_wq
);
2814 iscsit_unregister_transport(&iser_target_transport
);
2815 pr_debug("iSER_TARGET[0] - Released iser_target_transport\n");
2818 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
2819 MODULE_VERSION("0.1");
2820 MODULE_AUTHOR("nab@Linux-iSCSI.org");
2821 MODULE_LICENSE("GPL");
2823 module_init(isert_init
);
2824 module_exit(isert_exit
);